File 9582.patch of Package powerdns-dnsdist-test

Overview Repositories Revisions Requests Users Attributes Meta

File 9582.patch of Package powerdns-dnsdist-test (Revision 28ae8741f8fc608fe90de5bea00e36bc)

Currently displaying revision 28ae8741f8fc608fe90de5bea00e36bc , Show latest

From 21ffdd52eedeae7c62fac67f899ab53a4359bc16 Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Fri, 5 Jun 2020 17:58:31 +0200
Subject: [PATCH 01/23] dnsdist: TCP refactoring, split Frontend/Backend
 handling [ci skip]

---
 pdns/dnsdist-tcp.cc                        | 996 +++++----------------
 pdns/dnsdist.hh                            |  59 +-
 pdns/dnsdistdist/Makefile.am               |   3 +
 pdns/dnsdistdist/dnsdist-backend.cc        |   3 +-
 pdns/dnsdistdist/dnsdist-proxy-protocol.cc |  22 +-
 pdns/dnsdistdist/dnsdist-proxy-protocol.hh |   4 +
 pdns/dnsdistdist/dnsdist-tcp-downstream.cc | 338 +++++++
 pdns/dnsdistdist/dnsdist-tcp-downstream.hh | 193 ++++
 pdns/dnsdistdist/dnsdist-tcp-upstream.hh   | 237 +++++
 pdns/dnsdistdist/doh.cc                    |   2 +-
 pdns/dnsdistdist/tcpiohandler-mplexer.hh   | 120 +++
 11 files changed, 1189 insertions(+), 788 deletions(-)
 create mode 100644 pdns/dnsdistdist/dnsdist-tcp-downstream.cc
 create mode 100644 pdns/dnsdistdist/dnsdist-tcp-downstream.hh
 create mode 100644 pdns/dnsdistdist/dnsdist-tcp-upstream.hh
 create mode 100644 pdns/dnsdistdist/tcpiohandler-mplexer.hh

diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc
index 71c669202f..8d6e7d6db8 100644
--- a/pdns/dnsdist-tcp.cc
+++ b/pdns/dnsdist-tcp.cc
@@ -19,27 +19,27 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  */
+
+#include <thread>
+#include <netinet/tcp.h>
+#include <queue>
+
 #include "dnsdist.hh"
 #include "dnsdist-ecs.hh"
 #include "dnsdist-proxy-protocol.hh"
 #include "dnsdist-rings.hh"
+#include "dnsdist-tcp-downstream.hh"
+#include "dnsdist-tcp-upstream.hh"
 #include "dnsdist-xpf.hh"
-
 #include "dnsparser.hh"
-#include "ednsoptions.hh"
 #include "dolog.hh"
-#include "lock.hh"
+#include "ednsoptions.hh"
 #include "gettime.hh"
+#include "lock.hh"
+#include "sstuff.hh"
 #include "tcpiohandler.hh"
+#include "tcpiohandler-mplexer.hh"
 #include "threadname.hh"
-#include <thread>
-#include <atomic>
-#include <netinet/tcp.h>
-
-#include "sstuff.hh"
-
-using std::thread;
-using std::atomic;
 
 /* TCP: the grand design.
    We forward 'messages' between clients and downstream servers. Messages are 65k bytes large, tops.
@@ -58,7 +58,7 @@ using std::atomic;
 
 static std::mutex tcpClientsCountMutex;
 static std::map<ComboAddress,size_t,ComboAddress::addressOnlyLessThan> tcpClientsCount;
-static const size_t g_maxCachedConnectionsPerDownstream = 20;
+
 uint64_t g_maxTCPQueuedConnections{1000};
 size_t g_maxTCPQueriesPerConn{0};
 size_t g_maxTCPConnectionDuration{0};
@@ -66,243 +66,162 @@ size_t g_maxTCPConnectionsPerClient{0};
 uint16_t g_downstreamTCPCleanupInterval{60};
 bool g_useTCPSinglePipe{false};
 
-static std::unique_ptr<Socket> setupTCPDownstream(shared_ptr<DownstreamState>& ds, uint16_t& downstreamFailures)
-{
-  std::unique_ptr<Socket> result;
-
-  do {
-    vinfolog("TCP connecting to downstream %s (%d)", ds->remote.toStringWithPort(), downstreamFailures);
-    try {
-      result = std::unique_ptr<Socket>(new Socket(ds->remote.sin4.sin_family, SOCK_STREAM, 0));
-      if (!IsAnyAddress(ds->sourceAddr)) {
-        SSetsockopt(result->getHandle(), SOL_SOCKET, SO_REUSEADDR, 1);
-#ifdef IP_BIND_ADDRESS_NO_PORT
-        if (ds->ipBindAddrNoPort) {
-          SSetsockopt(result->getHandle(), SOL_IP, IP_BIND_ADDRESS_NO_PORT, 1);
-        }
-#endif
-#ifdef SO_BINDTODEVICE
-        if (!ds->sourceItfName.empty()) {
-          int res = setsockopt(result->getHandle(), SOL_SOCKET, SO_BINDTODEVICE, ds->sourceItfName.c_str(), ds->sourceItfName.length());
-          if (res != 0) {
-            vinfolog("Error setting up the interface on backend TCP socket '%s': %s", ds->getNameWithAddr(), stringerror());
-          }
-        }
-#endif
-        result->bind(ds->sourceAddr, false);
-      }
-      result->setNonBlocking();
-#ifdef MSG_FASTOPEN
-      if (!ds->tcpFastOpen) {
-        SConnectWithTimeout(result->getHandle(), ds->remote, /* no timeout, we will handle it ourselves */ 0);
-      }
-#else
-      SConnectWithTimeout(result->getHandle(), ds->remote, /* no timeout, we will handle it ourselves */ 0);
-#endif /* MSG_FASTOPEN */
-      return result;
-    }
-    catch(const std::runtime_error& e) {
-      vinfolog("Connection to downstream server %s failed: %s", ds->getName(), e.what());
-      downstreamFailures++;
-      if (downstreamFailures > ds->retries) {
-        throw;
-      }
-    }
-  } while(downstreamFailures <= ds->retries);
-
-  return nullptr;
-}
-
-class TCPConnectionToBackend
+class DownstreamConnectionsManager
 {
 public:
-  TCPConnectionToBackend(std::shared_ptr<DownstreamState>& ds, uint16_t& downstreamFailures, const struct timeval& now): d_ds(ds), d_connectionStartTime(now), d_enableFastOpen(ds->tcpFastOpen)
-  {
-    d_socket = setupTCPDownstream(d_ds, downstreamFailures);
-    ++d_ds->tcpCurrentConnections;
-  }
 
-  ~TCPConnectionToBackend()
+  static std::unique_ptr<TCPConnectionToBackend> getConnectionToDownstream(std::unique_ptr<FDMultiplexer>& mplexer, std::shared_ptr<DownstreamState>& ds, const struct timeval& now)
   {
-    if (d_ds && d_socket) {
-      --d_ds->tcpCurrentConnections;
-      struct timeval now;
-      gettimeofday(&now, nullptr);
-
-      auto diff = now - d_connectionStartTime;
-      d_ds->updateTCPMetrics(d_queries, diff.tv_sec * 1000 + diff.tv_usec / 1000);
-    }
-  }
+    std::unique_ptr<TCPConnectionToBackend> result;
 
-  int getHandle() const
-  {
-    if (!d_socket) {
-      throw std::runtime_error("Attempt to get the socket handle from a non-established TCP connection");
+    const auto& it = t_downstreamConnections.find(ds->remote);
+    if (it != t_downstreamConnections.end()) {
+      auto& list = it->second;
+      if (!list.empty()) {
+        result = std::move(list.front());
+        list.pop_front();
+        result->setReused();
+        return result;
+      }
     }
 
-    return d_socket->getHandle();
-  }
-
-  const ComboAddress& getRemote() const
-  {
-    return d_ds->remote;
-  }
-
-  bool isFresh() const
-  {
-    return d_fresh;
-  }
-
-  void incQueries()
-  {
-    ++d_queries;
-  }
-
-  void setReused()
-  {
-    d_fresh = false;
+    return make_unique<TCPConnectionToBackend>(ds, now);
   }
 
-  void disableFastOpen()
+  static void releaseDownstreamConnection(std::unique_ptr<TCPConnectionToBackend>&& conn)
   {
-    d_enableFastOpen = false;
-  }
+    if (conn == nullptr) {
+      return;
+    }
 
-  bool isFastOpenEnabled()
-  {
-    return d_enableFastOpen;
-  }
+    if (!conn->canBeReused()) {
+      conn.reset();
+      return;
+    }
 
-  bool canBeReused() const
-  {
-    /* we can't reuse a connection where a proxy protocol payload has been sent,
-       since:
-       - it cannot be reused for a different client
-       - we might have different TLV values for each query
-    */
-    if (d_ds && d_ds->useProxyProtocol) {
-      return false;
+    const auto& remote = conn->getRemote();
+    const auto& it = t_downstreamConnections.find(remote);
+    if (it != t_downstreamConnections.end()) {
+      auto& list = it->second;
+      if (list.size() >= s_maxCachedConnectionsPerDownstream) {
+        /* too many connections queued already */
+        conn.reset();
+        return;
+      }
+      list.push_back(std::move(conn));
+    }
+    else {
+      t_downstreamConnections[remote].push_back(std::move(conn));
     }
-    return true;
   }
 
-  bool matches(const std::shared_ptr<DownstreamState>& ds) const
+  static void cleanupClosedTCPConnections()
   {
-    if (!ds || !d_ds) {
-      return false;
+    for(auto dsIt = t_downstreamConnections.begin(); dsIt != t_downstreamConnections.end(); ) {
+      for (auto connIt = dsIt->second.begin(); connIt != dsIt->second.end(); ) {
+        if (*connIt && isTCPSocketUsable((*connIt)->getHandle())) {
+          ++connIt;
+        }
+        else {
+          connIt = dsIt->second.erase(connIt);
+        }
+      }
+
+      if (!dsIt->second.empty()) {
+        ++dsIt;
+      }
+      else {
+        dsIt = t_downstreamConnections.erase(dsIt);
+      }
     }
-    return ds == d_ds;
   }
 
 private:
-  std::unique_ptr<Socket> d_socket{nullptr};
-  std::shared_ptr<DownstreamState> d_ds{nullptr};
-  struct timeval d_connectionStartTime;
-  uint64_t d_queries{0};
-  bool d_fresh{true};
-  bool d_enableFastOpen{false};
+  static thread_local map<ComboAddress, std::deque<std::unique_ptr<TCPConnectionToBackend>>> t_downstreamConnections;
+  static const size_t s_maxCachedConnectionsPerDownstream;
 };
 
-static thread_local map<ComboAddress, std::deque<std::unique_ptr<TCPConnectionToBackend>>> t_downstreamConnections;
+thread_local map<ComboAddress, std::deque<std::unique_ptr<TCPConnectionToBackend>>> DownstreamConnectionsManager::t_downstreamConnections;
+const size_t DownstreamConnectionsManager::s_maxCachedConnectionsPerDownstream{20};
 
-static std::unique_ptr<TCPConnectionToBackend> getConnectionToDownstream(std::shared_ptr<DownstreamState>& ds, uint16_t& downstreamFailures, const struct timeval& now)
+static void decrementTCPClientCount(const ComboAddress& client)
 {
-  std::unique_ptr<TCPConnectionToBackend> result;
-
-  const auto& it = t_downstreamConnections.find(ds->remote);
-  if (it != t_downstreamConnections.end()) {
-    auto& list = it->second;
-    if (!list.empty()) {
-      result = std::move(list.front());
-      list.pop_front();
-      result->setReused();
-      return result;
+  if (g_maxTCPConnectionsPerClient) {
+    std::lock_guard<std::mutex> lock(tcpClientsCountMutex);
+    tcpClientsCount[client]--;
+    if (tcpClientsCount[client] == 0) {
+      tcpClientsCount.erase(client);
     }
   }
-
-  return std::unique_ptr<TCPConnectionToBackend>(new TCPConnectionToBackend(ds, downstreamFailures, now));
 }
 
-static void releaseDownstreamConnection(std::unique_ptr<TCPConnectionToBackend>&& conn)
+IncomingTCPConnectionState::~IncomingTCPConnectionState()
 {
-  if (conn == nullptr) {
-    return;
+  decrementTCPClientCount(d_ci.remote);
+  if (d_ci.cs != nullptr) {
+    struct timeval now;
+    gettimeofday(&now, nullptr);
+    
+    auto diff = now - d_connectionStartTime;
+    d_ci.cs->updateTCPMetrics(d_queriesCount, diff.tv_sec * 1000.0 + diff.tv_usec / 1000.0);
   }
 
-  if (!conn->canBeReused()) {
-    conn.reset();
-    return;
+  if (d_ds != nullptr) {
+
+    if (d_downstreamConnection) {
+      try {
+        if (d_lastIOState == IOState::NeedRead) {
+          cerr<<__func__<<": removing leftover backend read FD "<<d_downstreamConnection->getHandle()<<endl;
+          d_threadData.mplexer->removeReadFD(d_downstreamConnection->getHandle());
+        }
+        else if (d_lastIOState == IOState::NeedWrite) {
+          cerr<<__func__<<": removing leftover backend write FD "<<d_downstreamConnection->getHandle()<<endl;
+          d_threadData.mplexer->removeWriteFD(d_downstreamConnection->getHandle());
+        }
+      }
+      catch(const FDMultiplexerException& e) {
+        vinfolog("Got an exception when trying to remove a pending IO operation on the socket to the %s backend: %s", d_ds->getName(), e.what());
+      }
+      catch(const std::runtime_error& e) {
+        /* might be thrown by getHandle() */
+        vinfolog("Got an exception when trying to remove a pending IO operation on the socket to the %s backend: %s", d_ds->getName(), e.what());
+      }
+    }
   }
 
-  const auto& remote = conn->getRemote();
-  const auto& it = t_downstreamConnections.find(remote);
-  if (it != t_downstreamConnections.end()) {
-    auto& list = it->second;
-    if (list.size() >= g_maxCachedConnectionsPerDownstream) {
-      /* too many connections queued already */
-      conn.reset();
-      return;
+  try {
+    if (d_lastIOState == IOState::NeedRead) {
+      cerr<<__func__<<": removing leftover client read FD "<<d_ci.fd<<endl;
+      d_threadData.mplexer->removeReadFD(d_ci.fd);
+    }
+    else if (d_lastIOState == IOState::NeedWrite) {
+      cerr<<__func__<<": removing leftover client write FD "<<d_ci.fd<<endl;
+      d_threadData.mplexer->removeWriteFD(d_ci.fd);
     }
-    list.push_back(std::move(conn));
   }
-  else {
-    t_downstreamConnections[remote].push_back(std::move(conn));
+  catch(const FDMultiplexerException& e) {
+    vinfolog("Got an exception when trying to remove a pending IO operation on an incoming TCP connection from %s: %s", d_ci.remote.toStringWithPort(), e.what());
   }
 }
 
-struct ConnectionInfo
+std::shared_ptr<TCPConnectionToBackend> IncomingTCPConnectionState::getDownstreamConnection(std::shared_ptr<DownstreamState>& ds, const struct timeval& now)
 {
-  ConnectionInfo(ClientState* cs_): cs(cs_), fd(-1)
-  {
-  }
-  ConnectionInfo(ConnectionInfo&& rhs): remote(rhs.remote), cs(rhs.cs), fd(rhs.fd)
-  {
-    rhs.cs = nullptr;
-    rhs.fd = -1;
-  }
-
-  ConnectionInfo(const ConnectionInfo& rhs) = delete;
-  ConnectionInfo& operator=(const ConnectionInfo& rhs) = delete;
+  std::shared_ptr<TCPConnectionToBackend> downstream{nullptr};
 
-  ConnectionInfo& operator=(ConnectionInfo&& rhs)
-  {
-    remote = rhs.remote;
-    cs = rhs.cs;
-    rhs.cs = nullptr;
-    fd = rhs.fd;
-    rhs.fd = -1;
-    return *this;
+  if (!ds->useProxyProtocol || !d_proxyProtocolPayloadHasTLV) {
+    downstream = getActiveDownstreamConnection(ds);
   }
 
-  ~ConnectionInfo()
-  {
-    if (fd != -1) {
-      close(fd);
-      fd = -1;
-    }
-    if (cs) {
-      --cs->tcpCurrentConnections;
-    }
+  if (!downstream) {
+    /* we don't have a connection to this backend active yet, let's ask one (it might not be a fresh one, though) */
+    downstream = DownstreamConnectionsManager::getConnectionToDownstream(d_threadData.mplexer, ds, now);
   }
 
-  ComboAddress remote;
-  ClientState* cs{nullptr};
-  int fd{-1};
-};
-
-void tcpClientThread(int pipefd);
-
-static void decrementTCPClientCount(const ComboAddress& client)
-{
-  if (g_maxTCPConnectionsPerClient) {
-    std::lock_guard<std::mutex> lock(tcpClientsCountMutex);
-    tcpClientsCount[client]--;
-    if (tcpClientsCount[client] == 0) {
-      tcpClientsCount.erase(client);
-    }
-  }
+  return downstream;
 }
 
+static void tcpClientThread(int pipefd);
+
 void TCPClientCollection::addTCPClientThread()
 {
   int pipefds[2] = { -1, -1};
@@ -349,7 +268,7 @@ void TCPClientCollection::addTCPClientThread()
     }
 
     try {
-      thread t1(tcpClientThread, pipefds[0]);
+      std::thread t1(tcpClientThread, pipefds[0]);
       t1.detach();
     }
     catch(const std::runtime_error& e) {
@@ -367,27 +286,6 @@ void TCPClientCollection::addTCPClientThread()
   }
 }
 
-static void cleanupClosedTCPConnections()
-{
-  for(auto dsIt = t_downstreamConnections.begin(); dsIt != t_downstreamConnections.end(); ) {
-    for (auto connIt = dsIt->second.begin(); connIt != dsIt->second.end(); ) {
-      if (*connIt && isTCPSocketUsable((*connIt)->getHandle())) {
-        ++connIt;
-      }
-      else {
-        connIt = dsIt->second.erase(connIt);
-      }
-    }
-
-    if (!dsIt->second.empty()) {
-      ++dsIt;
-    }
-    else {
-      dsIt = t_downstreamConnections.erase(dsIt);
-    }
-  }
-}
-
 /* Tries to read exactly toRead bytes into the buffer, starting at position pos.
    Updates pos everytime a successful read occurs,
    throws an std::runtime_error in case of IO error,
@@ -395,7 +293,7 @@ static void cleanupClosedTCPConnections()
    would block.
 */
 // XXX could probably be implemented as a TCPIOHandler
-static IOState tryRead(int fd, std::vector<uint8_t>& buffer, size_t& pos, size_t toRead)
+IOState tryRead(int fd, std::vector<uint8_t>& buffer, size_t& pos, size_t toRead)
 {
   if (buffer.size() < (pos + toRead)) {
     throw std::out_of_range("Calling tryRead() with a too small buffer (" + std::to_string(buffer.size()) + ") for a read of " + std::to_string(toRead) + " bytes starting at " + std::to_string(pos));
@@ -426,278 +324,13 @@ static IOState tryRead(int fd, std::vector<uint8_t>& buffer, size_t& pos, size_t
 
 std::unique_ptr<TCPClientCollection> g_tcpclientthreads;
 
-class TCPClientThreadData
-{
-public:
-  TCPClientThreadData(): localRespRulactions(g_resprulactions.getLocal()), mplexer(std::unique_ptr<FDMultiplexer>(FDMultiplexer::getMultiplexerSilent()))
-  {
-  }
-
-  LocalHolders holders;
-  LocalStateHolder<vector<DNSDistResponseRuleAction> > localRespRulactions;
-  std::unique_ptr<FDMultiplexer> mplexer{nullptr};
-};
-
-static void handleDownstreamIOCallback(int fd, FDMultiplexer::funcparam_t& param);
-
-class IncomingTCPConnectionState
-{
-public:
-  IncomingTCPConnectionState(ConnectionInfo&& ci, TCPClientThreadData& threadData, const struct timeval& now): d_buffer(s_maxPacketCacheEntrySize), d_responseBuffer(s_maxPacketCacheEntrySize), d_threadData(threadData), d_ci(std::move(ci)), d_handler(d_ci.fd, g_tcpRecvTimeout, d_ci.cs->tlsFrontend ? d_ci.cs->tlsFrontend->getContext() : nullptr, now.tv_sec), d_connectionStartTime(now)
-  {
-    d_ids.origDest.reset();
-    d_ids.origDest.sin4.sin_family = d_ci.remote.sin4.sin_family;
-    socklen_t socklen = d_ids.origDest.getSocklen();
-    if (getsockname(d_ci.fd, reinterpret_cast<sockaddr*>(&d_ids.origDest), &socklen)) {
-      d_ids.origDest = d_ci.cs->local;
-    }
-  }
-
-  IncomingTCPConnectionState(const IncomingTCPConnectionState& rhs) = delete;
-  IncomingTCPConnectionState& operator=(const IncomingTCPConnectionState& rhs) = delete;
-
-  ~IncomingTCPConnectionState()
-  {
-    decrementTCPClientCount(d_ci.remote);
-    if (d_ci.cs != nullptr) {
-      struct timeval now;
-      gettimeofday(&now, nullptr);
-
-      auto diff = now - d_connectionStartTime;
-      d_ci.cs->updateTCPMetrics(d_queriesCount, diff.tv_sec * 1000.0 + diff.tv_usec / 1000.0);
-    }
-
-    if (d_ds != nullptr) {
-      if (d_outstanding) {
-        --d_ds->outstanding;
-        d_outstanding = false;
-      }
-
-      if (d_downstreamConnection) {
-        try {
-          if (d_lastIOState == IOState::NeedRead) {
-            cerr<<__func__<<": removing leftover backend read FD "<<d_downstreamConnection->getHandle()<<endl;
-            d_threadData.mplexer->removeReadFD(d_downstreamConnection->getHandle());
-          }
-          else if (d_lastIOState == IOState::NeedWrite) {
-            cerr<<__func__<<": removing leftover backend write FD "<<d_downstreamConnection->getHandle()<<endl;
-            d_threadData.mplexer->removeWriteFD(d_downstreamConnection->getHandle());
-          }
-        }
-        catch(const FDMultiplexerException& e) {
-          vinfolog("Got an exception when trying to remove a pending IO operation on the socket to the %s backend: %s", d_ds->getName(), e.what());
-        }
-        catch(const std::runtime_error& e) {
-          /* might be thrown by getHandle() */
-          vinfolog("Got an exception when trying to remove a pending IO operation on the socket to the %s backend: %s", d_ds->getName(), e.what());
-        }
-      }
-    }
-
-    try {
-      if (d_lastIOState == IOState::NeedRead) {
-        cerr<<__func__<<": removing leftover client read FD "<<d_ci.fd<<endl;
-        d_threadData.mplexer->removeReadFD(d_ci.fd);
-      }
-      else if (d_lastIOState == IOState::NeedWrite) {
-        cerr<<__func__<<": removing leftover client write FD "<<d_ci.fd<<endl;
-        d_threadData.mplexer->removeWriteFD(d_ci.fd);
-      }
-    }
-    catch(const FDMultiplexerException& e) {
-      vinfolog("Got an exception when trying to remove a pending IO operation on an incoming TCP connection from %s: %s", d_ci.remote.toStringWithPort(), e.what());
-    }
-  }
-
-  void resetForNewQuery()
-  {
-    d_buffer.resize(sizeof(uint16_t));
-    d_currentPos = 0;
-    d_querySize = 0;
-    d_responseSize = 0;
-    d_downstreamFailures = 0;
-    d_state = State::readingQuerySize;
-    d_lastIOState = IOState::Done;
-    d_selfGeneratedResponse = false;
-  }
-
-  boost::optional<struct timeval> getClientReadTTD(struct timeval now) const
-  {
-    if (g_maxTCPConnectionDuration == 0 && g_tcpRecvTimeout == 0) {
-      return boost::none;
-    }
-
-    if (g_maxTCPConnectionDuration > 0) {
-      auto elapsed = now.tv_sec - d_connectionStartTime.tv_sec;
-      if (elapsed < 0 || (static_cast<size_t>(elapsed) >= g_maxTCPConnectionDuration)) {
-        return now;
-      }
-      auto remaining = g_maxTCPConnectionDuration - elapsed;
-      if (g_tcpRecvTimeout == 0 || remaining <= static_cast<size_t>(g_tcpRecvTimeout)) {
-        now.tv_sec += remaining;
-        return now;
-      }
-    }
-
-    now.tv_sec += g_tcpRecvTimeout;
-    return now;
-  }
-
-  boost::optional<struct timeval> getBackendReadTTD(const struct timeval& now) const
-  {
-    if (d_ds == nullptr) {
-      throw std::runtime_error("getBackendReadTTD() without any backend selected");
-    }
-    if (d_ds->tcpRecvTimeout == 0) {
-      return boost::none;
-    }
-
-    struct timeval res = now;
-    res.tv_sec += d_ds->tcpRecvTimeout;
-
-    return res;
-  }
-
-  boost::optional<struct timeval> getClientWriteTTD(const struct timeval& now) const
-  {
-    if (g_maxTCPConnectionDuration == 0 && g_tcpSendTimeout == 0) {
-      return boost::none;
-    }
-
-    struct timeval res = now;
-
-    if (g_maxTCPConnectionDuration > 0) {
-      auto elapsed = res.tv_sec - d_connectionStartTime.tv_sec;
-      if (elapsed < 0 || static_cast<size_t>(elapsed) >= g_maxTCPConnectionDuration) {
-        return res;
-      }
-      auto remaining = g_maxTCPConnectionDuration - elapsed;
-      if (g_tcpSendTimeout == 0 || remaining <= static_cast<size_t>(g_tcpSendTimeout)) {
-        res.tv_sec += remaining;
-        return res;
-      }
-    }
-
-    res.tv_sec += g_tcpSendTimeout;
-    return res;
-  }
-
-  boost::optional<struct timeval> getBackendWriteTTD(const struct timeval& now) const
-  {
-    if (d_ds == nullptr) {
-      throw std::runtime_error("getBackendReadTTD() called without any backend selected");
-    }
-    if (d_ds->tcpSendTimeout == 0) {
-      return boost::none;
-    }
-
-    struct timeval res = now;
-    res.tv_sec += d_ds->tcpSendTimeout;
-
-    return res;
-  }
-
-  bool maxConnectionDurationReached(unsigned int maxConnectionDuration, const struct timeval& now)
-  {
-    if (maxConnectionDuration) {
-      time_t curtime = now.tv_sec;
-      unsigned int elapsed = 0;
-      if (curtime > d_connectionStartTime.tv_sec) { // To prevent issues when time goes backward
-        elapsed = curtime - d_connectionStartTime.tv_sec;
-      }
-      if (elapsed >= maxConnectionDuration) {
-        return true;
-      }
-      d_remainingTime = maxConnectionDuration - elapsed;
-    }
-
-    return false;
-  }
-
-  void dump() const
-  {
-    static std::mutex s_mutex;
-
-    struct timeval now;
-    gettimeofday(&now, 0);
-
-    {
-      std::lock_guard<std::mutex> lock(s_mutex);
-      fprintf(stderr, "State is %p\n", this);
-      cerr << "Current state is " << static_cast<int>(d_state) << ", got "<<d_queriesCount<<" queries so far" << endl;
-      cerr << "Current time is " << now.tv_sec << " - " << now.tv_usec << endl;
-      cerr << "Connection started at " << d_connectionStartTime.tv_sec << " - " << d_connectionStartTime.tv_usec << endl;
-      if (d_state > State::doingHandshake) {
-        cerr << "Handshake done at " << d_handshakeDoneTime.tv_sec << " - " << d_handshakeDoneTime.tv_usec << endl;
-      }
-      if (d_state > State::readingQuerySize) {
-        cerr << "Got first query size at " << d_firstQuerySizeReadTime.tv_sec << " - " << d_firstQuerySizeReadTime.tv_usec << endl;
-      }
-      if (d_state > State::readingQuerySize) {
-        cerr << "Got query size at " << d_querySizeReadTime.tv_sec << " - " << d_querySizeReadTime.tv_usec << endl;
-      }
-      if (d_state > State::readingQuery) {
-        cerr << "Got query at " << d_queryReadTime.tv_sec << " - " << d_queryReadTime.tv_usec << endl;
-      }
-      if (d_state > State::sendingQueryToBackend) {
-        cerr << "Sent query at " << d_querySentTime.tv_sec << " - " << d_querySentTime.tv_usec << endl;
-      }
-      if (d_state > State::readingResponseFromBackend) {
-        cerr << "Got response at " << d_responseReadTime.tv_sec << " - " << d_responseReadTime.tv_usec << endl;
-      }
-    }
-  }
-
-  enum class State { doingHandshake, readingQuerySize, readingQuery, sendingQueryToBackend, readingResponseSizeFromBackend, readingResponseFromBackend, sendingResponse };
-
-  std::vector<uint8_t> d_buffer;
-  std::vector<uint8_t> d_responseBuffer;
-  TCPClientThreadData& d_threadData;
-  IDState d_ids;
-  ConnectionInfo d_ci;
-  TCPIOHandler d_handler;
-  std::unique_ptr<TCPConnectionToBackend> d_downstreamConnection{nullptr};
-  std::shared_ptr<DownstreamState> d_ds{nullptr};
-  dnsheader d_cleartextDH;
-  struct timeval d_connectionStartTime;
-  struct timeval d_handshakeDoneTime;
-  struct timeval d_firstQuerySizeReadTime;
-  struct timeval d_querySizeReadTime;
-  struct timeval d_queryReadTime;
-  struct timeval d_querySentTime;
-  struct timeval d_responseReadTime;
-  size_t d_currentPos{0};
-  size_t d_queriesCount{0};
-  unsigned int d_remainingTime{0};
-  uint16_t d_querySize{0};
-  uint16_t d_responseSize{0};
-  uint16_t d_downstreamFailures{0};
-  State d_state{State::doingHandshake};
-  IOState d_lastIOState{IOState::Done};
-  bool d_readingFirstQuery{true};
-  bool d_outstanding{false};
-  bool d_firstResponsePacket{true};
-  bool d_isXFR{false};
-  bool d_xfrStarted{false};
-  bool d_selfGeneratedResponse{false};
-  bool d_proxyProtocolPayloadAdded{false};
-  bool d_proxyProtocolPayloadHasTLV{false};
-};
-
-static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param);
-static void handleNewIOState(std::shared_ptr<IncomingTCPConnectionState>& state, IOState iostate, const int fd, FDMultiplexer::callbackfunc_t callback, boost::optional<struct timeval> ttd=boost::none);
-static void handleIO(std::shared_ptr<IncomingTCPConnectionState>& state, struct timeval& now);
-static void handleDownstreamIO(std::shared_ptr<IncomingTCPConnectionState>& state, struct timeval& now);
-
-static void handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& state, struct timeval& now)
+static void handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
 {
-  handleNewIOState(state, IOState::Done, state->d_ci.fd, handleIOCallback);
-
   if (state->d_isXFR && state->d_downstreamConnection) {
-    /* we need to resume reading from the backend! */
-    state->d_state = IncomingTCPConnectionState::State::readingResponseSizeFromBackend;
+    /* we should still be reading from the backend, and we don't want to read from the client */
+    #warning check that we are doing the right thing on the backend side
+    state->d_state = IncomingTCPConnectionState::State::idle;
     state->d_currentPos = 0;
-    handleDownstreamIO(state, now);
     return;
   }
 
@@ -735,45 +368,58 @@ static void handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& stat
   }
 
   state->resetForNewQuery();
+}
 
-  handleIO(state, now);
+/* this version is called when the buffer has been set and the rules have been processed */
+void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
+{
+  // if we already reading a query (not the query size, mind you), or sending a response we need to either queue the response
+  // otherwise we can start sending it right away
+  if (state->d_state == IncomingTCPConnectionState::State::idle ||
+      state->d_state == IncomingTCPConnectionState::State::readingQuerySize) {
+    state->d_state = IncomingTCPConnectionState::State::sendingResponse;
+    state->d_ioState->update(IOState::NeedWrite, handleIOCallback, state, getClientWriteTTD(now));
+  }
+  else {
+    // queue response
+  }
+}
+
+{
+  if (state->handleResponseFromBackend(state, now, response)) {
+    state->sendResponse(state, response, now);
+  }
 }
 
-static void sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, struct timeval& now)
+static void sendInternalResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
 {
-  state->d_state = IncomingTCPConnectionState::State::sendingResponse;
   const uint8_t sizeBytes[] = { static_cast<uint8_t>(state->d_responseSize / 256), static_cast<uint8_t>(state->d_responseSize % 256) };
   /* prepend the size. Yes, this is not the most efficient way but it prevents mistakes
      that could occur if we had to deal with the size during the processing,
      especially alignment issues */
   state->d_responseBuffer.insert(state->d_responseBuffer.begin(), sizeBytes, sizeBytes + 2);
-
   state->d_currentPos = 0;
-
-  handleIO(state, now);
+  state->sendResponse(now);
 }
 
-static void handleResponse(std::shared_ptr<IncomingTCPConnectionState>& state, struct timeval& now)
+/* this version is called from the backend code when a new response has been received */
+void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response, const ComboAddress& remote)
 {
-  if (state->d_responseSize < sizeof(dnsheader) || !state->d_ds) {
+  if (response.d_buffer.size() < sizeof(dnsheader)) {
     return;
   }
 
-  auto response = reinterpret_cast<char*>(&state->d_responseBuffer.at(0));
+  auto response = reinterpret_cast<char*>(&response.d_buffer.at(0));
+  auto& ids = response.d_idstate;
   unsigned int consumed;
-  if (state->d_firstResponsePacket && !responseContentMatches(response, state->d_responseSize, state->d_ids.qname, state->d_ids.qtype, state->d_ids.qclass, state->d_ds->remote, consumed)) {
+  if (ids.d_firstResponsePacket && !responseContentMatches(response, response.d_buffer.size(), ids.qname, ids.qtype, ids.qclass, remote, consumed)) {
     return;
   }
-  state->d_firstResponsePacket = false;
-
-  if (state->d_outstanding) {
-    --state->d_ds->outstanding;
-    state->d_outstanding = false;
-  }
+  ids.d_firstResponsePacket = false;
 
   auto dh = reinterpret_cast<struct dnsheader*>(response);
   uint16_t addRoom = 0;
-  DNSResponse dr = makeDNSResponseFromIDState(state->d_ids, dh, state->d_responseBuffer.size(), state->d_responseSize, true);
+  DNSResponse dr = makeDNSResponseFromIDState(ids, dh, response.d_buffer.size(), state->d_responseSize, true);
   if (dr.dnsCryptQuery) {
     addRoom = DNSCRYPT_MAX_RESPONSE_PADDING_AND_MAC_SIZE;
   }
@@ -813,51 +459,7 @@ static void handleResponse(std::shared_ptr<IncomingTCPConnectionState>& state, s
   sendResponse(state, now);
 }
 
-static void sendQueryToBackend(std::shared_ptr<IncomingTCPConnectionState>& state, struct timeval& now)
-{
-  auto ds = state->d_ds;
-  state->d_state = IncomingTCPConnectionState::State::sendingQueryToBackend;
-  state->d_currentPos = 0;
-  state->d_firstResponsePacket = true;
-
-  if (state->d_xfrStarted) {
-    /* sorry, but we are not going to resume a XFR if we have already sent some packets
-       to the client */
-    return;
-  }
-
-  if (!state->d_downstreamConnection) {
-    if (state->d_downstreamFailures < state->d_ds->retries) {
-      try {
-        state->d_downstreamConnection = getConnectionToDownstream(ds, state->d_downstreamFailures, now);
-      }
-      catch (const std::runtime_error& e) {
-        state->d_downstreamConnection.reset();
-      }
-    }
-
-    if (!state->d_downstreamConnection) {
-      ++ds->tcpGaveUp;
-      ++state->d_ci.cs->tcpGaveUp;
-      vinfolog("Downstream connection to %s failed %d times in a row, giving up.", ds->getName(), state->d_downstreamFailures);
-      return;
-    }
-
-    if (ds->useProxyProtocol && !state->d_proxyProtocolPayloadAdded) {
-      /* we know there is no TLV values to add, otherwise we would not have tried
-         to reuse the connection and d_proxyProtocolPayloadAdded would be true already */
-      addProxyProtocol(state->d_buffer, true, state->d_ci.remote, state->d_ids.origDest, std::vector<ProxyProtocolValue>());
-      state->d_proxyProtocolPayloadAdded = true;
-    }
-  }
-
-  vinfolog("Got query for %s|%s from %s (%s), relayed to %s", state->d_ids.qname.toLogString(), QType(state->d_ids.qtype).getName(), state->d_ci.remote.toStringWithPort(), (state->d_ci.cs->tlsFrontend ? "DoT" : "TCP"), ds->getName());
-
-  handleDownstreamIO(state, now);
-  return;
-}
-
-static void handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, struct timeval& now)
+static void handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
 {
   if (state->d_querySize < sizeof(dnsheader)) {
     ++g_stats.nonCompliantQueries;
@@ -902,7 +504,7 @@ static void handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, stru
   if (dnsCryptResponse) {
     state->d_responseBuffer = std::move(*dnsCryptResponse);
     state->d_responseSize = state->d_responseBuffer.size();
-    sendResponse(state, now);
+    sendInteralResponse(state, now);
     return;
   }
 
@@ -935,7 +537,7 @@ static void handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, stru
     state->d_buffer.resize(dq.len);
     state->d_responseBuffer = std::move(state->d_buffer);
     state->d_responseSize = state->d_responseBuffer.size();
-    sendResponse(state, now);
+    sendInternalResponse(state, now);
     return;
   }
 
@@ -943,7 +545,15 @@ static void handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, stru
     return;
   }
 
-  setIDStateFromDNSQuestion(state->d_ids, dq, std::move(qname));
+#warning move this, we should just never read another question again on this client connection
+  if (state->d_xfrStarted) {
+    /* sorry, but we are not going to resume a XFR if we have already sent some packets
+       to the client */
+    return;
+  }
+
+  IDState ids;
+  setIDStateFromDNSQuestion(ids, dq, std::move(qname));
 
   const uint8_t sizeBytes[] = { static_cast<uint8_t>(dq.len / 256), static_cast<uint8_t>(dq.len % 256) };
   /* prepend the size. Yes, this is not the most efficient way but it prevents mistakes
@@ -955,242 +565,69 @@ static void handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, stru
   dq.size = state->d_buffer.size();
   state->d_buffer.resize(dq.len);
 
+  bool proxyProtocolPayloadAdded = false;
+  std::string proxyProtocolPayload;
+
   if (state->d_ds->useProxyProtocol) {
     /* if we ever sent a TLV over a connection, we can never go back */
     if (!state->d_proxyProtocolPayloadHasTLV) {
       state->d_proxyProtocolPayloadHasTLV = dq.proxyProtocolValues && !dq.proxyProtocolValues->empty();
     }
 
-    if (state->d_downstreamConnection && !state->d_proxyProtocolPayloadHasTLV && state->d_downstreamConnection->matches(state->d_ds)) {
-      /* we have an existing connection, on which we already sent a Proxy Protocol header with no values
-         (in the previous query had TLV values we would have reset the connection afterwards),
-         so let's reuse it as long as we still don't have any values */
-      state->d_proxyProtocolPayloadAdded = false;
-    }
-    else {
-      state->d_downstreamConnection.reset();
-      addProxyProtocol(state->d_buffer, true, state->d_ci.remote, state->d_ids.origDest, dq.proxyProtocolValues ? *dq.proxyProtocolValues : std::vector<ProxyProtocolValue>());
-      state->d_proxyProtocolPayloadAdded = true;
-    }
-  }
-
-  sendQueryToBackend(state, now);
-}
+    proxyProtocolPayload = getProxyProtocolPayload(dq);
 
-static void handleNewIOState(std::shared_ptr<IncomingTCPConnectionState>& state, IOState iostate, const int fd, FDMultiplexer::callbackfunc_t callback, boost::optional<struct timeval> ttd)
-{
-  //cerr<<"in "<<__func__<<" for fd "<<fd<<", last state was "<<(int)state->d_lastIOState<<", new state is "<<(int)iostate<<endl;
-
-  if (state->d_lastIOState == IOState::NeedRead && iostate != IOState::NeedRead) {
-    state->d_threadData.mplexer->removeReadFD(fd);
-    //cerr<<__func__<<": remove read FD "<<fd<<endl;
-    state->d_lastIOState = IOState::Done;
-  }
-  else if (state->d_lastIOState == IOState::NeedWrite && iostate != IOState::NeedWrite) {
-    state->d_threadData.mplexer->removeWriteFD(fd);
-    //cerr<<__func__<<": remove write FD "<<fd<<endl;
-    state->d_lastIOState = IOState::Done;
-  }
-
-  if (iostate == IOState::NeedRead) {
-    if (state->d_lastIOState == IOState::NeedRead) {
-      if (ttd) {
-        /* let's update the TTD ! */
-        state->d_threadData.mplexer->setReadTTD(fd, *ttd, /* we pass 0 here because we already have a TTD */0);
-      }
-      return;
+    if (state->d_proxyProtocolPayloadHasTLV) {
+      /* we will not be able to reuse an existing connection anyway so let's add the payload right now */
+      addProxyProtocol(state->d_buffer, proxyProtocolPayload);
+      proxyProtocolPayloadAdded = true;
     }
-
-    state->d_lastIOState = IOState::NeedRead;
-    //cerr<<__func__<<": add read FD "<<fd<<endl;
-    state->d_threadData.mplexer->addReadFD(fd, callback, state, ttd ? &*ttd : nullptr);
   }
-  else if (iostate == IOState::NeedWrite) {
-    if (state->d_lastIOState == IOState::NeedWrite) {
-      return;
-    }
 
-    state->d_lastIOState = IOState::NeedWrite;
-    //cerr<<__func__<<": add write FD "<<fd<<endl;
-    state->d_threadData.mplexer->addWriteFD(fd, callback, state, ttd ? &*ttd : nullptr);
-  }
-  else if (iostate == IOState::Done) {
-    state->d_lastIOState = IOState::Done;
-  }
-}
+  auto downstreamConnection = state->getDownstreamConnection(state->d_ds, now);
+  downstreamConnection->assignToClientConnection(state);
 
-static void handleDownstreamIO(std::shared_ptr<IncomingTCPConnectionState>& state, struct timeval& now)
-{
-  if (state->d_downstreamConnection == nullptr) {
-    throw std::runtime_error("No downstream socket in " + std::string(__func__) + "!");
-  }
-
-  int fd = state->d_downstreamConnection->getHandle();
-  IOState iostate = IOState::Done;
-  bool connectionDied = false;
-
-  try {
-    if (state->d_state == IncomingTCPConnectionState::State::sendingQueryToBackend) {
-      int socketFlags = 0;
-#ifdef MSG_FASTOPEN
-      if (state->d_downstreamConnection->isFastOpenEnabled()) {
-        socketFlags |= MSG_FASTOPEN;
-      }
-#endif /* MSG_FASTOPEN */
-
-      size_t sent = sendMsgWithOptions(fd, reinterpret_cast<const char *>(&state->d_buffer.at(state->d_currentPos)), state->d_buffer.size() - state->d_currentPos, &state->d_ds->remote, &state->d_ds->sourceAddr, state->d_ds->sourceItf, socketFlags);
-      if (sent == state->d_buffer.size()) {
-        /* request sent ! */
-        state->d_downstreamConnection->incQueries();
-        state->d_state = IncomingTCPConnectionState::State::readingResponseSizeFromBackend;
-        state->d_currentPos = 0;
-        state->d_querySentTime = now;
-        iostate = IOState::NeedRead;
-        if (!state->d_isXFR && !state->d_outstanding) {
-          /* don't bother with the outstanding count for XFR queries */
-          ++state->d_ds->outstanding;
-          state->d_outstanding = true;
-        }
-      }
-      else {
-        state->d_currentPos += sent;
-        iostate = IOState::NeedWrite;
-        /* disable fast open on partial write */
-        state->d_downstreamConnection->disableFastOpen();
-      }
-    }
-
-    if (state->d_state == IncomingTCPConnectionState::State::readingResponseSizeFromBackend) {
-      // then we need to allocate a new buffer (new because we might need to re-send the query if the
-      // backend dies on us
-      // We also might need to read and send to the client more than one response in case of XFR (yeah!)
-      // should very likely be a TCPIOHandler d_downstreamHandler
-      iostate = tryRead(fd, state->d_responseBuffer, state->d_currentPos, sizeof(uint16_t) - state->d_currentPos);
-      if (iostate == IOState::Done) {
-        state->d_state = IncomingTCPConnectionState::State::readingResponseFromBackend;
-        state->d_responseSize = state->d_responseBuffer.at(0) * 256 + state->d_responseBuffer.at(1);
-        state->d_responseBuffer.resize((state->d_ids.dnsCryptQuery && (UINT16_MAX - state->d_responseSize) > static_cast<uint16_t>(DNSCRYPT_MAX_RESPONSE_PADDING_AND_MAC_SIZE)) ? state->d_responseSize + DNSCRYPT_MAX_RESPONSE_PADDING_AND_MAC_SIZE : state->d_responseSize);
-        state->d_currentPos = 0;
-      }
-    }
-
-    if (state->d_state == IncomingTCPConnectionState::State::readingResponseFromBackend) {
-      iostate = tryRead(fd, state->d_responseBuffer, state->d_currentPos, state->d_responseSize - state->d_currentPos);
-      if (iostate == IOState::Done) {
-        handleNewIOState(state, IOState::Done, fd, handleDownstreamIOCallback);
-
-        if (state->d_isXFR) {
-          /* Don't reuse the TCP connection after an {A,I}XFR */
-          /* but don't reset it either, we will need to read more messages */
-        }
-        else {
-          /* if we did not send a Proxy Protocol header, let's pool the connection */
-          if (state->d_ds && state->d_ds->useProxyProtocol == false) {
-            releaseDownstreamConnection(std::move(state->d_downstreamConnection));
-          }
-          else {
-            if (state->d_proxyProtocolPayloadHasTLV) {
-              /* sent a Proxy Protocol header with TLV values, we can't reuse it */
-              state->d_downstreamConnection.reset();
-            }
-            else {
-              /* if we did but there was no TLV values, let's try to reuse it but only
-                 for this incoming connection */
-            }
-          }
-        }
-        fd = -1;
-
-        state->d_responseReadTime = now;
-        try {
-          handleResponse(state, now);
-        }
-        catch (const std::exception& e) {
-          vinfolog("Got an exception while handling TCP response from %s (client is %s): %s", state->d_ds ? state->d_ds->getName() : "unknown", state->d_ci.remote.toStringWithPort(), e.what());
-        }
-        return;
-      }
-    }
-
-    if (state->d_state != IncomingTCPConnectionState::State::sendingQueryToBackend &&
-        state->d_state != IncomingTCPConnectionState::State::readingResponseSizeFromBackend &&
-        state->d_state != IncomingTCPConnectionState::State::readingResponseFromBackend) {
-      vinfolog("Unexpected state %d in handleDownstreamIOCallback", static_cast<int>(state->d_state));
-    }
-  }
-  catch(const std::exception& e) {
-    /* most likely an EOF because the other end closed the connection,
-       but it might also be a real IO error or something else.
-       Let's just drop the connection
-    */
-    vinfolog("Got an exception while handling (%s backend) TCP query from %s: %s", (state->d_lastIOState == IOState::NeedRead ? "reading from" : "writing to"), state->d_ci.remote.toStringWithPort(), e.what());
-    if (state->d_state == IncomingTCPConnectionState::State::sendingQueryToBackend) {
-      ++state->d_ds->tcpDiedSendingQuery;
-    }
-    else {
-      ++state->d_ds->tcpDiedReadingResponse;
-    }
-
-    /* don't increase this counter when reusing connections */
-    if (state->d_downstreamConnection && state->d_downstreamConnection->isFresh()) {
-      ++state->d_downstreamFailures;
-    }
-
-    if (state->d_outstanding) {
-      state->d_outstanding = false;
-
-      if (state->d_ds != nullptr) {
-        --state->d_ds->outstanding;
-      }
-    }
-    /* remove this FD from the IO multiplexer */
-    iostate = IOState::Done;
-    connectionDied = true;
-  }
-
-  if (iostate == IOState::Done) {
-    handleNewIOState(state, iostate, fd, handleDownstreamIOCallback);
+  if (proxyProtocolPayloadAdded) {
+    downstreamConnection->setProxyProtocolPayloadAdded(true);
   }
   else {
-    handleNewIOState(state, iostate, fd, handleDownstreamIOCallback, iostate == IOState::NeedRead ? state->getBackendReadTTD(now) : state->getBackendWriteTTD(now));
+    downstreamConnection->setProxyProtocolPayload(std::move(proxyProtocolPayload));
   }
+  downstreamConnection->queueQuery(TCPQuery(std::move(state->d_buffer), std::move(ids)), downstreamConnection);
 
-  if (connectionDied) {
-    state->d_downstreamConnection.reset();
-    sendQueryToBackend(state, now);
-  }
+  //sendQueryToBackend(state, now);
 }
 
-static void handleDownstreamIOCallback(int fd, FDMultiplexer::funcparam_t& param)
+void IncomingTCPConnectionState::handleIOCallback(int fd, FDMultiplexer::funcparam_t& param)
 {
-  auto state = boost::any_cast<std::shared_ptr<IncomingTCPConnectionState>>(param);
-  if (state->d_downstreamConnection == nullptr) {
-    throw std::runtime_error("No downstream socket in " + std::string(__func__) + "!");
-  }
-  if (fd != state->d_downstreamConnection->getHandle()) {
-    throw std::runtime_error("Unexpected socket descriptor " + std::to_string(fd) + " received in " + std::string(__func__) + ", expected " + std::to_string(state->d_downstreamConnection->getHandle()));
+  auto conn = boost::any_cast<std::shared_ptr<IncomingTCPConnectionState>>(param);
+  if (fd != conn->d_ci.fd) {
+    throw std::runtime_error("Unexpected socket descriptor " + std::to_string(fd) + " received in " + std::string(__func__) + ", expected " + std::to_string(conn->d_ci.fd));
   }
 
   struct timeval now;
   gettimeofday(&now, 0);
-  handleDownstreamIO(state, now);
+  handleIO(conn, now);
 }
 
-static void handleIO(std::shared_ptr<IncomingTCPConnectionState>& state, struct timeval& now)
+void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
 {
-  int fd = state->d_ci.fd;
+  cerr<<"in "<<__func__<<endl;
   IOState iostate = IOState::Done;
+  IOStateGuard ioGuard(state->d_ioState);
 
   if (state->maxConnectionDurationReached(g_maxTCPConnectionDuration, now)) {
     vinfolog("Terminating TCP connection from %s because it reached the maximum TCP connection duration", state->d_ci.remote.toStringWithPort());
-    handleNewIOState(state, IOState::Done, fd, handleIOCallback);
+    // will be handled by the ioGuard
+    //handleNewIOState(state, IOState::Done, fd, handleIOCallback);
     return;
   }
 
   try {
     if (state->d_state == IncomingTCPConnectionState::State::doingHandshake) {
+      cerr<<"doing handshake"<<endl;
       iostate = state->d_handler.tryHandshake();
       if (iostate == IOState::Done) {
+        cerr<<"handshake done"<<endl;
         if (state->d_handler.isTLS()) {
           if (!state->d_handler.hasTLSSessionBeenResumed()) {
             ++state->d_ci.cs->tlsNewSessions;
@@ -1212,8 +649,10 @@ static void handleIO(std::shared_ptr<IncomingTCPConnectionState>& state, struct
     }
 
     if (state->d_state == IncomingTCPConnectionState::State::readingQuerySize) {
+      cerr<<"reading query size"<<endl;
       iostate = state->d_handler.tryRead(state->d_buffer, state->d_currentPos, sizeof(uint16_t));
       if (iostate == IOState::Done) {
+        cerr<<"query size received"<<endl;
         state->d_state = IncomingTCPConnectionState::State::readingQuery;
         state->d_querySizeReadTime = now;
         if (state->d_queriesCount == 0) {
@@ -1222,7 +661,8 @@ static void handleIO(std::shared_ptr<IncomingTCPConnectionState>& state, struct
         state->d_querySize = state->d_buffer.at(0) * 256 + state->d_buffer.at(1);
         if (state->d_querySize < sizeof(dnsheader)) {
           /* go away */
-          handleNewIOState(state, IOState::Done, fd, handleIOCallback);
+          // will be handled by the guard
+          //handleNewIOState(state, IOState::Done, fd, handleIOCallback);
           return;
         }
 
@@ -1234,10 +674,15 @@ static void handleIO(std::shared_ptr<IncomingTCPConnectionState>& state, struct
     }
 
     if (state->d_state == IncomingTCPConnectionState::State::readingQuery) {
+      cerr<<"reading query"<<endl;
       iostate = state->d_handler.tryRead(state->d_buffer, state->d_currentPos, state->d_querySize);
       if (iostate == IOState::Done) {
-        handleNewIOState(state, IOState::Done, fd, handleIOCallback);
+        cerr<<"query received"<<endl;
+        //handleNewIOState(state, IOState::Done, fd, handleIOCallback);
         handleQuery(state, now);
+        state->resetForNewQuery();
+        state->d_ioState->update(IOState::NeedRead, handleIOCallback, state, state->getClientReadTTD(now));
+        ioGuard.release();
         return;
       }
     }
@@ -1246,6 +691,8 @@ static void handleIO(std::shared_ptr<IncomingTCPConnectionState>& state, struct
       iostate = state->d_handler.tryWrite(state->d_responseBuffer, state->d_currentPos, state->d_responseBuffer.size());
       if (iostate == IOState::Done) {
         handleResponseSent(state, now);
+        state->d_ioState->update(IOState::NeedRead, handleIOCallback, state, state->getClientReadTTD(now));
+        ioGuard.release();
         return;
       }
     }
@@ -1282,23 +729,22 @@ static void handleIO(std::shared_ptr<IncomingTCPConnectionState>& state, struct
   }
 
   if (iostate == IOState::Done) {
-    handleNewIOState(state, iostate, fd, handleIOCallback);
+    state->d_ioState->update(iostate, handleIOCallback, state);
   }
   else {
-    handleNewIOState(state, iostate, fd, handleIOCallback, iostate == IOState::NeedRead ? state->getClientReadTTD(now) : state->getClientWriteTTD(now));
+    state->d_ioState->update(iostate, handleIOCallback, state, iostate == IOState::NeedRead ? state->getClientReadTTD(now) : state->getClientWriteTTD(now));
   }
+  ioGuard.release();
 }
 
-static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param)
+void IncomingTCPConnectionState::notifyIOError(IDState&& query)
 {
-  auto state = boost::any_cast<std::shared_ptr<IncomingTCPConnectionState>>(param);
-  if (fd != state->d_ci.fd) {
-    throw std::runtime_error("Unexpected socket descriptor " + std::to_string(fd) + " received in " + std::string(__func__) + ", expected " + std::to_string(state->d_ci.fd));
-  }
-  struct timeval now;
-  gettimeofday(&now, 0);
+  #warning NEED TO BE IMPLEMENTED
+}
 
-  handleIO(state, now);
+void IncomingTCPConnectionState::sendXFRResponse(std::vector<uint8_t>&& response, const IDState& query)
+{
+  #warning NEED TO BE IMPLEMENTED
 }
 
 static void handleIncomingTCPQuery(int pipefd, FDMultiplexer::funcparam_t& param)
@@ -1333,7 +779,7 @@ static void handleIncomingTCPQuery(int pipefd, FDMultiplexer::funcparam_t& param
     /* let's update the remaining time */
     state->d_remainingTime = g_maxTCPConnectionDuration;
 
-    handleIO(state, now);
+    IncomingTCPConnectionState::handleIO(state, now);
   }
   catch(...) {
     delete citmp;
@@ -1342,7 +788,7 @@ static void handleIncomingTCPQuery(int pipefd, FDMultiplexer::funcparam_t& param
   }
 }
 
-void tcpClientThread(int pipefd)
+static void tcpClientThread(int pipefd)
 {
   /* we get launched with a pipe on which we receive file descriptors from clients that we own
      from that point on */
@@ -1361,7 +807,7 @@ void tcpClientThread(int pipefd)
     data.mplexer->run(&now);
 
     if (g_downstreamTCPCleanupInterval > 0 && (now.tv_sec > (lastTCPCleanup + g_downstreamTCPCleanupInterval))) {
-      cleanupClosedTCPConnections();
+      DownstreamConnectionsManager::cleanupClosedTCPConnections();
       lastTCPCleanup = now.tv_sec;
     }
 
diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh
index 55c0868647..fbdc720d5c 100644
--- a/pdns/dnsdist.hh
+++ b/pdns/dnsdist.hh
@@ -465,13 +465,60 @@ struct ClientState;
 struct IDState
 {
   IDState(): sentTime(true), delayMsec(0), tempFailureTTL(boost::none) { origDest.sin4.sin_family = 0;}
-  IDState(const IDState& orig): origRemote(orig.origRemote), origDest(orig.origDest), age(orig.age)
+  IDState(const IDState& orig) = delete;
+  IDState(IDState&& rhs): origRemote(rhs.origRemote), origDest(rhs.origDest), sentTime(rhs.sentTime), qname(std::move(rhs.qname)), dnsCryptQuery(std::move(rhs.dnsCryptQuery)), subnet(rhs.subnet), packetCache(std::move(rhs.packetCache)), qTag(std::move(qTag)), cs(rhs.cs), du(std::move(rhs.du)), cacheKey(rhs.cacheKey), cacheKeyNoECS(rhs.cacheKeyNoECS), age(rhs.age), qtype(rhs.qtype), qclass(rhs.qclass), origID(rhs.origID), origFlags(rhs.origFlags), origFD(rhs.origFD), delayMsec(rhs.delayMsec), tempFailureTTL(rhs.tempFailureTTL), ednsAdded(rhs.ednsAdded), ecsAdded(rhs.ecsAdded), skipCache(rhs.skipCache), destHarvested(rhs.destHarvested), dnssecOK(rhs.dnssecOK), useZeroScope(rhs.useZeroScope)
   {
-    usageIndicator.store(orig.usageIndicator.load());
-    origFD = orig.origFD;
-    origID = orig.origID;
-    delayMsec = orig.delayMsec;
-    tempFailureTTL = orig.tempFailureTTL;
+    if (rhs.isInUse()) {
+      throw std::runtime_error("Trying to move an in-use IDState");
+    }
+
+#ifdef HAVE_PROTOBUF
+    uniqueId = std::move(rhs.uniqueId);
+#endif
+  }
+
+  IDState& operator=(IDState&& rhs)
+  {
+    if (isInUse()) {
+      throw std::runtime_error("Trying to overwrite an in-use IDState");
+    }
+
+    if (rhs.isInUse()) {
+      throw std::runtime_error("Trying to move an in-use IDState");
+    }
+
+    origRemote = rhs.origRemote;
+    origDest = rhs.origDest;
+    sentTime = rhs.sentTime;
+    qname = std::move(rhs.qname);
+    dnsCryptQuery = std::move(rhs.dnsCryptQuery);
+    subnet = rhs.subnet;
+    packetCache = std::move(rhs.packetCache);
+    qTag = std::move(qTag);
+    cs = rhs.cs;
+    du = std::move(rhs.du);
+    cacheKey = rhs.cacheKey;
+    cacheKeyNoECS = rhs.cacheKeyNoECS;
+    age = rhs.age;
+    qtype = rhs.qtype;
+    qclass = rhs.qclass;
+    origID = rhs.origID;
+    origFlags = rhs.origFlags;
+    origFD = rhs.origFD;
+    delayMsec = rhs.delayMsec;
+    tempFailureTTL = rhs.tempFailureTTL;
+    ednsAdded = rhs.ednsAdded;
+    ecsAdded = rhs.ecsAdded;
+    skipCache = rhs.skipCache;
+    destHarvested = rhs.destHarvested;
+    dnssecOK = rhs.dnssecOK;
+    useZeroScope = rhs.useZeroScope;
+
+#ifdef HAVE_PROTOBUF
+    uniqueId = std::move(rhs.uniqueId);
+#endif
+
+    return *this;
   }
 
   static const int64_t unusedIndicator = -1;
diff --git a/pdns/dnsdistdist/Makefile.am b/pdns/dnsdistdist/Makefile.am
index 8471bea6d3..7ce6fb370d 100644
--- a/pdns/dnsdistdist/Makefile.am
+++ b/pdns/dnsdistdist/Makefile.am
@@ -159,6 +159,8 @@ dnsdist_SOURCES = \
 	dnsdist-snmp.cc dnsdist-snmp.hh \
 	dnsdist-systemd.cc dnsdist-systemd.hh \
 	dnsdist-tcp.cc \
+	dnsdist-tcp-downstream.cc dnsdist-tcp-downstream.hh \
+	dnsdist-tcp-upstream.hh \
 	dnsdist-web.cc dnsdist-web.hh \
 	dnsdist-xpf.cc dnsdist-xpf.hh \
 	dnslabeltext.cc \
@@ -194,6 +196,7 @@ dnsdist_SOURCES = \
 	svc-records.cc svc-records.hh \
 	statnode.cc statnode.hh \
 	tcpiohandler.cc tcpiohandler.hh \
+	tcpiohandler-mplexer.hh \
 	threadname.hh threadname.cc \
 	uuid-utils.hh uuid-utils.cc \
 	views.hh \
diff --git a/pdns/dnsdistdist/dnsdist-backend.cc b/pdns/dnsdistdist/dnsdist-backend.cc
index 38398e28eb..2eb19a8c59 100644
--- a/pdns/dnsdistdist/dnsdist-backend.cc
+++ b/pdns/dnsdistdist/dnsdist-backend.cc
@@ -150,7 +150,7 @@ void DownstreamState::setWeight(int newWeight)
   }
 }
 
-DownstreamState::DownstreamState(const ComboAddress& remote_, const ComboAddress& sourceAddr_, unsigned int sourceItf_, const std::string& sourceItfName_, size_t numberOfSockets, bool connect=true): sourceItfName(sourceItfName_), remote(remote_), sourceAddr(sourceAddr_), sourceItf(sourceItf_), name(remote_.toStringWithPort()), nameWithAddr(remote_.toStringWithPort())
+DownstreamState::DownstreamState(const ComboAddress& remote_, const ComboAddress& sourceAddr_, unsigned int sourceItf_, const std::string& sourceItfName_, size_t numberOfSockets, bool connect=true): sourceItfName(sourceItfName_), remote(remote_), idStates(g_maxOutstanding), sourceAddr(sourceAddr_), sourceItf(sourceItf_), name(remote_.toStringWithPort()), nameWithAddr(remote_.toStringWithPort())
 {
   id = getUniqueID();
   threadStarted.clear();
@@ -164,7 +164,6 @@ DownstreamState::DownstreamState(const ComboAddress& remote_, const ComboAddress
 
   if (connect && !IsAnyAddress(remote)) {
     reconnect();
-    idStates.resize(g_maxOutstanding);
     sw.start();
   }
 }
diff --git a/pdns/dnsdistdist/dnsdist-proxy-protocol.cc b/pdns/dnsdistdist/dnsdist-proxy-protocol.cc
index 083b0d345a..e7773a2003 100644
--- a/pdns/dnsdistdist/dnsdist-proxy-protocol.cc
+++ b/pdns/dnsdistdist/dnsdist-proxy-protocol.cc
@@ -22,9 +22,13 @@
 
 #include "dnsdist-proxy-protocol.hh"
 
-bool addProxyProtocol(DNSQuestion& dq)
+std::string getProxyProtocolPayload(const DNSQuestion& dq)
+{
+  return makeProxyHeader(dq.tcp, *dq.remote, *dq.local, dq.proxyProtocolValues ? *dq.proxyProtocolValues : std::vector<ProxyProtocolValue>());
+}
+
+bool addProxyProtocol(DNSQuestion& dq, const std::string& payload)
 {
-  auto payload = makeProxyHeader(dq.tcp, *dq.remote, *dq.local, dq.proxyProtocolValues ? *dq.proxyProtocolValues : std::vector<ProxyProtocolValue>());
   if ((dq.size - dq.len) < payload.size()) {
     return false;
   }
@@ -36,10 +40,14 @@ bool addProxyProtocol(DNSQuestion& dq)
   return true;
 }
 
-bool addProxyProtocol(std::vector<uint8_t>& buffer, bool tcp, const ComboAddress& source, const ComboAddress& destination, const std::vector<ProxyProtocolValue>& values)
+bool addProxyProtocol(DNSQuestion& dq)
 {
-  auto payload = makeProxyHeader(tcp, source, destination, values);
+  auto payload = getProxyProtocolPayload(dq);
+  return addProxyProtocol(dq, payload);
+}
 
+bool addProxyProtocol(std::vector<uint8_t>& buffer, const std::string& payload)
+{
   auto previousSize = buffer.size();
   if (payload.size() > (std::numeric_limits<size_t>::max() - previousSize)) {
     return false;
@@ -51,3 +59,9 @@ bool addProxyProtocol(std::vector<uint8_t>& buffer, bool tcp, const ComboAddress
 
   return true;
 }
+
+bool addProxyProtocol(std::vector<uint8_t>& buffer, bool tcp, const ComboAddress& source, const ComboAddress& destination, const std::vector<ProxyProtocolValue>& values)
+{
+  auto payload = makeProxyHeader(tcp, source, destination, values);
+  return addProxyProtocol(buffer, payload);
+}
diff --git a/pdns/dnsdistdist/dnsdist-proxy-protocol.hh b/pdns/dnsdistdist/dnsdist-proxy-protocol.hh
index 433a7d2394..a218a403d4 100644
--- a/pdns/dnsdistdist/dnsdist-proxy-protocol.hh
+++ b/pdns/dnsdistdist/dnsdist-proxy-protocol.hh
@@ -23,5 +23,9 @@
 
 #include "dnsdist.hh"
 
+std::string getProxyProtocolPayload(const DNSQuestion& dq);
+
 bool addProxyProtocol(DNSQuestion& dq);
+bool addProxyProtocol(DNSQuestion& dq, const std::string& payload);
+bool addProxyProtocol(std::vector<uint8_t>& buffer, const std::string& payload);
 bool addProxyProtocol(std::vector<uint8_t>& buffer, bool tcp, const ComboAddress& source, const ComboAddress& destination, const std::vector<ProxyProtocolValue>& values);
diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
new file mode 100644
index 0000000000..7e8741e380
--- /dev/null
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
@@ -0,0 +1,338 @@
+
+#include "dnsdist-tcp-downstream.hh"
+#include "dnsdist-tcp-upstream.hh"
+
+const uint16_t TCPConnectionToBackend::s_xfrID = 0;
+
+void TCPConnectionToBackend::assignToClientConnection(std::shared_ptr<IncomingTCPConnectionState>& clientConn)
+{
+  cerr<<"in "<<__func__<<endl;
+  // taking a WEAK pointer
+  d_clientConn = clientConn;
+  d_ioState = make_unique<IOStateHandler>(clientConn->getIOMPlexer(), d_socket->getHandle());
+}
+
+void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now)
+{
+  cerr<<"in "<<__func__<<endl;
+  if (conn->d_socket == nullptr) {
+    throw std::runtime_error("No downstream socket in " + std::string(__func__) + "!");
+  }
+
+  bool connectionDied = false;
+  IOState iostate = IOState::Done;
+  IOStateGuard ioGuard(conn->d_ioState);
+  int fd = conn->d_socket->getHandle();
+
+  try {
+    if (conn->d_state == State::sendingQueryToBackend) {
+      cerr<<"sending query to backend"<<endl;
+      int socketFlags = 0;
+#ifdef MSG_FASTOPEN
+      if (conn->isFastOpenEnabled()) {
+        socketFlags |= MSG_FASTOPEN;
+      }
+#endif /* MSG_FASTOPEN */
+
+      size_t sent = sendMsgWithOptions(fd, reinterpret_cast<const char *>(&conn->d_currentQuery.d_buffer.at(conn->d_currentPos)), conn->d_currentQuery.d_buffer.size() - conn->d_currentPos, &conn->d_ds->remote, &conn->d_ds->sourceAddr, conn->d_ds->sourceItf, socketFlags);
+      if (sent == conn->d_currentQuery.d_buffer.size()) {
+        cerr<<"query sent to backend"<<endl;
+        /* request sent ! */
+        conn->incQueries();
+        conn->d_currentPos = 0;
+        //conn->d_currentQuery.d_querySentTime = now;
+        conn->d_pendingResponses.insert({ conn->d_currentQuery.d_idstate.origID, std::move(conn->d_currentQuery.d_idstate) });
+        conn->d_currentQuery.d_buffer.clear();
+#if 0
+        if (!conn->d_usedForXFR) {
+          /* don't bother with the outstanding count for XFR queries */
+          ++conn->d_ds->outstanding;
+          ++conn->d_outstanding;
+        }
+#endif
+
+        if (conn->d_pendingQueries.empty()) {
+          conn->d_state = State::readingResponseSizeFromBackend;
+          iostate = IOState::NeedRead;
+        }
+        else {
+          conn->d_currentQuery = std::move(conn->d_pendingQueries.front());
+          conn->d_pendingQueries.pop_front();
+          conn->d_state = State::sendingQueryToBackend;
+          iostate = IOState::NeedWrite;
+        }
+      }
+      else {
+        conn->d_currentPos += sent;
+        iostate = IOState::NeedWrite;
+        /* disable fast open on partial write */
+        conn->disableFastOpen();
+      }
+    }
+
+    if (conn->d_state == State::readingResponseSizeFromBackend) {
+      cerr<<"reading response size from backend"<<endl;
+      // then we need to allocate a new buffer (new because we might need to re-send the query if the
+      // backend dies on us)
+      // We also might need to read and send to the client more than one response in case of XFR (yeah!)
+      // should very likely be a TCPIOHandler d_downstreamHandler
+      iostate = tryRead(fd, conn->d_responseBuffer, conn->d_currentPos, sizeof(uint16_t) - conn->d_currentPos);
+      if (iostate == IOState::Done) {
+        cerr<<"got response size from backend"<<endl;
+        conn->d_state = State::readingResponseFromBackend;
+        conn->d_responseSize = conn->d_responseBuffer.at(0) * 256 + conn->d_responseBuffer.at(1);
+        conn->d_responseBuffer.resize(conn->d_responseSize);
+        conn->d_currentPos = 0;
+      }
+    }
+
+    if (conn->d_state == State::readingResponseFromBackend) {
+      cerr<<"reading response from backend"<<endl;
+      iostate = tryRead(fd, conn->d_responseBuffer, conn->d_currentPos, conn->d_responseSize - conn->d_currentPos);
+      if (iostate == IOState::Done) {
+        cerr<<"got response from backend"<<endl;
+        //conn->d_responseReadTime = now;
+        try {
+          iostate = conn->handleResponse(now);
+        }
+        catch (const std::exception& e) {
+          vinfolog("Got an exception while handling TCP response from %s (client is %s): %s", conn->d_ds ? conn->d_ds->getName() : "unknown", conn->d_currentQuery.d_idstate.origRemote.toStringWithPort(), e.what());
+        }
+        return;
+      }
+    }
+
+    if (conn->d_state != State::sendingQueryToBackend &&
+        conn->d_state != State::readingResponseSizeFromBackend &&
+        conn->d_state != State::readingResponseFromBackend) {
+      vinfolog("Unexpected state %d in TCPConnectionToBackend::handleIO", static_cast<int>(conn->d_state));
+    }
+  }
+  catch (const std::exception& e) {
+    /* most likely an EOF because the other end closed the connection,
+       but it might also be a real IO error or something else.
+       Let's just drop the connection
+    */
+    vinfolog("Got an exception while handling (%s backend) TCP query from %s: %s", (conn->d_ioState->getState() == IOState::NeedRead ? "reading from" : "writing to"), conn->d_currentQuery.d_idstate.origRemote.toStringWithPort(), e.what());
+    if (conn->d_state == State::sendingQueryToBackend) {
+      ++conn->d_ds->tcpDiedSendingQuery;
+    }
+    else {
+      ++conn->d_ds->tcpDiedReadingResponse;
+    }
+
+    /* don't increase this counter when reusing connections */
+    if (conn->d_fresh) {
+      ++conn->d_downstreamFailures;
+    }
+
+#if 0
+    if (conn->d_outstanding) {
+      conn->d_outstanding = false;
+
+      if (conn->d_ds != nullptr) {
+        --conn->d_ds->outstanding;
+      }
+    }
+#endif
+    /* remove this FD from the IO multiplexer */
+    iostate = IOState::Done;
+    connectionDied = true;
+  }
+
+  if (iostate == IOState::Done) {
+    cerr<<"in "<<__func__<<", done"<<endl;
+    conn->d_ioState->update(iostate, handleIOCallback, conn);
+  }
+  else {
+    cerr<<"in "<<__func__<<", updating to "<<(int)iostate<<endl;
+    conn->d_ioState->update(iostate, handleIOCallback, conn, iostate == IOState::NeedRead ? conn->getBackendReadTTD(now) : conn->getBackendWriteTTD(now));
+  }
+  ioGuard.release();
+
+  if (connectionDied) {
+#warning FIXME: if we had only one question pending we might try sending it again
+    conn->d_connectionDied = true;
+
+    conn->notifyAllQueriesFailed();
+  }
+}
+
+void TCPConnectionToBackend::handleIOCallback(int fd, FDMultiplexer::funcparam_t& param)
+{
+  auto conn = boost::any_cast<std::shared_ptr<TCPConnectionToBackend>>(param);
+  if (fd != conn->getHandle()) {
+    throw std::runtime_error("Unexpected socket descriptor " + std::to_string(fd) + " received in " + std::string(__func__) + ", expected " + std::to_string(conn->getHandle()));
+  }
+
+  struct timeval now;
+  gettimeofday(&now, 0);
+  handleIO(conn, now);
+}
+
+void TCPConnectionToBackend::queueQuery(TCPQuery&& query, std::shared_ptr<TCPConnectionToBackend>& sharedSelf)
+{
+  cerr<<"in "<<__func__<<endl;
+  if (d_ioState == nullptr) {
+    throw std::runtime_error("Trying to queue a query to a TCP connection that has no incoming client connection assigned");
+  }
+
+  // if we are not already sending a query or in the middle of reading a response (so idle or doingHandshake),
+  // start sending the query
+  if (d_state == State::idle || d_state == State::waitingForResponseFromBackend) {
+    d_state = State::sendingQueryToBackend;
+    d_currentQuery = std::move(query);
+    cerr<<"need write"<<endl;
+
+    struct timeval now;
+    gettimeofday(&now, 0);
+
+    d_ioState->update(IOState::NeedWrite, handleIOCallback, sharedSelf, getBackendWriteTTD(now));
+  }
+  else {
+    // store query in the list of queries to send
+    d_pendingQueries.push_back(std::move(query));
+  }
+}
+
+bool TCPConnectionToBackend::reconnect()
+{
+  std::unique_ptr<Socket> result;
+
+  if (d_socket) {
+    d_socket.reset();
+    d_ioState.reset();
+    --d_ds->tcpCurrentConnections;
+  }
+
+  do {
+    vinfolog("TCP connecting to downstream %s (%d)", d_ds->remote.toStringWithPort(), d_downstreamFailures);
+    try {
+      result = std::unique_ptr<Socket>(new Socket(d_ds->remote.sin4.sin_family, SOCK_STREAM, 0));
+      if (!IsAnyAddress(d_ds->sourceAddr)) {
+        SSetsockopt(result->getHandle(), SOL_SOCKET, SO_REUSEADDR, 1);
+#ifdef IP_BIND_ADDRESS_NO_PORT
+        if (d_ds->ipBindAddrNoPort) {
+          SSetsockopt(result->getHandle(), SOL_IP, IP_BIND_ADDRESS_NO_PORT, 1);
+        }
+#endif
+#ifdef SO_BINDTODEVICE
+        if (!d_ds->sourceItfName.empty()) {
+          int res = setsockopt(result->getHandle(), SOL_SOCKET, SO_BINDTODEVICE, d_ds->sourceItfName.c_str(), d_ds->sourceItfName.length());
+          if (res != 0) {
+            vinfolog("Error setting up the interface on backend TCP socket '%s': %s", d_ds->getNameWithAddr(), stringerror());
+          }
+        }
+#endif
+        result->bind(d_ds->sourceAddr, false);
+      }
+      result->setNonBlocking();
+#ifdef MSG_FASTOPEN
+      if (!d_ds->tcpFastOpen) {
+        SConnectWithTimeout(result->getHandle(), d_ds->remote, /* no timeout, we will handle it ourselves */ 0);
+      }
+#else
+      SConnectWithTimeout(result->getHandle(), d_ds->remote, /* no timeout, we will handle it ourselves */ 0);
+#endif /* MSG_FASTOPEN */
+
+      d_socket = std::move(result);
+      //d_ioState->setSocket(d_socket->getHandle());
+      ++d_ds->tcpCurrentConnections;
+      return true;
+    }
+    catch(const std::runtime_error& e) {
+      vinfolog("Connection to downstream server %s failed: %s", d_ds->getName(), e.what());
+      d_downstreamFailures++;
+      if (d_downstreamFailures > d_ds->retries) {
+        throw;
+      }
+    }
+  }
+  while (d_downstreamFailures <= d_ds->retries);
+
+  return false;
+}
+
+void TCPConnectionToBackend::notifyAllQueriesFailed()
+{
+  d_connectionDied = true;
+  auto clientConn = d_clientConn.lock();
+  if (!clientConn) {
+    return;
+  }
+
+  for (auto& query : d_pendingQueries) {
+    clientConn->notifyIOError(std::move(query.d_idstate));
+  }
+
+  for (auto& response : d_pendingResponses) {
+    clientConn->notifyIOError(std::move(response.second));
+  }
+}
+
+IOState TCPConnectionToBackend::handleResponse(const struct timeval& now)
+{
+  cerr<<"in "<<__func__<<endl;
+  auto clientConn = d_clientConn.lock();
+  if (!clientConn) {
+    d_connectionDied = true;
+    cerr<<"connection to client died, bye bye"<<endl;
+    return IOState::Done;
+  }
+
+  if (d_usedForXFR) {
+    clientConn->sendXFRResponse(std::move(d_responseBuffer), d_pendingResponses.at(s_xfrID));
+    return IOState::NeedRead;
+    // get ready to read the next packet, if any
+  }
+  else {
+    uint16_t queryId = getQueryIdFromResponse();
+    if (d_pendingResponses.count(queryId) == 0) {
+      notifyAllQueriesFailed();
+      return IOState::Done;
+    }
+    cerr<<"passing response to client connection"<<endl;
+    clientConn->sendResponse(now, std::move(d_responseBuffer), std::move(d_pendingResponses.at(queryId)));
+    d_pendingResponses.erase(queryId);
+
+    if (!d_pendingQueries.empty()) {
+      cerr<<"still have some queries to send"<<endl;
+      d_state = State::sendingQueryToBackend;
+      d_currentQuery = std::move(d_pendingQueries.front());
+      d_pendingQueries.pop_front();
+      return IOState::NeedWrite;
+    }
+    else if (!d_pendingResponses.empty()) {
+      cerr<<"still have some responses to read"<<endl;
+      d_state = State::readingResponseSizeFromBackend;
+      return IOState::NeedRead;
+    }
+    else {
+      cerr<<"nothing to do, phewwwww"<<endl;
+      d_state = State::idle;
+      return IOState::Done;
+    }
+  }
+}
+
+uint16_t TCPConnectionToBackend::getQueryIdFromResponse()
+{
+  if (d_responseBuffer.size() < sizeof(dnsheader)) {
+    notifyAllQueriesFailed();
+    throw std::runtime_error("Unable to get query ID in a too small (" + std::to_string(d_responseBuffer.size()) + ") response from " + d_ds->getNameWithAddr());
+  }
+
+  dnsheader dh;
+  memcpy(&dh, &d_responseBuffer.at(0), sizeof(dh));
+  return ntohs(dh.id);
+}
+
+void TCPConnectionToBackend::setProxyProtocolPayload(std::string&& payload)
+{
+  #warning TO DO
+}
+
+void TCPConnectionToBackend::setProxyProtocolPayloadAdded(bool added)
+{
+  #warning TO DO
+}
diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
new file mode 100644
index 0000000000..84149d8943
--- /dev/null
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
@@ -0,0 +1,193 @@
+#pragma once
+
+#include <queue>
+
+#include "sstuff.hh"
+#include "tcpiohandler-mplexer.hh"
+#include "dnsdist.hh"
+
+struct TCPQuery
+{
+  TCPQuery()
+  {
+  }
+
+  TCPQuery(std::vector<uint8_t>&& buffer, IDState&& state): d_idstate(std::move(state)), d_buffer(std::move(buffer))
+  {
+  }
+
+  IDState d_idstate;
+  std::vector<uint8_t> d_buffer;
+};
+
+struct TCPResponse : public TCPQuery
+{
+};
+
+class IncomingTCPConnectionState;
+
+class TCPConnectionToBackend
+{
+public:
+  TCPConnectionToBackend(std::shared_ptr<DownstreamState>& ds, const struct timeval& now): d_responseBuffer(s_maxPacketCacheEntrySize), d_ds(ds), d_connectionStartTime(now), d_enableFastOpen(ds->tcpFastOpen)
+  {
+    reconnect();
+  }
+
+  ~TCPConnectionToBackend()
+  {
+    if (d_ds && d_socket) {
+      --d_ds->tcpCurrentConnections;
+      struct timeval now;
+      gettimeofday(&now, nullptr);
+
+      auto diff = now - d_connectionStartTime;
+      d_ds->updateTCPMetrics(d_queries, diff.tv_sec * 1000 + diff.tv_usec / 1000);
+    }
+  }
+
+  void assignToClientConnection(std::shared_ptr<IncomingTCPConnectionState>& clientConn);
+
+  int getHandle() const
+  {
+    if (!d_socket) {
+      throw std::runtime_error("Attempt to get the socket handle from a non-established TCP connection");
+    }
+
+    return d_socket->getHandle();
+  }
+
+  const ComboAddress& getRemote() const
+  {
+    return d_ds->remote;
+  }
+
+  bool isFresh() const
+  {
+    return d_fresh;
+  }
+
+  void incQueries()
+  {
+    ++d_queries;
+  }
+
+  void setReused()
+  {
+    d_fresh = false;
+  }
+
+  void disableFastOpen()
+  {
+    d_enableFastOpen = false;
+  }
+
+  bool isFastOpenEnabled()
+  {
+    return d_enableFastOpen;
+  }
+
+  bool canAcceptNewQueries() const
+  {
+    if (d_usedForXFR || d_connectionDied) {
+      return false;
+      /* Don't reuse the TCP connection after an {A,I}XFR */
+      /* but don't reset it either, we will need to read more messages */
+    }
+#warning FIXME: maximum number of pending queries
+    return true;
+  }
+
+  bool canBeReused() const 
+  {
+    if (d_usedForXFR || d_connectionDied) {
+      return false;
+    }
+    /* we can't reuse a connection where a proxy protocol payload has been sent,
+       since:
+       - it cannot be reused for a different client
+       - we might have different TLV values for each query
+    */
+    if (d_ds && d_ds->useProxyProtocol == true) {
+      return false;
+    }
+    return true;
+  }
+
+  bool matches(const std::shared_ptr<DownstreamState>& ds) const
+  {
+    if (!ds || !d_ds) {
+      return false;
+    }
+    return ds == d_ds;
+  }
+
+  static void handleIO(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now);
+  static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param);
+
+  void queueQuery(TCPQuery&& query, std::shared_ptr<TCPConnectionToBackend>& sharedSelf);
+  IOState handleResponse(const struct timeval& now);
+  void setProxyProtocolPayload(std::string&& payload);
+  void setProxyProtocolPayloadAdded(bool added);
+
+private:
+  uint16_t getQueryIdFromResponse();
+  bool reconnect();
+  void notifyAllQueriesFailed();
+
+  boost::optional<struct timeval> getBackendReadTTD(const struct timeval& now) const
+  {
+    if (d_ds == nullptr) {
+      throw std::runtime_error("getBackendReadTTD() without any backend selected");
+    }
+    if (d_ds->tcpRecvTimeout == 0) {
+      return boost::none;
+    }
+
+    struct timeval res = now;
+    res.tv_sec += d_ds->tcpRecvTimeout;
+
+    return res;
+  }
+
+  boost::optional<struct timeval> getBackendWriteTTD(const struct timeval& now) const
+  {
+    if (d_ds == nullptr) {
+      throw std::runtime_error("getBackendReadTTD() called without any backend selected");
+    }
+    if (d_ds->tcpSendTimeout == 0) {
+      return boost::none;
+    }
+
+    struct timeval res = now;
+    res.tv_sec += d_ds->tcpSendTimeout;
+
+    return res;
+  }
+
+  /* waitingForResponseFromBackend is a state where we have not yet started reading the size,
+     so we can still switch to sending instead */
+  enum class State { idle, doingHandshake, sendingQueryToBackend, waitingForResponseFromBackend, readingResponseSizeFromBackend, readingResponseFromBackend };
+  static const uint16_t s_xfrID;
+
+  std::vector<uint8_t> d_responseBuffer;
+  std::deque<TCPQuery> d_pendingQueries;
+  std::unordered_map<uint16_t, IDState> d_pendingResponses;
+  std::unique_ptr<Socket> d_socket{nullptr};
+  std::unique_ptr<IOStateHandler> d_ioState{nullptr};
+  std::shared_ptr<DownstreamState> d_ds{nullptr};
+  std::weak_ptr<IncomingTCPConnectionState> d_clientConn;
+  TCPQuery d_currentQuery;
+  struct timeval d_connectionStartTime;
+  size_t d_currentPos{0};
+  uint64_t d_queries{0};
+  uint64_t d_downstreamFailures{0};
+  uint16_t d_responseSize{0};
+  State d_state{State::idle};
+  bool d_fresh{true};
+  bool d_enableFastOpen{false};
+  bool d_connectionDied{true};
+#warning FIXME : must be set!
+  bool d_usedForXFR{false};
+};
+
diff --git a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
new file mode 100644
index 0000000000..cc624321d8
--- /dev/null
+++ b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
@@ -0,0 +1,237 @@
+#pragma once
+
+#include "dolog.hh"
+
+class TCPClientThreadData
+{
+public:
+  TCPClientThreadData(): localRespRulactions(g_resprulactions.getLocal()), mplexer(std::unique_ptr<FDMultiplexer>(FDMultiplexer::getMultiplexerSilent()))
+  {
+  }
+
+  LocalHolders holders;
+  LocalStateHolder<vector<DNSDistResponseRuleAction> > localRespRulactions;
+  std::unique_ptr<FDMultiplexer> mplexer{nullptr};
+};
+
+struct ConnectionInfo
+{
+  ConnectionInfo(ClientState* cs_): cs(cs_), fd(-1)
+  {
+  }
+  ConnectionInfo(ConnectionInfo&& rhs): remote(rhs.remote), cs(rhs.cs), fd(rhs.fd)
+  {
+    rhs.cs = nullptr;
+    rhs.fd = -1;
+  }
+
+  ConnectionInfo(const ConnectionInfo& rhs) = delete;
+  ConnectionInfo& operator=(const ConnectionInfo& rhs) = delete;
+
+  ConnectionInfo& operator=(ConnectionInfo&& rhs)
+  {
+    remote = rhs.remote;
+    cs = rhs.cs;
+    rhs.cs = nullptr;
+    fd = rhs.fd;
+    rhs.fd = -1;
+    return *this;
+  }
+
+  ~ConnectionInfo()
+  {
+    if (fd != -1) {
+      close(fd);
+      fd = -1;
+    }
+    if (cs) {
+      --cs->tcpCurrentConnections;
+    }
+  }
+
+  ComboAddress remote;
+  ClientState* cs{nullptr};
+  int fd{-1};
+};
+
+class IncomingTCPConnectionState
+{
+public:
+  //IncomingTCPConnectionState(ConnectionInfo&& ci, TCPClientThreadData& threadData, const struct timeval& now): d_buffer(s_maxPacketCacheEntrySize), d_responseBuffer(s_maxPacketCacheEntrySize), d_threadData(threadData), d_ci(std::move(ci)), d_handler(d_ci.fd, g_tcpRecvTimeout, d_ci.cs->tlsFrontend ? d_ci.cs->tlsFrontend->getContext() : nullptr, now.tv_sec), d_ioState(threadData.mplexer, d_ci.fd), _connectionStartTime(now)
+  IncomingTCPConnectionState(ConnectionInfo&& ci, TCPClientThreadData& threadData, const struct timeval& now): d_buffer(s_maxPacketCacheEntrySize), d_responseBuffer(s_maxPacketCacheEntrySize), d_threadData(threadData), d_ci(std::move(ci)), d_handler(d_ci.fd, g_tcpRecvTimeout, d_ci.cs->tlsFrontend ? d_ci.cs->tlsFrontend->getContext() : nullptr, now.tv_sec), d_ioState(make_unique<IOStateHandler>(threadData.mplexer, d_ci.fd)), d_connectionStartTime(now)
+  {
+    d_ids.origDest.reset();
+    d_ids.origDest.sin4.sin_family = d_ci.remote.sin4.sin_family;
+    socklen_t socklen = d_ids.origDest.getSocklen();
+    if (getsockname(d_ci.fd, reinterpret_cast<sockaddr*>(&d_ids.origDest), &socklen)) {
+      d_ids.origDest = d_ci.cs->local;
+    }
+  }
+
+  IncomingTCPConnectionState(const IncomingTCPConnectionState& rhs) = delete;
+  IncomingTCPConnectionState& operator=(const IncomingTCPConnectionState& rhs) = delete;
+
+  ~IncomingTCPConnectionState();
+
+  void resetForNewQuery()
+  {
+    d_buffer.resize(sizeof(uint16_t));
+    d_currentPos = 0;
+    d_querySize = 0;
+    d_responseSize = 0;
+    d_downstreamFailures = 0;
+    d_state = State::readingQuerySize;
+    d_lastIOState = IOState::Done;
+    d_selfGeneratedResponse = false;
+  }
+
+  boost::optional<struct timeval> getClientReadTTD(struct timeval now) const
+  {
+    if (g_maxTCPConnectionDuration == 0 && g_tcpRecvTimeout == 0) {
+      return boost::none;
+    }
+
+    if (g_maxTCPConnectionDuration > 0) {
+      auto elapsed = now.tv_sec - d_connectionStartTime.tv_sec;
+      if (elapsed < 0 || (static_cast<size_t>(elapsed) >= g_maxTCPConnectionDuration)) {
+        return now;
+      }
+      auto remaining = g_maxTCPConnectionDuration - elapsed;
+      if (g_tcpRecvTimeout == 0 || remaining <= static_cast<size_t>(g_tcpRecvTimeout)) {
+        now.tv_sec += remaining;
+        return now;
+      }
+    }
+
+    now.tv_sec += g_tcpRecvTimeout;
+    return now;
+  }
+
+  boost::optional<struct timeval> getClientWriteTTD(const struct timeval& now) const
+  {
+    if (g_maxTCPConnectionDuration == 0 && g_tcpSendTimeout == 0) {
+      return boost::none;
+    }
+
+    struct timeval res = now;
+
+    if (g_maxTCPConnectionDuration > 0) {
+      auto elapsed = res.tv_sec - d_connectionStartTime.tv_sec;
+      if (elapsed < 0 || static_cast<size_t>(elapsed) >= g_maxTCPConnectionDuration) {
+        return res;
+      }
+      auto remaining = g_maxTCPConnectionDuration - elapsed;
+      if (g_tcpSendTimeout == 0 || remaining <= static_cast<size_t>(g_tcpSendTimeout)) {
+        res.tv_sec += remaining;
+        return res;
+      }
+    }
+
+    res.tv_sec += g_tcpSendTimeout;
+    return res;
+  }
+
+  bool maxConnectionDurationReached(unsigned int maxConnectionDuration, const struct timeval& now)
+  {
+    if (maxConnectionDuration) {
+      time_t curtime = now.tv_sec;
+      unsigned int elapsed = 0;
+      if (curtime > d_connectionStartTime.tv_sec) { // To prevent issues when time goes backward
+        elapsed = curtime - d_connectionStartTime.tv_sec;
+      }
+      if (elapsed >= maxConnectionDuration) {
+        return true;
+      }
+      d_remainingTime = maxConnectionDuration - elapsed;
+    }
+
+    return false;
+  }
+
+  void dump() const
+  {
+    static std::mutex s_mutex;
+
+    struct timeval now;
+    gettimeofday(&now, 0);
+
+    {
+      std::lock_guard<std::mutex> lock(s_mutex);
+      fprintf(stderr, "State is %p\n", this);
+      cerr << "Current state is " << static_cast<int>(d_state) << ", got "<<d_queriesCount<<" queries so far" << endl;
+      cerr << "Current time is " << now.tv_sec << " - " << now.tv_usec << endl;
+      cerr << "Connection started at " << d_connectionStartTime.tv_sec << " - " << d_connectionStartTime.tv_usec << endl;
+      if (d_state > State::doingHandshake) {
+        cerr << "Handshake done at " << d_handshakeDoneTime.tv_sec << " - " << d_handshakeDoneTime.tv_usec << endl;
+      }
+      if (d_state > State::readingQuerySize) {
+        cerr << "Got first query size at " << d_firstQuerySizeReadTime.tv_sec << " - " << d_firstQuerySizeReadTime.tv_usec << endl;
+      }
+      if (d_state > State::readingQuerySize) {
+        cerr << "Got query size at " << d_querySizeReadTime.tv_sec << " - " << d_querySizeReadTime.tv_usec << endl;
+      }
+      if (d_state > State::readingQuery) {
+        cerr << "Got query at " << d_queryReadTime.tv_sec << " - " << d_queryReadTime.tv_usec << endl;
+      }
+    }
+  }
+
+  std::shared_ptr<TCPConnectionToBackend> getActiveDownstreamConnection(const std::shared_ptr<DownstreamState>& ds)
+  {
+#warning TODO: we need to find a connection to this DS, usable (no TLV values sent) and supporting OOR
+    return nullptr;
+  }
+
+  std::shared_ptr<TCPConnectionToBackend> getDownstreamConnection(std::shared_ptr<DownstreamState>& ds, const struct timeval& now);
+
+  std::unique_ptr<FDMultiplexer>& getIOMPlexer() const
+  {
+    return d_threadData.mplexer;
+  }
+
+  static void handleIO(std::shared_ptr<IncomingTCPConnectionState>& conn, const struct timeval& now);
+  static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param);
+
+  void notifyIOError(IDState&& query);
+  void queueQuery(TCPQuery&& query);
+  void sendXFRResponse(std::vector<uint8_t>&& response, const IDState& query);
+  void sendResponse(std::vector<uint8_t>&& response, IDState&& query);
+
+  enum class State { doingHandshake, readingQuerySize, readingQuery, sendingResponse, idle /* in case of XFR, we stop processing queries */ };
+
+  std::vector<uint8_t> d_buffer;
+  #warning unused?
+  std::vector<uint8_t> d_responseBuffer;
+  std::deque<TCPResponse> d_pendingResponses;
+  TCPClientThreadData& d_threadData;
+  IDState d_ids;
+  ConnectionInfo d_ci;
+  TCPIOHandler d_handler;
+  std::unique_ptr<IOStateHandler> d_ioState{nullptr};
+  std::unique_ptr<TCPConnectionToBackend> d_downstreamConnection{nullptr};
+  std::shared_ptr<DownstreamState> d_ds{nullptr};
+  dnsheader d_cleartextDH;
+  struct timeval d_connectionStartTime;
+  struct timeval d_handshakeDoneTime;
+  struct timeval d_firstQuerySizeReadTime;
+  struct timeval d_querySizeReadTime;
+  struct timeval d_queryReadTime;
+  size_t d_currentPos{0};
+  size_t d_queriesCount{0};
+  unsigned int d_remainingTime{0};
+  uint16_t d_querySize{0};
+  #warning unused?
+  uint16_t d_responseSize{0};
+  uint16_t d_downstreamFailures{0};
+  State d_state{State::doingHandshake};
+  IOState d_lastIOState{IOState::Done};
+  bool d_readingFirstQuery{true};
+  bool d_firstResponsePacket{true};
+  bool d_isXFR{false};
+  bool d_xfrStarted{false};
+  bool d_selfGeneratedResponse{false};
+  bool d_proxyProtocolPayloadAdded{false};
+  bool d_proxyProtocolPayloadHasTLV{false};
+};
+
+IOState tryRead(int fd, std::vector<uint8_t>& buffer, size_t& pos, size_t toRead);
diff --git a/pdns/dnsdistdist/doh.cc b/pdns/dnsdistdist/doh.cc
index 080c95e3b8..57c21606e0 100644
--- a/pdns/dnsdistdist/doh.cc
+++ b/pdns/dnsdistdist/doh.cc
@@ -866,7 +866,7 @@ try
       }
 
       string decoded;
-      /* rough estimate so we hopefully don't need a need allocation later */
+      /* rough estimate so we hopefully don't need a new allocation later */
       /* We reserve at least 512 additional bytes to be able to add EDNS, but we also want
          at least s_maxPacketCacheEntrySize bytes to be able to fill the answer from the packet cache */
       const size_t estimate = ((sdns.size() * 3) / 4);
diff --git a/pdns/dnsdistdist/tcpiohandler-mplexer.hh b/pdns/dnsdistdist/tcpiohandler-mplexer.hh
new file mode 100644
index 0000000000..a64cf563e7
--- /dev/null
+++ b/pdns/dnsdistdist/tcpiohandler-mplexer.hh
@@ -0,0 +1,120 @@
+
+#pragma once
+
+#include "mplexer.hh"
+#include "tcpiohandler.hh"
+
+class IOStateHandler
+{
+public:
+  IOStateHandler(std::unique_ptr<FDMultiplexer>& mplexer, const int fd): d_mplexer(mplexer), d_fd(fd), d_currentState(IOState::Done)
+  {
+  }
+
+  IOStateHandler(std::unique_ptr<FDMultiplexer>& mplexer): d_mplexer(mplexer), d_fd(-1), d_currentState(IOState::Done)
+  {
+  }
+
+  ~IOStateHandler()
+  {
+    /* be careful that this won't save us if the callback is still registered to the multiplexer,
+       because in that case the shared pointer count will never reach zero so this destructor won't
+       be called */
+    reset();
+  }
+
+  IOState getState() const
+  {
+    return d_currentState;
+  }
+
+  void setSocket(int fd)
+  {
+    if (d_fd != -1) {
+      throw std::runtime_error("Trying to set the socket descriptor on an already initialized IOStateHandler");
+    }
+    d_fd = fd;
+  }
+
+  void reset()
+  {
+    update(IOState::Done);
+  }  
+
+  void update(IOState iostate, FDMultiplexer::callbackfunc_t callback = FDMultiplexer::callbackfunc_t(), FDMultiplexer::funcparam_t callbackData = boost::any(), boost::optional<struct timeval> ttd = boost::none)
+  {
+    //cerr<<"in "<<__func__<<" for fd "<<d_fd<<", last state was "<<(int)d_currentState<<", new state is "<<(int)iostate<<endl;
+    if (d_currentState == IOState::NeedRead && iostate != IOState::NeedRead) {
+      d_mplexer->removeReadFD(d_fd);
+      //cerr<<__func__<<": remove read FD "<<d_fd<<endl;
+      d_currentState = IOState::Done;
+    }
+    else if (d_currentState == IOState::NeedWrite && iostate != IOState::NeedWrite) {
+      d_mplexer->removeWriteFD(d_fd);
+      //cerr<<__func__<<": remove write FD "<<d_fd<<endl;
+      d_currentState = IOState::Done;
+    }
+
+    if (iostate == IOState::NeedRead) {
+      if (d_currentState == IOState::NeedRead) {
+        if (ttd) {
+          /* let's update the TTD ! */
+          d_mplexer->setReadTTD(d_fd, *ttd, /* we pass 0 here because we already have a TTD */0);
+        }
+        return;
+      }
+
+      d_currentState = IOState::NeedRead;
+      //cerr<<__func__<<": add read FD "<<d_fd<<endl;
+      d_mplexer->addReadFD(d_fd, callback, callbackData, ttd ? &*ttd : nullptr);
+    }
+    else if (iostate == IOState::NeedWrite) {
+      if (d_currentState == IOState::NeedWrite) {
+        return;
+      }
+
+      d_currentState = IOState::NeedWrite;
+      //cerr<<__func__<<": add write FD "<<d_fd<<endl;
+      d_mplexer->addWriteFD(d_fd, callback, callbackData, ttd ? &*ttd : nullptr);
+    }
+    else if (iostate == IOState::Done) {
+      d_currentState = IOState::Done;
+    }
+  }
+
+private:
+  std::unique_ptr<FDMultiplexer>& d_mplexer;
+  int d_fd;
+  IOState d_currentState;
+};
+
+class IOStateGuard
+{
+public:
+  /* this class is using RAII to make sure we don't forget to release an IOStateHandler
+     from the IO multiplexer in case of exception / error handling */
+  IOStateGuard(std::unique_ptr<IOStateHandler>& handler): d_handler(handler), d_enabled(true)
+  {
+  }
+
+  ~IOStateGuard()
+  {
+    /* if we are still owning the state when we go out of scope,
+       let's reset the state so it's not registered to the IO multiplexer anymore
+       and its reference count goes to zero */
+    if (d_enabled && d_handler) {
+      cerr<<"IOStateGuard destroyed while holding a state, let's reset it"<<endl;
+      d_handler->reset();
+      d_enabled = false;
+    }
+  }
+
+  void release()
+  {
+    d_enabled = false;
+  }
+
+private:
+  std::unique_ptr<IOStateHandler>& d_handler;
+  bool d_enabled;
+};

From 459c8cae14e331b8812be668bcd31ce760dc3eef Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Tue, 1 Sep 2020 10:44:29 +0200
Subject: [PATCH 02/23] dnsdist: TCP out-of-order implementation

---
 pdns/dnsdist-tcp.cc                        | 584 +++++++++++++--------
 pdns/dnsdist.hh                            |   8 +-
 pdns/dnsdistdist/dnsdist-tcp-downstream.cc | 239 +++++++--
 pdns/dnsdistdist/dnsdist-tcp-downstream.hh |  33 +-
 pdns/dnsdistdist/dnsdist-tcp-upstream.hh   |  51 +-
 pdns/dnsdistdist/tcpiohandler-mplexer.hh   |  13 +-
 regression-tests.dnsdist/test_TCPShort.py  |   8 +-
 regression-tests.dnsdist/test_Tags.py      |   2 +-
 8 files changed, 611 insertions(+), 327 deletions(-)

diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc
index 8d6e7d6db8..a900105086 100644
--- a/pdns/dnsdist-tcp.cc
+++ b/pdns/dnsdist-tcp.cc
@@ -148,7 +148,7 @@ static void decrementTCPClientCount(const ComboAddress& client)
 {
   if (g_maxTCPConnectionsPerClient) {
     std::lock_guard<std::mutex> lock(tcpClientsCountMutex);
-    tcpClientsCount[client]--;
+    tcpClientsCount.at(client)--;
     if (tcpClientsCount[client] == 0) {
       tcpClientsCount.erase(client);
     }
@@ -157,25 +157,31 @@ static void decrementTCPClientCount(const ComboAddress& client)
 
 IncomingTCPConnectionState::~IncomingTCPConnectionState()
 {
+  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
   decrementTCPClientCount(d_ci.remote);
+  // DEBUG: cerr<<"decremented"<<endl;
+
   if (d_ci.cs != nullptr) {
     struct timeval now;
     gettimeofday(&now, nullptr);
-    
+
     auto diff = now - d_connectionStartTime;
+    // DEBUG: cerr<<"updating tcp metrics"<<endl;
     d_ci.cs->updateTCPMetrics(d_queriesCount, diff.tv_sec * 1000.0 + diff.tv_usec / 1000.0);
+    // DEBUG: cerr<<"updated tcp metrics"<<endl;
   }
 
+#if 0
   if (d_ds != nullptr) {
 
     if (d_downstreamConnection) {
       try {
         if (d_lastIOState == IOState::NeedRead) {
-          cerr<<__func__<<": removing leftover backend read FD "<<d_downstreamConnection->getHandle()<<endl;
+          // DEBUG: cerr<<__PRETTY_FUNCTION__<<": removing leftover backend read FD "<<d_downstreamConnection->getHandle()<<endl;
           d_threadData.mplexer->removeReadFD(d_downstreamConnection->getHandle());
         }
         else if (d_lastIOState == IOState::NeedWrite) {
-          cerr<<__func__<<": removing leftover backend write FD "<<d_downstreamConnection->getHandle()<<endl;
+          // DEBUG: cerr<<__PRETTY_FUNCTION__<<": removing leftover backend write FD "<<d_downstreamConnection->getHandle()<<endl;
           d_threadData.mplexer->removeWriteFD(d_downstreamConnection->getHandle());
         }
       }
@@ -188,20 +194,26 @@ IncomingTCPConnectionState::~IncomingTCPConnectionState()
       }
     }
   }
+#endif
 
+  // DEBUG: cerr<<"about to remove left over FDs"<<endl;
   try {
     if (d_lastIOState == IOState::NeedRead) {
-      cerr<<__func__<<": removing leftover client read FD "<<d_ci.fd<<endl;
+      // DEBUG: cerr<<__PRETTY_FUNCTION__<<": removing leftover client read FD "<<d_ci.fd<<endl;
       d_threadData.mplexer->removeReadFD(d_ci.fd);
     }
     else if (d_lastIOState == IOState::NeedWrite) {
-      cerr<<__func__<<": removing leftover client write FD "<<d_ci.fd<<endl;
+      // DEBUG: cerr<<__PRETTY_FUNCTION__<<": removing leftover client write FD "<<d_ci.fd<<endl;
       d_threadData.mplexer->removeWriteFD(d_ci.fd);
     }
   }
-  catch(const FDMultiplexerException& e) {
+  catch (const FDMultiplexerException& e) {
     vinfolog("Got an exception when trying to remove a pending IO operation on an incoming TCP connection from %s: %s", d_ci.remote.toStringWithPort(), e.what());
   }
+  catch (...) {
+    vinfolog("Got an unknown exception when trying to remove a pending IO operation on an incoming TCP connection from %s", d_ci.remote.toStringWithPort());
+  }
+  // DEBUG: cerr<<"done"<<endl;
 }
 
 std::shared_ptr<TCPConnectionToBackend> IncomingTCPConnectionState::getDownstreamConnection(std::shared_ptr<DownstreamState>& ds, const struct timeval& now)
@@ -324,122 +336,155 @@ IOState tryRead(int fd, std::vector<uint8_t>& buffer, size_t& pos, size_t toRead
 
 std::unique_ptr<TCPClientCollection> g_tcpclientthreads;
 
-static void handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
+static IOState handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
 {
-  if (state->d_isXFR && state->d_downstreamConnection) {
-    /* we should still be reading from the backend, and we don't want to read from the client */
-    #warning check that we are doing the right thing on the backend side
-    state->d_state = IncomingTCPConnectionState::State::idle;
-    state->d_currentPos = 0;
-    return;
-  }
+  if (!state->d_isXFR) {
+    const auto& currentResponse = state->d_currentResponse;
+    if (state->d_selfGeneratedResponse == false && currentResponse.d_ds) {
+      struct timespec answertime;
+      gettime(&answertime);
+      const auto& ids = currentResponse.d_idstate;
+      double udiff = ids.sentTime.udiff();
+      g_rings.insertResponse(answertime, state->d_ci.remote, ids.qname, ids.qtype, static_cast<unsigned int>(udiff), static_cast<unsigned int>(currentResponse.d_buffer.size()), currentResponse.d_cleartextDH, currentResponse.d_ds->remote);
+      vinfolog("Got answer from %s, relayed to %s (%s), took %f usec", currentResponse.d_ds->remote.toStringWithPort(), ids.origRemote.toStringWithPort(), (state->d_ci.cs->tlsFrontend ? "DoT" : "TCP"), udiff);
+    }
 
-  if (state->d_selfGeneratedResponse == false && state->d_ds) {
-    /* if we have no downstream server selected, this was a self-answered response
-       but cache hits have a selected server as well, so be careful */
-    struct timespec answertime;
-    gettime(&answertime);
-    double udiff = state->d_ids.sentTime.udiff();
-    g_rings.insertResponse(answertime, state->d_ci.remote, state->d_ids.qname, state->d_ids.qtype, static_cast<unsigned int>(udiff), static_cast<unsigned int>(state->d_responseBuffer.size()), state->d_cleartextDH, state->d_ds->remote);
-    vinfolog("Got answer from %s, relayed to %s (%s), took %f usec", state->d_ds->remote.toStringWithPort(), state->d_ids.origRemote.toStringWithPort(), (state->d_ci.cs->tlsFrontend ? "DoT" : "TCP"), udiff);
-  }
+    switch (currentResponse.d_cleartextDH.rcode) {
+    case RCode::NXDomain:
+      ++g_stats.frontendNXDomain;
+      break;
+    case RCode::ServFail:
+      ++g_stats.servfailResponses;
+      ++g_stats.frontendServFail;
+      break;
+    case RCode::NoError:
+      ++g_stats.frontendNoError;
+      break;
+    }
 
-  switch (state->d_cleartextDH.rcode) {
-  case RCode::NXDomain:
-    ++g_stats.frontendNXDomain;
-    break;
-  case RCode::ServFail:
-    ++g_stats.servfailResponses;
-    ++g_stats.frontendServFail;
-    break;
-  case RCode::NoError:
-    ++g_stats.frontendNoError;
-    break;
-  }
+    if (g_maxTCPQueriesPerConn && state->d_queriesCount > g_maxTCPQueriesPerConn) {
+      vinfolog("Terminating TCP connection from %s because it reached the maximum number of queries per conn (%d / %d)", state->d_ci.remote.toStringWithPort(), state->d_queriesCount, g_maxTCPQueriesPerConn);
+      return IOState::Done;
+    }
 
-  if (g_maxTCPQueriesPerConn && state->d_queriesCount > g_maxTCPQueriesPerConn) {
-    vinfolog("Terminating TCP connection from %s because it reached the maximum number of queries per conn (%d / %d)", state->d_ci.remote.toStringWithPort(), state->d_queriesCount, g_maxTCPQueriesPerConn);
-    return;
+    if (state->maxConnectionDurationReached(g_maxTCPConnectionDuration, now)) {
+      vinfolog("Terminating TCP connection from %s because it reached the maximum TCP connection duration", state->d_ci.remote.toStringWithPort());
+      return IOState::Done;
+    }
   }
 
-  if (state->maxConnectionDurationReached(g_maxTCPConnectionDuration, now)) {
-    vinfolog("Terminating TCP connection from %s because it reached the maximum TCP connection duration", state->d_ci.remote.toStringWithPort());
-    return;
+  if (state->d_queuedResponses.empty()) {
+    // DEBUG: cerr<<"no response remaining"<<endl;
+    if (state->d_isXFR) {
+      /* we should still be reading from the backend, and we don't want to read from the client */
+      state->d_state = IncomingTCPConnectionState::State::idle;
+      state->d_currentPos = 0;
+      // DEBUG: cerr<<"idling for XFR completion"<<endl;
+      return IOState::Done;
+    } else {
+      // DEBUG: cerr<<"reading new queries if any"<<endl;
+      state->resetForNewQuery();
+      return IOState::NeedRead;
+    }
   }
+  else {
+    // DEBUG: cerr<<"queue size is "<<state->d_queuedResponses.size()<<endl;
+    TCPResponse resp = std::move(state->d_queuedResponses.front());
+    state->d_queuedResponses.pop_front();
+    state->d_state = IncomingTCPConnectionState::State::idle;
+    state->sendResponse(state, now, std::move(resp));
+    return IOState::NeedWrite;
+  }
+}
 
-  state->resetForNewQuery();
+void IncomingTCPConnectionState::resetForNewQuery()
+{
+  d_buffer.resize(sizeof(uint16_t));
+  d_currentPos = 0;
+  d_querySize = 0;
+  //d_responseSize = 0;
+  d_downstreamFailures = 0;
+  d_state = State::readingQuerySize;
+  d_lastIOState = IOState::Done;
+  d_selfGeneratedResponse = false;
 }
 
 /* this version is called when the buffer has been set and the rules have been processed */
-void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
+void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
 {
+  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
   // if we already reading a query (not the query size, mind you), or sending a response we need to either queue the response
   // otherwise we can start sending it right away
   if (state->d_state == IncomingTCPConnectionState::State::idle ||
       state->d_state == IncomingTCPConnectionState::State::readingQuerySize) {
+
     state->d_state = IncomingTCPConnectionState::State::sendingResponse;
-    state->d_ioState->update(IOState::NeedWrite, handleIOCallback, state, getClientWriteTTD(now));
+
+    uint16_t responseSize = static_cast<uint16_t>(response.d_buffer.size());
+    const uint8_t sizeBytes[] = { static_cast<uint8_t>(responseSize / 256), static_cast<uint8_t>(responseSize % 256) };
+    /* prepend the size. Yes, this is not the most efficient way but it prevents mistakes
+       that could occur if we had to deal with the size during the processing,
+       especially alignment issues */
+    response.d_buffer.insert(response.d_buffer.begin(), sizeBytes, sizeBytes + 2);
+    state->d_currentPos = 0;
+    state->d_currentResponse = std::move(response);
+
+    //IncomingTCPConnectionState::handleIO(state, now);
+     state->d_ioState->update(IOState::NeedWrite, handleIOCallback, state, getClientWriteTTD(now));
+    // DEBUG: cerr<<"updated IO state"<<endl;
   }
   else {
     // queue response
+    state->d_queuedResponses.push_back(std::move(response));
+    // DEBUG: cerr<<"queueing response because state is "<<(int)state->d_state<<", queue size is now "<<state->d_queuedResponses.size()<<endl;
   }
 }
 
-{
-  if (state->handleResponseFromBackend(state, now, response)) {
-    state->sendResponse(state, response, now);
-  }
-}
-
-static void sendInternalResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
-{
-  const uint8_t sizeBytes[] = { static_cast<uint8_t>(state->d_responseSize / 256), static_cast<uint8_t>(state->d_responseSize % 256) };
-  /* prepend the size. Yes, this is not the most efficient way but it prevents mistakes
-     that could occur if we had to deal with the size during the processing,
-     especially alignment issues */
-  state->d_responseBuffer.insert(state->d_responseBuffer.begin(), sizeBytes, sizeBytes + 2);
-  state->d_currentPos = 0;
-  state->sendResponse(now);
-}
-
 /* this version is called from the backend code when a new response has been received */
-void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response, const ComboAddress& remote)
+void IncomingTCPConnectionState::handleResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
 {
+  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
   if (response.d_buffer.size() < sizeof(dnsheader)) {
+    // DEBUG: cerr<<"too small"<<endl;
     return;
   }
 
-  auto response = reinterpret_cast<char*>(&response.d_buffer.at(0));
+  uint16_t responseSize = response.d_buffer.size();
+  response.d_buffer.resize(responseSize + static_cast<size_t>(512));
+  size_t responseCapacity = response.d_buffer.size();
+  auto responseAsCharArray = reinterpret_cast<char*>(&response.d_buffer.at(0));
+
   auto& ids = response.d_idstate;
+  // DEBUG: cerr<<"IDS has "<<(ids.qTag?" TAGS ": "NO TAGS")<<endl;
   unsigned int consumed;
-  if (ids.d_firstResponsePacket && !responseContentMatches(response, response.d_buffer.size(), ids.qname, ids.qtype, ids.qclass, remote, consumed)) {
+  // DEBUG: cerr<<"about to match response for "<<ids.qname<<endl;
+  if (!responseContentMatches(responseAsCharArray, responseSize, ids.qname, ids.qtype, ids.qclass, response.d_ds->remote, consumed)) {
+    // DEBUG: cerr<<"content does not match"<<endl;
     return;
   }
-  ids.d_firstResponsePacket = false;
 
-  auto dh = reinterpret_cast<struct dnsheader*>(response);
+  auto dh = reinterpret_cast<struct dnsheader*>(responseAsCharArray);
   uint16_t addRoom = 0;
-  DNSResponse dr = makeDNSResponseFromIDState(ids, dh, response.d_buffer.size(), state->d_responseSize, true);
+  DNSResponse dr = makeDNSResponseFromIDState(ids, dh, responseCapacity, responseSize, true);
   if (dr.dnsCryptQuery) {
     addRoom = DNSCRYPT_MAX_RESPONSE_PADDING_AND_MAC_SIZE;
   }
 
-  memcpy(&state->d_cleartextDH, dr.dh, sizeof(state->d_cleartextDH));
+  memcpy(&response.d_cleartextDH, dr.dh, sizeof(response.d_cleartextDH));
 
   std::vector<uint8_t> rewrittenResponse;
-  size_t responseSize = state->d_responseBuffer.size();
-  if (!processResponse(&response, &state->d_responseSize, &responseSize, state->d_threadData.localRespRulactions, dr, addRoom, rewrittenResponse, false)) {
+  if (!processResponse(&responseAsCharArray, &responseSize, &responseCapacity, state->d_threadData.localRespRulactions, dr, addRoom, rewrittenResponse, false)) {
+    // DEBUG: cerr<<"process said to drop it"<<endl;
     return;
   }
 
   if (!rewrittenResponse.empty()) {
     /* responseSize has been updated as well but we don't really care since it will match
        the capacity of rewrittenResponse anyway */
-    state->d_responseBuffer = std::move(rewrittenResponse);
-    state->d_responseSize = state->d_responseBuffer.size();
+    response.d_buffer = std::move(rewrittenResponse);
   } else {
     /* the size might have been updated (shrinked) if we removed the whole OPT RR, for example) */
-    state->d_responseBuffer.resize(state->d_responseSize);
+    response.d_buffer.resize(responseSize);
   }
 
   if (state->d_isXFR && !state->d_xfrStarted) {
@@ -447,23 +492,23 @@ void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnect
     state->d_xfrStarted = true;
     ++g_stats.responses;
     ++state->d_ci.cs->responses;
-    ++state->d_ds->responses;
+    ++response.d_ds->responses;
   }
 
   if (!state->d_isXFR) {
     ++g_stats.responses;
     ++state->d_ci.cs->responses;
-    ++state->d_ds->responses;
+    ++response.d_ds->responses;
   }
 
-  sendResponse(state, now);
+  sendResponse(state, now, std::move(response));
 }
 
-static void handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
+static bool handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
 {
   if (state->d_querySize < sizeof(dnsheader)) {
     ++g_stats.nonCompliantQueries;
-    return;
+    return true;
   }
 
   state->d_readingFirstQuery = false;
@@ -502,21 +547,24 @@ static void handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
   std::shared_ptr<DNSCryptQuery> dnsCryptQuery{nullptr};
   auto dnsCryptResponse = checkDNSCryptQuery(*state->d_ci.cs, query, state->d_querySize, dnsCryptQuery, queryRealTime.tv_sec, true);
   if (dnsCryptResponse) {
-    state->d_responseBuffer = std::move(*dnsCryptResponse);
-    state->d_responseSize = state->d_responseBuffer.size();
-    sendInteralResponse(state, now);
-    return;
+    //state->d_responseBuffer = std::move(*dnsCryptResponse);
+    //state->d_responseSize = state->d_responseBuffer.size();
+    TCPResponse response;
+    response.d_buffer = std::move(*dnsCryptResponse);
+    state->d_state = IncomingTCPConnectionState::State::idle;
+    state->sendResponse(state, now, std::move(response));
+    return false;
   }
 
   const auto& dh = reinterpret_cast<dnsheader*>(query);
   if (!checkQueryHeaders(dh)) {
-    return;
+    return true;
   }
 
   uint16_t qtype, qclass;
   unsigned int consumed = 0;
   DNSName qname(query, state->d_querySize, sizeof(dnsheader), false, &qtype, &qclass, &consumed);
-  DNSQuestion dq(&qname, qtype, qclass, consumed, &state->d_ids.origDest, &state->d_ci.remote, reinterpret_cast<dnsheader*>(query), state->d_buffer.size(), state->d_querySize, true, &queryRealTime);
+  DNSQuestion dq(&qname, qtype, qclass, consumed, &state->d_origDest, &state->d_ci.remote, reinterpret_cast<dnsheader*>(query), state->d_buffer.size(), state->d_querySize, true, &queryRealTime);
   dq.dnsCryptQuery = std::move(dnsCryptQuery);
   dq.sni = state->d_handler.getServerNameIndication();
 
@@ -525,35 +573,39 @@ static void handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
     dq.skipCache = true;
   }
 
-  state->d_ds.reset();
-  auto result = processQuery(dq, *state->d_ci.cs, state->d_threadData.holders, state->d_ds);
+  std::shared_ptr<DownstreamState> ds;
+  auto result = processQuery(dq, *state->d_ci.cs, state->d_threadData.holders, ds);
 
   if (result == ProcessQueryResult::Drop) {
-    return;
+    return true;
   }
 
   if (result == ProcessQueryResult::SendAnswer) {
     state->d_selfGeneratedResponse = true;
     state->d_buffer.resize(dq.len);
-    state->d_responseBuffer = std::move(state->d_buffer);
-    state->d_responseSize = state->d_responseBuffer.size();
-    sendInternalResponse(state, now);
-    return;
+    TCPResponse response;
+    response.d_buffer = std::move(state->d_buffer);
+    state->d_state = IncomingTCPConnectionState::State::idle;
+    state->sendResponse(state, now, std::move(response));
+    return false;
   }
 
-  if (result != ProcessQueryResult::PassToBackend || state->d_ds == nullptr) {
-    return;
+  if (result != ProcessQueryResult::PassToBackend || ds == nullptr) {
+    return true;
   }
 
 #warning move this, we should just never read another question again on this client connection
   if (state->d_xfrStarted) {
     /* sorry, but we are not going to resume a XFR if we have already sent some packets
        to the client */
-    return;
+    return true;
   }
 
   IDState ids;
+  // DEBUG: cerr<<"DQ has "<<(dq.qTag?" TAGS ": "NO TAGS")<<endl;
   setIDStateFromDNSQuestion(ids, dq, std::move(qname));
+  // DEBUG: cerr<<"query IDS has "<<(ids.qTag?" TAGS ": "NO TAGS")<<endl;
+  ids.origID = ntohs(dh->id);
 
   const uint8_t sizeBytes[] = { static_cast<uint8_t>(dq.len / 256), static_cast<uint8_t>(dq.len % 256) };
   /* prepend the size. Yes, this is not the most efficient way but it prevents mistakes
@@ -568,7 +620,7 @@ static void handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
   bool proxyProtocolPayloadAdded = false;
   std::string proxyProtocolPayload;
 
-  if (state->d_ds->useProxyProtocol) {
+  if (ds->useProxyProtocol) {
     /* if we ever sent a TLV over a connection, we can never go back */
     if (!state->d_proxyProtocolPayloadHasTLV) {
       state->d_proxyProtocolPayloadHasTLV = dq.proxyProtocolValues && !dq.proxyProtocolValues->empty();
@@ -583,8 +635,8 @@ static void handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
     }
   }
 
-  auto downstreamConnection = state->getDownstreamConnection(state->d_ds, now);
-  downstreamConnection->assignToClientConnection(state);
+  auto downstreamConnection = state->getDownstreamConnection(ds, now);
+  downstreamConnection->assignToClientConnection(state, state->d_isXFR);
 
   if (proxyProtocolPayloadAdded) {
     downstreamConnection->setProxyProtocolPayloadAdded(true);
@@ -592,16 +644,19 @@ static void handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
   else {
     downstreamConnection->setProxyProtocolPayload(std::move(proxyProtocolPayload));
   }
+  // DEBUG: cerr<<"about to be queued query IDS has "<<(ids.qTag?" TAGS ": "NO TAGS")<<endl;
   downstreamConnection->queueQuery(TCPQuery(std::move(state->d_buffer), std::move(ids)), downstreamConnection);
 
   //sendQueryToBackend(state, now);
+  // DEBUG: cerr<<"out of "<<__PRETTY_FUNCTION__<<endl;
+  return true;
 }
 
 void IncomingTCPConnectionState::handleIOCallback(int fd, FDMultiplexer::funcparam_t& param)
 {
   auto conn = boost::any_cast<std::shared_ptr<IncomingTCPConnectionState>>(param);
   if (fd != conn->d_ci.fd) {
-    throw std::runtime_error("Unexpected socket descriptor " + std::to_string(fd) + " received in " + std::string(__func__) + ", expected " + std::to_string(conn->d_ci.fd));
+    throw std::runtime_error("Unexpected socket descriptor " + std::to_string(fd) + " received in " + std::string(__PRETTY_FUNCTION__) + ", expected " + std::to_string(conn->d_ci.fd));
   }
 
   struct timeval now;
@@ -611,140 +666,219 @@ void IncomingTCPConnectionState::handleIOCallback(int fd, FDMultiplexer::funcpar
 
 void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
 {
-  cerr<<"in "<<__func__<<endl;
+  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
+  // why do we loop? Because the TLS layer does buffering, and thus can have data ready to read
+  // even though the underlying socket is not ready, so we need to actually ask for the data first
+  bool wouldBlock = false;
   IOState iostate = IOState::Done;
-  IOStateGuard ioGuard(state->d_ioState);
+  do {
+    iostate = IOState::Done;
+    IOStateGuard ioGuard(state->d_ioState);
 
-  if (state->maxConnectionDurationReached(g_maxTCPConnectionDuration, now)) {
-    vinfolog("Terminating TCP connection from %s because it reached the maximum TCP connection duration", state->d_ci.remote.toStringWithPort());
-    // will be handled by the ioGuard
-    //handleNewIOState(state, IOState::Done, fd, handleIOCallback);
-    return;
-  }
+    if (state->maxConnectionDurationReached(g_maxTCPConnectionDuration, now)) {
+      vinfolog("Terminating TCP connection from %s because it reached the maximum TCP connection duration", state->d_ci.remote.toStringWithPort());
+      // will be handled by the ioGuard
+      //handleNewIOState(state, IOState::Done, fd, handleIOCallback);
+      return;
+    }
 
-  try {
-    if (state->d_state == IncomingTCPConnectionState::State::doingHandshake) {
-      cerr<<"doing handshake"<<endl;
-      iostate = state->d_handler.tryHandshake();
-      if (iostate == IOState::Done) {
-        cerr<<"handshake done"<<endl;
-        if (state->d_handler.isTLS()) {
-          if (!state->d_handler.hasTLSSessionBeenResumed()) {
-            ++state->d_ci.cs->tlsNewSessions;
-          }
-          else {
-            ++state->d_ci.cs->tlsResumptions;
+    try {
+      if (state->d_state == IncomingTCPConnectionState::State::doingHandshake) {
+        // DEBUG: cerr<<"doing handshake"<<endl;
+        iostate = state->d_handler.tryHandshake();
+        if (iostate == IOState::Done) {
+          // DEBUG: cerr<<"handshake done"<<endl;
+          if (state->d_handler.isTLS()) {
+            if (!state->d_handler.hasTLSSessionBeenResumed()) {
+              ++state->d_ci.cs->tlsNewSessions;
+            }
+            else {
+              ++state->d_ci.cs->tlsResumptions;
+            }
+            if (state->d_handler.getResumedFromInactiveTicketKey()) {
+              ++state->d_ci.cs->tlsInactiveTicketKey;
+            }
+            if (state->d_handler.getUnknownTicketKey()) {
+              ++state->d_ci.cs->tlsUnknownTicketKey;
+            }
           }
-          if (state->d_handler.getResumedFromInactiveTicketKey()) {
-            ++state->d_ci.cs->tlsInactiveTicketKey;
+
+          state->d_handshakeDoneTime = now;
+          state->d_state = IncomingTCPConnectionState::State::readingQuerySize;
+        }
+        else {
+          wouldBlock = true;
+        }
+      }
+
+      if (state->d_state == IncomingTCPConnectionState::State::readingQuerySize) {
+        // DEBUG: cerr<<"reading query size"<<endl;
+        iostate = state->d_handler.tryRead(state->d_buffer, state->d_currentPos, sizeof(uint16_t));
+        if (iostate == IOState::Done) {
+          // DEBUG: cerr<<"query size received"<<endl;
+          state->d_state = IncomingTCPConnectionState::State::readingQuery;
+          state->d_querySizeReadTime = now;
+          if (state->d_queriesCount == 0) {
+            state->d_firstQuerySizeReadTime = now;
           }
-          if (state->d_handler.getUnknownTicketKey()) {
-            ++state->d_ci.cs->tlsUnknownTicketKey;
+          state->d_querySize = state->d_buffer.at(0) * 256 + state->d_buffer.at(1);
+          if (state->d_querySize < sizeof(dnsheader)) {
+            /* go away */
+            // will be handled by the guard
+            //handleNewIOState(state, IOState::Done, fd, handleIOCallback);
+            return;
           }
-        }
 
-        state->d_handshakeDoneTime = now;
-        state->d_state = IncomingTCPConnectionState::State::readingQuerySize;
+          /* allocate a bit more memory to be able to spoof the content, get an answer from the cache
+             or to add ECS without allocating a new buffer */
+          state->d_buffer.resize(std::max(state->d_querySize + static_cast<size_t>(512), s_maxPacketCacheEntrySize));
+          state->d_currentPos = 0;
+        }
+        else {
+          wouldBlock = true;
+        }
       }
-    }
 
-    if (state->d_state == IncomingTCPConnectionState::State::readingQuerySize) {
-      cerr<<"reading query size"<<endl;
-      iostate = state->d_handler.tryRead(state->d_buffer, state->d_currentPos, sizeof(uint16_t));
-      if (iostate == IOState::Done) {
-        cerr<<"query size received"<<endl;
-        state->d_state = IncomingTCPConnectionState::State::readingQuery;
-        state->d_querySizeReadTime = now;
-        if (state->d_queriesCount == 0) {
-          state->d_firstQuerySizeReadTime = now;
-        }
-        state->d_querySize = state->d_buffer.at(0) * 256 + state->d_buffer.at(1);
-        if (state->d_querySize < sizeof(dnsheader)) {
-          /* go away */
-          // will be handled by the guard
+      if (state->d_state == IncomingTCPConnectionState::State::readingQuery) {
+        // DEBUG: cerr<<"reading query"<<endl;
+        iostate = state->d_handler.tryRead(state->d_buffer, state->d_currentPos, state->d_querySize);
+        if (iostate == IOState::Done) {
+          // DEBUG: cerr<<"query received"<<endl;
           //handleNewIOState(state, IOState::Done, fd, handleIOCallback);
-          return;
+          if (handleQuery(state, now)) {
+            // DEBUG: cerr<<"handle query returned true"<<endl;
+            // if the query has been passed to a backend, or dropped, we can start
+            // reading again, or sending queued responses
+            if (state->d_queuedResponses.empty()) {
+              state->resetForNewQuery();
+              // DEBUG: cerr<<__LINE__<<endl;
+              iostate = IOState::NeedRead;
+              //state->d_ioState->update(IOState::NeedRead, handleIOCallback, state, state->getClientReadTTD(now));
+              // DEBUG: cerr<<__LINE__<<endl;
+              //ioGuard.release();
+            }
+            else {
+              TCPResponse resp = std::move(state->d_queuedResponses.front());
+              state->d_queuedResponses.pop_front();
+              ioGuard.release();
+              state->sendResponse(state, now, std::move(resp));
+              return;
+            }
+          }
+          else {
+            /* otherwise the state should already be waiting for
+               the socket to be writable */
+            // DEBUG: cerr<<"should be waiting for writable socket"<<endl;
+            ioGuard.release();
+            return;
+          }
+        }
+        else {
+          wouldBlock = true;
         }
-
-        /* allocate a bit more memory to be able to spoof the content, get an answer from the cache
-           or to add ECS without allocating a new buffer */
-        state->d_buffer.resize(std::max(state->d_querySize + static_cast<size_t>(512), s_maxPacketCacheEntrySize));
-        state->d_currentPos = 0;
       }
-    }
 
-    if (state->d_state == IncomingTCPConnectionState::State::readingQuery) {
-      cerr<<"reading query"<<endl;
-      iostate = state->d_handler.tryRead(state->d_buffer, state->d_currentPos, state->d_querySize);
-      if (iostate == IOState::Done) {
-        cerr<<"query received"<<endl;
-        //handleNewIOState(state, IOState::Done, fd, handleIOCallback);
-        handleQuery(state, now);
-        state->resetForNewQuery();
-        state->d_ioState->update(IOState::NeedRead, handleIOCallback, state, state->getClientReadTTD(now));
-        ioGuard.release();
-        return;
+      if (state->d_state == IncomingTCPConnectionState::State::sendingResponse) {
+        // DEBUG: cerr<<"sending response"<<endl;
+        iostate = state->d_handler.tryWrite(state->d_currentResponse.d_buffer, state->d_currentPos, state->d_currentResponse.d_buffer.size());
+        if (iostate == IOState::Done) {
+          // DEBUG: cerr<<"response sent"<<endl;
+          iostate = handleResponseSent(state, now);
+        } else {
+          wouldBlock = true;
+          // DEBUG: cerr<<"partial write"<<endl;
+        }
+        // DEBUG: cerr<<__LINE__<<endl;
+        //state->d_ioState->update(IOState::NeedRead, handleIOCallback, state, state->getClientReadTTD(now));
+        //// DEBUG: cerr<<__LINE__<<endl;
       }
-    }
 
-    if (state->d_state == IncomingTCPConnectionState::State::sendingResponse) {
-      iostate = state->d_handler.tryWrite(state->d_responseBuffer, state->d_currentPos, state->d_responseBuffer.size());
-      if (iostate == IOState::Done) {
-        handleResponseSent(state, now);
-        state->d_ioState->update(IOState::NeedRead, handleIOCallback, state, state->getClientReadTTD(now));
-        ioGuard.release();
-        return;
+      if (state->d_state != IncomingTCPConnectionState::State::idle &&
+          state->d_state != IncomingTCPConnectionState::State::doingHandshake &&
+          state->d_state != IncomingTCPConnectionState::State::readingQuerySize &&
+          state->d_state != IncomingTCPConnectionState::State::readingQuery &&
+          state->d_state != IncomingTCPConnectionState::State::sendingResponse) {
+        vinfolog("Unexpected state %d in handleIOCallback", static_cast<int>(state->d_state));
       }
     }
+    catch(const std::exception& e) {
+      /* most likely an EOF because the other end closed the connection,
+         but it might also be a real IO error or something else.
+         Let's just drop the connection
+      */
+      if (state->d_state == IncomingTCPConnectionState::State::idle ||
+          state->d_state == IncomingTCPConnectionState::State::doingHandshake ||
+          state->d_state == IncomingTCPConnectionState::State::readingQuerySize ||
+          state->d_state == IncomingTCPConnectionState::State::readingQuery) {
+        ++state->d_ci.cs->tcpDiedReadingQuery;
+      }
+      else if (state->d_state == IncomingTCPConnectionState::State::sendingResponse) {
+        ++state->d_ci.cs->tcpDiedSendingResponse;
+      }
 
-    if (state->d_state != IncomingTCPConnectionState::State::doingHandshake &&
-        state->d_state != IncomingTCPConnectionState::State::readingQuerySize &&
-        state->d_state != IncomingTCPConnectionState::State::readingQuery &&
-        state->d_state != IncomingTCPConnectionState::State::sendingResponse) {
-      vinfolog("Unexpected state %d in handleIOCallback", static_cast<int>(state->d_state));
-    }
-  }
-  catch(const std::exception& e) {
-    /* most likely an EOF because the other end closed the connection,
-       but it might also be a real IO error or something else.
-       Let's just drop the connection
-    */
-    if (state->d_state == IncomingTCPConnectionState::State::doingHandshake ||
-        state->d_state == IncomingTCPConnectionState::State::readingQuerySize ||
-        state->d_state == IncomingTCPConnectionState::State::readingQuery) {
-      ++state->d_ci.cs->tcpDiedReadingQuery;
-    }
-    else if (state->d_state == IncomingTCPConnectionState::State::sendingResponse) {
-      ++state->d_ci.cs->tcpDiedSendingResponse;
+      if (state->d_lastIOState == IOState::NeedWrite || state->d_readingFirstQuery) {
+        // DEBUG: cerr<<"Got an exception while handling TCP query: "<<e.what()<<endl;
+        vinfolog("Got an exception while handling (%s) TCP query from %s: %s", (state->d_lastIOState == IOState::NeedRead ? "reading" : "writing"), state->d_ci.remote.toStringWithPort(), e.what());
+      }
+      else {
+        vinfolog("Closing TCP client connection with %s", state->d_ci.remote.toStringWithPort());
+        // DEBUG: cerr<<"Closing TCP client connection: "<<e.what()<<endl;
+      }
+      /* remove this FD from the IO multiplexer */
+      iostate = IOState::Done;
     }
 
-    if (state->d_lastIOState == IOState::NeedWrite || state->d_readingFirstQuery) {
-      vinfolog("Got an exception while handling (%s) TCP query from %s: %s", (state->d_lastIOState == IOState::NeedRead ? "reading" : "writing"), state->d_ci.remote.toStringWithPort(), e.what());
+    if (iostate == IOState::Done) {
+      // DEBUG: cerr<<__LINE__<<endl;
+      state->d_ioState->update(iostate, handleIOCallback, state);
+      // DEBUG: cerr<<__LINE__<<endl;
     }
     else {
-      vinfolog("Closing TCP client connection with %s", state->d_ci.remote.toStringWithPort());
+      // DEBUG: cerr<<__LINE__<<endl;
+      state->d_ioState->update(iostate, handleIOCallback, state, iostate == IOState::NeedRead ? state->getClientReadTTD(now) : state->getClientWriteTTD(now));
+      // DEBUG: cerr<<__LINE__<<endl;
     }
-    /* remove this FD from the IO multiplexer */
-    iostate = IOState::Done;
+    ioGuard.release();
   }
+  while (state->d_state == IncomingTCPConnectionState::State::readingQuerySize && iostate == IOState::NeedRead && !wouldBlock);
+}
 
-  if (iostate == IOState::Done) {
-    state->d_ioState->update(iostate, handleIOCallback, state);
+void IncomingTCPConnectionState::notifyIOError(std::shared_ptr<IncomingTCPConnectionState>& state, IDState&& query, const struct timeval& now)
+{
+  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
+  if (d_isXFR) {
+    d_xfrDone = true;
+  }
+
+  if (d_state == State::sendingResponse) {
+    /* if we have responses to send, let's do that first */
+  }
+  else if (!d_queuedResponses.empty()) {
+    /* stop reading and send what we have */
+    TCPResponse resp = std::move(d_queuedResponses.front());
+    d_queuedResponses.pop_front();
+    sendResponse(state, now, std::move(resp));
   }
   else {
-    state->d_ioState->update(iostate, handleIOCallback, state, iostate == IOState::NeedRead ? state->getClientReadTTD(now) : state->getClientWriteTTD(now));
+    // the backend code already tried to reconnect if it was possible
+    d_lastIOState = IOState::Done;
+    d_ioState->reset();
   }
-  ioGuard.release();
+
+  // DEBUG: cerr<<"out "<<__PRETTY_FUNCTION__<<endl;
 }
 
-void IncomingTCPConnectionState::notifyIOError(IDState&& query)
+void IncomingTCPConnectionState::handleXFRResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
 {
-  #warning NEED TO BE IMPLEMENTED
+  sendResponse(state, now, std::move(response));
 }
 
-void IncomingTCPConnectionState::sendXFRResponse(std::vector<uint8_t>&& response, const IDState& query)
+void IncomingTCPConnectionState::handleTimeout(bool write)
 {
-  #warning NEED TO BE IMPLEMENTED
+  // DEBUG: cerr<<"client timeout"<<endl;
+  ++d_ci.cs->tcpClientTimeouts;
+  d_lastIOState = IOState::Done;
+  d_ioState->reset();
 }
 
 static void handleIncomingTCPQuery(int pipefd, FDMultiplexer::funcparam_t& param)
@@ -814,35 +948,43 @@ static void tcpClientThread(int pipefd)
     if (now.tv_sec > lastTimeoutScan) {
       lastTimeoutScan = now.tv_sec;
       auto expiredReadConns = data.mplexer->getTimeouts(now, false);
-      for(const auto& conn : expiredReadConns) {
-        auto state = boost::any_cast<std::shared_ptr<IncomingTCPConnectionState>>(conn.second);
-        if (conn.first == state->d_ci.fd) {
-          vinfolog("Timeout (read) from remote TCP client %s", state->d_ci.remote.toStringWithPort());
-          ++state->d_ci.cs->tcpClientTimeouts;
+      for (const auto& cbData : expiredReadConns) {
+        if (cbData.second.type() == typeid(std::shared_ptr<IncomingTCPConnectionState>)) {
+          auto state = boost::any_cast<std::shared_ptr<IncomingTCPConnectionState>>(cbData.second);
+          if (cbData.first == state->d_ci.fd) {
+            vinfolog("Timeout (read) from remote TCP client %s", state->d_ci.remote.toStringWithPort());
+            state->handleTimeout(false);
+          }
         }
-        else if (state->d_ds) {
-          vinfolog("Timeout (read) from remote backend %s", state->d_ds->getName());
-          ++state->d_ci.cs->tcpDownstreamTimeouts;
-          ++state->d_ds->tcpReadTimeouts;
+        else if (cbData.second.type() == typeid(std::shared_ptr<TCPConnectionToBackend>)) {
+          auto conn = boost::any_cast<std::shared_ptr<TCPConnectionToBackend>>(cbData.second);
+          vinfolog("Timeout (read) from remote backend %s", conn->getBackendName());
+          conn->handleTimeout(now, false);
         }
-        data.mplexer->removeReadFD(conn.first);
-        state->d_lastIOState = IOState::Done;
       }
 
       auto expiredWriteConns = data.mplexer->getTimeouts(now, true);
-      for(const auto& conn : expiredWriteConns) {
-        auto state = boost::any_cast<std::shared_ptr<IncomingTCPConnectionState>>(conn.second);
-        if (conn.first == state->d_ci.fd) {
-          vinfolog("Timeout (write) from remote TCP client %s", state->d_ci.remote.toStringWithPort());
-          ++state->d_ci.cs->tcpClientTimeouts;
+      for (const auto& cbData : expiredWriteConns) {
+        if (cbData.second.type() == typeid(std::shared_ptr<IncomingTCPConnectionState>)) {
+          auto state = boost::any_cast<std::shared_ptr<IncomingTCPConnectionState>>(cbData.second);
+          if (cbData.first == state->d_ci.fd) {
+            vinfolog("Timeout (write) from remote TCP client %s", state->d_ci.remote.toStringWithPort());
+            state->handleTimeout(true);
+          }
+        }
+        else if (cbData.second.type() == typeid(std::shared_ptr<TCPConnectionToBackend>)) {
+          auto conn = boost::any_cast<std::shared_ptr<TCPConnectionToBackend>>(cbData.second);
+          vinfolog("Timeout (write) from remote backend %s", conn->getBackendName());
+          conn->handleTimeout(now, true);
         }
-        else if (state->d_ds) {
-          vinfolog("Timeout (write) from remote backend %s", state->d_ds->getName());
-          ++state->d_ci.cs->tcpDownstreamTimeouts;
-          ++state->d_ds->tcpWriteTimeouts;
+#if 0
+        try {
+          data.mplexer->removeWriteFD(cbData.first);
+        }
+        catch (const FDMultiplexerException& fde) {
+          warnlog("Exception while removing a socket (%d) after a write timeout: %s", cbData.first, fde.what());
         }
-        data.mplexer->removeWriteFD(conn.first);
-        state->d_lastIOState = IOState::Done;
+#endif
       }
     }
   }
diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh
index fbdc720d5c..0dee6cc8df 100644
--- a/pdns/dnsdist.hh
+++ b/pdns/dnsdist.hh
@@ -466,7 +466,7 @@ struct IDState
 {
   IDState(): sentTime(true), delayMsec(0), tempFailureTTL(boost::none) { origDest.sin4.sin_family = 0;}
   IDState(const IDState& orig) = delete;
-  IDState(IDState&& rhs): origRemote(rhs.origRemote), origDest(rhs.origDest), sentTime(rhs.sentTime), qname(std::move(rhs.qname)), dnsCryptQuery(std::move(rhs.dnsCryptQuery)), subnet(rhs.subnet), packetCache(std::move(rhs.packetCache)), qTag(std::move(qTag)), cs(rhs.cs), du(std::move(rhs.du)), cacheKey(rhs.cacheKey), cacheKeyNoECS(rhs.cacheKeyNoECS), age(rhs.age), qtype(rhs.qtype), qclass(rhs.qclass), origID(rhs.origID), origFlags(rhs.origFlags), origFD(rhs.origFD), delayMsec(rhs.delayMsec), tempFailureTTL(rhs.tempFailureTTL), ednsAdded(rhs.ednsAdded), ecsAdded(rhs.ecsAdded), skipCache(rhs.skipCache), destHarvested(rhs.destHarvested), dnssecOK(rhs.dnssecOK), useZeroScope(rhs.useZeroScope)
+  IDState(IDState&& rhs): origRemote(rhs.origRemote), origDest(rhs.origDest), sentTime(rhs.sentTime), qname(std::move(rhs.qname)), dnsCryptQuery(std::move(rhs.dnsCryptQuery)), subnet(rhs.subnet), packetCache(std::move(rhs.packetCache)), qTag(std::move(rhs.qTag)), cs(rhs.cs), du(std::move(rhs.du)), cacheKey(rhs.cacheKey), cacheKeyNoECS(rhs.cacheKeyNoECS), age(rhs.age), qtype(rhs.qtype), qclass(rhs.qclass), origID(rhs.origID), origFlags(rhs.origFlags), origFD(rhs.origFD), delayMsec(rhs.delayMsec), tempFailureTTL(rhs.tempFailureTTL), ednsAdded(rhs.ednsAdded), ecsAdded(rhs.ecsAdded), skipCache(rhs.skipCache), destHarvested(rhs.destHarvested), dnssecOK(rhs.dnssecOK), useZeroScope(rhs.useZeroScope)
   {
     if (rhs.isInUse()) {
       throw std::runtime_error("Trying to move an in-use IDState");
@@ -494,7 +494,7 @@ struct IDState
     dnsCryptQuery = std::move(rhs.dnsCryptQuery);
     subnet = rhs.subnet;
     packetCache = std::move(rhs.packetCache);
-    qTag = std::move(qTag);
+    qTag = std::move(rhs.qTag);
     cs = rhs.cs;
     du = std::move(rhs.du);
     cacheKey = rhs.cacheKey;
@@ -610,14 +610,14 @@ struct IDState
   uint16_t origID;                                            // 2
   uint16_t origFlags;                                         // 2
   int origFD{-1};
-  int delayMsec;
+  int delayMsec{0};
   boost::optional<uint32_t> tempFailureTTL;
   bool ednsAdded{false};
   bool ecsAdded{false};
   bool skipCache{false};
   bool destHarvested{false}; // if true, origDest holds the original dest addr, otherwise the listening addr
   bool dnssecOK{false};
-  bool useZeroScope;
+  bool useZeroScope{false};
 };
 
 typedef std::unordered_map<string, unsigned int> QueryCountRecords;
diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
index 7e8741e380..86dee17204 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
@@ -4,19 +4,30 @@
 
 const uint16_t TCPConnectionToBackend::s_xfrID = 0;
 
-void TCPConnectionToBackend::assignToClientConnection(std::shared_ptr<IncomingTCPConnectionState>& clientConn)
+void TCPConnectionToBackend::assignToClientConnection(std::shared_ptr<IncomingTCPConnectionState>& clientConn, bool isXFR)
 {
-  cerr<<"in "<<__func__<<endl;
-  // taking a WEAK pointer
+  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
+  if (isXFR) {
+    d_usedForXFR = true;
+  }
+
   d_clientConn = clientConn;
   d_ioState = make_unique<IOStateHandler>(clientConn->getIOMPlexer(), d_socket->getHandle());
 }
 
+IOState TCPConnectionToBackend::sendNextQuery(std::shared_ptr<TCPConnectionToBackend>& conn)
+{
+  conn->d_currentQuery = std::move(conn->d_pendingQueries.front());
+  conn->d_pendingQueries.pop_front();
+  conn->d_state = State::sendingQueryToBackend;
+  return IOState::NeedWrite;
+}
+
 void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now)
 {
-  cerr<<"in "<<__func__<<endl;
+  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
   if (conn->d_socket == nullptr) {
-    throw std::runtime_error("No downstream socket in " + std::string(__func__) + "!");
+    throw std::runtime_error("No downstream socket in " + std::string(__PRETTY_FUNCTION__) + "!");
   }
 
   bool connectionDied = false;
@@ -26,7 +37,7 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
 
   try {
     if (conn->d_state == State::sendingQueryToBackend) {
-      cerr<<"sending query to backend"<<endl;
+      // DEBUG: cerr<<"sending query to backend over FD "<<fd<<endl;
       int socketFlags = 0;
 #ifdef MSG_FASTOPEN
       if (conn->isFastOpenEnabled()) {
@@ -36,12 +47,14 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
 
       size_t sent = sendMsgWithOptions(fd, reinterpret_cast<const char *>(&conn->d_currentQuery.d_buffer.at(conn->d_currentPos)), conn->d_currentQuery.d_buffer.size() - conn->d_currentPos, &conn->d_ds->remote, &conn->d_ds->sourceAddr, conn->d_ds->sourceItf, socketFlags);
       if (sent == conn->d_currentQuery.d_buffer.size()) {
-        cerr<<"query sent to backend"<<endl;
+        // DEBUG: cerr<<"query sent to backend"<<endl;
         /* request sent ! */
         conn->incQueries();
         conn->d_currentPos = 0;
         //conn->d_currentQuery.d_querySentTime = now;
-        conn->d_pendingResponses.insert({ conn->d_currentQuery.d_idstate.origID, std::move(conn->d_currentQuery.d_idstate) });
+        // DEBUG: cerr<<"adding a pending response for ID "<<conn->d_currentQuery.d_idstate.origID<<" and QNAME "<<conn->d_currentQuery.d_idstate.qname<<endl;
+        // DEBUG: cerr<<"IDS has "<<(conn->d_currentQuery.d_idstate.qTag?"tags":"no tags")<<endl;
+        conn->d_pendingResponses[conn->d_currentQuery.d_idstate.origID] = std::move(conn->d_currentQuery);
         conn->d_currentQuery.d_buffer.clear();
 #if 0
         if (!conn->d_usedForXFR) {
@@ -53,13 +66,12 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
 
         if (conn->d_pendingQueries.empty()) {
           conn->d_state = State::readingResponseSizeFromBackend;
+          conn->d_currentPos = 0;
+          conn->d_responseBuffer.resize(sizeof(uint16_t));
           iostate = IOState::NeedRead;
         }
         else {
-          conn->d_currentQuery = std::move(conn->d_pendingQueries.front());
-          conn->d_pendingQueries.pop_front();
-          conn->d_state = State::sendingQueryToBackend;
-          iostate = IOState::NeedWrite;
+          iostate = sendNextQuery(conn);
         }
       }
       else {
@@ -71,26 +83,28 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
     }
 
     if (conn->d_state == State::readingResponseSizeFromBackend) {
-      cerr<<"reading response size from backend"<<endl;
+      // DEBUG: cerr<<"reading response size from backend"<<endl;
       // then we need to allocate a new buffer (new because we might need to re-send the query if the
       // backend dies on us)
       // We also might need to read and send to the client more than one response in case of XFR (yeah!)
       // should very likely be a TCPIOHandler d_downstreamHandler
+      conn->d_responseBuffer.resize(sizeof(uint16_t));
       iostate = tryRead(fd, conn->d_responseBuffer, conn->d_currentPos, sizeof(uint16_t) - conn->d_currentPos);
       if (iostate == IOState::Done) {
-        cerr<<"got response size from backend"<<endl;
+        // DEBUG: cerr<<"got response size from backend"<<endl;
         conn->d_state = State::readingResponseFromBackend;
         conn->d_responseSize = conn->d_responseBuffer.at(0) * 256 + conn->d_responseBuffer.at(1);
+        conn->d_responseBuffer.reserve(conn->d_responseSize + /* we will need to prepend the size later */ 2);
         conn->d_responseBuffer.resize(conn->d_responseSize);
         conn->d_currentPos = 0;
       }
     }
 
     if (conn->d_state == State::readingResponseFromBackend) {
-      cerr<<"reading response from backend"<<endl;
+      // DEBUG: cerr<<"reading response from backend"<<endl;
       iostate = tryRead(fd, conn->d_responseBuffer, conn->d_currentPos, conn->d_responseSize - conn->d_currentPos);
       if (iostate == IOState::Done) {
-        cerr<<"got response from backend"<<endl;
+        // DEBUG: cerr<<"got response from backend"<<endl;
         //conn->d_responseReadTime = now;
         try {
           iostate = conn->handleResponse(now);
@@ -98,11 +112,12 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
         catch (const std::exception& e) {
           vinfolog("Got an exception while handling TCP response from %s (client is %s): %s", conn->d_ds ? conn->d_ds->getName() : "unknown", conn->d_currentQuery.d_idstate.origRemote.toStringWithPort(), e.what());
         }
-        return;
+        //return;
       }
     }
 
-    if (conn->d_state != State::sendingQueryToBackend &&
+    if (conn->d_state != State::idle &&
+        conn->d_state != State::sendingQueryToBackend &&
         conn->d_state != State::readingResponseSizeFromBackend &&
         conn->d_state != State::readingResponseFromBackend) {
       vinfolog("Unexpected state %d in TCPConnectionToBackend::handleIO", static_cast<int>(conn->d_state));
@@ -140,29 +155,71 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
     connectionDied = true;
   }
 
+  if (connectionDied) {
+    bool reconnected = false;
+    // DEBUG: cerr<<"connection died, number of failures is "<<conn->d_downstreamFailures<<", retries is "<<conn->d_ds->retries<<endl;
+
+    if ((!conn->d_usedForXFR || conn->d_queries == 0) && conn->d_downstreamFailures < conn->d_ds->retries) {
+      // DEBUG: cerr<<"reconnecting"<<endl;
+      conn->d_ioState->reset();
+      ioGuard.release();
+
+      if (conn->reconnect()) {
+        // DEBUG: cerr<<"reconnected"<<endl;
+
+        conn->d_ioState = make_unique<IOStateHandler>(conn->d_clientConn->getIOMPlexer(), conn->d_socket->getHandle());
+        // DEBUG: cerr<<"new state"<<endl;
+
+        for (auto& pending : conn->d_pendingResponses) {
+          conn->d_pendingQueries.push_back(std::move(pending.second));
+        }
+        conn->d_pendingResponses.clear();
+        conn->d_currentPos = 0;
+
+        if (conn->d_state == State::doingHandshake ||
+            conn->d_state == State::sendingQueryToBackend) {
+          iostate = IOState::NeedWrite;
+          // resume sending query
+        }
+        else {
+          // DEBUG: cerr<<"sending next query"<<endl;
+          iostate = sendNextQuery(conn);
+          // DEBUG: cerr<<"after call to sendNextQuery"<<endl;
+        }
+
+        if (!conn->d_proxyProtocolPayloadAdded && !conn->d_proxyProtocolPayload.empty()) {
+          conn->d_currentQuery.d_buffer.insert(conn->d_currentQuery.d_buffer.begin(), conn->d_proxyProtocolPayload.begin(), conn->d_proxyProtocolPayload.end());
+          conn->d_proxyProtocolPayloadAdded = true;
+        }
+
+        reconnected = true;
+      }
+    }
+
+    if (!reconnected) {
+      /* reconnect failed, we give up */
+      conn->d_connectionDied = true;
+      conn->notifyAllQueriesFailed(now);
+    }
+  }
+
   if (iostate == IOState::Done) {
-    cerr<<"in "<<__func__<<", done"<<endl;
+    // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<", done"<<endl;
     conn->d_ioState->update(iostate, handleIOCallback, conn);
   }
   else {
-    cerr<<"in "<<__func__<<", updating to "<<(int)iostate<<endl;
+    // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<", updating to "<<(int)iostate<<endl;
     conn->d_ioState->update(iostate, handleIOCallback, conn, iostate == IOState::NeedRead ? conn->getBackendReadTTD(now) : conn->getBackendWriteTTD(now));
   }
   ioGuard.release();
 
-  if (connectionDied) {
-#warning FIXME: if we had only one question pending we might try sending it again
-    conn->d_connectionDied = true;
-
-    conn->notifyAllQueriesFailed();
-  }
 }
 
 void TCPConnectionToBackend::handleIOCallback(int fd, FDMultiplexer::funcparam_t& param)
 {
   auto conn = boost::any_cast<std::shared_ptr<TCPConnectionToBackend>>(param);
   if (fd != conn->getHandle()) {
-    throw std::runtime_error("Unexpected socket descriptor " + std::to_string(fd) + " received in " + std::string(__func__) + ", expected " + std::to_string(conn->getHandle()));
+    throw std::runtime_error("Unexpected socket descriptor " + std::to_string(fd) + " received in " + std::string(__PRETTY_FUNCTION__) + ", expected " + std::to_string(conn->getHandle()));
   }
 
   struct timeval now;
@@ -172,7 +229,8 @@ void TCPConnectionToBackend::handleIOCallback(int fd, FDMultiplexer::funcparam_t
 
 void TCPConnectionToBackend::queueQuery(TCPQuery&& query, std::shared_ptr<TCPConnectionToBackend>& sharedSelf)
 {
-  cerr<<"in "<<__func__<<endl;
+  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
+  // DEBUG: cerr<<"IDS has "<<(query.d_idstate.qTag?"tags":"no tags")<<endl;
   if (d_ioState == nullptr) {
     throw std::runtime_error("Trying to queue a query to a TCP connection that has no incoming client connection assigned");
   }
@@ -182,7 +240,7 @@ void TCPConnectionToBackend::queueQuery(TCPQuery&& query, std::shared_ptr<TCPCon
   if (d_state == State::idle || d_state == State::waitingForResponseFromBackend) {
     d_state = State::sendingQueryToBackend;
     d_currentQuery = std::move(query);
-    cerr<<"need write"<<endl;
+    // DEBUG: cerr<<"need write"<<endl;
 
     struct timeval now;
     gettimeofday(&now, 0);
@@ -193,6 +251,7 @@ void TCPConnectionToBackend::queueQuery(TCPQuery&& query, std::shared_ptr<TCPCon
     // store query in the list of queries to send
     d_pendingQueries.push_back(std::move(query));
   }
+  // DEBUG: cerr<<"out of "<<__PRETTY_FUNCTION__<<endl;
 }
 
 bool TCPConnectionToBackend::reconnect()
@@ -200,15 +259,18 @@ bool TCPConnectionToBackend::reconnect()
   std::unique_ptr<Socket> result;
 
   if (d_socket) {
+    // DEBUG: cerr<<"closing socket "<<d_socket->getHandle()<<endl;
+    shutdown(d_socket->getHandle(), SHUT_RDWR);
     d_socket.reset();
     d_ioState.reset();
     --d_ds->tcpCurrentConnections;
   }
 
   do {
-    vinfolog("TCP connecting to downstream %s (%d)", d_ds->remote.toStringWithPort(), d_downstreamFailures);
+    vinfolog("TCP connecting to downstream %s (%d)", d_ds->getNameWithAddr(), d_downstreamFailures);
     try {
       result = std::unique_ptr<Socket>(new Socket(d_ds->remote.sin4.sin_family, SOCK_STREAM, 0));
+      // DEBUG: cerr<<"result of connect is "<<result->getHandle()<<endl;
       if (!IsAnyAddress(d_ds->sourceAddr)) {
         SSetsockopt(result->getHandle(), SOL_SOCKET, SO_REUSEADDR, 1);
 #ifdef IP_BIND_ADDRESS_NO_PORT
@@ -228,7 +290,7 @@ bool TCPConnectionToBackend::reconnect()
       }
       result->setNonBlocking();
 #ifdef MSG_FASTOPEN
-      if (!d_ds->tcpFastOpen) {
+      if (!d_ds->tcpFastOpen || !isFastOpenEnabled()) {
         SConnectWithTimeout(result->getHandle(), d_ds->remote, /* no timeout, we will handle it ourselves */ 0);
       }
 #else
@@ -236,7 +298,7 @@ bool TCPConnectionToBackend::reconnect()
 #endif /* MSG_FASTOPEN */
 
       d_socket = std::move(result);
-      //d_ioState->setSocket(d_socket->getHandle());
+      // DEBUG: cerr<<"connected new socket "<<d_socket->getHandle()<<endl;
       ++d_ds->tcpCurrentConnections;
       return true;
     }
@@ -253,63 +315,133 @@ bool TCPConnectionToBackend::reconnect()
   return false;
 }
 
-void TCPConnectionToBackend::notifyAllQueriesFailed()
+void TCPConnectionToBackend::handleTimeout(const struct timeval& now, bool write)
+{
+  if (write) {
+    ++d_ds->tcpWriteTimeouts;
+  }
+  else {
+    ++d_ds->tcpReadTimeouts;
+  }
+
+  if (d_ioState) {
+    d_ioState->reset();
+  }
+
+  notifyAllQueriesFailed(now, true);
+}
+
+void TCPConnectionToBackend::notifyAllQueriesFailed(const struct timeval& now, bool timeout)
 {
   d_connectionDied = true;
-  auto clientConn = d_clientConn.lock();
-  if (!clientConn) {
+  //auto clientConn = d_clientConn.lock();
+  //if (!clientConn) {
+  //  d_clientConn.reset();
+  //  return;
+  //}
+  auto& clientConn = d_clientConn;
+  if (!clientConn->active()) {
+    // a client timeout occured, or something like that */
+    d_connectionDied = true;
+    d_clientConn.reset();
     return;
   }
 
+  if (timeout) {
+    ++clientConn->d_ci.cs->tcpDownstreamTimeouts;
+  }
+
+  if (d_state == State::doingHandshake || d_state == State::sendingQueryToBackend) {
+    clientConn->notifyIOError(clientConn, std::move(d_currentQuery.d_idstate), now);
+  }
+
   for (auto& query : d_pendingQueries) {
-    clientConn->notifyIOError(std::move(query.d_idstate));
+    clientConn->notifyIOError(clientConn, std::move(query.d_idstate), now);
   }
 
   for (auto& response : d_pendingResponses) {
-    clientConn->notifyIOError(std::move(response.second));
+    clientConn->notifyIOError(clientConn, std::move(response.second.d_idstate), now);
   }
+
+  d_pendingQueries.clear();
+  d_pendingResponses.clear();
+
+  d_clientConn.reset();
 }
 
 IOState TCPConnectionToBackend::handleResponse(const struct timeval& now)
 {
-  cerr<<"in "<<__func__<<endl;
-  auto clientConn = d_clientConn.lock();
-  if (!clientConn) {
+  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
+  //auto clientConn = d_clientConn.lock();
+  //if (!clientConn) {
+  //d_clientConn.reset();
+  //  d_connectionDied = true;
+  //  // DEBUG: cerr<<"connection to client died, bye bye"<<endl;
+  //  return IOState::Done;
+  //}
+
+  auto& clientConn = d_clientConn;
+  if (!clientConn->active()) {
+    // DEBUG: cerr<<"client is not active"<<endl;
+    // a client timeout occured, or something like that */
     d_connectionDied = true;
-    cerr<<"connection to client died, bye bye"<<endl;
+    d_clientConn.reset();
     return IOState::Done;
   }
 
   if (d_usedForXFR) {
-    clientConn->sendXFRResponse(std::move(d_responseBuffer), d_pendingResponses.at(s_xfrID));
+    // DEBUG: cerr<<"XFR!"<<endl;
+    TCPResponse response;
+    response.d_buffer = std::move(d_responseBuffer);
+    response.d_ds = d_ds;
+    clientConn->handleXFRResponse(clientConn, now, std::move(response));
+    d_state = State::readingResponseSizeFromBackend;
+    d_currentPos = 0;
+    d_responseBuffer.resize(sizeof(uint16_t));
     return IOState::NeedRead;
     // get ready to read the next packet, if any
   }
   else {
-    uint16_t queryId = getQueryIdFromResponse();
-    if (d_pendingResponses.count(queryId) == 0) {
-      notifyAllQueriesFailed();
+    // DEBUG: cerr<<"not XFR, phew"<<endl;
+    uint16_t queryId = 0;
+    try {
+      queryId = getQueryIdFromResponse();
+    }
+    catch (const std::exception& e) {
+      notifyAllQueriesFailed(now);
+      throw;
+    }
+
+    auto it = d_pendingResponses.find(queryId);
+    if (it == d_pendingResponses.end()) {
+      // DEBUG: cerr<<"could not found any corresponding query for ID "<<queryId<<endl;
+      notifyAllQueriesFailed(now);
       return IOState::Done;
     }
-    cerr<<"passing response to client connection"<<endl;
-    clientConn->sendResponse(now, std::move(d_responseBuffer), std::move(d_pendingResponses.at(queryId)));
-    d_pendingResponses.erase(queryId);
+    auto ids = std::move(it->second.d_idstate);
+    // DEBUG: cerr<<"IDS has "<<(ids.qTag?" TAGS ": "NO TAGS")<<endl;
+    // DEBUG: cerr<<"passing response to client connection for "<<ids.qname<<endl;
+    clientConn->handleResponse(clientConn, now, TCPResponse(std::move(d_responseBuffer), std::move(ids), d_ds));
+    d_pendingResponses.erase(it);
 
     if (!d_pendingQueries.empty()) {
-      cerr<<"still have some queries to send"<<endl;
+      // DEBUG: cerr<<"still have some queries to send"<<endl;
       d_state = State::sendingQueryToBackend;
       d_currentQuery = std::move(d_pendingQueries.front());
       d_pendingQueries.pop_front();
       return IOState::NeedWrite;
     }
     else if (!d_pendingResponses.empty()) {
-      cerr<<"still have some responses to read"<<endl;
+      // DEBUG: cerr<<"still have some responses to read"<<endl;
       d_state = State::readingResponseSizeFromBackend;
+      d_currentPos = 0;
+      d_responseBuffer.resize(sizeof(uint16_t));
       return IOState::NeedRead;
     }
     else {
-      cerr<<"nothing to do, phewwwww"<<endl;
+      // DEBUG: cerr<<"nothing to do, phewwwww"<<endl;
       d_state = State::idle;
+      d_clientConn.reset();
       return IOState::Done;
     }
   }
@@ -318,7 +450,6 @@ IOState TCPConnectionToBackend::handleResponse(const struct timeval& now)
 uint16_t TCPConnectionToBackend::getQueryIdFromResponse()
 {
   if (d_responseBuffer.size() < sizeof(dnsheader)) {
-    notifyAllQueriesFailed();
     throw std::runtime_error("Unable to get query ID in a too small (" + std::to_string(d_responseBuffer.size()) + ") response from " + d_ds->getNameWithAddr());
   }
 
@@ -329,10 +460,10 @@ uint16_t TCPConnectionToBackend::getQueryIdFromResponse()
 
 void TCPConnectionToBackend::setProxyProtocolPayload(std::string&& payload)
 {
-  #warning TO DO
+  d_proxyProtocolPayload = std::move(payload);
 }
 
 void TCPConnectionToBackend::setProxyProtocolPayloadAdded(bool added)
 {
-  #warning TO DO
+  d_proxyProtocolPayloadAdded = added;
 }
diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
index 84149d8943..9d24de5f17 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
@@ -22,6 +22,17 @@ struct TCPQuery
 
 struct TCPResponse : public TCPQuery
 {
+  TCPResponse()
+  {
+  }
+
+  TCPResponse(std::vector<uint8_t>&& buffer, IDState&& state, std::shared_ptr<DownstreamState> ds): TCPQuery(std::move(buffer), std::move(state)), d_ds(ds)
+  {
+  }
+
+  std::shared_ptr<DownstreamState> d_ds{nullptr};
+  dnsheader d_cleartextDH;
+  bool d_selfGenerated{false};
 };
 
 class IncomingTCPConnectionState;
@@ -46,7 +57,7 @@ public:
     }
   }
 
-  void assignToClientConnection(std::shared_ptr<IncomingTCPConnectionState>& clientConn);
+  void assignToClientConnection(std::shared_ptr<IncomingTCPConnectionState>& clientConn, bool isXFR);
 
   int getHandle() const
   {
@@ -62,6 +73,11 @@ public:
     return d_ds->remote;
   }
 
+  const std::string& getBackendName() const
+  {
+    return d_ds->getName();
+  }
+
   bool isFresh() const
   {
     return d_fresh;
@@ -98,7 +114,7 @@ public:
     return true;
   }
 
-  bool canBeReused() const 
+  bool canBeReused() const
   {
     if (d_usedForXFR || d_connectionDied) {
       return false;
@@ -124,8 +140,10 @@ public:
 
   static void handleIO(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now);
   static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param);
+  static IOState sendNextQuery(std::shared_ptr<TCPConnectionToBackend>& conn);
 
   void queueQuery(TCPQuery&& query, std::shared_ptr<TCPConnectionToBackend>& sharedSelf);
+  void handleTimeout(const struct timeval& now, bool write);
   IOState handleResponse(const struct timeval& now);
   void setProxyProtocolPayload(std::string&& payload);
   void setProxyProtocolPayloadAdded(bool added);
@@ -133,7 +151,7 @@ public:
 private:
   uint16_t getQueryIdFromResponse();
   bool reconnect();
-  void notifyAllQueriesFailed();
+  void notifyAllQueriesFailed(const struct timeval& now, bool timeout = false);
 
   boost::optional<struct timeval> getBackendReadTTD(const struct timeval& now) const
   {
@@ -172,11 +190,13 @@ private:
 
   std::vector<uint8_t> d_responseBuffer;
   std::deque<TCPQuery> d_pendingQueries;
-  std::unordered_map<uint16_t, IDState> d_pendingResponses;
+  std::unordered_map<uint16_t, TCPQuery> d_pendingResponses;
   std::unique_ptr<Socket> d_socket{nullptr};
   std::unique_ptr<IOStateHandler> d_ioState{nullptr};
   std::shared_ptr<DownstreamState> d_ds{nullptr};
-  std::weak_ptr<IncomingTCPConnectionState> d_clientConn;
+  //std::weak_ptr<IncomingTCPConnectionState> d_clientConn;
+  std::shared_ptr<IncomingTCPConnectionState> d_clientConn;
+  std::string d_proxyProtocolPayload;
   TCPQuery d_currentQuery;
   struct timeval d_connectionStartTime;
   size_t d_currentPos{0};
@@ -187,7 +207,6 @@ private:
   bool d_fresh{true};
   bool d_enableFastOpen{false};
   bool d_connectionDied{true};
-#warning FIXME : must be set!
   bool d_usedForXFR{false};
+  bool d_proxyProtocolPayloadAdded{false};
 };
-
diff --git a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
index cc624321d8..9479f0a403 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
@@ -58,13 +58,13 @@ class IncomingTCPConnectionState
 {
 public:
   //IncomingTCPConnectionState(ConnectionInfo&& ci, TCPClientThreadData& threadData, const struct timeval& now): d_buffer(s_maxPacketCacheEntrySize), d_responseBuffer(s_maxPacketCacheEntrySize), d_threadData(threadData), d_ci(std::move(ci)), d_handler(d_ci.fd, g_tcpRecvTimeout, d_ci.cs->tlsFrontend ? d_ci.cs->tlsFrontend->getContext() : nullptr, now.tv_sec), d_ioState(threadData.mplexer, d_ci.fd), _connectionStartTime(now)
-  IncomingTCPConnectionState(ConnectionInfo&& ci, TCPClientThreadData& threadData, const struct timeval& now): d_buffer(s_maxPacketCacheEntrySize), d_responseBuffer(s_maxPacketCacheEntrySize), d_threadData(threadData), d_ci(std::move(ci)), d_handler(d_ci.fd, g_tcpRecvTimeout, d_ci.cs->tlsFrontend ? d_ci.cs->tlsFrontend->getContext() : nullptr, now.tv_sec), d_ioState(make_unique<IOStateHandler>(threadData.mplexer, d_ci.fd)), d_connectionStartTime(now)
+  IncomingTCPConnectionState(ConnectionInfo&& ci, TCPClientThreadData& threadData, const struct timeval& now): d_buffer(s_maxPacketCacheEntrySize), d_threadData(threadData), d_ci(std::move(ci)), d_handler(d_ci.fd, g_tcpRecvTimeout, d_ci.cs->tlsFrontend ? d_ci.cs->tlsFrontend->getContext() : nullptr, now.tv_sec), d_ioState(make_unique<IOStateHandler>(threadData.mplexer, d_ci.fd)), d_connectionStartTime(now)
   {
-    d_ids.origDest.reset();
-    d_ids.origDest.sin4.sin_family = d_ci.remote.sin4.sin_family;
-    socklen_t socklen = d_ids.origDest.getSocklen();
-    if (getsockname(d_ci.fd, reinterpret_cast<sockaddr*>(&d_ids.origDest), &socklen)) {
-      d_ids.origDest = d_ci.cs->local;
+    d_origDest.reset();
+    d_origDest.sin4.sin_family = d_ci.remote.sin4.sin_family;
+    socklen_t socklen = d_origDest.getSocklen();
+    if (getsockname(d_ci.fd, reinterpret_cast<sockaddr*>(&d_origDest), &socklen)) {
+      d_origDest = d_ci.cs->local;
     }
   }
 
@@ -73,17 +73,7 @@ public:
 
   ~IncomingTCPConnectionState();
 
-  void resetForNewQuery()
-  {
-    d_buffer.resize(sizeof(uint16_t));
-    d_currentPos = 0;
-    d_querySize = 0;
-    d_responseSize = 0;
-    d_downstreamFailures = 0;
-    d_state = State::readingQuerySize;
-    d_lastIOState = IOState::Done;
-    d_selfGeneratedResponse = false;
-  }
+  void resetForNewQuery();
 
   boost::optional<struct timeval> getClientReadTTD(struct timeval now) const
   {
@@ -192,25 +182,28 @@ public:
   static void handleIO(std::shared_ptr<IncomingTCPConnectionState>& conn, const struct timeval& now);
   static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param);
 
-  void notifyIOError(IDState&& query);
   void queueQuery(TCPQuery&& query);
-  void sendXFRResponse(std::vector<uint8_t>&& response, const IDState& query);
-  void sendResponse(std::vector<uint8_t>&& response, IDState&& query);
+  void notifyIOError(std::shared_ptr<IncomingTCPConnectionState>& state, IDState&& query, const struct timeval& now);
+  void sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
+  void handleResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
+  void handleXFRResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
+  void handleTimeout(bool write);
+
+  bool active() const
+  {
+    return d_ioState != nullptr;
+  }
 
   enum class State { doingHandshake, readingQuerySize, readingQuery, sendingResponse, idle /* in case of XFR, we stop processing queries */ };
 
   std::vector<uint8_t> d_buffer;
-  #warning unused?
-  std::vector<uint8_t> d_responseBuffer;
-  std::deque<TCPResponse> d_pendingResponses;
+  std::deque<TCPResponse> d_queuedResponses;
   TCPClientThreadData& d_threadData;
-  IDState d_ids;
+  TCPResponse d_currentResponse;
   ConnectionInfo d_ci;
+  ComboAddress d_origDest;
   TCPIOHandler d_handler;
   std::unique_ptr<IOStateHandler> d_ioState{nullptr};
-  std::unique_ptr<TCPConnectionToBackend> d_downstreamConnection{nullptr};
-  std::shared_ptr<DownstreamState> d_ds{nullptr};
-  dnsheader d_cleartextDH;
   struct timeval d_connectionStartTime;
   struct timeval d_handshakeDoneTime;
   struct timeval d_firstQuerySizeReadTime;
@@ -220,15 +213,13 @@ public:
   size_t d_queriesCount{0};
   unsigned int d_remainingTime{0};
   uint16_t d_querySize{0};
-  #warning unused?
-  uint16_t d_responseSize{0};
   uint16_t d_downstreamFailures{0};
   State d_state{State::doingHandshake};
   IOState d_lastIOState{IOState::Done};
   bool d_readingFirstQuery{true};
-  bool d_firstResponsePacket{true};
   bool d_isXFR{false};
   bool d_xfrStarted{false};
+  bool d_xfrDone{false};
   bool d_selfGeneratedResponse{false};
   bool d_proxyProtocolPayloadAdded{false};
   bool d_proxyProtocolPayloadHasTLV{false};
diff --git a/pdns/dnsdistdist/tcpiohandler-mplexer.hh b/pdns/dnsdistdist/tcpiohandler-mplexer.hh
index a64cf563e7..fdb1c5ed97 100644
--- a/pdns/dnsdistdist/tcpiohandler-mplexer.hh
+++ b/pdns/dnsdistdist/tcpiohandler-mplexer.hh
@@ -39,19 +39,19 @@ public:
   void reset()
   {
     update(IOState::Done);
-  }  
+  }
 
   void update(IOState iostate, FDMultiplexer::callbackfunc_t callback = FDMultiplexer::callbackfunc_t(), FDMultiplexer::funcparam_t callbackData = boost::any(), boost::optional<struct timeval> ttd = boost::none)
   {
-    //cerr<<"in "<<__func__<<" for fd "<<d_fd<<", last state was "<<(int)d_currentState<<", new state is "<<(int)iostate<<endl;
+    cerr<<"in "<<__PRETTY_FUNCTION__<<" for fd "<<d_fd<<", last state was "<<(int)d_currentState<<", new state is "<<(int)iostate<<endl;
     if (d_currentState == IOState::NeedRead && iostate != IOState::NeedRead) {
+      cerr<<__PRETTY_FUNCTION__<<": remove read FD "<<d_fd<<endl;
       d_mplexer->removeReadFD(d_fd);
-      //cerr<<__func__<<": remove read FD "<<d_fd<<endl;
       d_currentState = IOState::Done;
     }
     else if (d_currentState == IOState::NeedWrite && iostate != IOState::NeedWrite) {
+      cerr<<__PRETTY_FUNCTION__<<": remove write FD "<<d_fd<<endl;
       d_mplexer->removeWriteFD(d_fd);
-      //cerr<<__func__<<": remove write FD "<<d_fd<<endl;
       d_currentState = IOState::Done;
     }
 
@@ -65,7 +65,7 @@ public:
       }
 
       d_currentState = IOState::NeedRead;
-      //cerr<<__func__<<": add read FD "<<d_fd<<endl;
+      cerr<<__PRETTY_FUNCTION__<<": add read FD "<<d_fd<<endl;
       d_mplexer->addReadFD(d_fd, callback, callbackData, ttd ? &*ttd : nullptr);
     }
     else if (iostate == IOState::NeedWrite) {
@@ -74,11 +74,12 @@ public:
       }
 
       d_currentState = IOState::NeedWrite;
-      //cerr<<__func__<<": add write FD "<<d_fd<<endl;
+      cerr<<__PRETTY_FUNCTION__<<": add write FD "<<d_fd<<endl;
       d_mplexer->addWriteFD(d_fd, callback, callbackData, ttd ? &*ttd : nullptr);
     }
     else if (iostate == IOState::Done) {
       d_currentState = IOState::Done;
+      cerr<<__PRETTY_FUNCTION__<<": done"<<endl;
     }
   }
 
diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc
index a900105086..41692fec32 100644
--- a/pdns/dnsdist-tcp.cc
+++ b/pdns/dnsdist-tcp.cc
@@ -70,9 +70,10 @@ class DownstreamConnectionsManager
 {
 public:
 
-  static std::unique_ptr<TCPConnectionToBackend> getConnectionToDownstream(std::unique_ptr<FDMultiplexer>& mplexer, std::shared_ptr<DownstreamState>& ds, const struct timeval& now)
+  static std::shared_ptr<TCPConnectionToBackend> getConnectionToDownstream(std::unique_ptr<FDMultiplexer>& mplexer, std::shared_ptr<DownstreamState>& ds, const struct timeval& now)
   {
-    std::unique_ptr<TCPConnectionToBackend> result;
+    DEBUGLOG("in "<<__PRETTY_FUNCTION__);
+    std::shared_ptr<TCPConnectionToBackend> result;
 
     const auto& it = t_downstreamConnections.find(ds->remote);
     if (it != t_downstreamConnections.end()) {
@@ -81,20 +82,24 @@ class DownstreamConnectionsManager
         result = std::move(list.front());
         list.pop_front();
         result->setReused();
+        DEBUGLOG("Found");
         return result;
       }
     }
 
-    return make_unique<TCPConnectionToBackend>(ds, now);
+    DEBUGLOG("nothing found");
+    return std::make_shared<TCPConnectionToBackend>(ds, now);
   }
 
-  static void releaseDownstreamConnection(std::unique_ptr<TCPConnectionToBackend>&& conn)
+  static void releaseDownstreamConnection(std::shared_ptr<TCPConnectionToBackend>&& conn)
   {
+    DEBUGLOG("in "<<__PRETTY_FUNCTION__);
     if (conn == nullptr) {
       return;
     }
 
     if (!conn->canBeReused()) {
+      DEBUGLOG("cannot be reused");
       conn.reset();
       return;
     }
@@ -105,12 +110,15 @@ class DownstreamConnectionsManager
       auto& list = it->second;
       if (list.size() >= s_maxCachedConnectionsPerDownstream) {
         /* too many connections queued already */
+        DEBUGLOG("too many queued");
         conn.reset();
         return;
       }
+      DEBUGLOG("adding");
       list.push_back(std::move(conn));
     }
     else {
+      DEBUGLOG("adding");
       t_downstreamConnections[remote].push_back(std::move(conn));
     }
   }
@@ -137,11 +145,11 @@ class DownstreamConnectionsManager
   }
 
 private:
-  static thread_local map<ComboAddress, std::deque<std::unique_ptr<TCPConnectionToBackend>>> t_downstreamConnections;
+  static thread_local map<ComboAddress, std::deque<std::shared_ptr<TCPConnectionToBackend>>> t_downstreamConnections;
   static const size_t s_maxCachedConnectionsPerDownstream;
 };
 
-thread_local map<ComboAddress, std::deque<std::unique_ptr<TCPConnectionToBackend>>> DownstreamConnectionsManager::t_downstreamConnections;
+thread_local map<ComboAddress, std::deque<std::shared_ptr<TCPConnectionToBackend>>> DownstreamConnectionsManager::t_downstreamConnections;
 const size_t DownstreamConnectionsManager::s_maxCachedConnectionsPerDownstream{20};
 
 static void decrementTCPClientCount(const ComboAddress& client)
@@ -157,18 +165,18 @@ static void decrementTCPClientCount(const ComboAddress& client)
 
 IncomingTCPConnectionState::~IncomingTCPConnectionState()
 {
-  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
+  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   decrementTCPClientCount(d_ci.remote);
-  // DEBUG: cerr<<"decremented"<<endl;
+  DEBUGLOG("decremented");
 
   if (d_ci.cs != nullptr) {
     struct timeval now;
     gettimeofday(&now, nullptr);
 
     auto diff = now - d_connectionStartTime;
-    // DEBUG: cerr<<"updating tcp metrics"<<endl;
+    DEBUGLOG("updating tcp metrics");
     d_ci.cs->updateTCPMetrics(d_queriesCount, diff.tv_sec * 1000.0 + diff.tv_usec / 1000.0);
-    // DEBUG: cerr<<"updated tcp metrics"<<endl;
+    DEBUGLOG("updated tcp metrics");
   }
 
 #if 0
@@ -177,11 +185,11 @@ IncomingTCPConnectionState::~IncomingTCPConnectionState()
     if (d_downstreamConnection) {
       try {
         if (d_lastIOState == IOState::NeedRead) {
-          // DEBUG: cerr<<__PRETTY_FUNCTION__<<": removing leftover backend read FD "<<d_downstreamConnection->getHandle()<<endl;
+          DEBUGLOG(__PRETTY_FUNCTION__<<": removing leftover backend read FD "<<d_downstreamConnection->getHandle());
           d_threadData.mplexer->removeReadFD(d_downstreamConnection->getHandle());
         }
         else if (d_lastIOState == IOState::NeedWrite) {
-          // DEBUG: cerr<<__PRETTY_FUNCTION__<<": removing leftover backend write FD "<<d_downstreamConnection->getHandle()<<endl;
+          DEBUGLOG(__PRETTY_FUNCTION__<<": removing leftover backend write FD "<<d_downstreamConnection->getHandle());
           d_threadData.mplexer->removeWriteFD(d_downstreamConnection->getHandle());
         }
       }
@@ -196,14 +204,14 @@ IncomingTCPConnectionState::~IncomingTCPConnectionState()
   }
 #endif
 
-  // DEBUG: cerr<<"about to remove left over FDs"<<endl;
+  DEBUGLOG("about to remove left over FDs");
   try {
     if (d_lastIOState == IOState::NeedRead) {
-      // DEBUG: cerr<<__PRETTY_FUNCTION__<<": removing leftover client read FD "<<d_ci.fd<<endl;
+      DEBUGLOG(__PRETTY_FUNCTION__<<": removing leftover client read FD "<<d_ci.fd);
       d_threadData.mplexer->removeReadFD(d_ci.fd);
     }
     else if (d_lastIOState == IOState::NeedWrite) {
-      // DEBUG: cerr<<__PRETTY_FUNCTION__<<": removing leftover client write FD "<<d_ci.fd<<endl;
+      DEBUGLOG(__PRETTY_FUNCTION__<<": removing leftover client write FD "<<d_ci.fd);
       d_threadData.mplexer->removeWriteFD(d_ci.fd);
     }
   }
@@ -213,7 +221,7 @@ IncomingTCPConnectionState::~IncomingTCPConnectionState()
   catch (...) {
     vinfolog("Got an unknown exception when trying to remove a pending IO operation on an incoming TCP connection from %s", d_ci.remote.toStringWithPort());
   }
-  // DEBUG: cerr<<"done"<<endl;
+  DEBUGLOG("done");
 }
 
 std::shared_ptr<TCPConnectionToBackend> IncomingTCPConnectionState::getDownstreamConnection(std::shared_ptr<DownstreamState>& ds, const struct timeval& now)
@@ -221,12 +229,20 @@ std::shared_ptr<TCPConnectionToBackend> IncomingTCPConnectionState::getDownstrea
   std::shared_ptr<TCPConnectionToBackend> downstream{nullptr};
 
   if (!ds->useProxyProtocol || !d_proxyProtocolPayloadHasTLV) {
+    DEBUGLOG("looking for an active connection");
     downstream = getActiveDownstreamConnection(ds);
+    if (downstream) {
+      DEBUGLOG("FOUND an active connection");
+    }
+    else {
+      DEBUGLOG("no active connection");
+    }
   }
 
   if (!downstream) {
     /* we don't have a connection to this backend active yet, let's ask one (it might not be a fresh one, though) */
     downstream = DownstreamConnectionsManager::getConnectionToDownstream(d_threadData.mplexer, ds, now);
+    registerActiveDownstreamConnection(downstream);
   }
 
   return downstream;
@@ -338,15 +354,18 @@ std::unique_ptr<TCPClientCollection> g_tcpclientthreads;
 
 static IOState handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
 {
+  --state->d_currentQueriesCount;
+
   if (!state->d_isXFR) {
     const auto& currentResponse = state->d_currentResponse;
-    if (state->d_selfGeneratedResponse == false && currentResponse.d_ds) {
+    if (state->d_selfGeneratedResponse == false && currentResponse.d_connection && currentResponse.d_connection->getDS()) {
+      const auto& ds = currentResponse.d_connection->getDS();
       struct timespec answertime;
       gettime(&answertime);
       const auto& ids = currentResponse.d_idstate;
       double udiff = ids.sentTime.udiff();
-      g_rings.insertResponse(answertime, state->d_ci.remote, ids.qname, ids.qtype, static_cast<unsigned int>(udiff), static_cast<unsigned int>(currentResponse.d_buffer.size()), currentResponse.d_cleartextDH, currentResponse.d_ds->remote);
-      vinfolog("Got answer from %s, relayed to %s (%s), took %f usec", currentResponse.d_ds->remote.toStringWithPort(), ids.origRemote.toStringWithPort(), (state->d_ci.cs->tlsFrontend ? "DoT" : "TCP"), udiff);
+      g_rings.insertResponse(answertime, state->d_ci.remote, ids.qname, ids.qtype, static_cast<unsigned int>(udiff), static_cast<unsigned int>(currentResponse.d_buffer.size()), currentResponse.d_cleartextDH, ds->remote);
+      vinfolog("Got answer from %s, relayed to %s (%s), took %f usec", ds->remote.toStringWithPort(), ids.origRemote.toStringWithPort(), (state->d_ci.cs->tlsFrontend ? "DoT" : "TCP"), udiff);
     }
 
     switch (currentResponse.d_cleartextDH.rcode) {
@@ -374,21 +393,27 @@ static IOState handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& s
   }
 
   if (state->d_queuedResponses.empty()) {
-    // DEBUG: cerr<<"no response remaining"<<endl;
+    DEBUGLOG("no response remaining");
     if (state->d_isXFR) {
       /* we should still be reading from the backend, and we don't want to read from the client */
       state->d_state = IncomingTCPConnectionState::State::idle;
       state->d_currentPos = 0;
-      // DEBUG: cerr<<"idling for XFR completion"<<endl;
+      DEBUGLOG("idling for XFR completion");
       return IOState::Done;
     } else {
-      // DEBUG: cerr<<"reading new queries if any"<<endl;
-      state->resetForNewQuery();
-      return IOState::NeedRead;
+      DEBUGLOG("reading new queries if any");
+      if (state->canAcceptNewQueries()) {
+        state->resetForNewQuery();
+        return IOState::NeedRead;
+      }
+      else {
+        state->d_state = IncomingTCPConnectionState::State::idle;
+        return IOState::Done;
+      }
     }
   }
   else {
-    // DEBUG: cerr<<"queue size is "<<state->d_queuedResponses.size()<<endl;
+    DEBUGLOG("queue size is "<<state->d_queuedResponses.size());
     TCPResponse resp = std::move(state->d_queuedResponses.front());
     state->d_queuedResponses.pop_front();
     state->d_state = IncomingTCPConnectionState::State::idle;
@@ -397,6 +422,24 @@ static IOState handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& s
   }
 }
 
+
+bool IncomingTCPConnectionState::canAcceptNewQueries() const
+{
+  if (d_isXFR) {
+    DEBUGLOG("not accepting new queries because used for XFR");
+    return false;
+  }
+
+  // d_state ?
+  if (d_currentQueriesCount >= d_ci.cs->d_maxInFlightQueriesPerConn) {
+    DEBUGLOG("not accepting new queries because we already have "<<d_currentQueriesCount<<" out of "<<d_ci.cs->d_maxInFlightQueriesPerConn);
+    return false;
+  }
+
+  DEBUGLOG("accepting new queries");
+  return true;
+}
+
 void IncomingTCPConnectionState::resetForNewQuery()
 {
   d_buffer.resize(sizeof(uint16_t));
@@ -409,10 +452,34 @@ void IncomingTCPConnectionState::resetForNewQuery()
   d_selfGeneratedResponse = false;
 }
 
+std::shared_ptr<TCPConnectionToBackend> IncomingTCPConnectionState::getActiveDownstreamConnection(const std::shared_ptr<DownstreamState>& ds)
+{
+  auto it = d_activeConnectionsToBackend.find(ds->remote);
+  if (it == d_activeConnectionsToBackend.end()) {
+    DEBUGLOG("no active connection found for "<<ds->remote.toString());
+    return nullptr;
+  }
+
+  for (auto& conn : it->second) {
+    if (conn->canAcceptNewQueries()) {
+      DEBUGLOG("Got one active connection accepting more for "<<ds->remote.toString());
+      return conn;
+    }
+    DEBUGLOG("not accepting more for "<<ds->remote.toString());
+  }
+
+  return nullptr;
+}
+
+void IncomingTCPConnectionState::registerActiveDownstreamConnection(std::shared_ptr<TCPConnectionToBackend>& conn)
+{
+  d_activeConnectionsToBackend[conn->getRemote()].push_front(conn);
+}
+
 /* this version is called when the buffer has been set and the rules have been processed */
 void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
 {
-  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
+  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   // if we already reading a query (not the query size, mind you), or sending a response we need to either queue the response
   // otherwise we can start sending it right away
   if (state->d_state == IncomingTCPConnectionState::State::idle ||
@@ -431,21 +498,32 @@ void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnect
 
     //IncomingTCPConnectionState::handleIO(state, now);
      state->d_ioState->update(IOState::NeedWrite, handleIOCallback, state, getClientWriteTTD(now));
-    // DEBUG: cerr<<"updated IO state"<<endl;
+    DEBUGLOG("updated IO state");
   }
   else {
     // queue response
     state->d_queuedResponses.push_back(std::move(response));
-    // DEBUG: cerr<<"queueing response because state is "<<(int)state->d_state<<", queue size is now "<<state->d_queuedResponses.size()<<endl;
+    DEBUGLOG("queueing response because state is "<<(int)state->d_state<<", queue size is now "<<state->d_queuedResponses.size());
   }
 }
 
 /* this version is called from the backend code when a new response has been received */
 void IncomingTCPConnectionState::handleResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
 {
-  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
+  if (response.d_connection && response.d_connection->isIdle()) {
+    auto& list = d_activeConnectionsToBackend.at(response.d_connection->getRemote());
+    for (auto it = list.begin(); it != list.end(); ++it) {
+      if (*it == response.d_connection) {
+        DownstreamConnectionsManager::releaseDownstreamConnection(std::move(*it));
+        list.erase(it);
+        break;
+      }
+    }
+  }
+
+  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   if (response.d_buffer.size() < sizeof(dnsheader)) {
-    // DEBUG: cerr<<"too small"<<endl;
+    DEBUGLOG("too small");
     return;
   }
 
@@ -455,11 +533,10 @@ void IncomingTCPConnectionState::handleResponse(std::shared_ptr<IncomingTCPConne
   auto responseAsCharArray = reinterpret_cast<char*>(&response.d_buffer.at(0));
 
   auto& ids = response.d_idstate;
-  // DEBUG: cerr<<"IDS has "<<(ids.qTag?" TAGS ": "NO TAGS")<<endl;
   unsigned int consumed;
-  // DEBUG: cerr<<"about to match response for "<<ids.qname<<endl;
-  if (!responseContentMatches(responseAsCharArray, responseSize, ids.qname, ids.qtype, ids.qclass, response.d_ds->remote, consumed)) {
-    // DEBUG: cerr<<"content does not match"<<endl;
+  DEBUGLOG("about to match response for "<<ids.qname);
+  if (!responseContentMatches(responseAsCharArray, responseSize, ids.qname, ids.qtype, ids.qclass, response.d_connection->getRemote(), consumed)) {
+    DEBUGLOG("content does not match");
     return;
   }
 
@@ -474,7 +551,7 @@ void IncomingTCPConnectionState::handleResponse(std::shared_ptr<IncomingTCPConne
 
   std::vector<uint8_t> rewrittenResponse;
   if (!processResponse(&responseAsCharArray, &responseSize, &responseCapacity, state->d_threadData.localRespRulactions, dr, addRoom, rewrittenResponse, false)) {
-    // DEBUG: cerr<<"process said to drop it"<<endl;
+    DEBUGLOG("process said to drop it");
     return;
   }
 
@@ -492,13 +569,17 @@ void IncomingTCPConnectionState::handleResponse(std::shared_ptr<IncomingTCPConne
     state->d_xfrStarted = true;
     ++g_stats.responses;
     ++state->d_ci.cs->responses;
-    ++response.d_ds->responses;
+    if (response.d_connection->getDS()) {
+      ++response.d_connection->getDS()->responses;
+    }
   }
 
   if (!state->d_isXFR) {
     ++g_stats.responses;
     ++state->d_ci.cs->responses;
-    ++response.d_ds->responses;
+    if (response.d_connection->getDS()) {
+      ++response.d_connection->getDS()->responses;
+    }
   }
 
   sendResponse(state, now, std::move(response));
@@ -552,6 +633,7 @@ static bool handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
     TCPResponse response;
     response.d_buffer = std::move(*dnsCryptResponse);
     state->d_state = IncomingTCPConnectionState::State::idle;
+    ++state->d_currentQueriesCount;
     state->sendResponse(state, now, std::move(response));
     return false;
   }
@@ -586,6 +668,7 @@ static bool handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
     TCPResponse response;
     response.d_buffer = std::move(state->d_buffer);
     state->d_state = IncomingTCPConnectionState::State::idle;
+    ++state->d_currentQueriesCount;
     state->sendResponse(state, now, std::move(response));
     return false;
   }
@@ -594,17 +677,8 @@ static bool handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
     return true;
   }
 
-#warning move this, we should just never read another question again on this client connection
-  if (state->d_xfrStarted) {
-    /* sorry, but we are not going to resume a XFR if we have already sent some packets
-       to the client */
-    return true;
-  }
-
   IDState ids;
-  // DEBUG: cerr<<"DQ has "<<(dq.qTag?" TAGS ": "NO TAGS")<<endl;
   setIDStateFromDNSQuestion(ids, dq, std::move(qname));
-  // DEBUG: cerr<<"query IDS has "<<(ids.qTag?" TAGS ": "NO TAGS")<<endl;
   ids.origID = ntohs(dh->id);
 
   const uint8_t sizeBytes[] = { static_cast<uint8_t>(dq.len / 256), static_cast<uint8_t>(dq.len % 256) };
@@ -644,11 +718,12 @@ static bool handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
   else {
     downstreamConnection->setProxyProtocolPayload(std::move(proxyProtocolPayload));
   }
-  // DEBUG: cerr<<"about to be queued query IDS has "<<(ids.qTag?" TAGS ": "NO TAGS")<<endl;
+
+  ++state->d_currentQueriesCount;
   downstreamConnection->queueQuery(TCPQuery(std::move(state->d_buffer), std::move(ids)), downstreamConnection);
 
   //sendQueryToBackend(state, now);
-  // DEBUG: cerr<<"out of "<<__PRETTY_FUNCTION__<<endl;
+  DEBUGLOG("out of "<<__PRETTY_FUNCTION__);
   return true;
 }
 
@@ -666,7 +741,7 @@ void IncomingTCPConnectionState::handleIOCallback(int fd, FDMultiplexer::funcpar
 
 void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
 {
-  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
+  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   // why do we loop? Because the TLS layer does buffering, and thus can have data ready to read
   // even though the underlying socket is not ready, so we need to actually ask for the data first
   bool wouldBlock = false;
@@ -684,10 +759,10 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
 
     try {
       if (state->d_state == IncomingTCPConnectionState::State::doingHandshake) {
-        // DEBUG: cerr<<"doing handshake"<<endl;
+        DEBUGLOG("doing handshake");
         iostate = state->d_handler.tryHandshake();
         if (iostate == IOState::Done) {
-          // DEBUG: cerr<<"handshake done"<<endl;
+          DEBUGLOG("handshake done");
           if (state->d_handler.isTLS()) {
             if (!state->d_handler.hasTLSSessionBeenResumed()) {
               ++state->d_ci.cs->tlsNewSessions;
@@ -712,10 +787,10 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
       }
 
       if (state->d_state == IncomingTCPConnectionState::State::readingQuerySize) {
-        // DEBUG: cerr<<"reading query size"<<endl;
+        DEBUGLOG("reading query size");
         iostate = state->d_handler.tryRead(state->d_buffer, state->d_currentPos, sizeof(uint16_t));
         if (iostate == IOState::Done) {
-          // DEBUG: cerr<<"query size received"<<endl;
+          DEBUGLOG("query size received");
           state->d_state = IncomingTCPConnectionState::State::readingQuery;
           state->d_querySizeReadTime = now;
           if (state->d_queriesCount == 0) {
@@ -740,21 +815,27 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
       }
 
       if (state->d_state == IncomingTCPConnectionState::State::readingQuery) {
-        // DEBUG: cerr<<"reading query"<<endl;
+        DEBUGLOG("reading query");
         iostate = state->d_handler.tryRead(state->d_buffer, state->d_currentPos, state->d_querySize);
         if (iostate == IOState::Done) {
-          // DEBUG: cerr<<"query received"<<endl;
+          DEBUGLOG("query received");
           //handleNewIOState(state, IOState::Done, fd, handleIOCallback);
           if (handleQuery(state, now)) {
-            // DEBUG: cerr<<"handle query returned true"<<endl;
+            DEBUGLOG("handle query returned true");
             // if the query has been passed to a backend, or dropped, we can start
             // reading again, or sending queued responses
             if (state->d_queuedResponses.empty()) {
-              state->resetForNewQuery();
-              // DEBUG: cerr<<__LINE__<<endl;
-              iostate = IOState::NeedRead;
+              if (state->canAcceptNewQueries()) {
+                state->resetForNewQuery();
+                iostate = IOState::NeedRead;
+              }
+              else {
+                state->d_state = IncomingTCPConnectionState::State::idle;
+                iostate = IOState::Done;
+              }
+              DEBUGLOG(__LINE__);
               //state->d_ioState->update(IOState::NeedRead, handleIOCallback, state, state->getClientReadTTD(now));
-              // DEBUG: cerr<<__LINE__<<endl;
+              DEBUGLOG(__LINE__);
               //ioGuard.release();
             }
             else {
@@ -768,7 +849,7 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
           else {
             /* otherwise the state should already be waiting for
                the socket to be writable */
-            // DEBUG: cerr<<"should be waiting for writable socket"<<endl;
+            DEBUGLOG("should be waiting for writable socket");
             ioGuard.release();
             return;
           }
@@ -779,18 +860,18 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
       }
 
       if (state->d_state == IncomingTCPConnectionState::State::sendingResponse) {
-        // DEBUG: cerr<<"sending response"<<endl;
+        DEBUGLOG("sending response");
         iostate = state->d_handler.tryWrite(state->d_currentResponse.d_buffer, state->d_currentPos, state->d_currentResponse.d_buffer.size());
         if (iostate == IOState::Done) {
-          // DEBUG: cerr<<"response sent"<<endl;
+          DEBUGLOG("response sent");
           iostate = handleResponseSent(state, now);
         } else {
           wouldBlock = true;
-          // DEBUG: cerr<<"partial write"<<endl;
+          DEBUGLOG("partial write");
         }
-        // DEBUG: cerr<<__LINE__<<endl;
+        DEBUGLOG(__LINE__);
         //state->d_ioState->update(IOState::NeedRead, handleIOCallback, state, state->getClientReadTTD(now));
-        //// DEBUG: cerr<<__LINE__<<endl;
+        //DEBUGLOG(__LINE__);
       }
 
       if (state->d_state != IncomingTCPConnectionState::State::idle &&
@@ -817,26 +898,26 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
       }
 
       if (state->d_lastIOState == IOState::NeedWrite || state->d_readingFirstQuery) {
-        // DEBUG: cerr<<"Got an exception while handling TCP query: "<<e.what()<<endl;
+        DEBUGLOG("Got an exception while handling TCP query: "<<e.what());
         vinfolog("Got an exception while handling (%s) TCP query from %s: %s", (state->d_lastIOState == IOState::NeedRead ? "reading" : "writing"), state->d_ci.remote.toStringWithPort(), e.what());
       }
       else {
         vinfolog("Closing TCP client connection with %s", state->d_ci.remote.toStringWithPort());
-        // DEBUG: cerr<<"Closing TCP client connection: "<<e.what()<<endl;
+        DEBUGLOG("Closing TCP client connection: "<<e.what());
       }
       /* remove this FD from the IO multiplexer */
       iostate = IOState::Done;
     }
 
     if (iostate == IOState::Done) {
-      // DEBUG: cerr<<__LINE__<<endl;
+      DEBUGLOG(__LINE__);
       state->d_ioState->update(iostate, handleIOCallback, state);
-      // DEBUG: cerr<<__LINE__<<endl;
+      DEBUGLOG(__LINE__);
     }
     else {
-      // DEBUG: cerr<<__LINE__<<endl;
+      DEBUGLOG(__LINE__);
       state->d_ioState->update(iostate, handleIOCallback, state, iostate == IOState::NeedRead ? state->getClientReadTTD(now) : state->getClientWriteTTD(now));
-      // DEBUG: cerr<<__LINE__<<endl;
+      DEBUGLOG(__LINE__);
     }
     ioGuard.release();
   }
@@ -845,11 +926,7 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
 
 void IncomingTCPConnectionState::notifyIOError(std::shared_ptr<IncomingTCPConnectionState>& state, IDState&& query, const struct timeval& now)
 {
-  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
-  if (d_isXFR) {
-    d_xfrDone = true;
-  }
-
+  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   if (d_state == State::sendingResponse) {
     /* if we have responses to send, let's do that first */
   }
@@ -865,7 +942,7 @@ void IncomingTCPConnectionState::notifyIOError(std::shared_ptr<IncomingTCPConnec
     d_ioState->reset();
   }
 
-  // DEBUG: cerr<<"out "<<__PRETTY_FUNCTION__<<endl;
+  DEBUGLOG("out "<<__PRETTY_FUNCTION__);
 }
 
 void IncomingTCPConnectionState::handleXFRResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
@@ -875,7 +952,7 @@ void IncomingTCPConnectionState::handleXFRResponse(std::shared_ptr<IncomingTCPCo
 
 void IncomingTCPConnectionState::handleTimeout(bool write)
 {
-  // DEBUG: cerr<<"client timeout"<<endl;
+  DEBUGLOG("client timeout");
   ++d_ci.cs->tcpClientTimeouts;
   d_lastIOState = IOState::Done;
   d_ioState->reset();
diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh
index 0dee6cc8df..4640254aec 100644
--- a/pdns/dnsdist.hh
+++ b/pdns/dnsdist.hh
@@ -669,6 +669,7 @@ struct ClientState
   std::atomic<double> tcpAvgQueriesPerConnection{0.0};
   /* in ms */
   std::atomic<double> tcpAvgConnectionDuration{0.0};
+  size_t d_maxInFlightQueriesPerConn{1};
   int udpFD{-1};
   int tcpFD{-1};
   int tcpListenQueueSize{SOMAXCONN};
@@ -852,6 +853,7 @@ struct DownstreamState
   /* in ms */
   std::atomic<double> tcpAvgConnectionDuration{0.0};
   size_t socketsOffset{0};
+  size_t d_maxInFlightQueriesPerConn{1};
   double queryLoad{0.0};
   double dropRate{0.0};
   double latencyUsec{0.0};
diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
index 86dee17204..b1468ed1f1 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
@@ -6,7 +6,7 @@ const uint16_t TCPConnectionToBackend::s_xfrID = 0;
 
 void TCPConnectionToBackend::assignToClientConnection(std::shared_ptr<IncomingTCPConnectionState>& clientConn, bool isXFR)
 {
-  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
+  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   if (isXFR) {
     d_usedForXFR = true;
   }
@@ -20,12 +20,14 @@ IOState TCPConnectionToBackend::sendNextQuery(std::shared_ptr<TCPConnectionToBac
   conn->d_currentQuery = std::move(conn->d_pendingQueries.front());
   conn->d_pendingQueries.pop_front();
   conn->d_state = State::sendingQueryToBackend;
+  conn->d_currentPos = 0;
+
   return IOState::NeedWrite;
 }
 
 void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now)
 {
-  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
+  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   if (conn->d_socket == nullptr) {
     throw std::runtime_error("No downstream socket in " + std::string(__PRETTY_FUNCTION__) + "!");
   }
@@ -37,7 +39,7 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
 
   try {
     if (conn->d_state == State::sendingQueryToBackend) {
-      // DEBUG: cerr<<"sending query to backend over FD "<<fd<<endl;
+      DEBUGLOG("sending query to backend over FD "<<fd);
       int socketFlags = 0;
 #ifdef MSG_FASTOPEN
       if (conn->isFastOpenEnabled()) {
@@ -47,13 +49,12 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
 
       size_t sent = sendMsgWithOptions(fd, reinterpret_cast<const char *>(&conn->d_currentQuery.d_buffer.at(conn->d_currentPos)), conn->d_currentQuery.d_buffer.size() - conn->d_currentPos, &conn->d_ds->remote, &conn->d_ds->sourceAddr, conn->d_ds->sourceItf, socketFlags);
       if (sent == conn->d_currentQuery.d_buffer.size()) {
-        // DEBUG: cerr<<"query sent to backend"<<endl;
+        DEBUGLOG("query sent to backend");
         /* request sent ! */
         conn->incQueries();
         conn->d_currentPos = 0;
         //conn->d_currentQuery.d_querySentTime = now;
-        // DEBUG: cerr<<"adding a pending response for ID "<<conn->d_currentQuery.d_idstate.origID<<" and QNAME "<<conn->d_currentQuery.d_idstate.qname<<endl;
-        // DEBUG: cerr<<"IDS has "<<(conn->d_currentQuery.d_idstate.qTag?"tags":"no tags")<<endl;
+        DEBUGLOG("adding a pending response for ID "<<conn->d_currentQuery.d_idstate.origID<<" and QNAME "<<conn->d_currentQuery.d_idstate.qname);
         conn->d_pendingResponses[conn->d_currentQuery.d_idstate.origID] = std::move(conn->d_currentQuery);
         conn->d_currentQuery.d_buffer.clear();
 #if 0
@@ -83,7 +84,7 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
     }
 
     if (conn->d_state == State::readingResponseSizeFromBackend) {
-      // DEBUG: cerr<<"reading response size from backend"<<endl;
+      DEBUGLOG("reading response size from backend");
       // then we need to allocate a new buffer (new because we might need to re-send the query if the
       // backend dies on us)
       // We also might need to read and send to the client more than one response in case of XFR (yeah!)
@@ -91,7 +92,7 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
       conn->d_responseBuffer.resize(sizeof(uint16_t));
       iostate = tryRead(fd, conn->d_responseBuffer, conn->d_currentPos, sizeof(uint16_t) - conn->d_currentPos);
       if (iostate == IOState::Done) {
-        // DEBUG: cerr<<"got response size from backend"<<endl;
+        DEBUGLOG("got response size from backend");
         conn->d_state = State::readingResponseFromBackend;
         conn->d_responseSize = conn->d_responseBuffer.at(0) * 256 + conn->d_responseBuffer.at(1);
         conn->d_responseBuffer.reserve(conn->d_responseSize + /* we will need to prepend the size later */ 2);
@@ -101,13 +102,13 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
     }
 
     if (conn->d_state == State::readingResponseFromBackend) {
-      // DEBUG: cerr<<"reading response from backend"<<endl;
+      DEBUGLOG("reading response from backend");
       iostate = tryRead(fd, conn->d_responseBuffer, conn->d_currentPos, conn->d_responseSize - conn->d_currentPos);
       if (iostate == IOState::Done) {
-        // DEBUG: cerr<<"got response from backend"<<endl;
+        DEBUGLOG("got response from backend");
         //conn->d_responseReadTime = now;
         try {
-          iostate = conn->handleResponse(now);
+          iostate = conn->handleResponse(conn, now);
         }
         catch (const std::exception& e) {
           vinfolog("Got an exception while handling TCP response from %s (client is %s): %s", conn->d_ds ? conn->d_ds->getName() : "unknown", conn->d_currentQuery.d_idstate.origRemote.toStringWithPort(), e.what());
@@ -157,18 +158,18 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
 
   if (connectionDied) {
     bool reconnected = false;
-    // DEBUG: cerr<<"connection died, number of failures is "<<conn->d_downstreamFailures<<", retries is "<<conn->d_ds->retries<<endl;
+    DEBUGLOG("connection died, number of failures is "<<conn->d_downstreamFailures<<", retries is "<<conn->d_ds->retries);
 
     if ((!conn->d_usedForXFR || conn->d_queries == 0) && conn->d_downstreamFailures < conn->d_ds->retries) {
-      // DEBUG: cerr<<"reconnecting"<<endl;
+      DEBUGLOG("reconnecting");
       conn->d_ioState->reset();
       ioGuard.release();
 
       if (conn->reconnect()) {
-        // DEBUG: cerr<<"reconnected"<<endl;
+        DEBUGLOG("reconnected");
 
         conn->d_ioState = make_unique<IOStateHandler>(conn->d_clientConn->getIOMPlexer(), conn->d_socket->getHandle());
-        // DEBUG: cerr<<"new state"<<endl;
+        DEBUGLOG("new state");
 
         for (auto& pending : conn->d_pendingResponses) {
           conn->d_pendingQueries.push_back(std::move(pending.second));
@@ -182,9 +183,9 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
           // resume sending query
         }
         else {
-          // DEBUG: cerr<<"sending next query"<<endl;
+          DEBUGLOG("sending next query");
           iostate = sendNextQuery(conn);
-          // DEBUG: cerr<<"after call to sendNextQuery"<<endl;
+          DEBUGLOG("after call to sendNextQuery");
         }
 
         if (!conn->d_proxyProtocolPayloadAdded && !conn->d_proxyProtocolPayload.empty()) {
@@ -198,17 +199,17 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
 
     if (!reconnected) {
       /* reconnect failed, we give up */
-      conn->d_connectionDied = true;
+      DEBUGLOG("reconnect failed, we give up");
       conn->notifyAllQueriesFailed(now);
     }
   }
 
   if (iostate == IOState::Done) {
-    // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<", done"<<endl;
+    DEBUGLOG("in "<<__PRETTY_FUNCTION__<<", done");
     conn->d_ioState->update(iostate, handleIOCallback, conn);
   }
   else {
-    // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<", updating to "<<(int)iostate<<endl;
+    DEBUGLOG("in "<<__PRETTY_FUNCTION__<<", updating to "<<(int)iostate);
     conn->d_ioState->update(iostate, handleIOCallback, conn, iostate == IOState::NeedRead ? conn->getBackendReadTTD(now) : conn->getBackendWriteTTD(now));
   }
   ioGuard.release();
@@ -229,8 +230,7 @@ void TCPConnectionToBackend::handleIOCallback(int fd, FDMultiplexer::funcparam_t
 
 void TCPConnectionToBackend::queueQuery(TCPQuery&& query, std::shared_ptr<TCPConnectionToBackend>& sharedSelf)
 {
-  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
-  // DEBUG: cerr<<"IDS has "<<(query.d_idstate.qTag?"tags":"no tags")<<endl;
+  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   if (d_ioState == nullptr) {
     throw std::runtime_error("Trying to queue a query to a TCP connection that has no incoming client connection assigned");
   }
@@ -238,9 +238,16 @@ void TCPConnectionToBackend::queueQuery(TCPQuery&& query, std::shared_ptr<TCPCon
   // if we are not already sending a query or in the middle of reading a response (so idle or doingHandshake),
   // start sending the query
   if (d_state == State::idle || d_state == State::waitingForResponseFromBackend) {
+
     d_state = State::sendingQueryToBackend;
+    d_currentPos = 0;
     d_currentQuery = std::move(query);
-    // DEBUG: cerr<<"need write"<<endl;
+    if (!d_proxyProtocolPayloadAdded && !d_proxyProtocolPayload.empty()) {
+      d_currentQuery.d_buffer.insert(d_currentQuery.d_buffer.begin(), d_proxyProtocolPayload.begin(), d_proxyProtocolPayload.end());
+      d_proxyProtocolPayloadAdded = true;
+    }
+
+    DEBUGLOG("need write");
 
     struct timeval now;
     gettimeofday(&now, 0);
@@ -251,7 +258,7 @@ void TCPConnectionToBackend::queueQuery(TCPQuery&& query, std::shared_ptr<TCPCon
     // store query in the list of queries to send
     d_pendingQueries.push_back(std::move(query));
   }
-  // DEBUG: cerr<<"out of "<<__PRETTY_FUNCTION__<<endl;
+  DEBUGLOG("out of "<<__PRETTY_FUNCTION__);
 }
 
 bool TCPConnectionToBackend::reconnect()
@@ -259,7 +266,7 @@ bool TCPConnectionToBackend::reconnect()
   std::unique_ptr<Socket> result;
 
   if (d_socket) {
-    // DEBUG: cerr<<"closing socket "<<d_socket->getHandle()<<endl;
+    DEBUGLOG("closing socket "<<d_socket->getHandle());
     shutdown(d_socket->getHandle(), SHUT_RDWR);
     d_socket.reset();
     d_ioState.reset();
@@ -270,7 +277,7 @@ bool TCPConnectionToBackend::reconnect()
     vinfolog("TCP connecting to downstream %s (%d)", d_ds->getNameWithAddr(), d_downstreamFailures);
     try {
       result = std::unique_ptr<Socket>(new Socket(d_ds->remote.sin4.sin_family, SOCK_STREAM, 0));
-      // DEBUG: cerr<<"result of connect is "<<result->getHandle()<<endl;
+      DEBUGLOG("result of connect is "<<result->getHandle());
       if (!IsAnyAddress(d_ds->sourceAddr)) {
         SSetsockopt(result->getHandle(), SOL_SOCKET, SO_REUSEADDR, 1);
 #ifdef IP_BIND_ADDRESS_NO_PORT
@@ -298,7 +305,7 @@ bool TCPConnectionToBackend::reconnect()
 #endif /* MSG_FASTOPEN */
 
       d_socket = std::move(result);
-      // DEBUG: cerr<<"connected new socket "<<d_socket->getHandle()<<endl;
+      DEBUGLOG("connected new socket "<<d_socket->getHandle());
       ++d_ds->tcpCurrentConnections;
       return true;
     }
@@ -333,6 +340,7 @@ void TCPConnectionToBackend::handleTimeout(const struct timeval& now, bool write
 
 void TCPConnectionToBackend::notifyAllQueriesFailed(const struct timeval& now, bool timeout)
 {
+  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   d_connectionDied = true;
   //auto clientConn = d_clientConn.lock();
   //if (!clientConn) {
@@ -342,7 +350,6 @@ void TCPConnectionToBackend::notifyAllQueriesFailed(const struct timeval& now, b
   auto& clientConn = d_clientConn;
   if (!clientConn->active()) {
     // a client timeout occured, or something like that */
-    d_connectionDied = true;
     d_clientConn.reset();
     return;
   }
@@ -369,20 +376,22 @@ void TCPConnectionToBackend::notifyAllQueriesFailed(const struct timeval& now, b
   d_clientConn.reset();
 }
 
-IOState TCPConnectionToBackend::handleResponse(const struct timeval& now)
+IOState TCPConnectionToBackend::handleResponse(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now)
 {
-  // DEBUG: cerr<<"in "<<__PRETTY_FUNCTION__<<endl;
+  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   //auto clientConn = d_clientConn.lock();
   //if (!clientConn) {
   //d_clientConn.reset();
   //  d_connectionDied = true;
-  //  // DEBUG: cerr<<"connection to client died, bye bye"<<endl;
+  //  DEBUGLOG("connection to client died, bye bye");
   //  return IOState::Done;
   //}
 
+  d_downstreamFailures = 0;
+
   auto& clientConn = d_clientConn;
   if (!clientConn->active()) {
-    // DEBUG: cerr<<"client is not active"<<endl;
+    DEBUGLOG("client is not active");
     // a client timeout occured, or something like that */
     d_connectionDied = true;
     d_clientConn.reset();
@@ -390,10 +399,10 @@ IOState TCPConnectionToBackend::handleResponse(const struct timeval& now)
   }
 
   if (d_usedForXFR) {
-    // DEBUG: cerr<<"XFR!"<<endl;
+    DEBUGLOG("XFR!");
     TCPResponse response;
     response.d_buffer = std::move(d_responseBuffer);
-    response.d_ds = d_ds;
+    response.d_connection = conn;
     clientConn->handleXFRResponse(clientConn, now, std::move(response));
     d_state = State::readingResponseSizeFromBackend;
     d_currentPos = 0;
@@ -402,44 +411,45 @@ IOState TCPConnectionToBackend::handleResponse(const struct timeval& now)
     // get ready to read the next packet, if any
   }
   else {
-    // DEBUG: cerr<<"not XFR, phew"<<endl;
+    DEBUGLOG("not XFR, phew");
     uint16_t queryId = 0;
     try {
       queryId = getQueryIdFromResponse();
     }
     catch (const std::exception& e) {
+      DEBUGLOG("Unable to get query ID");
       notifyAllQueriesFailed(now);
       throw;
     }
 
     auto it = d_pendingResponses.find(queryId);
     if (it == d_pendingResponses.end()) {
-      // DEBUG: cerr<<"could not found any corresponding query for ID "<<queryId<<endl;
+      DEBUGLOG("could not found any corresponding query for ID "<<queryId);
       notifyAllQueriesFailed(now);
       return IOState::Done;
     }
     auto ids = std::move(it->second.d_idstate);
-    // DEBUG: cerr<<"IDS has "<<(ids.qTag?" TAGS ": "NO TAGS")<<endl;
-    // DEBUG: cerr<<"passing response to client connection for "<<ids.qname<<endl;
-    clientConn->handleResponse(clientConn, now, TCPResponse(std::move(d_responseBuffer), std::move(ids), d_ds));
+    DEBUGLOG("passing response to client connection for "<<ids.qname);
+    clientConn->handleResponse(clientConn, now, TCPResponse(std::move(d_responseBuffer), std::move(ids), conn));
     d_pendingResponses.erase(it);
 
     if (!d_pendingQueries.empty()) {
-      // DEBUG: cerr<<"still have some queries to send"<<endl;
+      DEBUGLOG("still have some queries to send");
       d_state = State::sendingQueryToBackend;
       d_currentQuery = std::move(d_pendingQueries.front());
+      d_currentPos = 0;
       d_pendingQueries.pop_front();
       return IOState::NeedWrite;
     }
     else if (!d_pendingResponses.empty()) {
-      // DEBUG: cerr<<"still have some responses to read"<<endl;
+      DEBUGLOG("still have some responses to read");
       d_state = State::readingResponseSizeFromBackend;
       d_currentPos = 0;
       d_responseBuffer.resize(sizeof(uint16_t));
       return IOState::NeedRead;
     }
     else {
-      // DEBUG: cerr<<"nothing to do, phewwwww"<<endl;
+      DEBUGLOG("nothing to do, phewwwww");
       d_state = State::idle;
       d_clientConn.reset();
       return IOState::Done;
diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
index 9d24de5f17..bd0955547f 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
@@ -20,17 +20,19 @@ struct TCPQuery
   std::vector<uint8_t> d_buffer;
 };
 
+class TCPConnectionToBackend;
+
 struct TCPResponse : public TCPQuery
 {
   TCPResponse()
   {
   }
 
-  TCPResponse(std::vector<uint8_t>&& buffer, IDState&& state, std::shared_ptr<DownstreamState> ds): TCPQuery(std::move(buffer), std::move(state)), d_ds(ds)
+  TCPResponse(std::vector<uint8_t>&& buffer, IDState&& state, std::shared_ptr<TCPConnectionToBackend> conn): TCPQuery(std::move(buffer), std::move(state)), d_connection(conn)
   {
   }
 
-  std::shared_ptr<DownstreamState> d_ds{nullptr};
+  std::shared_ptr<TCPConnectionToBackend> d_connection{nullptr};
   dnsheader d_cleartextDH;
   bool d_selfGenerated{false};
 };
@@ -68,6 +70,11 @@ public:
     return d_socket->getHandle();
   }
 
+  const std::shared_ptr<DownstreamState>& getDS() const
+  {
+    return d_ds;
+  }
+
   const ComboAddress& getRemote() const
   {
     return d_ds->remote;
@@ -106,14 +113,25 @@ public:
   bool canAcceptNewQueries() const
   {
     if (d_usedForXFR || d_connectionDied) {
+      DEBUGLOG("Used for XFR: "<<d_usedForXFR<<", connection died: "<<d_connectionDied);
       return false;
       /* Don't reuse the TCP connection after an {A,I}XFR */
       /* but don't reset it either, we will need to read more messages */
     }
-#warning FIXME: maximum number of pending queries
+
+    if ((d_pendingQueries.size() + d_pendingResponses.size()) >= d_ds->d_maxInFlightQueriesPerConn) {
+      DEBUGLOG("Pending queries: "<<d_pendingQueries.size()<<", pending responses: "<<d_pendingResponses.size());
+      return false;
+    }
+
     return true;
   }
 
+  bool isIdle() const
+  {
+    return d_pendingQueries.size() == 0 && d_pendingResponses.size() == 0;
+  }
+
   bool canBeReused() const
   {
     if (d_usedForXFR || d_connectionDied) {
@@ -138,17 +156,17 @@ public:
     return ds == d_ds;
   }
 
-  static void handleIO(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now);
-  static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param);
-  static IOState sendNextQuery(std::shared_ptr<TCPConnectionToBackend>& conn);
-
   void queueQuery(TCPQuery&& query, std::shared_ptr<TCPConnectionToBackend>& sharedSelf);
   void handleTimeout(const struct timeval& now, bool write);
-  IOState handleResponse(const struct timeval& now);
   void setProxyProtocolPayload(std::string&& payload);
   void setProxyProtocolPayloadAdded(bool added);
 
 private:
+  static void handleIO(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now);
+  static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param);
+  static IOState sendNextQuery(std::shared_ptr<TCPConnectionToBackend>& conn);
+
+  IOState handleResponse(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now);
   uint16_t getQueryIdFromResponse();
   bool reconnect();
   void notifyAllQueriesFailed(const struct timeval& now, bool timeout = false);
@@ -206,7 +224,7 @@ private:
   State d_state{State::idle};
   bool d_fresh{true};
   bool d_enableFastOpen{false};
-  bool d_connectionDied{true};
+  bool d_connectionDied{false};
   bool d_usedForXFR{false};
   bool d_proxyProtocolPayloadAdded{false};
 };
diff --git a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
index 9479f0a403..5092bfe406 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
@@ -166,13 +166,9 @@ public:
     }
   }
 
-  std::shared_ptr<TCPConnectionToBackend> getActiveDownstreamConnection(const std::shared_ptr<DownstreamState>& ds)
-  {
-#warning TODO: we need to find a connection to this DS, usable (no TLV values sent) and supporting OOR
-    return nullptr;
-  }
-
+  std::shared_ptr<TCPConnectionToBackend> getActiveDownstreamConnection(const std::shared_ptr<DownstreamState>& ds);
   std::shared_ptr<TCPConnectionToBackend> getDownstreamConnection(std::shared_ptr<DownstreamState>& ds, const struct timeval& now);
+  void registerActiveDownstreamConnection(std::shared_ptr<TCPConnectionToBackend>& conn);
 
   std::unique_ptr<FDMultiplexer>& getIOMPlexer() const
   {
@@ -189,6 +185,8 @@ public:
   void handleXFRResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
   void handleTimeout(bool write);
 
+  bool canAcceptNewQueries() const;
+
   bool active() const
   {
     return d_ioState != nullptr;
@@ -196,6 +194,7 @@ public:
 
   enum class State { doingHandshake, readingQuerySize, readingQuery, sendingResponse, idle /* in case of XFR, we stop processing queries */ };
 
+  std::map<ComboAddress, std::deque<std::shared_ptr<TCPConnectionToBackend>>> d_activeConnectionsToBackend;
   std::vector<uint8_t> d_buffer;
   std::deque<TCPResponse> d_queuedResponses;
   TCPClientThreadData& d_threadData;
@@ -211,6 +210,7 @@ public:
   struct timeval d_queryReadTime;
   size_t d_currentPos{0};
   size_t d_queriesCount{0};
+  size_t d_currentQueriesCount{0};
   unsigned int d_remainingTime{0};
   uint16_t d_querySize{0};
   uint16_t d_downstreamFailures{0};
@@ -219,7 +219,6 @@ public:
   bool d_readingFirstQuery{true};
   bool d_isXFR{false};
   bool d_xfrStarted{false};
-  bool d_xfrDone{false};
   bool d_selfGeneratedResponse{false};
   bool d_proxyProtocolPayloadAdded{false};
   bool d_proxyProtocolPayloadHasTLV{false};
diff --git a/pdns/dnsdistdist/tcpiohandler-mplexer.hh b/pdns/dnsdistdist/tcpiohandler-mplexer.hh
index fdb1c5ed97..e0e366dd78 100644
--- a/pdns/dnsdistdist/tcpiohandler-mplexer.hh
+++ b/pdns/dnsdistdist/tcpiohandler-mplexer.hh
@@ -4,6 +4,12 @@
 #include "mplexer.hh"
 #include "tcpiohandler.hh"
 
+#if 0
+#define DEBUGLOG(x) cerr<<x<<endl;
+#else
+#define DEBUGLOG(x)
+#endif
+
 class IOStateHandler
 {
 public:
@@ -43,14 +49,14 @@ public:
 
   void update(IOState iostate, FDMultiplexer::callbackfunc_t callback = FDMultiplexer::callbackfunc_t(), FDMultiplexer::funcparam_t callbackData = boost::any(), boost::optional<struct timeval> ttd = boost::none)
   {
-    cerr<<"in "<<__PRETTY_FUNCTION__<<" for fd "<<d_fd<<", last state was "<<(int)d_currentState<<", new state is "<<(int)iostate<<endl;
+    DEBUGLOG("in "<<__PRETTY_FUNCTION__<<" for fd "<<d_fd<<", last state was "<<(int)d_currentState<<", new state is "<<(int)iostate);
     if (d_currentState == IOState::NeedRead && iostate != IOState::NeedRead) {
-      cerr<<__PRETTY_FUNCTION__<<": remove read FD "<<d_fd<<endl;
+      DEBUGLOG(__PRETTY_FUNCTION__<<": remove read FD "<<d_fd);
       d_mplexer->removeReadFD(d_fd);
       d_currentState = IOState::Done;
     }
     else if (d_currentState == IOState::NeedWrite && iostate != IOState::NeedWrite) {
-      cerr<<__PRETTY_FUNCTION__<<": remove write FD "<<d_fd<<endl;
+      DEBUGLOG(__PRETTY_FUNCTION__<<": remove write FD "<<d_fd);
       d_mplexer->removeWriteFD(d_fd);
       d_currentState = IOState::Done;
     }
@@ -65,7 +71,7 @@ public:
       }
 
       d_currentState = IOState::NeedRead;
-      cerr<<__PRETTY_FUNCTION__<<": add read FD "<<d_fd<<endl;
+      DEBUGLOG(__PRETTY_FUNCTION__<<": add read FD "<<d_fd);
       d_mplexer->addReadFD(d_fd, callback, callbackData, ttd ? &*ttd : nullptr);
     }
     else if (iostate == IOState::NeedWrite) {
@@ -74,12 +80,12 @@ public:
       }
 
       d_currentState = IOState::NeedWrite;
-      cerr<<__PRETTY_FUNCTION__<<": add write FD "<<d_fd<<endl;
+      DEBUGLOG(__PRETTY_FUNCTION__<<": add write FD "<<d_fd);
       d_mplexer->addWriteFD(d_fd, callback, callbackData, ttd ? &*ttd : nullptr);
     }
     else if (iostate == IOState::Done) {
       d_currentState = IOState::Done;
-      cerr<<__PRETTY_FUNCTION__<<": done"<<endl;
+      DEBUGLOG(__PRETTY_FUNCTION__<<": done");
     }
   }
 
@@ -104,7 +110,7 @@ public:
        let's reset the state so it's not registered to the IO multiplexer anymore
        and its reference count goes to zero */
     if (d_enabled && d_handler) {
-      cerr<<"IOStateGuard destroyed while holding a state, let's reset it"<<endl;
+      DEBUGLOG("IOStateGuard destroyed while holding a state, let's reset it");
       d_handler->reset();
       d_enabled = false;
     }

diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc
index 41692fec32..fbcd858d42 100644
--- a/pdns/dnsdist-tcp.cc
+++ b/pdns/dnsdist-tcp.cc
@@ -72,7 +72,6 @@ class DownstreamConnectionsManager
 
   static std::shared_ptr<TCPConnectionToBackend> getConnectionToDownstream(std::unique_ptr<FDMultiplexer>& mplexer, std::shared_ptr<DownstreamState>& ds, const struct timeval& now)
   {
-    DEBUGLOG("in "<<__PRETTY_FUNCTION__);
     std::shared_ptr<TCPConnectionToBackend> result;
 
     const auto& it = t_downstreamConnections.find(ds->remote);
@@ -82,24 +81,20 @@ class DownstreamConnectionsManager
         result = std::move(list.front());
         list.pop_front();
         result->setReused();
-        DEBUGLOG("Found");
         return result;
       }
     }
 
-    DEBUGLOG("nothing found");
     return std::make_shared<TCPConnectionToBackend>(ds, now);
   }
 
   static void releaseDownstreamConnection(std::shared_ptr<TCPConnectionToBackend>&& conn)
   {
-    DEBUGLOG("in "<<__PRETTY_FUNCTION__);
     if (conn == nullptr) {
       return;
     }
 
     if (!conn->canBeReused()) {
-      DEBUGLOG("cannot be reused");
       conn.reset();
       return;
     }
@@ -110,15 +105,13 @@ class DownstreamConnectionsManager
       auto& list = it->second;
       if (list.size() >= s_maxCachedConnectionsPerDownstream) {
         /* too many connections queued already */
-        DEBUGLOG("too many queued");
         conn.reset();
         return;
       }
-      DEBUGLOG("adding");
+
       list.push_back(std::move(conn));
     }
     else {
-      DEBUGLOG("adding");
       t_downstreamConnections[remote].push_back(std::move(conn));
     }
   }
@@ -167,51 +160,20 @@ IncomingTCPConnectionState::~IncomingTCPConnectionState()
 {
   DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   decrementTCPClientCount(d_ci.remote);
-  DEBUGLOG("decremented");
 
   if (d_ci.cs != nullptr) {
     struct timeval now;
     gettimeofday(&now, nullptr);
 
     auto diff = now - d_connectionStartTime;
-    DEBUGLOG("updating tcp metrics");
     d_ci.cs->updateTCPMetrics(d_queriesCount, diff.tv_sec * 1000.0 + diff.tv_usec / 1000.0);
-    DEBUGLOG("updated tcp metrics");
   }
 
-#if 0
-  if (d_ds != nullptr) {
-
-    if (d_downstreamConnection) {
-      try {
-        if (d_lastIOState == IOState::NeedRead) {
-          DEBUGLOG(__PRETTY_FUNCTION__<<": removing leftover backend read FD "<<d_downstreamConnection->getHandle());
-          d_threadData.mplexer->removeReadFD(d_downstreamConnection->getHandle());
-        }
-        else if (d_lastIOState == IOState::NeedWrite) {
-          DEBUGLOG(__PRETTY_FUNCTION__<<": removing leftover backend write FD "<<d_downstreamConnection->getHandle());
-          d_threadData.mplexer->removeWriteFD(d_downstreamConnection->getHandle());
-        }
-      }
-      catch(const FDMultiplexerException& e) {
-        vinfolog("Got an exception when trying to remove a pending IO operation on the socket to the %s backend: %s", d_ds->getName(), e.what());
-      }
-      catch(const std::runtime_error& e) {
-        /* might be thrown by getHandle() */
-        vinfolog("Got an exception when trying to remove a pending IO operation on the socket to the %s backend: %s", d_ds->getName(), e.what());
-      }
-    }
-  }
-#endif
-
-  DEBUGLOG("about to remove left over FDs");
   try {
     if (d_lastIOState == IOState::NeedRead) {
-      DEBUGLOG(__PRETTY_FUNCTION__<<": removing leftover client read FD "<<d_ci.fd);
       d_threadData.mplexer->removeReadFD(d_ci.fd);
     }
     else if (d_lastIOState == IOState::NeedWrite) {
-      DEBUGLOG(__PRETTY_FUNCTION__<<": removing leftover client write FD "<<d_ci.fd);
       d_threadData.mplexer->removeWriteFD(d_ci.fd);
     }
   }
@@ -221,7 +183,6 @@ IncomingTCPConnectionState::~IncomingTCPConnectionState()
   catch (...) {
     vinfolog("Got an unknown exception when trying to remove a pending IO operation on an incoming TCP connection from %s", d_ci.remote.toStringWithPort());
   }
-  DEBUGLOG("done");
 }
 
 std::shared_ptr<TCPConnectionToBackend> IncomingTCPConnectionState::getDownstreamConnection(std::shared_ptr<DownstreamState>& ds, const struct timeval& now)
@@ -229,14 +190,7 @@ std::shared_ptr<TCPConnectionToBackend> IncomingTCPConnectionState::getDownstrea
   std::shared_ptr<TCPConnectionToBackend> downstream{nullptr};
 
   if (!ds->useProxyProtocol || !d_proxyProtocolPayloadHasTLV) {
-    DEBUGLOG("looking for an active connection");
     downstream = getActiveDownstreamConnection(ds);
-    if (downstream) {
-      DEBUGLOG("FOUND an active connection");
-    }
-    else {
-      DEBUGLOG("no active connection");
-    }
   }
 
   if (!downstream) {
@@ -314,42 +268,6 @@ void TCPClientCollection::addTCPClientThread()
   }
 }
 
-/* Tries to read exactly toRead bytes into the buffer, starting at position pos.
-   Updates pos everytime a successful read occurs,
-   throws an std::runtime_error in case of IO error,
-   return Done when toRead bytes have been read, needRead or needWrite if the IO operation
-   would block.
-*/
-// XXX could probably be implemented as a TCPIOHandler
-IOState tryRead(int fd, std::vector<uint8_t>& buffer, size_t& pos, size_t toRead)
-{
-  if (buffer.size() < (pos + toRead)) {
-    throw std::out_of_range("Calling tryRead() with a too small buffer (" + std::to_string(buffer.size()) + ") for a read of " + std::to_string(toRead) + " bytes starting at " + std::to_string(pos));
-  }
-
-  size_t got = 0;
-  do {
-    ssize_t res = ::read(fd, reinterpret_cast<char*>(&buffer.at(pos)), toRead - got);
-    if (res == 0) {
-      throw runtime_error("EOF while reading message");
-    }
-    if (res < 0) {
-      if (errno == EAGAIN || errno == EWOULDBLOCK || errno == ENOTCONN) {
-        return IOState::NeedRead;
-      }
-      else {
-        throw std::runtime_error(std::string("Error while reading message: ") + stringerror());
-      }
-    }
-
-    pos += static_cast<size_t>(res);
-    got += static_cast<size_t>(res);
-  }
-  while (got < toRead);
-
-  return IOState::Done;
-}
-
 std::unique_ptr<TCPClientCollection> g_tcpclientthreads;
 
 static IOState handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
@@ -445,7 +363,6 @@ void IncomingTCPConnectionState::resetForNewQuery()
   d_buffer.resize(sizeof(uint16_t));
   d_currentPos = 0;
   d_querySize = 0;
-  //d_responseSize = 0;
   d_downstreamFailures = 0;
   d_state = State::readingQuerySize;
   d_lastIOState = IOState::Done;
@@ -496,9 +413,7 @@ void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnect
     state->d_currentPos = 0;
     state->d_currentResponse = std::move(response);
 
-    //IncomingTCPConnectionState::handleIO(state, now);
-     state->d_ioState->update(IOState::NeedWrite, handleIOCallback, state, getClientWriteTTD(now));
-    DEBUGLOG("updated IO state");
+    state->d_ioState->update(IOState::NeedWrite, handleIOCallback, state, getClientWriteTTD(now));
   }
   else {
     // queue response
@@ -510,6 +425,7 @@ void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnect
 /* this version is called from the backend code when a new response has been received */
 void IncomingTCPConnectionState::handleResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
 {
+  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   if (response.d_connection && response.d_connection->isIdle()) {
     auto& list = d_activeConnectionsToBackend.at(response.d_connection->getRemote());
     for (auto it = list.begin(); it != list.end(); ++it) {
@@ -521,9 +437,7 @@ void IncomingTCPConnectionState::handleResponse(std::shared_ptr<IncomingTCPConne
     }
   }
 
-  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   if (response.d_buffer.size() < sizeof(dnsheader)) {
-    DEBUGLOG("too small");
     return;
   }
 
@@ -534,9 +448,7 @@ void IncomingTCPConnectionState::handleResponse(std::shared_ptr<IncomingTCPConne
 
   auto& ids = response.d_idstate;
   unsigned int consumed;
-  DEBUGLOG("about to match response for "<<ids.qname);
   if (!responseContentMatches(responseAsCharArray, responseSize, ids.qname, ids.qtype, ids.qclass, response.d_connection->getRemote(), consumed)) {
-    DEBUGLOG("content does not match");
     return;
   }
 
@@ -551,7 +463,6 @@ void IncomingTCPConnectionState::handleResponse(std::shared_ptr<IncomingTCPConne
 
   std::vector<uint8_t> rewrittenResponse;
   if (!processResponse(&responseAsCharArray, &responseSize, &responseCapacity, state->d_threadData.localRespRulactions, dr, addRoom, rewrittenResponse, false)) {
-    DEBUGLOG("process said to drop it");
     return;
   }
 
@@ -628,8 +539,6 @@ static bool handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
   std::shared_ptr<DNSCryptQuery> dnsCryptQuery{nullptr};
   auto dnsCryptResponse = checkDNSCryptQuery(*state->d_ci.cs, query, state->d_querySize, dnsCryptQuery, queryRealTime.tv_sec, true);
   if (dnsCryptResponse) {
-    //state->d_responseBuffer = std::move(*dnsCryptResponse);
-    //state->d_responseSize = state->d_responseBuffer.size();
     TCPResponse response;
     response.d_buffer = std::move(*dnsCryptResponse);
     state->d_state = IncomingTCPConnectionState::State::idle;
@@ -722,8 +631,6 @@ static bool handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
   ++state->d_currentQueriesCount;
   downstreamConnection->queueQuery(TCPQuery(std::move(state->d_buffer), std::move(ids)), downstreamConnection);
 
-  //sendQueryToBackend(state, now);
-  DEBUGLOG("out of "<<__PRETTY_FUNCTION__);
   return true;
 }
 
@@ -799,8 +706,6 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
           state->d_querySize = state->d_buffer.at(0) * 256 + state->d_buffer.at(1);
           if (state->d_querySize < sizeof(dnsheader)) {
             /* go away */
-            // will be handled by the guard
-            //handleNewIOState(state, IOState::Done, fd, handleIOCallback);
             return;
           }
 
@@ -819,7 +724,7 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
         iostate = state->d_handler.tryRead(state->d_buffer, state->d_currentPos, state->d_querySize);
         if (iostate == IOState::Done) {
           DEBUGLOG("query received");
-          //handleNewIOState(state, IOState::Done, fd, handleIOCallback);
+
           if (handleQuery(state, now)) {
             DEBUGLOG("handle query returned true");
             // if the query has been passed to a backend, or dropped, we can start
@@ -833,10 +738,6 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
                 state->d_state = IncomingTCPConnectionState::State::idle;
                 iostate = IOState::Done;
               }
-              DEBUGLOG(__LINE__);
-              //state->d_ioState->update(IOState::NeedRead, handleIOCallback, state, state->getClientReadTTD(now));
-              DEBUGLOG(__LINE__);
-              //ioGuard.release();
             }
             else {
               TCPResponse resp = std::move(state->d_queuedResponses.front());
@@ -869,9 +770,6 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
           wouldBlock = true;
           DEBUGLOG("partial write");
         }
-        DEBUGLOG(__LINE__);
-        //state->d_ioState->update(IOState::NeedRead, handleIOCallback, state, state->getClientReadTTD(now));
-        //DEBUGLOG(__LINE__);
       }
 
       if (state->d_state != IncomingTCPConnectionState::State::idle &&
@@ -910,14 +808,10 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
     }
 
     if (iostate == IOState::Done) {
-      DEBUGLOG(__LINE__);
       state->d_ioState->update(iostate, handleIOCallback, state);
-      DEBUGLOG(__LINE__);
     }
     else {
-      DEBUGLOG(__LINE__);
       state->d_ioState->update(iostate, handleIOCallback, state, iostate == IOState::NeedRead ? state->getClientReadTTD(now) : state->getClientWriteTTD(now));
-      DEBUGLOG(__LINE__);
     }
     ioGuard.release();
   }
@@ -941,8 +835,6 @@ void IncomingTCPConnectionState::notifyIOError(std::shared_ptr<IncomingTCPConnec
     d_lastIOState = IOState::Done;
     d_ioState->reset();
   }
-
-  DEBUGLOG("out "<<__PRETTY_FUNCTION__);
 }
 
 void IncomingTCPConnectionState::handleXFRResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
@@ -1054,14 +946,6 @@ static void tcpClientThread(int pipefd)
           vinfolog("Timeout (write) from remote backend %s", conn->getBackendName());
           conn->handleTimeout(now, true);
         }
-#if 0
-        try {
-          data.mplexer->removeWriteFD(cbData.first);
-        }
-        catch (const FDMultiplexerException& fde) {
-          warnlog("Exception while removing a socket (%d) after a write timeout: %s", cbData.first, fde.what());
-        }
-#endif
       }
     }
   }
diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
index b1468ed1f1..03a4c41b24 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
@@ -25,6 +25,42 @@ IOState TCPConnectionToBackend::sendNextQuery(std::shared_ptr<TCPConnectionToBac
   return IOState::NeedWrite;
 }
 
+/* Tries to read exactly toRead bytes into the buffer, starting at position pos.
+   Updates pos everytime a successful read occurs,
+   throws an std::runtime_error in case of IO error,
+   return Done when toRead bytes have been read, needRead or needWrite if the IO operation
+   would block.
+*/
+// XXX could probably be implemented as a TCPIOHandler
+static IOState tryRead(int fd, std::vector<uint8_t>& buffer, size_t& pos, size_t toRead)
+{
+  if (buffer.size() < (pos + toRead)) {
+    throw std::out_of_range("Calling tryRead() with a too small buffer (" + std::to_string(buffer.size()) + ") for a read of " + std::to_string(toRead) + " bytes starting at " + std::to_string(pos));
+  }
+
+  size_t got = 0;
+  do {
+    ssize_t res = ::read(fd, reinterpret_cast<char*>(&buffer.at(pos)), toRead - got);
+    if (res == 0) {
+      throw runtime_error("EOF while reading message");
+    }
+    if (res < 0) {
+      if (errno == EAGAIN || errno == EWOULDBLOCK || errno == ENOTCONN) {
+        return IOState::NeedRead;
+      }
+      else {
+        throw std::runtime_error(std::string("Error while reading message: ") + stringerror());
+      }
+    }
+
+    pos += static_cast<size_t>(res);
+    got += static_cast<size_t>(res);
+  }
+  while (got < toRead);
+
+  return IOState::Done;
+}
+
 void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now)
 {
   DEBUGLOG("in "<<__PRETTY_FUNCTION__);
@@ -57,13 +93,6 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
         DEBUGLOG("adding a pending response for ID "<<conn->d_currentQuery.d_idstate.origID<<" and QNAME "<<conn->d_currentQuery.d_idstate.qname);
         conn->d_pendingResponses[conn->d_currentQuery.d_idstate.origID] = std::move(conn->d_currentQuery);
         conn->d_currentQuery.d_buffer.clear();
-#if 0
-        if (!conn->d_usedForXFR) {
-          /* don't bother with the outstanding count for XFR queries */
-          ++conn->d_ds->outstanding;
-          ++conn->d_outstanding;
-        }
-#endif
 
         if (conn->d_pendingQueries.empty()) {
           conn->d_state = State::readingResponseSizeFromBackend;
@@ -113,7 +142,6 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
         catch (const std::exception& e) {
           vinfolog("Got an exception while handling TCP response from %s (client is %s): %s", conn->d_ds ? conn->d_ds->getName() : "unknown", conn->d_currentQuery.d_idstate.origRemote.toStringWithPort(), e.what());
         }
-        //return;
       }
     }
 
@@ -169,7 +197,6 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
         DEBUGLOG("reconnected");
 
         conn->d_ioState = make_unique<IOStateHandler>(conn->d_clientConn->getIOMPlexer(), conn->d_socket->getHandle());
-        DEBUGLOG("new state");
 
         for (auto& pending : conn->d_pendingResponses) {
           conn->d_pendingQueries.push_back(std::move(pending.second));
@@ -183,9 +210,7 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
           // resume sending query
         }
         else {
-          DEBUGLOG("sending next query");
           iostate = sendNextQuery(conn);
-          DEBUGLOG("after call to sendNextQuery");
         }
 
         if (!conn->d_proxyProtocolPayloadAdded && !conn->d_proxyProtocolPayload.empty()) {
@@ -205,15 +230,12 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
   }
 
   if (iostate == IOState::Done) {
-    DEBUGLOG("in "<<__PRETTY_FUNCTION__<<", done");
     conn->d_ioState->update(iostate, handleIOCallback, conn);
   }
   else {
-    DEBUGLOG("in "<<__PRETTY_FUNCTION__<<", updating to "<<(int)iostate);
     conn->d_ioState->update(iostate, handleIOCallback, conn, iostate == IOState::NeedRead ? conn->getBackendReadTTD(now) : conn->getBackendWriteTTD(now));
   }
   ioGuard.release();
-
 }
 
 void TCPConnectionToBackend::handleIOCallback(int fd, FDMultiplexer::funcparam_t& param)
@@ -342,11 +364,7 @@ void TCPConnectionToBackend::notifyAllQueriesFailed(const struct timeval& now, b
 {
   DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   d_connectionDied = true;
-  //auto clientConn = d_clientConn.lock();
-  //if (!clientConn) {
-  //  d_clientConn.reset();
-  //  return;
-  //}
+
   auto& clientConn = d_clientConn;
   if (!clientConn->active()) {
     // a client timeout occured, or something like that */
@@ -379,13 +397,6 @@ void TCPConnectionToBackend::notifyAllQueriesFailed(const struct timeval& now, b
 IOState TCPConnectionToBackend::handleResponse(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now)
 {
   DEBUGLOG("in "<<__PRETTY_FUNCTION__);
-  //auto clientConn = d_clientConn.lock();
-  //if (!clientConn) {
-  //d_clientConn.reset();
-  //  d_connectionDied = true;
-  //  DEBUGLOG("connection to client died, bye bye");
-  //  return IOState::Done;
-  //}
 
   d_downstreamFailures = 0;
 
diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
index bd0955547f..c815364831 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
@@ -110,17 +110,16 @@ public:
     return d_enableFastOpen;
   }
 
+  /* whether we can acept new queries FOR THE SAME CLIENT */
   bool canAcceptNewQueries() const
   {
     if (d_usedForXFR || d_connectionDied) {
-      DEBUGLOG("Used for XFR: "<<d_usedForXFR<<", connection died: "<<d_connectionDied);
       return false;
       /* Don't reuse the TCP connection after an {A,I}XFR */
       /* but don't reset it either, we will need to read more messages */
     }
 
     if ((d_pendingQueries.size() + d_pendingResponses.size()) >= d_ds->d_maxInFlightQueriesPerConn) {
-      DEBUGLOG("Pending queries: "<<d_pendingQueries.size()<<", pending responses: "<<d_pendingResponses.size());
       return false;
     }
 
@@ -132,6 +131,7 @@ public:
     return d_pendingQueries.size() == 0 && d_pendingResponses.size() == 0;
   }
 
+  /* whether a connection can be reused for a different client */
   bool canBeReused() const
   {
     if (d_usedForXFR || d_connectionDied) {
@@ -212,7 +212,6 @@ private:
   std::unique_ptr<Socket> d_socket{nullptr};
   std::unique_ptr<IOStateHandler> d_ioState{nullptr};
   std::shared_ptr<DownstreamState> d_ds{nullptr};
-  //std::weak_ptr<IncomingTCPConnectionState> d_clientConn;
   std::shared_ptr<IncomingTCPConnectionState> d_clientConn;
   std::string d_proxyProtocolPayload;
   TCPQuery d_currentQuery;
diff --git a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
index 5092bfe406..721559256e 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
@@ -57,7 +57,6 @@ struct ConnectionInfo
 class IncomingTCPConnectionState
 {
 public:
-  //IncomingTCPConnectionState(ConnectionInfo&& ci, TCPClientThreadData& threadData, const struct timeval& now): d_buffer(s_maxPacketCacheEntrySize), d_responseBuffer(s_maxPacketCacheEntrySize), d_threadData(threadData), d_ci(std::move(ci)), d_handler(d_ci.fd, g_tcpRecvTimeout, d_ci.cs->tlsFrontend ? d_ci.cs->tlsFrontend->getContext() : nullptr, now.tv_sec), d_ioState(threadData.mplexer, d_ci.fd), _connectionStartTime(now)
   IncomingTCPConnectionState(ConnectionInfo&& ci, TCPClientThreadData& threadData, const struct timeval& now): d_buffer(s_maxPacketCacheEntrySize), d_threadData(threadData), d_ci(std::move(ci)), d_handler(d_ci.fd, g_tcpRecvTimeout, d_ci.cs->tlsFrontend ? d_ci.cs->tlsFrontend->getContext() : nullptr, now.tv_sec), d_ioState(make_unique<IOStateHandler>(threadData.mplexer, d_ci.fd)), d_connectionStartTime(now)
   {
     d_origDest.reset();
@@ -138,6 +137,7 @@ public:
     return false;
   }
 
+#if 0
   void dump() const
   {
     static std::mutex s_mutex;
@@ -165,6 +165,7 @@ public:
       }
     }
   }
+#endif
 
   std::shared_ptr<TCPConnectionToBackend> getActiveDownstreamConnection(const std::shared_ptr<DownstreamState>& ds);
   std::shared_ptr<TCPConnectionToBackend> getDownstreamConnection(std::shared_ptr<DownstreamState>& ds, const struct timeval& now);
@@ -224,4 +225,3 @@ public:
   bool d_proxyProtocolPayloadHasTLV{false};
 };
 
-IOState tryRead(int fd, std::vector<uint8_t>& buffer, size_t& pos, size_t toRead);

From 0abe1e37642abeafbced6e999252f68fdad12562 Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Wed, 9 Sep 2020 17:20:30 +0200
Subject: [PATCH 05/23] dnsdist: Add 'maxInFlight' parameter to 'newServer()'
 and '*Local()'

---
 pdns/dnsdist-lua.cc                        | 34 ++++++++++++++++++----
 pdns/dnsdistdist/docs/reference/config.rst | 13 ++++++++-
 2 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/pdns/dnsdist-lua.cc b/pdns/dnsdist-lua.cc
index b16d466c87..ac1e48e762 100644
--- a/pdns/dnsdist-lua.cc
+++ b/pdns/dnsdist-lua.cc
@@ -98,7 +98,7 @@ void resetLuaSideEffect()
 
 typedef std::unordered_map<std::string, boost::variant<bool, int, std::string, std::vector<std::pair<int,int> >, std::vector<std::pair<int, std::string> >, std::map<std::string,std::string>  > > localbind_t;
 
-static void parseLocalBindVars(boost::optional<localbind_t> vars, bool& reusePort, int& tcpFastOpenQueueSize, std::string& interface, std::set<int>& cpus, int& tcpListenQueueSize)
+static void parseLocalBindVars(boost::optional<localbind_t> vars, bool& reusePort, int& tcpFastOpenQueueSize, std::string& interface, std::set<int>& cpus, int& tcpListenQueueSize, size_t& maxInFlightQueriesPerConnection)
 {
   if (vars) {
     if (vars->count("reusePort")) {
@@ -110,6 +110,9 @@ static void parseLocalBindVars(boost::optional<localbind_t> vars, bool& reusePor
     if (vars->count("tcpListenQueueSize")) {
       tcpListenQueueSize = boost::get<int>((*vars)["tcpListenQueueSize"]);
     }
+    if (vars->count("maxInFlight")) {
+      maxInFlightQueriesPerConnection = boost::get<int>((*vars)["maxInFlight"]);
+    }
     if (vars->count("interface")) {
       interface = boost::get<std::string>((*vars)["interface"]);
     }
@@ -401,6 +404,10 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
         }
       }
 
+      if(vars.count("maxInFlight")) {
+        ret->d_maxInFlightQueriesPerConn = std::stoi(boost::get<string>(vars["maxInFlight"]));
+      }
+
       if(vars.count("name")) {
         ret->setName(boost::get<string>(vars["name"]));
       }
@@ -585,10 +592,11 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
       bool reusePort = false;
       int tcpFastOpenQueueSize = 0;
       int tcpListenQueueSize = 0;
+      size_t maxInFlightQueriesPerConn = 0;
       std::string interface;
       std::set<int> cpus;
 
-      parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize);
+      parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize, maxInFlightQueriesPerConn);
 
       try {
 	ComboAddress loc(addr, 53);
@@ -608,6 +616,9 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
         if (tcpListenQueueSize > 0) {
           tcpCS->tcpListenQueueSize = tcpListenQueueSize;
         }
+        if (maxInFlightQueriesPerConn > 0) {
+          tcpCS->d_maxInFlightQueriesPerConn = maxInFlightQueriesPerConn;
+        }
         g_frontends.push_back(std::move(tcpCS));
       }
       catch(const std::exception& e) {
@@ -626,10 +637,11 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
       bool reusePort = false;
       int tcpFastOpenQueueSize = 0;
       int tcpListenQueueSize = 0;
+      size_t maxInFlightQueriesPerConn = 0;
       std::string interface;
       std::set<int> cpus;
 
-      parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize);
+      parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize, maxInFlightQueriesPerConn);
 
       try {
 	ComboAddress loc(addr, 53);
@@ -639,6 +651,9 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
         if (tcpListenQueueSize > 0) {
           tcpCS->tcpListenQueueSize = tcpListenQueueSize;
         }
+        if (maxInFlightQueriesPerConn > 0) {
+          tcpCS->d_maxInFlightQueriesPerConn = maxInFlightQueriesPerConn;
+        }
         g_frontends.push_back(std::move(tcpCS));
       }
       catch(std::exception& e) {
@@ -1248,11 +1263,12 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
       bool reusePort = false;
       int tcpFastOpenQueueSize = 0;
       int tcpListenQueueSize = 0;
+      size_t maxInFlightQueriesPerConn = 0;
       std::string interface;
       std::set<int> cpus;
       std::vector<DNSCryptContext::CertKeyPaths> certKeys;
 
-      parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize);
+      parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize, maxInFlightQueriesPerConn);
 
       if (certFiles.type() == typeid(std::string) && keyFiles.type() == typeid(std::string)) {
         auto certFile = boost::get<std::string>(certFiles);
@@ -1950,11 +1966,12 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
     bool reusePort = false;
     int tcpFastOpenQueueSize = 0;
     int tcpListenQueueSize = 0;
+    size_t maxInFlightQueriesPerConn = 0;
     std::string interface;
     std::set<int> cpus;
 
     if (vars) {
-      parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize);
+      parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize, maxInFlightQueriesPerConn);
 
       if (vars->count("idleTimeout")) {
         frontend->d_idleTimeout = boost::get<int>((*vars)["idleTimeout"]);
@@ -2136,11 +2153,12 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
         bool reusePort = false;
         int tcpFastOpenQueueSize = 0;
         int tcpListenQueueSize = 0;
+        size_t maxInFlightQueriesPerConn = 0;
         std::string interface;
         std::set<int> cpus;
 
         if (vars) {
-          parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize);
+          parseLocalBindVars(vars, reusePort, tcpFastOpenQueueSize, interface, cpus, tcpListenQueueSize, maxInFlightQueriesPerConn);
 
           if (vars->count("provider")) {
             frontend->d_provider = boost::get<const string>((*vars)["provider"]);
@@ -2168,6 +2186,10 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
           if (tcpListenQueueSize > 0) {
             cs->tcpListenQueueSize = tcpListenQueueSize;
           }
+          if (maxInFlightQueriesPerConn > 0) {
+            cs->d_maxInFlightQueriesPerConn = maxInFlightQueriesPerConn;
+          }
+
           g_tlslocals.push_back(cs->tlsFrontend);
           g_frontends.push_back(std::move(cs));
         }
diff --git a/pdns/dnsdistdist/docs/reference/config.rst b/pdns/dnsdistdist/docs/reference/config.rst
index 359c50877e..36e0c7b5a1 100644
--- a/pdns/dnsdistdist/docs/reference/config.rst
+++ b/pdns/dnsdistdist/docs/reference/config.rst
@@ -73,6 +73,9 @@ Listen Sockets
   .. versionchanged:: 1.5.0
     Added ``tcpListenQueueSize`` parameter.
 
+  .. versionchanged:: 1.6.0
+    Added ``maxInFlight`` parameter.
+
   Add to the list of listen addresses.
 
   :param str address: The IP Address with an optional port to listen on.
@@ -87,6 +90,7 @@ Listen Sockets
   * ``interface=""``: str - Set the network interface to use.
   * ``cpus={}``: table - Set the CPU affinity for this listener thread, asking the scheduler to run it on a single CPU id, or a set of CPU ids. This parameter is only available if the OS provides the pthread_setaffinity_np() function.
   * ``tcpListenQueueSize=SOMAXCONN``: int - Set the size of the listen queue. Default is ``SOMAXCONN``.
+  * ``maxInFlight=0``: int - Maximum number of in-flight queries. The default is 0, which disables out-of-order processing.
 
   .. code-block:: lua
 
@@ -164,6 +168,8 @@ Listen Sockets
     ``ciphersTLS13``, ``minTLSVersion``, ``ocspResponses``, ``preferServerCiphers``, ``keyLogFile`` options added.
   .. versionchanged:: 1.5.0
     ``sessionTimeout`` and ``tcpListenQueueSize`` options added.
+  .. versionchanged:: 1.6.0
+    Added ``maxInFlight`` parameter.
 
   Listen on the specified address and TCP port for incoming DNS over TLS connections, presenting the specified X.509 certificate.
 
@@ -193,6 +199,7 @@ Listen Sockets
   * ``preferServerCiphers``: bool - Whether to prefer the order of ciphers set by the server instead of the one set by the client. Default is true, meaning that the order of the server is used. For OpenSSL >= 1.1.1, setting this option also enables the temporary re-prioritization of the ChaCha20-Poly1305 cipher if the client prioritizes it.
   * ``keyLogFile``: str - Write the TLS keys in the specified file so that an external program can decrypt TLS exchanges, in the format described in https://developer.mozilla.org/en-US/docs/Mozilla/Projects/NSS/Key_Log_Format. Note that this feature requires OpenSSL >= 1.1.1.
   * ``tcpListenQueueSize=SOMAXCONN``: int - Set the size of the listen queue. Default is ``SOMAXCONN``.
+  * ``maxInFlight=0``: int - Maximum number of in-flight queries. The default is 0, which disables out-of-order processing.
 
 .. function:: setLocal(address[, options])
 
@@ -438,6 +445,9 @@ Servers
   .. versionchanged:: 1.5.0
     Added ``useProxyProtocol`` to server_table.
 
+  .. versionchanged:: 1.6.0
+    Added ``maxInFlight`` to server_table.
+
   Add a new backend server. Call this function with either a string::
 
     newServer(
@@ -482,7 +492,8 @@ Servers
       disableZeroScope=BOOL, -- Disable the EDNS Client Subnet 'zero scope' feature, which does a cache lookup for an answer valid for all subnets (ECS scope of 0) before adding ECS information to the query and doing the regular lookup. This requires the ``parseECS`` option of the corresponding cache to be set to true
       rise=NUM,              -- Require NUM consecutive successful checks before declaring the backend up, default: 1
       useProxyProtocol=BOOL, -- Add a proxy protocol header to the query, passing along the client's IP address and port along with the original destination address and port. Default is disabled.
-      reconnectOnUp=BOOL     -- Close and reopen the sockets when a server transits from Down to Up. This helps when an interface is missing when dnsdist is started. Default is disabled.
+      reconnectOnUp=BOOL,    -- Close and reopen the sockets when a server transits from Down to Up. This helps when an interface is missing when dnsdist is started. Default is disabled.
+      maxInFlight            -- Maximum number of in-flight queries. The default is 0, which disables out-of-order processing. It should only be enabled if the backend does support out-of-order processing.
     })
 
   :param str server_string: A simple IP:PORT string.

From 4138339490efd1edab8e6104acd2a938bad24888 Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Wed, 9 Sep 2020 17:21:28 +0200
Subject: [PATCH 06/23] dnsdist: Fix the accounting of 'active' connections

---
 pdns/dnsdistdist/dnsdist-tcp-downstream.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
index 03a4c41b24..0a5a11b1df 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
@@ -440,9 +440,9 @@ IOState TCPConnectionToBackend::handleResponse(std::shared_ptr<TCPConnectionToBa
       return IOState::Done;
     }
     auto ids = std::move(it->second.d_idstate);
+    d_pendingResponses.erase(it);
     DEBUGLOG("passing response to client connection for "<<ids.qname);
     clientConn->handleResponse(clientConn, now, TCPResponse(std::move(d_responseBuffer), std::move(ids), conn));
-    d_pendingResponses.erase(it);
 
     if (!d_pendingQueries.empty()) {
       DEBUGLOG("still have some queries to send");

From 244d610b4bddcc15ba8f9e62bd55fe6ddda376ea Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Thu, 10 Sep 2020 12:20:57 +0200
Subject: [PATCH 07/23] dnsdist: Add some tests for out of order processing

---
 regression-tests.dnsdist/dnsdisttests.py      |  16 +-
 regression-tests.dnsdist/test_OOOR.py         | 154 ++++++++++++++++++
 .../test_ProxyProtocol.py                     |   2 +-
 3 files changed, 167 insertions(+), 5 deletions(-)
 create mode 100644 regression-tests.dnsdist/test_OOOR.py

diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc
index fbcd858d42..02e23fa42d 100644
--- a/pdns/dnsdist-tcp.cc
+++ b/pdns/dnsdist-tcp.cc
@@ -33,7 +33,6 @@
 #include "dnsdist-xpf.hh"
 #include "dnsparser.hh"
 #include "dolog.hh"
-#include "ednsoptions.hh"
 #include "gettime.hh"
 #include "lock.hh"
 #include "sstuff.hh"
@@ -74,7 +73,7 @@ class DownstreamConnectionsManager
   {
     std::shared_ptr<TCPConnectionToBackend> result;
 
-    const auto& it = t_downstreamConnections.find(ds->remote);
+    const auto& it = t_downstreamConnections.find(ds);
     if (it != t_downstreamConnections.end()) {
       auto& list = it->second;
       if (!list.empty()) {
@@ -99,8 +98,8 @@ class DownstreamConnectionsManager
       return;
     }
 
-    const auto& remote = conn->getRemote();
-    const auto& it = t_downstreamConnections.find(remote);
+    const auto& ds = conn->getDS();
+    const auto& it = t_downstreamConnections.find(ds);
     if (it != t_downstreamConnections.end()) {
       auto& list = it->second;
       if (list.size() >= s_maxCachedConnectionsPerDownstream) {
@@ -112,7 +111,7 @@ class DownstreamConnectionsManager
       list.push_back(std::move(conn));
     }
     else {
-      t_downstreamConnections[remote].push_back(std::move(conn));
+      t_downstreamConnections[ds].push_back(std::move(conn));
     }
   }
 
@@ -138,11 +137,11 @@ class DownstreamConnectionsManager
   }
 
 private:
-  static thread_local map<ComboAddress, std::deque<std::shared_ptr<TCPConnectionToBackend>>> t_downstreamConnections;
+  static thread_local map<std::shared_ptr<DownstreamState>, std::deque<std::shared_ptr<TCPConnectionToBackend>>> t_downstreamConnections;
   static const size_t s_maxCachedConnectionsPerDownstream;
 };
 
-thread_local map<ComboAddress, std::deque<std::shared_ptr<TCPConnectionToBackend>>> DownstreamConnectionsManager::t_downstreamConnections;
+thread_local map<std::shared_ptr<DownstreamState>, std::deque<std::shared_ptr<TCPConnectionToBackend>>> DownstreamConnectionsManager::t_downstreamConnections;
 const size_t DownstreamConnectionsManager::s_maxCachedConnectionsPerDownstream{20};
 
 static void decrementTCPClientCount(const ComboAddress& client)
@@ -158,7 +157,6 @@ static void decrementTCPClientCount(const ComboAddress& client)
 
 IncomingTCPConnectionState::~IncomingTCPConnectionState()
 {
-  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   decrementTCPClientCount(d_ci.remote);
 
   if (d_ci.cs != nullptr) {
@@ -311,7 +309,6 @@ static IOState handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& s
   }
 
   if (state->d_queuedResponses.empty()) {
-    DEBUGLOG("no response remaining");
     if (state->d_isXFR) {
       /* we should still be reading from the backend, and we don't want to read from the client */
       state->d_state = IncomingTCPConnectionState::State::idle;
@@ -319,12 +316,13 @@ static IOState handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& s
       DEBUGLOG("idling for XFR completion");
       return IOState::Done;
     } else {
-      DEBUGLOG("reading new queries if any");
       if (state->canAcceptNewQueries()) {
+        DEBUGLOG("waiting for new queries");
         state->resetForNewQuery();
         return IOState::NeedRead;
       }
       else {
+        DEBUGLOG("idling");
         state->d_state = IncomingTCPConnectionState::State::idle;
         return IOState::Done;
       }
@@ -348,13 +346,11 @@ bool IncomingTCPConnectionState::canAcceptNewQueries() const
     return false;
   }
 
-  // d_state ?
   if (d_currentQueriesCount >= d_ci.cs->d_maxInFlightQueriesPerConn) {
     DEBUGLOG("not accepting new queries because we already have "<<d_currentQueriesCount<<" out of "<<d_ci.cs->d_maxInFlightQueriesPerConn);
     return false;
   }
 
-  DEBUGLOG("accepting new queries");
   return true;
 }
 
@@ -371,18 +367,18 @@ void IncomingTCPConnectionState::resetForNewQuery()
 
 std::shared_ptr<TCPConnectionToBackend> IncomingTCPConnectionState::getActiveDownstreamConnection(const std::shared_ptr<DownstreamState>& ds)
 {
-  auto it = d_activeConnectionsToBackend.find(ds->remote);
+  auto it = d_activeConnectionsToBackend.find(ds);
   if (it == d_activeConnectionsToBackend.end()) {
-    DEBUGLOG("no active connection found for "<<ds->remote.toString());
+    DEBUGLOG("no active connection found for "<<ds->getName());
     return nullptr;
   }
 
   for (auto& conn : it->second) {
     if (conn->canAcceptNewQueries()) {
-      DEBUGLOG("Got one active connection accepting more for "<<ds->remote.toString());
+      DEBUGLOG("Got one active connection accepting more for "<<ds->getName());
       return conn;
     }
-    DEBUGLOG("not accepting more for "<<ds->remote.toString());
+    DEBUGLOG("not accepting more for "<<ds->getName());
   }
 
   return nullptr;
@@ -390,13 +386,12 @@ std::shared_ptr<TCPConnectionToBackend> IncomingTCPConnectionState::getActiveDow
 
 void IncomingTCPConnectionState::registerActiveDownstreamConnection(std::shared_ptr<TCPConnectionToBackend>& conn)
 {
-  d_activeConnectionsToBackend[conn->getRemote()].push_front(conn);
+  d_activeConnectionsToBackend[conn->getDS()].push_front(conn);
 }
 
 /* this version is called when the buffer has been set and the rules have been processed */
 void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
 {
-  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   // if we already reading a query (not the query size, mind you), or sending a response we need to either queue the response
   // otherwise we can start sending it right away
   if (state->d_state == IncomingTCPConnectionState::State::idle ||
@@ -425,9 +420,9 @@ void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnect
 /* this version is called from the backend code when a new response has been received */
 void IncomingTCPConnectionState::handleResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
 {
-  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
-  if (response.d_connection && response.d_connection->isIdle()) {
-    auto& list = d_activeConnectionsToBackend.at(response.d_connection->getRemote());
+  // if we have added a TCP Proxy Protocol payload to a connection, don't release it yet, no one else will be able to use it anyway
+  if (!state->d_isXFR && response.d_connection && response.d_connection->isIdle() && response.d_connection->canBeReused()) {
+    auto& list = d_activeConnectionsToBackend.at(response.d_connection->getDS());
     for (auto it = list.begin(); it != list.end(); ++it) {
       if (*it == response.d_connection) {
         DownstreamConnectionsManager::releaseDownstreamConnection(std::move(*it));
@@ -648,7 +643,6 @@ void IncomingTCPConnectionState::handleIOCallback(int fd, FDMultiplexer::funcpar
 
 void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
 {
-  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   // why do we loop? Because the TLS layer does buffering, and thus can have data ready to read
   // even though the underlying socket is not ready, so we need to actually ask for the data first
   bool wouldBlock = false;
@@ -726,7 +720,6 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
           DEBUGLOG("query received");
 
           if (handleQuery(state, now)) {
-            DEBUGLOG("handle query returned true");
             // if the query has been passed to a backend, or dropped, we can start
             // reading again, or sending queued responses
             if (state->d_queuedResponses.empty()) {
@@ -750,7 +743,6 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
           else {
             /* otherwise the state should already be waiting for
                the socket to be writable */
-            DEBUGLOG("should be waiting for writable socket");
             ioGuard.release();
             return;
           }
@@ -820,7 +812,6 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
 
 void IncomingTCPConnectionState::notifyIOError(std::shared_ptr<IncomingTCPConnectionState>& state, IDState&& query, const struct timeval& now)
 {
-  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   if (d_state == State::sendingResponse) {
     /* if we have responses to send, let's do that first */
   }
diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
index 0a5a11b1df..8b5e56e0bb 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
@@ -6,13 +6,21 @@ const uint16_t TCPConnectionToBackend::s_xfrID = 0;
 
 void TCPConnectionToBackend::assignToClientConnection(std::shared_ptr<IncomingTCPConnectionState>& clientConn, bool isXFR)
 {
-  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
+  if (d_usedForXFR == true) {
+    throw std::runtime_error("Trying to send a query over a backend connection used for XFR");
+  }
+
   if (isXFR) {
     d_usedForXFR = true;
   }
 
-  d_clientConn = clientConn;
-  d_ioState = make_unique<IOStateHandler>(clientConn->getIOMPlexer(), d_socket->getHandle());
+  if (!d_clientConn) {
+    d_clientConn = clientConn;
+    d_ioState = make_unique<IOStateHandler>(clientConn->getIOMPlexer(), d_socket->getHandle());
+  }
+  else if (d_clientConn != clientConn) {
+    throw std::runtime_error("Assigning a query from a different client to an existing backend connection with pending queries");
+  }
 }
 
 IOState TCPConnectionToBackend::sendNextQuery(std::shared_ptr<TCPConnectionToBackend>& conn)
@@ -63,7 +71,6 @@ static IOState tryRead(int fd, std::vector<uint8_t>& buffer, size_t& pos, size_t
 
 void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now)
 {
-  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   if (conn->d_socket == nullptr) {
     throw std::runtime_error("No downstream socket in " + std::string(__PRETTY_FUNCTION__) + "!");
   }
@@ -75,7 +82,7 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
 
   try {
     if (conn->d_state == State::sendingQueryToBackend) {
-      DEBUGLOG("sending query to backend over FD "<<fd);
+      DEBUGLOG("sending query to backend "<<conn->getDS()->getName()<<" over FD "<<fd);
       int socketFlags = 0;
 #ifdef MSG_FASTOPEN
       if (conn->isFastOpenEnabled()) {
@@ -89,7 +96,7 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
         /* request sent ! */
         conn->incQueries();
         conn->d_currentPos = 0;
-        //conn->d_currentQuery.d_querySentTime = now;
+
         DEBUGLOG("adding a pending response for ID "<<conn->d_currentQuery.d_idstate.origID<<" and QNAME "<<conn->d_currentQuery.d_idstate.qname);
         conn->d_pendingResponses[conn->d_currentQuery.d_idstate.origID] = std::move(conn->d_currentQuery);
         conn->d_currentQuery.d_buffer.clear();
@@ -117,7 +124,7 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
       // then we need to allocate a new buffer (new because we might need to re-send the query if the
       // backend dies on us)
       // We also might need to read and send to the client more than one response in case of XFR (yeah!)
-      // should very likely be a TCPIOHandler d_downstreamHandler
+      // should very likely be a TCPIOHandler
       conn->d_responseBuffer.resize(sizeof(uint16_t));
       iostate = tryRead(fd, conn->d_responseBuffer, conn->d_currentPos, sizeof(uint16_t) - conn->d_currentPos);
       if (iostate == IOState::Done) {
@@ -135,7 +142,6 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
       iostate = tryRead(fd, conn->d_responseBuffer, conn->d_currentPos, conn->d_responseSize - conn->d_currentPos);
       if (iostate == IOState::Done) {
         DEBUGLOG("got response from backend");
-        //conn->d_responseReadTime = now;
         try {
           iostate = conn->handleResponse(conn, now);
         }
@@ -170,15 +176,6 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
       ++conn->d_downstreamFailures;
     }
 
-#if 0
-    if (conn->d_outstanding) {
-      conn->d_outstanding = false;
-
-      if (conn->d_ds != nullptr) {
-        --conn->d_ds->outstanding;
-      }
-    }
-#endif
     /* remove this FD from the IO multiplexer */
     iostate = IOState::Done;
     connectionDied = true;
@@ -189,15 +186,14 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
     DEBUGLOG("connection died, number of failures is "<<conn->d_downstreamFailures<<", retries is "<<conn->d_ds->retries);
 
     if ((!conn->d_usedForXFR || conn->d_queries == 0) && conn->d_downstreamFailures < conn->d_ds->retries) {
-      DEBUGLOG("reconnecting");
+
       conn->d_ioState->reset();
       ioGuard.release();
 
       if (conn->reconnect()) {
-        DEBUGLOG("reconnected");
-
         conn->d_ioState = make_unique<IOStateHandler>(conn->d_clientConn->getIOMPlexer(), conn->d_socket->getHandle());
 
+        /* we need to resend the queries that were in flight, if any */
         for (auto& pending : conn->d_pendingResponses) {
           conn->d_pendingQueries.push_back(std::move(pending.second));
         }
@@ -252,7 +248,6 @@ void TCPConnectionToBackend::handleIOCallback(int fd, FDMultiplexer::funcparam_t
 
 void TCPConnectionToBackend::queueQuery(TCPQuery&& query, std::shared_ptr<TCPConnectionToBackend>& sharedSelf)
 {
-  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   if (d_ioState == nullptr) {
     throw std::runtime_error("Trying to queue a query to a TCP connection that has no incoming client connection assigned");
   }
@@ -269,8 +264,6 @@ void TCPConnectionToBackend::queueQuery(TCPQuery&& query, std::shared_ptr<TCPCon
       d_proxyProtocolPayloadAdded = true;
     }
 
-    DEBUGLOG("need write");
-
     struct timeval now;
     gettimeofday(&now, 0);
 
@@ -280,7 +273,6 @@ void TCPConnectionToBackend::queueQuery(TCPQuery&& query, std::shared_ptr<TCPCon
     // store query in the list of queries to send
     d_pendingQueries.push_back(std::move(query));
   }
-  DEBUGLOG("out of "<<__PRETTY_FUNCTION__);
 }
 
 bool TCPConnectionToBackend::reconnect()
@@ -297,9 +289,11 @@ bool TCPConnectionToBackend::reconnect()
 
   do {
     vinfolog("TCP connecting to downstream %s (%d)", d_ds->getNameWithAddr(), d_downstreamFailures);
+    DEBUGLOG("Opening TCP connection to backend "<<d_ds->getNameWithAddr());
     try {
       result = std::unique_ptr<Socket>(new Socket(d_ds->remote.sin4.sin_family, SOCK_STREAM, 0));
       DEBUGLOG("result of connect is "<<result->getHandle());
+
       if (!IsAnyAddress(d_ds->sourceAddr)) {
         SSetsockopt(result->getHandle(), SOL_SOCKET, SO_REUSEADDR, 1);
 #ifdef IP_BIND_ADDRESS_NO_PORT
@@ -327,7 +321,6 @@ bool TCPConnectionToBackend::reconnect()
 #endif /* MSG_FASTOPEN */
 
       d_socket = std::move(result);
-      DEBUGLOG("connected new socket "<<d_socket->getHandle());
       ++d_ds->tcpCurrentConnections;
       return true;
     }
@@ -362,7 +355,6 @@ void TCPConnectionToBackend::handleTimeout(const struct timeval& now, bool write
 
 void TCPConnectionToBackend::notifyAllQueriesFailed(const struct timeval& now, bool timeout)
 {
-  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
   d_connectionDied = true;
 
   auto& clientConn = d_clientConn;
@@ -376,7 +368,7 @@ void TCPConnectionToBackend::notifyAllQueriesFailed(const struct timeval& now, b
     ++clientConn->d_ci.cs->tcpDownstreamTimeouts;
   }
 
-  if (d_state == State::doingHandshake || d_state == State::sendingQueryToBackend) {
+  if (d_state == State::sendingQueryToBackend) {
     clientConn->notifyIOError(clientConn, std::move(d_currentQuery.d_idstate), now);
   }
 
@@ -396,13 +388,10 @@ void TCPConnectionToBackend::notifyAllQueriesFailed(const struct timeval& now, b
 
 IOState TCPConnectionToBackend::handleResponse(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now)
 {
-  DEBUGLOG("in "<<__PRETTY_FUNCTION__);
-
   d_downstreamFailures = 0;
 
   auto& clientConn = d_clientConn;
   if (!clientConn->active()) {
-    DEBUGLOG("client is not active");
     // a client timeout occured, or something like that */
     d_connectionDied = true;
     d_clientConn.reset();
@@ -418,11 +407,10 @@ IOState TCPConnectionToBackend::handleResponse(std::shared_ptr<TCPConnectionToBa
     d_state = State::readingResponseSizeFromBackend;
     d_currentPos = 0;
     d_responseBuffer.resize(sizeof(uint16_t));
-    return IOState::NeedRead;
     // get ready to read the next packet, if any
+    return IOState::NeedRead;
   }
   else {
-    DEBUGLOG("not XFR, phew");
     uint16_t queryId = 0;
     try {
       queryId = getQueryIdFromResponse();
@@ -439,6 +427,7 @@ IOState TCPConnectionToBackend::handleResponse(std::shared_ptr<TCPConnectionToBa
       notifyAllQueriesFailed(now);
       return IOState::Done;
     }
+
     auto ids = std::move(it->second.d_idstate);
     d_pendingResponses.erase(it);
     DEBUGLOG("passing response to client connection for "<<ids.qname);
@@ -460,7 +449,7 @@ IOState TCPConnectionToBackend::handleResponse(std::shared_ptr<TCPConnectionToBa
       return IOState::NeedRead;
     }
     else {
-      DEBUGLOG("nothing to do, phewwwww");
+      DEBUGLOG("nothing to do, waiting for a new query");
       d_state = State::idle;
       d_clientConn.reset();
       return IOState::Done;
diff --git a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
index 721559256e..a8f69ef4ff 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
@@ -195,7 +195,7 @@ public:
 
   enum class State { doingHandshake, readingQuerySize, readingQuery, sendingResponse, idle /* in case of XFR, we stop processing queries */ };
 
-  std::map<ComboAddress, std::deque<std::shared_ptr<TCPConnectionToBackend>>> d_activeConnectionsToBackend;
+  std::map<std::shared_ptr<DownstreamState>, std::deque<std::shared_ptr<TCPConnectionToBackend>>> d_activeConnectionsToBackend;
   std::vector<uint8_t> d_buffer;
   std::deque<TCPResponse> d_queuedResponses;
   TCPClientThreadData& d_threadData;

From 50620a070c329b0eec8399529ff2e13f393a8923 Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Thu, 10 Sep 2020 17:05:48 +0200
Subject: [PATCH 09/23] dnsdist: More regression tests for out of order
 processing

---
 regression-tests.dnsdist/test_OOOR.py | 216 ++++++++++++++++++++++----
 1 file changed, 184 insertions(+), 32 deletions(-)

diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc
index 02e23fa42d..b68b88f795 100644
--- a/pdns/dnsdist-tcp.cc
+++ b/pdns/dnsdist-tcp.cc
@@ -166,21 +166,6 @@ IncomingTCPConnectionState::~IncomingTCPConnectionState()
     auto diff = now - d_connectionStartTime;
     d_ci.cs->updateTCPMetrics(d_queriesCount, diff.tv_sec * 1000.0 + diff.tv_usec / 1000.0);
   }
-
-  try {
-    if (d_lastIOState == IOState::NeedRead) {
-      d_threadData.mplexer->removeReadFD(d_ci.fd);
-    }
-    else if (d_lastIOState == IOState::NeedWrite) {
-      d_threadData.mplexer->removeWriteFD(d_ci.fd);
-    }
-  }
-  catch (const FDMultiplexerException& e) {
-    vinfolog("Got an exception when trying to remove a pending IO operation on an incoming TCP connection from %s: %s", d_ci.remote.toStringWithPort(), e.what());
-  }
-  catch (...) {
-    vinfolog("Got an unknown exception when trying to remove a pending IO operation on an incoming TCP connection from %s", d_ci.remote.toStringWithPort());
-  }
 }
 
 std::shared_ptr<TCPConnectionToBackend> IncomingTCPConnectionState::getDownstreamConnection(std::shared_ptr<DownstreamState>& ds, const struct timeval& now)
@@ -329,7 +314,7 @@ static IOState handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& s
     }
   }
   else {
-    DEBUGLOG("queue size is "<<state->d_queuedResponses.size());
+    DEBUGLOG("queue size is "<<state->d_queuedResponses.size()<<", sending the next one");
     TCPResponse resp = std::move(state->d_queuedResponses.front());
     state->d_queuedResponses.pop_front();
     state->d_state = IncomingTCPConnectionState::State::idle;
@@ -361,7 +346,6 @@ void IncomingTCPConnectionState::resetForNewQuery()
   d_querySize = 0;
   d_downstreamFailures = 0;
   d_state = State::readingQuerySize;
-  d_lastIOState = IOState::Done;
   d_selfGeneratedResponse = false;
 }
 
@@ -624,6 +608,7 @@ static bool handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
   }
 
   ++state->d_currentQueriesCount;
+  vinfolog("Got query for %s|%s from %s, relayed to %s", ids.qname.toLogString(), QType(ids.qtype).getName(), state->d_ci.remote.toStringWithPort(), ds->getName());
   downstreamConnection->queueQuery(TCPQuery(std::move(state->d_buffer), std::move(ids)), downstreamConnection);
 
   return true;
@@ -736,6 +721,7 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
               TCPResponse resp = std::move(state->d_queuedResponses.front());
               state->d_queuedResponses.pop_front();
               ioGuard.release();
+              state->d_state = IncomingTCPConnectionState::State::idle;
               state->sendResponse(state, now, std::move(resp));
               return;
             }
@@ -787,12 +773,12 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
         ++state->d_ci.cs->tcpDiedSendingResponse;
       }
 
-      if (state->d_lastIOState == IOState::NeedWrite || state->d_readingFirstQuery) {
+      if (state->d_ioState->getState() == IOState::NeedWrite || state->d_readingFirstQuery) {
         DEBUGLOG("Got an exception while handling TCP query: "<<e.what());
-        vinfolog("Got an exception while handling (%s) TCP query from %s: %s", (state->d_lastIOState == IOState::NeedRead ? "reading" : "writing"), state->d_ci.remote.toStringWithPort(), e.what());
+        vinfolog("Got an exception while handling (%s) TCP query from %s: %s", (state->d_ioState->getState() == IOState::NeedRead ? "reading" : "writing"), state->d_ci.remote.toStringWithPort(), e.what());
       }
       else {
-        vinfolog("Closing TCP client connection with %s", state->d_ci.remote.toStringWithPort());
+        vinfolog("Closing TCP client connection with %s: %s", state->d_ci.remote.toStringWithPort(), e.what());
         DEBUGLOG("Closing TCP client connection: "<<e.what());
       }
       /* remove this FD from the IO multiplexer */
@@ -819,11 +805,11 @@ void IncomingTCPConnectionState::notifyIOError(std::shared_ptr<IncomingTCPConnec
     /* stop reading and send what we have */
     TCPResponse resp = std::move(d_queuedResponses.front());
     d_queuedResponses.pop_front();
+    state->d_state = IncomingTCPConnectionState::State::idle;
     sendResponse(state, now, std::move(resp));
   }
   else {
     // the backend code already tried to reconnect if it was possible
-    d_lastIOState = IOState::Done;
     d_ioState->reset();
   }
 }
@@ -837,7 +823,6 @@ void IncomingTCPConnectionState::handleTimeout(bool write)
 {
   DEBUGLOG("client timeout");
   ++d_ci.cs->tcpClientTimeouts;
-  d_lastIOState = IOState::Done;
   d_ioState->reset();
 }
 
diff --git a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
index a8f69ef4ff..3f31f5dbca 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
@@ -216,7 +216,6 @@ public:
   uint16_t d_querySize{0};
   uint16_t d_downstreamFailures{0};
   State d_state{State::doingHandshake};
-  IOState d_lastIOState{IOState::Done};
   bool d_readingFirstQuery{true};
   bool d_isXFR{false};
   bool d_xfrStarted{false};

From a013f721aa8b666ca9349f1f53cf38c3b1bf5e24 Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Fri, 11 Sep 2020 15:01:38 +0200
Subject: [PATCH 11/23] dnsdist: Fix OOOR regression tests (connections count,
 ECONNRESET)

---
 regression-tests.dnsdist/test_OOOR.py | 129 +++++++++++++-------------
 1 file changed, 67 insertions(+), 62 deletions(-)

diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc
index b68b88f795..642550ae68 100644
--- a/pdns/dnsdist-tcp.cc
+++ b/pdns/dnsdist-tcp.cc
@@ -392,7 +392,7 @@ void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnect
     state->d_currentPos = 0;
     state->d_currentResponse = std::move(response);
 
-    state->d_ioState->update(IOState::NeedWrite, handleIOCallback, state, getClientWriteTTD(now));
+    state->d_ioState->update(IOState::NeedWrite, handleIOCallback, state, state->getClientWriteTTD(now));
   }
   else {
     // queue response
@@ -406,7 +406,7 @@ void IncomingTCPConnectionState::handleResponse(std::shared_ptr<IncomingTCPConne
 {
   // if we have added a TCP Proxy Protocol payload to a connection, don't release it yet, no one else will be able to use it anyway
   if (!state->d_isXFR && response.d_connection && response.d_connection->isIdle() && response.d_connection->canBeReused()) {
-    auto& list = d_activeConnectionsToBackend.at(response.d_connection->getDS());
+    auto& list = state->d_activeConnectionsToBackend.at(response.d_connection->getDS());
     for (auto it = list.begin(); it != list.end(); ++it) {
       if (*it == response.d_connection) {
         DownstreamConnectionsManager::releaseDownstreamConnection(std::move(*it));
@@ -798,19 +798,19 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
 
 void IncomingTCPConnectionState::notifyIOError(std::shared_ptr<IncomingTCPConnectionState>& state, IDState&& query, const struct timeval& now)
 {
-  if (d_state == State::sendingResponse) {
+  if (state->d_state == State::sendingResponse) {
     /* if we have responses to send, let's do that first */
   }
-  else if (!d_queuedResponses.empty()) {
+  else if (!state->d_queuedResponses.empty()) {
     /* stop reading and send what we have */
-    TCPResponse resp = std::move(d_queuedResponses.front());
-    d_queuedResponses.pop_front();
+    TCPResponse resp = std::move(state->d_queuedResponses.front());
+    state->d_queuedResponses.pop_front();
     state->d_state = IncomingTCPConnectionState::State::idle;
     sendResponse(state, now, std::move(resp));
   }
   else {
     // the backend code already tried to reconnect if it was possible
-    d_ioState->reset();
+    state->d_ioState->reset();
   }
 }
 
@@ -819,11 +819,26 @@ void IncomingTCPConnectionState::handleXFRResponse(std::shared_ptr<IncomingTCPCo
   sendResponse(state, now, std::move(response));
 }
 
-void IncomingTCPConnectionState::handleTimeout(bool write)
+void IncomingTCPConnectionState::handleTimeout(std::shared_ptr<IncomingTCPConnectionState>& state, bool write)
 {
   DEBUGLOG("client timeout");
-  ++d_ci.cs->tcpClientTimeouts;
-  d_ioState->reset();
+  DEBUGLOG("Processed "<<state->d_queriesCount<<" queries, current count is "<<state->d_currentQueriesCount<<", "<<state->d_activeConnectionsToBackend.size()<<" active connections, "<<state->d_queuedResponses.size()<<" response queued");
+
+  if (write || state->d_currentQueriesCount == 0) {
+    ++state->d_ci.cs->tcpClientTimeouts;
+    state->d_ioState->reset();
+  }
+  else {
+    DEBUGLOG("Going idle");
+    /* we still have some queries in flight, let's just stop reading for now */
+    state->d_state = IncomingTCPConnectionState::State::idle;
+    state->d_ioState->update(IOState::Done, handleIOCallback, state);
+    for (const auto& active : state->d_activeConnectionsToBackend) {
+      for (const auto& conn: active.second) {
+        DEBUGLOG("Connection to "<<active.first->getName()<<" is "<<(conn->isIdle() ? "idle" : "not idle"));
+      }
+    }
+  }
 }
 
 static void handleIncomingTCPQuery(int pipefd, FDMultiplexer::funcparam_t& param)
@@ -898,7 +913,7 @@ static void tcpClientThread(int pipefd)
           auto state = boost::any_cast<std::shared_ptr<IncomingTCPConnectionState>>(cbData.second);
           if (cbData.first == state->d_ci.fd) {
             vinfolog("Timeout (read) from remote TCP client %s", state->d_ci.remote.toStringWithPort());
-            state->handleTimeout(false);
+            state->handleTimeout(state, false);
           }
         }
         else if (cbData.second.type() == typeid(std::shared_ptr<TCPConnectionToBackend>)) {
@@ -914,7 +929,7 @@ static void tcpClientThread(int pipefd)
           auto state = boost::any_cast<std::shared_ptr<IncomingTCPConnectionState>>(cbData.second);
           if (cbData.first == state->d_ci.fd) {
             vinfolog("Timeout (write) from remote TCP client %s", state->d_ci.remote.toStringWithPort());
-            state->handleTimeout(true);
+            state->handleTimeout(state, true);
           }
         }
         else if (cbData.second.type() == typeid(std::shared_ptr<TCPConnectionToBackend>)) {
diff --git a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
index 3f31f5dbca..704e197476 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
@@ -178,13 +178,13 @@ public:
 
   static void handleIO(std::shared_ptr<IncomingTCPConnectionState>& conn, const struct timeval& now);
   static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param);
+  static void notifyIOError(std::shared_ptr<IncomingTCPConnectionState>& state, IDState&& query, const struct timeval& now);
+  static void sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
+  static void handleResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
+  static void handleXFRResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
+  static void handleTimeout(std::shared_ptr<IncomingTCPConnectionState>& state, bool write);
 
   void queueQuery(TCPQuery&& query);
-  void notifyIOError(std::shared_ptr<IncomingTCPConnectionState>& state, IDState&& query, const struct timeval& now);
-  void sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
-  void handleResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
-  void handleXFRResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
-  void handleTimeout(bool write);
 
   bool canAcceptNewQueries() const;

From 312eb38dde7786b695351c50919220d81c70868f Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Fri, 11 Sep 2020 17:03:57 +0200
Subject: [PATCH 13/23] dnsdist: Gracefully handle dnsdist closing the test
 backend connection

---
 regression-tests.dnsdist/test_OOOR.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc
index 642550ae68..6bf3b80eb3 100644
--- a/pdns/dnsdist-tcp.cc
+++ b/pdns/dnsdist-tcp.cc
@@ -798,6 +798,8 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
 
 void IncomingTCPConnectionState::notifyIOError(std::shared_ptr<IncomingTCPConnectionState>& state, IDState&& query, const struct timeval& now)
 {
+  --state->d_currentQueriesCount;
+
   if (state->d_state == State::sendingResponse) {
     /* if we have responses to send, let's do that first */
   }

From 0b619f5b72f4beb804dafcd5ea58a37c67c06bee Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Mon, 14 Sep 2020 11:42:29 +0200
Subject: [PATCH 15/23] dnsdist: Use a counter for pipelined TCP queries

It turns out that collisions do happen with a random ID.
---
 pdns/dnsdistdist/dnsdist-tcp-downstream.cc |  2 +-
 regression-tests.dnsdist/test_OOOR.py      | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
index 8b5e56e0bb..4fa155c55c 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
@@ -423,7 +423,7 @@ IOState TCPConnectionToBackend::handleResponse(std::shared_ptr<TCPConnectionToBa
 
     auto it = d_pendingResponses.find(queryId);
     if (it == d_pendingResponses.end()) {
-      DEBUGLOG("could not found any corresponding query for ID "<<queryId);
+      DEBUGLOG("could not find any corresponding query for ID "<<queryId<<". This is likely a duplicated ID over the same TCP connection, giving up!");
       notifyAllQueriesFailed(now);
       return IOState::Done;
     }

diff --git a/pdns/epollmplexer.cc b/pdns/epollmplexer.cc
index 433687d214..1a4b2a3b50 100644
--- a/pdns/epollmplexer.cc
+++ b/pdns/epollmplexer.cc
@@ -47,6 +47,8 @@ class EpollFDMultiplexer : public FDMultiplexer
 
   virtual void addFD(callbackmap_t& cbmap, int fd, callbackfunc_t toDo, const funcparam_t& parameter, const struct timeval* ttd=nullptr) override;
   virtual void removeFD(callbackmap_t& cbmap, int fd) override;
+  virtual void alterFD(callbackmap_t& from, callbackmap_t& to, int fd, callbackfunc_t toDo, const funcparam_t& parameter, const struct timeval* ttd) override;
+
   string getName() const override
   {
     return "epoll";
@@ -91,7 +93,7 @@ EpollFDMultiplexer::EpollFDMultiplexer() : d_eevents(new epoll_event[s_maxevents
     close(d_epollfd);
     throw FDMultiplexerException("epoll multiplexer failed self-test: "+string(fe.what()));
   }
-    
+
 }
 
 void EpollFDMultiplexer::addFD(callbackmap_t& cbmap, int fd, callbackfunc_t toDo, const funcparam_t& parameter, const struct timeval* ttd)
@@ -99,11 +101,11 @@ void EpollFDMultiplexer::addFD(callbackmap_t& cbmap, int fd, callbackfunc_t toDo
   accountingAddFD(cbmap, fd, toDo, parameter, ttd);
 
   struct epoll_event eevent;
-  
+
   eevent.events = (&cbmap == &d_readCallbacks) ? EPOLLIN : EPOLLOUT;
-  
+
   eevent.data.u64=0; // placate valgrind (I love it so much)
-  eevent.data.fd=fd; 
+  eevent.data.fd=fd;
 
   if(epoll_ctl(d_epollfd, EPOLL_CTL_ADD, fd, &eevent) < 0) {
     cbmap.erase(fd);
@@ -113,8 +115,7 @@ void EpollFDMultiplexer::addFD(callbackmap_t& cbmap, int fd, callbackfunc_t toDo
 
 void EpollFDMultiplexer::removeFD(callbackmap_t& cbmap, int fd)
 {
-  if(!cbmap.erase(fd))
-    throw FDMultiplexerException("Tried to remove unlisted fd "+std::to_string(fd)+ " from multiplexer");
+  accountingRemoveFD(cbmap, fd);
 
   struct epoll_event dummy;
   dummy.events = 0;
@@ -124,6 +125,22 @@ void EpollFDMultiplexer::removeFD(callbackmap_t& cbmap, int fd)
     throw FDMultiplexerException("Removing fd from epoll set: "+stringerror());
 }
 
+void EpollFDMultiplexer::alterFD(callbackmap_t& from, callbackmap_t& to, int fd, callbackfunc_t toDo, const funcparam_t& parameter, const struct timeval* ttd)
+{
+  accountingRemoveFD(from, fd);
+  accountingAddFD(to, fd, toDo, parameter, ttd);
+
+  struct epoll_event eevent;
+  eevent.events = (&to == &d_readCallbacks) ? EPOLLIN : EPOLLOUT;
+  eevent.data.u64 = 0; // placate valgrind (I love it so much)
+  eevent.data.fd = fd;
+
+  if (epoll_ctl(d_epollfd, EPOLL_CTL_MOD, fd, &eevent) < 0) {
+    to.erase(fd);
+    throw FDMultiplexerException("Altering fd in epoll set: "+stringerror());
+  }
+}
+
 void EpollFDMultiplexer::getAvailableFDs(std::vector<int>& fds, int timeout)
 {
   int ret=epoll_wait(d_epollfd, d_eevents.get(), s_maxevents, timeout);
@@ -141,7 +158,7 @@ int EpollFDMultiplexer::run(struct timeval* now, int timeout)
   if(d_inrun) {
     throw FDMultiplexerException("FDMultiplexer::run() is not reentrant!\n");
   }
-  
+
   int ret=epoll_wait(d_epollfd, d_eevents.get(), s_maxevents, timeout);
   gettimeofday(now,0); // MANDATORY
 
@@ -154,13 +171,13 @@ int EpollFDMultiplexer::run(struct timeval* now, int timeout)
   d_inrun=true;
   for(int n=0; n < ret; ++n) {
     d_iter=d_readCallbacks.find(d_eevents[n].data.fd);
-    
+
     if(d_iter != d_readCallbacks.end()) {
       d_iter->d_callback(d_iter->d_fd, d_iter->d_parameter);
       continue; // so we don't refind ourselves as writable!
     }
     d_iter=d_writeCallbacks.find(d_eevents[n].data.fd);
-    
+
     if(d_iter != d_writeCallbacks.end()) {
       d_iter->d_callback(d_iter->d_fd, d_iter->d_parameter);
     }
@@ -184,7 +201,7 @@ void acceptData(int fd, funcparam_t& parameter)
 int main()
 {
   Socket s(AF_INET, SOCK_DGRAM);
-  
+
   IPEndpoint loc("0.0.0.0", 2000);
   s.bind(loc);
 
@@ -199,5 +216,3 @@ int main()
   sfm.removeReadFD(s.getHandle());
 }
 #endif
-
-
diff --git a/pdns/mplexer.hh b/pdns/mplexer.hh
index a2cd0b7507..a948c9f075 100644
--- a/pdns/mplexer.hh
+++ b/pdns/mplexer.hh
@@ -136,6 +136,16 @@ public:
     d_writeCallbacks.replace(it, newEntry);
   }
 
+  virtual void alterFDToRead(int fd, callbackfunc_t toDo, const funcparam_t& parameter=funcparam_t(), const struct timeval* ttd=nullptr)
+  {
+    this->alterFD(d_writeCallbacks, d_readCallbacks, fd, toDo, parameter, ttd);
+  }
+
+  virtual void alterFDToWrite(int fd, callbackfunc_t toDo, const funcparam_t& parameter=funcparam_t(), const struct timeval* ttd=nullptr)
+  {
+    this->alterFD(d_readCallbacks, d_writeCallbacks, fd, toDo, parameter, ttd);
+  }
+
   virtual std::vector<std::pair<int, funcparam_t> > getTimeouts(const struct timeval& tv, bool writes=false)
   {
     std::vector<std::pair<int, funcparam_t> > ret;
@@ -209,6 +219,7 @@ protected:
 
   virtual void addFD(callbackmap_t& cbmap, int fd, callbackfunc_t toDo, const funcparam_t& parameter, const struct timeval* ttd=nullptr)=0;
   virtual void removeFD(callbackmap_t& cbmap, int fd)=0;
+
   bool d_inrun;
   callbackmap_t::iterator d_iter;
 
@@ -223,7 +234,7 @@ protected:
       cb.d_ttd = *ttd;
     }
 
-    auto pair = cbmap.insert(cb);
+    auto pair = cbmap.insert(std::move(cb));
     if (!pair.second) {
       throw FDMultiplexerException("Tried to add fd "+std::to_string(fd)+ " to multiplexer twice");
     }
@@ -235,4 +246,12 @@ protected:
       throw FDMultiplexerException("Tried to remove unlisted fd "+std::to_string(fd)+ " from multiplexer");
     }
   }
+
+  virtual void alterFD(callbackmap_t& from, callbackmap_t& to, int fd, callbackfunc_t toDo, const funcparam_t& parameter, const struct timeval* ttd)
+  {
+    /* naive implementation */
+    removeFD(from, fd);
+    addFD(to, fd, toDo, parameter, ttd);
+  }
+
 };
diff --git a/pdns/pollmplexer.cc b/pdns/pollmplexer.cc
index 399ad132ae..665b4b823c 100644
--- a/pdns/pollmplexer.cc
+++ b/pdns/pollmplexer.cc
@@ -70,8 +70,7 @@ void PollFDMultiplexer::removeFD(callbackmap_t& cbmap, int fd)
   if(d_inrun && d_iter->d_fd==fd)  // trying to remove us!
     ++d_iter;
 
-  if(!cbmap.erase(fd))
-    throw FDMultiplexerException("Tried to remove unlisted fd "+std::to_string(fd)+ " from multiplexer");
+  accountingRemoveFD(cbmap, fd);
 }
 
 vector<struct pollfd> PollFDMultiplexer::preparePollFD() const

From cb8ec658513b5f04ab7d518e58d15aa11830a2de Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Tue, 15 Sep 2020 15:38:53 +0200
Subject: [PATCH 17/23] dnsdist: Use the more efficient epoll update mechanism

---
 pdns/dnsdistdist/tcpiohandler-mplexer.hh | 27 ++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/pdns/dnsdistdist/tcpiohandler-mplexer.hh b/pdns/dnsdistdist/tcpiohandler-mplexer.hh
index e0e366dd78..1971a3a152 100644
--- a/pdns/dnsdistdist/tcpiohandler-mplexer.hh
+++ b/pdns/dnsdistdist/tcpiohandler-mplexer.hh
@@ -50,12 +50,12 @@ public:
   void update(IOState iostate, FDMultiplexer::callbackfunc_t callback = FDMultiplexer::callbackfunc_t(), FDMultiplexer::funcparam_t callbackData = boost::any(), boost::optional<struct timeval> ttd = boost::none)
   {
     DEBUGLOG("in "<<__PRETTY_FUNCTION__<<" for fd "<<d_fd<<", last state was "<<(int)d_currentState<<", new state is "<<(int)iostate);
-    if (d_currentState == IOState::NeedRead && iostate != IOState::NeedRead) {
+    if (d_currentState == IOState::NeedRead && iostate == IOState::Done) {
       DEBUGLOG(__PRETTY_FUNCTION__<<": remove read FD "<<d_fd);
       d_mplexer->removeReadFD(d_fd);
       d_currentState = IOState::Done;
     }
-    else if (d_currentState == IOState::NeedWrite && iostate != IOState::NeedWrite) {
+    else if (d_currentState == IOState::NeedWrite && iostate == IOState::Done) {
       DEBUGLOG(__PRETTY_FUNCTION__<<": remove write FD "<<d_fd);
       d_mplexer->removeWriteFD(d_fd);
       d_currentState = IOState::Done;
@@ -70,18 +70,33 @@ public:
         return;
       }
 
+      if (d_currentState == IOState::NeedWrite) {
+        d_mplexer->alterFDToRead(d_fd, callback, callbackData, ttd ? &*ttd : nullptr);
+        DEBUGLOG(__PRETTY_FUNCTION__<<": alter from write to read FD "<<d_fd);
+      }
+      else {
+        d_mplexer->addReadFD(d_fd, callback, callbackData, ttd ? &*ttd : nullptr);
+        DEBUGLOG(__PRETTY_FUNCTION__<<": add read FD "<<d_fd);
+      }
+
       d_currentState = IOState::NeedRead;
-      DEBUGLOG(__PRETTY_FUNCTION__<<": add read FD "<<d_fd);
-      d_mplexer->addReadFD(d_fd, callback, callbackData, ttd ? &*ttd : nullptr);
+
     }
     else if (iostate == IOState::NeedWrite) {
       if (d_currentState == IOState::NeedWrite) {
         return;
       }
 
+      if (d_currentState == IOState::NeedRead) {
+        d_mplexer->alterFDToWrite(d_fd, callback, callbackData, ttd ? &*ttd : nullptr);
+        DEBUGLOG(__PRETTY_FUNCTION__<<": alter from read to write FD "<<d_fd);
+      }
+      else {
+        d_mplexer->addWriteFD(d_fd, callback, callbackData, ttd ? &*ttd : nullptr);
+        DEBUGLOG(__PRETTY_FUNCTION__<<": add write FD "<<d_fd);
+      }
+
       d_currentState = IOState::NeedWrite;
-      DEBUGLOG(__PRETTY_FUNCTION__<<": add write FD "<<d_fd);
-      d_mplexer->addWriteFD(d_fd, callback, callbackData, ttd ? &*ttd : nullptr);
     }
     else if (iostate == IOState::Done) {
       d_currentState = IOState::Done;

From 89bd37536042aa5ee4f7e9ec11e7f0fa0ac9749b Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Tue, 15 Sep 2020 16:33:07 +0200
Subject: [PATCH 18/23] dnsdist: Only copy what we need from the response
 buffer

---
 pdns/dnsdist-tcp.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc
index 6bf3b80eb3..f3e3d6a1a1 100644
--- a/pdns/dnsdist-tcp.cc
+++ b/pdns/dnsdist-tcp.cc
@@ -323,7 +323,6 @@ static IOState handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& s
   }
 }
 
-
 bool IncomingTCPConnectionState::canAcceptNewQueries() const
 {
   if (d_isXFR) {
@@ -573,10 +572,13 @@ static bool handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
   /* prepend the size. Yes, this is not the most efficient way but it prevents mistakes
      that could occur if we had to deal with the size during the processing,
      especially alignment issues */
+  /* first we need to resize to the size that is actually used, since we allocated more to be able to insert
+     EDNS or Proxy Protocol values */
+  dq.size = state->d_buffer.size();
+  state->d_buffer.resize(dq.len);
   state->d_buffer.insert(state->d_buffer.begin(), sizeBytes, sizeBytes + 2);
   dq.len = dq.len + 2;
   dq.dh = reinterpret_cast<dnsheader*>(&state->d_buffer.at(0));
-  dq.size = state->d_buffer.size();
   state->d_buffer.resize(dq.len);
 
   bool proxyProtocolPayloadAdded = false;

From 75a34870ece20876bdc7bb835539ea1bffbbac29 Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Wed, 16 Sep 2020 11:41:01 +0200
Subject: [PATCH 19/23] dnsdist: Add 'hasBufferedData' method to TCPIOHandler

---
 pdns/dnsdist-tcp.cc                      |  3 +++
 pdns/dnsdistdist/tcpiohandler-mplexer.hh |  1 +
 pdns/dnsdistdist/tcpiohandler.cc         | 18 ++++++++++++++++++
 pdns/tcpiohandler.hh                     |  9 +++++++++
 4 files changed, 31 insertions(+)

diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc
index f3e3d6a1a1..395676d794 100644
--- a/pdns/dnsdist-tcp.cc
+++ b/pdns/dnsdist-tcp.cc
@@ -837,11 +837,14 @@ void IncomingTCPConnectionState::handleTimeout(std::shared_ptr<IncomingTCPConnec
     /* we still have some queries in flight, let's just stop reading for now */
     state->d_state = IncomingTCPConnectionState::State::idle;
     state->d_ioState->update(IOState::Done, handleIOCallback, state);
+
+#ifdef DEBUGLOG_ENABLED
     for (const auto& active : state->d_activeConnectionsToBackend) {
       for (const auto& conn: active.second) {
         DEBUGLOG("Connection to "<<active.first->getName()<<" is "<<(conn->isIdle() ? "idle" : "not idle"));
       }
     }
+#endif
   }
 }
 
diff --git a/pdns/dnsdistdist/tcpiohandler-mplexer.hh b/pdns/dnsdistdist/tcpiohandler-mplexer.hh
index 1971a3a152..0b85fe68ae 100644
--- a/pdns/dnsdistdist/tcpiohandler-mplexer.hh
+++ b/pdns/dnsdistdist/tcpiohandler-mplexer.hh
@@ -5,6 +5,7 @@
 #include "tcpiohandler.hh"
 
 #if 0
+#define DEBUGLOG_ENABLED
 #define DEBUGLOG(x) cerr<<x<<endl;
 #else
 #define DEBUGLOG(x)
diff --git a/pdns/dnsdistdist/tcpiohandler.cc b/pdns/dnsdistdist/tcpiohandler.cc
index b4ea4c9677..c1ff709344 100644
--- a/pdns/dnsdistdist/tcpiohandler.cc
+++ b/pdns/dnsdistdist/tcpiohandler.cc
@@ -224,6 +224,15 @@ class OpenSSLTLSConnection: public TLSConnection
     return got;
   }
 
+  bool hasBufferedData() const override
+  {
+    if (d_conn) {
+      return SSL_pending(d_conn.get()) > 0;
+    }
+
+    return false;
+  }
+
   void close() override
   {
     if (d_conn) {
@@ -690,6 +699,15 @@ class GnuTLSConnection: public TLSConnection
     return got;
   }
 
+  bool hasBufferedData() const override
+  {
+    if (d_conn) {
+      return gnutls_record_check_pending(d_conn.get()) > 0;
+    }
+
+    return false;
+  }
+
   std::string getServerNameIndication() const override
   {
     if (d_conn) {
diff --git a/pdns/tcpiohandler.hh b/pdns/tcpiohandler.hh
index 5a937368f0..a5e8320898 100644
--- a/pdns/tcpiohandler.hh
+++ b/pdns/tcpiohandler.hh
@@ -17,6 +17,7 @@ public:
   virtual size_t write(const void* buffer, size_t bufferSize, unsigned int writeTimeout) = 0;
   virtual IOState tryWrite(std::vector<uint8_t>& buffer, size_t& pos, size_t toWrite) = 0;
   virtual IOState tryRead(std::vector<uint8_t>& buffer, size_t& pos, size_t toRead) = 0;
+  virtual bool hasBufferedData() const = 0;
   virtual std::string getServerNameIndication() const = 0;
   virtual LibsslTLSVersion getTLSVersion() const = 0;
   virtual bool hasSessionBeenResumed() const = 0;
@@ -293,6 +294,14 @@ public:
     }
   }
 
+  bool hasBufferedData() const
+  {
+    if (d_conn) {
+      return d_conn->hasBufferedData();
+    }
+    return false;
+  }
+
   std::string getServerNameIndication() const
   {
     if (d_conn) {

From e28f7ddd6d70106bfe1997b0540633713f7495b4 Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Thu, 17 Sep 2020 17:04:04 +0200
Subject: [PATCH 20/23] dnsdist: Try to send before calling epoll_wait

It usually works and saves adding the descriptor to the set, calling
epoll_wait() then removing the descriptor.
---
 pdns/dnsdist-tcp.cc                        | 189 ++++++++++++---------
 pdns/dnsdistdist/dnsdist-tcp-downstream.cc | 110 +++++++-----
 pdns/dnsdistdist/dnsdist-tcp-downstream.hh |   7 +-
 pdns/dnsdistdist/dnsdist-tcp-upstream.hh   |   7 +-
 regression-tests.dnsdist/test_OOOR.py      |   2 +-
 5 files changed, 188 insertions(+), 127 deletions(-)

diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc
index 395676d794..18c68da79e 100644
--- a/pdns/dnsdist-tcp.cc
+++ b/pdns/dnsdist-tcp.cc
@@ -253,7 +253,42 @@ void TCPClientCollection::addTCPClientThread()
 
 std::unique_ptr<TCPClientCollection> g_tcpclientthreads;
 
-static IOState handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
+static IOState sendQueuedResponses(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
+{
+  IOState result = IOState::Done;
+
+  while (!state->d_queuedResponses.empty()) {
+    DEBUGLOG("queue size is "<<state->d_queuedResponses.size()<<", sending the next one");
+    TCPResponse resp = std::move(state->d_queuedResponses.front());
+    state->d_queuedResponses.pop_front();
+    state->d_state = IncomingTCPConnectionState::State::idle;
+    result = state->sendResponse(state, now, std::move(resp));
+    if (result != IOState::Done) {
+      return result;
+    }
+  }
+
+  if (state->d_isXFR) {
+    /* we should still be reading from the backend, and we don't want to read from the client */
+    state->d_state = IncomingTCPConnectionState::State::idle;
+    state->d_currentPos = 0;
+    DEBUGLOG("idling for XFR completion");
+    return IOState::Done;
+  } else {
+    if (state->canAcceptNewQueries()) {
+      DEBUGLOG("waiting for new queries");
+      state->resetForNewQuery();
+      return IOState::NeedRead;
+    }
+    else {
+      DEBUGLOG("idling");
+      state->d_state = IncomingTCPConnectionState::State::idle;
+      return IOState::Done;
+    }
+  }
+}
+
+static bool handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
 {
   --state->d_currentQueriesCount;
 
@@ -284,43 +319,16 @@ static IOState handleResponseSent(std::shared_ptr<IncomingTCPConnectionState>& s
 
     if (g_maxTCPQueriesPerConn && state->d_queriesCount > g_maxTCPQueriesPerConn) {
       vinfolog("Terminating TCP connection from %s because it reached the maximum number of queries per conn (%d / %d)", state->d_ci.remote.toStringWithPort(), state->d_queriesCount, g_maxTCPQueriesPerConn);
-      return IOState::Done;
+      return false;
     }
 
     if (state->maxConnectionDurationReached(g_maxTCPConnectionDuration, now)) {
       vinfolog("Terminating TCP connection from %s because it reached the maximum TCP connection duration", state->d_ci.remote.toStringWithPort());
-      return IOState::Done;
+      return false;
     }
   }
 
-  if (state->d_queuedResponses.empty()) {
-    if (state->d_isXFR) {
-      /* we should still be reading from the backend, and we don't want to read from the client */
-      state->d_state = IncomingTCPConnectionState::State::idle;
-      state->d_currentPos = 0;
-      DEBUGLOG("idling for XFR completion");
-      return IOState::Done;
-    } else {
-      if (state->canAcceptNewQueries()) {
-        DEBUGLOG("waiting for new queries");
-        state->resetForNewQuery();
-        return IOState::NeedRead;
-      }
-      else {
-        DEBUGLOG("idling");
-        state->d_state = IncomingTCPConnectionState::State::idle;
-        return IOState::Done;
-      }
-    }
-  }
-  else {
-    DEBUGLOG("queue size is "<<state->d_queuedResponses.size()<<", sending the next one");
-    TCPResponse resp = std::move(state->d_queuedResponses.front());
-    state->d_queuedResponses.pop_front();
-    state->d_state = IncomingTCPConnectionState::State::idle;
-    state->sendResponse(state, now, std::move(resp));
-    return IOState::NeedWrite;
-  }
+  return true;
 }
 
 bool IncomingTCPConnectionState::canAcceptNewQueries() const
@@ -372,26 +380,43 @@ void IncomingTCPConnectionState::registerActiveDownstreamConnection(std::shared_
   d_activeConnectionsToBackend[conn->getDS()].push_front(conn);
 }
 
-/* this version is called when the buffer has been set and the rules have been processed */
-void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
+/* called when the buffer has been set and the rules have been processed, and only from handleIO (sometimes indirectly via handleQuery) */
+IOState IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
+{
+  state->d_state = IncomingTCPConnectionState::State::sendingResponse;
+
+  uint16_t responseSize = static_cast<uint16_t>(response.d_buffer.size());
+  const uint8_t sizeBytes[] = { static_cast<uint8_t>(responseSize / 256), static_cast<uint8_t>(responseSize % 256) };
+  /* prepend the size. Yes, this is not the most efficient way but it prevents mistakes
+     that could occur if we had to deal with the size during the processing,
+     especially alignment issues */
+  response.d_buffer.insert(response.d_buffer.begin(), sizeBytes, sizeBytes + 2);
+  state->d_currentPos = 0;
+  state->d_currentResponse = std::move(response);
+
+  auto iostate = state->d_handler.tryWrite(state->d_currentResponse.d_buffer, state->d_currentPos, state->d_currentResponse.d_buffer.size());
+  if (iostate == IOState::Done) {
+    DEBUGLOG("response sent");
+    if (!handleResponseSent(state, now)) {
+      return IOState::Done;
+    }
+    return sendQueuedResponses(state, now);
+  } else {
+    return IOState::NeedWrite;
+    DEBUGLOG("partial write");
+  }
+}
+
+/* called when handling a response or error coming from a backend */
+void IncomingTCPConnectionState::sendOrQueueResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
 {
   // if we already reading a query (not the query size, mind you), or sending a response we need to either queue the response
   // otherwise we can start sending it right away
   if (state->d_state == IncomingTCPConnectionState::State::idle ||
       state->d_state == IncomingTCPConnectionState::State::readingQuerySize) {
 
-    state->d_state = IncomingTCPConnectionState::State::sendingResponse;
-
-    uint16_t responseSize = static_cast<uint16_t>(response.d_buffer.size());
-    const uint8_t sizeBytes[] = { static_cast<uint8_t>(responseSize / 256), static_cast<uint8_t>(responseSize % 256) };
-    /* prepend the size. Yes, this is not the most efficient way but it prevents mistakes
-       that could occur if we had to deal with the size during the processing,
-       especially alignment issues */
-    response.d_buffer.insert(response.d_buffer.begin(), sizeBytes, sizeBytes + 2);
-    state->d_currentPos = 0;
-    state->d_currentResponse = std::move(response);
-
-    state->d_ioState->update(IOState::NeedWrite, handleIOCallback, state, state->getClientWriteTTD(now));
+    auto iostate = sendResponse(state, now, std::move(response));
+    state->d_ioState->update(iostate, handleIOCallback, state, iostate == IOState::NeedWrite ? state->getClientWriteTTD(now) : state->getClientReadTTD(now));
   }
   else {
     // queue response
@@ -400,17 +425,21 @@ void IncomingTCPConnectionState::sendResponse(std::shared_ptr<IncomingTCPConnect
   }
 }
 
-/* this version is called from the backend code when a new response has been received */
-void IncomingTCPConnectionState::handleResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
+/* called from the backend code when a new response has been received */
+void IncomingTCPConnectionState::handleResponse(std::shared_ptr<IncomingTCPConnectionState> state, const struct timeval& now, TCPResponse&& response)
 {
-  // if we have added a TCP Proxy Protocol payload to a connection, don't release it yet, no one else will be able to use it anyway
-  if (!state->d_isXFR && response.d_connection && response.d_connection->isIdle() && response.d_connection->canBeReused()) {
-    auto& list = state->d_activeConnectionsToBackend.at(response.d_connection->getDS());
-    for (auto it = list.begin(); it != list.end(); ++it) {
-      if (*it == response.d_connection) {
-        DownstreamConnectionsManager::releaseDownstreamConnection(std::move(*it));
-        list.erase(it);
-        break;
+  if (!state->d_isXFR && response.d_connection && response.d_connection->isIdle()) {
+    // if we have added a TCP Proxy Protocol payload to a connection, don't release it to the general pool yet, no one else will be able to use it anyway
+    if (response.d_connection->canBeReused()) {
+      auto& list = state->d_activeConnectionsToBackend.at(response.d_connection->getDS());
+
+      for (auto it = list.begin(); it != list.end(); ++it) {
+        if (*it == response.d_connection) {
+          response.d_connection->release();
+          DownstreamConnectionsManager::releaseDownstreamConnection(std::move(*it));
+          list.erase(it);
+          break;
+        }
       }
     }
   }
@@ -471,14 +500,14 @@ void IncomingTCPConnectionState::handleResponse(std::shared_ptr<IncomingTCPConne
     }
   }
 
-  sendResponse(state, now, std::move(response));
+  sendOrQueueResponse(state, now, std::move(response));
 }
 
-static bool handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
+static IOState handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now)
 {
   if (state->d_querySize < sizeof(dnsheader)) {
     ++g_stats.nonCompliantQueries;
-    return true;
+    return IOState::NeedRead;
   }
 
   state->d_readingFirstQuery = false;
@@ -521,13 +550,12 @@ static bool handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
     response.d_buffer = std::move(*dnsCryptResponse);
     state->d_state = IncomingTCPConnectionState::State::idle;
     ++state->d_currentQueriesCount;
-    state->sendResponse(state, now, std::move(response));
-    return false;
+    return state->sendResponse(state, now, std::move(response));
   }
 
   const auto& dh = reinterpret_cast<dnsheader*>(query);
   if (!checkQueryHeaders(dh)) {
-    return true;
+    return IOState::NeedRead;
   }
 
   uint16_t qtype, qclass;
@@ -546,7 +574,7 @@ static bool handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
   auto result = processQuery(dq, *state->d_ci.cs, state->d_threadData.holders, ds);
 
   if (result == ProcessQueryResult::Drop) {
-    return true;
+    return IOState::Done;
   }
 
   if (result == ProcessQueryResult::SendAnswer) {
@@ -556,12 +584,11 @@ static bool handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
     response.d_buffer = std::move(state->d_buffer);
     state->d_state = IncomingTCPConnectionState::State::idle;
     ++state->d_currentQueriesCount;
-    state->sendResponse(state, now, std::move(response));
-    return false;
+    return state->sendResponse(state, now, std::move(response));
   }
 
   if (result != ProcessQueryResult::PassToBackend || ds == nullptr) {
-    return true;
+    return IOState::Done;
   }
 
   IDState ids;
@@ -613,7 +640,7 @@ static bool handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
   vinfolog("Got query for %s|%s from %s, relayed to %s", ids.qname.toLogString(), QType(ids.qtype).getName(), state->d_ci.remote.toStringWithPort(), ds->getName());
   downstreamConnection->queueQuery(TCPQuery(std::move(state->d_buffer), std::move(ids)), downstreamConnection);
 
-  return true;
+  return IOState::NeedRead;
 }
 
 void IncomingTCPConnectionState::handleIOCallback(int fd, FDMultiplexer::funcparam_t& param)
@@ -706,13 +733,13 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
         if (iostate == IOState::Done) {
           DEBUGLOG("query received");
 
-          if (handleQuery(state, now)) {
-            // if the query has been passed to a backend, or dropped, we can start
-            // reading again, or sending queued responses
+          iostate = handleQuery(state, now);
+          // if the query has been passed to a backend, or dropped, we can start
+          // reading again, or sending queued responses
+          if (iostate == IOState::NeedRead) {
             if (state->d_queuedResponses.empty()) {
               if (state->canAcceptNewQueries()) {
                 state->resetForNewQuery();
-                iostate = IOState::NeedRead;
               }
               else {
                 state->d_state = IncomingTCPConnectionState::State::idle;
@@ -724,16 +751,9 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
               state->d_queuedResponses.pop_front();
               ioGuard.release();
               state->d_state = IncomingTCPConnectionState::State::idle;
-              state->sendResponse(state, now, std::move(resp));
-              return;
+              iostate = sendResponse(state, now, std::move(resp));
             }
           }
-          else {
-            /* otherwise the state should already be waiting for
-               the socket to be writable */
-            ioGuard.release();
-            return;
-          }
         }
         else {
           wouldBlock = true;
@@ -745,7 +765,12 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
         iostate = state->d_handler.tryWrite(state->d_currentResponse.d_buffer, state->d_currentPos, state->d_currentResponse.d_buffer.size());
         if (iostate == IOState::Done) {
           DEBUGLOG("response sent");
-          iostate = handleResponseSent(state, now);
+          if (!handleResponseSent(state, now)) {
+            iostate = IOState::Done;
+          }
+          else {
+            iostate = sendQueuedResponses(state, now);
+          }
         } else {
           wouldBlock = true;
           DEBUGLOG("partial write");
@@ -795,7 +820,7 @@ void IncomingTCPConnectionState::handleIO(std::shared_ptr<IncomingTCPConnectionS
     }
     ioGuard.release();
   }
-  while (state->d_state == IncomingTCPConnectionState::State::readingQuerySize && iostate == IOState::NeedRead && !wouldBlock);
+  while ((iostate == IOState::NeedRead || iostate == IOState::NeedWrite) && !wouldBlock);
 }
 
 void IncomingTCPConnectionState::notifyIOError(std::shared_ptr<IncomingTCPConnectionState>& state, IDState&& query, const struct timeval& now)
@@ -810,7 +835,7 @@ void IncomingTCPConnectionState::notifyIOError(std::shared_ptr<IncomingTCPConnec
     TCPResponse resp = std::move(state->d_queuedResponses.front());
     state->d_queuedResponses.pop_front();
     state->d_state = IncomingTCPConnectionState::State::idle;
-    sendResponse(state, now, std::move(resp));
+    sendOrQueueResponse(state, now, std::move(resp));
   }
   else {
     // the backend code already tried to reconnect if it was possible
@@ -820,7 +845,7 @@ void IncomingTCPConnectionState::notifyIOError(std::shared_ptr<IncomingTCPConnec
 
 void IncomingTCPConnectionState::handleXFRResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response)
 {
-  sendResponse(state, now, std::move(response));
+  sendOrQueueResponse(state, now, std::move(response));
 }
 
 void IncomingTCPConnectionState::handleTimeout(std::shared_ptr<IncomingTCPConnectionState>& state, bool write)
diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
index 4fa155c55c..8b19b8d7de 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
@@ -23,7 +23,15 @@ void TCPConnectionToBackend::assignToClientConnection(std::shared_ptr<IncomingTC
   }
 }
 
-IOState TCPConnectionToBackend::sendNextQuery(std::shared_ptr<TCPConnectionToBackend>& conn)
+void TCPConnectionToBackend::release()
+{
+  d_clientConn.reset();
+  if (d_ioState) {
+    d_ioState.reset();
+  }
+}
+
+IOState TCPConnectionToBackend::queueNextQuery(std::shared_ptr<TCPConnectionToBackend>& conn)
 {
   conn->d_currentQuery = std::move(conn->d_pendingQueries.front());
   conn->d_pendingQueries.pop_front();
@@ -69,6 +77,38 @@ static IOState tryRead(int fd, std::vector<uint8_t>& buffer, size_t& pos, size_t
   return IOState::Done;
 }
 
+IOState TCPConnectionToBackend::sendQuery(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now)
+{
+  int fd = conn->d_socket->getHandle();
+  DEBUGLOG("sending query to backend "<<conn->getDS()->getName()<<" over FD "<<fd);
+  int socketFlags = 0;
+#ifdef MSG_FASTOPEN
+  if (conn->isFastOpenEnabled()) {
+    socketFlags |= MSG_FASTOPEN;
+  }
+#endif /* MSG_FASTOPEN */
+
+  size_t sent = sendMsgWithOptions(fd, reinterpret_cast<const char *>(&conn->d_currentQuery.d_buffer.at(conn->d_currentPos)), conn->d_currentQuery.d_buffer.size() - conn->d_currentPos, &conn->d_ds->remote, &conn->d_ds->sourceAddr, conn->d_ds->sourceItf, socketFlags);
+  if (sent == conn->d_currentQuery.d_buffer.size()) {
+    DEBUGLOG("query sent to backend");
+    /* request sent ! */
+    conn->incQueries();
+    conn->d_currentPos = 0;
+
+    DEBUGLOG("adding a pending response for ID "<<conn->d_currentQuery.d_idstate.origID<<" and QNAME "<<conn->d_currentQuery.d_idstate.qname);
+    conn->d_pendingResponses[conn->d_currentQuery.d_idstate.origID] = std::move(conn->d_currentQuery);
+    conn->d_currentQuery.d_buffer.clear();
+
+    return IOState::Done;
+  }
+  else {
+    conn->d_currentPos += sent;
+    /* disable fast open on partial write */
+    conn->disableFastOpen();
+    return IOState::NeedWrite;
+  }
+}
+
 void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now)
 {
   if (conn->d_socket == nullptr) {
@@ -82,40 +122,18 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
 
   try {
     if (conn->d_state == State::sendingQueryToBackend) {
-      DEBUGLOG("sending query to backend "<<conn->getDS()->getName()<<" over FD "<<fd);
-      int socketFlags = 0;
-#ifdef MSG_FASTOPEN
-      if (conn->isFastOpenEnabled()) {
-        socketFlags |= MSG_FASTOPEN;
+      iostate = sendQuery(conn, now);
+
+      while (iostate == IOState::Done && !conn->d_pendingQueries.empty()) {
+        queueNextQuery(conn);
+        iostate = sendQuery(conn, now);
       }
-#endif /* MSG_FASTOPEN */
 
-      size_t sent = sendMsgWithOptions(fd, reinterpret_cast<const char *>(&conn->d_currentQuery.d_buffer.at(conn->d_currentPos)), conn->d_currentQuery.d_buffer.size() - conn->d_currentPos, &conn->d_ds->remote, &conn->d_ds->sourceAddr, conn->d_ds->sourceItf, socketFlags);
-      if (sent == conn->d_currentQuery.d_buffer.size()) {
-        DEBUGLOG("query sent to backend");
-        /* request sent ! */
-        conn->incQueries();
+      if (iostate == IOState::Done && conn->d_pendingQueries.empty()) {
+        conn->d_state = State::readingResponseSizeFromBackend;
         conn->d_currentPos = 0;
-
-        DEBUGLOG("adding a pending response for ID "<<conn->d_currentQuery.d_idstate.origID<<" and QNAME "<<conn->d_currentQuery.d_idstate.qname);
-        conn->d_pendingResponses[conn->d_currentQuery.d_idstate.origID] = std::move(conn->d_currentQuery);
-        conn->d_currentQuery.d_buffer.clear();
-
-        if (conn->d_pendingQueries.empty()) {
-          conn->d_state = State::readingResponseSizeFromBackend;
-          conn->d_currentPos = 0;
-          conn->d_responseBuffer.resize(sizeof(uint16_t));
-          iostate = IOState::NeedRead;
-        }
-        else {
-          iostate = sendNextQuery(conn);
-        }
-      }
-      else {
-        conn->d_currentPos += sent;
-        iostate = IOState::NeedWrite;
-        /* disable fast open on partial write */
-        conn->disableFastOpen();
+        conn->d_responseBuffer.resize(sizeof(uint16_t));
+        iostate = IOState::NeedRead;
       }
     }
 
@@ -206,7 +224,11 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
           // resume sending query
         }
         else {
-          iostate = sendNextQuery(conn);
+          if (conn->d_pendingQueries.empty()) {
+            throw std::runtime_error("TCP connection to a backend in state " + std::to_string((int)conn->d_state) + " with no pending queries");
+          }
+
+          iostate = queueNextQuery(conn);
         }
 
         if (!conn->d_proxyProtocolPayloadAdded && !conn->d_proxyProtocolPayload.empty()) {
@@ -225,12 +247,15 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
     }
   }
 
-  if (iostate == IOState::Done) {
-    conn->d_ioState->update(iostate, handleIOCallback, conn);
-  }
-  else {
-    conn->d_ioState->update(iostate, handleIOCallback, conn, iostate == IOState::NeedRead ? conn->getBackendReadTTD(now) : conn->getBackendWriteTTD(now));
+  if (conn->d_ioState) {
+    if (iostate == IOState::Done) {
+      conn->d_ioState->update(iostate, handleIOCallback, conn);
+    }
+    else {
+      conn->d_ioState->update(iostate, handleIOCallback, conn, iostate == IOState::NeedRead ? conn->getBackendReadTTD(now) : conn->getBackendWriteTTD(now));
+    }
   }
+
   ioGuard.release();
 }
 
@@ -267,7 +292,8 @@ void TCPConnectionToBackend::queueQuery(TCPQuery&& query, std::shared_ptr<TCPCon
     struct timeval now;
     gettimeofday(&now, 0);
 
-    d_ioState->update(IOState::NeedWrite, handleIOCallback, sharedSelf, getBackendWriteTTD(now));
+    handleIO(sharedSelf, now);
+    // d_ioState->update(IOState::NeedWrite, handleIOCallback, sharedSelf, getBackendWriteTTD(now));
   }
   else {
     // store query in the list of queries to send
@@ -391,7 +417,7 @@ IOState TCPConnectionToBackend::handleResponse(std::shared_ptr<TCPConnectionToBa
   d_downstreamFailures = 0;
 
   auto& clientConn = d_clientConn;
-  if (!clientConn->active()) {
+  if (!clientConn || !clientConn->active()) {
     // a client timeout occured, or something like that */
     d_connectionDied = true;
     d_clientConn.reset();
@@ -431,6 +457,10 @@ IOState TCPConnectionToBackend::handleResponse(std::shared_ptr<TCPConnectionToBa
     auto ids = std::move(it->second.d_idstate);
     d_pendingResponses.erase(it);
     DEBUGLOG("passing response to client connection for "<<ids.qname);
+    /* marking as idle for now, so we can accept new queries if our queues are empty */
+    if (d_pendingQueries.empty() && d_pendingResponses.empty()) {
+      d_state = State::idle;
+    }
     clientConn->handleResponse(clientConn, now, TCPResponse(std::move(d_responseBuffer), std::move(ids), conn));
 
     if (!d_pendingQueries.empty()) {
diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
index c815364831..bc8d6d442d 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
@@ -128,7 +128,7 @@ public:
 
   bool isIdle() const
   {
-    return d_pendingQueries.size() == 0 && d_pendingResponses.size() == 0;
+    return d_state == State::idle && d_pendingQueries.size() == 0 && d_pendingResponses.size() == 0;
   }
 
   /* whether a connection can be reused for a different client */
@@ -158,13 +158,16 @@ public:
 
   void queueQuery(TCPQuery&& query, std::shared_ptr<TCPConnectionToBackend>& sharedSelf);
   void handleTimeout(const struct timeval& now, bool write);
+  void release();
+
   void setProxyProtocolPayload(std::string&& payload);
   void setProxyProtocolPayloadAdded(bool added);
 
 private:
   static void handleIO(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now);
   static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param);
-  static IOState sendNextQuery(std::shared_ptr<TCPConnectionToBackend>& conn);
+  static IOState queueNextQuery(std::shared_ptr<TCPConnectionToBackend>& conn);
+  static IOState sendQuery(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now);
 
   IOState handleResponse(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now);
   uint16_t getQueryIdFromResponse();
diff --git a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
index 704e197476..91b090ff9c 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
@@ -179,8 +179,11 @@ public:
   static void handleIO(std::shared_ptr<IncomingTCPConnectionState>& conn, const struct timeval& now);
   static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param);
   static void notifyIOError(std::shared_ptr<IncomingTCPConnectionState>& state, IDState&& query, const struct timeval& now);
-  static void sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
-  static void handleResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
+  static IOState sendResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
+  static void sendOrQueueResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
+
+  /* we take a copy of a shared pointer, not a reference, because the initial shared pointer might be released during the handling of the response */
+  static void handleResponse(std::shared_ptr<IncomingTCPConnectionState> state, const struct timeval& now, TCPResponse&& response);
   static void handleXFRResponse(std::shared_ptr<IncomingTCPConnectionState>& state, const struct timeval& now, TCPResponse&& response);
   static void handleTimeout(std::shared_ptr<IncomingTCPConnectionState>& state, bool write);
 
diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
index 8b19b8d7de..2864c706d6 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
@@ -99,6 +99,10 @@ IOState TCPConnectionToBackend::sendQuery(std::shared_ptr<TCPConnectionToBackend
     conn->d_pendingResponses[conn->d_currentQuery.d_idstate.origID] = std::move(conn->d_currentQuery);
     conn->d_currentQuery.d_buffer.clear();
 
+    if (!conn->d_usedForXFR) {
+      ++conn->d_ds->outstanding;
+    }
+
     return IOState::Done;
   }
   else {
@@ -200,6 +204,7 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
   }
 
   if (connectionDied) {
+
     bool reconnected = false;
     DEBUGLOG("connection died, number of failures is "<<conn->d_downstreamFailures<<", retries is "<<conn->d_ds->retries);
 
@@ -214,6 +219,9 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
         /* we need to resend the queries that were in flight, if any */
         for (auto& pending : conn->d_pendingResponses) {
           conn->d_pendingQueries.push_back(std::move(pending.second));
+          if (!conn->d_usedForXFR) {
+            --conn->d_ds->outstanding;
+          }
         }
         conn->d_pendingResponses.clear();
         conn->d_currentPos = 0;
@@ -383,6 +391,10 @@ void TCPConnectionToBackend::notifyAllQueriesFailed(const struct timeval& now, b
 {
   d_connectionDied = true;
 
+  if (!d_usedForXFR) {
+    d_ds->outstanding -= d_pendingResponses.size();
+  }
+
   auto& clientConn = d_clientConn;
   if (!clientConn->active()) {
     // a client timeout occured, or something like that */
@@ -421,6 +433,11 @@ IOState TCPConnectionToBackend::handleResponse(std::shared_ptr<TCPConnectionToBa
     // a client timeout occured, or something like that */
     d_connectionDied = true;
     d_clientConn.reset();
+
+    if (!conn->d_usedForXFR) {
+      --conn->d_ds->outstanding;
+    }
+
     return IOState::Done;
   }
 
@@ -454,6 +471,10 @@ IOState TCPConnectionToBackend::handleResponse(std::shared_ptr<TCPConnectionToBa
       return IOState::Done;
     }
 
+    if (!conn->d_usedForXFR) {
+      --conn->d_ds->outstanding;
+    }
+
     auto ids = std::move(it->second.d_idstate);
     d_pendingResponses.erase(it);
     DEBUGLOG("passing response to client connection for "<<ids.qname);

From 139c22c2af583189321789249301197925309056 Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Fri, 2 Oct 2020 15:44:38 +0200
Subject: [PATCH 22/23] dnsdist: Mark the TCP downstream connection as fresh
 after reconnecting

---
 pdns/dnsdistdist/dnsdist-tcp-downstream.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
index 2864c706d6..1d5271360a 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
@@ -321,6 +321,8 @@ bool TCPConnectionToBackend::reconnect()
     --d_ds->tcpCurrentConnections;
   }
 
+  d_fresh = true;
+
   do {
     vinfolog("TCP connecting to downstream %s (%d)", d_ds->getNameWithAddr(), d_downstreamFailures);
     DEBUGLOG("Opening TCP connection to backend "<<d_ds->getNameWithAddr());

From 4822504fe78b4009cfb321fe7300772230c37223 Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Fri, 2 Oct 2020 17:03:06 +0200
Subject: [PATCH 23/23] dnsdist: Fix tcpGaveUp and tcpDownstreamTimeout
 counters

---
 pdns/dnsdistdist/dnsdist-tcp-downstream.cc | 17 +++++++++++------
 pdns/dnsdistdist/dnsdist-tcp-downstream.hh | 10 ++++++----
 2 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
index 1d5271360a..8e63caf70e 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.cc
@@ -251,7 +251,8 @@ void TCPConnectionToBackend::handleIO(std::shared_ptr<TCPConnectionToBackend>& c
     if (!reconnected) {
       /* reconnect failed, we give up */
       DEBUGLOG("reconnect failed, we give up");
-      conn->notifyAllQueriesFailed(now);
+      ++conn->d_ds->tcpGaveUp;
+      conn->notifyAllQueriesFailed(now, FailureReason::gaveUp);
     }
   }
 
@@ -375,6 +376,7 @@ bool TCPConnectionToBackend::reconnect()
 
 void TCPConnectionToBackend::handleTimeout(const struct timeval& now, bool write)
 {
+  /* in some cases we could retry, here, reconnecting and sending our pending responses again */
   if (write) {
     ++d_ds->tcpWriteTimeouts;
   }
@@ -386,10 +388,10 @@ void TCPConnectionToBackend::handleTimeout(const struct timeval& now, bool write
     d_ioState->reset();
   }
 
-  notifyAllQueriesFailed(now, true);
+  notifyAllQueriesFailed(now, FailureReason::timeout);
 }
 
-void TCPConnectionToBackend::notifyAllQueriesFailed(const struct timeval& now, bool timeout)
+void TCPConnectionToBackend::notifyAllQueriesFailed(const struct timeval& now, FailureReason reason)
 {
   d_connectionDied = true;
 
@@ -404,9 +406,12 @@ void TCPConnectionToBackend::notifyAllQueriesFailed(const struct timeval& now, b
     return;
   }
 
-  if (timeout) {
+  if (reason == FailureReason::timeout) {
     ++clientConn->d_ci.cs->tcpDownstreamTimeouts;
   }
+  else if (reason == FailureReason::gaveUp) {
+    ++clientConn->d_ci.cs->tcpGaveUp;
+  }
 
   if (d_state == State::sendingQueryToBackend) {
     clientConn->notifyIOError(clientConn, std::move(d_currentQuery.d_idstate), now);
@@ -462,14 +467,14 @@ IOState TCPConnectionToBackend::handleResponse(std::shared_ptr<TCPConnectionToBa
     }
     catch (const std::exception& e) {
       DEBUGLOG("Unable to get query ID");
-      notifyAllQueriesFailed(now);
+      notifyAllQueriesFailed(now, FailureReason::unexpectedQueryID);
       throw;
     }
 
     auto it = d_pendingResponses.find(queryId);
     if (it == d_pendingResponses.end()) {
       DEBUGLOG("could not find any corresponding query for ID "<<queryId<<". This is likely a duplicated ID over the same TCP connection, giving up!");
-      notifyAllQueriesFailed(now);
+      notifyAllQueriesFailed(now, FailureReason::unexpectedQueryID);
       return IOState::Done;
     }
 
diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
index bc8d6d442d..fdfcfbbd86 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
@@ -164,6 +164,11 @@ public:
   void setProxyProtocolPayloadAdded(bool added);
 
 private:
+  /* waitingForResponseFromBackend is a state where we have not yet started reading the size,
+     so we can still switch to sending instead */
+  enum class State : uint8_t { idle, doingHandshake, sendingQueryToBackend, waitingForResponseFromBackend, readingResponseSizeFromBackend, readingResponseFromBackend };
+  enum class FailureReason : uint8_t { /* too many attempts */ gaveUp, timeout, unexpectedQueryID };
+
   static void handleIO(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now);
   static void handleIOCallback(int fd, FDMultiplexer::funcparam_t& param);
   static IOState queueNextQuery(std::shared_ptr<TCPConnectionToBackend>& conn);
@@ -172,7 +177,7 @@ private:
   IOState handleResponse(std::shared_ptr<TCPConnectionToBackend>& conn, const struct timeval& now);
   uint16_t getQueryIdFromResponse();
   bool reconnect();
-  void notifyAllQueriesFailed(const struct timeval& now, bool timeout = false);
+  void notifyAllQueriesFailed(const struct timeval& now, FailureReason reason);
 
   boost::optional<struct timeval> getBackendReadTTD(const struct timeval& now) const
   {
@@ -204,9 +209,6 @@ private:
     return res;
   }
 
-  /* waitingForResponseFromBackend is a state where we have not yet started reading the size,
-     so we can still switch to sending instead */
-  enum class State { idle, doingHandshake, sendingQueryToBackend, waitingForResponseFromBackend, readingResponseSizeFromBackend, readingResponseFromBackend };
   static const uint16_t s_xfrID;
 
   std::vector<uint8_t> d_responseBuffer;

Places

File 9582.patch of Package powerdns-dnsdist-test (Revision 28ae8741f8fc608fe90de5bea00e36bc)

Places