merge: !4 fix/robustness into deploy/all-2026-08-24

1 files changed, 99 insertions, 4 deletions

diff --git a/src/btftunnel.cpp b/src/btftunnel.cpp
index 63d178b..4acca1a 100644
--- a/src/btftunnel.cpp
+++ b/src/btftunnel.cpp
@@ -25,8 +25,12 @@ typedef int SOCKET;
 #endif
 #include "sockcount.h"
 #include <cstring>
+#include <cstdio>
+#include <memory>
+#include <new>
 #include <vector>
 #include <thread>
+#include <system_error>
 
 namespace btf
 {
@@ -119,20 +123,111 @@ static void PumpIn(SOCKET pumpEnd, RvSocket rv, std::vector<unsigned char> key)
     shutdown((SOCKET)rv, SD_BOTH);
 }
 
+// The pump end and the rendezvous socket, owned jointly by whoever is still
+// using them. Closed once, by the last reference to go.
+//
+// Joint ownership rather than "the inbound pump owns the cleanup", because
+// either pump can reach its exit first and BOTH touch BOTH descriptors on the
+// way out -- PumpOut ends in shutdown(rv), shutdown(pumpEnd) and PumpIn ends in
+// the mirror image. A single owner therefore has no exit at which the other is
+// guaranteed to be finished. Holding a reference for the whole body of each
+// pump gives it one: the close cannot happen until the last pump has stopped
+// touching the descriptors, so no shutdown() can land on a number that has
+// already been closed and handed back out by the kernel to an unrelated
+// connection.
+struct TunnelEnds
+{
+    SOCKET   pumpEnd;
+    RvSocket rv;
+    TunnelEnds(SOCKET p, RvSocket r) : pumpEnd(p), rv(r) {}
+    ~TunnelEnds() { BtfCloseSocket(pumpEnd); RvClose(rv); }
+private:
+    TunnelEnds(const TunnelEnds&);
+    TunnelEnds& operator=(const TunnelEnds&);
+};
+
 // Start the two pumps and hand back the app-facing socket. When both pumps
 // finish, close the pump end and the rendezvous socket.
+//
+// That sentence used to be a comment describing something the code did not do.
+// The inbound pump closed both descriptors as soon as it exited, without
+// waiting for the outbound pump, which then called shutdown() on both at its
+// own exit. Whether that shutdown hit a live descriptor or a closed one came
+// down to which pump got there first, and the rendezvous socket dying before
+// the app socket -- a relay restart, the far side hanging up -- is precisely
+// the order that puts the inbound pump ahead. Measured over 25 such teardowns
+// against the real code: 29 shutdowns landed on an already-closed descriptor.
+//
+// appEnd is not part of this. It is handed to the caller on success and closed
+// here on every failure path; no pump ever touches it.
+//
+// Every tunnel is two threads, and a tunnel is one peer connection (a .btf
+// dial or accept, a pool miner, a participant's pool session) -- so the
+// thread count scales with peer count, which is attacker-influenced. When
+// pthread_create is refused std::thread throws std::system_error; uncaught it
+// reached std::terminate and took the whole process down over one connection
+// it could not serve. Both spawns are guarded, and the failure paths return
+// INVALID_SOCKET, which every caller already handles by dropping just that
+// connection.
 static btf_socket_t StartTunnel(RvSocket rv, const unsigned char shared[32])
 {
     btf_socket_t appEnd, pumpEnd;
     if (!MakeLocalPair(appEnd, pumpEnd)) { RvClose(rv); return INVALID_SOCKET; }
 
     std::vector<unsigned char> key(shared, shared + 32);
-    std::thread(PumpOut, pumpEnd, rv, key).detach();
-    std::thread([pumpEnd, rv, key]() {
-        PumpIn(pumpEnd, rv, key);
+
+    // make_shared, not shared_ptr(new ...): it allocates before it constructs,
+    // so it either comes back owning both descriptors or throws having taken
+    // ownership of neither. The two-step form can run the deleter and then
+    // throw, which would close both here and again below.
+    std::shared_ptr<TunnelEnds> ends;
+    try
+    {
+        ends = std::make_shared<TunnelEnds>(pumpEnd, rv);
+    }
+    catch (const std::bad_alloc&)
+    {
+        fprintf(stderr, "btf tunnel: out of memory starting tunnel, tunnel dropped\n");
         BtfCloseSocket(pumpEnd);
+        BtfCloseSocket(appEnd);
         RvClose(rv);
-    }).detach();
+        return INVALID_SOCKET;
+    }
+
+    try
+    {
+        std::thread([ends, key]() { PumpIn(ends->pumpEnd, ends->rv, key); }).detach();
+    }
+    catch (const std::system_error& e)
+    {
+        // Nothing is running yet: `ends` is the only reference, and returning
+        // drops it, which is what closes pumpEnd and rv.
+        fprintf(stderr, "btf tunnel: cannot spawn inbound pump (%s), tunnel dropped\n",
+                e.what());
+        BtfCloseSocket(appEnd);
+        return INVALID_SOCKET;
+    }
+
+    try
+    {
+        std::thread([ends, key]() { PumpOut(ends->pumpEnd, ends->rv, key); }).detach();
+    }
+    catch (const std::system_error& e)
+    {
+        // The inbound pump may be parked in RvReadN(rv) or may already have
+        // exited -- when the spawn is refused because the process is loaded,
+        // the rendezvous socket is often dead for the same reason, and then
+        // PumpIn returns on its first length read. Either way this thread
+        // still holds a reference, so both descriptors are still open: the
+        // shutdown wakes the pump if it is parked and is harmless if it is
+        // not. The close belongs to whichever reference goes last.
+        fprintf(stderr, "btf tunnel: cannot spawn outbound pump (%s), tunnel dropped\n",
+                e.what());
+        shutdown((SOCKET)ends->rv, SD_BOTH);
+        shutdown(ends->pumpEnd, SD_BOTH);
+        BtfCloseSocket(appEnd);
+        return INVALID_SOCKET;
+    }
     return appEnd;
 }