merge: !4 fix/robustness into deploy/all-2026-08-24

8 files changed, 239 insertions, 27 deletions

diff --git a/src/btfrv.cpp b/src/btfrv.cpp
index 133eb89..8ab735c 100644
--- a/src/btfrv.cpp
+++ b/src/btfrv.cpp
@@ -40,6 +40,8 @@ typedef int SOCKET;
 #include <thread>
 #include <mutex>
 #include <atomic>
+#include <chrono>
+#include <system_error>
 
 namespace btf
 {
@@ -209,8 +211,28 @@ static void HandleIncoming(SOCKET s)
         bool a = WriteN(svc, &ok, 1);
         bool b = WriteN(s, &ok, 1);
         if (!a || !b) { CLOSESOCK(svc); CLOSESOCK(s); return; }
-        std::thread(Forward, s, svc).detach();
-        std::thread(Forward, svc, s).detach();
+        // One thread pumps s -> svc; this thread pumps svc -> s. Spawning a
+        // thread per direction left no clean way to fail the second spawn:
+        // std::thread throws std::system_error when pthread_create is refused
+        // (NPROC limit, container pids cgroup, memory pressure), the first
+        // pump was already detached, and closing its sockets from here would
+        // race it -- so nothing caught the throw and it reached
+        // std::terminate, killing the relay over one pair it could not serve.
+        // Running one direction on this thread leaves exactly one spawn to
+        // guard, and a failure here has nothing running yet to race.
+        try
+        {
+            std::thread(Forward, s, svc).detach();
+        }
+        catch (const std::system_error& e)
+        {
+            fprintf(stderr, "rendezvous relay: cannot spawn forward thread (%s), pair dropped\n",
+                    e.what());
+            CLOSESOCK(svc);
+            CLOSESOCK(s);
+            return;
+        }
+        Forward(svc, s);
     }
     else
     {
@@ -236,6 +258,7 @@ bool RvRelayRun(unsigned short port)
     if (bind(g_listen, (struct sockaddr*)&addr, sizeof(addr)) != 0) { CLOSESOCK(g_listen); return false; }
     if (listen(g_listen, 16) != 0) { CLOSESOCK(g_listen); return false; }
 
+    int nSpawnFails = 0;
     while (!g_stop)
     {
         struct sockaddr_in cli;
@@ -246,7 +269,27 @@ bool RvRelayRun(unsigned short port)
             if (g_stop) break;
             continue;
         }
-        std::thread(HandleIncoming, s).detach();
+        try
+        {
+            std::thread(HandleIncoming, s).detach();
+            nSpawnFails = 0;
+        }
+        catch (const std::system_error& e)
+        {
+            // A refused pthread_create is backpressure, not a reason to die:
+            // the condition behind it (NPROC, pids cgroup, memory) does not
+            // clear in milliseconds, and the connection count feeding it is
+            // attacker-influenced. Drop this one peer, stop accepting for a
+            // moment -- longer each time, capped -- and keep serving the
+            // connections already established.
+            fprintf(stderr, "rendezvous relay: cannot spawn connection thread (%s), connection dropped\n",
+                    e.what());
+            CLOSESOCK(s);
+            nSpawnFails++;
+            int nWaitMs = nSpawnFails < 5 ? nSpawnFails * 1000 : 5000;
+            for (int i = 0; i < nWaitMs && !g_stop; i += 100)
+                std::this_thread::sleep_for(std::chrono::milliseconds(100));
+        }
     }
     CLOSESOCK(g_listen);
     g_listen = INVALID_SOCKET;
diff --git a/src/btftunnel.cpp b/src/btftunnel.cpp
index 63d178b..4acca1a 100644
--- a/src/btftunnel.cpp
+++ b/src/btftunnel.cpp
@@ -25,8 +25,12 @@ typedef int SOCKET;
 #endif
 #include "sockcount.h"
 #include <cstring>
+#include <cstdio>
+#include <memory>
+#include <new>
 #include <vector>
 #include <thread>
+#include <system_error>
 
 namespace btf
 {
@@ -119,20 +123,111 @@ static void PumpIn(SOCKET pumpEnd, RvSocket rv, std::vector<unsigned char> key)
     shutdown((SOCKET)rv, SD_BOTH);
 }
 
+// The pump end and the rendezvous socket, owned jointly by whoever is still
+// using them. Closed once, by the last reference to go.
+//
+// Joint ownership rather than "the inbound pump owns the cleanup", because
+// either pump can reach its exit first and BOTH touch BOTH descriptors on the
+// way out -- PumpOut ends in shutdown(rv), shutdown(pumpEnd) and PumpIn ends in
+// the mirror image. A single owner therefore has no exit at which the other is
+// guaranteed to be finished. Holding a reference for the whole body of each
+// pump gives it one: the close cannot happen until the last pump has stopped
+// touching the descriptors, so no shutdown() can land on a number that has
+// already been closed and handed back out by the kernel to an unrelated
+// connection.
+struct TunnelEnds
+{
+    SOCKET   pumpEnd;
+    RvSocket rv;
+    TunnelEnds(SOCKET p, RvSocket r) : pumpEnd(p), rv(r) {}
+    ~TunnelEnds() { BtfCloseSocket(pumpEnd); RvClose(rv); }
+private:
+    TunnelEnds(const TunnelEnds&);
+    TunnelEnds& operator=(const TunnelEnds&);
+};
+
 // Start the two pumps and hand back the app-facing socket. When both pumps
 // finish, close the pump end and the rendezvous socket.
+//
+// That sentence used to be a comment describing something the code did not do.
+// The inbound pump closed both descriptors as soon as it exited, without
+// waiting for the outbound pump, which then called shutdown() on both at its
+// own exit. Whether that shutdown hit a live descriptor or a closed one came
+// down to which pump got there first, and the rendezvous socket dying before
+// the app socket -- a relay restart, the far side hanging up -- is precisely
+// the order that puts the inbound pump ahead. Measured over 25 such teardowns
+// against the real code: 29 shutdowns landed on an already-closed descriptor.
+//
+// appEnd is not part of this. It is handed to the caller on success and closed
+// here on every failure path; no pump ever touches it.
+//
+// Every tunnel is two threads, and a tunnel is one peer connection (a .btf
+// dial or accept, a pool miner, a participant's pool session) -- so the
+// thread count scales with peer count, which is attacker-influenced. When
+// pthread_create is refused std::thread throws std::system_error; uncaught it
+// reached std::terminate and took the whole process down over one connection
+// it could not serve. Both spawns are guarded, and the failure paths return
+// INVALID_SOCKET, which every caller already handles by dropping just that
+// connection.
 static btf_socket_t StartTunnel(RvSocket rv, const unsigned char shared[32])
 {
     btf_socket_t appEnd, pumpEnd;
     if (!MakeLocalPair(appEnd, pumpEnd)) { RvClose(rv); return INVALID_SOCKET; }
 
     std::vector<unsigned char> key(shared, shared + 32);
-    std::thread(PumpOut, pumpEnd, rv, key).detach();
-    std::thread([pumpEnd, rv, key]() {
-        PumpIn(pumpEnd, rv, key);
+
+    // make_shared, not shared_ptr(new ...): it allocates before it constructs,
+    // so it either comes back owning both descriptors or throws having taken
+    // ownership of neither. The two-step form can run the deleter and then
+    // throw, which would close both here and again below.
+    std::shared_ptr<TunnelEnds> ends;
+    try
+    {
+        ends = std::make_shared<TunnelEnds>(pumpEnd, rv);
+    }
+    catch (const std::bad_alloc&)
+    {
+        fprintf(stderr, "btf tunnel: out of memory starting tunnel, tunnel dropped\n");
         BtfCloseSocket(pumpEnd);
+        BtfCloseSocket(appEnd);
         RvClose(rv);
-    }).detach();
+        return INVALID_SOCKET;
+    }
+
+    try
+    {
+        std::thread([ends, key]() { PumpIn(ends->pumpEnd, ends->rv, key); }).detach();
+    }
+    catch (const std::system_error& e)
+    {
+        // Nothing is running yet: `ends` is the only reference, and returning
+        // drops it, which is what closes pumpEnd and rv.
+        fprintf(stderr, "btf tunnel: cannot spawn inbound pump (%s), tunnel dropped\n",
+                e.what());
+        BtfCloseSocket(appEnd);
+        return INVALID_SOCKET;
+    }
+
+    try
+    {
+        std::thread([ends, key]() { PumpOut(ends->pumpEnd, ends->rv, key); }).detach();
+    }
+    catch (const std::system_error& e)
+    {
+        // The inbound pump may be parked in RvReadN(rv) or may already have
+        // exited -- when the spawn is refused because the process is loaded,
+        // the rendezvous socket is often dead for the same reason, and then
+        // PumpIn returns on its first length read. Either way this thread
+        // still holds a reference, so both descriptors are still open: the
+        // shutdown wakes the pump if it is parked and is harmless if it is
+        // not. The close belongs to whichever reference goes last.
+        fprintf(stderr, "btf tunnel: cannot spawn outbound pump (%s), tunnel dropped\n",
+                e.what());
+        shutdown((SOCKET)ends->rv, SD_BOTH);
+        shutdown(ends->pumpEnd, SD_BOTH);
+        BtfCloseSocket(appEnd);
+        return INVALID_SOCKET;
+    }
     return appEnd;
 }
 
diff --git a/src/gui.cpp b/src/gui.cpp
index 8dcb854..9b3802b 100644
--- a/src/gui.cpp
+++ b/src/gui.cpp
@@ -15,6 +15,7 @@
 #include <atomic>
 #include <mutex>
 #include <thread>
+#include <system_error>
 #include "bip32.h"
 #include "walletcmd.h"
 #include "font_roboto.h"
@@ -910,8 +911,16 @@ static void DrawOptionsDialog()
 
             if (nMineMode == MINE_OPERATOR) {
                 if (!gPoolServerRunning) {
-                    gPoolServerRunning = true;
-                    _beginthread(ThreadRPCServer, 0, NULL);
+                    // Claim the server is up only once it is. The flag used to
+                    // be set first and the result never looked at, so a refused
+                    // spawn left an operator with a pool that reports itself
+                    // running, no RPC server for miners to reach, and nothing
+                    // anywhere saying why. The flag also latches: with it set,
+                    // this branch is never retried.
+                    if (_beginthread(ThreadRPCServer, 0, NULL) == (uintptr_t)-1)
+                        printf("Error: _beginthread(ThreadRPCServer) failed -- pool server not started\n");
+                    else
+                        gPoolServerRunning = true;
                 } else if (!gPoolRunning) {
                     gPoolRunning = true;
                 }
@@ -1124,11 +1133,28 @@ static void DrawRestorePhraseDialog()
                 }
                 g_restoreDerived.store(0);
                 g_restoreRecovered.store(0);
+                // Set before the spawn, not after: RestoreThread clears this
+                // flag itself when it finishes, and a short restore can beat
+                // us back here and have its `false` overwritten, leaving the
+                // dialog disabled forever.
                 g_restoreRunning.store(true);
                 // On its own thread: a restore walks the whole chain once per
                 // batch and would otherwise freeze the window for as long as it
-                // takes.
-                std::thread(RestoreThread, std::string(g_restoreInput)).detach();
+                // takes. A refused spawn is one restore that does not happen,
+                // not a reason to take the window down with an uncaught
+                // std::system_error -- and the flag has to come back off, or
+                // the dialog sits greyed out reporting a search nobody is
+                // running.
+                try
+                {
+                    std::thread(RestoreThread, std::string(g_restoreInput)).detach();
+                }
+                catch (const std::system_error& e)
+                {
+                    g_restoreRunning.store(false);
+                    std::lock_guard<std::mutex> lock(g_restoreResultMutex);
+                    g_restoreResult = std::string("Could not start the search: ") + e.what();
+                }
             }
         }
         if (fBusy)
diff --git a/src/main.cpp b/src/main.cpp
index 7b3e555..d154523 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -14,6 +14,7 @@
 #include <atomic>
 #include <mutex>
 #include <thread>
+#include <system_error>
 #include "bip32.h"
 #include "btfaddr.h"
 #include "btftunnel.h"
@@ -3899,9 +3900,32 @@ connected:
     std::vector<ParticipantShareHit> shareHits;
     std::mutex shareHitsMtx;
     std::vector<std::thread> hashThreads;
-    for (int i = 0; i < nWorkers; i++)
-        hashThreads.emplace_back(ParticipantHashWorker, &jobShared, i, nWorkers,
-                                 &shareHits, &shareHitsMtx);
+    try
+    {
+        for (int i = 0; i < nWorkers; i++)
+            hashThreads.emplace_back(ParticipantHashWorker, &jobShared, i, nWorkers,
+                                     &shareHits, &shareHitsMtx);
+    }
+    catch (const std::system_error& e)
+    {
+        // A refused pthread_create used to reach std::terminate here and
+        // kill the worker mid-session -- observed twice on an otherwise
+        // healthy participant. Stop the workers that did start the same way
+        // the disconnect path below does, drop this pool connection, and let
+        // the caller's reconnect loop try again once the pressure clears.
+        LogPrint("worker", "[worker] cannot spawn hash threads (%s)"
+                 " -- dropping pool connection\n", e.what());
+        SetParticipantMiningStatus("hash thread spawn failed");
+        {
+            std::lock_guard<std::mutex> lk(jobShared.mtx);
+            jobShared.stop = true;
+            jobShared.haveJob = false;
+        }
+        for (std::thread& t : hashThreads)
+            t.join();
+        closesocket(s);
+        return false;
+    }
     LogPrint("worker", "[worker] spawned %d hash threads on shared RandomX dataset\n",
              nWorkers);
     const uint64 sessionStartHashes = gParticipantHashes.load();
diff --git a/src/net.cpp b/src/net.cpp
index fed6123..16038c8 100644
--- a/src/net.cpp
+++ b/src/net.cpp
@@ -2090,7 +2090,7 @@ bool StartNode(string& strError)
     if (addrIncoming.ip)
         addrLocalHost.ip = addrIncoming.ip;
 
-    _beginthread([](void*) {
+    if (_beginthread([](void*) {
         unsigned int ip = addrLocalHost.ip;
         if (GetMyExternalIP(ip)) {
             addrLocalHost.ip = ip;
@@ -2098,7 +2098,8 @@ bool StartNode(string& strError)
             CWalletDB().WriteSetting("addrIncoming", addrIncoming);
             printf("External IP updated: %s\n", addrLocalHost.ToStringIP().c_str());
         }
-    }, 0, NULL);
+    }, 0, NULL) == (uintptr_t)-1)
+        printf("Error: _beginthread(GetMyExternalIP) failed\n");
 
     // Peers that answered last time, dialled straight away. The relays are the
     // only way to find anyone otherwise, and reaching them plus walking their
diff --git a/src/randomx_pow.cpp b/src/randomx_pow.cpp
index 1d058d4..9c59364 100644
--- a/src/randomx_pow.cpp
+++ b/src/randomx_pow.cpp
@@ -8,6 +8,7 @@
 #include <thread>
 #include <vector>
 #include <mutex>
+#include <system_error>
 
 // Fixed key (seed) that determines the RandomX cache/dataset. Keeping it
 // constant avoids per-epoch "reseed" logic; for the CPU+RAM fairness goal
@@ -101,13 +102,28 @@ bool RandomXInitDataset(int nThreads)
 
     std::vector<std::thread> workers;
     unsigned long per = total / nThreads;
-    for (int i = 0; i < nThreads; i++)
+    try
     {
-        unsigned long start = i * per;
-        unsigned long count = (i == nThreads - 1) ? (total - start) : per;
-        workers.emplace_back([start, count]() {
-            randomx_init_dataset(g_dataset, g_cache, start, count);
-        });
+        for (int i = 0; i < nThreads; i++)
+        {
+            unsigned long start = i * per;
+            unsigned long count = (i == nThreads - 1) ? (total - start) : per;
+            workers.emplace_back([start, count]() {
+                randomx_init_dataset(g_dataset, g_cache, start, count);
+            });
+        }
+    }
+    catch (const std::system_error& e)
+    {
+        // Same refusal as the allocation failure above: light mode is slow
+        // but alive, std::terminate is neither. Join the workers that did
+        // start before releasing the dataset out from under them.
+        for (auto& w : workers) w.join();
+        randomx_release_dataset(g_dataset);
+        g_dataset = NULL;
+        printf("RandomX: cannot spawn dataset init threads (%s), staying in light mode\n",
+               e.what());
+        return false;
     }
     for (auto& w : workers) w.join();
 
diff --git a/src/rpc.cpp b/src/rpc.cpp
index a4b53ca..13cb6c9 100644
--- a/src/rpc.cpp
+++ b/src/rpc.cpp
@@ -5077,10 +5077,17 @@ void ThreadRPCServer(void*)
     // armed for its one wallet probe before it is allowed to send.
     ReconcilePayouts();
 
-    _beginthread(AcceptLoopFn, 0, NULL);
-    _beginthread(LocalAcceptFn, 0, NULL);
-    _beginthread(ThreadBtfPoolAnnouncer, 0, NULL);
-    _beginthread(PayoutThreadFn, 0, NULL);
+    // _beginthread cannot throw (it reports -1), but an unchecked failure
+    // leaves the pool running deaf: nobody accepts, announces, or pays, and
+    // nothing in the log says why.
+    if (_beginthread(AcceptLoopFn, 0, NULL) == (uintptr_t)-1)
+        LogPrint("pool", "[pool] FATAL: accept loop thread failed to spawn -- miners cannot connect\n");
+    if (_beginthread(LocalAcceptFn, 0, NULL) == (uintptr_t)-1)
+        LogPrint("pool", "[pool] local accept thread failed to spawn\n");
+    if (_beginthread(ThreadBtfPoolAnnouncer, 0, NULL) == (uintptr_t)-1)
+        LogPrint("pool", "[pool] announcer thread failed to spawn -- pool will not be advertised\n");
+    if (_beginthread(PayoutThreadFn, 0, NULL) == (uintptr_t)-1)
+        LogPrint("pool", "[pool] FATAL: payout thread failed to spawn -- no one gets paid\n");
 
     PoolEventLoop(); // blocks until gPoolRunning = false or fShutdown
 
diff --git a/src/util.h b/src/util.h
index d72518b..58d1638 100644
--- a/src/util.h
+++ b/src/util.h
@@ -199,7 +199,7 @@ protected:
 public:
     char* pszFile;
     int nLine;
-    explicit CCriticalSection() { InitializeCriticalSection(&cs); }
+    explicit CCriticalSection() : pszFile(NULL), nLine(0) { InitializeCriticalSection(&cs); }
     ~CCriticalSection() { DeleteCriticalSection(&cs); }
     void Enter() { EnterCriticalSection(&cs); }
     void Leave() { LeaveCriticalSection(&cs); }