From 0b566b4869b9e86c90162befd87f9882b6dd143c Mon Sep 17 00:00:00 2001 From: bitflash-maintainer Date: Wed, 2 Sep 2026 07:09:45 +0000 Subject: net: a refused thread spawn drops one peer, not the node std::thread throws std::system_error when pthread_create is refused (RLIMIT_NPROC, a container's pids cgroup, memory pressure), and nothing caught it, so the node died by std::terminate. Threads are spawned per peer connection, so the pressure that triggers this scales with peer count, which is attacker-influenced. Observed killing a healthy participant worker mid-mining. Every spawn site in the node build, and what a failure now does: - btftunnel.cpp: both tunnel pumps are guarded. The inbound pump starts first because it owns cleanup of the pump end and rendezvous socket; if the second spawn is refused, shutdown() wakes the first out of its read and it cleans up itself -- closing those descriptors from the failure path instead would race it onto reused descriptors. Both paths return INVALID_SOCKET, which every caller already treats as "this connection failed". - btfrv.cpp: the relay's accept loop drops the connection it cannot serve and stops accepting for a moment -- longer after consecutive failures, capped at 5s -- instead of dying at an arbitrary one. The per-pair second pump now runs on the handling thread itself, leaving exactly one spawn to guard and nothing running yet to race. - main.cpp: a participant that cannot spawn its hash workers stops the ones that did start, drops the pool connection, and returns to the reconnect loop, the same way an ordinary disconnect does. - randomx_pow.cpp: dataset init joins the workers it got, releases the dataset, and stays in light mode -- the same trade the allocation failure path above it already makes. - rpc.cpp, net.cpp: the remaining unchecked _beginthread calls (which report failure by value, they cannot throw) now say precisely what did not start. The pool's accept and payout threads failing silently left the pool running deaf -- nobody accepts, nobody gets paid, and nothing in the log says why. Left alone deliberately: the per-relay accept spawns (rpc.cpp) already check, log and retry on the next poll; startup-critical spawns in net.cpp keep failing hard; the selftest-only std::thread uses may abort a selftest run, which is acceptable where dying over one peer is not. Verified: full node rebuilds clean and links; pool-stratum selftest passes; a scratch harness under RLIMIT_NPROC=1 confirms _beginthread reports -1 and std::thread throws system_error, the premises of every path above. --- src/btfrv.cpp | 49 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) (limited to 'src/btfrv.cpp') diff --git a/src/btfrv.cpp b/src/btfrv.cpp index 133eb89..8ab735c 100644 --- a/src/btfrv.cpp +++ b/src/btfrv.cpp @@ -40,6 +40,8 @@ typedef int SOCKET; #include #include #include +#include +#include namespace btf { @@ -209,8 +211,28 @@ static void HandleIncoming(SOCKET s) bool a = WriteN(svc, &ok, 1); bool b = WriteN(s, &ok, 1); if (!a || !b) { CLOSESOCK(svc); CLOSESOCK(s); return; } - std::thread(Forward, s, svc).detach(); - std::thread(Forward, svc, s).detach(); + // One thread pumps s -> svc; this thread pumps svc -> s. Spawning a + // thread per direction left no clean way to fail the second spawn: + // std::thread throws std::system_error when pthread_create is refused + // (NPROC limit, container pids cgroup, memory pressure), the first + // pump was already detached, and closing its sockets from here would + // race it -- so nothing caught the throw and it reached + // std::terminate, killing the relay over one pair it could not serve. + // Running one direction on this thread leaves exactly one spawn to + // guard, and a failure here has nothing running yet to race. + try + { + std::thread(Forward, s, svc).detach(); + } + catch (const std::system_error& e) + { + fprintf(stderr, "rendezvous relay: cannot spawn forward thread (%s), pair dropped\n", + e.what()); + CLOSESOCK(svc); + CLOSESOCK(s); + return; + } + Forward(svc, s); } else { @@ -236,6 +258,7 @@ bool RvRelayRun(unsigned short port) if (bind(g_listen, (struct sockaddr*)&addr, sizeof(addr)) != 0) { CLOSESOCK(g_listen); return false; } if (listen(g_listen, 16) != 0) { CLOSESOCK(g_listen); return false; } + int nSpawnFails = 0; while (!g_stop) { struct sockaddr_in cli; @@ -246,7 +269,27 @@ bool RvRelayRun(unsigned short port) if (g_stop) break; continue; } - std::thread(HandleIncoming, s).detach(); + try + { + std::thread(HandleIncoming, s).detach(); + nSpawnFails = 0; + } + catch (const std::system_error& e) + { + // A refused pthread_create is backpressure, not a reason to die: + // the condition behind it (NPROC, pids cgroup, memory) does not + // clear in milliseconds, and the connection count feeding it is + // attacker-influenced. Drop this one peer, stop accepting for a + // moment -- longer each time, capped -- and keep serving the + // connections already established. + fprintf(stderr, "rendezvous relay: cannot spawn connection thread (%s), connection dropped\n", + e.what()); + CLOSESOCK(s); + nSpawnFails++; + int nWaitMs = nSpawnFails < 5 ? nSpawnFails * 1000 : 5000; + for (int i = 0; i < nWaitMs && !g_stop; i += 100) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } } CLOSESOCK(g_listen); g_listen = INVALID_SOCKET; -- cgit v1.2.3