From 0b566b4869b9e86c90162befd87f9882b6dd143c Mon Sep 17 00:00:00 2001 From: bitflash-maintainer Date: Wed, 2 Sep 2026 07:09:45 +0000 Subject: net: a refused thread spawn drops one peer, not the node std::thread throws std::system_error when pthread_create is refused (RLIMIT_NPROC, a container's pids cgroup, memory pressure), and nothing caught it, so the node died by std::terminate. Threads are spawned per peer connection, so the pressure that triggers this scales with peer count, which is attacker-influenced. Observed killing a healthy participant worker mid-mining. Every spawn site in the node build, and what a failure now does: - btftunnel.cpp: both tunnel pumps are guarded. The inbound pump starts first because it owns cleanup of the pump end and rendezvous socket; if the second spawn is refused, shutdown() wakes the first out of its read and it cleans up itself -- closing those descriptors from the failure path instead would race it onto reused descriptors. Both paths return INVALID_SOCKET, which every caller already treats as "this connection failed". - btfrv.cpp: the relay's accept loop drops the connection it cannot serve and stops accepting for a moment -- longer after consecutive failures, capped at 5s -- instead of dying at an arbitrary one. The per-pair second pump now runs on the handling thread itself, leaving exactly one spawn to guard and nothing running yet to race. - main.cpp: a participant that cannot spawn its hash workers stops the ones that did start, drops the pool connection, and returns to the reconnect loop, the same way an ordinary disconnect does. - randomx_pow.cpp: dataset init joins the workers it got, releases the dataset, and stays in light mode -- the same trade the allocation failure path above it already makes. - rpc.cpp, net.cpp: the remaining unchecked _beginthread calls (which report failure by value, they cannot throw) now say precisely what did not start. The pool's accept and payout threads failing silently left the pool running deaf -- nobody accepts, nobody gets paid, and nothing in the log says why. Left alone deliberately: the per-relay accept spawns (rpc.cpp) already check, log and retry on the next poll; startup-critical spawns in net.cpp keep failing hard; the selftest-only std::thread uses may abort a selftest run, which is acceptable where dying over one peer is not. Verified: full node rebuilds clean and links; pool-stratum selftest passes; a scratch harness under RLIMIT_NPROC=1 confirms _beginthread reports -1 and std::thread throws system_error, the premises of every path above. --- src/btfrv.cpp | 49 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 3 deletions(-) (limited to 'src/btfrv.cpp') diff --git a/src/btfrv.cpp b/src/btfrv.cpp index 133eb89..8ab735c 100644 --- a/src/btfrv.cpp +++ b/src/btfrv.cpp @@ -40,6 +40,8 @@ typedef int SOCKET; #include #include #include +#include +#include namespace btf { @@ -209,8 +211,28 @@ static void HandleIncoming(SOCKET s) bool a = WriteN(svc, &ok, 1); bool b = WriteN(s, &ok, 1); if (!a || !b) { CLOSESOCK(svc); CLOSESOCK(s); return; } - std::thread(Forward, s, svc).detach(); - std::thread(Forward, svc, s).detach(); + // One thread pumps s -> svc; this thread pumps svc -> s. Spawning a + // thread per direction left no clean way to fail the second spawn: + // std::thread throws std::system_error when pthread_create is refused + // (NPROC limit, container pids cgroup, memory pressure), the first + // pump was already detached, and closing its sockets from here would + // race it -- so nothing caught the throw and it reached + // std::terminate, killing the relay over one pair it could not serve. + // Running one direction on this thread leaves exactly one spawn to + // guard, and a failure here has nothing running yet to race. + try + { + std::thread(Forward, s, svc).detach(); + } + catch (const std::system_error& e) + { + fprintf(stderr, "rendezvous relay: cannot spawn forward thread (%s), pair dropped\n", + e.what()); + CLOSESOCK(svc); + CLOSESOCK(s); + return; + } + Forward(svc, s); } else { @@ -236,6 +258,7 @@ bool RvRelayRun(unsigned short port) if (bind(g_listen, (struct sockaddr*)&addr, sizeof(addr)) != 0) { CLOSESOCK(g_listen); return false; } if (listen(g_listen, 16) != 0) { CLOSESOCK(g_listen); return false; } + int nSpawnFails = 0; while (!g_stop) { struct sockaddr_in cli; @@ -246,7 +269,27 @@ bool RvRelayRun(unsigned short port) if (g_stop) break; continue; } - std::thread(HandleIncoming, s).detach(); + try + { + std::thread(HandleIncoming, s).detach(); + nSpawnFails = 0; + } + catch (const std::system_error& e) + { + // A refused pthread_create is backpressure, not a reason to die: + // the condition behind it (NPROC, pids cgroup, memory) does not + // clear in milliseconds, and the connection count feeding it is + // attacker-influenced. Drop this one peer, stop accepting for a + // moment -- longer each time, capped -- and keep serving the + // connections already established. + fprintf(stderr, "rendezvous relay: cannot spawn connection thread (%s), connection dropped\n", + e.what()); + CLOSESOCK(s); + nSpawnFails++; + int nWaitMs = nSpawnFails < 5 ? nSpawnFails * 1000 : 5000; + for (int i = 0; i < nWaitMs && !g_stop; i += 100) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } } CLOSESOCK(g_listen); g_listen = INVALID_SOCKET; -- cgit v1.2.3 From 7c70eb4498e88725241493033edfe89554ff1b18 Mon Sep 17 00:00:00 2001 From: bitflash-maintainer Date: Wed, 2 Sep 2026 09:03:01 +0000 Subject: tunnel: report which stage failed instead of guessing the cause --- src/btfrv.cpp | 16 ++++++++++++++-- src/btfrv.h | 19 +++++++++++++++++-- src/btftunnel.cpp | 37 +++++++++++++++++++++++++++++++++---- src/btftunnel.h | 25 +++++++++++++++++++++++-- src/main.cpp | 11 ++++++----- 5 files changed, 93 insertions(+), 15 deletions(-) (limited to 'src/btfrv.cpp') diff --git a/src/btfrv.cpp b/src/btfrv.cpp index 133eb89..864fe3d 100644 --- a/src/btfrv.cpp +++ b/src/btfrv.cpp @@ -334,17 +334,29 @@ bool RvServiceWaitPaired(RvSocket s, int nTimeoutSecs) } RvSocket RvClientConnect(const char* relay_host, unsigned short port, - const unsigned char target_pubkey[32]) + const unsigned char target_pubkey[32], + RvClientFail* pFail) { + if (pFail) *pFail = RVC_FAIL_DIAL; if (!RvInit()) return RV_INVALID; SOCKET s = ConnectTo(relay_host, port); if (s == INVALID_SOCKET) return RV_INVALID; + if (pFail) *pFail = RVC_FAIL_PAIR; unsigned char hdr[33]; hdr[0] = (unsigned char)'C'; memcpy(hdr + 1, target_pubkey, 32); if (!WriteN(s, hdr, 33)) { CLOSESOCK(s); return RV_INVALID; } unsigned char paired = 0; - if (!ReadN(s, &paired, 1) || paired != 0x01) { CLOSESOCK(s); return RV_INVALID; } + if (!ReadN(s, &paired, 1)) { CLOSESOCK(s); return RV_INVALID; } + if (paired != 0x01) + { + // The relay answered and declined: HandleIncoming writes 0x00 and + // closes when no service is parked under this pubkey right now. + if (pFail) *pFail = RVC_FAIL_REFUSED; + CLOSESOCK(s); + return RV_INVALID; + } + if (pFail) *pFail = RVC_FAIL_NONE; return (RvSocket)s; } diff --git a/src/btfrv.h b/src/btfrv.h index 9cd7476..1e85cb0 100644 --- a/src/btfrv.h +++ b/src/btfrv.h @@ -57,10 +57,25 @@ RvSocket RvServiceRegister(const char* relay_host, unsigned short port, bool RvServiceWaitPaired(RvSocket s, int nTimeoutSecs = 0); // ---- client side (the node dialing a .btf address) ---- +// Where a failed RvClientConnect stopped. RV_INVALID alone collapses three +// different situations -- relay unreachable, relay reachable but nobody +// registered, connection died mid-pairing -- and they point at different +// components. The caller can only report the right one if this side says +// which it saw. +enum RvClientFail +{ + RVC_FAIL_NONE = 0, + RVC_FAIL_DIAL, // TCP connect to the relay itself failed + RVC_FAIL_REFUSED, // relay answered 0x00: no service registered under this pubkey + RVC_FAIL_PAIR, // relay connection died before pairing completed +}; + // Connect through the relay to the service registered under `target_pubkey`. -// Returns the paired socket, or RV_INVALID on error. +// Returns the paired socket, or RV_INVALID on error -- with `pFail`, if +// given, naming the stage that failed. RvSocket RvClientConnect(const char* relay_host, unsigned short port, - const unsigned char target_pubkey[32]); + const unsigned char target_pubkey[32], + RvClientFail* pFail = 0); // Blocking read/write of exactly n bytes over a rendezvous socket. Return false // on error/EOF. diff --git a/src/btftunnel.cpp b/src/btftunnel.cpp index 63d178b..408476e 100644 --- a/src/btftunnel.cpp +++ b/src/btftunnel.cpp @@ -136,18 +136,47 @@ static btf_socket_t StartTunnel(RvSocket rv, const unsigned char shared[32]) return appEnd; } +const char* TunnelFailText(TunnelFail f) +{ + switch (f) + { + case TUNNEL_FAIL_NONE: return "no failure"; + case TUNNEL_FAIL_INIT: return "local socket/crypto init failed"; + case TUNNEL_FAIL_DIAL: return "meeting node unreachable (tcp dial failed)"; + case TUNNEL_FAIL_REFUSED: return "relay reachable, but no service registered there for this address"; + case TUNNEL_FAIL_PAIR: return "relay connection died before pairing completed"; + case TUNNEL_FAIL_HANDSHAKE: return "end-to-end handshake failed"; + case TUNNEL_FAIL_LOCAL: return "local tunnel setup failed"; + } + return "unknown failure"; +} + btf_socket_t BtfClientTunnel(const char* meeting_host, unsigned short port, const unsigned char target_pubkey[32], - const unsigned char service_enc_pub[32]) + const unsigned char service_enc_pub[32], + TunnelFail* pFail) { + if (pFail) *pFail = TUNNEL_FAIL_INIT; if (!TunnelInit()) return INVALID_SOCKET; - RvSocket rv = RvClientConnect(meeting_host, port, target_pubkey); - if (rv == RV_INVALID) return INVALID_SOCKET; + RvClientFail rvFail = RVC_FAIL_NONE; + RvSocket rv = RvClientConnect(meeting_host, port, target_pubkey, &rvFail); + if (rv == RV_INVALID) + { + if (pFail) + *pFail = rvFail == RVC_FAIL_DIAL ? TUNNEL_FAIL_DIAL + : rvFail == RVC_FAIL_REFUSED ? TUNNEL_FAIL_REFUSED + : TUNNEL_FAIL_PAIR; + return INVALID_SOCKET; + } unsigned char eph_pub[32], shared[32]; + if (pFail) *pFail = TUNNEL_FAIL_HANDSHAKE; if (!ChanClientHandshake(service_enc_pub, eph_pub, shared)) { RvClose(rv); return INVALID_SOCKET; } if (!RvWriteN(rv, eph_pub, 32)) { RvClose(rv); return INVALID_SOCKET; } - return StartTunnel(rv, shared); + if (pFail) *pFail = TUNNEL_FAIL_LOCAL; + btf_socket_t app = StartTunnel(rv, shared); + if (app != INVALID_SOCKET && pFail) *pFail = TUNNEL_FAIL_NONE; + return app; } btf_socket_t BtfServiceWrap(RvSocket rv, const unsigned char my_enc_sk[32]) diff --git a/src/btftunnel.h b/src/btftunnel.h index 24a7509..3cc87cd 100644 --- a/src/btftunnel.h +++ b/src/btftunnel.h @@ -36,12 +36,33 @@ namespace btf // Initialize (sockets + libsodium). Idempotent. bool TunnelInit(); +// Which stage of BtfClientTunnel failed. INVALID_SOCKET alone says only "no +// tunnel"; a caller that then guesses at causes in its log line sends whoever +// reads it to the wrong component (issue #17: "relay unreachable or pool not +// listening" blamed two healthy parts). The stage is what this side actually +// knows -- report that. +enum TunnelFail +{ + TUNNEL_FAIL_NONE = 0, + TUNNEL_FAIL_INIT, // local socket/crypto init failed + TUNNEL_FAIL_DIAL, // TCP connect to the meeting node failed + TUNNEL_FAIL_REFUSED, // relay reachable, but no service registered under the pubkey + TUNNEL_FAIL_PAIR, // relay connection died before pairing completed + TUNNEL_FAIL_HANDSHAKE, // end-to-end channel handshake failed + TUNNEL_FAIL_LOCAL, // local pump plumbing (socket pair / threads) failed +}; + +// Short description of a TunnelFail for log lines. +const char* TunnelFailText(TunnelFail f); + // Client: dial the service registered under `target_pubkey` at the meeting node // (host:port), box to its static x25519 `service_enc_pub`, and return a local -// plaintext SOCKET tunneled end-to-end to the service. INVALID_SOCKET on error. +// plaintext SOCKET tunneled end-to-end to the service. INVALID_SOCKET on error +// -- with `pFail`, if given, naming the stage that failed. btf_socket_t BtfClientTunnel(const char* meeting_host, unsigned short port, const unsigned char target_pubkey[32], - const unsigned char service_enc_pub[32]); + const unsigned char service_enc_pub[32], + TunnelFail* pFail = 0); // Service: wrap an already-paired rendezvous socket (RvServiceRegister followed // by RvServiceWaitPaired -- the pairing byte MUST already be consumed) with diff --git a/src/main.cpp b/src/main.cpp index 7b3e555..c0a8825 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -3758,13 +3758,14 @@ static bool PoolParticipantMiner() // Step 2b: open encrypted tunnel to pool // ------------------------------------------------------------------ SetParticipantMiningStatus("opening tunnel"); + btf::TunnelFail tunnelFail = btf::TUNNEL_FAIL_NONE; s = btf::BtfClientTunnel(host.c_str(), (unsigned short)port, - target_pubkey, service_enc_pub); + target_pubkey, service_enc_pub, &tunnelFail); if (s == INVALID_SOCKET) { - LogPrint("worker", "[worker] tunnel connect FAILED via %s" - " (relay unreachable or pool not listening)\n", - meetingHostPort.c_str()); - SetParticipantMiningStatus("tunnel connect failed"); + LogPrint("worker", "[worker] tunnel connect FAILED via %s: %s\n", + meetingHostPort.c_str(), btf::TunnelFailText(tunnelFail)); + SetParticipantMiningStatus(std::string("tunnel connect failed: ") + + btf::TunnelFailText(tunnelFail)); return false; } LogPrint("worker", "[worker] tunnel open via %s\n", meetingHostPort.c_str()); -- cgit v1.2.3