From 51d617e70f90a972185917ea0096c1ecef2bfa8b Mon Sep 17 00:00:00 2001 From: Bitflash-sh Date: Fri, 24 Jul 2026 11:18:11 -0300 Subject: Bitflash: fair-launch release Satoshi's bitcoin-0.1.0 reborn with RandomX CPU proof of work, anonymous .btf addressing over Nostr with CGNAT rendezvous, automatic peer discovery, a headless Linux node, and Windows GUI wallet. Fair launch, no premine, 21M cap. --- src/btftunnel.cpp | 161 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 src/btftunnel.cpp (limited to 'src/btftunnel.cpp') diff --git a/src/btftunnel.cpp b/src/btftunnel.cpp new file mode 100644 index 0000000..65c4b89 --- /dev/null +++ b/src/btftunnel.cpp @@ -0,0 +1,161 @@ +// Copyright (c) 2026 Bitflash developers +// Distributed under the MIT/X11 software license. +// +// Transparent anonymous tunnel. See btftunnel.h. + +#include "btftunnel.h" +#include "btfchan.h" + +#ifdef _WIN32 +#include +#include +#else +#include +#include +#include +#include +typedef int SOCKET; +#ifndef INVALID_SOCKET +#define INVALID_SOCKET (-1) +#endif +#define closesocket(s) ::close(s) +#ifndef SD_BOTH +#define SD_BOTH SHUT_RDWR +#endif +#endif +#include +#include +#include + +namespace btf +{ + +bool TunnelInit() +{ + return RvInit() && ChanInit(); +} + +// Create a connected loopback socket pair (no socketpair() on Windows). +// Returns app end and pump end; both are ordinary full-duplex TCP sockets. +static bool MakeLocalPair(btf_socket_t& appEnd, btf_socket_t& pumpEnd) +{ + SOCKET listener = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (listener == INVALID_SOCKET) return false; + + struct sockaddr_in addr; + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = 0; // ephemeral + if (bind(listener, (struct sockaddr*)&addr, sizeof(addr)) != 0) { closesocket(listener); return false; } + if (listen(listener, 1) != 0) { closesocket(listener); return false; } + +#ifdef _WIN32 + int len = sizeof(addr); +#else + socklen_t len = sizeof(addr); +#endif + if (getsockname(listener, (struct sockaddr*)&addr, &len) != 0) { closesocket(listener); return false; } + + SOCKET a = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (a == INVALID_SOCKET) { closesocket(listener); return false; } + if (connect(a, (struct sockaddr*)&addr, sizeof(addr)) != 0) { closesocket(a); closesocket(listener); return false; } + + SOCKET b = accept(listener, NULL, NULL); + closesocket(listener); + if (b == INVALID_SOCKET) { closesocket(a); return false; } + + appEnd = a; + pumpEnd = b; + return true; +} + +static bool SockReadN(SOCKET s, void* buf, int n) +{ + char* p = (char*)buf; int off = 0; + while (off < n) { int r = recv(s, p + off, n - off, 0); if (r <= 0) return false; off += r; } + return true; +} +static bool SockWriteN(SOCKET s, const void* buf, int n) +{ + const char* p = (const char*)buf; int off = 0; + while (off < n) { int r = send(s, p + off, n - off, 0); if (r <= 0) return false; off += r; } + return true; +} + +// Outbound pump: plaintext from the app -> echan frame -> rendezvous socket. +static void PumpOut(SOCKET pumpEnd, RvSocket rv, std::vector key) +{ + char buf[16 * 1024]; + for (;;) + { + int r = recv(pumpEnd, buf, sizeof(buf), 0); + if (r <= 0) break; + std::vector frame = ChanEncryptFrame(key.data(), (const unsigned char*)buf, r); + if (!RvWriteN(rv, frame.data(), (int)frame.size())) break; + } + // Tear down: unblock the inbound pump and the app. + shutdown((SOCKET)rv, SD_BOTH); + shutdown(pumpEnd, SD_BOTH); +} + +// Inbound pump: echan frames from the rendezvous socket -> plaintext to the app. +static void PumpIn(SOCKET pumpEnd, RvSocket rv, std::vector key) +{ + for (;;) + { + unsigned char len[4]; + if (!RvReadN(rv, len, 4)) break; + size_t n = ((size_t)len[0]<<24)|((size_t)len[1]<<16)|((size_t)len[2]<<8)|len[3]; + if (n < (size_t)(CHAN_NONCEBYTES + CHAN_MACBYTES) || n > (1u<<20)) break; + std::vector body(n); + if (!RvReadN(rv, body.data(), (int)n)) break; + std::vector pt; + if (!ChanDecryptBody(key.data(), body.data(), (int)n, pt)) break; // auth fail -> drop + if (!pt.empty() && !SockWriteN(pumpEnd, pt.data(), (int)pt.size())) break; + } + shutdown(pumpEnd, SD_BOTH); + shutdown((SOCKET)rv, SD_BOTH); +} + +// Start the two pumps and hand back the app-facing socket. When both pumps +// finish, close the pump end and the rendezvous socket. +static btf_socket_t StartTunnel(RvSocket rv, const unsigned char shared[32]) +{ + btf_socket_t appEnd, pumpEnd; + if (!MakeLocalPair(appEnd, pumpEnd)) { RvClose(rv); return INVALID_SOCKET; } + + std::vector key(shared, shared + 32); + std::thread(PumpOut, pumpEnd, rv, key).detach(); + std::thread([pumpEnd, rv, key]() { + PumpIn(pumpEnd, rv, key); + closesocket(pumpEnd); + RvClose(rv); + }).detach(); + return appEnd; +} + +btf_socket_t BtfClientTunnel(const char* meeting_host, unsigned short port, + const unsigned char target_pubkey[32], + const unsigned char service_enc_pub[32]) +{ + if (!TunnelInit()) return INVALID_SOCKET; + RvSocket rv = RvClientConnect(meeting_host, port, target_pubkey); + if (rv == RV_INVALID) return INVALID_SOCKET; + + unsigned char eph_pub[32], shared[32]; + if (!ChanClientHandshake(service_enc_pub, eph_pub, shared)) { RvClose(rv); return INVALID_SOCKET; } + if (!RvWriteN(rv, eph_pub, 32)) { RvClose(rv); return INVALID_SOCKET; } + return StartTunnel(rv, shared); +} + +btf_socket_t BtfServiceWrap(RvSocket rv, const unsigned char my_enc_sk[32]) +{ + if (!TunnelInit()) { RvClose(rv); return INVALID_SOCKET; } + unsigned char eph_pub[32], shared[32]; + if (!RvReadN(rv, eph_pub, 32)) { RvClose(rv); return INVALID_SOCKET; } + if (!ChanEndpointHandshake(eph_pub, my_enc_sk, shared)) { RvClose(rv); return INVALID_SOCKET; } + return StartTunnel(rv, shared); +} + +} // namespace btf -- cgit v1.2.3 From 21c8bb63cb85f3001ddee9237b1d7e689675d293 Mon Sep 17 00:00:00 2001 From: Bitflash Date: Sat, 1 Aug 2026 13:10:23 -0300 Subject: diag: count sockets by where they were created, and whether the close took (#86) A node was found holding 1502 sockets in Windows bound state, one ephemeral port each, 9% of the machine range in nineteen hours. Reading the code did not find it: every connect() failure path here closes its socket. So count. Every socket is tagged at creation and untagged at close, across all ten sites, with a counter for closes that fail. Already rules out three explanations: nothing is created or closed behind the counter (untagged closes are zero), no close is failing, and the leaked sockets are not in the live set -- tracked live matches what the OS reports as established, while the bound pile is something the program already closed successfully. --- src/btfrv.cpp | 12 +++-- src/btftunnel.cpp | 23 ++++----- src/net.cpp | 43 ++++++++++++++--- src/net.h | 2 + src/nostr.cpp | 6 +-- src/rpc.cpp | 4 +- src/sockcount.h | 137 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 199 insertions(+), 28 deletions(-) create mode 100644 src/sockcount.h (limited to 'src/btftunnel.cpp') diff --git a/src/btfrv.cpp b/src/btfrv.cpp index 7efd38b..6ac3280 100644 --- a/src/btfrv.cpp +++ b/src/btfrv.cpp @@ -11,7 +11,7 @@ #ifdef _WIN32 #include #include -#define CLOSESOCK closesocket +#define CLOSESOCK BtfCloseSocket #else #include #include @@ -22,12 +22,14 @@ typedef int SOCKET; #define INVALID_SOCKET (-1) #define SOCKET_ERROR (-1) -#define CLOSESOCK ::close +#define CLOSESOCK BtfCloseSocket #ifndef SD_BOTH #define SD_BOTH SHUT_RDWR #endif #endif +#include "sockcount.h" + #include #include #include @@ -96,7 +98,7 @@ static SOCKET ConnectTo(const char* host, unsigned short port) hints.ai_socktype = SOCK_STREAM; char portstr[16]; sprintf(portstr, "%u", (unsigned)port); if (getaddrinfo(host, portstr, &hints, &res) != 0 || !res) return INVALID_SOCKET; - SOCKET s = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + SOCKET s = BtfSocketTag(socket(res->ai_family, res->ai_socktype, res->ai_protocol), SOCK_RV_DIAL); if (s == INVALID_SOCKET) { freeaddrinfo(res); return INVALID_SOCKET; } // Apply a short timeout only for the connect() and initial handshake so an @@ -201,7 +203,7 @@ bool RvRelayRun(unsigned short port) if (!RvInit()) return false; g_stop = false; - g_listen = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + g_listen = BtfSocketTag(socket(AF_INET, SOCK_STREAM, IPPROTO_TCP), SOCK_RV_LISTEN); if (g_listen == INVALID_SOCKET) return false; int yes = 1; setsockopt(g_listen, SOL_SOCKET, SO_REUSEADDR, (char*)&yes, sizeof(yes)); @@ -218,7 +220,7 @@ bool RvRelayRun(unsigned short port) { struct sockaddr_in cli; socklen_t len = sizeof(cli); - SOCKET s = accept(g_listen, (struct sockaddr*)&cli, &len); + SOCKET s = BtfSocketTag(accept(g_listen, (struct sockaddr*)&cli, &len), SOCK_RV_ACCEPT); if (s == INVALID_SOCKET) { if (g_stop) break; diff --git a/src/btftunnel.cpp b/src/btftunnel.cpp index 65c4b89..63d178b 100644 --- a/src/btftunnel.cpp +++ b/src/btftunnel.cpp @@ -23,6 +23,7 @@ typedef int SOCKET; #define SD_BOTH SHUT_RDWR #endif #endif +#include "sockcount.h" #include #include #include @@ -39,7 +40,7 @@ bool TunnelInit() // Returns app end and pump end; both are ordinary full-duplex TCP sockets. static bool MakeLocalPair(btf_socket_t& appEnd, btf_socket_t& pumpEnd) { - SOCKET listener = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + SOCKET listener = BtfSocketTag(socket(AF_INET, SOCK_STREAM, IPPROTO_TCP), SOCK_PAIR_LISTEN); if (listener == INVALID_SOCKET) return false; struct sockaddr_in addr; @@ -47,23 +48,23 @@ static bool MakeLocalPair(btf_socket_t& appEnd, btf_socket_t& pumpEnd) addr.sin_family = AF_INET; addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); addr.sin_port = 0; // ephemeral - if (bind(listener, (struct sockaddr*)&addr, sizeof(addr)) != 0) { closesocket(listener); return false; } - if (listen(listener, 1) != 0) { closesocket(listener); return false; } + if (bind(listener, (struct sockaddr*)&addr, sizeof(addr)) != 0) { BtfCloseSocket(listener); return false; } + if (listen(listener, 1) != 0) { BtfCloseSocket(listener); return false; } #ifdef _WIN32 int len = sizeof(addr); #else socklen_t len = sizeof(addr); #endif - if (getsockname(listener, (struct sockaddr*)&addr, &len) != 0) { closesocket(listener); return false; } + if (getsockname(listener, (struct sockaddr*)&addr, &len) != 0) { BtfCloseSocket(listener); return false; } - SOCKET a = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); - if (a == INVALID_SOCKET) { closesocket(listener); return false; } - if (connect(a, (struct sockaddr*)&addr, sizeof(addr)) != 0) { closesocket(a); closesocket(listener); return false; } + SOCKET a = BtfSocketTag(socket(AF_INET, SOCK_STREAM, IPPROTO_TCP), SOCK_PAIR_APP); + if (a == INVALID_SOCKET) { BtfCloseSocket(listener); return false; } + if (connect(a, (struct sockaddr*)&addr, sizeof(addr)) != 0) { BtfCloseSocket(a); BtfCloseSocket(listener); return false; } - SOCKET b = accept(listener, NULL, NULL); - closesocket(listener); - if (b == INVALID_SOCKET) { closesocket(a); return false; } + SOCKET b = BtfSocketTag(accept(listener, NULL, NULL), SOCK_PAIR_PUMP); + BtfCloseSocket(listener); + if (b == INVALID_SOCKET) { BtfCloseSocket(a); return false; } appEnd = a; pumpEnd = b; @@ -129,7 +130,7 @@ static btf_socket_t StartTunnel(RvSocket rv, const unsigned char shared[32]) std::thread(PumpOut, pumpEnd, rv, key).detach(); std::thread([pumpEnd, rv, key]() { PumpIn(pumpEnd, rv, key); - closesocket(pumpEnd); + BtfCloseSocket(pumpEnd); RvClose(rv); }).detach(); return appEnd; diff --git a/src/net.cpp b/src/net.cpp index 19893ee..7a13257 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -51,6 +51,33 @@ int64 nBlocksReceived = 0; int64 nBlocksWithoutParent = 0; static int64 nNodeStartTime = 0; +// --- Socket accounting (implementation lives in sockcount.h) -------------- +static const char* pszSockSite[SOCK_SITES] = { + "external-ip probe", "listen socket", "inbound accept", + "rendezvous dial", "rendezvous listen", "rendezvous accept", + "loopback pair listener", "loopback pair app end", "loopback pair pump end", + "nostr relay" +}; + +static string SockAccountingText() +{ + long long nOpened[SOCK_SITES], nClosed[SOCK_SITES], nFailed[SOCK_SITES], nUntagged = 0; + BtfSockSnapshot(nOpened, nClosed, nFailed, &nUntagged); + + string str = "\n sockets by where they were created\n"; + str += " site opened closed live close failed\n"; + for (int i = 0; i < SOCK_SITES; i++) + { + if (nOpened[i] == 0) + continue; + str += strprintf(" %-24s %8lld %8lld %8lld %13lld\n", pszSockSite[i], + nOpened[i], nClosed[i], nOpened[i] - nClosed[i], nFailed[i]); + } + str += strprintf(" %-24s %8s %8lld %8s %13s\n", "closed but never tagged", + "-", nUntagged, "-", "-"); + return str; +} + // Defined further down, next to the counter it reads. The miner keeps its own // live count -- threads bump it on the way in and on the way out -- which is // the honest number here: how many are hashing now, not how many were asked to. @@ -119,6 +146,8 @@ string GetDiagnosticsText() (RandomXFastReady() ? 2080 : 256) + 2 * nMining).c_str() : ", not mining"); + str += SockAccountingText(); + str += "\n peer dir height last recv last send unsent\n"; foreach(CNode* pnode, vCopy) { @@ -157,7 +186,7 @@ bool GetMyExternalIP(unsigned int& ipRet) if (getaddrinfo(svc.host, "80", &hints, &res) != 0 || !res) continue; - SOCKET hSocket = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + SOCKET hSocket = BtfSocketTag(socket(res->ai_family, res->ai_socktype, res->ai_protocol), SOCK_EXTIP); if (hSocket == INVALID_SOCKET) { freeaddrinfo(res); continue; } // 5-second timeout so a dead service doesn't stall startup @@ -173,7 +202,7 @@ bool GetMyExternalIP(unsigned int& ipRet) if (connect(hSocket, res->ai_addr, (int)res->ai_addrlen) != 0) { - freeaddrinfo(res); closesocket(hSocket); continue; + freeaddrinfo(res); BtfCloseSocket(hSocket); continue; } freeaddrinfo(res); @@ -190,7 +219,7 @@ bool GetMyExternalIP(unsigned int& ipRet) response += buf; if (response.size() > 4096) break; } - closesocket(hSocket); + BtfCloseSocket(hSocket); // Find blank line separating headers from body size_t bodyPos = response.find("\r\n\r\n"); @@ -950,7 +979,7 @@ void CNode::Disconnect() { LogPrint("net", "disconnecting node %s\n", addr.ToString().c_str()); - closesocket(hSocket); + BtfCloseSocket(hSocket); // All of a nodes broadcasts and subscriptions are automatically torn down // when it goes down, so a node has to stay up to keep its broadcast going. @@ -1230,7 +1259,7 @@ void ThreadSocketHandler2(void* parg) #else socklen_t len = sizeof(sockaddr); #endif - SOCKET hSocket = accept(hListenSocket, (struct sockaddr*)&sockaddr, &len); + SOCKET hSocket = BtfSocketTag(accept(hListenSocket, (struct sockaddr*)&sockaddr, &len), SOCK_ACCEPT); CAddress addr(sockaddr); if (hSocket == INVALID_SOCKET) { @@ -1251,7 +1280,7 @@ void ThreadSocketHandler2(void* parg) { LogPrint("net", "refusing connection from %s, already at %u\n", addr.ToString().c_str(), nNodes); - closesocket(hSocket); + BtfCloseSocket(hSocket); } else { @@ -1639,7 +1668,7 @@ bool StartNode(string& strError) printf("addrLocalHost = %s\n", addrLocalHost.ToString().c_str()); // Create socket for listening for incoming connections - SOCKET hListenSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + SOCKET hListenSocket = BtfSocketTag(socket(AF_INET, SOCK_STREAM, IPPROTO_TCP), SOCK_LISTEN); if (hListenSocket == INVALID_SOCKET) { strError = strprintf("Error: Couldn't open socket for incoming connections (socket returned error %d)", WSAGetLastError()); diff --git a/src/net.h b/src/net.h index bd4eb77..314c709 100644 --- a/src/net.h +++ b/src/net.h @@ -121,6 +121,8 @@ extern int64 nBlocksWithoutParent; // ...of those, how many arrived orphaned // user pastes into an issue is what a developer already knows how to read. string GetDiagnosticsText(); +#include "sockcount.h" + // Defined in main.cpp. Declared here because CNode announces it in the version // message, and net.h is included before main.h. extern int nBestHeight; diff --git a/src/nostr.cpp b/src/nostr.cpp index 17139ad..dc96be2 100644 --- a/src/nostr.cpp +++ b/src/nostr.cpp @@ -462,7 +462,7 @@ public: { if (ssl) { SSL_shutdown(ssl); SSL_free(ssl); ssl = NULL; } if (sslctx) { SSL_CTX_free(sslctx); sslctx = NULL; } - if (hSocket != INVALID_SOCKET) { closesocket(hSocket); hSocket = INVALID_SOCKET; } + if (hSocket != INVALID_SOCKET) { BtfCloseSocket(hSocket); hSocket = INVALID_SOCKET; } } int RawRead(char* buf, int len) @@ -518,7 +518,7 @@ public: bool fConnected = false; for (rp = res; rp != NULL; rp = rp->ai_next) { - hSocket = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); + hSocket = BtfSocketTag(socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol), SOCK_NOSTR); if (hSocket == INVALID_SOCKET) continue; @@ -542,7 +542,7 @@ public: break; } - closesocket(hSocket); + BtfCloseSocket(hSocket); hSocket = INVALID_SOCKET; } freeaddrinfo(res); diff --git a/src/rpc.cpp b/src/rpc.cpp index 97d8429..41c0328 100644 --- a/src/rpc.cpp +++ b/src/rpc.cpp @@ -46,10 +46,10 @@ using json = nlohmann::json; #ifdef _WIN32 -#define sock_close(s) closesocket(s) +#define sock_close(s) BtfCloseSocket(s) #define SEND_FLAGS 0 #else -#define sock_close(s) ::close(s) +#define sock_close(s) BtfCloseSocket(s) #define SEND_FLAGS MSG_NOSIGNAL #endif diff --git a/src/sockcount.h b/src/sockcount.h new file mode 100644 index 0000000..19f0827 --- /dev/null +++ b/src/sockcount.h @@ -0,0 +1,137 @@ +// Copyright (c) 2026 Bitflash developers +// Distributed under the MIT/X11 software license. +// +// Socket accounting -- which part of the program is holding the sockets. +// +// A node on a 32-core machine was found with 1502 sockets in Windows' "bound" +// state: created, never connected, never closed, each holding a distinct +// ephemeral port. That is 9% of the machine's whole dynamic port range gone in +// nineteen hours, and when the range runs out nothing on that machine can open +// an outbound connection -- not just this program. +// +// Reading the code did not find it. Every connect() failure path in this tree +// closes its socket. So count instead: tag each socket where it is created, +// untag it where it is closed, and report the difference per site. Whichever +// site's live count tracks the leak is the answer. +// +// Header-only on purpose. btfrv.cpp is compiled a second time, on its own, into +// the standalone rendezvous relay -- a separate tree that does not link net.cpp. +// Anything here that needed a .cpp would break that build, and it would break +// it on the VPS rather than on this machine. Include after the platform socket +// headers; each of those files defines SOCKET for itself. + +#ifndef BITFLASH_SOCKCOUNT_H +#define BITFLASH_SOCKCOUNT_H + +#include +#include + +#ifndef _WIN32 +#include +#endif + +enum +{ + SOCK_EXTIP, // GetMyExternalIP probe + SOCK_LISTEN, // the node's own listening socket + SOCK_ACCEPT, // inbound connections + SOCK_RV_DIAL, // outbound to a rendezvous relay + SOCK_RV_LISTEN, // rendezvous listener + SOCK_RV_ACCEPT, // inbound at the rendezvous + SOCK_PAIR_LISTEN, // loopback pair: the temporary listener + SOCK_PAIR_APP, // loopback pair: the app end handed to CNode + SOCK_PAIR_PUMP, // loopback pair: the pump end + SOCK_NOSTR, // outbound to a Nostr relay + SOCK_SITES +}; + +struct BtfSockAccount +{ + std::mutex mtx; + std::map mapSite; + long long nOpened[SOCK_SITES]; + long long nClosed[SOCK_SITES]; + long long nCloseFailed[SOCK_SITES]; + long long nClosedUntagged; + BtfSockAccount() : nClosedUntagged(0) + { + for (int i = 0; i < SOCK_SITES; i++) { nOpened[i] = 0; nClosed[i] = 0; nCloseFailed[i] = 0; } + } +}; + +// One instance across every translation unit: a function-local static in an +// inline function is the same object everywhere it is used. +inline BtfSockAccount& BtfSockAccounting() +{ + static BtfSockAccount account; + return account; +} + +// Record a freshly created socket and return it unchanged, so call sites read +// as they did before. INVALID_SOCKET passes through untouched. +inline SOCKET BtfSocketTag(SOCKET hSocket, int nSite) +{ + if (hSocket == INVALID_SOCKET || nSite < 0 || nSite >= SOCK_SITES) + return hSocket; + BtfSockAccount& a = BtfSockAccounting(); + std::lock_guard lock(a.mtx); + a.mapSite[hSocket] = nSite; + a.nOpened[nSite]++; + return hSocket; +} + +// Close a socket and forget it. Closing one nobody tagged is counted too -- +// that number staying at zero is what says the accounting is complete. +inline int BtfCloseSocket(SOCKET hSocket) +{ + if (hSocket == INVALID_SOCKET) + return 0; + int nSite = -1; + { + BtfSockAccount& a = BtfSockAccounting(); + std::lock_guard lock(a.mtx); + std::map::iterator mi = a.mapSite.find(hSocket); + if (mi == a.mapSite.end()) + a.nClosedUntagged++; + else + { + nSite = mi->second; + a.nClosed[nSite]++; + a.mapSite.erase(mi); + } + } +#ifdef _WIN32 + int nRet = ::closesocket(hSocket); +#else + int nRet = ::close(hSocket); +#endif + // A close that fails leaves the descriptor -- and on Windows the ephemeral + // port it holds -- in place. Counted because a socket the program believes + // it released and the operating system still shows is exactly the shape of + // the leak being chased, and nothing in this codebase has ever looked at + // what closesocket returns. + if (nRet != 0 && nSite >= 0) + { + BtfSockAccount& a = BtfSockAccounting(); + std::lock_guard lock(a.mtx); + a.nCloseFailed[nSite]++; + } + return nRet; +} + +// Copy the counters out. Formatting lives in net.cpp, which has strprintf -- +// and which the relay build does not compile. +inline void BtfSockSnapshot(long long* pOpened, long long* pClosed, long long* pFailed, long long* pUntagged) +{ + BtfSockAccount& a = BtfSockAccounting(); + std::lock_guard lock(a.mtx); + for (int i = 0; i < SOCK_SITES; i++) + { + pOpened[i] = a.nOpened[i]; + pClosed[i] = a.nClosed[i]; + pFailed[i] = a.nCloseFailed[i]; + } + *pUntagged = a.nClosedUntagged; +} + +#endif -- cgit v1.2.3 From 0b566b4869b9e86c90162befd87f9882b6dd143c Mon Sep 17 00:00:00 2001 From: bitflash-maintainer Date: Wed, 2 Sep 2026 07:09:45 +0000 Subject: net: a refused thread spawn drops one peer, not the node std::thread throws std::system_error when pthread_create is refused (RLIMIT_NPROC, a container's pids cgroup, memory pressure), and nothing caught it, so the node died by std::terminate. Threads are spawned per peer connection, so the pressure that triggers this scales with peer count, which is attacker-influenced. Observed killing a healthy participant worker mid-mining. Every spawn site in the node build, and what a failure now does: - btftunnel.cpp: both tunnel pumps are guarded. The inbound pump starts first because it owns cleanup of the pump end and rendezvous socket; if the second spawn is refused, shutdown() wakes the first out of its read and it cleans up itself -- closing those descriptors from the failure path instead would race it onto reused descriptors. Both paths return INVALID_SOCKET, which every caller already treats as "this connection failed". - btfrv.cpp: the relay's accept loop drops the connection it cannot serve and stops accepting for a moment -- longer after consecutive failures, capped at 5s -- instead of dying at an arbitrary one. The per-pair second pump now runs on the handling thread itself, leaving exactly one spawn to guard and nothing running yet to race. - main.cpp: a participant that cannot spawn its hash workers stops the ones that did start, drops the pool connection, and returns to the reconnect loop, the same way an ordinary disconnect does. - randomx_pow.cpp: dataset init joins the workers it got, releases the dataset, and stays in light mode -- the same trade the allocation failure path above it already makes. - rpc.cpp, net.cpp: the remaining unchecked _beginthread calls (which report failure by value, they cannot throw) now say precisely what did not start. The pool's accept and payout threads failing silently left the pool running deaf -- nobody accepts, nobody gets paid, and nothing in the log says why. Left alone deliberately: the per-relay accept spawns (rpc.cpp) already check, log and retry on the next poll; startup-critical spawns in net.cpp keep failing hard; the selftest-only std::thread uses may abort a selftest run, which is acceptable where dying over one peer is not. Verified: full node rebuilds clean and links; pool-stratum selftest passes; a scratch harness under RLIMIT_NPROC=1 confirms _beginthread reports -1 and std::thread throws system_error, the premises of every path above. --- src/btfrv.cpp | 49 ++++++++++++++++++++++++++++++++++++++++++++++--- src/btftunnel.cpp | 52 ++++++++++++++++++++++++++++++++++++++++++++++++---- src/main.cpp | 30 +++++++++++++++++++++++++++--- src/net.cpp | 5 +++-- src/randomx_pow.cpp | 28 ++++++++++++++++++++++------ src/rpc.cpp | 15 +++++++++++---- 6 files changed, 157 insertions(+), 22 deletions(-) (limited to 'src/btftunnel.cpp') diff --git a/src/btfrv.cpp b/src/btfrv.cpp index 133eb89..8ab735c 100644 --- a/src/btfrv.cpp +++ b/src/btfrv.cpp @@ -40,6 +40,8 @@ typedef int SOCKET; #include #include #include +#include +#include namespace btf { @@ -209,8 +211,28 @@ static void HandleIncoming(SOCKET s) bool a = WriteN(svc, &ok, 1); bool b = WriteN(s, &ok, 1); if (!a || !b) { CLOSESOCK(svc); CLOSESOCK(s); return; } - std::thread(Forward, s, svc).detach(); - std::thread(Forward, svc, s).detach(); + // One thread pumps s -> svc; this thread pumps svc -> s. Spawning a + // thread per direction left no clean way to fail the second spawn: + // std::thread throws std::system_error when pthread_create is refused + // (NPROC limit, container pids cgroup, memory pressure), the first + // pump was already detached, and closing its sockets from here would + // race it -- so nothing caught the throw and it reached + // std::terminate, killing the relay over one pair it could not serve. + // Running one direction on this thread leaves exactly one spawn to + // guard, and a failure here has nothing running yet to race. + try + { + std::thread(Forward, s, svc).detach(); + } + catch (const std::system_error& e) + { + fprintf(stderr, "rendezvous relay: cannot spawn forward thread (%s), pair dropped\n", + e.what()); + CLOSESOCK(svc); + CLOSESOCK(s); + return; + } + Forward(svc, s); } else { @@ -236,6 +258,7 @@ bool RvRelayRun(unsigned short port) if (bind(g_listen, (struct sockaddr*)&addr, sizeof(addr)) != 0) { CLOSESOCK(g_listen); return false; } if (listen(g_listen, 16) != 0) { CLOSESOCK(g_listen); return false; } + int nSpawnFails = 0; while (!g_stop) { struct sockaddr_in cli; @@ -246,7 +269,27 @@ bool RvRelayRun(unsigned short port) if (g_stop) break; continue; } - std::thread(HandleIncoming, s).detach(); + try + { + std::thread(HandleIncoming, s).detach(); + nSpawnFails = 0; + } + catch (const std::system_error& e) + { + // A refused pthread_create is backpressure, not a reason to die: + // the condition behind it (NPROC, pids cgroup, memory) does not + // clear in milliseconds, and the connection count feeding it is + // attacker-influenced. Drop this one peer, stop accepting for a + // moment -- longer each time, capped -- and keep serving the + // connections already established. + fprintf(stderr, "rendezvous relay: cannot spawn connection thread (%s), connection dropped\n", + e.what()); + CLOSESOCK(s); + nSpawnFails++; + int nWaitMs = nSpawnFails < 5 ? nSpawnFails * 1000 : 5000; + for (int i = 0; i < nWaitMs && !g_stop; i += 100) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } } CLOSESOCK(g_listen); g_listen = INVALID_SOCKET; diff --git a/src/btftunnel.cpp b/src/btftunnel.cpp index 63d178b..2b3f818 100644 --- a/src/btftunnel.cpp +++ b/src/btftunnel.cpp @@ -25,8 +25,10 @@ typedef int SOCKET; #endif #include "sockcount.h" #include +#include #include #include +#include namespace btf { @@ -121,18 +123,60 @@ static void PumpIn(SOCKET pumpEnd, RvSocket rv, std::vector key) // Start the two pumps and hand back the app-facing socket. When both pumps // finish, close the pump end and the rendezvous socket. +// +// Every tunnel is two threads, and a tunnel is one peer connection (a .btf +// dial or accept, a pool miner, a participant's pool session) -- so the +// thread count scales with peer count, which is attacker-influenced. When +// pthread_create is refused std::thread throws std::system_error; uncaught it +// reached std::terminate and took the whole process down over one connection +// it could not serve. Both spawns are guarded, and the failure paths return +// INVALID_SOCKET, which every caller already handles by dropping just that +// connection. static btf_socket_t StartTunnel(RvSocket rv, const unsigned char shared[32]) { btf_socket_t appEnd, pumpEnd; if (!MakeLocalPair(appEnd, pumpEnd)) { RvClose(rv); return INVALID_SOCKET; } std::vector key(shared, shared + 32); - std::thread(PumpOut, pumpEnd, rv, key).detach(); - std::thread([pumpEnd, rv, key]() { - PumpIn(pumpEnd, rv, key); + + // The inbound pump starts first because it owns the cleanup: it closes + // pumpEnd and rv when it exits. That ownership is what makes the second + // spawn's failure path below race-free. + try + { + std::thread([pumpEnd, rv, key]() { + PumpIn(pumpEnd, rv, key); + BtfCloseSocket(pumpEnd); + RvClose(rv); + }).detach(); + } + catch (const std::system_error& e) + { + // Nothing is running yet, so closing everything here is safe. + fprintf(stderr, "btf tunnel: cannot spawn inbound pump (%s), tunnel dropped\n", + e.what()); BtfCloseSocket(pumpEnd); + BtfCloseSocket(appEnd); RvClose(rv); - }).detach(); + return INVALID_SOCKET; + } + + try + { + std::thread(PumpOut, pumpEnd, rv, key).detach(); + } + catch (const std::system_error& e) + { + // The inbound pump is parked in RvReadN(rv). shutdown() wakes it, and + // it closes pumpEnd and rv itself -- closing them here instead would + // race exactly that, and a reused descriptor is how an unrelated + // socket gets closed. + fprintf(stderr, "btf tunnel: cannot spawn outbound pump (%s), tunnel dropped\n", + e.what()); + shutdown((SOCKET)rv, SD_BOTH); + BtfCloseSocket(appEnd); + return INVALID_SOCKET; + } return appEnd; } diff --git a/src/main.cpp b/src/main.cpp index 7b3e555..d154523 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include "bip32.h" #include "btfaddr.h" #include "btftunnel.h" @@ -3899,9 +3900,32 @@ connected: std::vector shareHits; std::mutex shareHitsMtx; std::vector hashThreads; - for (int i = 0; i < nWorkers; i++) - hashThreads.emplace_back(ParticipantHashWorker, &jobShared, i, nWorkers, - &shareHits, &shareHitsMtx); + try + { + for (int i = 0; i < nWorkers; i++) + hashThreads.emplace_back(ParticipantHashWorker, &jobShared, i, nWorkers, + &shareHits, &shareHitsMtx); + } + catch (const std::system_error& e) + { + // A refused pthread_create used to reach std::terminate here and + // kill the worker mid-session -- observed twice on an otherwise + // healthy participant. Stop the workers that did start the same way + // the disconnect path below does, drop this pool connection, and let + // the caller's reconnect loop try again once the pressure clears. + LogPrint("worker", "[worker] cannot spawn hash threads (%s)" + " -- dropping pool connection\n", e.what()); + SetParticipantMiningStatus("hash thread spawn failed"); + { + std::lock_guard lk(jobShared.mtx); + jobShared.stop = true; + jobShared.haveJob = false; + } + for (std::thread& t : hashThreads) + t.join(); + closesocket(s); + return false; + } LogPrint("worker", "[worker] spawned %d hash threads on shared RandomX dataset\n", nWorkers); const uint64 sessionStartHashes = gParticipantHashes.load(); diff --git a/src/net.cpp b/src/net.cpp index fed6123..16038c8 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -2090,7 +2090,7 @@ bool StartNode(string& strError) if (addrIncoming.ip) addrLocalHost.ip = addrIncoming.ip; - _beginthread([](void*) { + if (_beginthread([](void*) { unsigned int ip = addrLocalHost.ip; if (GetMyExternalIP(ip)) { addrLocalHost.ip = ip; @@ -2098,7 +2098,8 @@ bool StartNode(string& strError) CWalletDB().WriteSetting("addrIncoming", addrIncoming); printf("External IP updated: %s\n", addrLocalHost.ToStringIP().c_str()); } - }, 0, NULL); + }, 0, NULL) == (uintptr_t)-1) + printf("Error: _beginthread(GetMyExternalIP) failed\n"); // Peers that answered last time, dialled straight away. The relays are the // only way to find anyone otherwise, and reaching them plus walking their diff --git a/src/randomx_pow.cpp b/src/randomx_pow.cpp index 1d058d4..9c59364 100644 --- a/src/randomx_pow.cpp +++ b/src/randomx_pow.cpp @@ -8,6 +8,7 @@ #include #include #include +#include // Fixed key (seed) that determines the RandomX cache/dataset. Keeping it // constant avoids per-epoch "reseed" logic; for the CPU+RAM fairness goal @@ -101,13 +102,28 @@ bool RandomXInitDataset(int nThreads) std::vector workers; unsigned long per = total / nThreads; - for (int i = 0; i < nThreads; i++) + try { - unsigned long start = i * per; - unsigned long count = (i == nThreads - 1) ? (total - start) : per; - workers.emplace_back([start, count]() { - randomx_init_dataset(g_dataset, g_cache, start, count); - }); + for (int i = 0; i < nThreads; i++) + { + unsigned long start = i * per; + unsigned long count = (i == nThreads - 1) ? (total - start) : per; + workers.emplace_back([start, count]() { + randomx_init_dataset(g_dataset, g_cache, start, count); + }); + } + } + catch (const std::system_error& e) + { + // Same refusal as the allocation failure above: light mode is slow + // but alive, std::terminate is neither. Join the workers that did + // start before releasing the dataset out from under them. + for (auto& w : workers) w.join(); + randomx_release_dataset(g_dataset); + g_dataset = NULL; + printf("RandomX: cannot spawn dataset init threads (%s), staying in light mode\n", + e.what()); + return false; } for (auto& w : workers) w.join(); diff --git a/src/rpc.cpp b/src/rpc.cpp index a4b53ca..13cb6c9 100644 --- a/src/rpc.cpp +++ b/src/rpc.cpp @@ -5077,10 +5077,17 @@ void ThreadRPCServer(void*) // armed for its one wallet probe before it is allowed to send. ReconcilePayouts(); - _beginthread(AcceptLoopFn, 0, NULL); - _beginthread(LocalAcceptFn, 0, NULL); - _beginthread(ThreadBtfPoolAnnouncer, 0, NULL); - _beginthread(PayoutThreadFn, 0, NULL); + // _beginthread cannot throw (it reports -1), but an unchecked failure + // leaves the pool running deaf: nobody accepts, announces, or pays, and + // nothing in the log says why. + if (_beginthread(AcceptLoopFn, 0, NULL) == (uintptr_t)-1) + LogPrint("pool", "[pool] FATAL: accept loop thread failed to spawn -- miners cannot connect\n"); + if (_beginthread(LocalAcceptFn, 0, NULL) == (uintptr_t)-1) + LogPrint("pool", "[pool] local accept thread failed to spawn\n"); + if (_beginthread(ThreadBtfPoolAnnouncer, 0, NULL) == (uintptr_t)-1) + LogPrint("pool", "[pool] announcer thread failed to spawn -- pool will not be advertised\n"); + if (_beginthread(PayoutThreadFn, 0, NULL) == (uintptr_t)-1) + LogPrint("pool", "[pool] FATAL: payout thread failed to spawn -- no one gets paid\n"); PoolEventLoop(); // blocks until gPoolRunning = false or fShutdown -- cgit v1.2.3 From 7c70eb4498e88725241493033edfe89554ff1b18 Mon Sep 17 00:00:00 2001 From: bitflash-maintainer Date: Wed, 2 Sep 2026 09:03:01 +0000 Subject: tunnel: report which stage failed instead of guessing the cause --- src/btfrv.cpp | 16 ++++++++++++++-- src/btfrv.h | 19 +++++++++++++++++-- src/btftunnel.cpp | 37 +++++++++++++++++++++++++++++++++---- src/btftunnel.h | 25 +++++++++++++++++++++++-- src/main.cpp | 11 ++++++----- 5 files changed, 93 insertions(+), 15 deletions(-) (limited to 'src/btftunnel.cpp') diff --git a/src/btfrv.cpp b/src/btfrv.cpp index 133eb89..864fe3d 100644 --- a/src/btfrv.cpp +++ b/src/btfrv.cpp @@ -334,17 +334,29 @@ bool RvServiceWaitPaired(RvSocket s, int nTimeoutSecs) } RvSocket RvClientConnect(const char* relay_host, unsigned short port, - const unsigned char target_pubkey[32]) + const unsigned char target_pubkey[32], + RvClientFail* pFail) { + if (pFail) *pFail = RVC_FAIL_DIAL; if (!RvInit()) return RV_INVALID; SOCKET s = ConnectTo(relay_host, port); if (s == INVALID_SOCKET) return RV_INVALID; + if (pFail) *pFail = RVC_FAIL_PAIR; unsigned char hdr[33]; hdr[0] = (unsigned char)'C'; memcpy(hdr + 1, target_pubkey, 32); if (!WriteN(s, hdr, 33)) { CLOSESOCK(s); return RV_INVALID; } unsigned char paired = 0; - if (!ReadN(s, &paired, 1) || paired != 0x01) { CLOSESOCK(s); return RV_INVALID; } + if (!ReadN(s, &paired, 1)) { CLOSESOCK(s); return RV_INVALID; } + if (paired != 0x01) + { + // The relay answered and declined: HandleIncoming writes 0x00 and + // closes when no service is parked under this pubkey right now. + if (pFail) *pFail = RVC_FAIL_REFUSED; + CLOSESOCK(s); + return RV_INVALID; + } + if (pFail) *pFail = RVC_FAIL_NONE; return (RvSocket)s; } diff --git a/src/btfrv.h b/src/btfrv.h index 9cd7476..1e85cb0 100644 --- a/src/btfrv.h +++ b/src/btfrv.h @@ -57,10 +57,25 @@ RvSocket RvServiceRegister(const char* relay_host, unsigned short port, bool RvServiceWaitPaired(RvSocket s, int nTimeoutSecs = 0); // ---- client side (the node dialing a .btf address) ---- +// Where a failed RvClientConnect stopped. RV_INVALID alone collapses three +// different situations -- relay unreachable, relay reachable but nobody +// registered, connection died mid-pairing -- and they point at different +// components. The caller can only report the right one if this side says +// which it saw. +enum RvClientFail +{ + RVC_FAIL_NONE = 0, + RVC_FAIL_DIAL, // TCP connect to the relay itself failed + RVC_FAIL_REFUSED, // relay answered 0x00: no service registered under this pubkey + RVC_FAIL_PAIR, // relay connection died before pairing completed +}; + // Connect through the relay to the service registered under `target_pubkey`. -// Returns the paired socket, or RV_INVALID on error. +// Returns the paired socket, or RV_INVALID on error -- with `pFail`, if +// given, naming the stage that failed. RvSocket RvClientConnect(const char* relay_host, unsigned short port, - const unsigned char target_pubkey[32]); + const unsigned char target_pubkey[32], + RvClientFail* pFail = 0); // Blocking read/write of exactly n bytes over a rendezvous socket. Return false // on error/EOF. diff --git a/src/btftunnel.cpp b/src/btftunnel.cpp index 63d178b..408476e 100644 --- a/src/btftunnel.cpp +++ b/src/btftunnel.cpp @@ -136,18 +136,47 @@ static btf_socket_t StartTunnel(RvSocket rv, const unsigned char shared[32]) return appEnd; } +const char* TunnelFailText(TunnelFail f) +{ + switch (f) + { + case TUNNEL_FAIL_NONE: return "no failure"; + case TUNNEL_FAIL_INIT: return "local socket/crypto init failed"; + case TUNNEL_FAIL_DIAL: return "meeting node unreachable (tcp dial failed)"; + case TUNNEL_FAIL_REFUSED: return "relay reachable, but no service registered there for this address"; + case TUNNEL_FAIL_PAIR: return "relay connection died before pairing completed"; + case TUNNEL_FAIL_HANDSHAKE: return "end-to-end handshake failed"; + case TUNNEL_FAIL_LOCAL: return "local tunnel setup failed"; + } + return "unknown failure"; +} + btf_socket_t BtfClientTunnel(const char* meeting_host, unsigned short port, const unsigned char target_pubkey[32], - const unsigned char service_enc_pub[32]) + const unsigned char service_enc_pub[32], + TunnelFail* pFail) { + if (pFail) *pFail = TUNNEL_FAIL_INIT; if (!TunnelInit()) return INVALID_SOCKET; - RvSocket rv = RvClientConnect(meeting_host, port, target_pubkey); - if (rv == RV_INVALID) return INVALID_SOCKET; + RvClientFail rvFail = RVC_FAIL_NONE; + RvSocket rv = RvClientConnect(meeting_host, port, target_pubkey, &rvFail); + if (rv == RV_INVALID) + { + if (pFail) + *pFail = rvFail == RVC_FAIL_DIAL ? TUNNEL_FAIL_DIAL + : rvFail == RVC_FAIL_REFUSED ? TUNNEL_FAIL_REFUSED + : TUNNEL_FAIL_PAIR; + return INVALID_SOCKET; + } unsigned char eph_pub[32], shared[32]; + if (pFail) *pFail = TUNNEL_FAIL_HANDSHAKE; if (!ChanClientHandshake(service_enc_pub, eph_pub, shared)) { RvClose(rv); return INVALID_SOCKET; } if (!RvWriteN(rv, eph_pub, 32)) { RvClose(rv); return INVALID_SOCKET; } - return StartTunnel(rv, shared); + if (pFail) *pFail = TUNNEL_FAIL_LOCAL; + btf_socket_t app = StartTunnel(rv, shared); + if (app != INVALID_SOCKET && pFail) *pFail = TUNNEL_FAIL_NONE; + return app; } btf_socket_t BtfServiceWrap(RvSocket rv, const unsigned char my_enc_sk[32]) diff --git a/src/btftunnel.h b/src/btftunnel.h index 24a7509..3cc87cd 100644 --- a/src/btftunnel.h +++ b/src/btftunnel.h @@ -36,12 +36,33 @@ namespace btf // Initialize (sockets + libsodium). Idempotent. bool TunnelInit(); +// Which stage of BtfClientTunnel failed. INVALID_SOCKET alone says only "no +// tunnel"; a caller that then guesses at causes in its log line sends whoever +// reads it to the wrong component (issue #17: "relay unreachable or pool not +// listening" blamed two healthy parts). The stage is what this side actually +// knows -- report that. +enum TunnelFail +{ + TUNNEL_FAIL_NONE = 0, + TUNNEL_FAIL_INIT, // local socket/crypto init failed + TUNNEL_FAIL_DIAL, // TCP connect to the meeting node failed + TUNNEL_FAIL_REFUSED, // relay reachable, but no service registered under the pubkey + TUNNEL_FAIL_PAIR, // relay connection died before pairing completed + TUNNEL_FAIL_HANDSHAKE, // end-to-end channel handshake failed + TUNNEL_FAIL_LOCAL, // local pump plumbing (socket pair / threads) failed +}; + +// Short description of a TunnelFail for log lines. +const char* TunnelFailText(TunnelFail f); + // Client: dial the service registered under `target_pubkey` at the meeting node // (host:port), box to its static x25519 `service_enc_pub`, and return a local -// plaintext SOCKET tunneled end-to-end to the service. INVALID_SOCKET on error. +// plaintext SOCKET tunneled end-to-end to the service. INVALID_SOCKET on error +// -- with `pFail`, if given, naming the stage that failed. btf_socket_t BtfClientTunnel(const char* meeting_host, unsigned short port, const unsigned char target_pubkey[32], - const unsigned char service_enc_pub[32]); + const unsigned char service_enc_pub[32], + TunnelFail* pFail = 0); // Service: wrap an already-paired rendezvous socket (RvServiceRegister followed // by RvServiceWaitPaired -- the pairing byte MUST already be consumed) with diff --git a/src/main.cpp b/src/main.cpp index 7b3e555..c0a8825 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -3758,13 +3758,14 @@ static bool PoolParticipantMiner() // Step 2b: open encrypted tunnel to pool // ------------------------------------------------------------------ SetParticipantMiningStatus("opening tunnel"); + btf::TunnelFail tunnelFail = btf::TUNNEL_FAIL_NONE; s = btf::BtfClientTunnel(host.c_str(), (unsigned short)port, - target_pubkey, service_enc_pub); + target_pubkey, service_enc_pub, &tunnelFail); if (s == INVALID_SOCKET) { - LogPrint("worker", "[worker] tunnel connect FAILED via %s" - " (relay unreachable or pool not listening)\n", - meetingHostPort.c_str()); - SetParticipantMiningStatus("tunnel connect failed"); + LogPrint("worker", "[worker] tunnel connect FAILED via %s: %s\n", + meetingHostPort.c_str(), btf::TunnelFailText(tunnelFail)); + SetParticipantMiningStatus(std::string("tunnel connect failed: ") + + btf::TunnelFailText(tunnelFail)); return false; } LogPrint("worker", "[worker] tunnel open via %s\n", meetingHostPort.c_str()); -- cgit v1.2.3 From 0a8c445742ce871c6b9352bf406d2232a6fd3134 Mon Sep 17 00:00:00 2001 From: bitflash-maintainer Date: Wed, 2 Sep 2026 17:39:50 +0000 Subject: tunnel: give both pumps shared ownership of the sockets they close PumpIn's wrapper closed pumpEnd and the rendezvous socket the moment it exited, without waiting for PumpOut, which then called shutdown() on both at its own exit. Which of the two got there first decided whether that shutdown addressed a live descriptor or a number the kernel had already handed back out -- in which case it tears down an unrelated connection. The inbound pump wins that race whenever the rendezvous socket dies before the app socket, which is the ordinary shape of a relay restart or a far side hanging up. The failure path added in 0b566b4 has the same hole. Its catch shuts down rv to wake an inbound pump it assumes is parked in RvReadN. PumpIn does not park when the rendezvous connection is already dead: it returns on its first length read and its wrapper closes both descriptors. A refused pthread_create and a dead rv are not independent -- both are symptoms of a loaded process -- so that is the likely case, not the exotic one. Both descriptors now belong to a shared_ptr the two pumps hold for the whole of their bodies, so the close happens after the last pump has stopped touching them and no shutdown can precede it. appEnd is unchanged: the caller gets it on success, StartTunnel closes it on every failure path, no pump ever touches it. 0b566b4's message says of that catch that "closing those descriptors from the failure path instead would race it". True as far as it goes, but it reads as though what the code did instead was safe, and it was not -- the shutdown raced the same close. There was no ordering there to prefer, only a missing owner. Measured against the real code with the relay in a separate process, every close() and shutdown() traced and pthread_create refused on demand: normal teardown, 25 tunnels 29 shutdowns on a closed fd -> 0 refused outbound spawn, 25 25 shutdowns on a closed fd -> 0 Close counts are identical before and after (8 and 5 per tunnel, every one accounted for) and live descriptors return to their starting count, so the defect was only ever the ordering, never a leak or a double close. --- src/btftunnel.cpp | 85 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 17 deletions(-) (limited to 'src/btftunnel.cpp') diff --git a/src/btftunnel.cpp b/src/btftunnel.cpp index 2b3f818..4acca1a 100644 --- a/src/btftunnel.cpp +++ b/src/btftunnel.cpp @@ -26,6 +26,8 @@ typedef int SOCKET; #include "sockcount.h" #include #include +#include +#include #include #include #include @@ -121,9 +123,44 @@ static void PumpIn(SOCKET pumpEnd, RvSocket rv, std::vector key) shutdown((SOCKET)rv, SD_BOTH); } +// The pump end and the rendezvous socket, owned jointly by whoever is still +// using them. Closed once, by the last reference to go. +// +// Joint ownership rather than "the inbound pump owns the cleanup", because +// either pump can reach its exit first and BOTH touch BOTH descriptors on the +// way out -- PumpOut ends in shutdown(rv), shutdown(pumpEnd) and PumpIn ends in +// the mirror image. A single owner therefore has no exit at which the other is +// guaranteed to be finished. Holding a reference for the whole body of each +// pump gives it one: the close cannot happen until the last pump has stopped +// touching the descriptors, so no shutdown() can land on a number that has +// already been closed and handed back out by the kernel to an unrelated +// connection. +struct TunnelEnds +{ + SOCKET pumpEnd; + RvSocket rv; + TunnelEnds(SOCKET p, RvSocket r) : pumpEnd(p), rv(r) {} + ~TunnelEnds() { BtfCloseSocket(pumpEnd); RvClose(rv); } +private: + TunnelEnds(const TunnelEnds&); + TunnelEnds& operator=(const TunnelEnds&); +}; + // Start the two pumps and hand back the app-facing socket. When both pumps // finish, close the pump end and the rendezvous socket. // +// That sentence used to be a comment describing something the code did not do. +// The inbound pump closed both descriptors as soon as it exited, without +// waiting for the outbound pump, which then called shutdown() on both at its +// own exit. Whether that shutdown hit a live descriptor or a closed one came +// down to which pump got there first, and the rendezvous socket dying before +// the app socket -- a relay restart, the far side hanging up -- is precisely +// the order that puts the inbound pump ahead. Measured over 25 such teardowns +// against the real code: 29 shutdowns landed on an already-closed descriptor. +// +// appEnd is not part of this. It is handed to the caller on success and closed +// here on every failure path; no pump ever touches it. +// // Every tunnel is two threads, and a tunnel is one peer connection (a .btf // dial or accept, a pool miner, a participant's pool session) -- so the // thread count scales with peer count, which is attacker-influenced. When @@ -139,41 +176,55 @@ static btf_socket_t StartTunnel(RvSocket rv, const unsigned char shared[32]) std::vector key(shared, shared + 32); - // The inbound pump starts first because it owns the cleanup: it closes - // pumpEnd and rv when it exits. That ownership is what makes the second - // spawn's failure path below race-free. + // make_shared, not shared_ptr(new ...): it allocates before it constructs, + // so it either comes back owning both descriptors or throws having taken + // ownership of neither. The two-step form can run the deleter and then + // throw, which would close both here and again below. + std::shared_ptr ends; + try + { + ends = std::make_shared(pumpEnd, rv); + } + catch (const std::bad_alloc&) + { + fprintf(stderr, "btf tunnel: out of memory starting tunnel, tunnel dropped\n"); + BtfCloseSocket(pumpEnd); + BtfCloseSocket(appEnd); + RvClose(rv); + return INVALID_SOCKET; + } + try { - std::thread([pumpEnd, rv, key]() { - PumpIn(pumpEnd, rv, key); - BtfCloseSocket(pumpEnd); - RvClose(rv); - }).detach(); + std::thread([ends, key]() { PumpIn(ends->pumpEnd, ends->rv, key); }).detach(); } catch (const std::system_error& e) { - // Nothing is running yet, so closing everything here is safe. + // Nothing is running yet: `ends` is the only reference, and returning + // drops it, which is what closes pumpEnd and rv. fprintf(stderr, "btf tunnel: cannot spawn inbound pump (%s), tunnel dropped\n", e.what()); - BtfCloseSocket(pumpEnd); BtfCloseSocket(appEnd); - RvClose(rv); return INVALID_SOCKET; } try { - std::thread(PumpOut, pumpEnd, rv, key).detach(); + std::thread([ends, key]() { PumpOut(ends->pumpEnd, ends->rv, key); }).detach(); } catch (const std::system_error& e) { - // The inbound pump is parked in RvReadN(rv). shutdown() wakes it, and - // it closes pumpEnd and rv itself -- closing them here instead would - // race exactly that, and a reused descriptor is how an unrelated - // socket gets closed. + // The inbound pump may be parked in RvReadN(rv) or may already have + // exited -- when the spawn is refused because the process is loaded, + // the rendezvous socket is often dead for the same reason, and then + // PumpIn returns on its first length read. Either way this thread + // still holds a reference, so both descriptors are still open: the + // shutdown wakes the pump if it is parked and is harmless if it is + // not. The close belongs to whichever reference goes last. fprintf(stderr, "btf tunnel: cannot spawn outbound pump (%s), tunnel dropped\n", e.what()); - shutdown((SOCKET)rv, SD_BOTH); + shutdown((SOCKET)ends->rv, SD_BOTH); + shutdown(ends->pumpEnd, SD_BOTH); BtfCloseSocket(appEnd); return INVALID_SOCKET; } -- cgit v1.2.3