From 21c8bb63cb85f3001ddee9237b1d7e689675d293 Mon Sep 17 00:00:00 2001 From: Bitflash Date: Sat, 1 Aug 2026 13:10:23 -0300 Subject: diag: count sockets by where they were created, and whether the close took (#86) A node was found holding 1502 sockets in Windows bound state, one ephemeral port each, 9% of the machine range in nineteen hours. Reading the code did not find it: every connect() failure path here closes its socket. So count. Every socket is tagged at creation and untagged at close, across all ten sites, with a counter for closes that fail. Already rules out three explanations: nothing is created or closed behind the counter (untagged closes are zero), no close is failing, and the leaked sockets are not in the live set -- tracked live matches what the OS reports as established, while the bound pile is something the program already closed successfully. --- src/btfrv.cpp | 12 +++-- src/btftunnel.cpp | 23 ++++----- src/net.cpp | 43 ++++++++++++++--- src/net.h | 2 + src/nostr.cpp | 6 +-- src/rpc.cpp | 4 +- src/sockcount.h | 137 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 199 insertions(+), 28 deletions(-) create mode 100644 src/sockcount.h (limited to 'src') diff --git a/src/btfrv.cpp b/src/btfrv.cpp index 7efd38b..6ac3280 100644 --- a/src/btfrv.cpp +++ b/src/btfrv.cpp @@ -11,7 +11,7 @@ #ifdef _WIN32 #include #include -#define CLOSESOCK closesocket +#define CLOSESOCK BtfCloseSocket #else #include #include @@ -22,12 +22,14 @@ typedef int SOCKET; #define INVALID_SOCKET (-1) #define SOCKET_ERROR (-1) -#define CLOSESOCK ::close +#define CLOSESOCK BtfCloseSocket #ifndef SD_BOTH #define SD_BOTH SHUT_RDWR #endif #endif +#include "sockcount.h" + #include #include #include @@ -96,7 +98,7 @@ static SOCKET ConnectTo(const char* host, unsigned short port) hints.ai_socktype = SOCK_STREAM; char portstr[16]; sprintf(portstr, "%u", (unsigned)port); if (getaddrinfo(host, portstr, &hints, &res) != 0 || !res) return INVALID_SOCKET; - SOCKET s = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + SOCKET s = BtfSocketTag(socket(res->ai_family, res->ai_socktype, res->ai_protocol), SOCK_RV_DIAL); if (s == INVALID_SOCKET) { freeaddrinfo(res); return INVALID_SOCKET; } // Apply a short timeout only for the connect() and initial handshake so an @@ -201,7 +203,7 @@ bool RvRelayRun(unsigned short port) if (!RvInit()) return false; g_stop = false; - g_listen = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + g_listen = BtfSocketTag(socket(AF_INET, SOCK_STREAM, IPPROTO_TCP), SOCK_RV_LISTEN); if (g_listen == INVALID_SOCKET) return false; int yes = 1; setsockopt(g_listen, SOL_SOCKET, SO_REUSEADDR, (char*)&yes, sizeof(yes)); @@ -218,7 +220,7 @@ bool RvRelayRun(unsigned short port) { struct sockaddr_in cli; socklen_t len = sizeof(cli); - SOCKET s = accept(g_listen, (struct sockaddr*)&cli, &len); + SOCKET s = BtfSocketTag(accept(g_listen, (struct sockaddr*)&cli, &len), SOCK_RV_ACCEPT); if (s == INVALID_SOCKET) { if (g_stop) break; diff --git a/src/btftunnel.cpp b/src/btftunnel.cpp index 65c4b89..63d178b 100644 --- a/src/btftunnel.cpp +++ b/src/btftunnel.cpp @@ -23,6 +23,7 @@ typedef int SOCKET; #define SD_BOTH SHUT_RDWR #endif #endif +#include "sockcount.h" #include #include #include @@ -39,7 +40,7 @@ bool TunnelInit() // Returns app end and pump end; both are ordinary full-duplex TCP sockets. static bool MakeLocalPair(btf_socket_t& appEnd, btf_socket_t& pumpEnd) { - SOCKET listener = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + SOCKET listener = BtfSocketTag(socket(AF_INET, SOCK_STREAM, IPPROTO_TCP), SOCK_PAIR_LISTEN); if (listener == INVALID_SOCKET) return false; struct sockaddr_in addr; @@ -47,23 +48,23 @@ static bool MakeLocalPair(btf_socket_t& appEnd, btf_socket_t& pumpEnd) addr.sin_family = AF_INET; addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); addr.sin_port = 0; // ephemeral - if (bind(listener, (struct sockaddr*)&addr, sizeof(addr)) != 0) { closesocket(listener); return false; } - if (listen(listener, 1) != 0) { closesocket(listener); return false; } + if (bind(listener, (struct sockaddr*)&addr, sizeof(addr)) != 0) { BtfCloseSocket(listener); return false; } + if (listen(listener, 1) != 0) { BtfCloseSocket(listener); return false; } #ifdef _WIN32 int len = sizeof(addr); #else socklen_t len = sizeof(addr); #endif - if (getsockname(listener, (struct sockaddr*)&addr, &len) != 0) { closesocket(listener); return false; } + if (getsockname(listener, (struct sockaddr*)&addr, &len) != 0) { BtfCloseSocket(listener); return false; } - SOCKET a = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); - if (a == INVALID_SOCKET) { closesocket(listener); return false; } - if (connect(a, (struct sockaddr*)&addr, sizeof(addr)) != 0) { closesocket(a); closesocket(listener); return false; } + SOCKET a = BtfSocketTag(socket(AF_INET, SOCK_STREAM, IPPROTO_TCP), SOCK_PAIR_APP); + if (a == INVALID_SOCKET) { BtfCloseSocket(listener); return false; } + if (connect(a, (struct sockaddr*)&addr, sizeof(addr)) != 0) { BtfCloseSocket(a); BtfCloseSocket(listener); return false; } - SOCKET b = accept(listener, NULL, NULL); - closesocket(listener); - if (b == INVALID_SOCKET) { closesocket(a); return false; } + SOCKET b = BtfSocketTag(accept(listener, NULL, NULL), SOCK_PAIR_PUMP); + BtfCloseSocket(listener); + if (b == INVALID_SOCKET) { BtfCloseSocket(a); return false; } appEnd = a; pumpEnd = b; @@ -129,7 +130,7 @@ static btf_socket_t StartTunnel(RvSocket rv, const unsigned char shared[32]) std::thread(PumpOut, pumpEnd, rv, key).detach(); std::thread([pumpEnd, rv, key]() { PumpIn(pumpEnd, rv, key); - closesocket(pumpEnd); + BtfCloseSocket(pumpEnd); RvClose(rv); }).detach(); return appEnd; diff --git a/src/net.cpp b/src/net.cpp index 19893ee..7a13257 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -51,6 +51,33 @@ int64 nBlocksReceived = 0; int64 nBlocksWithoutParent = 0; static int64 nNodeStartTime = 0; +// --- Socket accounting (implementation lives in sockcount.h) -------------- +static const char* pszSockSite[SOCK_SITES] = { + "external-ip probe", "listen socket", "inbound accept", + "rendezvous dial", "rendezvous listen", "rendezvous accept", + "loopback pair listener", "loopback pair app end", "loopback pair pump end", + "nostr relay" +}; + +static string SockAccountingText() +{ + long long nOpened[SOCK_SITES], nClosed[SOCK_SITES], nFailed[SOCK_SITES], nUntagged = 0; + BtfSockSnapshot(nOpened, nClosed, nFailed, &nUntagged); + + string str = "\n sockets by where they were created\n"; + str += " site opened closed live close failed\n"; + for (int i = 0; i < SOCK_SITES; i++) + { + if (nOpened[i] == 0) + continue; + str += strprintf(" %-24s %8lld %8lld %8lld %13lld\n", pszSockSite[i], + nOpened[i], nClosed[i], nOpened[i] - nClosed[i], nFailed[i]); + } + str += strprintf(" %-24s %8s %8lld %8s %13s\n", "closed but never tagged", + "-", nUntagged, "-", "-"); + return str; +} + // Defined further down, next to the counter it reads. The miner keeps its own // live count -- threads bump it on the way in and on the way out -- which is // the honest number here: how many are hashing now, not how many were asked to. @@ -119,6 +146,8 @@ string GetDiagnosticsText() (RandomXFastReady() ? 2080 : 256) + 2 * nMining).c_str() : ", not mining"); + str += SockAccountingText(); + str += "\n peer dir height last recv last send unsent\n"; foreach(CNode* pnode, vCopy) { @@ -157,7 +186,7 @@ bool GetMyExternalIP(unsigned int& ipRet) if (getaddrinfo(svc.host, "80", &hints, &res) != 0 || !res) continue; - SOCKET hSocket = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + SOCKET hSocket = BtfSocketTag(socket(res->ai_family, res->ai_socktype, res->ai_protocol), SOCK_EXTIP); if (hSocket == INVALID_SOCKET) { freeaddrinfo(res); continue; } // 5-second timeout so a dead service doesn't stall startup @@ -173,7 +202,7 @@ bool GetMyExternalIP(unsigned int& ipRet) if (connect(hSocket, res->ai_addr, (int)res->ai_addrlen) != 0) { - freeaddrinfo(res); closesocket(hSocket); continue; + freeaddrinfo(res); BtfCloseSocket(hSocket); continue; } freeaddrinfo(res); @@ -190,7 +219,7 @@ bool GetMyExternalIP(unsigned int& ipRet) response += buf; if (response.size() > 4096) break; } - closesocket(hSocket); + BtfCloseSocket(hSocket); // Find blank line separating headers from body size_t bodyPos = response.find("\r\n\r\n"); @@ -950,7 +979,7 @@ void CNode::Disconnect() { LogPrint("net", "disconnecting node %s\n", addr.ToString().c_str()); - closesocket(hSocket); + BtfCloseSocket(hSocket); // All of a nodes broadcasts and subscriptions are automatically torn down // when it goes down, so a node has to stay up to keep its broadcast going. @@ -1230,7 +1259,7 @@ void ThreadSocketHandler2(void* parg) #else socklen_t len = sizeof(sockaddr); #endif - SOCKET hSocket = accept(hListenSocket, (struct sockaddr*)&sockaddr, &len); + SOCKET hSocket = BtfSocketTag(accept(hListenSocket, (struct sockaddr*)&sockaddr, &len), SOCK_ACCEPT); CAddress addr(sockaddr); if (hSocket == INVALID_SOCKET) { @@ -1251,7 +1280,7 @@ void ThreadSocketHandler2(void* parg) { LogPrint("net", "refusing connection from %s, already at %u\n", addr.ToString().c_str(), nNodes); - closesocket(hSocket); + BtfCloseSocket(hSocket); } else { @@ -1639,7 +1668,7 @@ bool StartNode(string& strError) printf("addrLocalHost = %s\n", addrLocalHost.ToString().c_str()); // Create socket for listening for incoming connections - SOCKET hListenSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + SOCKET hListenSocket = BtfSocketTag(socket(AF_INET, SOCK_STREAM, IPPROTO_TCP), SOCK_LISTEN); if (hListenSocket == INVALID_SOCKET) { strError = strprintf("Error: Couldn't open socket for incoming connections (socket returned error %d)", WSAGetLastError()); diff --git a/src/net.h b/src/net.h index bd4eb77..314c709 100644 --- a/src/net.h +++ b/src/net.h @@ -121,6 +121,8 @@ extern int64 nBlocksWithoutParent; // ...of those, how many arrived orphaned // user pastes into an issue is what a developer already knows how to read. string GetDiagnosticsText(); +#include "sockcount.h" + // Defined in main.cpp. Declared here because CNode announces it in the version // message, and net.h is included before main.h. extern int nBestHeight; diff --git a/src/nostr.cpp b/src/nostr.cpp index 17139ad..dc96be2 100644 --- a/src/nostr.cpp +++ b/src/nostr.cpp @@ -462,7 +462,7 @@ public: { if (ssl) { SSL_shutdown(ssl); SSL_free(ssl); ssl = NULL; } if (sslctx) { SSL_CTX_free(sslctx); sslctx = NULL; } - if (hSocket != INVALID_SOCKET) { closesocket(hSocket); hSocket = INVALID_SOCKET; } + if (hSocket != INVALID_SOCKET) { BtfCloseSocket(hSocket); hSocket = INVALID_SOCKET; } } int RawRead(char* buf, int len) @@ -518,7 +518,7 @@ public: bool fConnected = false; for (rp = res; rp != NULL; rp = rp->ai_next) { - hSocket = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol); + hSocket = BtfSocketTag(socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol), SOCK_NOSTR); if (hSocket == INVALID_SOCKET) continue; @@ -542,7 +542,7 @@ public: break; } - closesocket(hSocket); + BtfCloseSocket(hSocket); hSocket = INVALID_SOCKET; } freeaddrinfo(res); diff --git a/src/rpc.cpp b/src/rpc.cpp index 97d8429..41c0328 100644 --- a/src/rpc.cpp +++ b/src/rpc.cpp @@ -46,10 +46,10 @@ using json = nlohmann::json; #ifdef _WIN32 -#define sock_close(s) closesocket(s) +#define sock_close(s) BtfCloseSocket(s) #define SEND_FLAGS 0 #else -#define sock_close(s) ::close(s) +#define sock_close(s) BtfCloseSocket(s) #define SEND_FLAGS MSG_NOSIGNAL #endif diff --git a/src/sockcount.h b/src/sockcount.h new file mode 100644 index 0000000..19f0827 --- /dev/null +++ b/src/sockcount.h @@ -0,0 +1,137 @@ +// Copyright (c) 2026 Bitflash developers +// Distributed under the MIT/X11 software license. +// +// Socket accounting -- which part of the program is holding the sockets. +// +// A node on a 32-core machine was found with 1502 sockets in Windows' "bound" +// state: created, never connected, never closed, each holding a distinct +// ephemeral port. That is 9% of the machine's whole dynamic port range gone in +// nineteen hours, and when the range runs out nothing on that machine can open +// an outbound connection -- not just this program. +// +// Reading the code did not find it. Every connect() failure path in this tree +// closes its socket. So count instead: tag each socket where it is created, +// untag it where it is closed, and report the difference per site. Whichever +// site's live count tracks the leak is the answer. +// +// Header-only on purpose. btfrv.cpp is compiled a second time, on its own, into +// the standalone rendezvous relay -- a separate tree that does not link net.cpp. +// Anything here that needed a .cpp would break that build, and it would break +// it on the VPS rather than on this machine. Include after the platform socket +// headers; each of those files defines SOCKET for itself. + +#ifndef BITFLASH_SOCKCOUNT_H +#define BITFLASH_SOCKCOUNT_H + +#include +#include + +#ifndef _WIN32 +#include +#endif + +enum +{ + SOCK_EXTIP, // GetMyExternalIP probe + SOCK_LISTEN, // the node's own listening socket + SOCK_ACCEPT, // inbound connections + SOCK_RV_DIAL, // outbound to a rendezvous relay + SOCK_RV_LISTEN, // rendezvous listener + SOCK_RV_ACCEPT, // inbound at the rendezvous + SOCK_PAIR_LISTEN, // loopback pair: the temporary listener + SOCK_PAIR_APP, // loopback pair: the app end handed to CNode + SOCK_PAIR_PUMP, // loopback pair: the pump end + SOCK_NOSTR, // outbound to a Nostr relay + SOCK_SITES +}; + +struct BtfSockAccount +{ + std::mutex mtx; + std::map mapSite; + long long nOpened[SOCK_SITES]; + long long nClosed[SOCK_SITES]; + long long nCloseFailed[SOCK_SITES]; + long long nClosedUntagged; + BtfSockAccount() : nClosedUntagged(0) + { + for (int i = 0; i < SOCK_SITES; i++) { nOpened[i] = 0; nClosed[i] = 0; nCloseFailed[i] = 0; } + } +}; + +// One instance across every translation unit: a function-local static in an +// inline function is the same object everywhere it is used. +inline BtfSockAccount& BtfSockAccounting() +{ + static BtfSockAccount account; + return account; +} + +// Record a freshly created socket and return it unchanged, so call sites read +// as they did before. INVALID_SOCKET passes through untouched. +inline SOCKET BtfSocketTag(SOCKET hSocket, int nSite) +{ + if (hSocket == INVALID_SOCKET || nSite < 0 || nSite >= SOCK_SITES) + return hSocket; + BtfSockAccount& a = BtfSockAccounting(); + std::lock_guard lock(a.mtx); + a.mapSite[hSocket] = nSite; + a.nOpened[nSite]++; + return hSocket; +} + +// Close a socket and forget it. Closing one nobody tagged is counted too -- +// that number staying at zero is what says the accounting is complete. +inline int BtfCloseSocket(SOCKET hSocket) +{ + if (hSocket == INVALID_SOCKET) + return 0; + int nSite = -1; + { + BtfSockAccount& a = BtfSockAccounting(); + std::lock_guard lock(a.mtx); + std::map::iterator mi = a.mapSite.find(hSocket); + if (mi == a.mapSite.end()) + a.nClosedUntagged++; + else + { + nSite = mi->second; + a.nClosed[nSite]++; + a.mapSite.erase(mi); + } + } +#ifdef _WIN32 + int nRet = ::closesocket(hSocket); +#else + int nRet = ::close(hSocket); +#endif + // A close that fails leaves the descriptor -- and on Windows the ephemeral + // port it holds -- in place. Counted because a socket the program believes + // it released and the operating system still shows is exactly the shape of + // the leak being chased, and nothing in this codebase has ever looked at + // what closesocket returns. + if (nRet != 0 && nSite >= 0) + { + BtfSockAccount& a = BtfSockAccounting(); + std::lock_guard lock(a.mtx); + a.nCloseFailed[nSite]++; + } + return nRet; +} + +// Copy the counters out. Formatting lives in net.cpp, which has strprintf -- +// and which the relay build does not compile. +inline void BtfSockSnapshot(long long* pOpened, long long* pClosed, long long* pFailed, long long* pUntagged) +{ + BtfSockAccount& a = BtfSockAccounting(); + std::lock_guard lock(a.mtx); + for (int i = 0; i < SOCK_SITES; i++) + { + pOpened[i] = a.nOpened[i]; + pClosed[i] = a.nClosed[i]; + pFailed[i] = a.nCloseFailed[i]; + } + *pUntagged = a.nClosedUntagged; +} + +#endif -- cgit v1.2.3