upleb.uk

Public git repos — served from a NIP-34 GRASP relay at git.upleb.uk

summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/Makefile5
-rw-r--r--src/btfaddr.cpp40
-rw-r--r--src/btfrv.cpp41
-rw-r--r--src/btftunnel.h13
-rw-r--r--src/db.cpp16
-rw-r--r--src/db.h23
-rw-r--r--src/headers_core.h1
-rw-r--r--src/main.cpp608
-rw-r--r--src/main.h59
-rw-r--r--src/main_gui.cpp75
-rw-r--r--src/makefile.mingw4
-rw-r--r--src/market.cpp264
-rw-r--r--src/market.h182
-rw-r--r--src/net.cpp239
-rw-r--r--src/net.h204
-rw-r--r--src/nostr.cpp123
-rw-r--r--src/nostr.h8
-rw-r--r--src/rpc.cpp3444
-rw-r--r--src/selftest.cpp42
-rw-r--r--src/test_btfaddr.cpp19
-rw-r--r--src/test_rvaccept.cpp180
-rw-r--r--src/tor.cpp25
-rw-r--r--src/util.h35
23 files changed, 4449 insertions, 1201 deletions
diff --git a/src/Makefile b/src/Makefile
index b83ff58..fe48041 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -52,14 +52,14 @@ LIBS = -L$(SECP_LIB) -lsecp256k1 \
52# binary. Nothing crashed here, and nothing would say so if it had. 52# binary. Nothing crashed here, and nothing would say so if it had.
53HEADERS = headers_core.h headers.h util.h main.h serialize.h uint256.h \ 53HEADERS = headers_core.h headers.h util.h main.h serialize.h uint256.h \
54 key.h bignum.h script.h db.h base58.h net.h nostr.h \ 54 key.h bignum.h script.h db.h base58.h net.h nostr.h \
55 randomx_pow.h btfaddr.h btfchan.h btfrv.h btftunnel.h market.h \ 55 randomx_pow.h btfaddr.h btfchan.h btfrv.h btftunnel.h \
56 selftest.h walletcmd.h wallet_sqlite.h compat.h sockcount.h proxy.h tor.h font_roboto.h 56 selftest.h walletcmd.h wallet_sqlite.h compat.h sockcount.h proxy.h tor.h font_roboto.h
57 57
58IMGUI_OBJS = obj/imgui.o obj/imgui_draw.o obj/imgui_tables.o \ 58IMGUI_OBJS = obj/imgui.o obj/imgui_draw.o obj/imgui_tables.o \
59 obj/imgui_widgets.o obj/imgui_impl_glfw.o obj/imgui_impl_opengl3.o 59 obj/imgui_widgets.o obj/imgui_impl_glfw.o obj/imgui_impl_opengl3.o
60 60
61NODE_OBJS = obj/util.o obj/script.o obj/db.o obj/net.o obj/main.o \ 61NODE_OBJS = obj/util.o obj/script.o obj/db.o obj/net.o obj/main.o \
62 obj/market.o obj/sha.o obj/nostr.o obj/randomx_pow.o \ 62 obj/sha.o obj/nostr.o obj/randomx_pow.o \
63 obj/btfaddr.o obj/btfchan.o obj/btfrv.o obj/btftunnel.o obj/rpc.o \ 63 obj/btfaddr.o obj/btfchan.o obj/btfrv.o obj/btftunnel.o obj/rpc.o \
64 obj/selftest.o obj/bip32.o obj/walletcmd.o obj/wallet_sqlite.o \ 64 obj/selftest.o obj/bip32.o obj/walletcmd.o obj/wallet_sqlite.o \
65 obj/proxy.o obj/tor.o 65 obj/proxy.o obj/tor.o
@@ -195,6 +195,7 @@ tests: test_bip32 bitflash-node
195 ./bitflash-node -selftest=net-message 195 ./bitflash-node -selftest=net-message
196 ./bitflash-node -selftest=consensus-limits 196 ./bitflash-node -selftest=consensus-limits
197 ./bitflash-node -selftest=pool-stratum 197 ./bitflash-node -selftest=pool-stratum
198 ./bitflash-node -selftest=payout-ledger
198 ./bitflash-node -selftest=parse-money 199 ./bitflash-node -selftest=parse-money
199 ./bitflash-node -selftest=socks5-proxy 200 ./bitflash-node -selftest=socks5-proxy
200 ./bitflash-node -selftest=managed-tor 201 ./bitflash-node -selftest=managed-tor
diff --git a/src/btfaddr.cpp b/src/btfaddr.cpp
index ce2ac86..502220e 100644
--- a/src/btfaddr.cpp
+++ b/src/btfaddr.cpp
@@ -319,20 +319,32 @@ bool VerifyDescriptor(void* ctxv, const std::string& jsonStr,
319 std::string onion; 319 std::string onion;
320 if (v.contains("onion")) 320 if (v.contains("onion"))
321 { 321 {
322 if (v["onion"].is_string() && v.contains("sig3") && v["sig3"].is_string()) 322 // A descriptor that carries an onion endpoint must prove it. The
323 { 323 // endpoint is covered by sig3 and only sig3, so any failure here --
324 std::string onionNorm; 324 // missing sig3, bad hex, unnormalised endpoint, bad signature --
325 std::string onionRaw = v["onion"].get<std::string>(); 325 // means the descriptor was tampered with, and a tampered
326 if (NormalizeOnionEndpoint(onionRaw, onionNorm) && onionNorm == onionRaw) 326 // descriptor is rejected outright. Upstream dropped just the onion
327 { 327 // and accepted the rest, which let any relay convert an onion
328 unsigned char sig3[64]; 328 // descriptor into a rendezvous-only one by deleting a field.
329 unsigned char digest3[32]; 329 //
330 DescriptorDigestV3(expect_pubkey, enc, meeting_node, onionNorm, created, digest3); 330 // Residual, by design: removing BOTH "onion" and "sig3" still
331 if (HexDecode(v["sig3"].get<std::string>(), sig3, 64) && 331 // verifies, because the result is indistinguishable from a peer
332 secp256k1_schnorrsig_verify(ctx, sig3, digest3, 32, &xpub)) 332 // that never had an onion endpoint. Closing that requires binding
333 onion = onionNorm; 333 // the onion into the primary sig (the V2 digest), which every
334 } 334 // upstream 1.2.18/1.2.19 verifier would reject -- an upstream
335 } 335 // protocol change, not something this fork can do unilaterally.
336 if (!v["onion"].is_string()) return false;
337 if (!v.contains("sig3") || !v["sig3"].is_string()) return false;
338 std::string onionNorm;
339 std::string onionRaw = v["onion"].get<std::string>();
340 if (!NormalizeOnionEndpoint(onionRaw, onionNorm) || onionNorm != onionRaw)
341 return false;
342 unsigned char sig3[64];
343 unsigned char digest3[32];
344 DescriptorDigestV3(expect_pubkey, enc, meeting_node, onionNorm, created, digest3);
345 if (!HexDecode(v["sig3"].get<std::string>(), sig3, 64)) return false;
346 if (!secp256k1_schnorrsig_verify(ctx, sig3, digest3, 32, &xpub)) return false;
347 onion = onionNorm;
336 } 348 }
337 349
338 memcpy(out.pubkey, expect_pubkey, 32); 350 memcpy(out.pubkey, expect_pubkey, 32);
diff --git a/src/btfrv.cpp b/src/btfrv.cpp
index d4f079f..133eb89 100644
--- a/src/btfrv.cpp
+++ b/src/btfrv.cpp
@@ -102,31 +102,42 @@ static SOCKET ConnectTo(const char* host, unsigned short port)
102#else 102#else
103 struct addrinfo hints, *res = NULL; 103 struct addrinfo hints, *res = NULL;
104 memset(&hints, 0, sizeof(hints)); 104 memset(&hints, 0, sizeof(hints));
105 hints.ai_family = AF_INET; 105 hints.ai_family = AF_UNSPEC; // IPv4 or IPv6 (some relays are v6-only)
106 hints.ai_socktype = SOCK_STREAM; 106 hints.ai_socktype = SOCK_STREAM;
107 char portstr[16]; sprintf(portstr, "%u", (unsigned)port); 107 char portstr[16]; sprintf(portstr, "%u", (unsigned)port);
108 if (getaddrinfo(host, portstr, &hints, &res) != 0 || !res) return INVALID_SOCKET; 108 if (getaddrinfo(host, portstr, &hints, &res) != 0 || !res) return INVALID_SOCKET;
109 SOCKET s = BtfSocketTag(socket(res->ai_family, res->ai_socktype, res->ai_protocol), SOCK_RV_DIAL);
110 if (s == INVALID_SOCKET) { freeaddrinfo(res); return INVALID_SOCKET; }
111 109
112 // Apply a short timeout only for the connect() and initial handshake so an 110 // Try each resolved address until one connects.
113 // unresponsive relay doesn't block the thread. Once the tunnel is live the 111 SOCKET s = INVALID_SOCKET;
114 // timeout is cleared -- a forwarding pipe must block indefinitely on recv(). 112 for (struct addrinfo* rp = res; rp != NULL; rp = rp->ai_next)
113 {
114 s = BtfSocketTag(socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol), SOCK_RV_DIAL);
115 if (s == INVALID_SOCKET)
116 continue;
117
118 // Apply a short timeout only for the connect() and initial handshake so an
119 // unresponsive relay doesn't block the thread. Once the tunnel is live the
120 // timeout is cleared -- a forwarding pipe must block indefinitely on recv().
115#ifdef _WIN32 121#ifdef _WIN32
116 DWORD tvShort = 10000; 122 DWORD tvShort = 10000;
117 setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tvShort, sizeof(tvShort)); 123 setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tvShort, sizeof(tvShort));
118 setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, (const char*)&tvShort, sizeof(tvShort)); 124 setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, (const char*)&tvShort, sizeof(tvShort));
119#else 125#else
120 struct timeval tvShort; tvShort.tv_sec = 10; tvShort.tv_usec = 0; 126 struct timeval tvShort; tvShort.tv_sec = 10; tvShort.tv_usec = 0;
121 setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tvShort, sizeof(tvShort)); 127 setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, (char*)&tvShort, sizeof(tvShort));
122 setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, &tvShort, sizeof(tvShort)); 128 setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, (char*)&tvShort, sizeof(tvShort));
123#endif 129#endif
124 130
125 if (connect(s, res->ai_addr, (int)res->ai_addrlen) != 0) 131 if (connect(s, rp->ai_addr, (int)rp->ai_addrlen) != 0)
126 { 132 {
127 CLOSESOCK(s); freeaddrinfo(res); return INVALID_SOCKET; 133 CLOSESOCK(s);
134 s = INVALID_SOCKET;
135 continue;
136 }
137 break;
128 } 138 }
129 freeaddrinfo(res); 139 freeaddrinfo(res);
140 if (s == INVALID_SOCKET) return INVALID_SOCKET;
130#endif 141#endif
131 142
132 // Clear the timeout -- the socket is now a live tunnel and must not 143 // Clear the timeout -- the socket is now a live tunnel and must not
diff --git a/src/btftunnel.h b/src/btftunnel.h
index 1fa4f01..24a7509 100644
--- a/src/btftunnel.h
+++ b/src/btftunnel.h
@@ -10,8 +10,12 @@
10// caller), then BtfClientTunnel connects through the relay, runs the echan 10// caller), then BtfClientTunnel connects through the relay, runs the echan
11// handshake, and returns a local SOCKET to hand a new outbound CNode. 11// handshake, and returns a local SOCKET to hand a new outbound CNode.
12// 12//
13// Service side: after RvServiceRegister pairs us with a client, BtfServiceWrap 13// Service side: RvServiceRegister only registers -- callers must then block in
14// completes the echan handshake and returns a local SOCKET for an inbound CNode. 14// RvServiceWaitPaired until a client is actually paired to them. BtfServiceWrap
15// completes the echan handshake on that paired socket and returns a local
16// SOCKET for an inbound CNode. Wrapping a registered-but-unpaired socket reads
17// the relay's 0x01 pairing byte as the first byte of the client's ephemeral
18// key, which silently corrupts the tunnel instead of failing.
15 19
16#ifndef BITFLASH_BTFTUNNEL_H 20#ifndef BITFLASH_BTFTUNNEL_H
17#define BITFLASH_BTFTUNNEL_H 21#define BITFLASH_BTFTUNNEL_H
@@ -39,8 +43,9 @@ btf_socket_t BtfClientTunnel(const char* meeting_host, unsigned short port,
39 const unsigned char target_pubkey[32], 43 const unsigned char target_pubkey[32],
40 const unsigned char service_enc_pub[32]); 44 const unsigned char service_enc_pub[32]);
41 45
42// Service: wrap an already-paired rendezvous socket (from RvServiceRegister) 46// Service: wrap an already-paired rendezvous socket (RvServiceRegister followed
43// with echan using my static x25519 secret, returning a local plaintext SOCKET 47// by RvServiceWaitPaired -- the pairing byte MUST already be consumed) with
48// echan using my static x25519 secret, returning a local plaintext SOCKET
44// for an inbound CNode. Consumes `rv` (closed when the tunnel tears down). 49// for an inbound CNode. Consumes `rv` (closed when the tunnel tears down).
45btf_socket_t BtfServiceWrap(RvSocket rv, const unsigned char my_enc_sk[32]); 50btf_socket_t BtfServiceWrap(RvSocket rv, const unsigned char my_enc_sk[32]);
46 51
diff --git a/src/db.cpp b/src/db.cpp
index 949865f..bf1958d 100644
--- a/src/db.cpp
+++ b/src/db.cpp
@@ -806,20 +806,12 @@ bool CTxDB::LoadBlockIndex()
806 806
807 807
808// 808//
809// CReviewDB 809// CReviewDB is gone with the "review" handler. Its write path was
810// read-whole-vector / append-one / write-whole-vector, driven straight from an
811// unauthenticated peer, which is quadratic disk on an unbounded record.
812// Existing reviews.dat files are simply left alone and never opened.
810// 813//
811 814
812bool CReviewDB::ReadReviews(uint256 hash, vector<CReview>& vReviews)
813{
814 vReviews.size(); // msvc workaround, just need to do anything with vReviews
815 return Read(make_pair(string("reviews"), hash), vReviews);
816}
817
818bool CReviewDB::WriteReviews(uint256 hash, const vector<CReview>& vReviews)
819{
820 return Write(make_pair(string("reviews"), hash), vReviews);
821}
822
823 815
824 816
825 817
diff --git a/src/db.h b/src/db.h
index c50065b..859e3fb 100644
--- a/src/db.h
+++ b/src/db.h
@@ -8,8 +8,6 @@ class CTxIndex;
8class CDiskBlockIndex; 8class CDiskBlockIndex;
9class CDiskTxPos; 9class CDiskTxPos;
10class COutPoint; 10class COutPoint;
11class CUser;
12class CReview;
13class CAddress; 11class CAddress;
14class CWalletTx; 12class CWalletTx;
15 13
@@ -314,27 +312,6 @@ public:
314 312
315 313
316 314
317class CReviewDB : public CDB
318{
319public:
320 CReviewDB(const char* pszMode="r+", bool fTxn=false) : CDB("reviews.dat", pszMode, fTxn) { }
321private:
322 CReviewDB(const CReviewDB&);
323 void operator=(const CReviewDB&);
324public:
325 bool ReadUser(uint256 hash, CUser& user)
326 {
327 return Read(make_pair(string("user"), hash), user);
328 }
329
330 bool WriteUser(uint256 hash, const CUser& user)
331 {
332 return Write(make_pair(string("user"), hash), user);
333 }
334
335 bool ReadReviews(uint256 hash, vector<CReview>& vReviews);
336 bool WriteReviews(uint256 hash, const vector<CReview>& vReviews);
337};
338 315
339 316
340 317
diff --git a/src/headers_core.h b/src/headers_core.h
index f890c2f..2824126 100644
--- a/src/headers_core.h
+++ b/src/headers_core.h
@@ -71,7 +71,6 @@ inline typename std::common_type<T1,T2>::type max(const T1& a, const T2& b)
71#include "nostr.h" 71#include "nostr.h"
72#include "randomx_pow.h" 72#include "randomx_pow.h"
73#include "main.h" 73#include "main.h"
74#include "market.h"
75 74
76// GUI callback -- implemented in gui.cpp (or as no-op in headless builds) 75// GUI callback -- implemented in gui.cpp (or as no-op in headless builds)
77void MainFrameRepaint(); 76void MainFrameRepaint();
diff --git a/src/main.cpp b/src/main.cpp
index 5bd4028..3011111 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -87,6 +87,8 @@ CKey keyUser;
87string strSetDataDir; 87string strSetDataDir;
88int nDropMessagesTest = 0; 88int nDropMessagesTest = 0;
89bool fSoloMineTest = false; // /solomine: mine without requiring a peer (local test) 89bool fSoloMineTest = false; // /solomine: mine without requiring a peer (local test)
90bool fPoolNoAnnounce = false; // /noannounce: operator serves miners but never publishes pool announcements
91bool fPoolMineBlind = false; // /poolmineblind: issue jobs even when no peer has reported a height
90 92
91// Settings 93// Settings
92int fGenerateBitcoins; 94int fGenerateBitcoins;
@@ -102,6 +104,9 @@ CAddress addrIncoming;
102int nMineMode = MINE_RELAY; 104int nMineMode = MINE_RELAY;
103bool fMineModeFromCommandLine = false; 105bool fMineModeFromCommandLine = false;
104string strParticipantPool; // participant mode: pool .btf address 106string strParticipantPool; // participant mode: pool .btf address
107string strPayoutAddress; // participant: /payoutaddr override (mine without a wallet)
108string strPayoutFile; // participant: /payoutfile with one address per line, rotated per reconnect
109int nHashThreads = 0; // participant: /hashthreads override (0 = cores-1)
105string strPoolName = "Bitflash Pool"; 110string strPoolName = "Bitflash Pool";
106string strPoolDashboardUrl; 111string strPoolDashboardUrl;
107string strPoolStatusFile; 112string strPoolStatusFile;
@@ -163,12 +168,32 @@ int MinerThreadCount()
163{ 168{
164 if (nMinerThreads > 0) 169 if (nMinerThreads > 0)
165 return nMinerThreads; 170 return nMinerThreads;
166 unsigned int nCores = std::thread::hardware_concurrency(); 171 unsigned int nCores = AvailableCoreCount();
167 if (nCores <= 1) 172 if (nCores <= 1)
168 return 1; 173 return 1;
169 return (int)(nCores - 1); 174 return (int)(nCores - 1);
170} 175}
171 176
177// Hash threads one participant session runs.
178//
179// Explicit /hashthreads=N is honored exactly, including 1 -- a miner who says
180// one means one. Automatic leaves a core to the node, the Stratum I/O loop and
181// everything else the process still has to do, for the same reason
182// MinerThreadCount() does.
183//
184// Lives here, next to that function, because the two are easy to confuse and
185// StartMinerThreads() needs to be able to tell them apart: only ONE session
186// runs in participant mode, and this is what sizes it.
187int ParticipantHashThreadCount()
188{
189 if (nHashThreads > 0)
190 return nHashThreads;
191 int nWorkers = (int)AvailableCoreCount();
192 if (nWorkers < 2)
193 nWorkers = 2;
194 return nWorkers - 1;
195}
196
172static void SecureClearBytes(vector<unsigned char>& vch); 197static void SecureClearBytes(vector<unsigned char>& vch);
173static void SecureClearKeyingMaterial(CKeyingMaterial& vch); 198static void SecureClearKeyingMaterial(CKeyingMaterial& vch);
174 199
@@ -1551,7 +1576,7 @@ int CMerkleTx::GetBlocksToMaturity() const
1551{ 1576{
1552 if (!IsCoinBase()) 1577 if (!IsCoinBase())
1553 return 0; 1578 return 0;
1554 return max(0, (COINBASE_MATURITY+20) - GetDepthInMainChain()); 1579 return max(0, COINBASE_SPENDABLE_DEPTH - GetDepthInMainChain());
1555} 1580}
1556 1581
1557 1582
@@ -2819,9 +2844,10 @@ bool AlreadyHave(CTxDB& txdb, const CInv& inv)
2819 { 2844 {
2820 case MSG_TX: return mapTransactions.count(inv.hash) || txdb.ContainsTx(inv.hash); 2845 case MSG_TX: return mapTransactions.count(inv.hash) || txdb.ContainsTx(inv.hash);
2821 case MSG_BLOCK: return mapBlockIndex.count(inv.hash) || mapOrphanBlocks.count(inv.hash); 2846 case MSG_BLOCK: return mapBlockIndex.count(inv.hash) || mapOrphanBlocks.count(inv.hash);
2822 case MSG_REVIEW: return true;
2823 case MSG_PRODUCT: return mapProducts.count(inv.hash);
2824 } 2847 }
2848 // MSG_REVIEW / MSG_PRODUCT / MSG_TABLE are the 2009 marketplace inventory
2849 // types. Nothing announces or serves them any more (see ProcessMessage),
2850 // so they fall through to the default and are never requested.
2825 // Don't know what it is, just say we already got one 2851 // Don't know what it is, just say we already got one
2826 return true; 2852 return true;
2827} 2853}
@@ -2864,11 +2890,13 @@ bool ProcessMessages(CNode* pfrom)
2864 if (LogAcceptsCategory("net")) printf("\n\nPROCESSMESSAGE SKIPPED %d BYTES\n\n", (int)(pstart - vRecv.begin())); 2890 if (LogAcceptsCategory("net")) printf("\n\nPROCESSMESSAGE SKIPPED %d BYTES\n\n", (int)(pstart - vRecv.begin()));
2865 vRecv.erase(vRecv.begin(), pstart); 2891 vRecv.erase(vRecv.begin(), pstart);
2866 2892
2867 // Read the header from a copy first. The old path consumed the header, 2893 // Read the header from a copy first, and do not consume it until the
2868 // noticed the payload was incomplete, inserted the header back at the 2894 // whole body is here. The old path consumed the header, noticed the
2869 // front of the receive buffer, then slept inside the shared message 2895 // payload was incomplete, inserted the header back at the front of the
2870 // thread. A peer could announce a large payload and drip bytes forever, 2896 // receive buffer, then slept inside the shared message thread. A peer
2871 // making every pass pay for an O(n) insert and a 100 ms stall. 2897 // could announce a large payload and drip bytes forever, making every
2898 // pass pay for an O(n) insert and a 100 ms stall -- a cheap way for one
2899 // peer to slow the thread that serves all of them. (upstream #109)
2872 CMessageHeader hdr; 2900 CMessageHeader hdr;
2873 CDataStream vHeader(vRecv.begin(), vRecv.begin() + sizeof(CMessageHeader), 2901 CDataStream vHeader(vRecv.begin(), vRecv.begin() + sizeof(CMessageHeader),
2874 vRecv.nType, vRecv.nVersion); 2902 vRecv.nType, vRecv.nVersion);
@@ -2878,7 +2906,10 @@ bool ProcessMessages(CNode* pfrom)
2878 if (LogAcceptsCategory("net")) printf("\n\nPROCESSMESSAGE: ERRORS IN HEADER %s\n\n\n", hdr.GetCommand().c_str()); 2906 if (LogAcceptsCategory("net")) printf("\n\nPROCESSMESSAGE: ERRORS IN HEADER %s\n\n\n", hdr.GetCommand().c_str());
2879 if (hdr.nMessageSize > MAX_PROTOCOL_MESSAGE_SIZE) 2907 if (hdr.nMessageSize > MAX_PROTOCOL_MESSAGE_SIZE)
2880 pfrom->fDisconnect = true; 2908 pfrom->fDisconnect = true;
2881 pfrom->nIncompleteMessageStart = 0; 2909 pfrom->nMsgWaitSince = 0;
2910 pfrom->nMsgWaitSize = 0;
2911 // The header was read from a copy, so it is still in vRecv. Skip it
2912 // explicitly or this loop never advances.
2882 vRecv.ignore(sizeof(CMessageHeader)); 2913 vRecv.ignore(sizeof(CMessageHeader));
2883 continue; 2914 continue;
2884 } 2915 }
@@ -2889,30 +2920,63 @@ bool ProcessMessages(CNode* pfrom)
2889 unsigned int nPayloadAvailable = vRecv.size() - sizeof(CMessageHeader); 2920 unsigned int nPayloadAvailable = vRecv.size() - sizeof(CMessageHeader);
2890 if (nMessageSize > nPayloadAvailable) 2921 if (nMessageSize > nPayloadAvailable)
2891 { 2922 {
2892 if (pfrom->nIncompleteMessageStart == 0 || 2923 // Wait for the rest of the message. The header stays in vRecv.
2893 pfrom->nIncompleteMessageSize != nMessageSize || 2924 //
2894 pfrom->strIncompleteMessageCommand != strCommand) 2925 // This is the mechanism the original note asked for: "need a
2926 // mechanism to give up waiting for overlong message size error".
2927 // Without one the wait is unbounded, and unbounded here means
2928 // permanent. The header stays at the front of vRecv and the loop
2929 // breaks; if the body never completes, every later pass re-reads
2930 // that same header and breaks in the same place. The peer is never
2931 // parsed again -- no blocks, no inv, not even its version message,
2932 // so it never reports a height either, which is why a wedged node
2933 // has peers and GetPeerMedianHeight() still returns -1.
2934 //
2935 // Nothing rescues it. A header could claim up to 256 MB and still
2936 // pass CMessageHeader::IsValid() -- now capped at
2937 // MAX_PROTOCOL_MESSAGE_SIZE -- and any byte that arrives
2938 // refreshes nLastRecv, so the inactivity timeout that drops a
2939 // silent peer never fires on one that is merely unreadable. The
2940 // socket stays ESTABLISHED and select() keeps offering it.
2941 // Measured on a wedged pool: this branch ran 2014 times in the
2942 // deaf window, against 607 in the healthy window before it.
2943 if (LogAcceptsCategory("net")) printf("MESSAGE-BREAK 2\n");
2944 //
2945 // Time the STALL, not the wait. A big message on a slow link is
2946 // still arriving, and dropping that peer would be a bug of our own
2947 // making; what must be caught is a buffer that has stopped growing
2948 // while we go on expecting more. So the clock restarts whenever
2949 // even one more byte has landed since the last look.
2950 //
2951 // Upstream #109 times this differently: it restarts the clock only
2952 // when the (size, command) pair changes, which drops a peer sending
2953 // one big message slowly but honestly. Progress is the better test,
2954 // so that half is kept as it was.
2955 int64 nNow = GetTime();
2956 if (pfrom->nMsgWaitSince == 0 || nPayloadAvailable > pfrom->nMsgWaitSize)
2895 { 2957 {
2896 pfrom->nIncompleteMessageStart = GetTime(); 2958 pfrom->nMsgWaitSince = nNow;
2897 pfrom->nIncompleteMessageSize = nMessageSize; 2959 pfrom->nMsgWaitSize = nPayloadAvailable;
2898 pfrom->strIncompleteMessageCommand = strCommand;
2899 } 2960 }
2900 else if (GetTime() - pfrom->nIncompleteMessageStart > 2961 else if (nNow - pfrom->nMsgWaitSince > BTF_MSG_WAIT_MAX_SECS)
2901 BTF_INCOMPLETE_MESSAGE_TIMEOUT_SECS)
2902 { 2962 {
2903 LogPrint("net", 2963 // Deliberately not behind a debug category. This is the node
2904 "disconnecting %s: incomplete %s message, %u of %u bytes after %d seconds\n", 2964 // going deaf to a peer, which is the thing nobody could see.
2905 pfrom->addr.ToString().c_str(), strCommand.c_str(), 2965 printf("peer %s announced a %u byte '%s' and has sent nothing more "
2906 nPayloadAvailable, nMessageSize, 2966 "for %d s (%d of %u payload bytes) -- dropping, this peer is unreadable\n",
2907 BTF_INCOMPLETE_MESSAGE_TIMEOUT_SECS); 2967 pfrom->addr.ToString().c_str(), nMessageSize,
2968 strCommand.c_str(), (int)(nNow - pfrom->nMsgWaitSince),
2969 (int)nPayloadAvailable, nMessageSize);
2908 pfrom->fDisconnect = true; 2970 pfrom->fDisconnect = true;
2909 } 2971 }
2972 // No re-insert, no Sleep(): the header was never consumed, so there
2973 // is nothing to put back, and sleeping here stalls every peer.
2910 break; 2974 break;
2911 } 2975 }
2912 pfrom->nIncompleteMessageStart = 0;
2913 pfrom->nIncompleteMessageSize = 0;
2914 pfrom->strIncompleteMessageCommand.clear();
2915 vRecv.ignore(sizeof(CMessageHeader)); 2976 vRecv.ignore(sizeof(CMessageHeader));
2977 // The body arrived, so this peer is readable again.
2978 pfrom->nMsgWaitSince = 0;
2979 pfrom->nMsgWaitSize = 0;
2916 2980
2917 // Copy message to its own buffer 2981 // Copy message to its own buffer
2918 CDataStream vMsg(vRecv.begin(), vRecv.begin() + nMessageSize, vRecv.nType, vRecv.nVersion); 2982 CDataStream vMsg(vRecv.begin(), vRecv.begin() + nMessageSize, vRecv.nType, vRecv.nVersion);
@@ -2941,7 +3005,6 @@ bool ProcessMessages(CNode* pfrom)
2941 3005
2942bool ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) 3006bool ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv)
2943{ 3007{
2944 static map<unsigned int, vector<unsigned char> > mapReuseKey;
2945 if (LogAcceptsCategory("net")) { printf("received: %-12s (%d bytes) ", strCommand.c_str(), (int)vRecv.size()); for (int i = 0; i < min(vRecv.size(), (unsigned int)25); i++) printf("%02x ", vRecv[i] & 0xff); printf("\n"); } 3008 if (LogAcceptsCategory("net")) { printf("received: %-12s (%d bytes) ", strCommand.c_str(), (int)vRecv.size()); for (int i = 0; i < min(vRecv.size(), (unsigned int)25); i++) printf("%02x ", vRecv[i] & 0xff); printf("\n"); }
2946 if (nDropMessagesTest > 0 && GetRand(nDropMessagesTest) == 0) 3009 if (nDropMessagesTest > 0 && GetRand(nDropMessagesTest) == 0)
2947 { 3010 {
@@ -3180,22 +3243,18 @@ bool ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv)
3180 } 3243 }
3181 3244
3182 3245
3183 else if (strCommand == "review") 3246 // "review" used to be handled here. It took a CReview off the wire from an
3184 { 3247 // unauthenticated peer, checked a signature the sender had made with a key
3185 CDataStream vMsg(vRecv); 3248 // of its own choosing -- which proves possession of that key and authorises
3186 CReview review; 3249 // nothing -- and then appended the message to reviews.dat by reading the
3187 vRecv >> review; 3250 // whole vector for the target hash, pushing one entry, and writing all of
3188 3251 // it back. That is quadratic disk for a stream an attacker controls, on a
3189 CInv inv(MSG_REVIEW, review.GetHash()); 3252 // record whose mapValue is an unbounded map<string,string> read straight
3190 pfrom->AddInventoryKnown(inv); 3253 // from the socket, followed by AddAtomsAndPropagate walking a link graph
3191 3254 // the same peer supplied, and finally RelayMessage forwarding it to every
3192 if (review.AcceptReview()) 3255 // other node. Unbounded remote disk growth with network amplification and
3193 { 3256 // no authorisation anywhere in the path. The marketplace it belonged to has
3194 // Relay the original message as-is in case it's a higher version than we know how to parse 3257 // not existed since 2009; the handler is gone rather than bounded.
3195 RelayMessage(inv, vMsg);
3196 mapAlreadyAskedFor.erase(inv);
3197 }
3198 }
3199 3258
3200 3259
3201 else if (strCommand == "block") 3260 else if (strCommand == "block")
@@ -3210,41 +3269,46 @@ bool ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv)
3210 pfrom->AddInventoryKnown(inv); 3269 pfrom->AddInventoryKnown(inv);
3211 3270
3212 if (ProcessBlock(pfrom, pblock.release())) 3271 if (ProcessBlock(pfrom, pblock.release()))
3272 {
3213 mapAlreadyAskedFor.erase(inv); 3273 mapAlreadyAskedFor.erase(inv);
3214 }
3215 3274
3216 3275 // A peer that just handed us a block is demonstrably at least at
3217 else if (strCommand == "checkorder" || strCommand == "submitorder") 3276 // that block's height. Without this, nStartingHeight is a snapshot
3218 { 3277 // taken once during the version handshake and never touched again,
3219 // Legacy Bitcoin 0.1 marketplace messages are not part of Bitflash's 3278 // so on a long-lived connection it is the peer's height from
3220 // supported protocol. They used to let a remote peer make this wallet 3279 // however many hours ago it happened to connect. The pool's
3221 // reserve a key or import/broadcast a wallet transaction. Keep the 3280 // behind-the-network check compares against the median of those
3222 // command names explicit so old peers get a quiet no-op instead of a 3281 // numbers, so as connections age it drifts toward "everyone is
3223 // wallet side effect. 3282 // behind us" and quietly stops firing -- the opposite of its job.
3224 if (LogAcceptsCategory("net")) 3283 map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(inv.hash);
3225 printf("ProcessMessage(%s) : disabled legacy order message\n", strCommand.c_str()); 3284 if (mi != mapBlockIndex.end() && mi->second)
3226 vRecv.clear(); 3285 pfrom->nStartingHeight = max(pfrom->nStartingHeight, mi->second->nHeight);
3286 }
3227 } 3287 }
3228 3288
3229 3289
3230 else if (strCommand == "reply") 3290 // "checkorder", "submitorder" and "reply" -- the 2009 buy-from-a-merchant
3231 { 3291 // exchange -- used to be handled here. All three are gone.
3232 uint256 hashReply; 3292 //
3233 vRecv >> hashReply; 3293 // "checkorder" called GenerateNewKey() for any peer that asked, which is
3234 3294 // AddKey() and a wallet.dat write, and handed the pubkey back. The comment
3235 CRequestTracker tracker; 3295 // said "keep giving the same key to the same ip until they use it", and on
3236 CRITICAL_BLOCK(pfrom->cs_mapRequests) 3296 // a 2009 network of routable IPs that held. It does not hold here: every
3237 { 3297 // inbound peer arrives through a rendezvous tunnel and is tagged with
3238 map<uint256, CRequestTracker>::iterator mi = pfrom->mapRequests.find(hashReply); 3298 // BtfMarkerAddr() over three random bytes (net.cpp), so addr.ip is freshly
3239 if (mi != pfrom->mapRequests.end()) 3299 // random per connection and the cache never hits. Reconnect in a loop and
3240 { 3300 // the node mints a key per connection -- unbounded wallet growth, unbounded
3241 tracker = (*mi).second; 3301 // growth of the static map that held them, and every one of those keys
3242 pfrom->mapRequests.erase(mi); 3302 // missing from every backup taken beforehand, which for this pre-BIP32
3243 } 3303 // wallet is how coins are lost.
3244 } 3304 //
3245 if (!tracker.IsNull()) 3305 // "submitorder" took a CWalletTx from an unauthenticated peer and, if it
3246 tracker.fn(tracker.param1, vRecv); 3306 // was merely a valid transaction, AddToWallet()'d it. Valid is not the same
3247 } 3307 // as ours: any peer could write chosen transactions into wallet.dat.
3308 //
3309 // "reply" dispatched to a CRequestTracker callback. Nothing in this tree
3310 // has ever called PushRequest, so mapRequests was permanently empty and the
3311 // handler could only ever be a parser for hostile input.
3248 3312
3249 3313
3250 else if (strCommand == "ping") 3314 else if (strCommand == "ping")
@@ -3502,6 +3566,102 @@ static bool StratumRecvLine(SOCKET s, std::string& buf,
3502// How long to wait for a mining.notify after authorize before giving up 3566// How long to wait for a mining.notify after authorize before giving up
3503static const int64 STUCK_NO_JOB_TIMEOUT_SECS = 120; 3567static const int64 STUCK_NO_JOB_TIMEOUT_SECS = 120;
3504 3568
3569// ------------------------------------------------------------------
3570// Multi-threaded pool worker. The original loop hashed a single nonce
3571// per blocking 50ms socket poll, capping one instance at ~20 H/s no
3572// matter how many cores the machine has. The socket and job bookkeeping
3573// stay on the calling thread; N hash threads grind nonce ranges on their
3574// own RandomX VMs (all sharing the one ~2 GB dataset) and drop found
3575// shares on a queue the I/O thread drains and submits.
3576// ------------------------------------------------------------------
3577struct ParticipantJobShared
3578{
3579 std::mutex mtx;
3580 unsigned char header[80];
3581 uint256 target;
3582 std::string jobId;
3583 uint64 jobSeq;
3584 bool haveJob;
3585 bool stop;
3586 ParticipantJobShared() : jobSeq(0), haveJob(false), stop(false)
3587 {
3588 memset(header, 0, sizeof(header));
3589 }
3590};
3591
3592struct ParticipantShareHit
3593{
3594 uint64 jobSeq;
3595 std::string jobId;
3596 unsigned int nonce;
3597};
3598
3599static void ParticipantHashWorker(ParticipantJobShared* sh, int idx, int nWorkers,
3600 std::vector<ParticipantShareHit>* hits,
3601 std::mutex* hitsMtx)
3602{
3603 void* vm = RandomXCreateMinerVM();
3604 if (!vm)
3605 {
3606 LogPrint("worker", "[worker] hash thread %d: RandomX VM creation failed\n", idx);
3607 return;
3608 }
3609
3610 unsigned char header[80];
3611 uint256 target;
3612 std::string jobId;
3613 uint64 mySeq = 0;
3614 bool active = false;
3615 unsigned int nNonce = 0;
3616 uint64 localHashes = 0;
3617
3618 while (!fShutdown)
3619 {
3620 {
3621 std::lock_guard<std::mutex> lk(sh->mtx);
3622 if (sh->stop)
3623 break;
3624 if (sh->haveJob && sh->jobSeq != mySeq)
3625 {
3626 memcpy(header, sh->header, 80);
3627 target = sh->target;
3628 jobId = sh->jobId;
3629 mySeq = sh->jobSeq;
3630 nNonce = (unsigned int)(idx + 1);
3631 active = true;
3632 }
3633 }
3634 if (!active)
3635 {
3636 Sleep(50);
3637 continue;
3638 }
3639
3640 memcpy(header + 76, &nNonce, 4);
3641 uint256 hash = RandomXHashWithVM(vm, header, 80);
3642 localHashes++;
3643 if ((localHashes & 0x3ff) == 0)
3644 gParticipantHashes.fetch_add(0x400);
3645
3646 if (hash <= target)
3647 {
3648 LogPrint("worker", "[worker] share found! nonce=%u job=%s (thread %d)\n",
3649 nNonce, jobId.c_str(), idx);
3650 std::lock_guard<std::mutex> lk(*hitsMtx);
3651 hits->push_back(ParticipantShareHit{mySeq, jobId, nNonce});
3652 }
3653
3654 unsigned int next = nNonce + (unsigned int)nWorkers;
3655 if (next < nNonce) // wrapped past 2^32: range exhausted, wait for next job
3656 active = false;
3657 else
3658 nNonce = next;
3659 }
3660
3661 gParticipantHashes.fetch_add(localHashes & 0x3ff);
3662 RandomXDestroyMinerVM(vm);
3663}
3664
3505static bool PoolParticipantMiner() 3665static bool PoolParticipantMiner()
3506{ 3666{
3507 // ------------------------------------------------------------------ 3667 // ------------------------------------------------------------------
@@ -3516,6 +3676,51 @@ static bool PoolParticipantMiner()
3516 LogPrint("worker", "[worker] connecting to pool %s\n", strParticipantPool.c_str()); 3676 LogPrint("worker", "[worker] connecting to pool %s\n", strParticipantPool.c_str());
3517 SetParticipantMiningStatus("resolving pool address"); 3677 SetParticipantMiningStatus("resolving pool address");
3518 3678
3679 // ------------------------------------------------------------------
3680 // Steps 1-2: connect to the pool. Two address forms:
3681 // tcp:HOST:PORT -- plain TCP straight to the pool's stratum port
3682 // (e.g. our own operator node on this machine),
3683 // no .btf resolution or rendezvous tunnel involved
3684 // <name>.btf -- resolve via Nostr, dial through a rendezvous relay
3685 // ------------------------------------------------------------------
3686 btf_socket_t s = INVALID_SOCKET;
3687 if (strParticipantPool.compare(0, 4, "tcp:") == 0)
3688 {
3689 std::string hp = strParticipantPool.substr(4);
3690 size_t colon = hp.rfind(':');
3691 if (colon == std::string::npos) {
3692 LogPrint("worker", "[worker] bad tcp pool address '%s' (want tcp:host:port)\n",
3693 strParticipantPool.c_str());
3694 SetParticipantMiningStatus("bad pool address");
3695 return false;
3696 }
3697 std::string host = hp.substr(0, colon);
3698 int port = atoi(hp.substr(colon + 1).c_str());
3699 unsigned long ip = inet_addr(host.c_str());
3700 if (port <= 0 || port > 65535 || ip == INADDR_NONE) {
3701 LogPrint("worker", "[worker] bad tcp pool endpoint '%s'\n", hp.c_str());
3702 SetParticipantMiningStatus("bad pool endpoint");
3703 return false;
3704 }
3705 SetParticipantMiningStatus("connecting to pool (direct tcp)");
3706 struct sockaddr_in sa;
3707 memset(&sa, 0, sizeof(sa));
3708 sa.sin_family = AF_INET;
3709 sa.sin_port = htons((unsigned short)port);
3710 sa.sin_addr.s_addr = ip;
3711 SOCKET cs = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
3712 if (cs == INVALID_SOCKET || connect(cs, (struct sockaddr*)&sa, sizeof(sa)) == SOCKET_ERROR) {
3713 LogPrint("worker", "[worker] direct connect to %s FAILED\n", hp.c_str());
3714 SetParticipantMiningStatus("pool connect failed");
3715 if (cs != INVALID_SOCKET) closesocket(cs);
3716 return false;
3717 }
3718 LogPrint("worker", "[worker] connected directly to pool %s\n", hp.c_str());
3719 s = (btf_socket_t)cs;
3720 goto connected;
3721 }
3722
3723 {
3519 unsigned char target_pubkey[32]; 3724 unsigned char target_pubkey[32];
3520 if (!btf::ParseAddress(strParticipantPool, target_pubkey)) { 3725 if (!btf::ParseAddress(strParticipantPool, target_pubkey)) {
3521 LogPrint("worker", "[worker] invalid .btf address: %s\n", strParticipantPool.c_str()); 3726 LogPrint("worker", "[worker] invalid .btf address: %s\n", strParticipantPool.c_str());
@@ -3550,11 +3755,11 @@ static bool PoolParticipantMiner()
3550 } 3755 }
3551 3756
3552 // ------------------------------------------------------------------ 3757 // ------------------------------------------------------------------
3553 // Step 2: open encrypted tunnel to pool 3758 // Step 2b: open encrypted tunnel to pool
3554 // ------------------------------------------------------------------ 3759 // ------------------------------------------------------------------
3555 SetParticipantMiningStatus("opening tunnel"); 3760 SetParticipantMiningStatus("opening tunnel");
3556 btf_socket_t s = btf::BtfClientTunnel(host.c_str(), (unsigned short)port, 3761 s = btf::BtfClientTunnel(host.c_str(), (unsigned short)port,
3557 target_pubkey, service_enc_pub); 3762 target_pubkey, service_enc_pub);
3558 if (s == INVALID_SOCKET) { 3763 if (s == INVALID_SOCKET) {
3559 LogPrint("worker", "[worker] tunnel connect FAILED via %s" 3764 LogPrint("worker", "[worker] tunnel connect FAILED via %s"
3560 " (relay unreachable or pool not listening)\n", 3765 " (relay unreachable or pool not listening)\n",
@@ -3564,11 +3769,48 @@ static bool PoolParticipantMiner()
3564 } 3769 }
3565 LogPrint("worker", "[worker] tunnel open via %s\n", meetingHostPort.c_str()); 3770 LogPrint("worker", "[worker] tunnel open via %s\n", meetingHostPort.c_str());
3566 SetParticipantMiningStatus("tunnel open"); 3771 SetParticipantMiningStatus("tunnel open");
3772 }
3773
3774connected:
3567 3775
3568 // ------------------------------------------------------------------ 3776 // ------------------------------------------------------------------
3569 // Step 3: determine our payout address (mining username) 3777 // Step 3: determine our payout address (mining username). /payoutaddr
3778 // wins -- it lets a machine mine to an address whose keys live
3779 // elsewhere, so no wallet file is needed on the mining machine.
3780 // /payoutfile rotates through a list of such addresses: one is picked
3781 // per (re)connection, and the session self-limits (see rotation break
3782 // in the main loop) so consecutive rounds land on fresh addresses.
3570 // ------------------------------------------------------------------ 3783 // ------------------------------------------------------------------
3571 std::string username; 3784 static std::vector<std::string> vPayoutAddrs;
3785 static size_t nPayoutRot = 0;
3786 if (!strPayoutFile.empty() && vPayoutAddrs.empty())
3787 {
3788 FILE* f = fopen(strPayoutFile.c_str(), "r");
3789 if (f)
3790 {
3791 char line[256];
3792 while (fgets(line, sizeof(line), f))
3793 {
3794 std::string a = line;
3795 while (!a.empty() && (a.back() == '\n' || a.back() == '\r' ||
3796 a.back() == ' ' || a.back() == '\t'))
3797 a.pop_back();
3798 if (!a.empty())
3799 vPayoutAddrs.push_back(a);
3800 }
3801 fclose(f);
3802 LogPrint("worker", "[worker] loaded %zu payout addresses from %s\n",
3803 vPayoutAddrs.size(), strPayoutFile.c_str());
3804 }
3805 else
3806 LogPrint("worker", "[worker] cannot open payout file %s\n",
3807 strPayoutFile.c_str());
3808 }
3809
3810 std::string username = strPayoutAddress;
3811 if (username.empty() && !vPayoutAddrs.empty())
3812 username = vPayoutAddrs[nPayoutRot++ % vPayoutAddrs.size()];
3813 if (username.empty())
3572 { 3814 {
3573 std::vector<unsigned char> vchPubKey; 3815 std::vector<unsigned char> vchPubKey;
3574 if (CWalletDB("r").ReadDefaultKey(vchPubKey)) 3816 if (CWalletDB("r").ReadDefaultKey(vchPubKey))
@@ -3606,16 +3848,14 @@ static bool PoolParticipantMiner()
3606 // ------------------------------------------------------------------ 3848 // ------------------------------------------------------------------
3607 if (!RandomXFastReady()) { 3849 if (!RandomXFastReady()) {
3608 SetParticipantMiningStatus("building RandomX dataset (~2GB, one-time)"); 3850 SetParticipantMiningStatus("building RandomX dataset (~2GB, one-time)");
3609 unsigned int nThreads = std::thread::hardware_concurrency(); 3851 // Fill with the cores we are permitted to use, not the ones the host
3852 // happens to have. More threads than the cpuset allows does not build
3853 // the dataset any faster -- the CPU budget is the same either way --
3854 // it just oversubscribes, and this runs alongside the pool node.
3855 unsigned int nThreads = AvailableCoreCount();
3610 RandomXInitDataset(nThreads > 0 ? (int)nThreads : 1); 3856 RandomXInitDataset(nThreads > 0 ? (int)nThreads : 1);
3611 } 3857 }
3612 void* rxvm = RandomXCreateMinerVM(); 3858 LogPrint("worker", "[worker] RandomX dataset ready (%s)\n",
3613 if (!rxvm) {
3614 LogPrint("worker", "[worker] failed to create RandomX VM\n");
3615 SetParticipantMiningStatus("RandomX init failed");
3616 closesocket(s); return false;
3617 }
3618 LogPrint("worker", "[worker] RandomX VM ready (%s)\n",
3619 RandomXFastReady() ? "fast 2GB" : "light 256MB"); 3859 RandomXFastReady() ? "fast 2GB" : "light 256MB");
3620 3860
3621 // ------------------------------------------------------------------ 3861 // ------------------------------------------------------------------
@@ -3633,7 +3873,6 @@ static bool PoolParticipantMiner()
3633 {"params",json::array()}})) { 3873 {"params",json::array()}})) {
3634 LogPrint("worker", "[worker] subscribe send FAILED -- pool dropped connection\n"); 3874 LogPrint("worker", "[worker] subscribe send FAILED -- pool dropped connection\n");
3635 SetParticipantMiningStatus("subscribe send failed"); 3875 SetParticipantMiningStatus("subscribe send failed");
3636 RandomXDestroyMinerVM(rxvm);
3637 closesocket(s); return false; 3876 closesocket(s); return false;
3638 } 3877 }
3639 3878
@@ -3643,11 +3882,31 @@ static bool PoolParticipantMiner()
3643 {"params",json::array({username,"x"})}})) { 3882 {"params",json::array({username,"x"})}})) {
3644 LogPrint("worker", "[worker] authorize send FAILED -- pool dropped connection\n"); 3883 LogPrint("worker", "[worker] authorize send FAILED -- pool dropped connection\n");
3645 SetParticipantMiningStatus("authorize send failed"); 3884 SetParticipantMiningStatus("authorize send failed");
3646 RandomXDestroyMinerVM(rxvm);
3647 closesocket(s); return false; 3885 closesocket(s); return false;
3648 } 3886 }
3649 3887
3650 // ------------------------------------------------------------------ 3888 // ------------------------------------------------------------------
3889 // Step 6: spawn hash threads (each with its own RandomX VM on the
3890 // shared dataset), then run the I/O loop below: receive pool messages,
3891 // publish jobs to the threads, submit the shares they find.
3892 // ------------------------------------------------------------------
3893 // One count, decided in one place (main.cpp:MinerThreadCount's neighbour),
3894 // so that what StartMinerThreads() reports at startup and what actually
3895 // hashes here can never drift apart. Explicit /hashthreads=N is honored
3896 // exactly, including 1; automatic leaves one core for node + I/O threads.
3897 int nWorkers = ParticipantHashThreadCount();
3898 ParticipantJobShared jobShared;
3899 std::vector<ParticipantShareHit> shareHits;
3900 std::mutex shareHitsMtx;
3901 std::vector<std::thread> hashThreads;
3902 for (int i = 0; i < nWorkers; i++)
3903 hashThreads.emplace_back(ParticipantHashWorker, &jobShared, i, nWorkers,
3904 &shareHits, &shareHitsMtx);
3905 LogPrint("worker", "[worker] spawned %d hash threads on shared RandomX dataset\n",
3906 nWorkers);
3907 const uint64 sessionStartHashes = gParticipantHashes.load();
3908
3909 // ------------------------------------------------------------------
3651 // Step 6: main loop -- receive messages, hash, submit shares 3910 // Step 6: main loop -- receive messages, hash, submit shares
3652 // ------------------------------------------------------------------ 3911 // ------------------------------------------------------------------
3653 std::string jobId; 3912 std::string jobId;
@@ -3664,6 +3923,8 @@ static bool PoolParticipantMiner()
3664 uint64 sessionSharesSent = 0; // this connection only 3923 uint64 sessionSharesSent = 0; // this connection only
3665 uint64 sessionSharesAccepted = 0; 3924 uint64 sessionSharesAccepted = 0;
3666 int64 sampleStart = GetTime(); 3925 int64 sampleStart = GetTime();
3926 int64 connStart = GetTime();
3927 const int64 PAYOUT_ROTATE_SECS = 300; // reconnect with next /payoutfile address
3667 3928
3668 SetParticipantMiningStatus("waiting for pool response"); 3929 SetParticipantMiningStatus("waiting for pool response");
3669 3930
@@ -3771,6 +4032,14 @@ static bool PoolParticipantMiner()
3771 memcpy(&target, tgtBytes.data(), 32); 4032 memcpy(&target, tgtBytes.data(), 32);
3772 haveJob = true; 4033 haveJob = true;
3773 nNonce = 1; 4034 nNonce = 1;
4035 {
4036 std::lock_guard<std::mutex> lk(jobShared.mtx);
4037 memcpy(jobShared.header, header, 80);
4038 jobShared.target = target;
4039 jobShared.jobId = jobId;
4040 jobShared.jobSeq++;
4041 jobShared.haveJob = true;
4042 }
3774 LogPrint("worker", "[pool->worker] new job %s received\n", 4043 LogPrint("worker", "[pool->worker] new job %s received\n",
3775 jobId.c_str()); 4044 jobId.c_str());
3776 SetParticipantMiningStatus("hashing job " + jobId); 4045 SetParticipantMiningStatus("hashing job " + jobId);
@@ -3826,46 +4095,71 @@ static bool PoolParticipantMiner()
3826 } 4095 }
3827 } 4096 }
3828 4097
4098 // -- Payout address rotation: end this session cleanly so the outer
4099 // loop reconnects and authorizes the next /payoutfile address --
4100 if (!vPayoutAddrs.empty() && GetTime() - connStart > PAYOUT_ROTATE_SECS)
4101 {
4102 LogPrint("worker", "[worker] payout rotation interval reached"
4103 " -- reconnecting with next address\n");
4104 break;
4105 }
4106
3829 if (!haveJob) { Sleep(100); continue; } 4107 if (!haveJob) { Sleep(100); continue; }
3830 4108
3831 // -- Hash one nonce -- 4109 // -- Hash rate accounting (hashing happens on the worker threads) --
3832 memcpy(header+76, &nNonce, 4); 4110 hashesSinceSample = gParticipantHashes.load() - sessionStartHashes;
3833 uint256 hash = RandomXHashWithVM(rxvm, header, 80); 4111 UpdateParticipantHashRate(hashesSinceSample, sampleStart);
3834 hashesSinceSample++;
3835 gParticipantHashes.fetch_add(1);
3836 if ((hashesSinceSample & 0x3ff) == 0)
3837 UpdateParticipantHashRate(hashesSinceSample, sampleStart);
3838
3839 if (hash <= target) {
3840 LogPrint("worker", "[worker] share found! nonce=%u job=%s difficulty=%.4f\n",
3841 nNonce, jobId.c_str(), currentDifficulty);
3842 SetParticipantMiningStatus("share found; submitting");
3843 int submitId = msgId++;
3844 pendingSubmitIds.insert(submitId);
3845 gParticipantSharesSent.fetch_add(1);
3846 sessionSharesSent++;
3847 json submit = {{"id",submitId},{"method","mining.submit"},
3848 {"params",json::array({username, jobId, ToHexStr(&nNonce,4)})}};
3849 if (!StratumSendLine(s, submit)) {
3850 LogPrint("worker", "[worker] share submit send FAILED"
3851 " -- pool connection lost\n");
3852 break;
3853 }
3854 }
3855 4112
3856 nNonce++; 4113 // -- Submit any shares the hash threads found --
3857 if (nNonce == 0) { 4114 bool submitFailed = false;
3858 LogPrint("worker", "[worker] nonce space exhausted for job %s" 4115 {
3859 " -- waiting for new job\n", jobId.c_str()); 4116 std::vector<ParticipantShareHit> pending;
3860 haveJob = false; 4117 {
4118 std::lock_guard<std::mutex> lk(shareHitsMtx);
4119 pending.swap(shareHits);
4120 }
4121 for (ParticipantShareHit& hit : pending)
4122 {
4123 if (hit.jobId != jobId)
4124 {
4125 LogPrint("worker", "[worker] dropping stale share for old job %s\n",
4126 hit.jobId.c_str());
4127 continue;
4128 }
4129 LogPrint("worker", "[worker] share found! nonce=%u job=%s difficulty=%.4f\n",
4130 hit.nonce, hit.jobId.c_str(), currentDifficulty);
4131 SetParticipantMiningStatus("share found; submitting");
4132 int submitId = msgId++;
4133 pendingSubmitIds.insert(submitId);
4134 gParticipantSharesSent.fetch_add(1);
4135 sessionSharesSent++;
4136 json submit = {{"id",submitId},{"method","mining.submit"},
4137 {"params",json::array({username, hit.jobId,
4138 ToHexStr(&hit.nonce,4)})}};
4139 if (!StratumSendLine(s, submit)) {
4140 LogPrint("worker", "[worker] share submit send FAILED"
4141 " -- pool connection lost\n");
4142 submitFailed = true;
4143 break;
4144 }
4145 }
3861 } 4146 }
4147 if (submitFailed)
4148 break;
3862 } 4149 }
3863 4150
3864 // ------------------------------------------------------------------ 4151 // ------------------------------------------------------------------
3865 // Cleanup 4152 // Cleanup: stop the hash threads, then drop the pool connection
3866 // ------------------------------------------------------------------ 4153 // ------------------------------------------------------------------
3867 RandomXDestroyMinerVM(rxvm); 4154 {
4155 std::lock_guard<std::mutex> lk(jobShared.mtx);
4156 jobShared.stop = true;
4157 jobShared.haveJob = false;
4158 }
4159 for (std::thread& t : hashThreads)
4160 t.join();
3868 closesocket(s); 4161 closesocket(s);
4162 hashesSinceSample = gParticipantHashes.load() - sessionStartHashes;
3869 UpdateParticipantHashRate(hashesSinceSample, sampleStart); 4163 UpdateParticipantHashRate(hashesSinceSample, sampleStart);
3870 SetParticipantMiningStatus("disconnected"); 4164 SetParticipantMiningStatus("disconnected");
3871 LogPrint("worker", "[worker] disconnected from pool" 4165 LogPrint("worker", "[worker] disconnected from pool"
@@ -4090,7 +4384,7 @@ bool BitcoinMiner(int nThreadId)
4090 // Build the fast (~2 GB dataset) RandomX mode once, up front, so we're 4384 // Build the fast (~2 GB dataset) RandomX mode once, up front, so we're
4091 // not stuck mining in the much slower light/cache-only mode forever. 4385 // not stuck mining in the much slower light/cache-only mode forever.
4092 if (!RandomXFastReady()) { 4386 if (!RandomXFastReady()) {
4093 unsigned int nThreads = std::thread::hardware_concurrency(); 4387 unsigned int nThreads = AvailableCoreCount();
4094 RandomXInitDataset(nThreads > 0 ? (int)nThreads : 1); 4388 RandomXInitDataset(nThreads > 0 ? (int)nThreads : 1);
4095 } 4389 }
4096 4390
@@ -4177,7 +4471,19 @@ bool BitcoinMiner(int nThreadId)
4177 vector<char> vfAlreadyAdded(mapTransactions.size()); 4471 vector<char> vfAlreadyAdded(mapTransactions.size());
4178 bool fFoundSomething = true; 4472 bool fFoundSomething = true;
4179 unsigned int nBlockSize = 0; 4473 unsigned int nBlockSize = 0;
4180 while (fFoundSomething && nBlockSize < MAX_SIZE/2) 4474 // Assembly must stop below the consensus cap that CheckBlock()
4475 // enforces, or we mine a block our own node then rejects.
4476 //
4477 // This used to read MAX_SIZE/2 (16 MB), which was consistent while
4478 // CheckBlock() also tested against MAX_SIZE (32 MB). Adding
4479 // MAX_BLOCK_SIZE inverted it: assembly could fill 16 MB and
4480 // validation would refuse anything over 1 MB, so a full mempool
4481 // meant solving a block and throwing it away -- with this pool at
4482 // ~90% of hashrate, that is the network stalling, not just us.
4483 //
4484 // Half the cap, keeping the original margin: the test runs before
4485 // a transaction is added, so the loop can overshoot by one tx.
4486 while (fFoundSomething && nBlockSize < MAX_BLOCK_SIZE/2)
4181 { 4487 {
4182 fFoundSomething = false; 4488 fFoundSomething = false;
4183 unsigned int n = 0; 4489 unsigned int n = 0;
@@ -4604,9 +4910,17 @@ bool SelectCoins(int64 nTargetValue, set<CWalletTx*>& setCoinsRet)
4604 4910
4605 4911
4606 4912
4607bool CreateTransaction(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew, int64& nFeeRequiredRet) 4913// presultRet, when given, says WHY this returned false: the caller cannot work
4914// it out afterwards. "Coin selection came up short" and "the wallet could not
4915// build the transaction" are indistinguishable from the outside -- both are
4916// just `false` -- and the difference decides whether a pool defers a payment
4917// or counts it against the payment's retry budget. See WalletSendResult.
4918bool CreateTransaction(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew, int64& nFeeRequiredRet,
4919 WalletSendResult* presultRet)
4608{ 4920{
4609 nFeeRequiredRet = 0; 4921 nFeeRequiredRet = 0;
4922 if (presultRet)
4923 *presultRet = WALLET_SEND_FAILED;
4610 CRITICAL_BLOCK(cs_main) 4924 CRITICAL_BLOCK(cs_main)
4611 { 4925 {
4612 // txdb must be opened before the mapWallet lock 4926 // txdb must be opened before the mapWallet lock
@@ -4637,10 +4951,21 @@ bool CreateTransaction(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew, in
4637 // fixed this in 107d9e288. 4951 // fixed this in 107d9e288.
4638 int64 nTotalValue = nValue + nFee; 4952 int64 nTotalValue = nValue + nFee;
4639 4953
4640 // Choose coins to use 4954 // Choose coins to use.
4955 //
4956 // Failing here is a funding shortfall and nothing else: there
4957 // is not enough spendable coin for nValue plus the fee this
4958 // pass is working with. On the second and later passes that
4959 // fee is GetMinFee() of the transaction the first pass built,
4960 // so this is also how a wallet that can afford the amount but
4961 // not the 11 * CENT charged at 10,000 bytes reports itself.
4641 set<CWalletTx*> setCoins; 4962 set<CWalletTx*> setCoins;
4642 if (!SelectCoins(nTotalValue, setCoins)) 4963 if (!SelectCoins(nTotalValue, setCoins))
4964 {
4965 if (presultRet)
4966 *presultRet = WALLET_SEND_INSUFFICIENT_FUNDS;
4643 return false; 4967 return false;
4968 }
4644 int64 nValueIn = 0; 4969 int64 nValueIn = 0;
4645 foreach(CWalletTx* pcoin, setCoins) 4970 foreach(CWalletTx* pcoin, setCoins)
4646 nValueIn += pcoin->GetCredit(); 4971 nValueIn += pcoin->GetCredit();
@@ -4710,6 +5035,8 @@ bool CreateTransaction(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew, in
4710 } 5035 }
4711 } 5036 }
4712 } 5037 }
5038 if (presultRet)
5039 *presultRet = WALLET_SEND_OK;
4713 return true; 5040 return true;
4714} 5041}
4715 5042
@@ -4772,16 +5099,38 @@ bool CommitTransactionSpent(const CWalletTx& wtxNew)
4772 5099
4773 5100
4774 5101
4775bool SendMoney(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew) 5102// outcomeRet reports what happened rather than only whether it worked. A
5103// caller that has to decide what a failure MEANS -- the pool payout thread --
5104// cannot reconstruct it from a balance reading afterwards, because the fee
5105// that was actually required depends on the size of a transaction that was
5106// never built. See WalletSendResult.
5107bool SendMoney(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew,
5108 SendMoneyOutcome& outcomeRet)
4776{ 5109{
5110 outcomeRet.result = WALLET_SEND_FAILED;
5111 outcomeRet.nFeeRequired = 0;
4777 CRITICAL_BLOCK(cs_main) 5112 CRITICAL_BLOCK(cs_main)
4778 { 5113 {
4779 int64 nFeeRequired; 5114 int64 nFeeRequired;
4780 if (!CreateTransaction(scriptPubKey, nValue, wtxNew, nFeeRequired)) 5115 WalletSendResult createResult = WALLET_SEND_FAILED;
5116 if (!CreateTransaction(scriptPubKey, nValue, wtxNew, nFeeRequired, &createResult))
4781 { 5117 {
5118 outcomeRet.result = createResult;
5119 outcomeRet.nFeeRequired = nFeeRequired;
4782 string strError; 5120 string strError;
4783 if (nValue + nFeeRequired > GetBalance()) 5121 if (nValue + nFeeRequired > GetBalance())
4784 strError = strprintf("Error: This is an oversized transaction that requires a transaction fee of %s ", FormatMoney(nFeeRequired).c_str()); 5122 // Not an oversized transaction -- this branch is simply "the
5123 // spendable balance is too small". Saying otherwise (with a fee
5124 // of 0.00, because CreateTransaction failed before setting one)
5125 // sends anyone reading the log after a failed payout hunting a
5126 // transaction-size problem that does not exist.
5127 strError = strprintf("Error: insufficient spendable balance:"
5128 " need %s (plus fee %s) but only %s is spendable"
5129 " -- mined coin counts as zero until it is %d deep",
5130 FormatMoney(nValue).c_str(),
5131 FormatMoney(nFeeRequired).c_str(),
5132 FormatMoney(GetBalance()).c_str(),
5133 COINBASE_SPENDABLE_DEPTH);
4785 else 5134 else
4786 strError = "Error: Transaction creation failed "; 5135 strError = "Error: Transaction creation failed ";
4787 return error("SendMoney() : %s\n", strError.c_str()); 5136 return error("SendMoney() : %s\n", strError.c_str());
@@ -4805,6 +5154,13 @@ bool SendMoney(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew)
4805 } 5154 }
4806 wtxNew.RelayWalletTransaction(); 5155 wtxNew.RelayWalletTransaction();
4807 } 5156 }
5157 outcomeRet.result = WALLET_SEND_OK;
4808 MainFrameRepaint(); 5158 MainFrameRepaint();
4809 return true; 5159 return true;
4810} 5160}
5161
5162bool SendMoney(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew)
5163{
5164 SendMoneyOutcome ignored;
5165 return SendMoney(scriptPubKey, nValue, wtxNew, ignored);
5166}
diff --git a/src/main.h b/src/main.h
index dc5527e..75b6db7 100644
--- a/src/main.h
+++ b/src/main.h
@@ -41,6 +41,12 @@ static const int64 MAX_MONEY = 21000000 * COIN;
41inline bool MoneyRange(int64 nValue) { return (nValue >= 0 && nValue <= MAX_MONEY); } 41inline bool MoneyRange(int64 nValue) { return (nValue >= 0 && nValue <= MAX_MONEY); }
42static const int64 CENT = 1000000; 42static const int64 CENT = 1000000;
43static const int COINBASE_MATURITY = 100; 43static const int COINBASE_MATURITY = 100;
44// Depth at which a coinbase we mined is actually usable. CMerkleTx::GetCredit
45// values a coinbase at zero until GetBlocksToMaturity() reaches 0, which is
46// this depth -- not COINBASE_MATURITY. Anything that plans to *spend* mined
47// coin must gate on this, or it will try to pay out of a balance the wallet
48// still reports as zero. Both rules live here so they cannot drift apart.
49static const int COINBASE_SPENDABLE_DEPTH = COINBASE_MATURITY + 20;
44 50
45// Bitflash difficulty floor. The PoW is RandomX (memory-hard), ~1000x slower 51// Bitflash difficulty floor. The PoW is RandomX (memory-hard), ~1000x slower
46// per hash than SHA-256d, so the target is much easier than Bitcoin's to let a 52// per hash than SHA-256d, so the target is much easier than Bitcoin's to let a
@@ -84,6 +90,11 @@ extern int nMineMode;
84// something a previous session wrote. 90// something a previous session wrote.
85extern bool fMineModeFromCommandLine; 91extern bool fMineModeFromCommandLine;
86extern string strParticipantPool; // participant: pool .btf address 92extern string strParticipantPool; // participant: pool .btf address
93extern string strPayoutAddress; // participant: /payoutaddr override (mine without a wallet)
94extern string strPayoutFile; // participant: /payoutfile address list, rotated per reconnect
95extern bool fPoolNoAnnounce; // /noannounce: suppress pool announcements
96extern bool fPoolMineBlind; // /poolmineblind: mine on even when deaf to every peer
97extern int nHashThreads; // participant: /hashthreads override (0 = cores-1)
87extern string strPoolName; // operator: announced pool name 98extern string strPoolName; // operator: announced pool name
88extern string strPoolDashboardUrl; // operator: optional dashboard URL 99extern string strPoolDashboardUrl; // operator: optional dashboard URL
89extern string strPoolStatusFile; // operator: optional JSON status output path 100extern string strPoolStatusFile; // operator: optional JSON status output path
@@ -96,6 +107,9 @@ extern bool fSoloMineTest; // /solomine: mine without requiring a peer (local te
96extern int nMinerThreads; 107extern int nMinerThreads;
97// Resolves the automatic case into a real count. 108// Resolves the automatic case into a real count.
98int MinerThreadCount(); 109int MinerThreadCount();
110// Hash threads one participant session runs; resolves /hashthreads. Not the
111// same number as MinerThreadCount(), and only ONE session ever runs.
112int ParticipantHashThreadCount();
99extern bool gPoolServerRunning; // GUI/startup state 113extern bool gPoolServerRunning; // GUI/startup state
100extern volatile bool gPoolRunning; // runtime pool server loop flag (rpc.cpp) 114extern volatile bool gPoolRunning; // runtime pool server loop flag (rpc.cpp)
101 115
@@ -330,9 +344,52 @@ struct WalletRecoveryAudit
330 } 344 }
331}; 345};
332WalletRecoveryAudit GetWalletRecoveryAudit(); 346WalletRecoveryAudit GetWalletRecoveryAudit();
333bool CreateTransaction(CScript scriptPubKey, int64 nValue, CWalletTx& txNew, int64& nFeeRequiredRet); 347
348// Why a send did not go through.
349//
350// The pool's payout thread has to decide, on every failed SendMoney, whether
351// the failure was the wallet's condition (wait for it to change) or a fault
352// (something is wrong with this payment). It used to guess by re-reading
353// GetBalance() afterwards and allowing a fixed CENT of headroom for the fee.
354// That cannot work: the fee here is a step function of the SERIALISED SIZE of
355// a transaction that no longer exists, and the step is from 0 to 11 * CENT at
356// 10,000 bytes -- so a one-CENT allowance is either unnecessary or an order of
357// magnitude short, never right. A pool wallet close to broke selects many
358// small inputs BECAUSE it is close to broke, which is what pushes the
359// transaction over 10 KB, so the guess is wrong exactly when the money is
360// tight.
361//
362// The wallet already knows the answer; it just did not say. SelectCoins()
363// failing is "there is not enough spendable coin to cover the amount plus
364// whatever fee this size of transaction requires", and it is the only reason
365// CreateTransaction() can fail that is about funding rather than about the
366// wallet being broken.
367enum WalletSendResult
368{
369 WALLET_SEND_OK = 0,
370 WALLET_SEND_INSUFFICIENT_FUNDS, // coin selection could not cover amount+fee
371 WALLET_SEND_FAILED // anything else: could not build, sign or accept
372};
373
374// What the wallet did with a send request. nFeeRequired is the fee the
375// transaction turned out to need, which for a failed send is what coin
376// selection was asked for on top of the amount and could not find -- 0 if it
377// never got that far.
378struct SendMoneyOutcome
379{
380 WalletSendResult result = WALLET_SEND_OK;
381 int64 nFeeRequired = 0;
382};
383
384bool CreateTransaction(CScript scriptPubKey, int64 nValue, CWalletTx& txNew, int64& nFeeRequiredRet,
385 WalletSendResult* presultRet=NULL);
334bool CommitTransactionSpent(const CWalletTx& wtxNew); 386bool CommitTransactionSpent(const CWalletTx& wtxNew);
387// The three-argument form is kept, and kept as a distinct declaration rather
388// than a defaulted argument, because gui.cpp declares SendMoney itself instead
389// of including this header.
335bool SendMoney(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew); 390bool SendMoney(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew);
391bool SendMoney(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew,
392 SendMoneyOutcome& outcomeRet);
336 393
337 394
338 395
diff --git a/src/main_gui.cpp b/src/main_gui.cpp
index 943799f..2743429 100644
--- a/src/main_gui.cpp
+++ b/src/main_gui.cpp
@@ -123,6 +123,9 @@ static void PrintUsage()
123 printf("Mining mode:\n"); 123 printf("Mining mode:\n");
124 printf(" /operator\n"); 124 printf(" /operator\n");
125 printf(" /participant=POOL_BTF_ADDRESS\n"); 125 printf(" /participant=POOL_BTF_ADDRESS\n");
126 printf(" /payoutaddr=BTF_ADDRESS (participant: pay shares to this address, no wallet needed)\n");
127 printf(" /payoutfile=FILE (participant: rotate payout addresses from file)\n");
128 printf(" /hashthreads=N (participant: mining threads, default cores-1)\n");
126 printf(" /stratumbridge=POOL_BTF_ADDRESS\n"); 129 printf(" /stratumbridge=POOL_BTF_ADDRESS\n");
127 printf(" /stratumbridgeport=N (default 3333; listen on 127.0.0.1)\n"); 130 printf(" /stratumbridgeport=N (default 3333; listen on 127.0.0.1)\n");
128 printf(" /solomine\n"); 131 printf(" /solomine\n");
@@ -147,6 +150,7 @@ static void PrintUsage()
147 printf(" /socks=HOST:PORT (SOCKS5 proxy for Nostr, .btf relay, and onion dials)\n"); 150 printf(" /socks=HOST:PORT (SOCKS5 proxy for Nostr, .btf relay, and onion dials)\n");
148 printf(" /tor[=HOST:PORT] (Tor mode; default SOCKS5 proxy is 127.0.0.1:9050)\n"); 151 printf(" /tor[=HOST:PORT] (Tor mode; default SOCKS5 proxy is 127.0.0.1:9050)\n");
149 printf(" /managedtor[=PATH] (start Tor, create a hidden service, advertise its onion)\n"); 152 printf(" /managedtor[=PATH] (start Tor, create a hidden service, advertise its onion)\n");
153 printf(" /automanagedtor (auto-start a bundled Tor if one sits next to the executable)\n");
150 printf(" /nomanagedtor (disable automatic bundled Tor startup)\n"); 154 printf(" /nomanagedtor (disable automatic bundled Tor startup)\n");
151 printf(" /oniononly (do not fall back to rendezvous when a .btf onion dial fails)\n"); 155 printf(" /oniononly (do not fall back to rendezvous when a .btf onion dial fails)\n");
152 printf(" /btfseed=ADDRESS:ENCHEX (extra bootstrap peer, repeatable)\n"); 156 printf(" /btfseed=ADDRESS:ENCHEX (extra bootstrap peer, repeatable)\n");
@@ -234,7 +238,7 @@ static void ParseStartupArguments(int argc, char* argv[])
234 if (!strProcLimit.empty()) 238 if (!strProcLimit.empty())
235 { 239 {
236 int n = atoi(strProcLimit.c_str()); 240 int n = atoi(strProcLimit.c_str());
237 unsigned int nCores = std::thread::hardware_concurrency(); 241 unsigned int nCores = AvailableCoreCount();
238 if (n < 0) 242 if (n < 0)
239 fprintf(stderr, "Ignoring /genproclimit=%s: not a thread count\n", 243 fprintf(stderr, "Ignoring /genproclimit=%s: not a thread count\n",
240 strProcLimit.c_str()); 244 strProcLimit.c_str());
@@ -260,6 +264,18 @@ static void ParseStartupArguments(int argc, char* argv[])
260 fMineModeFromCommandLine = true; 264 fMineModeFromCommandLine = true;
261 } 265 }
262 266
267 string payoutAddr = argval2(argc, argv, "/payoutaddr", "-payoutaddr");
268 if (!payoutAddr.empty())
269 strPayoutAddress = payoutAddr;
270
271 string payoutFile = argval2(argc, argv, "/payoutfile", "-payoutfile");
272 if (!payoutFile.empty())
273 strPayoutFile = payoutFile;
274
275 string hashThreads = argval2(argc, argv, "/hashthreads", "-hashthreads");
276 if (!hashThreads.empty())
277 nHashThreads = atoi(hashThreads.c_str());
278
263 if (arg(argc,argv,"/stratumbridge") || arg(argc,argv,"-stratumbridge")) 279 if (arg(argc,argv,"/stratumbridge") || arg(argc,argv,"-stratumbridge"))
264 { 280 {
265 fStratumBridge = true; 281 fStratumBridge = true;
@@ -286,6 +302,12 @@ static void ParseStartupArguments(int argc, char* argv[])
286 if (!poolDash.empty()) 302 if (!poolDash.empty())
287 strPoolDashboardUrl = poolDash; 303 strPoolDashboardUrl = poolDash;
288 304
305 if (arg(argc,argv,"/noannounce") || arg(argc,argv,"-noannounce"))
306 fPoolNoAnnounce = true;
307
308 if (arg(argc,argv,"/poolmineblind") || arg(argc,argv,"-poolmineblind"))
309 fPoolMineBlind = true;
310
289 string poolFee = argval2(argc, argv, "/poolfee", "-poolfee"); 311 string poolFee = argval2(argc, argv, "/poolfee", "-poolfee");
290 if (!poolFee.empty()) 312 if (!poolFee.empty())
291 dPoolFeePercent = atof(poolFee.c_str()); 313 dPoolFeePercent = atof(poolFee.c_str());
@@ -306,8 +328,20 @@ static void ParseStartupArguments(int argc, char* argv[])
306 string rvRelay = argval2(argc, argv, "/rvrelay", "-rvrelay"); 328 string rvRelay = argval2(argc, argv, "/rvrelay", "-rvrelay");
307 if (!rvRelay.empty()) 329 if (!rvRelay.empty())
308 { 330 {
309 vBtfMeetingRelays.clear(); 331 if (!LooksLikeRelay(rvRelay))
310 vBtfMeetingRelays.push_back(rvRelay); 332 {
333 // A malformed pin must not replace the seed list: with the pin
334 // set, no discovered relay can ever mask the typo, and the pool
335 // would publish a garbage (or no) meeting node forever.
336 fprintf(stderr, "Ignoring /rvrelay=%s: expected host:port\n",
337 rvRelay.c_str());
338 }
339 else
340 {
341 vBtfMeetingRelays.clear();
342 vBtfMeetingRelays.push_back(rvRelay);
343 fBtfRelayPinned = true;
344 }
311 } 345 }
312 346
313 // net.cpp has described nListenPort as "tunable via /port" since it was 347 // net.cpp has described nListenPort as "tunable via /port" since it was
@@ -348,8 +382,16 @@ static void ParseStartupArguments(int argc, char* argv[])
348 bool fTorMode = arg(argc, argv, "/tor") || arg(argc, argv, "-tor"); 382 bool fTorMode = arg(argc, argv, "/tor") || arg(argc, argv, "-tor");
349 string socksProxy = argval2(argc, argv, "/socks", "-socks"); 383 string socksProxy = argval2(argc, argv, "/socks", "-socks");
350 string bundledTorPath; 384 string bundledTorPath;
385 // Managed Tor is opt-in on this fork. Upstream 1.2.19 auto-starts a
386 // bundled tor whenever one sits next to the executable, which silently
387 // moves Nostr/.btf traffic onto Tor, creates a hidden service, and starts
388 // publishing descriptors with meeting_node="rendezvous-pending" on nodes
389 // that never asked for it -- against our pinned-relay posture. The
390 // upstream behavior remains available under the explicit /automanagedtor.
351 bool fAutoManagedTor = !fManagedTor && !fNoManagedTor && !fTorMode && 391 bool fAutoManagedTor = !fManagedTor && !fNoManagedTor && !fTorMode &&
352 socksProxy.empty() && BtfBundledTorPath(bundledTorPath); 392 socksProxy.empty() &&
393 (arg(argc, argv, "/automanagedtor") || arg(argc, argv, "-automanagedtor")) &&
394 BtfBundledTorPath(bundledTorPath);
353 if (fManagedTor || fAutoManagedTor) 395 if (fManagedTor || fAutoManagedTor)
354 { 396 {
355 if (fManagedTor && (fTorMode || !socksProxy.empty())) 397 if (fManagedTor && (fTorMode || !socksProxy.empty()))
@@ -1030,6 +1072,31 @@ int main(int argc, char* argv[])
1030 1072
1031 ReacceptWalletTransactions(); 1073 ReacceptWalletTransactions();
1032 1074
1075 // /dumpaddresses=N:FILE -- generate N fresh wallet keys and write their
1076 // addresses (one per line) to FILE, then exit. Keys stay in this wallet;
1077 // the file alone is enough for a miner elsewhere to rotate payout
1078 // addresses via /payoutfile. (No HD/xpub exists in this wallet format.)
1079 string dumpSpec = argval2(argc, argv, "/dumpaddresses", "-dumpaddresses");
1080 if (!dumpSpec.empty())
1081 {
1082 size_t colon = dumpSpec.rfind(':');
1083 if (colon == string::npos) { fprintf(stderr, "usage: /dumpaddresses=N:FILE\n"); return 1; }
1084 int n = atoi(dumpSpec.substr(0, colon).c_str());
1085 string file = dumpSpec.substr(colon + 1);
1086 FILE* f = fopen(file.c_str(), "w");
1087 if (!f) { fprintf(stderr, "cannot open %s\n", file.c_str()); return 1; }
1088 for (int i = 0; i < n; i++)
1089 {
1090 CKey key;
1091 key.MakeNewKey();
1092 if (!AddKey(key)) { fclose(f); fprintf(stderr, "AddKey failed at %d\n", i); return 1; }
1093 fprintf(f, "%s\n", PubKeyToAddress(key.GetPubKey()).c_str());
1094 }
1095 fclose(f);
1096 printf("wrote %d fresh wallet addresses to %s\n", n, file.c_str());
1097 return 0;
1098 }
1099
1033 if (!StartNode(strErrors)) 1100 if (!StartNode(strErrors))
1034 { 1101 {
1035 fprintf(stderr,"StartNode: %s\n",strErrors.c_str()); 1102 fprintf(stderr,"StartNode: %s\n",strErrors.c_str());
diff --git a/src/makefile.mingw b/src/makefile.mingw
index 55f186b..740bba6 100644
--- a/src/makefile.mingw
+++ b/src/makefile.mingw
@@ -35,7 +35,7 @@ IMGUI_OBJS = obj/imgui.o obj/imgui_draw.o obj/imgui_tables.o \
35 obj/imgui_widgets.o obj/imgui_impl_glfw.o obj/imgui_impl_opengl3.o 35 obj/imgui_widgets.o obj/imgui_impl_glfw.o obj/imgui_impl_opengl3.o
36 36
37NODE_OBJS = obj/util.o obj/script.o obj/db.o obj/net.o obj/main.o \ 37NODE_OBJS = obj/util.o obj/script.o obj/db.o obj/net.o obj/main.o \
38 obj/market.o obj/sha.o obj/nostr.o obj/randomx_pow.o \ 38 obj/sha.o obj/nostr.o obj/randomx_pow.o \
39 obj/btfaddr.o obj/btfchan.o obj/btfrv.o obj/btftunnel.o obj/rpc.o \ 39 obj/btfaddr.o obj/btfchan.o obj/btfrv.o obj/btftunnel.o obj/rpc.o \
40 obj/selftest.o obj/bip32.o obj/walletcmd.o obj/wallet_sqlite.o \ 40 obj/selftest.o obj/bip32.o obj/walletcmd.o obj/wallet_sqlite.o \
41 obj/proxy.o obj/tor.o 41 obj/proxy.o obj/tor.o
@@ -51,7 +51,7 @@ GUI_OBJS = obj/gui.o obj/main_gui.o obj/bitflash_res.o
51# binary. Nothing crashed here, and nothing would say so if it had. 51# binary. Nothing crashed here, and nothing would say so if it had.
52HEADERS = headers_core.h headers.h util.h main.h serialize.h uint256.h \ 52HEADERS = headers_core.h headers.h util.h main.h serialize.h uint256.h \
53 key.h bignum.h script.h db.h base58.h net.h nostr.h \ 53 key.h bignum.h script.h db.h base58.h net.h nostr.h \
54 randomx_pow.h btfaddr.h btfchan.h btfrv.h btftunnel.h market.h \ 54 randomx_pow.h btfaddr.h btfchan.h btfrv.h btftunnel.h \
55 selftest.h walletcmd.h wallet_sqlite.h sockcount.h proxy.h tor.h compat.h font_roboto.h 55 selftest.h walletcmd.h wallet_sqlite.h sockcount.h proxy.h tor.h compat.h font_roboto.h
56 56
57all: bitflash.exe 57all: bitflash.exe
diff --git a/src/market.cpp b/src/market.cpp
deleted file mode 100644
index e5d34f4..0000000
--- a/src/market.cpp
+++ /dev/null
@@ -1,264 +0,0 @@
1// Copyright (c) 2009 Satoshi Nakamoto
2// Distributed under the MIT/X11 software license, see the accompanying
3// file license.txt or http://www.opensource.org/licenses/mit-license.php.
4
5#include "headers.h"
6
7
8
9
10
11
12
13
14
15
16//
17// Global state variables
18//
19
20//// later figure out how these are persisted
21map<uint256, CProduct> mapMyProducts;
22
23
24
25
26map<uint256, CProduct> mapProducts;
27CCriticalSection cs_mapProducts;
28
29bool AdvertInsert(const CProduct& product)
30{
31 uint256 hash = product.GetHash();
32 bool fNew = false;
33 bool fUpdated = false;
34
35 CRITICAL_BLOCK(cs_mapProducts)
36 {
37 // Insert or find existing product
38 pair<map<uint256, CProduct>::iterator, bool> item = mapProducts.insert(make_pair(hash, product));
39 CProduct* pproduct = &(*(item.first)).second;
40 fNew = item.second;
41
42 // Update if newer
43 if (product.nSequence > pproduct->nSequence)
44 {
45 *pproduct = product;
46 fUpdated = true;
47 }
48 }
49
50 //if (fNew)
51 // NotifyProductAdded(hash);
52 //else if (fUpdated)
53 // NotifyProductUpdated(hash);
54
55 return (fNew || fUpdated);
56}
57
58void AdvertErase(const CProduct& product)
59{
60 uint256 hash = product.GetHash();
61 CRITICAL_BLOCK(cs_mapProducts)
62 mapProducts.erase(hash);
63 //NotifyProductDeleted(hash);
64}
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84template<typename T>
85unsigned int Union(T& v1, T& v2)
86{
87 // v1 = v1 union v2
88 // v1 and v2 must be sorted
89 // returns the number of elements added to v1
90
91 ///// need to check that this is equivalent, then delete this comment
92 //vector<unsigned short> vUnion(v1.size() + v2.size());
93 //vUnion.erase(set_union(v1.begin(), v1.end(),
94 // v2.begin(), v2.end(),
95 // vUnion.begin()),
96 // vUnion.end());
97
98 T vUnion;
99 vUnion.reserve(v1.size() + v2.size());
100 set_union(v1.begin(), v1.end(),
101 v2.begin(), v2.end(),
102 back_inserter(vUnion));
103 unsigned int nAdded = vUnion.size() - v1.size();
104 if (nAdded > 0)
105 v1 = vUnion;
106 return nAdded;
107}
108
109void CUser::AddAtom(unsigned short nAtom, bool fOrigin)
110{
111 // Ignore duplicates
112 if (binary_search(vAtomsIn.begin(), vAtomsIn.end(), nAtom) ||
113 find(vAtomsNew.begin(), vAtomsNew.end(), nAtom) != vAtomsNew.end())
114 return;
115
116 //// instead of zero atom, should change to free atom that propagates,
117 //// limited to lower than a certain value like 5 so conflicts quickly
118 // The zero atom never propagates,
119 // new atoms always propagate through the user that created them
120 if (nAtom == 0 || fOrigin)
121 {
122 vector<unsigned short> vTmp(1, nAtom);
123 Union(vAtomsIn, vTmp);
124 if (fOrigin)
125 vAtomsOut.push_back(nAtom);
126 return;
127 }
128
129 vAtomsNew.push_back(nAtom);
130
131 if (vAtomsNew.size() >= nFlowthroughRate || vAtomsOut.empty())
132 {
133 // Select atom to flow through to vAtomsOut
134 vAtomsOut.push_back(vAtomsNew[GetRand(vAtomsNew.size())]);
135
136 // Merge vAtomsNew into vAtomsIn
137 sort(vAtomsNew.begin(), vAtomsNew.end());
138 Union(vAtomsIn, vAtomsNew);
139 vAtomsNew.clear();
140 }
141}
142
143bool AddAtomsAndPropagate(uint256 hashUserStart, const vector<unsigned short>& vAtoms, bool fOrigin)
144{
145 CReviewDB reviewdb;
146 map<uint256, vector<unsigned short> > pmapPropagate[2];
147 pmapPropagate[0][hashUserStart] = vAtoms;
148
149 for (int side = 0; !pmapPropagate[side].empty(); side = 1 - side)
150 {
151 map<uint256, vector<unsigned short> >& mapFrom = pmapPropagate[side];
152 map<uint256, vector<unsigned short> >& mapTo = pmapPropagate[1 - side];
153
154 for (map<uint256, vector<unsigned short> >::iterator mi = mapFrom.begin(); mi != mapFrom.end(); ++mi)
155 {
156 const uint256& hashUser = (*mi).first;
157 const vector<unsigned short>& vReceived = (*mi).second;
158
159 ///// this would be a lot easier on the database if it put the new atom at the beginning of the list,
160 ///// so the change would be right next to the vector size.
161
162 // Read user
163 CUser user;
164 reviewdb.ReadUser(hashUser, user);
165 unsigned int nIn = user.vAtomsIn.size();
166 unsigned int nNew = user.vAtomsNew.size();
167 unsigned int nOut = user.vAtomsOut.size();
168
169 // Add atoms received
170 foreach(unsigned short nAtom, vReceived)
171 user.AddAtom(nAtom, fOrigin);
172 fOrigin = false;
173
174 // Don't bother writing to disk if no changes
175 if (user.vAtomsIn.size() == nIn && user.vAtomsNew.size() == nNew)
176 continue;
177
178 // Propagate
179 if (user.vAtomsOut.size() > nOut)
180 foreach(const uint256& hash, user.vLinksOut)
181 mapTo[hash].insert(mapTo[hash].end(), user.vAtomsOut.begin() + nOut, user.vAtomsOut.end());
182
183 // Write back
184 if (!reviewdb.WriteUser(hashUser, user))
185 return false;
186 }
187 mapFrom.clear();
188 }
189 return true;
190}
191
192
193
194
195
196
197bool CReview::AcceptReview()
198{
199 // Timestamp
200 nTime = GetTime();
201
202 // Check signature
203 if (!CKey::Verify(vchPubKeyFrom, GetSigHash(), vchSig))
204 return false;
205
206 CReviewDB reviewdb;
207
208 // Add review text to recipient
209 vector<CReview> vReviews;
210 reviewdb.ReadReviews(hashTo, vReviews);
211 vReviews.push_back(*this);
212 if (!reviewdb.WriteReviews(hashTo, vReviews))
213 return false;
214
215 // Add link from sender
216 CUser user;
217 uint256 hashFrom = Hash(vchPubKeyFrom.begin(), vchPubKeyFrom.end());
218 reviewdb.ReadUser(hashFrom, user);
219 user.vLinksOut.push_back(hashTo);
220 if (!reviewdb.WriteUser(hashFrom, user))
221 return false;
222
223 reviewdb.Close();
224
225 // Propagate atoms to recipient
226 vector<unsigned short> vZeroAtom(1, 0);
227 if (!AddAtomsAndPropagate(hashTo, user.vAtomsOut.size() ? user.vAtomsOut : vZeroAtom, false))
228 return false;
229
230 return true;
231}
232
233
234
235
236
237bool CProduct::CheckSignature()
238{
239 return (CKey::Verify(vchPubKeyFrom, GetSigHash(), vchSig));
240}
241
242bool CProduct::CheckProduct()
243{
244 if (!CheckSignature())
245 return false;
246
247 // Make sure it's a summary product
248 if (!mapDetails.empty() || !vOrderForm.empty())
249 return false;
250
251 // Look up seller's atom count
252 CReviewDB reviewdb("r");
253 CUser user;
254 reviewdb.ReadUser(GetUserHash(), user);
255 nAtoms = user.GetAtomCount();
256 reviewdb.Close();
257
258 ////// delme, this is now done by AdvertInsert
259 //// Store to memory
260 //CRITICAL_BLOCK(cs_mapProducts)
261 // mapProducts[GetHash()] = *this;
262
263 return true;
264}
diff --git a/src/market.h b/src/market.h
deleted file mode 100644
index b5c8b3b..0000000
--- a/src/market.h
+++ /dev/null
@@ -1,182 +0,0 @@
1// Copyright (c) 2009 Satoshi Nakamoto
2// Distributed under the MIT/X11 software license, see the accompanying
3// file license.txt or http://www.opensource.org/licenses/mit-license.php.
4
5class CUser;
6class CReview;
7class CProduct;
8
9static const unsigned int nFlowthroughRate = 2;
10
11
12
13
14bool AdvertInsert(const CProduct& product);
15void AdvertErase(const CProduct& product);
16bool AddAtomsAndPropagate(uint256 hashUserStart, const vector<unsigned short>& vAtoms, bool fOrigin);
17
18
19
20
21
22
23
24
25class CUser
26{
27public:
28 vector<unsigned short> vAtomsIn;
29 vector<unsigned short> vAtomsNew;
30 vector<unsigned short> vAtomsOut;
31 vector<uint256> vLinksOut;
32
33 CUser()
34 {
35 }
36
37 IMPLEMENT_SERIALIZE
38 (
39 if (!(nType & SER_GETHASH))
40 READWRITE(nVersion);
41 READWRITE(vAtomsIn);
42 READWRITE(vAtomsNew);
43 READWRITE(vAtomsOut);
44 READWRITE(vLinksOut);
45 )
46
47 void SetNull()
48 {
49 vAtomsIn.clear();
50 vAtomsNew.clear();
51 vAtomsOut.clear();
52 vLinksOut.clear();
53 }
54
55 uint256 GetHash() const { return SerializeHash(*this); }
56
57
58 int GetAtomCount() const
59 {
60 return (vAtomsIn.size() + vAtomsNew.size());
61 }
62
63 void AddAtom(unsigned short nAtom, bool fOrigin);
64};
65
66
67
68
69
70
71
72class CReview
73{
74public:
75 int nVersion;
76 uint256 hashTo;
77 map<string, string> mapValue;
78 vector<unsigned char> vchPubKeyFrom;
79 vector<unsigned char> vchSig;
80
81 // memory only
82 unsigned int nTime;
83 int nAtoms;
84
85
86 CReview()
87 {
88 nVersion = 1;
89 hashTo = 0;
90 nTime = 0;
91 nAtoms = 0;
92 }
93
94 IMPLEMENT_SERIALIZE
95 (
96 READWRITE(this->nVersion);
97 nVersion = this->nVersion;
98 if (!(nType & SER_DISK))
99 READWRITE(hashTo);
100 READWRITE(mapValue);
101 READWRITE(vchPubKeyFrom);
102 if (!(nType & SER_GETHASH))
103 READWRITE(vchSig);
104 )
105
106 uint256 GetHash() const { return SerializeHash(*this); }
107 uint256 GetSigHash() const { return SerializeHash(*this, SER_GETHASH|SER_SKIPSIG); }
108 uint256 GetUserHash() const { return Hash(vchPubKeyFrom.begin(), vchPubKeyFrom.end()); }
109
110
111 bool AcceptReview();
112};
113
114
115
116
117
118
119
120class CProduct
121{
122public:
123 int nVersion;
124 CAddress addr;
125 map<string, string> mapValue;
126 map<string, string> mapDetails;
127 vector<pair<string, string> > vOrderForm;
128 unsigned int nSequence;
129 vector<unsigned char> vchPubKeyFrom;
130 vector<unsigned char> vchSig;
131
132 // disk only
133 int nAtoms;
134
135 // memory only
136 set<unsigned int> setSources;
137
138 CProduct()
139 {
140 nVersion = 1;
141 nAtoms = 0;
142 nSequence = 0;
143 }
144
145 IMPLEMENT_SERIALIZE
146 (
147 READWRITE(this->nVersion);
148 nVersion = this->nVersion;
149 READWRITE(addr);
150 READWRITE(mapValue);
151 if (!(nType & SER_GETHASH))
152 {
153 READWRITE(mapDetails);
154 READWRITE(vOrderForm);
155 READWRITE(nSequence);
156 }
157 READWRITE(vchPubKeyFrom);
158 if (!(nType & SER_GETHASH))
159 READWRITE(vchSig);
160 if (nType & SER_DISK)
161 READWRITE(nAtoms);
162 )
163
164 uint256 GetHash() const { return SerializeHash(*this); }
165 uint256 GetSigHash() const { return SerializeHash(*this, SER_GETHASH|SER_SKIPSIG); }
166 uint256 GetUserHash() const { return Hash(vchPubKeyFrom.begin(), vchPubKeyFrom.end()); }
167
168
169 bool CheckSignature();
170 bool CheckProduct();
171};
172
173
174
175
176
177
178
179
180extern map<uint256, CProduct> mapProducts;
181extern CCriticalSection cs_mapProducts;
182extern map<uint256, CProduct> mapMyProducts;
diff --git a/src/net.cpp b/src/net.cpp
index 4603d2d..4741f09 100644
--- a/src/net.cpp
+++ b/src/net.cpp
@@ -492,28 +492,9 @@ bool GetMyExternalIP(unsigned int& ipRet)
492 492
493 493
494 494
495void AbandonRequests(void (*fn)(void*, CDataStream&), void* param1) 495// AbandonRequests() lived here, sweeping CNode::mapRequests for a callback a
496{ 496// closing dialog had registered. Nothing ever registered one and nothing ever
497 // If the dialog might get closed before the reply comes back, 497// called this, so it went with CRequestTracker.
498 // call this in the destructor so it doesn't get called after it's deleted.
499 CRITICAL_BLOCK(cs_vNodes)
500 {
501 foreach(CNode* pnode, vNodes)
502 {
503 CRITICAL_BLOCK(pnode->cs_mapRequests)
504 {
505 for (map<uint256, CRequestTracker>::iterator mi = pnode->mapRequests.begin(); mi != pnode->mapRequests.end();)
506 {
507 CRequestTracker& tracker = (*mi).second;
508 if (tracker.fn == fn && tracker.param1 == param1)
509 pnode->mapRequests.erase(mi++);
510 else
511 mi++;
512 }
513 }
514 }
515 }
516}
517 498
518 499
519 500
@@ -521,25 +502,10 @@ void AbandonRequests(void (*fn)(void*, CDataStream&), void* param1)
521 502
522 503
523 504
524// 505// The broadcast/subscribe channel system (AnySubscribed, CNode::IsSubscribed,
525// Subscription methods for the broadcast and subscription system. 506// Subscribe, CancelSubscribe) stood here. Its channels were MSG_TABLE and
526// Channel numbers are message numbers, i.e. MSG_TABLE and MSG_PRODUCT. 507// MSG_PRODUCT, there was never an inbound handler for "subscribe",
527// 508// "sub-cancel", "publish" or "pub-cancel", and the objects it carried are gone.
528// The subscription system uses a meet-in-the-middle strategy.
529// With 100,000 nodes, if senders broadcast to 1000 random nodes and receivers
530// subscribe to 1000 random nodes, 99.995% (1 - 0.99^1000) of messages will get through.
531//
532
533bool AnySubscribed(unsigned int nChannel)
534{
535 if (pnodeLocalHost->IsSubscribed(nChannel))
536 return true;
537 CRITICAL_BLOCK(cs_vNodes)
538 foreach(CNode* pnode, vNodes)
539 if (pnode->IsSubscribed(nChannel))
540 return true;
541 return false;
542}
543 509
544void CNode::PushGetBlocks(CBlockIndex* pindexBegin, uint256 hashEnd) 510void CNode::PushGetBlocks(CBlockIndex* pindexBegin, uint256 hashEnd)
545{ 511{
@@ -552,55 +518,6 @@ void CNode::PushGetBlocks(CBlockIndex* pindexBegin, uint256 hashEnd)
552 PushMessage("getblocks", CBlockLocator(pindexBegin), hashEnd); 518 PushMessage("getblocks", CBlockLocator(pindexBegin), hashEnd);
553} 519}
554 520
555bool CNode::IsSubscribed(unsigned int nChannel)
556{
557 if (nChannel >= vfSubscribe.size())
558 return false;
559 return vfSubscribe[nChannel];
560}
561
562void CNode::Subscribe(unsigned int nChannel, unsigned int nHops)
563{
564 if (nChannel >= vfSubscribe.size())
565 return;
566
567 if (!AnySubscribed(nChannel))
568 {
569 // Relay subscribe
570 CRITICAL_BLOCK(cs_vNodes)
571 foreach(CNode* pnode, vNodes)
572 if (pnode != this)
573 pnode->PushMessage("subscribe", nChannel, nHops);
574 }
575
576 vfSubscribe[nChannel] = true;
577}
578
579void CNode::CancelSubscribe(unsigned int nChannel)
580{
581 if (nChannel >= vfSubscribe.size())
582 return;
583
584 // Prevent from relaying cancel if wasn't subscribed
585 if (!vfSubscribe[nChannel])
586 return;
587 vfSubscribe[nChannel] = false;
588
589 if (!AnySubscribed(nChannel))
590 {
591 // Relay subscription cancel
592 CRITICAL_BLOCK(cs_vNodes)
593 foreach(CNode* pnode, vNodes)
594 if (pnode != this)
595 pnode->PushMessage("sub-cancel", nChannel);
596
597 // Clear memory, no longer subscribed
598 if (nChannel == MSG_PRODUCT)
599 CRITICAL_BLOCK(cs_mapProducts)
600 mapProducts.clear();
601 }
602}
603
604 521
605 522
606 523
@@ -1176,6 +1093,18 @@ CNode* ConnectNodeBtfResolved(const string& strBtfAddr, const string& strMeeting
1176void ThreadBtfAccept(void* parg) 1093void ThreadBtfAccept(void* parg)
1177{ 1094{
1178 printf("ThreadBtfAccept started\n"); 1095 printf("ThreadBtfAccept started\n");
1096 // In operator mode the pool server registers this same node identity as a
1097 // stratum service at the rendezvous relays, and a relay only holds ONE
1098 // service registration per pubkey -- peer-accept and pool-accept would
1099 // kick each other off forever, breaking both. The pool's registration is
1100 // the one that must be stable (that's what miners resolve and dial), so
1101 // the node skips inbound rendezvous here and relies on outbound .btf
1102 // connections for block sync.
1103 if (nMineMode == MINE_OPERATOR)
1104 {
1105 printf("ThreadBtfAccept: operator mode -- skipping peer rendezvous registration\n");
1106 return;
1107 }
1179 unsigned char pk[32]; 1108 unsigned char pk[32];
1180 unsigned char sk[32]; 1109 unsigned char sk[32];
1181 while (!BtfGetIdentity(pk, sk)) 1110 while (!BtfGetIdentity(pk, sk))
@@ -1325,17 +1254,6 @@ void CNode::Disconnect()
1325 BtfCloseSocket(hSocket); 1254 BtfCloseSocket(hSocket);
1326 hSocket = INVALID_SOCKET; 1255 hSocket = INVALID_SOCKET;
1327 1256
1328 // All of a nodes broadcasts and subscriptions are automatically torn down
1329 // when it goes down, so a node has to stay up to keep its broadcast going.
1330
1331 CRITICAL_BLOCK(cs_mapProducts)
1332 for (map<uint256, CProduct>::iterator mi = mapProducts.begin(); mi != mapProducts.end();)
1333 AdvertRemoveSource(this, MSG_PRODUCT, 0, (*(mi++)).second);
1334
1335 // Cancel subscriptions
1336 for (unsigned int nChannel = 0; nChannel < vfSubscribe.size(); nChannel++)
1337 if (vfSubscribe[nChannel])
1338 CancelSubscribe(nChannel);
1339} 1257}
1340 1258
1341 1259
@@ -1460,7 +1378,6 @@ void ThreadSocketHandler2(void* parg)
1460 bool fDelete = false; 1378 bool fDelete = false;
1461 TRY_CRITICAL_BLOCK(pnode->cs_vSend) 1379 TRY_CRITICAL_BLOCK(pnode->cs_vSend)
1462 TRY_CRITICAL_BLOCK(pnode->cs_vRecv) 1380 TRY_CRITICAL_BLOCK(pnode->cs_vRecv)
1463 TRY_CRITICAL_BLOCK(pnode->cs_mapRequests)
1464 TRY_CRITICAL_BLOCK(pnode->cs_inventory) 1381 TRY_CRITICAL_BLOCK(pnode->cs_inventory)
1465 fDelete = true; 1382 fDelete = true;
1466 if (fDelete) 1383 if (fDelete)
@@ -1568,6 +1485,27 @@ void ThreadSocketHandler2(void* parg)
1568 printf("dropping node %s: its socket is no longer valid\n", 1485 printf("dropping node %s: its socket is no longer valid\n",
1569 pnode->addr.ToString().c_str()); 1486 pnode->addr.ToString().c_str());
1570 pnode->fDisconnect = true; 1487 pnode->fDisconnect = true;
1488
1489 // Stop referring to it, or it goes into the next poll
1490 // set and fails it again.
1491 //
1492 // Marking fDisconnect is not enough: a node is only
1493 // removed from vNodes once it has drained or once
1494 // DISCONNECT_DRAIN_SECS has passed, so one dead
1495 // descriptor kept failing the wait for a full minute --
1496 // and a failed wait skips the service loop below
1497 // entirely, which means NO peer is read on that pass.
1498 // Measured on a wedged pool under the old select()
1499 // implementation: 1,027 consecutive passes over 61
1500 // seconds, one bad socket, the entire node deaf. poll()
1501 // reports a bad descriptor as POLLNVAL on that entry
1502 // rather than failing the whole call, so this path is
1503 // now the backstop rather than the common case.
1504 //
1505 // Not closed, only forgotten. getsockopt just told us
1506 // this descriptor is not ours any more; closing it now
1507 // is how an unrelated socket gets closed instead.
1508 pnode->hSocket = INVALID_SOCKET;
1571 } 1509 }
1572 } 1510 }
1573 } 1511 }
@@ -1841,6 +1779,13 @@ void ThreadMessageHandler(void* parg)
1841 } 1779 }
1842} 1780}
1843 1781
1782// When any peer last told us a height. A wedged node -- one that still has
1783// peers but has stopped hearing them -- is exactly a node where this stops
1784// advancing, and that is the one thing the height comparison itself cannot
1785// see: there is no height to compare.
1786static int64 nLastPeerHeightSeen = 0;
1787static int64 nPeerHeightFirstAsked = 0;
1788
1844int GetPeerMedianHeight() 1789int GetPeerMedianHeight()
1845{ 1790{
1846 vector<int> vHeights; 1791 vector<int> vHeights;
@@ -1852,10 +1797,26 @@ int GetPeerMedianHeight()
1852 if (vHeights.empty()) 1797 if (vHeights.empty())
1853 return -1; 1798 return -1;
1854 1799
1800 nLastPeerHeightSeen = GetTime();
1855 sort(vHeights.begin(), vHeights.end()); 1801 sort(vHeights.begin(), vHeights.end());
1856 return vHeights[vHeights.size() / 2]; 1802 return vHeights[vHeights.size() / 2];
1857} 1803}
1858 1804
1805// Seconds since any peer last reported a height.
1806//
1807// Before ever hearing one, measure from the first time somebody asked rather
1808// than from the epoch -- otherwise a node refuses jobs the instant it starts,
1809// before it has had a chance to complete a single handshake.
1810int64 GetPeerHeightAgeSecs()
1811{
1812 int64 nNow = GetTime();
1813 if (nPeerHeightFirstAsked == 0)
1814 nPeerHeightFirstAsked = nNow;
1815 if (nLastPeerHeightSeen == 0)
1816 return nNow - nPeerHeightFirstAsked;
1817 return nNow - nLastPeerHeightSeen;
1818}
1819
1859 1820
1860// Say out loud when we are behind the network. 1821// Say out loud when we are behind the network.
1861// 1822//
@@ -1992,7 +1953,21 @@ void ThreadBitcoinMiner(void* parg)
1992// Start the miners. Returns how many actually got off the ground. 1953// Start the miners. Returns how many actually got off the ground.
1993int StartMinerThreads() 1954int StartMinerThreads()
1994{ 1955{
1995 int nThreads = MinerThreadCount(); 1956 // MinerThreadCount() sizes the SOLO miner, where each thread hashes a
1957 // template of its own and more threads is simply more hashrate.
1958 //
1959 // Participant mode is not that shape. Every thread that enters
1960 // BitcoinMiner() takes the MINE_PARTICIPANT branch, and that branch is a
1961 // whole miner: its own Stratum session to the pool, its own payout address
1962 // from the rotation, and its own pool of /hashthreads worker threads. So
1963 // the two counts multiply instead of dividing, and the machine ends up
1964 // running MinerThreadCount() x nHashThreads hashing threads plus one pool
1965 // connection each. On a 12-core host with /hashthreads=2 that is 11
1966 // sessions and 22 hash threads -- roughly 2x the cores, which is not more
1967 // work done, only more contention -- and with /hashthreads unset it is 11
1968 // sessions of 11 workers. One session is the entirety of participant mode;
1969 // /hashthreads is what sizes it, and it can then mean what it says.
1970 int nThreads = (nMineMode == MINE_PARTICIPANT) ? 1 : MinerThreadCount();
1996 int nStarted = 0; 1971 int nStarted = 0;
1997 for (int i = 0; i < nThreads; i++) 1972 for (int i = 0; i < nThreads; i++)
1998 { 1973 {
@@ -2004,9 +1979,24 @@ int StartMinerThreads()
2004 // The dataset is one allocation shared by all of them; only the 2 MB 1979 // The dataset is one allocation shared by all of them; only the 2 MB
2005 // scratchpads multiply. Printed because "why is it using 2 GB" is the first 1980 // scratchpads multiply. Printed because "why is it using 2 GB" is the first
2006 // question anyone asks. 1981 // question anyone asks.
2007 printf("Mining with %d thread(s) of %u core(s) -- about %d MB " 1982 //
2008 "(~2080 MB shared RandomX dataset + ~2 MB per thread)\n", 1983 // Participant mode reports the workers that session will run, not the one
2009 nStarted, std::thread::hardware_concurrency(), 2080 + 2 * nStarted); 1984 // thread that runs it: the hash thread count is the number an operator
1985 // watches, and it is the number that was wrong here. It is otherwise only
1986 // logged under -debug, which is how 11 sessions ran unnoticed.
1987 if (nMineMode == MINE_PARTICIPANT)
1988 {
1989 int nWorkers = ParticipantHashThreadCount();
1990 printf("Mining for the pool with %d hash thread(s) of %u core(s) -- about %d MB "
1991 "(~2080 MB shared RandomX dataset + ~2 MB per thread)\n",
1992 nWorkers, AvailableCoreCount(), 2080 + 2 * nWorkers);
1993 }
1994 else
1995 {
1996 printf("Mining with %d thread(s) of %u core(s) -- about %d MB "
1997 "(~2080 MB shared RandomX dataset + ~2 MB per thread)\n",
1998 nStarted, AvailableCoreCount(), 2080 + 2 * nStarted);
1999 }
2010 return nStarted; 2000 return nStarted;
2011} 2001}
2012 2002
@@ -2077,20 +2067,41 @@ bool StartNode(string& strError)
2077 // IP address, and port for the socket that is being bound 2067 // IP address, and port for the socket that is being bound
2078 int nRetryLimit = 15; 2068 int nRetryLimit = 15;
2079 struct sockaddr_in sockaddr = addrLocalHost.GetSockAddr(); 2069 struct sockaddr_in sockaddr = addrLocalHost.GetSockAddr();
2080 // Keep addrLocalHost as this node's advertised address, but listen on all 2070 // Managed Tor forwards hidden-service traffic to 127.0.0.1:<port>, which a
2081 // local interfaces. Managed Tor forwards hidden-service traffic to 2071 // LAN-only bind cannot receive -- so a node that advertises an onion
2082 // 127.0.0.1:<port>, and binding only to the LAN address made that onion 2072 // endpoint must listen on all local interfaces. Upstream 1.2.19 bound
2083 // endpoint unreachable even though the P2P listener was running. 2073 // INADDR_ANY unconditionally, which also widened the P2P listener of every
2084 sockaddr.sin_addr.s_addr = INADDR_ANY; 2074 // non-Tor node to VPN tunnels, docker bridges and secondary NICs. Keep the
2075 // wildcard bind conditional on actually having an onion endpoint.
2076 if (!BtfLocalOnionEndpoint().empty())
2077 sockaddr.sin_addr.s_addr = INADDR_ANY;
2085 if (bind(hListenSocket, (struct sockaddr*)&sockaddr, sizeof(sockaddr)) == SOCKET_ERROR) 2078 if (bind(hListenSocket, (struct sockaddr*)&sockaddr, sizeof(sockaddr)) == SOCKET_ERROR)
2086 { 2079 {
2087 int nErr = WSAGetLastError(); 2080 int nErr = WSAGetLastError();
2088 if (nErr == WSAEADDRINUSE) 2081 if (nErr == WSAEADDRINUSE)
2089 strError = strprintf("Error: Unable to bind to port %s on this computer. The program is probably already running.", addrLocalHost.ToString().c_str()); 2082 {
2083 // Another local instance holds the default port. Fall back to an
2084 // ephemeral port: inbound direct-IP peering is optional, and .btf
2085 // rendezvous connectivity is unaffected by the local listen port.
2086 sockaddr.sin_port = htons(0);
2087 if (bind(hListenSocket, (struct sockaddr*)&sockaddr, sizeof(sockaddr)) == SOCKET_ERROR)
2088 {
2089 strError = strprintf("Error: Unable to bind any port on this computer (bind returned error %d)", WSAGetLastError());
2090 printf("%s\n", strError.c_str());
2091 return false;
2092 }
2093 struct sockaddr_in bound;
2094 socklen_t nBoundLen = sizeof(bound);
2095 if (getsockname(hListenSocket, (struct sockaddr*)&bound, &nBoundLen) == 0)
2096 addrLocalHost.port = bound.sin_port;
2097 printf("default port in use by another instance; bound to ephemeral %s instead\n", addrLocalHost.ToString().c_str());
2098 }
2090 else 2099 else
2100 {
2091 strError = strprintf("Error: Unable to bind to port %s on this computer (bind returned error %d)", addrLocalHost.ToString().c_str(), nErr); 2101 strError = strprintf("Error: Unable to bind to port %s on this computer (bind returned error %d)", addrLocalHost.ToString().c_str(), nErr);
2092 printf("%s\n", strError.c_str()); 2102 printf("%s\n", strError.c_str());
2093 return false; 2103 return false;
2104 }
2094 } 2105 }
2095 printf("bound to addrLocalHost = %s\n\n", addrLocalHost.ToString().c_str()); 2106 printf("bound to addrLocalHost = %s\n\n", addrLocalHost.ToString().c_str());
2096 2107
diff --git a/src/net.h b/src/net.h
index e3d4a93..72e13eb 100644
--- a/src/net.h
+++ b/src/net.h
@@ -5,7 +5,6 @@
5class CMessageHeader; 5class CMessageHeader;
6class CAddress; 6class CAddress;
7class CInv; 7class CInv;
8class CRequestTracker;
9class CNode; 8class CNode;
10// Defined in main.h, which is included after this file. Only ever used here as 9// Defined in main.h, which is included after this file. Only ever used here as
11// a pointer, so the forward declaration is enough. 10// a pointer, so the forward declaration is enough.
@@ -14,7 +13,6 @@ class CBlockIndex;
14 13
15 14
16static const unsigned short DEFAULT_PORT = htons(8433); 15static const unsigned short DEFAULT_PORT = htons(8433);
17static const unsigned int PUBLISH_HOPS = 5;
18enum 16enum
19{ 17{
20 NODE_NETWORK = (1 << 0), 18 NODE_NETWORK = (1 << 0),
@@ -93,10 +91,9 @@ static const int BTF_HANDSHAKE_GRACE_SECS = 60;
93 91
94// A serialized object cannot be larger than MAX_SIZE, so no valid network 92// A serialized object cannot be larger than MAX_SIZE, so no valid network
95// message needs the old 256 MB header allowance. Keep the wire cap aligned with 93// message needs the old 256 MB header allowance. Keep the wire cap aligned with
96// what the deserializer can accept and bound how long one peer may sit on a 94// what the deserializer can accept. How long a peer may sit on a partial
97// partial payload after a complete header has arrived. 95// payload after a complete header has arrived is BTF_MSG_WAIT_MAX_SECS, below.
98static const unsigned int MAX_PROTOCOL_MESSAGE_SIZE = MAX_SIZE; 96static const unsigned int MAX_PROTOCOL_MESSAGE_SIZE = MAX_SIZE;
99static const int BTF_INCOMPLETE_MESSAGE_TIMEOUT_SECS = 2 * 60;
100 97
101// Ceiling on simultaneous connections. 98// Ceiling on simultaneous connections.
102// 99//
@@ -136,6 +133,42 @@ static const unsigned int MAX_CONNECTIONS_PER_IP = 32;
136// this long regardless of what is still buffered. 133// this long regardless of what is still buffered.
137static const int64 DISCONNECT_DRAIN_SECS = 60; 134static const int64 DISCONNECT_DRAIN_SECS = 60;
138 135
136// How long a pool may go without hearing a height from ANY peer before it
137// stops issuing jobs. Long enough that ordinary peer churn and a quiet
138// handshake window do not trip it; short enough that a deaf pool cannot build
139// more than a few blocks of private fork before it is stopped. Heights
140// 3966-4085 -- 120 blocks, 5890.50 BTF -- were mined in exactly this state.
141static const int64 BTF_NO_PEER_HEIGHT_MAX_SECS = 20 * 60;
142
143// How long a peer may leave this node waiting for the rest of a message it
144// announced in a header.
145//
146// ProcessMessages rewinds the header and breaks when the body has not all
147// arrived, which is correct for a message still in flight and unbounded for
148// one that never completes: the same header is re-read, the loop breaks
149// again, and that peer is never parsed again for the life of the process.
150// Satoshi left the note -- "need a mechanism to give up waiting for overlong
151// message size error" -- and this is it.
152//
153// It never self-corrects, because any byte that arrives refreshes nLastRecv,
154// so the inactivity timeout that would otherwise drop the peer never fires.
155// The socket stays healthy, select() keeps reporting it, and the node goes
156// silently deaf one peer at a time.
157//
158// A header may legitimately outrun its body by a round trip or two; two
159// minutes is far beyond that, and well inside BTF_RECV_TIMEOUT_SECS.
160static const int64 BTF_MSG_WAIT_MAX_SECS = 2 * 60;
161
162// Largest payload a peer may announce in a message header: MAX_PROTOCOL_MESSAGE_SIZE
163// is defined further up this file. This tree reached the same cap by the same
164// reasoning as upstream #109 -- a 256 MB allowance is eight times what the
165// deserializer will accept (serialize.h throws above MAX_SIZE), so the header
166// check admitted an allocation the parser would only reject later, after
167// BTF_MSG_WAIT_MAX_SECS of a stalled peer slot -- so only the note survives.
168//
169// Both definitions were textually clean and 59 lines apart, so the replay onto
170// upstream merged them without a conflict and only the compiler objected.
171
139// Send a ping after this long with nothing to say, so a peer running the 172// Send a ping after this long with nothing to say, so a peer running the
140// inactivity check above does not mistake a quiet node for a dead one. Must 173// inactivity check above does not mistake a quiet node for a dead one. Must
141// stay well under BTF_RECV_TIMEOUT_SECS. 174// stay well under BTF_RECV_TIMEOUT_SECS.
@@ -176,6 +209,7 @@ extern int nBestHeight;
176// has told us anything. Median rather than maximum so one peer claiming an 209// has told us anything. Median rather than maximum so one peer claiming an
177// absurd height cannot move it. 210// absurd height cannot move it.
178int GetPeerMedianHeight(); 211int GetPeerMedianHeight();
212int64 GetPeerHeightAgeSecs();
179 213
180// Lightweight .btf churn accounting. These counters are diagnostics only: they 214// Lightweight .btf churn accounting. These counters are diagnostics only: they
181// let an operator tell stale descriptors, relay pairing failures, and paired 215// let an operator tell stale descriptors, relay pairing failures, and paired
@@ -193,8 +227,6 @@ void BtfChurnNoteHandshakeTimeout(const string& strBtfAddr, const string& strMee
193void BtfPexCollect(std::vector<std::string>& vDescOut); 227void BtfPexCollect(std::vector<std::string>& vDescOut);
194// Verify descriptors a peer sent and remember the good ones. Returns how many. 228// Verify descriptors a peer sent and remember the good ones. Returns how many.
195int BtfPexAccept(const std::vector<std::string>& vDesc); 229int BtfPexAccept(const std::vector<std::string>& vDesc);
196void AbandonRequests(void (*fn)(void*, CDataStream&), void* param1);
197bool AnySubscribed(unsigned int nChannel);
198void ThreadBitcoinMiner(void* parg); 230void ThreadBitcoinMiner(void* parg);
199// Starts MinerThreadCount() miners; returns how many came up. 231// Starts MinerThreadCount() miners; returns how many came up.
200int StartMinerThreads(); 232int StartMinerThreads();
@@ -468,13 +500,19 @@ public:
468 500
469 501
470 502
503// MSG_REVIEW, MSG_PRODUCT and MSG_TABLE no longer have a handler, a producer
504// or a store -- the marketplace they belonged to is gone. They stay in the
505// enum, and their names stay in ppszTypeName below, because CInv serialises
506// `type` as a plain integer: dropping them would renumber nothing today but
507// would make it tempting to renumber later, and MSG_TX and MSG_BLOCK are wire
508// values shared with every other node. Reserved, never emitted, never served.
471enum 509enum
472{ 510{
473 MSG_TX = 1, 511 MSG_TX = 1,
474 MSG_BLOCK, 512 MSG_BLOCK,
475 MSG_REVIEW, 513 MSG_REVIEW, // reserved, dead
476 MSG_PRODUCT, 514 MSG_PRODUCT, // reserved, dead
477 MSG_TABLE, 515 MSG_TABLE, // reserved, dead
478}; 516};
479 517
480static const char* ppszTypeName[] = 518static const char* ppszTypeName[] =
@@ -561,23 +599,9 @@ public:
561 599
562 600
563 601
564class CRequestTracker 602// CRequestTracker and the PushRequest/"reply" round trip lived here. Nothing
565{ 603// in this tree ever called PushRequest, so the table was always empty and the
566public: 604// "reply" handler was pure inbound parser. Both removed.
567 void (*fn)(void*, CDataStream&);
568 void* param1;
569
570 explicit CRequestTracker(void (*fnIn)(void*, CDataStream&)=NULL, void* param1In=NULL)
571 {
572 fn = fnIn;
573 param1 = param1In;
574 }
575
576 bool IsNull()
577 {
578 return fn == NULL;
579 }
580};
581 605
582 606
583 607
@@ -623,9 +647,6 @@ protected:
623public: 647public:
624 int64 nReleaseTime; 648 int64 nReleaseTime;
625 int64 nDisconnectSince; 649 int64 nDisconnectSince;
626 map<uint256, CRequestTracker> mapRequests;
627 CCriticalSection cs_mapRequests;
628
629 // inventory based relay 650 // inventory based relay
630 set<CInv> setInventoryKnown; 651 set<CInv> setInventoryKnown;
631 set<CInv> setInventoryKnown2; 652 set<CInv> setInventoryKnown2;
@@ -633,9 +654,6 @@ public:
633 CCriticalSection cs_inventory; 654 CCriticalSection cs_inventory;
634 multimap<int64, CInv> mapAskFor; 655 multimap<int64, CInv> mapAskFor;
635 656
636 // publish and subscription
637 vector<char> vfSubscribe;
638
639 // Last "btfpeers" we accepted from this node, to rate-limit the exchange. 657 // Last "btfpeers" we accepted from this node, to rate-limit the exchange.
640 int64 nLastPexRecv; 658 int64 nLastPexRecv;
641 659
@@ -653,12 +671,17 @@ public:
653 int64 nLastSend; 671 int64 nLastSend;
654 int64 nLastRecv; 672 int64 nLastRecv;
655 int64 nLastSendEmpty; 673 int64 nLastSendEmpty;
656 int64 nIncompleteMessageStart;
657 unsigned int nIncompleteMessageSize;
658 string strIncompleteMessageCommand;
659 string strBtfAddr; 674 string strBtfAddr;
660 string strBtfMeeting; 675 string strBtfMeeting;
661 676
677 // When this peer's message loop last rewound a header without having made
678 // progress, or 0 when it is not waiting, plus how much was buffered then.
679 // The pair times a STALL rather than a duration, so a large message
680 // arriving slowly is never mistaken for one that is never arriving.
681 // See ProcessMessages.
682 int64 nMsgWaitSince;
683 unsigned int nMsgWaitSize;
684
662 // Height this peer announced in its version message, or -1 if it sent a 685 // Height this peer announced in its version message, or -1 if it sent a
663 // version message that predates the field. Knowing how far along everyone 686 // version message that predates the field. Knowing how far along everyone
664 // else is, is the only way a node can tell "in sync" from "not hearing". 687 // else is, is the only way a node can tell "in sync" from "not hearing".
@@ -688,13 +711,14 @@ public:
688 nLastSend = 0; 711 nLastSend = 0;
689 nLastRecv = 0; 712 nLastRecv = 0;
690 nLastSendEmpty = GetTime(); 713 nLastSendEmpty = GetTime();
691 nIncompleteMessageStart = 0;
692 nIncompleteMessageSize = 0;
693 strIncompleteMessageCommand.clear();
694 strBtfAddr.clear(); 714 strBtfAddr.clear();
695 strBtfMeeting.clear(); 715 strBtfMeeting.clear();
716 // Not waiting on anything yet. Left indeterminate, the first partial
717 // message this peer sends is measured against whatever was in this
718 // memory, and ProcessMessages drops it as unreadable on first contact.
719 nMsgWaitSince = 0;
720 nMsgWaitSize = 0;
696 nStartingHeight = -1; 721 nStartingHeight = -1;
697 vfSubscribe.assign(256, false);
698 722
699 // Push a version message 723 // Push a version message
700 /// when NTP implemented, change to just nTime = GetAdjustedTime() 724 /// when NTP implemented, change to just nTime = GetAdjustedTime()
@@ -722,6 +746,11 @@ public:
722 { 746 {
723 if (hSocket != INVALID_SOCKET) 747 if (hSocket != INVALID_SOCKET)
724 { 748 {
749 // Route through BtfCloseSocket so the accounting sees this close.
750 // A raw closesocket() here was invisible to sockcount, which is
751 // why the leak numbers never added up, and leaving the descriptor
752 // set afterwards is how a number that now belongs to somebody else
753 // gets closed a second time. (#103)
725 BtfCloseSocket(hSocket); 754 BtfCloseSocket(hSocket);
726 hSocket = INVALID_SOCKET; 755 hSocket = INVALID_SOCKET;
727 } 756 }
@@ -968,49 +997,6 @@ public:
968 } 997 }
969 998
970 999
971 void PushRequest(const char* pszCommand,
972 void (*fn)(void*, CDataStream&), void* param1)
973 {
974 uint256 hashReply;
975 RAND_bytes((unsigned char*)&hashReply, sizeof(hashReply));
976
977 CRITICAL_BLOCK(cs_mapRequests)
978 mapRequests[hashReply] = CRequestTracker(fn, param1);
979
980 PushMessage(pszCommand, hashReply);
981 }
982
983 template<typename T1>
984 void PushRequest(const char* pszCommand, const T1& a1,
985 void (*fn)(void*, CDataStream&), void* param1)
986 {
987 uint256 hashReply;
988 RAND_bytes((unsigned char*)&hashReply, sizeof(hashReply));
989
990 CRITICAL_BLOCK(cs_mapRequests)
991 mapRequests[hashReply] = CRequestTracker(fn, param1);
992
993 PushMessage(pszCommand, hashReply, a1);
994 }
995
996 template<typename T1, typename T2>
997 void PushRequest(const char* pszCommand, const T1& a1, const T2& a2,
998 void (*fn)(void*, CDataStream&), void* param1)
999 {
1000 uint256 hashReply;
1001 RAND_bytes((unsigned char*)&hashReply, sizeof(hashReply));
1002
1003 CRITICAL_BLOCK(cs_mapRequests)
1004 mapRequests[hashReply] = CRequestTracker(fn, param1);
1005
1006 PushMessage(pszCommand, hashReply, a1, a2);
1007 }
1008
1009
1010
1011 bool IsSubscribed(unsigned int nChannel);
1012 void Subscribe(unsigned int nChannel, unsigned int nHops=0);
1013 void CancelSubscribe(unsigned int nChannel);
1014 void Disconnect(); 1000 void Disconnect();
1015}; 1001};
1016 1002
@@ -1067,50 +1053,8 @@ inline void RelayMessage<>(const CInv& inv, const CDataStream& ss)
1067 1053
1068 1054
1069 1055
1070// 1056// The publish/subscribe advert templates that used to close this file
1071// Templates for the publish and subscription system. 1057// (AdvertStartPublish / AdvertStopPublish / AdvertRemoveSource) are gone along
1072// The object being published as T& obj needs to have: 1058// with the marketplace objects they carried. Nothing ever handled an inbound
1073// a set<unsigned int> setSources member 1059// "publish" or "pub-cancel", so the machinery only ever pushed; what it pushed
1074// specializations of AdvertInsert and AdvertErase 1060// was CProduct, which no longer exists.
1075// Currently implemented for CTable and CProduct.
1076//
1077
1078template<typename T>
1079void AdvertStartPublish(CNode* pfrom, unsigned int nChannel, unsigned int nHops, T& obj)
1080{
1081 // Add to sources
1082 obj.setSources.insert(pfrom->addr.ip);
1083
1084 if (!AdvertInsert(obj))
1085 return;
1086
1087 // Relay
1088 CRITICAL_BLOCK(cs_vNodes)
1089 foreach(CNode* pnode, vNodes)
1090 if (pnode != pfrom && (nHops < PUBLISH_HOPS || pnode->IsSubscribed(nChannel)))
1091 pnode->PushMessage("publish", nChannel, nHops, obj);
1092}
1093
1094template<typename T>
1095void AdvertStopPublish(CNode* pfrom, unsigned int nChannel, unsigned int nHops, T& obj)
1096{
1097 uint256 hash = obj.GetHash();
1098
1099 CRITICAL_BLOCK(cs_vNodes)
1100 foreach(CNode* pnode, vNodes)
1101 if (pnode != pfrom && (nHops < PUBLISH_HOPS || pnode->IsSubscribed(nChannel)))
1102 pnode->PushMessage("pub-cancel", nChannel, nHops, hash);
1103
1104 AdvertErase(obj);
1105}
1106
1107template<typename T>
1108void AdvertRemoveSource(CNode* pfrom, unsigned int nChannel, unsigned int nHops, T& obj)
1109{
1110 // Remove a source
1111 obj.setSources.erase(pfrom->addr.ip);
1112
1113 // If no longer supported by any sources, cancel it
1114 if (obj.setSources.empty())
1115 AdvertStopPublish(pfrom, nChannel, nHops, obj);
1116}
diff --git a/src/nostr.cpp b/src/nostr.cpp
index 35fde43..e2d839c 100644
--- a/src/nostr.cpp
+++ b/src/nostr.cpp
@@ -6,6 +6,7 @@
6#include "headers.h" 6#include "headers.h"
7#include <openssl/ssl.h> 7#include <openssl/ssl.h>
8#include <openssl/err.h> 8#include <openssl/err.h>
9#include <openssl/x509.h>
9#include <openssl/x509v3.h> 10#include <openssl/x509v3.h>
10#ifdef _WIN32 11#ifdef _WIN32
11#include <wincrypt.h> 12#include <wincrypt.h>
@@ -82,9 +83,20 @@ static CCriticalSection cs_btfPeers;
82// independent relays. Kept only one Cloudflare-fronted relay (damus.io, the 83// independent relays. Kept only one Cloudflare-fronted relay (damus.io, the
83// most established) and filled the rest with relays on different hosts 84// most established) and filled the rest with relays on different hosts
84// (Hetzner, various independents) so no single upstream provider is a 85// (Hetzner, various independents) so no single upstream provider is a
85// single point of failure for peer discovery. 7 relays lands inside the 86// single point of failure for peer discovery. 8 relays lands inside the
86// commonly recommended 5-10 range for redundancy without keeping an 87// commonly recommended 5-10 range for redundancy without keeping an
87// excessive number of persistent sockets open per node. 88// excessive number of persistent sockets open per node.
89//
90// Four entries were removed after probing all twelve from two independent
91// networks (a residential line and the pool's VPS, over both v4 and v6):
92// relay.current.fyi, relay.nostr.bg -- no A record, and the AAAA for both
93// lands on the same parked host (2a04:c300:1000::381) serving a
94// certificate that fails verification. The domains are gone.
95// relay.nostr.band, nostr.fmt.wiz.biz -- TCP connects, TLS handshake never
96// completes, from both networks. Whether they are down or refusing
97// non-browser clients, this node cannot use them.
98// A dead relay is not free: each one holds a connect/retry slot and delays
99// discovery, so carrying it costs latency for no redundancy.
88const char* pszNostrRelays[] = { 100const char* pszNostrRelays[] = {
89 "wss://relay.damus.io", 101 "wss://relay.damus.io",
90 "wss://nos.lol", 102 "wss://nos.lol",
@@ -97,6 +109,8 @@ const char* pszNostrRelays[] = {
97 "wss://offchain.pub", 109 "wss://offchain.pub",
98 "wss://nostr.mom", 110 "wss://nostr.mom",
99 "wss://relay.primal.net", 111 "wss://relay.primal.net",
112 "wss://nostr.oxtr.dev",
113 "wss://relay.nos.social",
100}; 114};
101const int nNostrRelays = ARRAYLEN(pszNostrRelays); 115const int nNostrRelays = ARRAYLEN(pszNostrRelays);
102 116
@@ -110,6 +124,12 @@ vector<string> vBtfMeetingRelays = {
110 "31.44.4.249:8434", // New Jersey, US 124 "31.44.4.249:8434", // New Jersey, US
111 "90.156.222.107:8434", // Almaty, KZ 125 "90.156.222.107:8434", // Almaty, KZ
112}; 126};
127// True when the operator pinned the relay list with /rvrelay: discovered
128// relays must NOT be merged in (BtfAllRelays), otherwise one accept thread
129// per discovered relay is spawned and each one optimistically overwrites the
130// advertised meeting node (BtfSetActiveRelay) before registering -- a
131// discovered but unreachable relay then flaps the published .btf descriptor.
132bool fBtfRelayPinned = false;
113string strBtfActiveRelay; 133string strBtfActiveRelay;
114static CCriticalSection cs_activeRelay; 134static CCriticalSection cs_activeRelay;
115static string strBtfOnionEndpoint; 135static string strBtfOnionEndpoint;
@@ -166,7 +186,7 @@ static map<string, BtfPoolAnnouncement> g_discoveredPools;
166static CCriticalSection cs_discoveredPools; 186static CCriticalSection cs_discoveredPools;
167 187
168// Basic sanity: "host:port" with a numeric 1..65535 port and a plausible host. 188// Basic sanity: "host:port" with a numeric 1..65535 port and a plausible host.
169static bool LooksLikeRelay(const string& r) 189bool LooksLikeRelay(const string& r)
170{ 190{
171 size_t colon = r.rfind(':'); 191 size_t colon = r.rfind(':');
172 if (colon == string::npos || colon == 0 || colon + 1 >= r.size()) 192 if (colon == string::npos || colon == 0 || colon + 1 >= r.size())
@@ -194,6 +214,8 @@ static bool LooksLikeRelay(const string& r)
194vector<string> BtfAllRelays() 214vector<string> BtfAllRelays()
195{ 215{
196 vector<string> all = vBtfMeetingRelays; 216 vector<string> all = vBtfMeetingRelays;
217 if (fBtfRelayPinned)
218 return all;
197 CRITICAL_BLOCK(cs_discoveredRelays) 219 CRITICAL_BLOCK(cs_discoveredRelays)
198 foreach(const string& r, g_discoveredRelays) 220 foreach(const string& r, g_discoveredRelays)
199 if (find(all.begin(), all.end(), r) == all.end()) 221 if (find(all.begin(), all.end(), r) == all.end())
@@ -578,26 +600,52 @@ public:
578 { 600 {
579 sslctx = SSL_CTX_new(TLS_client_method()); 601 sslctx = SSL_CTX_new(TLS_client_method());
580 if (!sslctx) return false; 602 if (!sslctx) return false;
603
604 // SSL_CTX_new defaults to SSL_VERIFY_NONE with no trust store
605 // loaded, so until now every wss:// relay connection was accepted
606 // without looking at the certificate at all -- any on-path
607 // attacker could impersonate a relay. Our own events are signed,
608 // so a MITM cannot forge them, but it could drop the rendezvous
609 // registration (leaving the pool unreachable to miners) or answer
610 // subscriptions with whatever it liked.
581 SSL_CTX_set_verify(sslctx, SSL_VERIFY_PEER, NULL); 611 SSL_CTX_set_verify(sslctx, SSL_VERIFY_PEER, NULL);
612
613 // OpenSSL's default verify paths are usually empty on Windows, so
614 // the system ROOT store is loaded too -- without it verification
615 // fails closed there and every relay becomes unreachable.
582 bool fTrustLoaded = (SSL_CTX_set_default_verify_paths(sslctx) == 1); 616 bool fTrustLoaded = (SSL_CTX_set_default_verify_paths(sslctx) == 1);
583#ifdef _WIN32 617#ifdef _WIN32
584 fTrustLoaded = BtfLoadWindowsRootCertificates(sslctx) || fTrustLoaded; 618 fTrustLoaded = BtfLoadWindowsRootCertificates(sslctx) || fTrustLoaded;
585#endif 619#endif
586 if (!fTrustLoaded) 620 if (!fTrustLoaded)
587 return error("Nostr: failed to load TLS trust store"); 621 return error("Nostr: failed to load TLS trust store");
622
623 // Floor the protocol version. Verification says who the peer is,
624 // not how weak a cipher suite we will agree to; without this a
625 // relay offering nothing better still gets TLS 1.0.
626 SSL_CTX_set_min_proto_version(sslctx, TLS1_2_VERSION);
627
588 ssl = SSL_new(sslctx); 628 ssl = SSL_new(sslctx);
589 if (!ssl) return false; 629 if (!ssl) return false;
590 SSL_set_fd(ssl, (int)hSocket); 630 SSL_set_fd(ssl, (int)hSocket);
591 if (SSL_set_tlsext_host_name(ssl, host.c_str()) != 1) 631 if (SSL_set_tlsext_host_name(ssl, host.c_str()) != 1) // SNI
592 return error("Nostr: failed to set TLS SNI for %s", host.c_str()); 632 return error("Nostr: failed to set TLS SNI for %s", host.c_str());
633
634 // Verifying the chain is not enough on its own: without this the
635 // relay could present any certificate a trusted CA ever issued,
636 // for any name. SSL_set1_host binds the check to the host we asked
637 // for.
593 if (SSL_set1_host(ssl, host.c_str()) != 1) 638 if (SSL_set1_host(ssl, host.c_str()) != 1)
594 return error("Nostr: failed to set TLS hostname for %s", host.c_str()); 639 return error("Nostr: failed to set TLS hostname for %s", host.c_str());
640
595 if (SSL_connect(ssl) != 1) 641 if (SSL_connect(ssl) != 1)
642 {
643 long nVerify = SSL_get_verify_result(ssl);
644 if (nVerify != X509_V_OK)
645 return error("Nostr: certificate for %s rejected: %s",
646 host.c_str(), X509_verify_cert_error_string(nVerify));
596 return error("Nostr: TLS handshake with %s failed", host.c_str()); 647 return error("Nostr: TLS handshake with %s failed", host.c_str());
597 long nVerify = SSL_get_verify_result(ssl); 648 }
598 if (nVerify != X509_V_OK)
599 return error("Nostr: TLS certificate verification failed for %s: %s",
600 host.c_str(), X509_verify_cert_error_string(nVerify));
601 } 649 }
602 650
603 // WebSocket handshake (HTTP Upgrade) 651 // WebSocket handshake (HTTP Upgrade)
@@ -1049,9 +1097,15 @@ static void HandlePoolAnnouncement(const json& ev)
1049 1097
1050// Resolve a `.btf` address via a relay: fetch the descriptor published by that 1098// Resolve a `.btf` address via a relay: fetch the descriptor published by that
1051// address's key and verify it self-certifies under the decoded pubkey. 1099// address's key and verify it self-certifies under the decoded pubkey.
1100// fExchangeOk reports whether the relay actually finished the conversation
1101// (an EOSE arrived) as opposed to the socket dying mid-read. The caller pools
1102// this socket for reuse, and only a completed exchange leaves it in a state
1103// the next caller can trust.
1052static bool ResolveDescriptor(CWebSocket& ws, void* ctx, const string& btfAddr, 1104static bool ResolveDescriptor(CWebSocket& ws, void* ctx, const string& btfAddr,
1053 btf::Descriptor& out, string* rawOut = NULL) 1105 btf::Descriptor& out, bool& fExchangeOk,
1106 string* rawOut = NULL)
1054{ 1107{
1108 fExchangeOk = false;
1055 unsigned char pk[32]; 1109 unsigned char pk[32];
1056 if (!btf::ParseAddress(btfAddr, pk)) 1110 if (!btf::ParseAddress(btfAddr, pk))
1057 return false; 1111 return false;
@@ -1063,30 +1117,46 @@ static bool ResolveDescriptor(CWebSocket& ws, void* ctx, const string& btfAddr,
1063 json req = json::array({ "REQ", "btf-resolve", filter }); 1117 json req = json::array({ "REQ", "btf-resolve", filter });
1064 if (!ws.SendText(req.dump())) 1118 if (!ws.SendText(req.dump()))
1065 return false; 1119 return false;
1120 bool fFound = false;
1066 for (;;) 1121 for (;;)
1067 { 1122 {
1068 string msg; 1123 string msg;
1069 if (!ws.RecvText(msg)) 1124 if (!ws.RecvText(msg))
1070 break; 1125 return fFound; // socket died mid-exchange; fExchangeOk stays false
1071 json j; 1126 json j;
1072 try { j = json::parse(msg); } catch (...) { continue; } 1127 try { j = json::parse(msg); } catch (...) { continue; }
1073 if (!j.is_array() || j.empty() || !j[0].is_string()) continue; 1128 if (!j.is_array() || j.empty() || !j[0].is_string()) continue;
1074 string t = j[0].get<string>(); 1129 string t = j[0].get<string>();
1130 // Every relay message names the subscription it belongs to. This socket
1131 // is pooled and reused, so a message left over from an earlier exchange
1132 // is the ordinary case here, not an exotic one -- and an unmatched EOSE
1133 // ends this query before it has begun, which is indistinguishable from
1134 // "that descriptor does not exist". Check the id.
1135 if (j.size() >= 2 && j[1].is_string() && j[1].get<string>() != "btf-resolve")
1136 continue;
1075 if (t == "EVENT" && j.size() >= 3) 1137 if (t == "EVENT" && j.size() >= 3)
1076 { 1138 {
1077 const json& ev = j[2]; 1139 const json& ev = j[2];
1078 if (ev.contains("content") && 1140 if (!fFound && ev.contains("content") &&
1079 btf::VerifyDescriptor(ctx, ev["content"].get<string>(), pk, out)) 1141 btf::VerifyDescriptor(ctx, ev["content"].get<string>(), pk, out))
1080 { 1142 {
1143 fFound = true;
1081 if (rawOut) 1144 if (rawOut)
1082 *rawOut = ev["content"].get<string>(); 1145 *rawOut = ev["content"].get<string>();
1083 return true;
1084 } 1146 }
1147 // Deliberately keep reading to EOSE rather than returning here.
1148 // Returning early leaves the rest of the relay's answer queued on a
1149 // socket that goes straight back into the pool, and the next caller
1150 // reads our leftovers as its own -- one message out of step, for
1151 // the life of the process.
1085 } 1152 }
1086 else if (t == "EOSE") 1153 else if (t == "EOSE")
1154 {
1155 fExchangeOk = true; // relay said "that is all": socket is clean
1087 break; 1156 break;
1157 }
1088 } 1158 }
1089 return false; 1159 return fFound;
1090} 1160}
1091 1161
1092// Resolve a `.btf` address into its rendezvous coordinates by querying the 1162// Resolve a `.btf` address into its rendezvous coordinates by querying the
@@ -1107,12 +1177,22 @@ bool BtfResolve(const std::string& btfAddr, std::string& meetingHostPort,
1107 if (!lease.Connected()) 1177 if (!lease.Connected())
1108 continue; 1178 continue;
1109 btf::Descriptor d; 1179 btf::Descriptor d;
1110 if (!ResolveDescriptor(lease.Ws(), g_nostrKey.ctx, btfAddr, d)) 1180 bool fExchangeOk = false;
1181 if (!ResolveDescriptor(lease.Ws(), g_nostrKey.ctx, btfAddr, d, fExchangeOk))
1111 { 1182 {
1112 lease.fOk = true; // exchange completed, just no match 1183 // Only keep the pooled socket if the relay actually finished
1184 // the conversation. This used to say "exchange completed, just
1185 // no match" on every failure -- including a dead socket -- so
1186 // ~CRelayLease never closed it and fConnected was never
1187 // cleared. A relay connection that died was then pinned in the
1188 // pool, reported healthy, and reused for every later lookup,
1189 // which is why "no Nostr descriptor found" persisted for a
1190 // descriptor that was demonstrably published, and why only a
1191 // restart cleared it.
1192 lease.fOk = fExchangeOk;
1113 continue; 1193 continue;
1114 } 1194 }
1115 lease.fOk = true; 1195 lease.fOk = fExchangeOk;
1116 if (d.meeting_node.empty() || d.meeting_node == "rendezvous-pending") 1196 if (d.meeting_node.empty() || d.meeting_node == "rendezvous-pending")
1117 { 1197 {
1118 if (!onionHostPort || d.onion.empty()) 1198 if (!onionHostPort || d.onion.empty())
@@ -1450,7 +1530,7 @@ void ThreadBtfPoolAnnouncer(void*)
1450 LogPrint("pool", "ThreadBtfPoolAnnouncer started\n"); 1530 LogPrint("pool", "ThreadBtfPoolAnnouncer started\n");
1451 while (!fShutdown) 1531 while (!fShutdown)
1452 { 1532 {
1453 if (nMineMode == MINE_OPERATOR && gPoolRunning) 1533 if (nMineMode == MINE_OPERATOR && gPoolRunning && !fPoolNoAnnounce)
1454 { 1534 {
1455 BtfPoolAnnouncement ann; 1535 BtfPoolAnnouncement ann;
1456 ann.btfAddress = BtfLocalAddress(); 1536 ann.btfAddress = BtfLocalAddress();
@@ -1785,14 +1865,17 @@ void ThreadNostrSeed(void* parg)
1785 PublishDescriptor(ws, key); 1865 PublishDescriptor(ws, key);
1786 Sleep(700); // let the relay store the replaceable event 1866 Sleep(700); // let the relay store the replaceable event
1787 btf::Descriptor d; 1867 btf::Descriptor d;
1788 if (ResolveDescriptor(ws, key.ctx, key.BtfAddress(), d)) 1868 bool fExchangeOk = false;
1869 if (ResolveDescriptor(ws, key.ctx, key.BtfAddress(), d, fExchangeOk))
1789 { 1870 {
1790 LogPrint("nostr", "Nostr: .btf self-resolve OK via %s (meeting_node=%s, enc=%s...)\n", 1871 LogPrint("nostr", "Nostr: .btf self-resolve OK via %s (meeting_node=%s, enc=%s...)\n",
1791 pszNostrRelays[i], d.meeting_node.c_str(), d.enc.substr(0, 16).c_str()); 1872 pszNostrRelays[i], d.meeting_node.c_str(), d.enc.substr(0, 16).c_str());
1792 lease.fOk = true; 1873 lease.fOk = fExchangeOk;
1793 break; 1874 break;
1794 } 1875 }
1795 lease.fOk = true; 1876 // Same rule as BtfResolve: a socket only goes back in the pool if
1877 // the relay finished the conversation on it.
1878 lease.fOk = fExchangeOk;
1796 } 1879 }
1797 CATCH_PRINT_EXCEPTION("btf self-test") 1880 CATCH_PRINT_EXCEPTION("btf self-test")
1798 } 1881 }
@@ -1824,7 +1907,7 @@ void ThreadNostrSeed(void* parg)
1824 } 1907 }
1825 CATCH_PRINT_EXCEPTION("ConnectDiscoveredBtfPeers") 1908 CATCH_PRINT_EXCEPTION("ConnectDiscoveredBtfPeers")
1826 1909
1827 if (nMineMode == MINE_OPERATOR && gPoolRunning) 1910 if (nMineMode == MINE_OPERATOR && gPoolRunning && !fPoolNoAnnounce)
1828 { 1911 {
1829 BtfPoolAnnouncement ann; 1912 BtfPoolAnnouncement ann;
1830 ann.btfAddress = BtfLocalAddress(); 1913 ann.btfAddress = BtfLocalAddress();
diff --git a/src/nostr.h b/src/nostr.h
index dc4288c..6dbbe26 100644
--- a/src/nostr.h
+++ b/src/nostr.h
@@ -50,6 +50,14 @@ std::string BtfLocalOnionEndpoint();
50// If set (via /announcerelay=host:port), this node announces that relay on 50// If set (via /announcerelay=host:port), this node announces that relay on
51// Nostr so other nodes discover it automatically -- no manual seed-list edit. 51// Nostr so other nodes discover it automatically -- no manual seed-list edit.
52extern std::string strBtfAnnounceRelay; 52extern std::string strBtfAnnounceRelay;
53// True when /rvrelay pinned the meeting relay list: BtfAllRelays then returns
54// ONLY vBtfMeetingRelays, never Nostr-discovered volunteer relays. Without
55// this, a pool that pins one public relay still spawns an accept thread per
56// discovered relay, and each thread's optimistic BtfSetActiveRelay makes the
57// published descriptor flap to relays the pool may not be reachable at.
58extern bool fBtfRelayPinned;
59// Basic sanity check on a "host:port" relay address (numeric port 1..65535).
60bool LooksLikeRelay(const std::string& r);
53// Every relay to try: curated seeds + relays discovered from Nostr announcements. 61// Every relay to try: curated seeds + relays discovered from Nostr announcements.
54std::vector<std::string> BtfAllRelays(); 62std::vector<std::string> BtfAllRelays();
55 63
diff --git a/src/rpc.cpp b/src/rpc.cpp
index 813f1d1..7a6b6a4 100644
--- a/src/rpc.cpp
+++ b/src/rpc.cpp
@@ -6,8 +6,8 @@
6// Architecture: 6// Architecture:
7// PoolEventLoop -- single thread, owns all miner fds, all job/share/payout 7// PoolEventLoop -- single thread, owns all miner fds, all job/share/payout
8// state. No locks needed on any of that data. 8// state. No locks needed on any of that data.
9// AcceptOneFn -- one thread per relay. Blocks on RvServiceRegister + 9// AcceptOneFn -- one thread per relay. Registers, waits to be paired,
10// BtfServiceWrap, pushes finished socket to gReadyQueue. 10// wraps the tunnel, pushes finished socket to gReadyQueue.
11// AcceptLoopFn -- spawns AcceptOneFn threads, re-polls relay list every 11// AcceptLoopFn -- spawns AcceptOneFn threads, re-polls relay list every
12// 60s so dynamically discovered relays get picked up. 12// 60s so dynamically discovered relays get picked up.
13// 13//
@@ -31,10 +31,15 @@
31#include <netinet/in.h> 31#include <netinet/in.h>
32#include <arpa/inet.h> 32#include <arpa/inet.h>
33#include <unistd.h> 33#include <unistd.h>
34#include <fcntl.h>
35#include <dirent.h>
36#else
37#include <io.h>
34#endif 38#endif
35 39
36#include <atomic> 40#include <atomic>
37#include <mutex> 41#include <mutex>
42#include <thread>
38#include <map> 43#include <map>
39#include <set> 44#include <set>
40#include <deque> 45#include <deque>
@@ -143,6 +148,75 @@ static bool RebuildJob()
143 return false; 148 return false;
144 } 149 }
145 150
151 // Don't issue jobs while we are behind the network: mining against a
152 // stale tip produces blocks the network will orphan, and payout
153 // entries that can never be paid (their coinbases never exist on the
154 // main chain). Require at least one peer, and require that no peer
155 // claims a longer chain than ours.
156 if (vNodes.empty()) {
157 LogPrint("pool", "RebuildJob: no peers yet -- not issuing jobs\n");
158 return false;
159 }
160 // This used to compare the tip's timestamp against 30 minutes, which
161 // was wrong in both directions and cost real money.
162 //
163 // It cannot tell "we are not synced" from "the network is slow". A
164 // pool mining its own fork keeps its tip timestamp fresh forever, so
165 // the check passed for the whole of heights 3966-4085 -- 120
166 // consecutive blocks, every one of them orphaned, 5890.50 BTF of
167 // payouts voided. Then, once the pool stopped finding blocks, the
168 // same check refused to issue jobs against a tip that was perfectly
169 // valid and was the network's own best, and the pool -- roughly 90%
170 // of the hashrate -- could not restart the chain it was the majority
171 // of. Stale tip, so no jobs; no jobs, so the tip stays stale.
172 //
173 // The height peers announce in their version message answers the
174 // question the age check was only guessing at. Median, so one peer
175 // claiming an absurd height cannot stall us, and one block of slack
176 // to match WarnIfBehind, because somebody is always mid-relay. A
177 // peer that predates the announcement reports -1 and is ignored; if
178 // nobody has told us anything we mine our best chain, which is what
179 // the node would do without a pool attached.
180 int nPeerHeight = GetPeerMedianHeight();
181 if (nPeerHeight >= 0 && nBestHeight < nPeerHeight - 1) {
182 LogPrint("pool", "RebuildJob: %d blocks behind the network "
183 "(height %d, peers report %d) -- not issuing jobs\n",
184 nPeerHeight - nBestHeight, nBestHeight, nPeerHeight);
185 return false;
186 }
187
188 // The comparison above is blind in the one case that matters most.
189 //
190 // When the node goes deaf -- peers still in vNodes, nothing arriving --
191 // no peer announces anything, GetPeerMedianHeight() returns -1, and the
192 // fallback ("nobody has told us anything, so mine our best chain") let
193 // the pool build a private fork for as long as the deafness lasted.
194 // That fallback is right for a node that has simply not been told yet
195 // and wrong for a node that has stopped being able to hear, and a
196 // height nobody sent cannot tell those apart. Time can.
197 //
198 // Deliberately not behind a debug category, and deliberately not a
199 // silent refusal: an operator whose pool has stopped issuing jobs needs
200 // the reason in the log. /poolmineblind is the escape hatch for a node
201 // that really is alone -- without one, a genuinely isolated pool that is
202 // the majority of the hashrate could never restart its own chain, which
203 // is the deadlock the tip-age check used to cause.
204 if (nPeerHeight < 0 && !fPoolMineBlind) {
205 int64 nAge = GetPeerHeightAgeSecs();
206 if (nAge >= BTF_NO_PEER_HEIGHT_MAX_SECS) {
207 static int64 nLastSaid = 0;
208 if (GetTime() - nLastSaid > 60) {
209 nLastSaid = GetTime();
210 printf("RebuildJob: no peer has reported a height for %d minutes "
211 "-- not issuing jobs (our height %d, %d peers connected). "
212 "This node is deaf; mining now would fork it. "
213 "Pass /poolmineblind to override.\n",
214 (int)(nAge / 60), nBestHeight, (int)vNodes.size());
215 }
216 return false;
217 }
218 }
219
146 CBlockIndex* pindexPrev = pindexBest; 220 CBlockIndex* pindexPrev = pindexBest;
147 unsigned int nBits = GetNextWorkRequired(pindexPrev); 221 unsigned int nBits = GetNextWorkRequired(pindexPrev);
148 222
@@ -194,7 +268,15 @@ static bool RebuildJob()
194 vector<char> vfAlreadyAdded(mapTransactions.size()); 268 vector<char> vfAlreadyAdded(mapTransactions.size());
195 bool fFoundSomething = true; 269 bool fFoundSomething = true;
196 unsigned int sz = 0; 270 unsigned int sz = 0;
197 while (fFoundSomething && sz < MAX_SIZE / 2) 271 // The cap is MAX_BLOCK_SIZE/2, not MAX_SIZE/2. MAX_SIZE is the
272 // 32 MB serialisation ceiling, so half of it is 16 MB -- this
273 // builder would happily assemble a template that its own
274 // CheckBlock() rejects at the 1 MB consensus cap. This is the
275 // pool's job builder, so a block over the cap is not one
276 // wasted solve, it is every miner on the pool working a
277 // template the network will reject. The inner break bounds the
278 // overshoot to a single transaction within a pass.
279 while (fFoundSomething && sz < MAX_BLOCK_SIZE / 2)
198 { 280 {
199 fFoundSomething = false; 281 fFoundSomething = false;
200 unsigned int n = 0; 282 unsigned int n = 0;
@@ -214,6 +296,7 @@ static bool RebuildJob()
214 sz += ::GetSerializeSize(tx, SER_NETWORK); 296 sz += ::GetSerializeSize(tx, SER_NETWORK);
215 vfAlreadyAdded[n] = true; 297 vfAlreadyAdded[n] = true;
216 fFoundSomething = true; 298 fFoundSomething = true;
299 if (sz > MAX_BLOCK_SIZE / 2) break;
217 } 300 }
218 } 301 }
219 } 302 }
@@ -251,17 +334,45 @@ static bool RebuildJob()
251// Payouts -- event loop thread only 334// Payouts -- event loop thread only
252// --------------------------------------------------------------------------- 335// ---------------------------------------------------------------------------
253struct PendingPayout { 336struct PendingPayout {
254 std::string id; 337 // The block that funds this payout, identified by hash and not by height
255 int blockHeight; 338 // alone. A reorg replaces the block at a height without changing the
256 int matureAtHeight; 339 // height, so a height-keyed obligation outlives the coinbase that was
340 // supposed to pay it: a 280-block reorg once left ~14.4k BTF of debt
341 // owed against blocks that no longer existed, which then drained the
342 // wallet paying fork-era miners out of real-chain coin.
343 //
344 // Upstream keys these on the string "block-<height>". That is the same
345 // bug this fork already paid for, and height keying additionally lets
346 // FindRecordedPoolPayout match the *previous* round's transaction after a
347 // reorg and silently skip a miner, so their `id` is deliberately dropped.
348 int blockHeight = 0;
349 // Must be explicitly zeroed: uint256's default constructor is empty
350 // (uint256.h:514), so leaving this unassigned gives every ledger entry
351 // loaded from an older file a garbage hash, which then matches no block
352 // and gets voided as "orphaned". That mistake voided a live payout.
353 uint256 blockHash = 0; // zero = entry predates hash keying
354 int matureAtHeight = 0;
257 std::map<std::string, int64> amounts; // address -> satoshis 355 std::map<std::string, int64> amounts; // address -> satoshis
356 // From upstream: which recipients already have a transaction. This is what
357 // lets an entry stay on disk until every address in `amounts` is paid,
358 // instead of being erased the moment the batch is handed off.
258 std::map<std::string, std::string> paidTxids; // address -> txid 359 std::map<std::string, std::string> paidTxids; // address -> txid
259 int64 nextAttempt; 360 // Retry and backoff state does NOT live here. It used to be declared here
260 361 // as well as on PayingRecord, and the copy on this side was never written:
261 PendingPayout() : blockHeight(0), matureAtHeight(0), nextAttempt(0) {} 362 // `retryAfter` was declared, serialised, loaded and tested at the flush
363 // gate, and the only assignment to it anywhere was `= 0`, so the gate could
364 // not fire. `retries` and `deferrals` were the same story. The real state
365 // is per (round, recipient) -- which is the granularity a payment can fail
366 // at -- and it lives on PayingRecord, which is now what persists it.
262}; 367};
263static std::vector<PendingPayout> gPendingPayouts; 368static std::vector<PendingPayout> gPendingPayouts;
264static std::mutex gPayoutLedgerMutex; 369static std::mutex gPayoutLedgerMutex;
370// Bumped on every serialisation, under gPayoutLedgerMutex. See PublishVersioned.
371static uint64 gPayoutLedgerSeq = 0;
372// Serialises writers of pending_payouts.json, and guards the sequence number
373// of the image currently on disk. Never taken while a data mutex is held.
374static std::mutex gPayoutLedgerFileMutex;
375static uint64 gPayoutLedgerPublishedSeq = 0;
265 376
266struct PoolRoundProof { 377struct PoolRoundProof {
267 std::string id; 378 std::string id;
@@ -282,6 +393,12 @@ struct PoolRoundProof {
282 std::map<std::string, uint64> shares; 393 std::map<std::string, uint64> shares;
283 std::map<std::string, int64> payouts; 394 std::map<std::string, int64> payouts;
284 std::map<std::string, std::string> paidTxids; 395 std::map<std::string, std::string> paidTxids;
396 // Recipients the ledger has written off, and why. Without this the public
397 // round proof goes on advertising an address as owed with "txid": null
398 // forever, while the ledger has already dropped the obligation -- the two
399 // records of the same round permanently disagreeing, with the operator
400 // holding the money.
401 std::map<std::string, std::string> forfeited; // address -> reason
285 402
286 PoolRoundProof() 403 PoolRoundProof()
287 : blockHeight(0), matureAtHeight(0), createdAt(0), feePercent(0.0), 404 : blockHeight(0), matureAtHeight(0), createdAt(0), feePercent(0.0),
@@ -290,6 +407,9 @@ struct PoolRoundProof {
290}; 407};
291static std::vector<PoolRoundProof> gPoolRounds; 408static std::vector<PoolRoundProof> gPoolRounds;
292static std::mutex gPoolRoundsMutex; 409static std::mutex gPoolRoundsMutex;
410static uint64 gPoolRoundsSeq = 0; // guarded by gPoolRoundsMutex
411static std::mutex gPoolRoundsFileMutex;
412static uint64 gPoolRoundsPublishedSeq = 0; // guarded by gPoolRoundsFileMutex
293static const size_t MAX_POOL_ROUNDS = 200; 413static const size_t MAX_POOL_ROUNDS = 200;
294 414
295static std::string PayoutFilePath() 415static std::string PayoutFilePath()
@@ -304,9 +424,302 @@ static std::string PoolRoundsFilePath()
304 return GetAppDir() + "/pool_rounds.json"; 424 return GetAppDir() + "/pool_rounds.json";
305} 425}
306 426
307static std::string PayoutIdForBlock(int blockHeight) 427// ---------------------------------------------------------------------------
428// Atomic, ordered publication of the payout state files
429// ---------------------------------------------------------------------------
430//
431// Every one of these files is a full dump of an in-memory structure, written
432// tmp-then-rename so a crash cannot truncate it. Two things have to be true
433// for that to actually be safe, and only the first was:
434//
435// 1. the rename must be atomic (it is), and
436// 2. exactly one thread at a time may be writing the temp file, and a write
437// must never publish an OLDER image over a newer one.
438//
439// (2) was broken for pending_payouts.json. The event loop wrote it with
440// gPayoutLedgerMutex held, the payout thread wrote it with the mutex
441// released, and nothing serialised the two: both opened the same
442// "<path>.tmp" with independent handles and both renamed it. The interleaved
443// file that lands is either a mixture of two images (LoadPendingPayouts then
444// throws mid-loop and comes up with a nearly empty ledger -- every obligation
445// not yet exploded, gone) or a whole newly-queued round silently overwritten
446// by an older snapshot that predates it.
447//
448// The fix is a per-file write mutex, held only across the I/O and never
449// across a data-structure mutex, plus a version on the snapshot. Serialising
450// happens under the data mutex and stamps a monotonically increasing sequence
451// number; publishing happens with the data mutex released and refuses any
452// blob whose sequence number is not newer than what is already on disk. So
453// writers can queue in any order and finish in any order -- the file only
454// ever moves forwards.
455//
456// The other half of the rule is that no thread waits for I/O while holding
457// gPayoutLedgerMutex, gPayoutExecMutex or gPoolRoundsMutex, which is the
458// contention the merge was right to remove from the event loop. That was
459// claimed here before it was true: pool_status.json is written by the same
460// WriteFileAtomic and RefreshStats held gPayoutLedgerMutex at function scope
461// across the call, so once the writes gained fsync() the event loop was doing
462// two of them every ten seconds with the payout thread's mutex held -- worse
463// than the buffered write the merge replaced. The status file now goes through
464// the same serialise-under-the-lock / publish-outside-it split as the ledgers,
465// and the rule holds for all four files. It is checked, not asserted: see the
466// "no payout mutex is held across a publish" section of the payout-ledger
467// self-test.
468
469// A serialised file image plus the version of the in-memory state it was
470// taken from.
471struct VersionedBlob {
472 std::string blob;
473 uint64 seq = 0;
474};
475
476// Push a file's contents out of the C library's buffer AND out of the kernel's
477// page cache, so the bytes exist on the device before anything points at them.
478static bool SyncFileHandle(FILE* f)
479{
480 if (fflush(f) != 0)
481 return false;
482#ifdef _WIN32
483 return _commit(_fileno(f)) == 0;
484#else
485 return fsync(fileno(f)) == 0;
486#endif
487}
488
489// Make the directory entry that rename() created durable too.
490//
491// A tmp+rename is only atomic with respect to a crash of the process. Against
492// a power cut or a hard VM reset the rename can reach the platter while the
493// data it points at has not, and the file that comes back is short or empty.
494// For pending_payouts.json that is not a corrupt file, it is a silent one:
495// LoadPendingPayouts catches the parse error, logs a single warning, and the
496// node comes up owing nobody anything.
497//
498// Best effort -- a filesystem that does not allow the directory to be opened
499// for this is not a reason to refuse to write the ledger.
500static void SyncDirectoryOf(const std::string& path)
501{
502#ifndef _WIN32
503 size_t slash = path.find_last_of('/');
504 std::string dir = (slash == std::string::npos) ? std::string(".")
505 : path.substr(0, slash);
506 if (dir.empty())
507 dir = "/";
508 int fd = open(dir.c_str(), O_RDONLY);
509 if (fd < 0)
510 return;
511 fsync(fd);
512 close(fd);
513#endif
514}
515
516// Replace `path` with `data` atomically and durably. Caller must hold the
517// write mutex for `path`: the temp name is derived from the target, so two
518// threads in here at once for one path is precisely the race this exists to
519// prevent.
520//
521// Every failure removes the temp file before returning. It used to be left
522// where it fell, so a disk that filled up once left a short, valid-looking
523// "pending_payouts.json.tmp" next to the good ledger for as long as the node
524// ran. That file is a trap: it has the name of a half-finished save, and an
525// operator cleaning up after an incident is exactly the person who will move
526// it into place.
527//
528// Failures print unconditionally rather than through LogPrint(). These three
529// files are written under two different debug categories ("payout" for the
530// ledgers, "pool" for the status file), and a write failure is not a trace of
531// normal operation -- it is the file the operator is about to read not being
532// there. Gating it behind a category that had to be chosen before the failure
533// happened is how "could not replace pool_status.json" went missing.
534static bool WriteFileAtomic(const std::string& path, const std::string& data,
535 const char* what)
536{
537 std::string tmp = path + ".tmp";
538 FILE* f = fopen(tmp.c_str(), "w");
539 if (!f) {
540 printf("[payout] WARNING: could not write %s (%s)\n",
541 tmp.c_str(), what);
542 return false;
543 }
544 if (!data.empty() && fwrite(data.c_str(), 1, data.size(), f) != data.size()) {
545 printf("[payout] WARNING: short write to %s (%s)\n",
546 tmp.c_str(), what);
547 fclose(f);
548 remove(tmp.c_str());
549 return false;
550 }
551 // Before the rename, not after: publishing a name that points at nothing
552 // is the failure this ordering exists to prevent. If the data cannot be
553 // made durable, do not rename -- the previous file is still intact and
554 // still correct, and the next write will try again.
555 if (!SyncFileHandle(f)) {
556 printf("[payout] WARNING: could not flush %s to disk (%s)"
557 " -- keeping the previous file\n", tmp.c_str(), what);
558 fclose(f);
559 remove(tmp.c_str());
560 return false;
561 }
562 // fclose() is where buffered data actually reaches the OS, so a failure
563 // here means the .tmp is short -- renaming it over the real file would
564 // destroy the good copy, which is exactly what these files exist to
565 // prevent.
566 if (fclose(f) != 0) {
567 printf("[payout] WARNING: could not close %s (%s)\n",
568 tmp.c_str(), what);
569 remove(tmp.c_str());
570 return false;
571 }
572#ifdef _WIN32
573 // rename() on Windows will not replace an existing file.
574 remove(path.c_str());
575#endif
576 if (rename(tmp.c_str(), path.c_str()) != 0) {
577 printf("[payout] WARNING: could not replace %s (%s)\n",
578 path.c_str(), what);
579 remove(tmp.c_str());
580 return false;
581 }
582 SyncDirectoryOf(path);
583 return true;
584}
585
586// Paths whose loader threw part-way through, and which must therefore never be
587// written again by this process.
588//
589// All three loaders push entries into the live structure INSIDE the try, so an
590// exception half way down a file leaves a PARTIAL ledger in memory -- the
591// entries before the bad one, and nothing after it. The first publish after
592// that then writes the partial state over the file, and the only record of
593// what was owed is gone. ReconcilePayouts publishes unconditionally at
594// startup, so "the first publish" is a few milliseconds later, before an
595// operator could possibly intervene. a8e2f58 named this failure mode in its
596// own commit message and then only fixed the write side.
597//
598// The file is renamed out of the way instead of being left in place, so the
599// evidence survives even if the operator restarts into the same fault, and so
600// a node that comes up owing nobody anything is visibly missing its ledger
601// rather than quietly appearing to have none.
602static std::set<std::string> gRefusedFiles;
603static std::mutex gRefusedFilesMutex;
604
605static bool FileIsRefused(const std::string& path)
606{
607 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
608 return gRefusedFiles.count(path) > 0;
609}
610
611// Quarantine `path` and refuse to write it for the rest of the process.
612//
613// Prints unconditionally: an operator who has just lost a payout ledger must
614// be told at whatever log settings they were already running with, and the
615// name it was moved to is the only way back to the data.
616static void RefuseFileAfterLoadFailure(const std::string& path, const char* what,
617 const char* why)
618{
619 {
620 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
621 gRefusedFiles.insert(path);
622 }
623 std::string bad = strprintf("%s.bad.%lld", path.c_str(), (long long)GetTime());
624 bool moved = (rename(path.c_str(), bad.c_str()) == 0);
625 printf("[payout] ERROR: %s (%s) did not parse: %s\n"
626 "[payout] ERROR: what loaded before the error is a PARTIAL picture,"
627 " so this file will not be written again by this process --"
628 " restart once the cause is fixed\n"
629 "[payout] ERROR: the file %s\n",
630 path.c_str(), what, why,
631 moved ? strprintf("has been kept as %s", bad.c_str()).c_str()
632 : "could NOT be renamed out of the way -- copy it now");
633}
634
635// Publish a versioned image of some in-memory state to `path`.
636//
637// `fileMutex` serialises writers of `path` and guards `publishedSeq`. A blob
638// older than what is already published is dropped: these files are whole-state
639// dumps, so a higher sequence number is by construction a later state of the
640// same structure and publishing it loses nothing. On a write failure
641// `publishedSeq` is deliberately left alone, so the next (newer) write still
642// gets a chance to land.
643static void PublishVersioned(const std::string& path, const VersionedBlob& v,
644 std::mutex& fileMutex, uint64& publishedSeq,
645 const char* what)
646{
647 // A default-constructed blob is not a snapshot of anything; it is a
648 // VersionedBlob nobody filled in. Today they are already dropped, because
649 // every serialiser stamps ++gSeq and so never produces 0, and 0 fails the
650 // seq <= publishedSeq test below -- but only for as long as publishedSeq
651 // starts at 0 and nobody writes `seq = gSeq++`. One character, and the
652 // ledger file becomes "[]". Say it out loud instead of relying on it.
653 if (!v.seq)
654 return;
655 // A file whose loader threw is not a file this process knows the contents
656 // of. Writing it would replace a full record of what was owed with
657 // whatever happened to parse before the error.
658 if (FileIsRefused(path))
659 return;
660 std::lock_guard<std::mutex> lk(fileMutex);
661 if (v.seq <= publishedSeq)
662 return;
663 if (!WriteFileAtomic(path, v.blob, what))
664 return;
665 publishedSeq = v.seq;
666}
667
668// Identity of a payout round. Used as the pool-round proof id and, more
669// importantly, as the `pool_payout_id` written into wallet transaction
670// metadata and matched by FindRecordedPoolPayout().
671//
672// Derived from the block HASH, not from its height alone. Upstream uses
673// "block-<height>", which names two different blocks identically across a
674// reorg -- and because this id is what the crash-recovery probe matches on, a
675// height-keyed id lets that probe find the *previous* round's transaction and
676// conclude a miner has already been paid when they have not. The height stays
677// in the string for readability only.
678//
679// A zero hash means the entry predates hash keying; those keep the old form so
680// anything already recorded against them still matches.
681static std::string PayoutIdForBlock(int blockHeight, const uint256& blockHash)
682{
683 if (blockHash == 0)
684 return strprintf("block-%d", blockHeight);
685 return strprintf("block-%d-%s", blockHeight,
686 blockHash.ToString().substr(0, 16).c_str());
687}
688
689// Do these two references name the same payout round?
690//
691// The block hash alone does not decide it, and every ledger lookup used to
692// assume it did. Entries grandfathered in from a file written before payouts
693// were keyed by hash all carry hash 0, so under a hash-only comparison the
694// first zero-hash entry answers for every other one: RecordPayoutTxid writes
695// a txid against the wrong round (which is then erased as "fully paid" while
696// the real debt still stands), and ReconcilePayouts drops a live in-flight
697// record because a *different* zero-hash round happened to pay that address.
698//
699// A non-zero hash is unique and decides on its own -- that is the whole point
700// of keying on it. A zero hash falls back to the height, which
701// LoadPendingPayouts supplies for any entry that can name its own maturity.
702// Mixed hash-ness never matches: refusing to match is the safe direction for
703// every caller here, because each of them treats a match as "this debt is
704// already dealt with".
705static bool SamePayoutRound(int aHeight, const uint256& aHash,
706 int bHeight, const uint256& bHash)
707{
708 if (aHash != bHash)
709 return false;
710 if (aHash != 0)
711 return true;
712 return aHeight == bHeight;
713}
714
715// An entry with neither a hash nor a height cannot be told apart from any
716// other entry in the same state, so nothing may be recorded against it: a
717// txid written to the wrong one reports a debt as settled that is still owed.
718// Such an entry is never paid and is logged instead -- refusing to act is the
719// safe failure, and no file this tree has ever written can produce one.
720static bool PayoutRoundIdentifiable(int height, const uint256& hash)
308{ 721{
309 return strprintf("block-%d", blockHeight); 722 return hash != 0 || height != 0;
310} 723}
311 724
312static bool PayoutFullyPaid(const PendingPayout& pp) 725static bool PayoutFullyPaid(const PendingPayout& pp)
@@ -317,15 +730,28 @@ static bool PayoutFullyPaid(const PendingPayout& pp)
317 return true; 730 return true;
318} 731}
319 732
320static void SavePendingPayoutsLocked() 733// Serialise the ledger and stamp it with the version it was taken at. Caller
734// must hold gPayoutLedgerMutex.
735//
736// Split from the file write so no thread holds the ledger mutex across the
737// I/O. Upstream writes the file inside the per-recipient loop with
738// gPayoutLedgerMutex held, while the event loop's FlushMaturePayouts blocks on
739// that same mutex -- one slow fsync per recipient, on the thread the miners'
740// I/O shares. The sequence number is what lets the write happen later, and
741// possibly out of order, without a stale image reaching the disk.
742static VersionedBlob SerializePendingPayoutsLocked()
321{ 743{
322 nlohmann::json arr = nlohmann::json::array(); 744 nlohmann::json arr = nlohmann::json::array();
323 for (const auto& pp : gPendingPayouts) { 745 for (const auto& pp : gPendingPayouts) {
324 nlohmann::json obj; 746 nlohmann::json obj;
325 obj["id"] = pp.id; 747 obj["blockHeight"] = pp.blockHeight;
326 obj["blockHeight"] = pp.blockHeight; 748 obj["blockHash"] = pp.blockHash.ToString();
327 obj["matureAtHeight"] = pp.matureAtHeight; 749 obj["matureAtHeight"] = pp.matureAtHeight;
328 obj["nextAttempt"] = pp.nextAttempt; 750 // No retryAfter: it was never written with anything but zero, and the
751 // retry clock that is actually used is on PayingRecord. Files that
752 // still carry the key load fine (it is ignored), and a build that
753 // expects it defaults an absent one to 0, which means "ready now" --
754 // the same answer it always got.
329 nlohmann::json amounts = nlohmann::json::object(); 755 nlohmann::json amounts = nlohmann::json::object();
330 for (const auto& kv : pp.amounts) 756 for (const auto& kv : pp.amounts)
331 amounts[kv.first] = kv.second; 757 amounts[kv.first] = kv.second;
@@ -336,24 +762,18 @@ static void SavePendingPayoutsLocked()
336 obj["paidTxids"] = paid; 762 obj["paidTxids"] = paid;
337 arr.push_back(obj); 763 arr.push_back(obj);
338 } 764 }
339 std::string path = PayoutFilePath(); 765 VersionedBlob v;
340 std::string tmp = path + ".tmp"; 766 v.blob = arr.dump(2);
341 FILE* f = fopen(tmp.c_str(), "w"); 767 v.seq = ++gPayoutLedgerSeq;
342 if (f) { 768 return v;
343 std::string s = arr.dump(2); 769}
344 fwrite(s.c_str(), 1, s.size(), f); 770
345 if (fclose(f) != 0) { 771// Publish a ledger snapshot. Holds no data mutex; safe -- and intended -- to
346 LogPrint("payout", "[payout] WARNING: could not close %s\n", tmp.c_str()); 772// call with gPayoutLedgerMutex released.
347 return; 773static void PublishPendingPayouts(const VersionedBlob& v)
348 } 774{
349#ifdef _WIN32 775 PublishVersioned(PayoutFilePath(), v, gPayoutLedgerFileMutex,
350 remove(path.c_str()); 776 gPayoutLedgerPublishedSeq, "pending payout ledger");
351#endif
352 if (rename(tmp.c_str(), path.c_str()) != 0)
353 LogPrint("payout", "[payout] WARNING: could not replace %s\n", path.c_str());
354 } else {
355 LogPrint("payout", "[payout] WARNING: could not write %s\n", tmp.c_str());
356 }
357} 777}
358 778
359static void LoadPendingPayouts() 779static void LoadPendingPayouts()
@@ -372,17 +792,26 @@ static void LoadPendingPayouts()
372 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); 792 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
373 for (const auto& obj : arr) { 793 for (const auto& obj : arr) {
374 PendingPayout pp; 794 PendingPayout pp;
375 if (obj.contains("blockHeight"))
376 pp.blockHeight = obj["blockHeight"].get<int>();
377 pp.matureAtHeight = obj["matureAtHeight"].get<int>(); 795 pp.matureAtHeight = obj["matureAtHeight"].get<int>();
378 if (obj.contains("id") && obj["id"].is_string()) 796 // Entries written before payouts were keyed by hash carry neither
379 pp.id = obj["id"].get<std::string>(); 797 // field. Leave blockHash zero and they are grandfathered past the
380 if (pp.blockHeight == 0) 798 // orphan check rather than voided on a guess.
799 if (obj.contains("blockHash") && obj["blockHash"].is_string())
800 pp.blockHash.SetHex(obj["blockHash"].get<std::string>());
801 if (obj.contains("blockHeight") && obj["blockHeight"].is_number())
802 pp.blockHeight = obj["blockHeight"].get<int>();
803 // Upstream's tolerance for an entry with no usable height, kept
804 // because it costs nothing and an entry that cannot name its block
805 // is worse than one derived from its maturity. COINBASE_MATURITY
806 // and not COINBASE_SPENDABLE_DEPTH: this reverses whatever the
807 // writer of *that* file used, and only files old enough to lack
808 // blockHeight are affected -- they were written when maturity was
809 // still COINBASE_MATURITY.
810 if (pp.blockHeight == 0 && pp.matureAtHeight > COINBASE_MATURITY)
381 pp.blockHeight = pp.matureAtHeight - COINBASE_MATURITY; 811 pp.blockHeight = pp.matureAtHeight - COINBASE_MATURITY;
382 if (pp.id.empty()) 812 // "retryAfter" is deliberately not read. Files written by the
383 pp.id = PayoutIdForBlock(pp.blockHeight); 813 // merged build carry it, always as 0, and there is nowhere left to
384 if (obj.contains("nextAttempt")) 814 // put it -- the retry clock is PayingRecord's.
385 pp.nextAttempt = obj["nextAttempt"].get<int64>();
386 for (auto it = obj["amounts"].begin(); it != obj["amounts"].end(); ++it) 815 for (auto it = obj["amounts"].begin(); it != obj["amounts"].end(); ++it)
387 pp.amounts[it.key()] = it.value().get<int64>(); 816 pp.amounts[it.key()] = it.value().get<int64>();
388 if (obj.contains("paidTxids") && obj["paidTxids"].is_object()) 817 if (obj.contains("paidTxids") && obj["paidTxids"].is_object())
@@ -393,11 +822,211 @@ static void LoadPendingPayouts()
393 LogPrint("payout", "[payout] loaded %zu pending payout(s) from disk\n", 822 LogPrint("payout", "[payout] loaded %zu pending payout(s) from disk\n",
394 gPendingPayouts.size()); 823 gPendingPayouts.size());
395 } catch (const std::exception& e) { 824 } catch (const std::exception& e) {
396 LogPrint("payout", "[payout] WARNING: failed to parse %s: %s\n", 825 RefuseFileAfterLoadFailure(path, "pending payout ledger", e.what());
397 path.c_str(), e.what()); 826 }
827}
828
829// A payment in flight: one record per (payout entry, address) pair, which is
830// the granularity SendMoney works at and therefore the granularity at which a
831// payment can be half-done when the process dies.
832//
833// This list is persisted. It used to be a memory-only queue, and matured
834// entries were erased from pending_payouts.json *before* being handed to it,
835// so any restart in between lost those payments with no record that anyone
836// was owed anything: a container restart on 2026-07-30 silently dropped
837// ~1,156 BTF that way. Writing the records before removing the ledger entry,
838// and reconciling against the wallet on startup, is what makes a restart
839// survivable.
840struct PayingRecord {
841 // Identity of this SLOT, distinct from the identity of the payment it
842 // describes. ForgetPayment and ReschedulePayment used to find their record
843 // by comparing every field, which is only an identity while no two records
844 // can be equal -- and a restart used to produce byte-identical duplicates
845 // (see FlushMaturePayouts), so the first match absorbed every update and
846 // the second was never touched again.
847 //
848 // Process-local and deliberately not persisted: gPaying is rebuilt from
849 // disk at every start, so an id only ever has to name a slot within one
850 // run, and reading ids back would mean trusting a file to contain no
851 // duplicates. Assigned under gPayoutExecMutex from gPayingNextId.
852 uint64 id = 0;
853 int blockHeight = 0;
854 uint256 blockHash = 0; // zero = predates hash keying, see PendingPayout
855 int matureAtHeight = 0;
856 std::string address;
857 int64 amount = 0;
858 int retries = 0;
859 int deferrals = 0;
860 int64 retryAfter = 0;
861 // Ask the wallet whether this payment already went out, once, before the
862 // first send attempt. Upstream asks on every attempt: FindRecordedPoolPayout
863 // is a linear scan of mapWallet under cs_mapWallet, per recipient, so with
864 // 2000 rotating payout addresses and a retry budget of 1440 that is a
865 // scan per address per minute for a day. The question it answers -- "did a
866 // previous process send this before it died?" -- can only change across a
867 // restart, and every restart re-sets this flag (ReconcilePayouts), so
868 // asking once per record is the whole of the answer.
869 //
870 // Not persisted: a record read back from disk is by definition a restart.
871 bool fProbeWallet = true;
872};
873static std::vector<PayingRecord> gPaying; // guarded by gPayoutExecMutex
874static std::mutex gPayoutExecMutex;
875static uint64 gPayingNextId = 0; // guarded by gPayoutExecMutex
876static uint64 gPayingSeq = 0; // guarded by gPayoutExecMutex
877static std::mutex gPayingFileMutex;
878static uint64 gPayingPublishedSeq = 0; // guarded by gPayingFileMutex
879static std::atomic<bool> gPayoutThreadRunning{false};
880
881// Upstream's gPayoutExecQueue/gPayoutExecMutex/gPayoutThreadRunning are
882// declared above with gPaying: the in-flight list IS the exec queue in this
883// tree, at per-recipient rather than per-round granularity, so a second queue
884// would be a second answer to "who is owed money". Only the dedupe set is new.
885//
886// Ids of rounds currently exploded into gPaying, so a flush cannot queue the
887// same round twice. Guarded by gPayoutExecMutex. Seeded from the restored
888// in-flight list by ReconcilePayouts -- it is memory-only, and leaving it
889// empty at startup was half of why a restart duplicated everything in flight.
890static std::set<std::string> gQueuedPayouts;
891
892// Is this exact payment already in flight? gPaying is keyed on
893// (round, address): the ledger holds one amount per address per round, so two
894// records for that pair are two standing orders to send the same money.
895// Caller must hold gPayoutExecMutex.
896static bool PaymentInFlightLocked(int blockHeight, const uint256& blockHash,
897 const std::string& address)
898{
899 for (const PayingRecord& e : gPaying)
900 if (e.address == address
901 && SamePayoutRound(e.blockHeight, e.blockHash, blockHeight, blockHash))
902 return true;
903 return false;
904}
905
906static std::string PayingFilePath()
907{
908 return GetAppDir() + "/paying_payouts.json";
909}
910
911// Serialise the in-flight list. Caller must hold gPayoutExecMutex.
912//
913// gPayoutExecMutex already serialises every writer of this file, so it never
914// had the interleaving race pending_payouts.json had. It is still split the
915// same way for a different reason: FlushMaturePayouts runs on the event loop
916// and holds gPayoutExecMutex, so writing (and, since this series, fsyncing)
917// the file under that mutex parks miner I/O behind the payout thread's disk.
918// The version stamp keeps the file monotonic now that the write is outside
919// the mutex that orders the writers.
920static VersionedBlob SerializePayingLocked()
921{
922 nlohmann::json arr = nlohmann::json::array();
923 for (const auto& r : gPaying) {
924 nlohmann::json obj;
925 obj["blockHeight"] = r.blockHeight;
926 obj["blockHash"] = r.blockHash.ToString();
927 obj["matureAtHeight"] = r.matureAtHeight;
928 obj["address"] = r.address;
929 obj["amount"] = r.amount;
930 obj["retries"] = r.retries;
931 obj["deferrals"] = r.deferrals;
932 // The live retry clock, which used not to be written at all: every
933 // restart handed every backing-off payment a retryAfter of 0 and put
934 // the whole set into the very next 500 ms batch, so a node restarted
935 // in a loop retried continuously instead of once a minute.
936 obj["retryAfter"] = r.retryAfter;
937 arr.push_back(obj);
938 }
939 VersionedBlob v;
940 v.blob = arr.dump(2);
941 v.seq = ++gPayingSeq;
942 return v;
943}
944
945// Publish an in-flight-list snapshot. Call with gPayoutExecMutex released.
946static void PublishPaying(const VersionedBlob& v)
947{
948 PublishVersioned(PayingFilePath(), v, gPayingFileMutex,
949 gPayingPublishedSeq, "in-flight payment list");
950}
951
952static void LoadPaying()
953{
954 std::string path = PayingFilePath();
955 FILE* f = fopen(path.c_str(), "r");
956 if (!f) return;
957 fseek(f, 0, SEEK_END);
958 long sz = ftell(f);
959 fseek(f, 0, SEEK_SET);
960 std::string s(sz, '\0');
961 fread(&s[0], 1, sz, f);
962 fclose(f);
963 try {
964 nlohmann::json arr = nlohmann::json::parse(s);
965 // gPaying and gPayingNextId are both annotated as guarded by
966 // gPayoutExecMutex and this was mutating them without it. Safe today
967 // -- LoadPaying only runs at startup, before the payout thread is
968 // spawned -- but "safe because of where it is called from" is a fact
969 // about the caller, not about this function, and the annotation is
970 // what every other reader will trust. The mutex is uncontended here by
971 // construction, so taking it costs nothing.
972 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
973 for (const auto& obj : arr) {
974 PayingRecord r;
975 r.address = obj["address"].get<std::string>();
976 r.amount = obj["amount"].get<int64>();
977 r.matureAtHeight = obj["matureAtHeight"].get<int>();
978 if (obj.contains("blockHash") && obj["blockHash"].is_string())
979 r.blockHash.SetHex(obj["blockHash"].get<std::string>());
980 if (obj.contains("blockHeight") && obj["blockHeight"].is_number())
981 r.blockHeight = obj["blockHeight"].get<int>();
982 if (obj.contains("retries") && obj["retries"].is_number())
983 r.retries = obj["retries"].get<int>();
984 if (obj.contains("deferrals") && obj["deferrals"].is_number())
985 r.deferrals = obj["deferrals"].get<int>();
986 // Absent in files written before this change; 0 means ready now,
987 // which is exactly the behaviour those files used to get.
988 if (obj.contains("retryAfter") && obj["retryAfter"].is_number())
989 r.retryAfter = obj["retryAfter"].get<int64>();
990 // Collapse duplicates as they are read. A build without the
991 // ReconcilePayouts seeding below wrote a second (and after another
992 // restart, a third) byte-identical record for every unpaid
993 // recipient, so a file written by one of those has to be cleaned
994 // up here or the pathology is inherited. The retry and deferral
995 // counts are merged upwards so collapsing does not hand the
996 // survivor a fresh budget.
997 PayingRecord* dup = NULL;
998 for (PayingRecord& e : gPaying)
999 if (e.address == r.address
1000 && SamePayoutRound(e.blockHeight, e.blockHash,
1001 r.blockHeight, r.blockHash)) {
1002 dup = &e;
1003 break;
1004 }
1005 if (dup) {
1006 dup->retries = std::max(dup->retries, r.retries);
1007 dup->deferrals = std::max(dup->deferrals, r.deferrals);
1008 dup->retryAfter = std::max(dup->retryAfter, r.retryAfter);
1009 LogPrint("payout", "[payout] collapsing duplicate in-flight"
1010 " record for %s to %s -- one payment, one record\n",
1011 FormatMoney(r.amount).c_str(), r.address.c_str());
1012 continue;
1013 }
1014 r.id = ++gPayingNextId;
1015 gPaying.push_back(r);
1016 }
1017 LogPrint("payout", "[payout] loaded %zu in-flight payment(s) from disk\n",
1018 gPaying.size());
1019 } catch (const std::exception& e) {
1020 RefuseFileAfterLoadFailure(path, "in-flight payment list", e.what());
398 } 1021 }
399} 1022}
400 1023
1024// ---------------------------------------------------------------------------
1025// Pool round proofs (upstream cbdd06d). Disjoint from the in-flight ledger
1026// above -- this is the operator's published record of how a round was split,
1027// not an obligation to pay. Both sides added code at this offset; both are
1028// kept.
1029// ---------------------------------------------------------------------------
401static nlohmann::json PoolRoundToJson(const PoolRoundProof& r) 1030static nlohmann::json PoolRoundToJson(const PoolRoundProof& r)
402{ 1031{
403 nlohmann::json j; 1032 nlohmann::json j;
@@ -436,12 +1065,26 @@ static nlohmann::json PoolRoundToJson(const PoolRoundProof& r)
436 p["txid"] = tx->second; 1065 p["txid"] = tx->second;
437 else 1066 else
438 p["txid"] = nullptr; 1067 p["txid"] = nullptr;
1068 // Only emitted when it happened, so the common shape of the file is
1069 // unchanged for anything already parsing it.
1070 std::map<std::string, std::string>::const_iterator fo = r.forfeited.find(kv.first);
1071 if (fo != r.forfeited.end()) {
1072 p["forfeited"] = true;
1073 p["forfeitReason"] = fo->second;
1074 }
439 j["payouts"][kv.first] = p; 1075 j["payouts"][kv.first] = p;
440 } 1076 }
441 return j; 1077 return j;
442} 1078}
443 1079
444static void SavePoolRoundsLocked() 1080// Serialise the published round proofs. Caller must hold gPoolRoundsMutex.
1081//
1082// Like the in-flight list, gPoolRoundsMutex already serialises the writers
1083// (RecordPoolRound on the event loop, MarkPoolRoundPaid on the payout thread),
1084// so this file never had the interleaving race either -- but the event loop
1085// should not be waiting on the payout thread's fsync to publish a round, so
1086// the write moves outside the mutex on the same versioned terms.
1087static VersionedBlob SerializePoolRoundsLocked()
445{ 1088{
446 nlohmann::json root; 1089 nlohmann::json root;
447 root["schema"] = "bitflash-pool-rounds-1"; 1090 root["schema"] = "bitflash-pool-rounds-1";
@@ -450,24 +1093,17 @@ static void SavePoolRoundsLocked()
450 for (const PoolRoundProof& r : gPoolRounds) 1093 for (const PoolRoundProof& r : gPoolRounds)
451 root["rounds"].push_back(PoolRoundToJson(r)); 1094 root["rounds"].push_back(PoolRoundToJson(r));
452 1095
453 std::string path = PoolRoundsFilePath(); 1096 VersionedBlob v;
454 std::string tmp = path + ".tmp"; 1097 v.blob = root.dump(2);
455 FILE* f = fopen(tmp.c_str(), "w"); 1098 v.seq = ++gPoolRoundsSeq;
456 if (!f) { 1099 return v;
457 LogPrint("payout", "[payout] WARNING: could not write %s\n", tmp.c_str()); 1100}
458 return; 1101
459 } 1102// Publish a round-proof snapshot. Call with gPoolRoundsMutex released.
460 std::string s = root.dump(2); 1103static void PublishPoolRounds(const VersionedBlob& v)
461 fwrite(s.c_str(), 1, s.size(), f); 1104{
462 if (fclose(f) != 0) { 1105 PublishVersioned(PoolRoundsFilePath(), v, gPoolRoundsFileMutex,
463 LogPrint("payout", "[payout] WARNING: could not close %s\n", tmp.c_str()); 1106 gPoolRoundsPublishedSeq, "pool round proofs");
464 return;
465 }
466#ifdef _WIN32
467 remove(path.c_str());
468#endif
469 if (rename(tmp.c_str(), path.c_str()) != 0)
470 LogPrint("payout", "[payout] WARNING: could not replace %s\n", path.c_str());
471} 1107}
472 1108
473static void LoadPoolRounds() 1109static void LoadPoolRounds()
@@ -512,6 +1148,9 @@ static void LoadPoolRounds()
512 r.payouts[it.key()] = it.value().value("amountSatoshis", (int64)0); 1148 r.payouts[it.key()] = it.value().value("amountSatoshis", (int64)0);
513 if (it.value().contains("txid") && it.value()["txid"].is_string()) 1149 if (it.value().contains("txid") && it.value()["txid"].is_string())
514 r.paidTxids[it.key()] = it.value()["txid"].get<std::string>(); 1150 r.paidTxids[it.key()] = it.value()["txid"].get<std::string>();
1151 if (it.value().value("forfeited", false))
1152 r.forfeited[it.key()] =
1153 it.value().value("forfeitReason", std::string("unspecified"));
515 } else if (it.value().is_number_integer()) { 1154 } else if (it.value().is_number_integer()) {
516 r.payouts[it.key()] = it.value().get<int64>(); 1155 r.payouts[it.key()] = it.value().get<int64>();
517 } 1156 }
@@ -524,55 +1163,283 @@ static void LoadPoolRounds()
524 LogPrint("payout", "[payout] loaded %zu pool round proof(s) from disk\n", 1163 LogPrint("payout", "[payout] loaded %zu pool round proof(s) from disk\n",
525 gPoolRounds.size()); 1164 gPoolRounds.size());
526 } catch (const std::exception& e) { 1165 } catch (const std::exception& e) {
527 LogPrint("payout", "[payout] WARNING: failed to parse %s: %s\n", 1166 RefuseFileAfterLoadFailure(path, "pool round proofs", e.what());
528 path.c_str(), e.what());
529 } 1167 }
530} 1168}
531 1169
1170// Is the block that funds this payout still on the main chain?
1171//
1172// A zero hash means the entry was written before payouts recorded one, so
1173// there is nothing to check and we assume it is good -- voiding those would
1174// mean refusing to pay miners on the strength of a missing field.
1175static bool PayoutBlockStillOnChain(const uint256& blockHash)
1176{
1177 if (blockHash == 0)
1178 return true;
1179 CRITICAL_BLOCK(cs_main)
1180 {
1181 std::map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(blockHash);
1182 if (mi == mapBlockIndex.end() || mi->second == NULL)
1183 return false;
1184 return mi->second->IsInMainChain();
1185 }
1186 return false;
1187}
1188
1189static std::string PayoutComment(int blockHeight)
1190{
1191 return strprintf("Pool payout height %d", blockHeight);
1192}
1193
1194// Startup reconciliation.
1195//
1196// This used to be the crash-recovery mechanism: it scanned the wallet with
1197// WalletAlreadyPaid() and then dropped ledger entries that had already been
1198// exploded into in-flight records. Neither job is right any more.
1199//
1200// The ledger is now authoritative and an entry survives until every recipient
1201// has a txid recorded against it (upstream cbdd06d), so dropping ledger
1202// entries here would reintroduce exactly the loss this merge is adopting that
1203// model to prevent. And the wallet scan is now FindRecordedPoolPayout(), done
1204// once per record on its first attempt, which is the same question asked in
1205// the one place that can act on the answer. Running both would be two
1206// authorities for the same debt.
1207//
1208// What is left is the useful part: the in-flight list restored from disk may
1209// contain records the ledger already shows as paid (we died between the send
1210// and the in-flight file being rewritten). The ledger wins.
1211//
1212// And the part that was missing: gQueuedPayouts is memory-only, so it came up
1213// empty while gPaying came up full. FlushMaturePayouts deduped against that
1214// empty set and against the ledger's paidTxids, and against nothing else --
1215// so a round that was exploded but not yet paid got a SECOND record per
1216// unpaid recipient on the first flush after every restart, deterministically.
1217// (Observed live: two byte-identical records for one (block, address).) The
1218// surviving in-flight list is exactly the statement of which rounds are
1219// already exploded, so it is what the set is rebuilt from.
1220static void ReconcilePayouts()
1221{
1222 VersionedBlob payingSnap;
1223 {
1224 std::lock_guard<std::mutex> exec(gPayoutExecMutex);
1225 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
1226
1227 size_t settled = 0;
1228 int64 settledAmount = 0;
1229 for (auto it = gPaying.begin(); it != gPaying.end(); ) {
1230 bool paid = false;
1231 for (const PendingPayout& pp : gPendingPayouts)
1232 if (SamePayoutRound(pp.blockHeight, pp.blockHash,
1233 it->blockHeight, it->blockHash)
1234 && pp.paidTxids.count(it->address)) {
1235 paid = true;
1236 break;
1237 }
1238 if (paid) {
1239 settledAmount += it->amount;
1240 settled++;
1241 it = gPaying.erase(it);
1242 } else {
1243 // Every restored record gets one wallet probe before it is
1244 // allowed to send: this is precisely the window where the
1245 // wallet may hold a transaction the ledger has not recorded.
1246 it->fProbeWallet = true;
1247 ++it;
1248 }
1249 }
1250 if (settled) {
1251 LogPrint("payout", "[payout] %zu in-flight payment(s) totalling %s"
1252 " are already recorded paid in the ledger -- not"
1253 " resending\n",
1254 settled, FormatMoney(settledAmount).c_str());
1255 }
1256
1257 // Rebuild the "already exploded" set from what survived. Without this
1258 // the first flush after a restart re-explodes every round in flight.
1259 gQueuedPayouts.clear();
1260 for (const PayingRecord& r : gPaying)
1261 gQueuedPayouts.insert(PayoutIdForBlock(r.blockHeight, r.blockHash));
1262
1263 payingSnap = SerializePayingLocked();
1264
1265 if (!gPaying.empty()) {
1266 int64 owed = 0;
1267 for (const auto& r : gPaying) owed += r.amount;
1268 LogPrint("payout", "[payout] resuming %zu in-flight payment(s)"
1269 " totalling %s\n",
1270 gPaying.size(), FormatMoney(owed).c_str());
1271 }
1272 }
1273 PublishPaying(payingSnap);
1274}
1275
532static void RecordPoolRound(const PoolRoundProof& round) 1276static void RecordPoolRound(const PoolRoundProof& round)
533{ 1277{
534 std::lock_guard<std::mutex> lk(gPoolRoundsMutex); 1278 VersionedBlob snap;
535 for (PoolRoundProof& existing : gPoolRounds) { 1279 {
536 if (existing.id == round.id) { 1280 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
537 existing = round; 1281 bool replaced = false;
538 SavePoolRoundsLocked(); 1282 for (PoolRoundProof& existing : gPoolRounds) {
539 return; 1283 if (existing.id == round.id) {
1284 existing = round;
1285 replaced = true;
1286 break;
1287 }
540 } 1288 }
1289 if (!replaced) {
1290 gPoolRounds.push_back(round);
1291 while (gPoolRounds.size() > MAX_POOL_ROUNDS)
1292 gPoolRounds.erase(gPoolRounds.begin());
1293 }
1294 snap = SerializePoolRoundsLocked();
541 } 1295 }
542 gPoolRounds.push_back(round); 1296 PublishPoolRounds(snap);
543 while (gPoolRounds.size() > MAX_POOL_ROUNDS)
544 gPoolRounds.erase(gPoolRounds.begin());
545 SavePoolRoundsLocked();
546} 1297}
547 1298
548static void MarkPoolRoundPaid(const std::string& id, 1299static void MarkPoolRoundPaid(const std::string& id,
549 const std::string& address, 1300 const std::string& address,
550 const std::string& txid) 1301 const std::string& txid)
551{ 1302{
552 std::lock_guard<std::mutex> lk(gPoolRoundsMutex); 1303 VersionedBlob snap;
553 for (PoolRoundProof& r : gPoolRounds) { 1304 {
554 if (r.id != id) 1305 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
555 continue; 1306 bool found = false;
556 r.paidTxids[address] = txid; 1307 for (PoolRoundProof& r : gPoolRounds) {
557 SavePoolRoundsLocked(); 1308 if (r.id != id)
558 return; 1309 continue;
1310 r.paidTxids[address] = txid;
1311 found = true;
1312 break;
1313 }
1314 if (!found)
1315 return;
1316 snap = SerializePoolRoundsLocked();
559 } 1317 }
1318 PublishPoolRounds(snap);
560} 1319}
561 1320
562// Payouts ready to execute are moved here under gPayoutExecMutex. 1321// Publish the fact that a recipient was written off, so the round proof and
563// A dedicated thread drains this list so SendMoney (which takes cs_main) 1322// the ledger agree. The ledger erases the amount (it is not owed any more and
564// never runs on the event loop thread and can't stall miner I/O. 1323// must not be reported as sent); the proof keeps the amount visible and marks
565static std::vector<PendingPayout> gPayoutExecQueue; 1324// it forfeited with a reason, because a miner who was promised a share is
566static std::mutex gPayoutExecMutex; 1325// entitled to see what happened to it.
567static std::atomic<bool> gPayoutThreadRunning{false}; 1326static void MarkPoolRoundForfeited(const std::string& id,
1327 const std::string& address,
1328 const std::string& reason)
1329{
1330 VersionedBlob snap;
1331 {
1332 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1333 bool found = false;
1334 for (PoolRoundProof& r : gPoolRounds) {
1335 if (r.id != id)
1336 continue;
1337 r.forfeited[address] = reason;
1338 found = true;
1339 break;
1340 }
1341 if (!found)
1342 return;
1343 snap = SerializePoolRoundsLocked();
1344 }
1345 PublishPoolRounds(snap);
1346}
568 1347
569static std::set<std::string> gQueuedPayouts; 1348// Which wallet transaction, if any, already paid this?
1349//
1350// Two kinds of evidence, in order of quality.
1351//
1352// Structured (pool_payout_id / pool_payout_address / pool_payout_amount) is
1353// upstream's, and it is exact: the id is derived from the block hash, so it
1354// names one round and one round only. Every transaction this build sends
1355// carries it.
1356//
1357// But NO transaction this pool has ever sent carries it. Every payout in the
1358// existing wallet has only a "comment". So on the first boot after this
1359// upgrade -- precisely the boot where the binary is swapped after an unclean
1360// shutdown with payments in flight -- a structured-only probe finds nothing
1361// and every one of those recipients is paid a second time. The deleted
1362// WalletAlreadyPaid() matched on what the payment WAS, and that evidence is
1363// still there; it just has to be read.
1364//
1365// The legacy fallback is therefore restored, with three constraints that
1366// together stop it answering for a round that is not this one:
1367//
1368// 1. it is only consulted when the structured lookup found nothing;
1369// 2. it ignores any transaction carrying pool_payout_id. That key means this
1370// build wrote the transaction, and this build's transactions are
1371// adjudicated by the exact path -- so a NEW round's payment can never be
1372// offered as evidence for an old one. Only pre-upgrade transactions,
1373// which is exactly the population the fallback exists for, are eligible;
1374// 3. `excluded` holds every txid already credited to some OTHER
1375// (round, address) pair -- from the ledger's paidTxids, which is durable,
1376// and from what this process has already claimed. One legacy transaction
1377// can therefore settle at most one debt, so N transactions can never
1378// satisfy more than N obligations.
1379//
1380// What is left unresolved by construction is two pre-upgrade rounds at the
1381// SAME height paying the same address the same amount, since the comment
1382// carries only the height. That needs a reorg.
1383//
1384// This used to say the losing twin's ledger entry is voided by the orphan
1385// check before it can be paid. That is wrong: PayoutBlockStillOnChain()
1386// returns true for a zero hash, and EVERY pre-upgrade entry has a zero hash,
1387// so the orphan check is a no-op for exactly the population this fallback
1388// serves.
1389//
1390// The outcome is still safe, by a different mechanism. SamePayoutRound()
1391// treats two zero-hash entries at the same height as the SAME round, so the
1392// pair is indistinguishable everywhere, not only here: FlushMaturePayouts
1393// explodes one of them and skips the other as already queued,
1394// PaymentInFlightLocked refuses a second record for that (round, address), and
1395// RecordPayoutTxid settles against the first match. The pair therefore
1396// presents as ONE debt and can be paid at most once -- which is the direction
1397// that matters for a fallback whose failure mode is treating one payment as
1398// proof of two.
1399//
1400// The cost of that mechanism, stated plainly: only one of a genuine
1401// same-height pair would ever be paid. It takes a reorg AND both rounds
1402// predating hash keying, and no file this tree has written since can produce
1403// one, so it is a property of history rather than a live risk.
1404//
1405// Where the evidence genuinely runs out the fallback simply does not match,
1406// and the payment is made -- paying a debt we cannot prove was settled is the
1407// correct default for a pool.
1408enum PayoutProbeResult {
1409 PAYOUT_PROBE_NONE = 0, // the wallet holds no record of this payment
1410 PAYOUT_PROBE_STRUCTURED, // matched this build's pool_payout_* metadata
1411 PAYOUT_PROBE_LEGACY // matched a pre-upgrade comment and output
1412};
570 1413
571static bool FindRecordedPoolPayout(const std::string& id, 1414static PayoutProbeResult FindRecordedPoolPayout(const std::string& id,
572 const std::string& address, 1415 int blockHeight,
573 int64 amount, 1416 const std::string& address,
574 std::string& txidRet) 1417 int64 amount,
1418 const std::set<std::string>& excluded,
1419 std::string& txidRet)
575{ 1420{
1421 // Built before cs_mapWallet is taken: an invalid address has no script to
1422 // compare against, and the caller has already rejected those.
1423 uint160 h160;
1424 bool fHaveScript = AddressToHash160(address, h160);
1425 CScript want;
1426 if (fHaveScript)
1427 want << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
1428 const std::string comment = PayoutComment(blockHeight);
1429
1430 // Candidate legacy matches, adjudicated after cs_mapWallet is released.
1431 //
1432 // The depth check cannot happen inside the scan: GetDepthInMainChain()
1433 // reads mapBlockIndex and pindexBest, which belong to cs_main, and cs_main
1434 // is taken BEFORE cs_mapWallet everywhere else in this tree
1435 // (CreateTransaction, ConnectBlock). Taking it the other way round here
1436 // would be a lock-order inversion in the payout thread. There is at most a
1437 // handful of these: a candidate has to carry this exact height's comment
1438 // and an output of exactly this amount to this address.
1439 std::vector<uint256> legacyCandidates;
1440
1441 // One pass, not two: this scan runs under cs_mapWallet, which the whole
1442 // node contends on, and the merge is already paying for it once per record.
576 CRITICAL_BLOCK(cs_mapWallet) 1443 CRITICAL_BLOCK(cs_mapWallet)
577 { 1444 {
578 for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); 1445 for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin();
@@ -580,23 +1447,130 @@ static bool FindRecordedPoolPayout(const std::string& id,
580 { 1447 {
581 const CWalletTx& wtx = it->second; 1448 const CWalletTx& wtx = it->second;
582 map<string, string>::const_iterator idIt = wtx.mapValue.find("pool_payout_id"); 1449 map<string, string>::const_iterator idIt = wtx.mapValue.find("pool_payout_id");
583 if (idIt == wtx.mapValue.end() || idIt->second != id) 1450 if (idIt != wtx.mapValue.end()) {
1451 // Written by this build. Adjudicated exactly, and never
1452 // offered to the fallback below.
1453 if (idIt->second != id)
1454 continue;
1455 map<string, string>::const_iterator addrIt = wtx.mapValue.find("pool_payout_address");
1456 if (addrIt == wtx.mapValue.end() || addrIt->second != address)
1457 continue;
1458 map<string, string>::const_iterator amtIt = wtx.mapValue.find("pool_payout_amount");
1459 if (amtIt == wtx.mapValue.end() || amtIt->second != strprintf("%lld", amount))
1460 continue;
1461 txidRet = wtx.GetHash().GetHex();
1462 return PAYOUT_PROBE_STRUCTURED;
1463 }
1464
1465 if (!fHaveScript)
584 continue; 1466 continue;
585 map<string, string>::const_iterator addrIt = wtx.mapValue.find("pool_payout_address"); 1467 // The comment is written only by the payout path, so its presence
586 if (addrIt == wtx.mapValue.end() || addrIt->second != address) 1468 // already identifies the transaction as one of ours.
1469 map<string, string>::const_iterator ci = wtx.mapValue.find("comment");
1470 if (ci == wtx.mapValue.end() || ci->second != comment)
587 continue; 1471 continue;
588 map<string, string>::const_iterator amtIt = wtx.mapValue.find("pool_payout_amount"); 1472 if (excluded.count(wtx.GetHash().GetHex()))
589 if (amtIt == wtx.mapValue.end() || amtIt->second != strprintf("%lld", amount)) 1473 continue; // already credited to a different debt
1474 foreach (const CTxOut& txout, wtx.vout)
1475 if (txout.nValue == amount && txout.scriptPubKey == want) {
1476 legacyCandidates.push_back(it->first);
1477 break;
1478 }
1479 }
1480 }
1481
1482 // A legacy match must have CONFIRMED.
1483 //
1484 // The structured path does not need this and deliberately does not have
1485 // it: this process sent that transaction moments ago, and a send that
1486 // failed leaves nothing behind, because SendMoney calls
1487 // RollbackTransactionSpent, which calls EraseFromWallet.
1488 //
1489 // The legacy path is the opposite case in every respect. Those
1490 // transactions are days or weeks old, so whether they confirmed is a
1491 // settled fact rather than an open question, and a pre-upgrade payout that
1492 // never confirmed is evidence of NON-payment about to be read as payment.
1493 // This chain has had a 280-block reorg (see FlushMaturePayouts): a payout
1494 // invalidated by it is still sitting in mapWallet with its comment intact,
1495 // and the first boot after this upgrade is exactly when it gets matched.
1496 // The cost of being wrong here is a miner who is never paid and no record
1497 // that they were owed.
1498 CRITICAL_BLOCK(cs_main)
1499 CRITICAL_BLOCK(cs_mapWallet)
1500 {
1501 for (const uint256& hash : legacyCandidates) {
1502 map<uint256, CWalletTx>::const_iterator it = mapWallet.find(hash);
1503 if (it == mapWallet.end())
1504 continue; // erased under us; it is not evidence of anything
1505 int depth = it->second.GetDepthInMainChain();
1506 if (depth < 1) {
1507 // Loud: this is the difference between a miner having been
1508 // paid and a miner about to be paid twice, decided on a fact
1509 // that is easy to check and impossible to recover afterwards.
1510 LogPrint("payout", "[payout] wallet tx %s carries the payout"
1511 " comment for height %d and an output of exactly %s to"
1512 " %s, but it is NOT in the main chain (depth %d) --"
1513 " refusing to read it as payment; this recipient will"
1514 " be paid\n",
1515 hash.GetHex().c_str(), blockHeight,
1516 FormatMoney(amount).c_str(), address.c_str(), depth);
590 continue; 1517 continue;
591 txidRet = wtx.GetHash().GetHex(); 1518 }
592 return true; 1519 txidRet = hash.GetHex();
1520 return PAYOUT_PROBE_LEGACY;
593 } 1521 }
594 } 1522 }
595 return false; 1523 return PAYOUT_PROBE_NONE;
596} 1524}
597 1525
1526// txid -> "<round id>|<address>", for payments this run has already credited
1527// to a debt on legacy evidence. Stops one pre-upgrade transaction being read
1528// as proof that two different obligations were met. The ledger's paidTxids
1529// covers the same ground durably; this covers the window before the recovered
1530// txid reaches the ledger, and the case where the ledger entry has already
1531// been retired.
1532static std::map<std::string, std::string> gLegacyClaimedTxids;
1533static std::mutex gClaimedTxidMutex;
1534
1535// Every txid already spoken for by some payment OTHER than this one. Collected
1536// before cs_mapWallet is taken: the ledger mutex must not be acquired under a
1537// wallet lock.
1538static void CollectForeignPayoutTxids(int blockHeight, const uint256& blockHash,
1539 const std::string& address,
1540 const std::string& claimKey,
1541 std::set<std::string>& out)
1542{
1543 {
1544 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1545 for (const PendingPayout& pp : gPendingPayouts) {
1546 bool sameRound = SamePayoutRound(pp.blockHeight, pp.blockHash,
1547 blockHeight, blockHash);
1548 for (const auto& kv : pp.paidTxids)
1549 if (!(sameRound && kv.first == address))
1550 out.insert(kv.second);
1551 }
1552 }
1553 {
1554 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
1555 for (const auto& kv : gLegacyClaimedTxids)
1556 if (kv.second != claimKey)
1557 out.insert(kv.first);
1558 }
1559}
1560
1561static void ClaimLegacyPayoutTxid(const std::string& txid, const std::string& claimKey)
1562{
1563 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
1564 gLegacyClaimedTxids[txid] = claimKey;
1565}
1566
1567// blockHash stays a uint256 and not upstream's display string: it is the key
1568// this fork's payout ledger is built on, and a key that has been through
1569// ToString() is a key that can be compared wrong. It is converted to text only
1570// where the round proof needs text. The other three arguments are upstream's,
1571// carried for the round-proof data.
598static void QueuePayouts(int blockHeight, 1572static void QueuePayouts(int blockHeight,
599 const std::string& blockHash, 1573 const uint256& blockHash,
600 const std::string& coinbaseTxid, 1574 const std::string& coinbaseTxid,
601 int64 coinbaseValue, 1575 int64 coinbaseValue,
602 const std::string& foundBy, 1576 const std::string& foundBy,
@@ -604,11 +1578,11 @@ static void QueuePayouts(int blockHeight,
604 uint64 total) 1578 uint64 total)
605{ 1579{
606 PoolRoundProof round; 1580 PoolRoundProof round;
607 round.id = PayoutIdForBlock(blockHeight); 1581 round.id = PayoutIdForBlock(blockHeight, blockHash);
608 round.blockHeight = blockHeight; 1582 round.blockHeight = blockHeight;
609 round.matureAtHeight = blockHeight + COINBASE_MATURITY; 1583 round.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH;
610 round.createdAt = GetTime(); 1584 round.createdAt = GetTime();
611 round.blockHash = blockHash; 1585 round.blockHash = blockHash.ToString();
612 round.coinbaseTxid = coinbaseTxid; 1586 round.coinbaseTxid = coinbaseTxid;
613 round.coinbaseValue = coinbaseValue; 1587 round.coinbaseValue = coinbaseValue;
614 round.foundBy = foundBy; 1588 round.foundBy = foundBy;
@@ -639,9 +1613,21 @@ static void QueuePayouts(int blockHeight,
639 round.operatorReward = operatorCut; 1613 round.operatorReward = operatorCut;
640 1614
641 PendingPayout pp; 1615 PendingPayout pp;
642 pp.id = round.id;
643 pp.blockHeight = blockHeight; 1616 pp.blockHeight = blockHeight;
644 pp.matureAtHeight = blockHeight + COINBASE_MATURITY; 1617 pp.blockHash = blockHash;
1618 // COINBASE_SPENDABLE_DEPTH, not COINBASE_MATURITY. Paying at depth 100
1619 // means the block's own coinbase is still 20 blocks short of spendable, so
1620 // the payment has to come out of some *other* mature coin -- the pool runs
1621 // a permanent ~19-block float and every payout fails until unrelated coin
1622 // matures. The earlier "96% of 934 payments went through at depth 100"
1623 // measurement that justified the shorter wait was taken while this node was
1624 // alone on a private fork paying itself, so it says nothing about a pool
1625 // competing on the real chain. Waiting the extra 20 blocks makes each
1626 // payout funded by construction, and puts the maturity gate past the
1627 // reorg depth that orphaned 280 blocks of obligations. Upstream uses
1628 // COINBASE_MATURITY here; this is a considered disagreement, not an
1629 // oversight -- see the merge commit message.
1630 pp.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH;
645 1631
646 int skipped = 0; 1632 int skipped = 0;
647 int64 burned = 0; // amount owed to miners with bad/empty addresses 1633 int64 burned = 0; // amount owed to miners with bad/empty addresses
@@ -668,9 +1654,13 @@ static void QueuePayouts(int blockHeight,
668 round.forfeitedBadAddress = burned; 1654 round.forfeitedBadAddress = burned;
669 1655
670 { 1656 {
671 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); 1657 VersionedBlob snap;
672 gPendingPayouts.push_back(pp); 1658 {
673 SavePendingPayoutsLocked(); 1659 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1660 gPendingPayouts.push_back(pp);
1661 snap = SerializePendingPayoutsLocked();
1662 }
1663 PublishPendingPayouts(snap);
674 } 1664 }
675 RecordPoolRound(round); 1665 RecordPoolRound(round);
676 1666
@@ -689,37 +1679,338 @@ static void QueuePayouts(int blockHeight,
689 : ""); 1679 : "");
690} 1680}
691 1681
692// Called from event loop: copy matured entries to the exec queue (fast). 1682// Called from the event loop: explode matured ledger entries into per-recipient
693// The ledger entry stays on disk until every recipient has a txid recorded. 1683// in-flight records (fast; no wallet or disk work beyond one small write).
694// On restart, FindRecordedPoolPayout() checks wallet metadata before sending, 1684//
695// so a crash after SendMoney but before the ledger update does not double-pay. 1685// The ledger entry SURVIVES here. That is upstream's model and it is the
1686// crash-safe one: this fork used to erase the entry as soon as its payments
1687// were handed off, so a death in that window left the debt described by
1688// nothing at all -- a container restart on 2026-07-30 silently dropped
1689// ~1,156 BTF exactly that way. An entry is removed only once every recipient
1690// in it has a txid recorded (PayoutFullyPaid), which the payout thread does.
1691//
1692// gPaying is kept on top of that as the in-flight working set: it carries the
1693// per-recipient retry/deferral state that a per-round `nextAttempt` cannot
1694// express, and it is what the payout thread iterates. It is a cache of work
1695// derived from the ledger, never an independent record of debt -- if it were
1696// lost entirely the ledger would rebuild it on the next flush.
696static void FlushMaturePayouts() 1697static void FlushMaturePayouts()
697{ 1698{
1699 // Which funding blocks are still on the main chain? Asked with no payout
1700 // lock held: PayoutBlockStillOnChain takes cs_main, and cs_main is held by
1701 // ProcessBlock and SendMoney for long stretches. Holding the ledger mutex
1702 // across it would let the event loop park the payout thread behind a chain
1703 // lock it has no interest in. (It is not a deadlock either way -- nothing
1704 // takes the ledger mutex while holding cs_main -- but it is exactly the
1705 // kind of stall this merge is removing from the ledger path.)
1706 std::vector<uint256> orphaned;
1707 {
1708 std::vector<uint256> hashes;
1709 {
1710 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
1711 for (const PendingPayout& pp : gPendingPayouts)
1712 hashes.push_back(pp.blockHash);
1713 }
1714 for (const uint256& h : hashes)
1715 if (!PayoutBlockStillOnChain(h))
1716 orphaned.push_back(h);
1717 }
1718
698 std::vector<PendingPayout> ready; 1719 std::vector<PendingPayout> ready;
1720 VersionedBlob ledgerSnap;
699 { 1721 {
700 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex); 1722 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
701 std::lock_guard<std::mutex> queue(gPayoutExecMutex); 1723
702 int64 now = GetTime(); 1724 // Void obligations whose funding block lost a reorg. The coin that was
1725 // meant to pay them does not exist, so paying anyway spends real-chain
1726 // coin the pool owes to somebody else.
1727 bool voided = false;
1728 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) {
1729 if (std::find(orphaned.begin(), orphaned.end(), it->blockHash) == orphaned.end()) {
1730 ++it;
1731 continue;
1732 }
1733 int64 total = 0;
1734 for (auto& kv : it->amounts) total += kv.second;
1735 LogPrint("payout", "[payout] block %d (%s) was orphaned -- voiding %zu"
1736 " payment(s) totalling %s\n",
1737 it->blockHeight, it->blockHash.ToString().substr(0, 14).c_str(),
1738 it->amounts.size(), FormatMoney(total).c_str());
1739 it = gPendingPayouts.erase(it);
1740 voided = true;
1741 }
1742 if (voided) ledgerSnap = SerializePendingPayoutsLocked();
1743
703 for (const PendingPayout& pp : gPendingPayouts) { 1744 for (const PendingPayout& pp : gPendingPayouts) {
704 if (nBestHeight < pp.matureAtHeight) 1745 if (nBestHeight < pp.matureAtHeight)
705 continue; 1746 continue;
706 if (pp.nextAttempt > now)
707 continue;
708 if (PayoutFullyPaid(pp)) 1747 if (PayoutFullyPaid(pp))
709 continue; 1748 continue;
710 if (gQueuedPayouts.count(pp.id)) 1749 // An entry with neither a hash nor a height cannot be told apart
1750 // from another in the same state, so a txid recorded against it
1751 // could settle the wrong debt. Paying it is the unsafe move;
1752 // leaving it standing and saying so is not.
1753 if (!PayoutRoundIdentifiable(pp.blockHeight, pp.blockHash)) {
1754 LogPrint("payout", "[payout] WARNING: ledger entry maturing at"
1755 " height %d names neither a block hash nor a height --"
1756 " refusing to pay %zu recipient(s) that cannot be told"
1757 " apart from another round's\n",
1758 pp.matureAtHeight, pp.amounts.size());
711 continue; 1759 continue;
712 LogPrint("payout", "[payout] block height %d matured -- queuing payout %s (%zu recipients)\n", 1760 }
713 pp.matureAtHeight, pp.id.c_str(), pp.amounts.size());
714 ready.push_back(pp); 1761 ready.push_back(pp);
715 gQueuedPayouts.insert(pp.id);
716 } 1762 }
717 for (auto& pp : ready)
718 gPayoutExecQueue.push_back(pp);
719 } 1763 }
1764 // Outside the ledger mutex: publishing is I/O and the event loop is the
1765 // one thread that must never wait on the payout thread's disk.
1766 PublishPendingPayouts(ledgerSnap);
1767
1768 if (ready.empty())
1769 return;
1770
1771 VersionedBlob payingSnap;
1772 {
1773 std::lock_guard<std::mutex> queue(gPayoutExecMutex);
1774 size_t added = 0;
1775 for (const PendingPayout& pp : ready) {
1776 const std::string id = PayoutIdForBlock(pp.blockHeight, pp.blockHash);
1777 if (gQueuedPayouts.count(id))
1778 continue; // already exploded into gPaying and still in flight
1779 size_t n = 0;
1780 for (const auto& kv : pp.amounts) {
1781 // Upstream's per-recipient dedupe: a recipient already carrying
1782 // a txid is done, even though the entry as a whole is not.
1783 if (pp.paidTxids.count(kv.first))
1784 continue;
1785 // The invariant, checked against the thing it is an invariant
1786 // of: at most one in-flight record per (round, address). The
1787 // round-level set above is an optimisation that can be wrong --
1788 // it is memory only, it is rebuilt from gPaying at startup, and
1789 // a stale `ready` snapshot can race a round finishing --
1790 // whereas gPaying is the list being protected. A second record
1791 // here is a second standing order to send the same money.
1792 if (PaymentInFlightLocked(pp.blockHeight, pp.blockHash, kv.first))
1793 continue;
1794 PayingRecord r;
1795 r.id = ++gPayingNextId;
1796 r.blockHeight = pp.blockHeight;
1797 r.blockHash = pp.blockHash;
1798 r.matureAtHeight = pp.matureAtHeight;
1799 r.address = kv.first;
1800 r.amount = kv.second;
1801 gPaying.push_back(r);
1802 n++;
1803 }
1804 gQueuedPayouts.insert(id);
1805 added += n;
1806 LogPrint("payout", "[payout] block height %d matured -- queuing"
1807 " payout %s (%zu of %zu recipients still owed)\n",
1808 pp.matureAtHeight, id.c_str(), n, pp.amounts.size());
1809 }
1810 if (added) payingSnap = SerializePayingLocked();
1811 }
1812 PublishPaying(payingSnap);
720} 1813}
721 1814
722// Payout execution thread: drains gPayoutExecQueue, calls SendMoney. 1815// Settle a payment: it is either done or abandoned, either way it stops being
1816// in flight. Note this only retires the working record -- whether the miner is
1817// still OWED the money is the ledger's business, and the ledger is updated
1818// separately by RecordPayoutTxid().
1819//
1820// Erases by record id, not by comparing fields. Value-matching was only an
1821// identity for as long as no two records could be equal, and the restart bug
1822// above produced records that were equal in every field: the first match then
1823// absorbed every erase and every state update, and the second was never
1824// touched again. The id names the slot the caller took its copy from, so the
1825// update lands where it belongs whatever else is in the list.
1826static void ForgetPayment(const PayingRecord& r)
1827{
1828 VersionedBlob snap;
1829 {
1830 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
1831 for (auto it = gPaying.begin(); it != gPaying.end(); ++it)
1832 if (it->id == r.id) { gPaying.erase(it); break; }
1833
1834 // When the last record for a round is gone the round is no longer in
1835 // flight, so a later flush may explode it again if the ledger still
1836 // shows recipients owed. That is the intended behaviour and it is now
1837 // the only way a stalled payment ever leaves this list: a payment that
1838 // keeps failing is retried forever rather than given up on, so its
1839 // record stays here, and the flush that would re-explode it is stopped
1840 // by PaymentInFlightLocked.
1841 bool stillInFlight = false;
1842 for (const auto& e : gPaying)
1843 if (SamePayoutRound(e.blockHeight, e.blockHash,
1844 r.blockHeight, r.blockHash)) {
1845 stillInFlight = true;
1846 break;
1847 }
1848 if (!stillInFlight)
1849 gQueuedPayouts.erase(PayoutIdForBlock(r.blockHeight, r.blockHash));
1850
1851 snap = SerializePayingLocked();
1852 }
1853 PublishPaying(snap);
1854}
1855
1856// Persist an updated attempt count / backoff for a payment still owed.
1857static void ReschedulePayment(const PayingRecord& r)
1858{
1859 VersionedBlob snap;
1860 {
1861 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
1862 for (auto& e : gPaying)
1863 if (e.id == r.id) {
1864 e.retries = r.retries;
1865 e.deferrals = r.deferrals;
1866 e.retryAfter = r.retryAfter;
1867 // Carry the cleared probe flag back, or every retry re-scans
1868 // the whole wallet -- the cost this merge is specifically
1869 // removing.
1870 e.fProbeWallet = r.fProbeWallet;
1871 break;
1872 }
1873 snap = SerializePayingLocked();
1874 }
1875 PublishPaying(snap);
1876}
1877
1878// Record that a recipient has been paid, in the ledger, which is the only
1879// durable statement that the debt is settled.
1880//
1881// The file write happens with gPayoutLedgerMutex RELEASED. Upstream writes the
1882// file inside the per-recipient loop with the mutex held, so every recipient
1883// costs the event loop a full serialise+fsync it has to wait for in
1884// FlushMaturePayouts. Ordering against the event loop's own writes is the
1885// snapshot version's job, not the mutex's -- see PublishVersioned.
1886static void RecordPayoutTxid(const PayingRecord& r, const std::string& txid)
1887{
1888 VersionedBlob blob;
1889 bool complete = false;
1890 {
1891 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1892 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ++it) {
1893 if (!SamePayoutRound(it->blockHeight, it->blockHash,
1894 r.blockHeight, r.blockHash))
1895 continue;
1896 it->paidTxids[r.address] = txid;
1897 if (PayoutFullyPaid(*it)) {
1898 LogPrint("payout", "[payout] payout for block %d complete --"
1899 " removing from ledger\n", it->blockHeight);
1900 gPendingPayouts.erase(it);
1901 complete = true;
1902 }
1903 break;
1904 }
1905 blob = SerializePendingPayoutsLocked();
1906 }
1907 PublishPendingPayouts(blob);
1908 // The round proof records a txid per address, so this is per recipient and
1909 // not only on completion.
1910 MarkPoolRoundPaid(PayoutIdForBlock(r.blockHeight, r.blockHash), r.address, txid);
1911 if (complete)
1912 LogPrint("payout", "[payout] block %d fully settled\n", r.blockHeight);
1913}
1914
1915// Write a recipient off.
1916//
1917// Only ever called for a state that is DECIDABLE -- currently one: the ledger
1918// names an address that is not a valid address, so no transaction paying it
1919// can be constructed by this or any other build. It is not called, and must
1920// not be, because a payment has failed some number of times. "The wallet said
1921// no N times" is not a decidable state, it is a description of how long we
1922// have been asking; a debt written off on the strength of it is a debt the
1923// operator keeps, and the arithmetic that decided WHY the wallet said no was
1924// wrong for a year (see PayoutThreadFn).
1925//
1926// The amount is removed from the entry rather than marked paid, so the money
1927// is not reported as sent. Once nothing is left owed the entry retires. The
1928// round proof is told separately, because dropping the obligation from the
1929// ledger while pool_rounds.json goes on publishing the address as owed with
1930// "txid": null leaves the pool's two records of the same round permanently
1931// disagreeing.
1932static void ForfeitPayoutRecipient(const PayingRecord& r, const std::string& reason)
1933{
1934 LogPrint("payout", "[payout] writing off %s owed to %s for block %d: %s\n",
1935 FormatMoney(r.amount).c_str(), r.address.c_str(), r.blockHeight,
1936 reason.c_str());
1937 VersionedBlob blob;
1938 {
1939 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1940 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ++it) {
1941 if (!SamePayoutRound(it->blockHeight, it->blockHash,
1942 r.blockHeight, r.blockHash))
1943 continue;
1944 it->amounts.erase(r.address);
1945 // PayoutFullyPaid() returns true for an empty amounts map -- there
1946 // is no recipient in it without a txid -- so the
1947 // `amounts.empty()` disjunct that used to be here decided nothing.
1948 if (PayoutFullyPaid(*it))
1949 gPendingPayouts.erase(it);
1950 break;
1951 }
1952 blob = SerializePendingPayoutsLocked();
1953 }
1954 PublishPendingPayouts(blob);
1955 MarkPoolRoundForfeited(PayoutIdForBlock(r.blockHeight, r.blockHash),
1956 r.address, reason);
1957}
1958
1959// What a failed send means for the payment's retry budget.
1960//
1961// A deferral is "the wallet's condition is temporarily wrong": wait, and do
1962// not hold it against this payment. A fault is "the wallet is funded and
1963// unlocked and still would not do it", which is worth logging loudly and
1964// retrying with backoff.
1965//
1966// This used to be decided by re-reading GetBalance() after the failure and
1967// comparing it against `amount + nTransactionFee + CENT`. Every term of that
1968// was wrong:
1969//
1970// * nTransactionFee is 0 in production. It is only ever assigned from the
1971// GUI preferences dialog (gui.cpp), which a headless pool node does not
1972// have, so the whole reserve was the CENT.
1973//
1974// * The fee the reserve is for is GetMinFee(fDiscount=true), a STEP
1975// function of the serialised size: 0 below 10,000 bytes, and
1976// (1 + bytes/1000) * CENT at or above it -- so 11 * CENT at the step.
1977// There is no required-fee value between 1 and 11 CENT. A one-CENT
1978// reserve is therefore never the right number: below the step nothing
1979// needs reserving, and at the step it is short by a factor of eleven.
1980//
1981// * A pool wallet close to broke selects many small inputs BECAUSE it is
1982// close to broke, and ~56 signed P2PKH inputs is 10 KB. The trigger and
1983// the failure are positively correlated: the guess is wrong precisely
1984// when the pool is nearly out of money.
1985//
1986// Concretely, with a balance of amount + 0.05: the pre-send gate passes,
1987// CreateTransaction builds a >10 KB transaction, the fee becomes 0.11,
1988// SelectCoins(amount + 0.11) fails, SendMoney returns false -- and the old
1989// arithmetic said (amount + 0.05) < (amount + 0.01) is false, so "funded",
1990// so a fault. The condition is stable, so it recurred every 60 s, and 1440 of
1991// them erased the debt with the coin still in the operator's wallet.
1992//
1993// So stop guessing. The wallet is asked to say what it did, and this only
1994// classifies the answer. WALLET_SEND_INSUFFICIENT_FUNDS is coin selection
1995// failing to cover amount plus whatever fee that size of transaction
1996// required, whatever that turned out to be -- no reserve to get wrong.
1997enum PayoutFailureClass {
1998 PAYOUT_FAILURE_DEFER = 0, // conditions are wrong; wait for them to change
1999 PAYOUT_FAILURE_FAULT // funded and unlocked, and it still would not go
2000};
2001
2002static PayoutFailureClass ClassifyPayoutSendFailure(WalletSendResult result,
2003 bool fWalletLocked)
2004{
2005 // A locked wallet cannot sign. Nothing about this payment is wrong.
2006 if (fWalletLocked)
2007 return PAYOUT_FAILURE_DEFER;
2008 if (result == WALLET_SEND_INSUFFICIENT_FUNDS)
2009 return PAYOUT_FAILURE_DEFER;
2010 return PAYOUT_FAILURE_FAULT;
2011}
2012
2013// Payout execution thread: works through gPaying, calls SendMoney.
723// Runs independently of the event loop so slow wallet operations 2014// Runs independently of the event loop so slow wallet operations
724// (SendMoney takes cs_main) never stall miner I/O. 2015// (SendMoney takes cs_main) never stall miner I/O.
725static void PayoutThreadFn(void*) 2016static void PayoutThreadFn(void*)
@@ -728,103 +2019,259 @@ static void PayoutThreadFn(void*)
728 LogPrint("payout", "[payout] payout thread started\n"); 2019 LogPrint("payout", "[payout] payout thread started\n");
729 2020
730 while (!fShutdown && gPoolRunning) { 2021 while (!fShutdown && gPoolRunning) {
731 std::vector<PendingPayout> batch; 2022 std::vector<PayingRecord> batch;
732 { 2023 {
733 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 2024 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
734 batch.swap(gPayoutExecQueue); 2025 int64 now = GetTime();
2026 for (const auto& r : gPaying)
2027 if (r.retryAfter <= now)
2028 batch.push_back(r);
735 } 2029 }
736 2030
737 for (auto& ppQueued : batch) { 2031 for (PayingRecord r : batch) {
738 LogPrint("payout", "[payout] executing payments for block matured at %d" 2032 const std::string id = PayoutIdForBlock(r.blockHeight, r.blockHash);
739 " (%zu miners)\n", ppQueued.matureAtHeight, ppQueued.amounts.size()); 2033
740 bool fAbortBatch = false; 2034 // Re-check here and not only at maturity: a payment can sit in this
741 for (auto& kv : ppQueued.amounts) { 2035 // list for hours across retries, and the reorg that kills its
2036 // funding block can land at any point in that window. Upstream has
2037 // no equivalent -- it decides once, when the entry matures.
2038 if (!PayoutBlockStillOnChain(r.blockHash)) {
2039 LogPrint("payout", "[payout] block %d (%s) orphaned while queued"
2040 " -- dropping %s to %s\n",
2041 r.blockHeight, r.blockHash.ToString().substr(0, 14).c_str(),
2042 FormatMoney(r.amount).c_str(), r.address.c_str());
2043 ForgetPayment(r);
2044 continue;
2045 }
2046
2047 // Upstream's per-recipient dedupe, against the ledger rather than
2048 // against the batch: if a txid is already recorded for this
2049 // address, the money went out and nothing more is owed.
2050 {
742 bool alreadyPaid = false; 2051 bool alreadyPaid = false;
743 { 2052 {
744 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); 2053 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
745 for (const PendingPayout& pp : gPendingPayouts) 2054 for (const PendingPayout& pp : gPendingPayouts)
746 if (pp.id == ppQueued.id && pp.paidTxids.count(kv.first)) 2055 if (SamePayoutRound(pp.blockHeight, pp.blockHash,
2056 r.blockHeight, r.blockHash)
2057 && pp.paidTxids.count(r.address)) {
747 alreadyPaid = true; 2058 alreadyPaid = true;
2059 break;
2060 }
748 } 2061 }
749 if (alreadyPaid) 2062 if (alreadyPaid) {
2063 ForgetPayment(r);
750 continue; 2064 continue;
751
752 uint160 h160;
753 if (!AddressToHash160(kv.first, h160)) {
754 // Should not happen -- bad addresses are filtered in QueuePayouts
755 LogPrint("payout", "[payout] SKIPPED bad address '%s'\n",
756 kv.first.c_str());
757 continue;
758 }
759 CScript sc;
760 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
761 CWalletTx wtx;
762 wtx.mapValue["comment"] = strprintf("Pool payout height %d", ppQueued.matureAtHeight);
763 wtx.mapValue["pool_payout_id"] = ppQueued.id;
764 wtx.mapValue["pool_payout_address"] = kv.first;
765 wtx.mapValue["pool_payout_amount"] = strprintf("%lld", kv.second);
766
767 std::string txid;
768 if (FindRecordedPoolPayout(ppQueued.id, kv.first, kv.second, txid))
769 {
770 LogPrint("payout", "[payout] recovered recorded payout %s to %s"
771 " from wallet tx %s\n",
772 FormatMoney(kv.second).c_str(), kv.first.c_str(), txid.c_str());
773 MarkPoolRoundPaid(ppQueued.id, kv.first, txid);
774 }
775 else if (SendMoney(sc, kv.second, wtx))
776 {
777 txid = wtx.GetHash().GetHex();
778 LogPrint("payout", "[payout] paid %s to %s\n",
779 FormatMoney(kv.second).c_str(), kv.first.c_str());
780 MarkPoolRoundPaid(ppQueued.id, kv.first, txid);
781 }
782 else
783 {
784 LogPrint("payout", "[payout] FAILED to pay %s to %s"
785 " (insufficient funds or wallet locked?)\n",
786 FormatMoney(kv.second).c_str(), kv.first.c_str());
787 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
788 for (PendingPayout& pp : gPendingPayouts)
789 if (pp.id == ppQueued.id)
790 pp.nextAttempt = GetTime() + 60;
791 SavePendingPayoutsLocked();
792 fAbortBatch = true;
793 break;
794 }
795
796 {
797 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
798 for (PendingPayout& pp : gPendingPayouts) {
799 if (pp.id != ppQueued.id)
800 continue;
801 pp.paidTxids[kv.first] = txid;
802 pp.nextAttempt = 0;
803 SavePendingPayoutsLocked();
804 break;
805 }
806 } 2065 }
807 } 2066 }
808 2067
809 { 2068 uint160 h160;
810 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex); 2069 if (!AddressToHash160(r.address, h160)) {
811 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { 2070 // Should not happen -- bad addresses are filtered in
812 if (it->id == ppQueued.id && PayoutFullyPaid(*it)) { 2071 // QueuePayouts. If one is here anyway it is decidable and
813 LogPrint("payout", "[payout] payout %s complete -- removing from ledger\n", 2072 // permanent: there is no transaction that pays a string which
814 it->id.c_str()); 2073 // is not an address, so retrying is not waiting for anything.
815 it = gPendingPayouts.erase(it); 2074 //
816 SavePendingPayoutsLocked(); 2075 // This used to only ForgetPayment(), which drops the in-flight
2076 // record and leaves the ledger entry claiming the debt and
2077 // pool_rounds.json advertising the recipient as unpaid
2078 // forever, with the next flush re-exploding it. Write it off
2079 // properly and say why.
2080 const std::string reason =
2081 strprintf("'%s' is not a valid address, so no transaction"
2082 " can pay it", r.address.c_str());
2083 LogPrint("payout", "[payout] SKIPPED bad address '%s' --"
2084 " %s owed for block %d cannot be paid\n",
2085 r.address.c_str(), FormatMoney(r.amount).c_str(),
2086 r.blockHeight);
2087 ForfeitPayoutRecipient(r, reason);
2088 ForgetPayment(r);
2089 continue;
2090 }
2091 CScript sc;
2092 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
2093
2094 // Did a previous process send this and die before recording it?
2095 // Only asked once per record -- see PayingRecord::fProbeWallet.
2096 if (r.fProbeWallet) {
2097 std::string recovered;
2098 const std::string claimKey = id + "|" + r.address;
2099 std::set<std::string> excluded;
2100 CollectForeignPayoutTxids(r.blockHeight, r.blockHash, r.address,
2101 claimKey, excluded);
2102 r.fProbeWallet = false;
2103 PayoutProbeResult probe = FindRecordedPoolPayout(
2104 id, r.blockHeight, r.address, r.amount, excluded, recovered);
2105 if (probe != PAYOUT_PROBE_NONE) {
2106 if (probe == PAYOUT_PROBE_LEGACY) {
2107 // Loud on purpose: this is the upgrade path, where the
2108 // only evidence a payment went out is a comment written
2109 // by a binary that is no longer running.
2110 LogPrint("payout", "[payout] %s to %s was already paid by"
2111 " a pre-upgrade build (wallet tx %s carries the"
2112 " payout comment for height %d and an output of"
2113 " exactly this amount, and no structured payout"
2114 " metadata) -- not resending\n",
2115 FormatMoney(r.amount).c_str(), r.address.c_str(),
2116 recovered.c_str(), r.blockHeight);
2117 ClaimLegacyPayoutTxid(recovered, claimKey);
817 } else { 2118 } else {
818 ++it; 2119 LogPrint("payout", "[payout] recovered recorded payout %s to %s"
2120 " from wallet tx %s -- not resending\n",
2121 FormatMoney(r.amount).c_str(), r.address.c_str(),
2122 recovered.c_str());
819 } 2123 }
2124 RecordPayoutTxid(r, recovered);
2125 ForgetPayment(r);
2126 continue;
820 } 2127 }
821 } 2128 }
822 { 2129
823 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 2130 // Waiting for coin to mature is not a failure, so it must not
824 gQueuedPayouts.erase(ppQueued.id); 2131 // consume the retry budget. Burning a retry every 60s means a
2132 // payment the pool fully intends to honour can hit GIVING UP
2133 // 24h later purely because the subsidy funding it had not
2134 // matured yet -- the miner is then silently never paid.
2135 //
2136 // The comparison includes the fee. It used to be against the
2137 // amount alone, so a balance anywhere between amount and
2138 // amount+fee sailed past this gate, failed in SendMoney, and was
2139 // counted as a send FAILURE rather than a wait -- 1440 of those is
2140 // 24 hours, and the recipient was erased with the money left in
2141 // the operator's wallet and nothing on disk saying it was owed.
2142 //
2143 // nTransactionFee is the part of the fee that can be known before
2144 // the transaction exists, and in production it is 0 -- it is only
2145 // ever assigned from the GUI preferences dialog, which a pool node
2146 // does not have -- so including it here changes nothing on this
2147 // pool and is kept only because a GUI operator can set it. The
2148 // rest is GetMinFee(), a function of the serialised size and
2149 // therefore of how many inputs SelectCoins ends up picking, which
2150 // cannot be predicted from here -- so this gate deliberately does
2151 // NOT try to reserve it. Refusing to attempt a payment the wallet
2152 // could actually have made would be its own way of not paying a
2153 // miner. That residual is handled after the fact, by asking the
2154 // wallet why it refused -- see ClassifyPayoutSendFailure.
2155 int64 nSpendable = GetBalance();
2156 if (nSpendable < r.amount + nTransactionFee) {
2157 r.deferrals++;
2158 r.retryAfter = GetTime() + 60;
2159 // Hourly, not every minute: a dry pool would otherwise
2160 // write one line per payment per minute forever.
2161 if (r.deferrals == 1 || r.deferrals % 60 == 0)
2162 LogPrint("payout", "[payout] deferring %s to %s -- only %s"
2163 " spendable, need %s with fee (waiting %d min so far)\n",
2164 FormatMoney(r.amount).c_str(), r.address.c_str(),
2165 FormatMoney(nSpendable).c_str(),
2166 FormatMoney(r.amount + nTransactionFee).c_str(),
2167 r.deferrals);
2168 ReschedulePayment(r);
2169 continue;
825 } 2170 }
826 if (fAbortBatch) 2171
2172 // Likewise a locked wallet: it cannot sign, so every attempt fails
2173 // for a reason that has nothing to do with this payment being
2174 // payable. Left to the retry budget, an operator who locks the
2175 // wallet overnight comes back to erased debts.
2176 if (IsWalletLocked()) {
2177 r.deferrals++;
2178 r.retryAfter = GetTime() + 60;
2179 if (r.deferrals == 1 || r.deferrals % 60 == 0)
2180 LogPrint("payout", "[payout] deferring %s to %s -- wallet is"
2181 " locked (waiting %d min so far)\n",
2182 FormatMoney(r.amount).c_str(), r.address.c_str(),
2183 r.deferrals);
2184 ReschedulePayment(r);
827 continue; 2185 continue;
2186 }
2187
2188 CWalletTx wtx;
2189 wtx.mapValue["comment"] = PayoutComment(r.blockHeight);
2190 // Upstream's structured metadata. This is what makes the wallet
2191 // probe above possible at all -- the old comment-only form could
2192 // not distinguish two rounds paying the same address the same
2193 // amount. Keyed by the hash-derived id, so a reorg cannot make one
2194 // round's transaction answer for another's.
2195 wtx.mapValue["pool_payout_id"] = id;
2196 wtx.mapValue["pool_payout_address"] = r.address;
2197 wtx.mapValue["pool_payout_amount"] = strprintf("%lld", r.amount);
2198
2199 SendMoneyOutcome outcome;
2200 if (SendMoney(sc, r.amount, wtx, outcome)) {
2201 // The wallet holds the transaction before this returns, so a
2202 // crash between here and the ledger write is recoverable: the
2203 // probe above finds it by id+address+amount and does not
2204 // resend. Ledger first, in-flight record second -- the debt is
2205 // described by at least one of them at every instant.
2206 RecordPayoutTxid(r, wtx.GetHash().GetHex());
2207 ForgetPayment(r);
2208 LogPrint("payout", "[payout] paid %s to %s (tx %s)\n",
2209 FormatMoney(r.amount).c_str(), r.address.c_str(),
2210 wtx.GetHash().ToString().substr(0, 12).c_str());
2211 } else {
2212 // Why did it fail? The answer used to decide whether the
2213 // payment eventually got erased, and it was decided by
2214 // guessing from a balance snapshot -- see
2215 // ClassifyPayoutSendFailure for the arithmetic and what it
2216 // cost. The wallet now says what it did, and nothing here
2217 // erases anything.
2218 //
2219 // The lock state is re-read rather than trusted from before
2220 // the attempt: SendMoney takes cs_main and can block for a
2221 // long time, so it may have moved underneath us. The funding
2222 // answer does not need re-reading -- it is a statement about
2223 // the attempt that was actually made.
2224 bool fLocked = IsWalletLocked();
2225 int64 nAfter = GetBalance();
2226
2227 if (ClassifyPayoutSendFailure(outcome.result, fLocked)
2228 == PAYOUT_FAILURE_DEFER) {
2229 // Conditions are temporarily wrong. Wait.
2230 r.deferrals++;
2231 r.retryAfter = GetTime() + 60;
2232 if (r.deferrals == 1 || r.deferrals % 60 == 0)
2233 LogPrint("payout", "[payout] deferring %s to %s -- send"
2234 " failed with %s (%s spendable; the wallet"
2235 " needed a fee of %s for a transaction this"
2236 " size); waiting %d min so far\n",
2237 FormatMoney(r.amount).c_str(), r.address.c_str(),
2238 fLocked ? "the wallet locked" : "too little coin",
2239 FormatMoney(nAfter).c_str(),
2240 FormatMoney(outcome.nFeeRequired).c_str(),
2241 r.deferrals);
2242 ReschedulePayment(r);
2243 } else {
2244 // Funded, unlocked, block on the main chain, and the
2245 // wallet still would not build, sign or accept the
2246 // transaction. That is worth shouting about; it is not
2247 // grounds to keep the money.
2248 //
2249 // There is deliberately no give-up branch. It used to
2250 // forfeit the debt after 1440 of these, which is not a
2251 // decidable state -- it is a count of how long we have
2252 // been asking, and it was reached by misclassifying a
2253 // funding shortfall as a fault. A debt that is never
2254 // written off cannot be written off wrongly. The payment
2255 // stays in flight, keeps its ledger entry, keeps being
2256 // reported in pool_status.json as owed, and keeps
2257 // retrying once a minute until the operator or the wallet
2258 // makes it possible.
2259 r.retries++;
2260 r.retryAfter = GetTime() + 60;
2261 ReschedulePayment(r);
2262 // Every minute at first, then hourly: a fault that is not
2263 // going away must not turn the log into one line a minute
2264 // forever, and must not go quiet either.
2265 if (r.retries <= 5 || r.retries % 60 == 0)
2266 LogPrint("payout", "[payout] FAILED to pay %s to %s"
2267 " (%s spendable, wallet unlocked, block %d on"
2268 " the main chain) -- attempt %d, retrying in"
2269 " 60s; this debt stands until it is paid\n",
2270 FormatMoney(r.amount).c_str(), r.address.c_str(),
2271 FormatMoney(nAfter).c_str(), r.blockHeight,
2272 r.retries);
2273 }
2274 }
828 } 2275 }
829 2276
830 Sleep(500); // check for new work every 500ms 2277 Sleep(500); // check for new work every 500ms
@@ -877,7 +2324,22 @@ static std::string PoolStatusFilePath()
877 return GetAppDir() + "/pool_status.json"; 2324 return GetAppDir() + "/pool_status.json";
878} 2325}
879 2326
880static void WritePoolStatusJsonLocked() 2327// Serialise the operator status file. Caller must hold gStatsMutex (this reads
2328// gStats / gWorkerStats / gPayoutViews); gPoolRoundsMutex is taken and
2329// released inside.
2330//
2331// Split from the write for the same reason as the three ledgers, and for a
2332// worse instance of the same problem. This used to be called from RefreshStats
2333// with gPayoutLedgerMutex held at function scope, and once the ledger writes
2334// gained fsync() that call became two fsyncs -- one on the file, one on its
2335// directory -- performed by the event loop, the thread that owns every miner's
2336// socket, with the mutex the payout thread needs in its hand. Measured on the
2337// build host (ext4 on LVM on SSD): 3.78 ms per publish, every 10 seconds, and
2338// unbounded on a shared virtual disk under load.
2339//
2340// Nothing here needs the ledger mutex at all: the pending-payout rows were
2341// already copied into gPayoutViews under it, further up RefreshStats.
2342static std::string SerializePoolStatusLocked()
881{ 2343{
882 nlohmann::json j; 2344 nlohmann::json j;
883 j["schema"] = "bitflash-pool-status-1"; 2345 j["schema"] = "bitflash-pool-status-1";
@@ -933,10 +2395,19 @@ static void WritePoolStatusJsonLocked()
933 size_t start = gPoolRounds.size() > 10 ? gPoolRounds.size() - 10 : 0; 2395 size_t start = gPoolRounds.size() > 10 ? gPoolRounds.size() - 10 : 0;
934 for (size_t i = start; i < gPoolRounds.size(); i++) { 2396 for (size_t i = start; i < gPoolRounds.size(); i++) {
935 const PoolRoundProof& r = gPoolRounds[i]; 2397 const PoolRoundProof& r = gPoolRounds[i];
2398 // A forfeited recipient is not "unpaid and pending" -- the ledger
2399 // has already dropped the obligation. Counting it as unpaid is
2400 // what made the round proof and the ledger disagree for good.
936 int unpaid = 0; 2401 int unpaid = 0;
937 for (const auto& kv : r.payouts) 2402 int forfeited = 0;
938 if (r.paidTxids.count(kv.first) == 0) 2403 for (const auto& kv : r.payouts) {
2404 if (r.paidTxids.count(kv.first))
2405 continue;
2406 if (r.forfeited.count(kv.first))
2407 forfeited++;
2408 else
939 unpaid++; 2409 unpaid++;
2410 }
940 j["recentRounds"].push_back({ 2411 j["recentRounds"].push_back({
941 {"id", r.id}, 2412 {"id", r.id},
942 {"blockHeight", r.blockHeight}, 2413 {"blockHeight", r.blockHeight},
@@ -945,30 +2416,34 @@ static void WritePoolStatusJsonLocked()
945 {"totalShares", r.totalShares}, 2416 {"totalShares", r.totalShares},
946 {"payouts", (int)r.payouts.size()}, 2417 {"payouts", (int)r.payouts.size()},
947 {"unpaidPayouts", unpaid}, 2418 {"unpaidPayouts", unpaid},
2419 {"forfeitedPayouts", forfeited},
948 {"coinbaseValueBtf", FormatMoney(r.coinbaseValue)}, 2420 {"coinbaseValueBtf", FormatMoney(r.coinbaseValue)},
949 {"minerRewardBtf", FormatMoney(r.minerReward)} 2421 {"minerRewardBtf", FormatMoney(r.minerReward)}
950 }); 2422 });
951 } 2423 }
952 } 2424 }
953 2425
954 std::string path = PoolStatusFilePath(); 2426 return j.dump(2);
955 std::string tmp = path + ".tmp"; 2427}
956 FILE* f = fopen(tmp.c_str(), "w"); 2428
957 if (!f) { 2429// Publish the operator status file. Call with every payout mutex released.
958 LogPrint("pool", "[pool] WARNING: could not write status file %s\n", tmp.c_str()); 2430//
959 return; 2431// Still no version guard and no write mutex, deliberately, and the split does
960 } 2432// not change the argument: RefreshStats remains the only caller of either
961 std::string s = j.dump(2); 2433// half, in production it runs only on the event loop, and it now calls this
962 fwrite(s.c_str(), 1, s.size(), f); 2434// after leaving the scope that serialised -- so the serialise and the write
963 if (fclose(f) != 0) { 2435// stay strictly ordered on one thread and pool_status.json still has a single
964 LogPrint("pool", "[pool] WARNING: could not close status file %s\n", tmp.c_str()); 2436// writer by construction. It shares WriteFileAtomic so the durability rules
965 return; 2437// stay in one place.
966 } 2438//
967#ifdef _WIN32 2439// "In production" is doing real work in that sentence. The payout-ledger
968 remove(path.c_str()); 2440// self-test also calls RefreshStats, from one thread at a time and with the
969#endif 2441// pool not running, which is why it is still a single writer there too. A
970 if (rename(tmp.c_str(), path.c_str()) != 0) 2442// second concurrent caller anywhere would invalidate this paragraph and want
971 LogPrint("pool", "[pool] WARNING: could not replace status file %s\n", path.c_str()); 2443// the same versioned publish the ledgers use.
2444static void PublishPoolStatus(const std::string& blob)
2445{
2446 WriteFileAtomic(PoolStatusFilePath(), blob, "pool status");
972} 2447}
973 2448
974// --------------------------------------------------------------------------- 2449// ---------------------------------------------------------------------------
@@ -1044,6 +2519,45 @@ static void PushReady(btf_socket_t s)
1044 gReadyQueue.push_back(s); 2519 gReadyQueue.push_back(s);
1045} 2520}
1046 2521
2522// Loopback stratum listener: lets a miner running on the SAME machine reach
2523// this pool over plain TCP (127.0.0.1:8334), skipping the .btf rendezvous
2524// entirely. The rendezvous relay only allows one service registration per
2525// pubkey, so the pool service and the node's own peer-accept fight over the
2526// node identity at the relay -- loopback avoids that fight for local miners.
2527static void LocalAcceptFn(void*)
2528{
2529 SOCKET ls = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
2530 if (ls == INVALID_SOCKET)
2531 return;
2532 int yes = 1;
2533 setsockopt(ls, SOL_SOCKET, SO_REUSEADDR, (char*)&yes, sizeof(yes));
2534 struct sockaddr_in addr;
2535 memset(&addr, 0, sizeof(addr));
2536 addr.sin_family = AF_INET;
2537 addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
2538 addr.sin_port = htons(8334);
2539 if (bind(ls, (struct sockaddr*)&addr, sizeof(addr)) != 0 || listen(ls, 16) != 0)
2540 {
2541 LogPrint("pool", "[pool] loopback stratum bind/listen failed on 127.0.0.1:8334\n");
2542 closesocket(ls);
2543 return;
2544 }
2545 LogPrint("pool", "[pool] loopback stratum listening on 127.0.0.1:8334\n");
2546 while (!fShutdown && gPoolRunning)
2547 {
2548 SOCKET c = accept(ls, NULL, NULL);
2549 if (c == INVALID_SOCKET)
2550 {
2551 if (fShutdown || !gPoolRunning)
2552 break;
2553 continue;
2554 }
2555 LogPrint("pool", "[pool] miner arrived via loopback -- pushing to event loop\n");
2556 PushReady((btf_socket_t)c);
2557 }
2558 closesocket(ls);
2559}
2560
1047// --------------------------------------------------------------------------- 2561// ---------------------------------------------------------------------------
1048// I/O helpers -- event loop thread only 2562// I/O helpers -- event loop thread only
1049// --------------------------------------------------------------------------- 2563// ---------------------------------------------------------------------------
@@ -1480,7 +2994,11 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1480 if (accepted) { 2994 if (accepted) {
1481 // Snapshot height before RebuildJob() (which increments it) 2995 // Snapshot height before RebuildJob() (which increments it)
1482 int foundHeight = gCurrentJob.height; 2996 int foundHeight = gCurrentJob.height;
1483 QueuePayouts(foundHeight, b.GetHash().GetHex(), b.vtx[0].GetHash().GetHex(), 2997 // b is the block we submitted, untouched by ProcessBlock (which
2998 // took a copy), so its hash is what the payout must be keyed on.
2999 // Passed as a uint256, not hex: the ledger key must stay a real
3000 // hash. The remaining arguments are upstream's round-proof data.
3001 QueuePayouts(foundHeight, b.GetHash(), b.vtx[0].GetHash().GetHex(),
1484 b.vtx[0].vout[0].nValue, m->address, 3002 b.vtx[0].vout[0].nValue, m->address,
1485 roundShareCount, roundShareTotal); 3003 roundShareCount, roundShareTotal);
1486 // Reset round shares 3004 // Reset round shares
@@ -1508,6 +3026,1352 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1508 return true; 3026 return true;
1509} 3027}
1510 3028
3029// ---------------------------------------------------------------------------
3030// payout-ledger self-test
3031//
3032// Regression cover for the payout defects fixed in this series. Everything
3033// here runs against a scratch data directory and an in-memory wallet map; it
3034// never opens wallet.dat, never touches a real datadir, and never sends
3035// anything.
3036// ---------------------------------------------------------------------------
3037
3038static bool PayoutTestReadFile(const std::string& path, std::string& out)
3039{
3040 FILE* f = fopen(path.c_str(), "r");
3041 if (!f) return false;
3042 fseek(f, 0, SEEK_END);
3043 long sz = ftell(f);
3044 fseek(f, 0, SEEK_SET);
3045 out.assign((size_t)(sz > 0 ? sz : 0), '\0');
3046 if (sz > 0 && fread(&out[0], 1, (size_t)sz, f) != (size_t)sz) {
3047 fclose(f);
3048 return false;
3049 }
3050 fclose(f);
3051 return true;
3052}
3053
3054// Delete the scratch data directory and everything in it. A sweep and not a
3055// list of known names: LogPrint binds debug.log to whatever GetAppDir() is
3056// when it first fires, so the test's own log lands in here too, and a
3057// directory left behind in src/ after `make tests` is litter.
3058static void PayoutTestRemoveDir(const std::string& dir)
3059{
3060#ifdef _WIN32
3061 WIN32_FIND_DATAA findData;
3062 HANDLE hFind = FindFirstFileA((dir + "\\*").c_str(), &findData);
3063 if (hFind != INVALID_HANDLE_VALUE) {
3064 do {
3065 std::string name = findData.cFileName;
3066 if (name == "." || name == "..") continue;
3067 DeleteFileA((dir + "\\" + name).c_str());
3068 } while (FindNextFileA(hFind, &findData));
3069 FindClose(hFind);
3070 }
3071 RemoveDirectoryA(dir.c_str());
3072#else
3073 // One level of recursion, because CreateTransaction() opens a Berkeley DB
3074 // environment and BDB puts its logs in a "database" subdirectory. Without
3075 // this the scratch directory cannot be removed and the test leaves one
3076 // behind on every run, which is what d5b3503 was for.
3077 DIR* d = opendir(dir.c_str());
3078 if (d) {
3079 struct dirent* ent;
3080 while ((ent = readdir(d)) != NULL) {
3081 std::string name = ent->d_name;
3082 if (name == "." || name == "..") continue;
3083 const std::string path = dir + "/" + name;
3084 if (unlink(path.c_str()) != 0)
3085 PayoutTestRemoveDir(path);
3086 }
3087 closedir(d);
3088 }
3089 rmdir(dir.c_str());
3090#endif
3091}
3092
3093static bool PayoutTestWriteFile(const std::string& path, const std::string& data)
3094{
3095 FILE* f = fopen(path.c_str(), "w");
3096 if (!f) return false;
3097 bool ok = data.empty() || fwrite(data.c_str(), 1, data.size(), f) == data.size();
3098 return fclose(f) == 0 && ok;
3099}
3100
3101// Wipe every piece of payout state this test touches, so each section starts
3102// from the same place and a "restart" can be simulated by reloading from disk.
3103static void PayoutTestResetState()
3104{
3105 {
3106 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3107 gPendingPayouts.clear();
3108 }
3109 {
3110 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3111 gPaying.clear();
3112 gQueuedPayouts.clear();
3113 }
3114 {
3115 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
3116 gPoolRounds.clear();
3117 }
3118 {
3119 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
3120 gLegacyClaimedTxids.clear();
3121 }
3122 {
3123 // A quarantine is meant to last the life of the process; each section
3124 // here is a fresh process as far as the code under test is concerned.
3125 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
3126 gRefusedFiles.clear();
3127 }
3128}
3129
3130// A BTF address whose hash160 is entirely under the test's control, so the
3131// output script the probe compares against can be built without a wallet.
3132static std::string PayoutTestAddress(unsigned char seed)
3133{
3134 uint160 h = 0;
3135 for (int i = 0; i < 20; i++)
3136 ((unsigned char*)&h)[i] = (unsigned char)(seed + i);
3137 return Hash160ToAddress(h);
3138}
3139
3140static CScript PayoutTestScriptFor(const std::string& address)
3141{
3142 uint160 h160;
3143 CScript sc;
3144 if (AddressToHash160(address, h160))
3145 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
3146 return sc;
3147}
3148
3149// Block index entries fabricated by PayoutTestAddWalletTx, so they can be
3150// freed and unhooked when the test finishes. gPayoutTestChainTip is the end of
3151// that chain: blocks are linked onto it rather than each replacing the tip,
3152// because IsInMainChain() is "pnext is set, or I am the tip".
3153static std::vector<CBlockIndex*> gPayoutTestBlockIndexes;
3154static CBlockIndex* gPayoutTestChainTip = NULL;
3155
3156static void PayoutTestClearBlockIndex()
3157{
3158 CRITICAL_BLOCK(cs_main)
3159 {
3160 for (CBlockIndex* p : gPayoutTestBlockIndexes) {
3161 for (std::map<uint256, CBlockIndex*>::iterator it = mapBlockIndex.begin();
3162 it != mapBlockIndex.end(); ) {
3163 if (it->second == p) mapBlockIndex.erase(it++);
3164 else ++it;
3165 }
3166 delete p;
3167 }
3168 gPayoutTestBlockIndexes.clear();
3169 gPayoutTestChainTip = NULL;
3170 pindexBest = NULL;
3171 }
3172}
3173
3174// Put a transaction in the wallet map that looks exactly like one this pool
3175// sent. `nonce` only exists to keep the hashes distinct.
3176//
3177// fConfirmed decides what GetDepthInMainChain() will answer for it, which is
3178// what the legacy probe now gates on. A confirmed one is given a block index
3179// entry of its own: an empty merkle branch hashes to the transaction itself,
3180// so a block whose merkle root IS that hash accepts it at index 0, and
3181// pointing pindexBest at that block puts it in the main chain one deep. An
3182// unconfirmed one keeps hashBlock 0, which is depth 0 -- the shape a payout
3183// left behind by a reorg has.
3184static std::string PayoutTestAddWalletTx(const std::string& comment,
3185 const std::string& address,
3186 int64 amount,
3187 int nonce,
3188 const std::string& structuredId,
3189 bool fConfirmed = true)
3190{
3191 CWalletTx wtx;
3192 wtx.vin.resize(1);
3193 wtx.vin[0].prevout.SetNull();
3194 wtx.vin[0].scriptSig << nonce;
3195 wtx.vout.resize(1);
3196 wtx.vout[0].nValue = amount;
3197 wtx.vout[0].scriptPubKey = PayoutTestScriptFor(address);
3198 if (!comment.empty())
3199 wtx.mapValue["comment"] = comment;
3200 if (!structuredId.empty()) {
3201 wtx.mapValue["pool_payout_id"] = structuredId;
3202 wtx.mapValue["pool_payout_address"] = address;
3203 wtx.mapValue["pool_payout_amount"] = strprintf("%lld", amount);
3204 }
3205 uint256 hash = wtx.GetHash();
3206 if (fConfirmed) {
3207 // A block hash distinct from the transaction's own.
3208 uint256 blockHash = hash;
3209 ((unsigned char*)&blockHash)[0] ^= 0xff;
3210 CRITICAL_BLOCK(cs_main)
3211 {
3212 CBlockIndex* pindex = new CBlockIndex();
3213 pindex->hashMerkleRoot = hash;
3214 // Linked onto the end of the fabricated chain rather than simply
3215 // becoming the new tip: IsInMainChain() is "pnext is set, or I am
3216 // the tip", so a second confirmed transaction would otherwise take
3217 // the first one out of the chain again.
3218 pindex->pprev = gPayoutTestChainTip;
3219 pindex->nHeight = gPayoutTestChainTip ? gPayoutTestChainTip->nHeight + 1 : 1;
3220 if (gPayoutTestChainTip)
3221 gPayoutTestChainTip->pnext = pindex;
3222 gPayoutTestChainTip = pindex;
3223 std::map<uint256, CBlockIndex*>::iterator mi =
3224 mapBlockIndex.insert(std::make_pair(blockHash, pindex)).first;
3225 pindex->phashBlock = &mi->first;
3226 pindexBest = pindex;
3227 gPayoutTestBlockIndexes.push_back(pindex);
3228 }
3229 wtx.hashBlock = blockHash;
3230 wtx.nIndex = 0;
3231 }
3232 CRITICAL_BLOCK(cs_mapWallet)
3233 mapWallet[hash] = wtx;
3234 return hash.GetHex();
3235}
3236
3237// A real key, in the keystore, so a test wallet's outputs are IsMine() and can
3238// actually be signed -- both are needed for CreateTransaction to build the
3239// large transaction the fee schedule is being tested against. Inserted
3240// straight into the key maps rather than through AddKey(), which would also
3241// write it to a wallet.dat.
3242static std::string PayoutTestAddSpendableKey()
3243{
3244 CKey key;
3245 key.MakeNewKey();
3246 std::vector<unsigned char> vchPubKey = key.GetPubKey();
3247 CRITICAL_BLOCK(cs_mapKeys)
3248 {
3249 mapKeys[vchPubKey] = key.GetPrivKey();
3250 mapPubKeys[Hash160(vchPubKey)] = vchPubKey;
3251 }
3252 return Hash160ToAddress(Hash160(vchPubKey));
3253}
3254
3255// Give the wallet `n` spendable outputs of `each`, one transaction apiece --
3256// which is the granularity SelectCoins works at, and therefore the number of
3257// inputs the transaction spending them will carry.
3258//
3259// Deliberately not coinbases: a null prevout would make CWalletTx::GetCredit
3260// return 0 until COINBASE_SPENDABLE_DEPTH, and there is no chain here.
3261static void PayoutTestFundWallet(int n, int64 each, const std::string& address)
3262{
3263 const CScript sc = PayoutTestScriptFor(address);
3264 CRITICAL_BLOCK(cs_mapWallet)
3265 {
3266 for (int i = 0; i < n; i++) {
3267 CWalletTx wtx;
3268 wtx.vin.resize(1);
3269 wtx.vin[0].prevout.hash = (uint64)(i + 1);
3270 wtx.vin[0].prevout.n = 0;
3271 wtx.vout.resize(1);
3272 wtx.vout[0].nValue = each;
3273 wtx.vout[0].scriptPubKey = sc;
3274 mapWallet[wtx.GetHash()] = wtx;
3275 }
3276 }
3277}
3278
3279// Defined below with the event loop; the status-file publication test drives
3280// it directly.
3281static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal);
3282
3283int RunPayoutLedgerSelfTest()
3284{
3285 int nFail = 0;
3286 auto check = [&](bool ok, const char* msg) {
3287 fprintf(stdout, " %s %s\n", ok ? "ok" : "FAIL", msg);
3288 if (!ok) nFail++;
3289 };
3290
3291 fprintf(stdout, "payout-ledger self-test\n");
3292
3293 const std::string savedDataDir = strSetDataDir;
3294 const int savedBestHeight = nBestHeight;
3295
3296 // Scratch data directory in the current working directory. GetAppDir()
3297 // follows strSetDataDir, so this is what PayoutFilePath() and
3298 // PayingFilePath() resolve against for the rest of the test.
3299#ifdef _WIN32
3300 unsigned long pid = (unsigned long)GetCurrentProcessId();
3301#else
3302 unsigned long pid = (unsigned long)getpid();
3303#endif
3304 // Absolute, and the process moves into it.
3305 //
3306 // Both are needed by the CreateTransaction section: Berkeley DB resolves
3307 // its log directory relative to the environment home, so a relative
3308 // datadir makes dbenv.open() fail with DB_RUNRECOVERY; and BDB opens
3309 // db.log relative to the working directory, which is src/ during `make
3310 // tests`. Moving in puts it inside the scratch directory, where it dies
3311 // with everything else.
3312 std::string cwd = ".";
3313 {
3314#ifdef _WIN32
3315 char buf[MAX_PATH];
3316 if (GetCurrentDirectoryA(MAX_PATH, buf) > 0) cwd = buf;
3317#else
3318 char buf[4096];
3319 if (getcwd(buf, sizeof(buf)) != NULL) cwd = buf;
3320#endif
3321 }
3322 std::string dir = cwd + "/" +
3323 strprintf("payout_selftest_%d_%lu", (int)GetTime(), pid);
3324#ifdef _WIN32
3325 CreateDirectoryA(dir.c_str(), NULL);
3326 SetCurrentDirectoryA(dir.c_str());
3327#else
3328 mkdir(dir.c_str(), 0700);
3329 if (chdir(dir.c_str()) != 0) { /* stay put; the paths are absolute anyway */ }
3330#endif
3331 strSetDataDir = dir;
3332 fprintf(stdout, " scratch datadir: %s\n", dir.c_str());
3333
3334 const std::string ledgerPath = PayoutFilePath();
3335 const std::string payingPath = PayingFilePath();
3336 const std::string addrA = PayoutTestAddress(0x11);
3337 const std::string addrB = PayoutTestAddress(0x55);
3338
3339 // -- 1. A restart must not duplicate an in-flight record -----------------
3340 //
3341 // The ledger entry is grandfathered (zero hash) on purpose: it keeps
3342 // PayoutBlockStillOnChain out of the picture, which needs a block index,
3343 // and it exercises the height fallback at the same time.
3344 {
3345 PayoutTestResetState();
3346 remove(ledgerPath.c_str());
3347 remove(payingPath.c_str());
3348
3349 PendingPayout pp;
3350 pp.blockHeight = 4242;
3351 pp.blockHash = 0;
3352 pp.matureAtHeight = 4242;
3353 pp.amounts[addrA] = 10 * COIN;
3354 pp.amounts[addrB] = 20 * COIN;
3355 VersionedBlob snap;
3356 {
3357 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3358 gPendingPayouts.push_back(pp);
3359 snap = SerializePendingPayoutsLocked();
3360 }
3361 PublishPendingPayouts(snap);
3362
3363 nBestHeight = 5000;
3364 FlushMaturePayouts();
3365 size_t afterFirstFlush = 0;
3366 {
3367 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3368 afterFirstFlush = gPaying.size();
3369 }
3370 check(afterFirstFlush == 2,
3371 "a matured round explodes into one record per unpaid recipient");
3372
3373 // Restart: drop everything in memory and come back from the files.
3374 for (int restart = 1; restart <= 2; restart++) {
3375 PayoutTestResetState();
3376 LoadPendingPayouts();
3377 LoadPaying();
3378 ReconcilePayouts();
3379
3380 size_t queued = 0;
3381 {
3382 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3383 queued = gQueuedPayouts.size();
3384 }
3385 if (restart == 1)
3386 check(queued == 1,
3387 "startup rebuilds the exploded-round set from the restored"
3388 " in-flight list");
3389
3390 FlushMaturePayouts();
3391 size_t n = 0;
3392 {
3393 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3394 n = gPaying.size();
3395 }
3396 check(n == 2, restart == 1
3397 ? "a restart does not duplicate an in-flight record"
3398 : "a second restart does not duplicate one either");
3399 }
3400
3401 // The other half of the guard: even with the round-level dedupe set
3402 // deliberately emptied, the per-recipient check against gPaying must
3403 // stop a second standing order for the same money.
3404 {
3405 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3406 gQueuedPayouts.clear();
3407 }
3408 FlushMaturePayouts();
3409 size_t n = 0;
3410 {
3411 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3412 n = gPaying.size();
3413 }
3414 check(n == 2,
3415 "the flush refuses a second record for a (round, address) already"
3416 " in flight even with the round set lost");
3417 }
3418
3419 // -- 2. Duplicates already on disk are collapsed as they are read --------
3420 {
3421 PayoutTestResetState();
3422 nlohmann::json arr = nlohmann::json::array();
3423 for (int i = 0; i < 2; i++) {
3424 nlohmann::json o;
3425 o["blockHeight"] = 4242;
3426 o["blockHash"] = uint256(0).ToString();
3427 o["matureAtHeight"] = 4242;
3428 o["address"] = addrA;
3429 o["amount"] = (int64)(10 * COIN);
3430 o["retries"] = i == 0 ? 7 : 0;
3431 o["deferrals"] = i == 0 ? 0 : 3;
3432 arr.push_back(o);
3433 }
3434 check(PayoutTestWriteFile(payingPath, arr.dump(2)),
3435 "wrote a paying file containing the duplicate the old build made");
3436 LoadPaying();
3437 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3438 check(gPaying.size() == 1,
3439 "loading collapses a duplicate in-flight record from disk");
3440 check(gPaying.size() == 1 && gPaying[0].retries == 7 && gPaying[0].deferrals == 3,
3441 "collapsing merges the attempt counts upwards, not resetting them");
3442 }
3443
3444 // -- 3. State updates address a slot, not a value -----------------------
3445 //
3446 // Build the exact pathology the old code produced -- two records equal in
3447 // every field -- and prove the update lands on the one asked for. Under
3448 // value matching the first record absorbed everything and the second was
3449 // untouchable.
3450 {
3451 PayoutTestResetState();
3452 PayingRecord a;
3453 a.blockHeight = 4242; a.blockHash = 0; a.matureAtHeight = 4242;
3454 a.address = addrA; a.amount = 10 * COIN;
3455 PayingRecord b = a;
3456 {
3457 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3458 a.id = ++gPayingNextId;
3459 b.id = ++gPayingNextId;
3460 gPaying.push_back(a);
3461 gPaying.push_back(b);
3462 }
3463 PayingRecord upd = b;
3464 upd.retries = 5;
3465 upd.retryAfter = 1234567;
3466 ReschedulePayment(upd);
3467 {
3468 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3469 check(gPaying.size() == 2 &&
3470 gPaying[0].retries == 0 && gPaying[1].retries == 5,
3471 "rescheduling updates the record it was given, not the first"
3472 " one that looks like it");
3473 }
3474 ForgetPayment(a);
3475 {
3476 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3477 check(gPaying.size() == 1 && gPaying[0].id == b.id,
3478 "forgetting erases the record it was given, not the first"
3479 " one that looks like it");
3480 }
3481 }
3482
3483 // -- 4. retryAfter survives a restart ------------------------------------
3484 {
3485 PayoutTestResetState();
3486 remove(payingPath.c_str());
3487 PayingRecord r;
3488 r.blockHeight = 77; r.blockHash = 0; r.matureAtHeight = 77;
3489 r.address = addrA; r.amount = 3 * COIN;
3490 r.retryAfter = 1700000000LL;
3491 r.deferrals = 9;
3492 VersionedBlob snap;
3493 {
3494 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3495 r.id = ++gPayingNextId;
3496 gPaying.push_back(r);
3497 snap = SerializePayingLocked();
3498 }
3499 PublishPaying(snap);
3500 PayoutTestResetState();
3501 LoadPaying();
3502 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3503 check(gPaying.size() == 1 && gPaying[0].retryAfter == 1700000000LL &&
3504 gPaying[0].deferrals == 9,
3505 "the retry clock that is actually used survives a restart");
3506 }
3507
3508 // -- 5. The ledger file cannot be published from a stale snapshot --------
3509 {
3510 PayoutTestResetState();
3511 remove(ledgerPath.c_str());
3512
3513 PendingPayout one;
3514 one.blockHeight = 100; one.blockHash = 0; one.matureAtHeight = 100;
3515 one.amounts[addrA] = 1 * COIN;
3516 VersionedBlob older, newer;
3517 {
3518 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3519 gPendingPayouts.push_back(one);
3520 older = SerializePendingPayoutsLocked();
3521 PendingPayout two;
3522 two.blockHeight = 200; two.blockHash = 0; two.matureAtHeight = 200;
3523 two.amounts[addrB] = 2 * COIN;
3524 gPendingPayouts.push_back(two);
3525 newer = SerializePendingPayoutsLocked();
3526 }
3527 check(newer.seq > older.seq,
3528 "a later serialisation carries a strictly higher version");
3529
3530 // The order the race produced: the payout thread's older image
3531 // arriving after the event loop's newer one.
3532 PublishPendingPayouts(newer);
3533 PublishPendingPayouts(older);
3534 std::string onDisk;
3535 check(PayoutTestReadFile(ledgerPath, onDisk) && onDisk == newer.blob,
3536 "an older snapshot cannot overwrite a newer published ledger");
3537
3538 // And the ordinary direction still works.
3539 VersionedBlob newest;
3540 {
3541 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3542 gPendingPayouts.clear();
3543 newest = SerializePendingPayoutsLocked();
3544 }
3545 PublishPendingPayouts(newest);
3546 check(PayoutTestReadFile(ledgerPath, onDisk) && onDisk == newest.blob,
3547 "a newer snapshot does replace the published ledger");
3548 }
3549
3550 // -- 6. Concurrent publishers cannot interleave a file -------------------
3551 //
3552 // Two threads opening and renaming the same "<path>.tmp" over the same
3553 // target was the original defect. This test was mutation-tested and did
3554 // not catch it: with the file mutex removed the suite still went green 2
3555 // runs in 5, and the "never leave it unparseable" assertion fired 0 times
3556 // in 5. Two reasons, both fixed here.
3557 //
3558 // First, every image had the same structure and near enough the same
3559 // length, so an interleaving of two of them was still valid JSON. The
3560 // images now alternate between ONE entry and FIVE THOUSAND, so a write
3561 // that lands on top of another leaves either a truncated array or a small
3562 // image followed by the tail of a large one -- neither of which parses.
3563 //
3564 // Second, and worse, almost nothing was being written. Each thread walked
3565 // a permutation of the images, and PublishVersioned drops any blob that is
3566 // not newer than what is on disk, so only the record highs in each
3567 // permutation ever reached the disk -- on the order of ten writes in
3568 // total, four threads, no contention to speak of. Every publish here is
3569 // stamped from a shared counter at the moment it is made, so every one of
3570 // them is newer than the last and every one of them writes.
3571 //
3572 // And the file is watched while the writers run, not only inspected after
3573 // they finish. "The published file is never observed unparseable" is the
3574 // actual invariant -- rename(2) is atomic, so a reader can only ever see a
3575 // completely written image -- and checking it once at the end samples the
3576 // one moment when a torn file is least likely to still be there.
3577 {
3578 const std::string racePath = dir + "/publish_race_selftest.json";
3579 std::mutex raceMutex;
3580 uint64 racePublished = 0;
3581
3582 std::string tiny, huge;
3583 {
3584 nlohmann::json one = nlohmann::json::array();
3585 nlohmann::json o;
3586 o["image"] = 0;
3587 o["entry"] = 0;
3588 o["address"] = addrA;
3589 o["amount"] = (int64)COIN;
3590 one.push_back(o);
3591 tiny = one.dump(2);
3592
3593 nlohmann::json many = nlohmann::json::array();
3594 for (int k = 0; k < 5000; k++) {
3595 nlohmann::json e;
3596 e["image"] = 1;
3597 e["entry"] = k;
3598 e["address"] = addrA;
3599 e["amount"] = (int64)(k + 1) * COIN;
3600 many.push_back(e);
3601 }
3602 huge = many.dump(2);
3603 }
3604
3605 const int kThreads = 4;
3606 const int kPerThread = 100;
3607 const uint64 kTotal = (uint64)kThreads * kPerThread;
3608
3609 std::atomic<uint64> nextSeq(0);
3610 std::atomic<bool> racing(true);
3611 std::atomic<int> tornReads(0);
3612 std::atomic<int> goodReads(0);
3613
3614 std::thread watcher([&]() {
3615 bool seenOnce = false;
3616 while (racing) {
3617 std::string s;
3618 if (!PayoutTestReadFile(racePath, s)) {
3619 // Only meaningful once the file exists at all.
3620 if (seenOnce) tornReads++;
3621 continue;
3622 }
3623 seenOnce = true;
3624 try {
3625 if (nlohmann::json::parse(s).is_array()) goodReads++;
3626 else tornReads++;
3627 } catch (...) {
3628 tornReads++;
3629 }
3630 }
3631 });
3632
3633 std::vector<std::thread> threads;
3634 for (int t = 0; t < kThreads; t++) {
3635 threads.push_back(std::thread([&]() {
3636 for (int i = 0; i < kPerThread; i++) {
3637 VersionedBlob v;
3638 v.seq = ++nextSeq;
3639 v.blob = (v.seq & 1) ? tiny : huge;
3640 PublishVersioned(racePath, v, raceMutex, racePublished,
3641 "publish race selftest");
3642 }
3643 }));
3644 }
3645 for (std::thread& th : threads)
3646 th.join();
3647 racing = false;
3648 watcher.join();
3649
3650 check(goodReads > 0,
3651 "the race test actually observed the file while it was being"
3652 " written");
3653 check(tornReads == 0,
3654 "four threads publishing the same file never leave it"
3655 " unparseable, at any instant while they are doing it");
3656
3657 // The highest sequence number issued is the one the file must settle
3658 // on: PublishVersioned refuses anything older, so whichever order the
3659 // writes complete in, that image is the last one that can land.
3660 const std::string& expected = (kTotal & 1) ? tiny : huge;
3661 std::string onDisk;
3662 bool read = PayoutTestReadFile(racePath, onDisk);
3663 check(read && onDisk == expected,
3664 "the published file settles on the highest version, whatever the"
3665 " completion order");
3666 remove(racePath.c_str());
3667 remove((racePath + ".tmp").c_str());
3668 }
3669
3670 // -- 7. Grandfathered zero-hash rounds do not answer for each other ------
3671 {
3672 uint256 h1 = 0, h2 = 0;
3673 h1.SetHex("00000000000000000000000000000000000000000000000000000000000000aa");
3674 h2.SetHex("00000000000000000000000000000000000000000000000000000000000000bb");
3675 check(!SamePayoutRound(100, 0, 200, 0),
3676 "two zero-hash entries at different heights are different rounds");
3677 check(SamePayoutRound(100, 0, 100, 0),
3678 "two zero-hash entries at the same height are the same round");
3679 check(SamePayoutRound(100, h1, 200, h1),
3680 "a real hash identifies the round on its own");
3681 check(!SamePayoutRound(100, h1, 100, h2),
3682 "different hashes at one height are different rounds");
3683 check(!SamePayoutRound(100, h1, 100, 0),
3684 "a grandfathered entry never matches a hash-keyed one");
3685 check(!PayoutRoundIdentifiable(0, 0) && PayoutRoundIdentifiable(0, h1) &&
3686 PayoutRoundIdentifiable(7, 0),
3687 "an entry with neither hash nor height is not adjudicable");
3688
3689 // What actually makes a grandfathered same-height pair safe, since the
3690 // comment on FindRecordedPoolPayout used to claim it was the orphan
3691 // check. It is not: the orphan check passes every zero-hash entry, and
3692 // every pre-upgrade entry has a zero hash.
3693 check(PayoutBlockStillOnChain(0),
3694 "the orphan check passes every grandfathered entry, so it cannot"
3695 " be what voids a losing twin");
3696 PayoutTestResetState();
3697 remove(ledgerPath.c_str());
3698 remove(payingPath.c_str());
3699 {
3700 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3701 PendingPayout twin;
3702 twin.blockHeight = 300; twin.blockHash = 0; twin.matureAtHeight = 300;
3703 twin.amounts[addrA] = 1 * COIN;
3704 gPendingPayouts.push_back(twin);
3705 gPendingPayouts.push_back(twin); // the losing side of a reorg
3706 }
3707 nBestHeight = 5000;
3708 FlushMaturePayouts();
3709 {
3710 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3711 check(gPaying.size() == 1,
3712 "two grandfathered rounds at one height are one round, so the"
3713 " pair presents as one debt and is paid at most once");
3714 }
3715
3716 // End to end: recording a payment against one grandfathered round must
3717 // not settle the other's debt.
3718 PayoutTestResetState();
3719 remove(ledgerPath.c_str());
3720 {
3721 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3722 PendingPayout first;
3723 first.blockHeight = 100; first.blockHash = 0; first.matureAtHeight = 100;
3724 first.amounts[addrA] = 1 * COIN;
3725 PendingPayout second;
3726 second.blockHeight = 200; second.blockHash = 0; second.matureAtHeight = 200;
3727 second.amounts[addrA] = 1 * COIN;
3728 gPendingPayouts.push_back(first);
3729 gPendingPayouts.push_back(second);
3730 }
3731 PayingRecord r;
3732 r.blockHeight = 200; r.blockHash = 0; r.matureAtHeight = 200;
3733 r.address = addrA; r.amount = 1 * COIN;
3734 RecordPayoutTxid(r, "txid-for-the-second-round");
3735 {
3736 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3737 // The second round was fully paid and retired; the first is
3738 // untouched and still owed.
3739 check(gPendingPayouts.size() == 1 &&
3740 gPendingPayouts[0].blockHeight == 100 &&
3741 gPendingPayouts[0].paidTxids.empty(),
3742 "a txid recorded against one zero-hash round leaves the other"
3743 " still owed");
3744 }
3745 }
3746
3747 // -- 8. A payment with only a legacy comment is recognised as paid -------
3748 {
3749 PayoutTestResetState();
3750 const int height = 8080;
3751 const int64 amount = 7 * COIN;
3752 const std::string id = PayoutIdForBlock(height, 0);
3753 const std::string comment = PayoutComment(height);
3754 std::set<std::string> none;
3755 std::string found;
3756
3757 check(FindRecordedPoolPayout(id, height, addrA, amount, none, found)
3758 == PAYOUT_PROBE_NONE,
3759 "an empty wallet is not evidence that anything was paid");
3760
3761 const std::string legacyTxid =
3762 PayoutTestAddWalletTx(comment, addrA, amount, 1, "");
3763 check(FindRecordedPoolPayout(id, height, addrA, amount, none, found)
3764 == PAYOUT_PROBE_LEGACY && found == legacyTxid,
3765 "a payment carrying only a pre-upgrade comment is recognised as"
3766 " already paid");
3767
3768 check(FindRecordedPoolPayout(id, height, addrB, amount, none, found)
3769 == PAYOUT_PROBE_NONE,
3770 "the legacy match requires the output to pay THIS recipient");
3771 check(FindRecordedPoolPayout(id, height, addrA, amount + COIN, none, found)
3772 == PAYOUT_PROBE_NONE,
3773 "the legacy match requires the output to pay exactly this amount");
3774 check(FindRecordedPoolPayout(id, height + 1, addrA, amount, none, found)
3775 == PAYOUT_PROBE_NONE,
3776 "the legacy match requires the comment for this block height");
3777
3778 std::set<std::string> spent;
3779 spent.insert(legacyTxid);
3780 check(FindRecordedPoolPayout(id, height, addrA, amount, spent, found)
3781 == PAYOUT_PROBE_NONE,
3782 "one legacy transaction cannot settle a second debt");
3783
3784 // A transaction written by THIS build carries pool_payout_id, and must
3785 // never be offered as legacy evidence for a different round -- that is
3786 // exactly what the structured metadata exists to prevent.
3787 const std::string otherId = PayoutIdForBlock(height, 0) + "-other";
3788 PayoutTestAddWalletTx(comment, addrB, amount, 2, otherId);
3789 check(FindRecordedPoolPayout(id, height, addrB, amount, none, found)
3790 == PAYOUT_PROBE_NONE,
3791 "another round's structured payment is not legacy evidence for"
3792 " this one");
3793
3794 // The structured path still wins where it applies.
3795 const std::string structuredTxid =
3796 PayoutTestAddWalletTx(comment, addrB, amount, 3, id);
3797 check(FindRecordedPoolPayout(id, height, addrB, amount, none, found)
3798 == PAYOUT_PROBE_STRUCTURED && found == structuredTxid,
3799 "structured payout metadata is matched in preference to a comment");
3800
3801 // A comment-matching transaction that never made it into the chain is
3802 // evidence of NON-payment. This tree has seen a 280-block reorg; a
3803 // pre-upgrade payout invalidated by one is still in mapWallet with its
3804 // comment intact, and the first boot after the upgrade is exactly when
3805 // it would be matched.
3806 CRITICAL_BLOCK(cs_mapWallet)
3807 mapWallet.clear();
3808 PayoutTestClearBlockIndex();
3809 const std::string addrC = PayoutTestAddress(0x33);
3810 PayoutTestAddWalletTx(comment, addrC, amount, 4, "", false);
3811 check(FindRecordedPoolPayout(id, height, addrC, amount, none, found)
3812 == PAYOUT_PROBE_NONE,
3813 "a pre-upgrade payout that never confirmed is not read as"
3814 " payment");
3815
3816 // The same transaction, confirmed, is.
3817 CRITICAL_BLOCK(cs_mapWallet)
3818 mapWallet.clear();
3819 const std::string confirmedTxid =
3820 PayoutTestAddWalletTx(comment, addrC, amount, 4, "", true);
3821 check(FindRecordedPoolPayout(id, height, addrC, amount, none, found)
3822 == PAYOUT_PROBE_LEGACY && found == confirmedTxid,
3823 "the same payout, confirmed, is");
3824
3825 // And an unconfirmed one does not shadow a confirmed one that would
3826 // have matched: the probe used to stop at the first comment match.
3827 const std::string alsoConfirmed =
3828 PayoutTestAddWalletTx(comment, addrC, amount, 6, "", true);
3829 PayoutTestAddWalletTx(comment, addrC, amount, 5, "", false);
3830 std::set<std::string> spentFirst;
3831 spentFirst.insert(confirmedTxid);
3832 check(FindRecordedPoolPayout(id, height, addrC, amount, spentFirst, found)
3833 == PAYOUT_PROBE_LEGACY && found == alsoConfirmed,
3834 "an unconfirmed transaction does not hide a confirmed one behind"
3835 " it");
3836
3837 CRITICAL_BLOCK(cs_mapWallet)
3838 mapWallet.clear();
3839 PayoutTestClearBlockIndex();
3840 }
3841
3842 // -- 9. On-disk compatibility with the live production ledger ------------
3843 {
3844 PayoutTestResetState();
3845 // Exactly the shape of the file on the pool node: amounts, blockHash,
3846 // blockHeight, matureAtHeight, and nothing else.
3847 const char* live =
3848 "[{\"amounts\":{\"%s\":1234500000},"
3849 "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000cd\","
3850 "\"blockHeight\":1000,"
3851 "\"matureAtHeight\":1120}]";
3852 check(PayoutTestWriteFile(ledgerPath, strprintf(live, addrA.c_str())),
3853 "wrote a ledger file in the live production shape");
3854 LoadPendingPayouts();
3855 {
3856 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3857 check(gPendingPayouts.size() == 1 &&
3858 gPendingPayouts[0].blockHeight == 1000 &&
3859 gPendingPayouts[0].matureAtHeight == 1120 &&
3860 gPendingPayouts[0].blockHash != 0 &&
3861 gPendingPayouts[0].amounts.size() == 1 &&
3862 gPendingPayouts[0].amounts[addrA] == 1234500000LL,
3863 "the live production pending_payouts.json still loads");
3864 }
3865
3866 // And the shape the merged build writes, retryAfter included, which
3867 // this build no longer writes but must still read past.
3868 PayoutTestResetState();
3869 const char* merged =
3870 "[{\"amounts\":{\"%s\":100000000},"
3871 "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000ef\","
3872 "\"blockHeight\":2000,\"matureAtHeight\":2120,\"retryAfter\":0,"
3873 "\"paidTxids\":{\"%s\":\"abc123\"}}]";
3874 check(PayoutTestWriteFile(ledgerPath,
3875 strprintf(merged, addrA.c_str(), addrA.c_str())),
3876 "wrote a ledger file in the merged build's shape");
3877 LoadPendingPayouts();
3878 {
3879 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3880 check(gPendingPayouts.size() == 1 &&
3881 gPendingPayouts[0].blockHeight == 2000 &&
3882 gPendingPayouts[0].paidTxids.size() == 1 &&
3883 gPendingPayouts[0].paidTxids[addrA] == "abc123",
3884 "a ledger file carrying retryAfter and paidTxids still loads");
3885 }
3886
3887 // A round trip through this build must not lose anything either.
3888 VersionedBlob snap;
3889 {
3890 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3891 snap = SerializePendingPayoutsLocked();
3892 }
3893 PublishPendingPayouts(snap);
3894 PayoutTestResetState();
3895 LoadPendingPayouts();
3896 {
3897 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3898 check(gPendingPayouts.size() == 1 &&
3899 gPendingPayouts[0].blockHeight == 2000 &&
3900 gPendingPayouts[0].matureAtHeight == 2120 &&
3901 gPendingPayouts[0].paidTxids[addrA] == "abc123",
3902 "what this build writes, this build reads back unchanged");
3903 }
3904 }
3905
3906 // -- 10. A forfeit is published, not just dropped ------------------------
3907 {
3908 PayoutTestResetState();
3909 remove(ledgerPath.c_str());
3910 const std::string roundsPath = dir + "/pool_rounds_selftest.json";
3911 const std::string savedRoundsFile = strPoolRoundsFile;
3912 strPoolRoundsFile = roundsPath;
3913 remove(roundsPath.c_str());
3914
3915 const int height = 606;
3916 PoolRoundProof round;
3917 round.id = PayoutIdForBlock(height, 0);
3918 round.blockHeight = height;
3919 round.matureAtHeight = height;
3920 round.payouts[addrA] = 5 * COIN;
3921 round.payouts[addrB] = 6 * COIN;
3922 RecordPoolRound(round);
3923
3924 {
3925 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3926 PendingPayout pp;
3927 pp.blockHeight = height; pp.blockHash = 0; pp.matureAtHeight = height;
3928 pp.amounts[addrA] = 5 * COIN;
3929 pp.amounts[addrB] = 6 * COIN;
3930 gPendingPayouts.push_back(pp);
3931 }
3932
3933 PayingRecord r;
3934 r.blockHeight = height; r.blockHash = 0; r.matureAtHeight = height;
3935 r.address = addrA; r.amount = 5 * COIN;
3936 ForfeitPayoutRecipient(r, "selftest reason");
3937
3938 {
3939 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3940 check(gPendingPayouts.size() == 1 &&
3941 gPendingPayouts[0].amounts.count(addrA) == 0 &&
3942 gPendingPayouts[0].amounts.count(addrB) == 1,
3943 "a forfeit removes only that recipient from the ledger entry");
3944 }
3945
3946 std::string onDisk;
3947 bool proofOk = false;
3948 if (PayoutTestReadFile(roundsPath, onDisk)) {
3949 try {
3950 nlohmann::json root = nlohmann::json::parse(onDisk);
3951 const nlohmann::json& p = root["rounds"][0]["payouts"];
3952 proofOk = p[addrA].value("forfeited", false) &&
3953 p[addrA].value("forfeitReason", "") == "selftest reason" &&
3954 p[addrA]["txid"].is_null() &&
3955 !p[addrB].contains("forfeited");
3956 } catch (...) {
3957 proofOk = false;
3958 }
3959 }
3960 check(proofOk,
3961 "the published round proof records the forfeit instead of"
3962 " advertising the address as still owed");
3963
3964 // And it survives a reload of the proof file.
3965 {
3966 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
3967 gPoolRounds.clear();
3968 }
3969 LoadPoolRounds();
3970 {
3971 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
3972 check(gPoolRounds.size() == 1 &&
3973 gPoolRounds[0].forfeited.count(addrA) == 1 &&
3974 gPoolRounds[0].forfeited[addrA] == "selftest reason",
3975 "the forfeit is read back from pool_rounds.json");
3976 }
3977
3978 remove(roundsPath.c_str());
3979 remove((roundsPath + ".tmp").c_str());
3980 strPoolRoundsFile = savedRoundsFile;
3981 }
3982
3983 // -- 11. A ledger that failed to load is never written back --------------
3984 //
3985 // The loaders push entries into the live vector inside the try, so an
3986 // exception part way down a file leaves a PARTIAL ledger in memory: the
3987 // entries before the bad one and nothing after it. Publishing that -- and
3988 // ReconcilePayouts publishes unconditionally, milliseconds into startup --
3989 // writes the partial state over the file and destroys the only record of
3990 // what was owed.
3991 {
3992 PayoutTestResetState();
3993 remove(ledgerPath.c_str());
3994
3995 // Two entries, the second with a matureAtHeight that is not a number.
3996 // The first loads; the second throws where it is read.
3997 const std::string corrupt = strprintf(
3998 "[{\"amounts\":{\"%s\":100000000},\"blockHash\":\"%s\","
3999 "\"blockHeight\":10,\"matureAtHeight\":10},"
4000 "{\"amounts\":{\"%s\":900000000},\"blockHash\":\"%s\","
4001 "\"blockHeight\":20,\"matureAtHeight\":\"twenty\"}]",
4002 addrA.c_str(), uint256(0).ToString().c_str(),
4003 addrB.c_str(), uint256(0).ToString().c_str());
4004 check(PayoutTestWriteFile(ledgerPath, corrupt),
4005 "wrote a ledger whose second entry does not parse");
4006
4007 const int64 t0 = GetTime();
4008 LoadPendingPayouts();
4009 const int64 t1 = GetTime();
4010
4011 size_t loaded = 0;
4012 {
4013 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4014 loaded = gPendingPayouts.size();
4015 }
4016 check(loaded == 1,
4017 "a mid-file parse error does leave a partial ledger in memory --"
4018 " which is the reason for everything below");
4019
4020 std::string gone;
4021 check(!PayoutTestReadFile(ledgerPath, gone),
4022 "the file that did not parse is moved out of the way");
4023
4024 // The rename stamps GetTime(); it was taken between t0 and t1.
4025 std::string kept, keptPath;
4026 for (int64 t = t0; t <= t1 && keptPath.empty(); t++) {
4027 const std::string candidate =
4028 strprintf("%s.bad.%lld", ledgerPath.c_str(), (long long)t);
4029 if (PayoutTestReadFile(candidate, kept))
4030 keptPath = candidate;
4031 }
4032 check(!keptPath.empty() && kept == corrupt,
4033 "it is kept, byte for byte, as <path>.bad.<timestamp>");
4034
4035 // The publish that used to destroy it.
4036 VersionedBlob partial;
4037 {
4038 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4039 partial = SerializePendingPayoutsLocked();
4040 }
4041 PublishPendingPayouts(partial);
4042 check(!PayoutTestReadFile(ledgerPath, gone),
4043 "and the partial ledger is not published over it");
4044
4045 // The refusal is per path and lasts the process: a second attempt,
4046 // with more state, is refused too.
4047 {
4048 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4049 PendingPayout pp;
4050 pp.blockHeight = 30; pp.blockHash = 0; pp.matureAtHeight = 30;
4051 pp.amounts[addrA] = 5 * COIN;
4052 gPendingPayouts.push_back(pp);
4053 partial = SerializePendingPayoutsLocked();
4054 }
4055 PublishPendingPayouts(partial);
4056 check(!PayoutTestReadFile(ledgerPath, gone),
4057 "the refusal is not a one-shot -- the path stays unwritable");
4058
4059 if (!keptPath.empty())
4060 remove(keptPath.c_str());
4061 PayoutTestResetState();
4062 remove(ledgerPath.c_str());
4063
4064 // And a file that parses is not quarantined, so the ordinary path is
4065 // still the ordinary path.
4066 check(PayoutTestWriteFile(ledgerPath, "[]"),
4067 "wrote a ledger that parses");
4068 LoadPendingPayouts();
4069 {
4070 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4071 PendingPayout pp;
4072 pp.blockHeight = 40; pp.blockHash = 0; pp.matureAtHeight = 40;
4073 pp.amounts[addrA] = 2 * COIN;
4074 gPendingPayouts.push_back(pp);
4075 partial = SerializePendingPayoutsLocked();
4076 }
4077 PublishPendingPayouts(partial);
4078 std::string written;
4079 check(PayoutTestReadFile(ledgerPath, written) && written == partial.blob,
4080 "a ledger that loaded cleanly is still published normally");
4081 }
4082
4083 // -- 12. A send that failed for want of coin is not a fault --------------
4084 //
4085 // The defect this replaces: the classifier decided "was this a funding
4086 // problem?" by re-reading GetBalance() and allowing amount +
4087 // nTransactionFee + CENT. nTransactionFee is 0 on a headless node, and
4088 // GetMinFee() is a step function -- 0 below 10,000 bytes, 11 * CENT at it
4089 // -- so the allowance was either unnecessary or eleven times too small,
4090 // and the ledger's own retry budget then wrote the debt off after 1440
4091 // repeats of the same stable condition.
4092 //
4093 // This is tested against a real wallet and the real CreateTransaction, not
4094 // against injected values: the honest question is not "does my enum
4095 // switch work" but "does a wallet holding many small coins actually report
4096 // the state I think it does". 60 outputs of 1 BTF give ~180 signed P2PKH
4097 // bytes each, which is where the fee schedule steps. Nothing is committed
4098 // or broadcast -- CreateTransaction only builds.
4099 {
4100 PayoutTestResetState();
4101 CRITICAL_BLOCK(cs_mapWallet)
4102 mapWallet.clear();
4103
4104 // CreateTransaction opens CTxDB("r"), which throws if blkindex.dat is
4105 // not there. Create it in the scratch datadir first.
4106 bool dbOk = true;
4107 try {
4108 CTxDB txdb("cr");
4109 } catch (const std::exception&) {
4110 dbOk = false;
4111 }
4112 check(dbOk, "a scratch block index exists for CreateTransaction to open");
4113
4114 const std::string walletAddr = PayoutTestAddSpendableKey();
4115 const std::string payee = PayoutTestAddress(0x77);
4116 const CScript payeeScript = PayoutTestScriptFor(payee);
4117
4118 // -- the >= 10 KB case: 60 small coins, balance = amount + 0.05 ------
4119 PayoutTestFundWallet(60, 1 * COIN, walletAddr);
4120 const int64 balance = GetBalance();
4121 check(balance == 60 * COIN,
4122 "the test wallet holds 60 separately spendable coins");
4123
4124 const int64 amount = balance - 5 * CENT;
4125 // The pre-send gate in PayoutThreadFn passes: the balance covers the
4126 // amount and the only fee that can be known in advance.
4127 check(balance >= amount + nTransactionFee,
4128 "the pre-send gate would let this payment through");
4129
4130 CWalletTx wtx;
4131 SendMoneyOutcome outcome;
4132 int64 feeRequired = 0;
4133 WalletSendResult result = WALLET_SEND_OK;
4134 bool built = true;
4135 if (dbOk) {
4136 try {
4137 built = CreateTransaction(payeeScript, amount, wtx, feeRequired,
4138 &result);
4139 } catch (const std::exception&) {
4140 built = true; // report as a failure below
4141 }
4142 }
4143 check(dbOk && !built,
4144 "a wallet of many small coins cannot fund amount + the fee its"
4145 " own size demands");
4146 check(dbOk && result == WALLET_SEND_INSUFFICIENT_FUNDS,
4147 "the wallet reports that as a funding shortfall, not as a"
4148 " failure to build");
4149 // GetMinFee(fDiscount) returns 0 below 10,000 bytes and
4150 // (1 + bytes/1000) * CENT at or above, so a required fee of 11 CENT or
4151 // more IS the statement that the transaction crossed 10 KB. Measured
4152 // through the real serialiser rather than asserted.
4153 check(dbOk && feeRequired >= 11 * CENT,
4154 "the transaction it built crossed 10,000 bytes, so the fee"
4155 " stepped from 0 to at least 11 CENT");
4156
4157 // The defect, stated as arithmetic. The old classifier asked whether
4158 // the balance covered amount + nTransactionFee + CENT; it did, so it
4159 // called a permanent funding shortfall a fault and started counting
4160 // towards a write-off.
4161 check(!(balance < amount + nTransactionFee + CENT),
4162 "the reserve this replaces would have judged this send a fault");
4163 check(balance < amount + feeRequired,
4164 "while the fee actually required was more than the balance had");
4165
4166 outcome.result = result;
4167 outcome.nFeeRequired = feeRequired;
4168 check(dbOk && ClassifyPayoutSendFailure(outcome.result, false)
4169 == PAYOUT_FAILURE_DEFER,
4170 "a >= 10 KB send that ran out of coin is deferred, not counted"
4171 " against the payment");
4172
4173 // -- the < 10 KB case: no reserve is needed at all -------------------
4174 //
4175 // The other half of the arithmetic. Below the step the required fee is
4176 // 0, so a reserve of any size is a reason to refuse a payment the
4177 // wallet could have made -- which is its own way of not paying a
4178 // miner.
4179 CRITICAL_BLOCK(cs_mapWallet)
4180 mapWallet.clear();
4181 PayoutTestFundWallet(3, 1 * COIN, walletAddr);
4182 const int64 smallAmount = 3 * COIN - 5 * CENT;
4183 CWalletTx smallTx;
4184 int64 smallFee = -1;
4185 WalletSendResult smallResult = WALLET_SEND_FAILED;
4186 bool smallBuilt = false;
4187 if (dbOk) {
4188 try {
4189 smallBuilt = CreateTransaction(payeeScript, smallAmount, smallTx,
4190 smallFee, &smallResult);
4191 } catch (const std::exception&) {
4192 smallBuilt = false;
4193 }
4194 }
4195 check(dbOk && smallBuilt && smallResult == WALLET_SEND_OK,
4196 "the same balance pays a small transaction with no headroom at"
4197 " all");
4198 check(dbOk && smallFee == 0,
4199 "because below 10,000 bytes the required fee is zero");
4200
4201 // -- and the classification of every other answer --------------------
4202 check(ClassifyPayoutSendFailure(WALLET_SEND_INSUFFICIENT_FUNDS, false)
4203 == PAYOUT_FAILURE_DEFER &&
4204 ClassifyPayoutSendFailure(WALLET_SEND_INSUFFICIENT_FUNDS, true)
4205 == PAYOUT_FAILURE_DEFER &&
4206 ClassifyPayoutSendFailure(WALLET_SEND_FAILED, true)
4207 == PAYOUT_FAILURE_DEFER,
4208 "short of coin or unable to sign is always a deferral");
4209 check(ClassifyPayoutSendFailure(WALLET_SEND_FAILED, false)
4210 == PAYOUT_FAILURE_FAULT,
4211 "funded and unlocked and still refused is a fault");
4212
4213 CRITICAL_BLOCK(cs_mapWallet)
4214 mapWallet.clear();
4215 }
4216
4217 // -- 13. No payout mutex is held across the status-file write ------------
4218 //
4219 // RefreshStats runs on the event loop -- the thread that owns every
4220 // miner's socket -- and used to hold gStatsMutex and gPayoutLedgerMutex at
4221 // function scope across WriteFileAtomic. Once that write gained fsync() on
4222 // the file and on its directory, publishing pool_status.json parked the
4223 // payout thread behind two disk flushes every ten seconds.
4224 //
4225 // Timing assertions would be flaky, so this does not measure anything. It
4226 // makes the write block for as long as the test likes: strPoolStatusFile
4227 // points at a path whose ".tmp" is a FIFO, and open(2) for writing on a
4228 // FIFO does not return until a reader arrives. While RefreshStats is
4229 // parked in there, every mutex it must NOT be holding has to be
4230 // acquirable. Against the previous code this test does not merely fail, it
4231 // cannot pass: the writer is inside fopen() with both mutexes held and
4232 // nothing will release them until the test lets the write finish.
4233#ifndef _WIN32
4234 {
4235 PayoutTestResetState();
4236 const std::string statusPath = dir + "/pool_status_selftest.json";
4237 const std::string statusTmp = statusPath + ".tmp";
4238 const std::string savedStatusFile = strPoolStatusFile;
4239 const int64 savedLastWrite = gLastPoolStatusWrite;
4240 remove(statusPath.c_str());
4241 remove(statusTmp.c_str());
4242 strPoolStatusFile = statusPath;
4243 gLastPoolStatusWrite = 0; // due a publish on the next refresh
4244
4245 // Give the status file something to report, so the ledger mutex is
4246 // genuinely taken on the way through rather than skipped over.
4247 {
4248 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4249 PendingPayout pp;
4250 pp.blockHeight = 909; pp.blockHash = 0; pp.matureAtHeight = 909;
4251 pp.amounts[addrA] = 3 * COIN;
4252 gPendingPayouts.push_back(pp);
4253 }
4254
4255 bool fifoOk = (mkfifo(statusTmp.c_str(), 0600) == 0);
4256 check(fifoOk, "a FIFO stands in for the status file's temp file, so the"
4257 " write can be held open");
4258 if (fifoOk) {
4259 std::atomic<bool> done(false);
4260 std::thread writer([&done]() {
4261 RefreshStats(0, 0);
4262 done = true;
4263 });
4264
4265 // Settle: the work before the write is a handful of vector
4266 // operations, so anything past this point is inside fopen(). The
4267 // !done check below is what actually proves it -- with no reader
4268 // there is no other way for this thread to still be running.
4269 Sleep(500);
4270 bool blocked = !done;
4271 bool gotLedger = false, gotStats = false, gotExec = false,
4272 gotRounds = false;
4273 for (int i = 0; i < 1000 && blocked; i++) {
4274 if (!gotLedger && gPayoutLedgerMutex.try_lock()) {
4275 gotLedger = true; gPayoutLedgerMutex.unlock();
4276 }
4277 if (!gotStats && gStatsMutex.try_lock()) {
4278 gotStats = true; gStatsMutex.unlock();
4279 }
4280 if (!gotExec && gPayoutExecMutex.try_lock()) {
4281 gotExec = true; gPayoutExecMutex.unlock();
4282 }
4283 if (!gotRounds && gPoolRoundsMutex.try_lock()) {
4284 gotRounds = true; gPoolRoundsMutex.unlock();
4285 }
4286 if (gotLedger && gotStats && gotExec && gotRounds)
4287 break;
4288 Sleep(2);
4289 }
4290
4291 check(blocked,
4292 "the status-file write is held open for the duration of this"
4293 " test");
4294 check(!blocked || gotLedger,
4295 "gPayoutLedgerMutex is free while pool_status.json is being"
4296 " written");
4297 check(!blocked || (gotExec && gotRounds),
4298 "gPayoutExecMutex and gPoolRoundsMutex are free while"
4299 " pool_status.json is being written");
4300 check(!blocked || gotStats,
4301 "gStatsMutex is free while pool_status.json is being written");
4302
4303 // Let the write finish. Drain to EOF rather than closing early:
4304 // the writer is still filling the pipe and a closed read end would
4305 // deliver it SIGPIPE.
4306 std::thread reader([&statusTmp]() {
4307 FILE* rf = fopen(statusTmp.c_str(), "r");
4308 if (!rf) return;
4309 char buf[4096];
4310 while (fread(buf, 1, sizeof(buf), rf) > 0) { }
4311 fclose(rf);
4312 });
4313 writer.join();
4314 reader.join();
4315 }
4316 remove(statusTmp.c_str());
4317
4318 // And the ordinary path still produces the file it always did.
4319 gLastPoolStatusWrite = 0;
4320 RefreshStats(0, 0);
4321 std::string onDisk;
4322 bool statusOk = false;
4323 if (PayoutTestReadFile(statusPath, onDisk)) {
4324 try {
4325 nlohmann::json j = nlohmann::json::parse(onDisk);
4326 statusOk = j.value("schema", "") == "bitflash-pool-status-1" &&
4327 j["pendingPayouts"].is_array() &&
4328 j["pendingPayouts"].size() == 1 &&
4329 j["pendingPayouts"][0].value("matureAtHeight", 0) == 909 &&
4330 j["pendingPayouts"][0].value("totalAmountSatoshis",
4331 (int64)0) == 3 * COIN;
4332 } catch (...) {
4333 statusOk = false;
4334 }
4335 }
4336 check(statusOk,
4337 "splitting the serialise from the write leaves pool_status.json"
4338 " saying the same thing");
4339
4340 remove(statusPath.c_str());
4341 remove((statusPath + ".tmp").c_str());
4342 strPoolStatusFile = savedStatusFile;
4343 gLastPoolStatusWrite = savedLastWrite;
4344 }
4345#endif
4346
4347 // -- clean up ------------------------------------------------------------
4348 PayoutTestResetState();
4349 CRITICAL_BLOCK(cs_mapWallet)
4350 mapWallet.clear();
4351 PayoutTestClearBlockIndex();
4352 // Out of the scratch directory before removing it, and out of the key maps
4353 // too: the keys this test generated are process-local, but leaving them in
4354 // the keystore would let a later test think a script was its own.
4355 CRITICAL_BLOCK(cs_mapKeys)
4356 {
4357 mapKeys.clear();
4358 mapPubKeys.clear();
4359 }
4360#ifdef _WIN32
4361 SetCurrentDirectoryA(cwd.c_str());
4362#else
4363 if (chdir(cwd.c_str()) != 0) { /* nothing useful to do about it */ }
4364#endif
4365 PayoutTestRemoveDir(dir);
4366 strSetDataDir = savedDataDir;
4367 nBestHeight = savedBestHeight;
4368
4369 fprintf(stdout, "%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
4370 nFail, nFail == 1 ? "" : "s");
4371 fflush(stdout);
4372 return nFail == 0 ? 0 : 1;
4373}
4374
1511int RunPoolStratumSelfTest() 4375int RunPoolStratumSelfTest()
1512{ 4376{
1513 int nFail = 0; 4377 int nFail = 0;
@@ -1654,56 +4518,71 @@ int RunPoolStratumSelfTest()
1654// --------------------------------------------------------------------------- 4518// ---------------------------------------------------------------------------
1655static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal) 4519static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal)
1656{ 4520{
1657 std::lock_guard<std::mutex> lk(gStatsMutex); 4521 std::string statusBlob;
4522 bool fPublish = false;
4523 {
4524 std::lock_guard<std::mutex> lk(gStatsMutex);
1658 4525
1659 gStats.authorizedMiners = 0; 4526 gStats.authorizedMiners = 0;
1660 gStats.blocksFound = blocksFoundThisSession; 4527 gStats.blocksFound = blocksFoundThisSession;
1661 gStats.roundShares = roundShareTotal; 4528 gStats.roundShares = roundShareTotal;
1662 gStats.totalHashRate = 0.0; 4529 gStats.totalHashRate = 0.0;
1663 for (Miner* m : gMiners) 4530 for (Miner* m : gMiners)
1664 if (m->authorised) gStats.authorizedMiners++; 4531 if (m->authorised) gStats.authorizedMiners++;
1665 4532
1666 gWorkerStats.clear(); 4533 gWorkerStats.clear();
1667 for (Miner* m : gMiners) { 4534 for (Miner* m : gMiners) {
1668 if (!m->authorised || m->lastSeen <= 0) continue; 4535 if (!m->authorised || m->lastSeen <= 0) continue;
1669 PoolWorkerStatView v; 4536 PoolWorkerStatView v;
1670 v.address = m->address; 4537 v.address = m->address;
1671 v.worker = m->worker; 4538 v.worker = m->worker;
1672 v.roundShares = m->roundShares; 4539 v.roundShares = m->roundShares;
1673 v.totalShares = m->totalShares; 4540 v.totalShares = m->totalShares;
1674 v.lastSeen = m->lastSeen; 4541 v.lastSeen = m->lastSeen;
1675 v.hashRate = m->hashRate; 4542 v.hashRate = m->hashRate;
1676 gWorkerStats.push_back(v); 4543 gWorkerStats.push_back(v);
1677 gStats.totalHashRate += m->hashRate; 4544 gStats.totalHashRate += m->hashRate;
1678 } 4545 }
1679 std::sort(gWorkerStats.begin(), gWorkerStats.end(), 4546 std::sort(gWorkerStats.begin(), gWorkerStats.end(),
1680 [](const PoolWorkerStatView& a, const PoolWorkerStatView& b) { 4547 [](const PoolWorkerStatView& a, const PoolWorkerStatView& b) {
1681 return a.roundShares != b.roundShares ? a.roundShares > b.roundShares 4548 return a.roundShares != b.roundShares ? a.roundShares > b.roundShares
1682 : a.lastSeen > b.lastSeen; 4549 : a.lastSeen > b.lastSeen;
1683 }); 4550 });
1684 4551
1685 gPayoutViews.clear(); 4552 // Scoped, not held to the end of the function. At function scope this
1686 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex); 4553 // lock covered the status-file write below, so the event loop did two
1687 for (const PendingPayout& pp : gPendingPayouts) { 4554 // fsyncs holding the mutex the payout thread takes to record that a
1688 PendingPayoutView pv; 4555 // miner has been paid.
1689 pv.matureAtHeight = pp.matureAtHeight; 4556 gPayoutViews.clear();
1690 pv.recipients = 0; 4557 {
1691 pv.totalAmount = 0; 4558 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
1692 for (auto& kv : pp.amounts) { 4559 for (const PendingPayout& pp : gPendingPayouts) {
1693 if (pp.paidTxids.count(kv.first)) 4560 PendingPayoutView pv;
1694 continue; 4561 pv.matureAtHeight = pp.matureAtHeight;
1695 pv.recipients++; 4562 pv.recipients = 0;
1696 pv.totalAmount += kv.second; 4563 pv.totalAmount = 0;
4564 for (auto& kv : pp.amounts) {
4565 if (pp.paidTxids.count(kv.first))
4566 continue;
4567 pv.recipients++;
4568 pv.totalAmount += kv.second;
4569 }
4570 if (pv.recipients > 0)
4571 gPayoutViews.push_back(pv);
4572 }
1697 } 4573 }
1698 if (pv.recipients > 0)
1699 gPayoutViews.push_back(pv);
1700 }
1701 4574
1702 int64 now = GetTime(); 4575 int64 now = GetTime();
1703 if (now - gLastPoolStatusWrite >= 10) { 4576 if (now - gLastPoolStatusWrite >= 10) {
1704 WritePoolStatusJsonLocked(); 4577 statusBlob = SerializePoolStatusLocked();
1705 gLastPoolStatusWrite = now; 4578 gLastPoolStatusWrite = now;
4579 fPublish = true;
4580 }
1706 } 4581 }
4582 // Every payout mutex is released here, and gStatsMutex with them. This is
4583 // where the two fsyncs happen, and no other thread is waiting behind them.
4584 if (fPublish)
4585 PublishPoolStatus(statusBlob);
1707} 4586}
1708 4587
1709// --------------------------------------------------------------------------- 4588// ---------------------------------------------------------------------------
@@ -1962,7 +4841,11 @@ static void AcceptOneFn(void* arg)
1962 LogPrint("pool", "[pool] accept thread started for relay %s\n", relay.c_str()); 4841 LogPrint("pool", "[pool] accept thread started for relay %s\n", relay.c_str());
1963 4842
1964 while (!fShutdown && gPoolRunning) { 4843 while (!fShutdown && gPoolRunning) {
1965 // Blocks until a miner connects to us at this relay 4844 // Advertise this relay as the pool's meeting node BEFORE registering,
4845 // so that miners which resolve our descriptor find a relay where the
4846 // pool (not the node's peer-accept) holds the registration.
4847 BtfSetActiveRelay(relay);
4848 // Returns as soon as the relay lists us -- registered, not yet paired.
1966 btf::RvSocket rv = btf::RvServiceRegister(host.c_str(), (unsigned short)port, pk); 4849 btf::RvSocket rv = btf::RvServiceRegister(host.c_str(), (unsigned short)port, pk);
1967 4850
1968 if (fShutdown || !gPoolRunning) { 4851 if (fShutdown || !gPoolRunning) {
@@ -1976,6 +4859,23 @@ static void AcceptOneFn(void* arg)
1976 continue; 4859 continue;
1977 } 4860 }
1978 4861
4862 // Block for a miner to actually arrive. RvServiceRegister used to do
4863 // this itself and returned with the relay's 0x01 pairing byte already
4864 // consumed; it no longer does. Without waiting here, BtfServiceWrap
4865 // reads that 0x01 as the first byte of the miner's 32-byte ephemeral
4866 // key -- every subsequent byte is shifted by one, so the X25519
4867 // handshake completes against a garbage shared secret (it has no
4868 // confirmation step to catch it), the tunnel is handed to the event
4869 // loop, and the miner's mining.subscribe never decrypts into a valid
4870 // line. The miner sees the pool close on it; the pool sees a peer that
4871 // never authorized.
4872 if (!btf::RvServiceWaitPaired(rv)) {
4873 btf::RvClose(rv);
4874 LogPrint("pool", "[pool] relay %s: dropped us before any miner dialed"
4875 " -- re-registering\n", relay.c_str());
4876 continue;
4877 }
4878
1979 // Echan handshake -- blocking but fast (local + relay round-trip) 4879 // Echan handshake -- blocking but fast (local + relay round-trip)
1980 btf_socket_t fd = btf::BtfServiceWrap(rv, sk); 4880 btf_socket_t fd = btf::BtfServiceWrap(rv, sk);
1981 if (fd == INVALID_SOCKET) { 4881 if (fd == INVALID_SOCKET) {
@@ -2062,8 +4962,14 @@ void ThreadRPCServer(void*)
2062 4962
2063 LoadPendingPayouts(); 4963 LoadPendingPayouts();
2064 LoadPoolRounds(); 4964 LoadPoolRounds();
4965 LoadPaying();
4966 // Must run before the payout thread starts, so nothing is resent that the
4967 // ledger already shows as sent, and every restored in-flight record is
4968 // armed for its one wallet probe before it is allowed to send.
4969 ReconcilePayouts();
2065 4970
2066 _beginthread(AcceptLoopFn, 0, NULL); 4971 _beginthread(AcceptLoopFn, 0, NULL);
4972 _beginthread(LocalAcceptFn, 0, NULL);
2067 _beginthread(ThreadBtfPoolAnnouncer, 0, NULL); 4973 _beginthread(ThreadBtfPoolAnnouncer, 0, NULL);
2068 _beginthread(PayoutThreadFn, 0, NULL); 4974 _beginthread(PayoutThreadFn, 0, NULL);
2069 4975
diff --git a/src/selftest.cpp b/src/selftest.cpp
index dc9936e..2ada8f3 100644
--- a/src/selftest.cpp
+++ b/src/selftest.cpp
@@ -19,6 +19,7 @@
19#include "wallet_sqlite.h" 19#include "wallet_sqlite.h"
20 20
21extern int RunPoolStratumSelfTest(); 21extern int RunPoolStratumSelfTest();
22extern int RunPayoutLedgerSelfTest();
22 23
23// Test results go to the terminal, not to debug.log. 24// Test results go to the terminal, not to debug.log.
24// 25//
@@ -3023,6 +3024,13 @@ static int RunNetMessageSelfTest()
3023 int nFail = 0; 3024 int nFail = 0;
3024 try 3025 try
3025 { 3026 {
3027 // A peer that has not yet been kept waiting must not look like one that
3028 // has been waiting since the epoch. This is the whole bug: the stall
3029 // clock lives in CNode, so only the constructor can start it at zero.
3030 CNode opened(INVALID_SOCKET, CAddress("127.0.0.1"));
3031 nFail += Check(opened.nMsgWaitSince == 0 && opened.nMsgWaitSize == 0,
3032 "a new peer starts with its stall clock stopped") ? 0 : 1;
3033
3026 CNode complete(INVALID_SOCKET, CAddress("127.0.0.1")); 3034 CNode complete(INVALID_SOCKET, CAddress("127.0.0.1"));
3027 complete.nVersion = VERSION; 3035 complete.nVersion = VERSION;
3028 complete.vRecv << CMessageHeader("ping", 0); 3036 complete.vRecv << CMessageHeader("ping", 0);
@@ -3037,20 +3045,38 @@ static int RunNetMessageSelfTest()
3037 nFail += Check(ProcessMessages(&partial), "an incomplete message returns cleanly") ? 0 : 1; 3045 nFail += Check(ProcessMessages(&partial), "an incomplete message returns cleanly") ? 0 : 1;
3038 nFail += Check(partial.vRecv.size() == nPartialBefore, 3046 nFail += Check(partial.vRecv.size() == nPartialBefore,
3039 "an incomplete message keeps one header in the buffer") ? 0 : 1; 3047 "an incomplete message keeps one header in the buffer") ? 0 : 1;
3040 nFail += Check(partial.nIncompleteMessageStart != 0, 3048 nFail += Check(partial.nMsgWaitSince != 0,
3041 "an incomplete message starts a timeout clock") ? 0 : 1; 3049 "an incomplete message starts the stall clock") ? 0 : 1;
3050 nFail += Check(partial.nMsgWaitSize == 0,
3051 "the stall clock records how much payload had arrived") ? 0 : 1;
3042 nFail += Check(!partial.fDisconnect, 3052 nFail += Check(!partial.fDisconnect,
3043 "a fresh incomplete message does not disconnect immediately") ? 0 : 1; 3053 "a fresh incomplete message does not disconnect immediately") ? 0 : 1;
3044 3054
3055 // Still arriving, just slowly: more payload is here than when the clock
3056 // was started, so the clock restarts however old it was. Timing the
3057 // wait instead of the stall would drop this peer.
3058 CNode slow(INVALID_SOCKET, CAddress("127.0.0.1"));
3059 slow.nVersion = VERSION;
3060 slow.vRecv << CMessageHeader("block", 100);
3061 slow.vRecv.insert(slow.vRecv.end(), 50, '\0');
3062 slow.nMsgWaitSince = GetTime() - BTF_MSG_WAIT_MAX_SECS - 1;
3063 slow.nMsgWaitSize = 10;
3064 ProcessMessages(&slow);
3065 nFail += Check(!slow.fDisconnect,
3066 "a peer still sending payload is not dropped") ? 0 : 1;
3067 nFail += Check(slow.nMsgWaitSize == 50,
3068 "progress restarts the stall clock") ? 0 : 1;
3069
3070 // Nothing more has arrived since the clock started, and the clock is
3071 // older than the limit.
3045 CNode stale(INVALID_SOCKET, CAddress("127.0.0.1")); 3072 CNode stale(INVALID_SOCKET, CAddress("127.0.0.1"));
3046 stale.nVersion = VERSION; 3073 stale.nVersion = VERSION;
3047 stale.vRecv << CMessageHeader("block", 100); 3074 stale.vRecv << CMessageHeader("block", 100);
3048 stale.nIncompleteMessageStart = GetTime() - BTF_INCOMPLETE_MESSAGE_TIMEOUT_SECS - 1; 3075 stale.nMsgWaitSince = GetTime() - BTF_MSG_WAIT_MAX_SECS - 1;
3049 stale.nIncompleteMessageSize = 100; 3076 stale.nMsgWaitSize = 0;
3050 stale.strIncompleteMessageCommand = "block";
3051 ProcessMessages(&stale); 3077 ProcessMessages(&stale);
3052 nFail += Check(stale.fDisconnect, 3078 nFail += Check(stale.fDisconnect,
3053 "a stale incomplete message disconnects the peer") ? 0 : 1; 3079 "a stalled incomplete message disconnects the peer") ? 0 : 1;
3054 3080
3055 CNode oversized(INVALID_SOCKET, CAddress("127.0.0.1")); 3081 CNode oversized(INVALID_SOCKET, CAddress("127.0.0.1"));
3056 oversized.nVersion = VERSION; 3082 oversized.nVersion = VERSION;
@@ -3521,6 +3547,8 @@ int RunSelfTest(const std::string& name)
3521 return RunConsensusLimitsSelfTest(); 3547 return RunConsensusLimitsSelfTest();
3522 if (name == "pool-stratum") 3548 if (name == "pool-stratum")
3523 return RunPoolStratumSelfTest(); 3549 return RunPoolStratumSelfTest();
3550 if (name == "payout-ledger")
3551 return RunPayoutLedgerSelfTest();
3524 if (name == "parse-money") 3552 if (name == "parse-money")
3525 return RunParseMoneySelfTest(); 3553 return RunParseMoneySelfTest();
3526 if (name == "socks5-proxy") 3554 if (name == "socks5-proxy")
@@ -3529,6 +3557,6 @@ int RunSelfTest(const std::string& name)
3529 return RunManagedTorSelfTest(); 3557 return RunManagedTorSelfTest();
3530 3558
3531 printf("Unknown self-test '%s'\n", name.c_str()); 3559 printf("Unknown self-test '%s'\n", name.c_str());
3532 printf("Known self-tests: wallet-keypool, wallet-hd, wallet-format, wallet-storage-sanity, db-env-reopen, wallet-sqlite, wallet-sqlite-migration, wallet-crypto, wallet-encrypt, wallet-sqlite-encrypt, wallet-backend-default, wallet-convert, wallet-portability, net-message, consensus-limits, pool-stratum, parse-money, socks5-proxy, managed-tor\n"); 3560 printf("Known self-tests: wallet-keypool, wallet-hd, wallet-format, wallet-storage-sanity, db-env-reopen, wallet-sqlite, wallet-sqlite-migration, wallet-crypto, wallet-encrypt, wallet-sqlite-encrypt, wallet-backend-default, wallet-convert, wallet-portability, net-message, consensus-limits, pool-stratum, payout-ledger, parse-money, socks5-proxy, managed-tor\n");
3533 return 1; 3561 return 1;
3534} 3562}
diff --git a/src/test_btfaddr.cpp b/src/test_btfaddr.cpp
index 8de426e..9d33d63 100644
--- a/src/test_btfaddr.cpp
+++ b/src/test_btfaddr.cpp
@@ -133,12 +133,23 @@ int main()
133 std::string d3 = btf::SignDescriptor(ctx, sk, ENC_HEX, "node-1", 5, ONION); 133 std::string d3 = btf::SignDescriptor(ctx, sk, ENC_HEX, "node-1", 5, ONION);
134 json j3 = json::parse(d3); 134 json j3 = json::parse(d3);
135 j3["onion"] = "bcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcde.onion:8433"; 135 j3["onion"] = "bcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcde.onion:8433";
136 CHECK(btf::VerifyDescriptor(ctx, j3.dump(), pk, o) && o.onion.empty(), 136 CHECK(!btf::VerifyDescriptor(ctx, j3.dump(), pk, o),
137 "tampered onion ignored rather than trusted"); 137 "tampered onion rejects the descriptor");
138 json j4 = json::parse(d3); 138 json j4 = json::parse(d3);
139 j4["sig3"] = std::string(128, '0'); 139 j4["sig3"] = std::string(128, '0');
140 CHECK(btf::VerifyDescriptor(ctx, j4.dump(), pk, o) && o.onion.empty(), 140 CHECK(!btf::VerifyDescriptor(ctx, j4.dump(), pk, o),
141 "bad onion signature ignored rather than trusted"); 141 "bad onion signature rejects the descriptor");
142 json j5 = json::parse(d3);
143 j5.erase("sig3");
144 CHECK(!btf::VerifyDescriptor(ctx, j5.dump(), pk, o),
145 "unsigned onion rejects the descriptor");
146 // Removing both fields still verifies: the result is indistinguishable
147 // from a peer that never had an onion endpoint. That residual strip
148 // needs the onion bound into the primary sig, an upstream change.
149 json j6 = json::parse(d3);
150 j6.erase("onion"); j6.erase("sig3");
151 CHECK(btf::VerifyDescriptor(ctx, j6.dump(), pk, o) && o.onion.empty(),
152 "full strip indistinguishable from no-onion (documented residual)");
142 } 153 }
143 154
144 printf("full_resolve_chain_from_address\n"); 155 printf("full_resolve_chain_from_address\n");
diff --git a/src/test_rvaccept.cpp b/src/test_rvaccept.cpp
new file mode 100644
index 0000000..afa9046
--- /dev/null
+++ b/src/test_rvaccept.cpp
@@ -0,0 +1,180 @@
1// Local regression test for the pool's rendezvous accept path.
2//
3// Reproduces the exact failure that took remote miners off the pool: since
4// upstream #30, RvServiceRegister returns as soon as the relay lists the
5// service and no longer consumes the relay's 0x01 pairing byte. A service that
6// goes straight from RvServiceRegister to BtfServiceWrap therefore reads that
7// 0x01 as the first byte of the client's 32-byte ephemeral key, shifting the
8// whole stream by one. X25519 has no confirmation step, so the handshake
9// "succeeds" against a garbage shared secret and the corruption only shows up
10// as a tunnel that never carries a valid byte.
11//
12// Build: g++ -std=gnu++14 -O2 -pthread test_rvaccept.cpp btftunnel.cpp btfrv.cpp \
13// btfchan.cpp -lsodium -o test_rvaccept
14//
15// Build: g++ -std=gnu++14 -O2 -pthread test_rvaccept.cpp btftunnel.cpp \
16// btfrv.cpp btfchan.cpp -lsodium -o test_rvaccept
17//
18// Run against a bitflashd relay already listening on 127.0.0.1:<port>.
19// test_rvaccept broken <port> -- the pre-fix pool code path, expected to FAIL
20// test_rvaccept fixed <port> -- with RvServiceWaitPaired, expected to PASS
21
22#include "btftunnel.h"
23#include "btfrv.h"
24#include "btfchan.h"
25
26#include <sys/socket.h>
27#include <sys/time.h>
28#include <unistd.h>
29#include <cstdio>
30#include <cstdlib>
31#include <cstring>
32#include <string>
33#include <vector>
34#include <thread>
35#include <chrono>
36
37using namespace btf;
38
39static const char* HOST = "127.0.0.1";
40static unsigned short PORT = 18446;
41static const char* MSG = "mining.subscribe stand-in payload";
42
43static void SetTimeout(int fd, int secs)
44{
45 struct timeval tv;
46 tv.tv_sec = secs;
47 tv.tv_usec = 0;
48 setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
49 setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
50}
51
52static bool RecvN(int fd, void* buf, int n)
53{
54 char* p = (char*)buf;
55 int off = 0;
56 while (off < n) {
57 int r = recv(fd, p + off, n - off, 0);
58 if (r <= 0) return false;
59 off += r;
60 }
61 return true;
62}
63
64static bool SendN(int fd, const void* buf, int n)
65{
66 const char* p = (const char*)buf;
67 int off = 0;
68 while (off < n) {
69 int r = send(fd, p + off, n - off, 0);
70 if (r <= 0) return false;
71 off += r;
72 }
73 return true;
74}
75
76static bool g_serviceEcho = false;
77static std::string g_serviceSaw;
78
79// The pool's AcceptOneFn, in both shapes.
80static void ServiceThread(bool fWaitPaired,
81 std::vector<unsigned char> id,
82 std::vector<unsigned char> enc_sk,
83 int len)
84{
85 RvSocket rv = RvServiceRegister(HOST, PORT, id.data());
86 if (rv == RV_INVALID) { printf(" service: RvServiceRegister failed\n"); return; }
87
88 if (fWaitPaired) {
89 if (!RvServiceWaitPaired(rv)) {
90 printf(" service: RvServiceWaitPaired failed\n");
91 RvClose(rv);
92 return;
93 }
94 printf(" service: paired (0x01 consumed by RvServiceWaitPaired)\n");
95 } else {
96 printf(" service: skipped RvServiceWaitPaired (pre-fix behaviour)\n");
97 }
98
99 btf_socket_t app = BtfServiceWrap(rv, enc_sk.data());
100 if (app == INVALID_SOCKET) { printf(" service: BtfServiceWrap failed\n"); return; }
101 printf(" service: BtfServiceWrap returned a tunnel\n");
102
103 SetTimeout(app, 5);
104 std::vector<char> buf(len);
105 if (!RecvN(app, buf.data(), len)) {
106 printf(" service: never received the client's plaintext\n");
107 close(app);
108 return;
109 }
110 g_serviceSaw.assign(buf.data(), len);
111 g_serviceEcho = SendN(app, buf.data(), len);
112 close(app);
113}
114
115int main(int argc, char* argv[])
116{
117 bool fWaitPaired = (argc >= 2 && strcmp(argv[1], "fixed") == 0);
118 if (argc >= 3) PORT = (unsigned short)atoi(argv[2]);
119
120 if (!TunnelInit()) { printf("TunnelInit failed\n"); return 2; }
121
122 unsigned char enc_pk[32], enc_sk[32];
123 ChanKeypair(enc_pk, enc_sk);
124
125 std::vector<unsigned char> id(32);
126 for (int i = 0; i < 32; i++) id[i] = (unsigned char)(i * 7 + 13);
127
128 const std::string msg(MSG);
129 const int len = (int)msg.size();
130
131 printf(" mode: %s relay: %s:%u\n", fWaitPaired ? "fixed" : "broken", HOST, PORT);
132
133 std::thread svc(ServiceThread, fWaitPaired, id,
134 std::vector<unsigned char>(enc_sk, enc_sk + 32), len);
135 std::this_thread::sleep_for(std::chrono::milliseconds(500));
136
137 int nFail = 0;
138
139 btf_socket_t app = BtfClientTunnel(HOST, PORT, id.data(), enc_pk);
140 if (app == INVALID_SOCKET) {
141 printf(" client: BtfClientTunnel failed\n");
142 nFail++;
143 } else {
144 printf(" client: tunnel established\n");
145 SetTimeout(app, 5);
146 if (!SendN(app, msg.data(), len)) { printf(" client: send failed\n"); nFail++; }
147 std::vector<char> echo(len);
148 if (!RecvN(app, echo.data(), len)) {
149 printf(" client: no echo came back\n");
150 nFail++;
151 } else if (std::string(echo.data(), len) != msg) {
152 printf(" client: echo did not match what was sent\n");
153 nFail++;
154 } else {
155 printf(" client: round-tripped the payload intact\n");
156 }
157 close(app);
158 }
159
160 svc.join();
161
162 if (g_serviceSaw != msg) {
163 printf(" service did not see the correct plaintext (saw %d bytes)\n",
164 (int)g_serviceSaw.size());
165 nFail++;
166 }
167 if (!g_serviceEcho) { printf(" service could not echo\n"); nFail++; }
168
169 bool fPassed = (nFail == 0);
170 printf(" RESULT: %s\n", fPassed ? "PASS -- tunnel carries plaintext end to end"
171 : "FAIL -- tunnel never carried a valid byte");
172
173 // A broken run is the expected outcome pre-fix, so exit 0 when the
174 // observed behaviour matches the expectation for the mode under test.
175 bool fAsExpected = (fWaitPaired ? fPassed : !fPassed);
176 printf(" EXPECTED %s -> %s\n",
177 fWaitPaired ? "PASS" : "FAIL",
178 fAsExpected ? "as expected" : "UNEXPECTED");
179 return fAsExpected ? 0 : 1;
180}
diff --git a/src/tor.cpp b/src/tor.cpp
index 20b4577..fea6912 100644
--- a/src/tor.cpp
+++ b/src/tor.cpp
@@ -827,17 +827,26 @@ void BtfStopManagedTor()
827 memset(&g_managedTorProcess, 0, sizeof(g_managedTorProcess)); 827 memset(&g_managedTorProcess, 0, sizeof(g_managedTorProcess));
828 } 828 }
829#else 829#else
830 if (g_managedTorPid > 0) 830 // Snapshot the pid under the lock. The watchdog's ManagedTorProcessAlive
831 // reaps the same child concurrently and resets g_managedTorPid under
832 // cs_managedTor; reading the global unlocked here could leave us waiting
833 // on ECHILD and then SIGKILLing a recycled PID.
834 pid_t pid = -1;
835 CRITICAL_BLOCK(cs_managedTor)
836 pid = g_managedTorPid;
837 if (pid > 0)
831 { 838 {
832 std::string shutdownErr; 839 std::string shutdownErr;
833 if (!RequestManagedTorShutdown(shutdownErr)) 840 if (!RequestManagedTorShutdown(shutdownErr))
834 kill(g_managedTorPid, SIGTERM); 841 kill(pid, SIGTERM);
835 bool fExited = false; 842 bool fExited = false;
836 for (int i = 0; i < 100; i++) 843 for (int i = 0; i < 100; i++)
837 { 844 {
838 int status = 0; 845 int status = 0;
839 pid_t r = waitpid(g_managedTorPid, &status, WNOHANG); 846 pid_t r = waitpid(pid, &status, WNOHANG);
840 if (r == g_managedTorPid) 847 // ECHILD means the watchdog already reaped it -- that is exit,
848 // not a reason to signal anything.
849 if (r == pid || (r == -1 && errno == ECHILD))
841 { 850 {
842 fExited = true; 851 fExited = true;
843 break; 852 break;
@@ -846,10 +855,12 @@ void BtfStopManagedTor()
846 } 855 }
847 if (!fExited) 856 if (!fExited)
848 { 857 {
849 kill(g_managedTorPid, SIGKILL); 858 kill(pid, SIGKILL);
850 waitpid(g_managedTorPid, NULL, 0); 859 waitpid(pid, NULL, 0);
851 } 860 }
852 g_managedTorPid = -1; 861 CRITICAL_BLOCK(cs_managedTor)
862 if (g_managedTorPid == pid)
863 g_managedTorPid = -1;
853 } 864 }
854#endif 865#endif
855 CRITICAL_BLOCK(cs_managedTor) 866 CRITICAL_BLOCK(cs_managedTor)
diff --git a/src/util.h b/src/util.h
index d95d3ae..d72518b 100644
--- a/src/util.h
+++ b/src/util.h
@@ -8,6 +8,13 @@
8#include <cxxabi.h> 8#include <cxxabi.h>
9#endif 9#endif
10 10
11// For AvailableCoreCount() below: sched_getaffinity on Linux, and
12// hardware_concurrency as the fallback everywhere else.
13#include <thread>
14#ifdef __linux__
15#include <sched.h>
16#endif
17
11#if defined(_MSC_VER) || defined(__BORLANDC__) 18#if defined(_MSC_VER) || defined(__BORLANDC__)
12typedef __int64 int64; 19typedef __int64 int64;
13typedef unsigned __int64 uint64; 20typedef unsigned __int64 uint64;
@@ -144,6 +151,34 @@ int64 GetTime();
144int64 GetAdjustedTime(); 151int64 GetAdjustedTime();
145void AddTimeData(unsigned int ip, int64 nTime); 152void AddTimeData(unsigned int ip, int64 nTime);
146 153
154// Cores this process is actually allowed to run on.
155//
156// std::thread::hardware_concurrency() reports the MACHINE's core count, which
157// is not the same number inside a container. With --cpuset-cpus 0-5 on a
158// 12-core host it still answers 12, so every caller that sized a thread pool
159// from it oversubscribed by 2x and then wondered why the box was thrashing:
160// the pool node started 11 miner threads on 6 permitted CPUs, and a burst
161// miner asked for 2 cores filled the ~2 GB RandomX dataset with 12 threads.
162//
163// sched_getaffinity reports what the cpuset permits, which is the number every
164// one of those callers meant. Falls back to hardware_concurrency where the
165// call is unavailable, and never returns 0 -- callers divide by this.
166inline unsigned int AvailableCoreCount()
167{
168#ifdef __linux__
169 cpu_set_t cpuset;
170 CPU_ZERO(&cpuset);
171 if (sched_getaffinity(0, sizeof(cpuset), &cpuset) == 0)
172 {
173 int nAllowed = CPU_COUNT(&cpuset);
174 if (nAllowed > 0)
175 return (unsigned int)nAllowed;
176 }
177#endif
178 unsigned int nCores = std::thread::hardware_concurrency();
179 return nCores > 0 ? nCores : 1;
180}
181
147 182
148 183
149 184