upleb.uk

Public git repos — served from a NIP-34 GRASP relay at git.upleb.uk

summaryrefslogtreecommitdiff
path: root/src/rpc.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/rpc.cpp')
-rw-r--r--src/rpc.cpp4317
1 files changed, 4031 insertions, 286 deletions
diff --git a/src/rpc.cpp b/src/rpc.cpp
index 813f1d1..3827cd5 100644
--- a/src/rpc.cpp
+++ b/src/rpc.cpp
@@ -6,8 +6,8 @@
6// Architecture: 6// Architecture:
7// PoolEventLoop -- single thread, owns all miner fds, all job/share/payout 7// PoolEventLoop -- single thread, owns all miner fds, all job/share/payout
8// state. No locks needed on any of that data. 8// state. No locks needed on any of that data.
9// AcceptOneFn -- one thread per relay. Blocks on RvServiceRegister + 9// AcceptOneFn -- one thread per relay. Registers, waits to be paired,
10// BtfServiceWrap, pushes finished socket to gReadyQueue. 10// wraps the tunnel, pushes finished socket to gReadyQueue.
11// AcceptLoopFn -- spawns AcceptOneFn threads, re-polls relay list every 11// AcceptLoopFn -- spawns AcceptOneFn threads, re-polls relay list every
12// 60s so dynamically discovered relays get picked up. 12// 60s so dynamically discovered relays get picked up.
13// 13//
@@ -31,10 +31,15 @@
31#include <netinet/in.h> 31#include <netinet/in.h>
32#include <arpa/inet.h> 32#include <arpa/inet.h>
33#include <unistd.h> 33#include <unistd.h>
34#include <fcntl.h>
35#include <dirent.h>
36#else
37#include <io.h>
34#endif 38#endif
35 39
36#include <atomic> 40#include <atomic>
37#include <mutex> 41#include <mutex>
42#include <thread>
38#include <map> 43#include <map>
39#include <set> 44#include <set>
40#include <deque> 45#include <deque>
@@ -110,15 +115,28 @@ static uint256 ShareTargetFromDifficulty(double difficulty)
110{ 115{
111 if (difficulty <= 0.0) difficulty = 1.0; 116 if (difficulty <= 0.0) difficulty = 1.0;
112 static const uint256 diff1 = CBigNum(bnProofOfWorkLimit).getuint256(); 117 static const uint256 diff1 = CBigNum(bnProofOfWorkLimit).getuint256();
118 // Representability floor. shareTarget = diff1 / difficulty, and diff1 is
119 // ~uint256(0) >> 12, i.e. a hair under 2^244 -- so the true target stops
120 // fitting in a uint256 once difficulty reaches 2^-12 (= 1/4096), and the
121 // fixed-point divide below breaks even earlier in a second way: the
122 // (uint64)(difficulty * 65536.0) divisor truncates to zero for any
123 // difficulty under 2^-16. Both thresholds sit far below VARDIFF_MIN
124 // (0.001), so live operation never comes near this, but a caller that
125 // bypasses vardiff (or a future VARDIFF_MIN) must not divide by zero or
126 // hand back getuint256()'s low-256-bit truncation of an oversized
127 // quotient -- a wrapped target is not merely wrong, it is arbitrarily
128 // HARDER than difficulty 1. At these difficulties every hash is meant to
129 // be a share, so the honest answer is the easiest representable target.
130 // (This replaces a dead branch that read `if (t < diff1 && difficulty <
131 // 1.0)`: t = diff1/difficulty, so difficulty < 1.0 implies t > diff1 and
132 // the two conditions could never both hold.)
133 if (difficulty < 1.0 / 4096.0) return ~uint256(0);
113 // shareTarget = diff1 / difficulty 134 // shareTarget = diff1 / difficulty
114 CBigNum bn; bn.setuint256(diff1); 135 CBigNum bn; bn.setuint256(diff1);
115 // Multiply by 2^16 to keep precision, divide, shift back 136 // Multiply by 2^16 to keep precision, divide, shift back
116 bn <<= 16; 137 bn <<= 16;
117 bn /= (uint64)(difficulty * 65536.0); 138 bn /= (uint64)(difficulty * 65536.0);
118 uint256 t = bn.getuint256(); 139 return bn.getuint256();
119 // Cap at all-ones (maximum, easiest possible)
120 if (t < diff1 && difficulty < 1.0) t = ~uint256(0);
121 return t;
122} 140}
123 141
124// Expected number of hashes to find one share at a given Stratum difficulty, 142// Expected number of hashes to find one share at a given Stratum difficulty,
@@ -143,12 +161,104 @@ static bool RebuildJob()
143 return false; 161 return false;
144 } 162 }
145 163
164 // Don't issue jobs while we are behind the network: mining against a
165 // stale tip produces blocks the network will orphan, and payout
166 // entries that can never be paid (their coinbases never exist on the
167 // main chain). Require at least one peer, and require that no peer
168 // claims a longer chain than ours.
169 if (vNodes.empty()) {
170 LogPrint("pool", "RebuildJob: no peers yet -- not issuing jobs\n");
171 return false;
172 }
173 // This used to compare the tip's timestamp against 30 minutes, which
174 // was wrong in both directions and cost real money.
175 //
176 // It cannot tell "we are not synced" from "the network is slow". A
177 // pool mining its own fork keeps its tip timestamp fresh forever, so
178 // the check passed for the whole of heights 3966-4085 -- 120
179 // consecutive blocks, every one of them orphaned, 5890.50 BTF of
180 // payouts voided. Then, once the pool stopped finding blocks, the
181 // same check refused to issue jobs against a tip that was perfectly
182 // valid and was the network's own best, and the pool -- roughly 90%
183 // of the hashrate -- could not restart the chain it was the majority
184 // of. Stale tip, so no jobs; no jobs, so the tip stays stale.
185 //
186 // The height peers announce in their version message answers the
187 // question the age check was only guessing at. Median, so one peer
188 // claiming an absurd height cannot stall us, and one block of slack
189 // to match WarnIfBehind, because somebody is always mid-relay. A
190 // peer that predates the announcement reports -1 and is ignored; if
191 // nobody has told us anything we mine our best chain, which is what
192 // the node would do without a pool attached.
193 int nPeerHeight = GetPeerMedianHeight();
194 if (nPeerHeight >= 0 && nBestHeight < nPeerHeight - 1) {
195 LogPrint("pool", "RebuildJob: %d blocks behind the network "
196 "(height %d, peers report %d) -- not issuing jobs\n",
197 nPeerHeight - nBestHeight, nBestHeight, nPeerHeight);
198 return false;
199 }
200
201 // The comparison above is blind in the one case that matters most.
202 //
203 // When the node goes deaf -- peers still in vNodes, nothing arriving --
204 // no peer announces anything, GetPeerMedianHeight() returns -1, and the
205 // fallback ("nobody has told us anything, so mine our best chain") let
206 // the pool build a private fork for as long as the deafness lasted.
207 // That fallback is right for a node that has simply not been told yet
208 // and wrong for a node that has stopped being able to hear, and a
209 // height nobody sent cannot tell those apart. Time can.
210 //
211 // Deliberately not behind a debug category, and deliberately not a
212 // silent refusal: an operator whose pool has stopped issuing jobs needs
213 // the reason in the log. /poolmineblind is the escape hatch for a node
214 // that really is alone -- without one, a genuinely isolated pool that is
215 // the majority of the hashrate could never restart its own chain, which
216 // is the deadlock the tip-age check used to cause.
217 if (nPeerHeight < 0 && !fPoolMineBlind) {
218 int64 nAge = GetPeerHeightAgeSecs();
219 if (nAge >= BTF_NO_PEER_HEIGHT_MAX_SECS) {
220 static int64 nLastSaid = 0;
221 if (GetTime() - nLastSaid > 60) {
222 nLastSaid = GetTime();
223 printf("RebuildJob: no peer has reported a height for %d minutes "
224 "-- not issuing jobs (our height %d, %d peers connected). "
225 "This node is deaf; mining now would fork it. "
226 "Pass /poolmineblind to override.\n",
227 (int)(nAge / 60), nBestHeight, (int)vNodes.size());
228 }
229 return false;
230 }
231 }
232
146 CBlockIndex* pindexPrev = pindexBest; 233 CBlockIndex* pindexPrev = pindexBest;
147 unsigned int nBits = GetNextWorkRequired(pindexPrev); 234 unsigned int nBits = GetNextWorkRequired(pindexPrev);
148 235
149 // From the pool, so the template's payout key is in wallet.dat before 236 // From the pool, so the template's payout key is in wallet.dat before
150 // any miner is handed work against it. 237 // any miner is handed work against it.
151 vector<unsigned char> vchPubKey = GetKeyFromPool(); 238 vector<unsigned char> vchPubKey = GetKeyFromPool();
239 if (vchPubKey.empty()) {
240 // Empty means the wallet could not produce a key: encrypted and
241 // locked, or key generation itself failed. Serialised, an empty
242 // vector is a zero-length push, so the coinbase would become
243 // `<empty> OP_CHECKSIG` -- a script nobody can satisfy. Every
244 // block mined against such a job burns its whole subsidy, while
245 // QueuePayouts still records the full obligation and the miners
246 // are then paid out of coin that block never contributed.
247 // BitcoinMiner refuses to mine in this state; refuse to hand the
248 // pool's miners work in it too. printf and not LogPrint for the
249 // reason given above the deaf-node refusal, and throttled the
250 // same way -- the no-job watchdog retries every few seconds for
251 // as long as the condition lasts.
252 static int64 nLastSaidNoKey = 0;
253 if (GetTime() - nLastSaidNoKey > 60) {
254 nLastSaidNoKey = GetTime();
255 printf("RebuildJob: no wallet key available for the coinbase "
256 "(wallet locked, or key generation failed -- the "
257 "GetKeyFromPool line above says which) -- not issuing "
258 "jobs\n");
259 }
260 return false;
261 }
152 static std::atomic<uint32_t> sExtra{0}; 262 static std::atomic<uint32_t> sExtra{0};
153 uint32_t extraNonce = ++sExtra; 263 uint32_t extraNonce = ++sExtra;
154 264
@@ -193,8 +303,29 @@ static bool RebuildJob()
193 map<uint256, CTxIndex> pool; 303 map<uint256, CTxIndex> pool;
194 vector<char> vfAlreadyAdded(mapTransactions.size()); 304 vector<char> vfAlreadyAdded(mapTransactions.size());
195 bool fFoundSomething = true; 305 bool fFoundSomething = true;
196 unsigned int sz = 0; 306 // The cap is MAX_BLOCK_SIZE_GEN, not MAX_SIZE/2. MAX_SIZE is
197 while (fFoundSomething && sz < MAX_SIZE / 2) 307 // the 32 MB serialisation ceiling, so half of it is 16 MB --
308 // this builder would happily assemble a template that its own
309 // CheckBlock() rejects at the 1 MB consensus cap. This is the
310 // pool's job builder, so a block over the cap is not one
311 // wasted solve, it is every miner on the pool working a
312 // template the network will reject.
313 //
314 // The inner break used to bound the overshoot to one
315 // transaction per pass. One transaction is still too many:
316 // nothing caps how big a single transaction may be, so that
317 // bound was worth up to MAX_SIZE. Asking whether each one fits
318 // before appending it bounds the overshoot at zero instead.
319 //
320 // Seeded with the header, tx count and coinbase, because
321 // CheckBlock() weighs the whole serialized block.
322 unsigned int sz = ::GetSerializeSize(block, SER_DISK);
323 // And the sigop budget, which CheckBlock() enforces and this
324 // builder never counted against. Same counting function
325 // CheckBlock() sums, seeded from the coinbase because it sums
326 // over all of vtx.
327 unsigned int nSigOps = block.vtx[0].GetSigOpCount();
328 while (fFoundSomething && sz < MAX_BLOCK_SIZE_GEN)
198 { 329 {
199 fFoundSomething = false; 330 fFoundSomething = false;
200 unsigned int n = 0; 331 unsigned int n = 0;
@@ -205,13 +336,24 @@ static bool RebuildJob()
205 continue; 336 continue;
206 CTransaction& tx = (*mi).second; 337 CTransaction& tx = (*mi).second;
207 if (tx.IsCoinBase() || !tx.IsFinal()) continue; 338 if (tx.IsCoinBase() || !tx.IsFinal()) continue;
339 // Skip rather than stop: one transaction too big for
340 // the room left does not mean the next is. Ordered
341 // before ConnectInputs because ConnectInputs credits
342 // nFees by reference on success, and a transaction we
343 // then decline would leave the coinbase claiming a fee
344 // this block does not collect.
345 unsigned int nTxSize = ::GetSerializeSize(tx, SER_NETWORK);
346 unsigned int nTxSigOps = tx.GetSigOpCount();
347 if (!BlockTemplateFits(sz, nSigOps, nTxSize, nTxSigOps))
348 continue;
208 map<uint256, CTxIndex> tmp(pool); 349 map<uint256, CTxIndex> tmp(pool);
209 if (!tx.ConnectInputs(txdb, tmp, CDiskTxPos(1,1,1), 0, nFees, 350 if (!tx.ConnectInputs(txdb, tmp, CDiskTxPos(1,1,1), 0, nFees,
210 false, true, tx.GetMinFee(block.vtx.size() < 100))) 351 false, true, tx.GetMinFee(block.vtx.size() < 100)))
211 continue; 352 continue;
212 swap(pool, tmp); 353 swap(pool, tmp);
213 block.vtx.push_back(tx); 354 block.vtx.push_back(tx);
214 sz += ::GetSerializeSize(tx, SER_NETWORK); 355 sz += nTxSize;
356 nSigOps += nTxSigOps;
215 vfAlreadyAdded[n] = true; 357 vfAlreadyAdded[n] = true;
216 fFoundSomething = true; 358 fFoundSomething = true;
217 } 359 }
@@ -251,17 +393,45 @@ static bool RebuildJob()
251// Payouts -- event loop thread only 393// Payouts -- event loop thread only
252// --------------------------------------------------------------------------- 394// ---------------------------------------------------------------------------
253struct PendingPayout { 395struct PendingPayout {
254 std::string id; 396 // The block that funds this payout, identified by hash and not by height
255 int blockHeight; 397 // alone. A reorg replaces the block at a height without changing the
256 int matureAtHeight; 398 // height, so a height-keyed obligation outlives the coinbase that was
399 // supposed to pay it: a 280-block reorg once left ~14.4k BTF of debt
400 // owed against blocks that no longer existed, which then drained the
401 // wallet paying fork-era miners out of real-chain coin.
402 //
403 // Upstream keys these on the string "block-<height>". That is the same
404 // bug this fork already paid for, and height keying additionally lets
405 // FindRecordedPoolPayout match the *previous* round's transaction after a
406 // reorg and silently skip a miner, so their `id` is deliberately dropped.
407 int blockHeight = 0;
408 // Must be explicitly zeroed: uint256's default constructor is empty
409 // (uint256.h:514), so leaving this unassigned gives every ledger entry
410 // loaded from an older file a garbage hash, which then matches no block
411 // and gets voided as "orphaned". That mistake voided a live payout.
412 uint256 blockHash = 0; // zero = entry predates hash keying
413 int matureAtHeight = 0;
257 std::map<std::string, int64> amounts; // address -> satoshis 414 std::map<std::string, int64> amounts; // address -> satoshis
415 // From upstream: which recipients already have a transaction. This is what
416 // lets an entry stay on disk until every address in `amounts` is paid,
417 // instead of being erased the moment the batch is handed off.
258 std::map<std::string, std::string> paidTxids; // address -> txid 418 std::map<std::string, std::string> paidTxids; // address -> txid
259 int64 nextAttempt; 419 // Retry and backoff state does NOT live here. It used to be declared here
260 420 // as well as on PayingRecord, and the copy on this side was never written:
261 PendingPayout() : blockHeight(0), matureAtHeight(0), nextAttempt(0) {} 421 // `retryAfter` was declared, serialised, loaded and tested at the flush
422 // gate, and the only assignment to it anywhere was `= 0`, so the gate could
423 // not fire. `retries` and `deferrals` were the same story. The real state
424 // is per (round, recipient) -- which is the granularity a payment can fail
425 // at -- and it lives on PayingRecord, which is now what persists it.
262}; 426};
263static std::vector<PendingPayout> gPendingPayouts; 427static std::vector<PendingPayout> gPendingPayouts;
264static std::mutex gPayoutLedgerMutex; 428static std::mutex gPayoutLedgerMutex;
429// Bumped on every serialisation, under gPayoutLedgerMutex. See PublishVersioned.
430static uint64 gPayoutLedgerSeq = 0;
431// Serialises writers of pending_payouts.json, and guards the sequence number
432// of the image currently on disk. Never taken while a data mutex is held.
433static std::mutex gPayoutLedgerFileMutex;
434static uint64 gPayoutLedgerPublishedSeq = 0;
265 435
266struct PoolRoundProof { 436struct PoolRoundProof {
267 std::string id; 437 std::string id;
@@ -282,6 +452,12 @@ struct PoolRoundProof {
282 std::map<std::string, uint64> shares; 452 std::map<std::string, uint64> shares;
283 std::map<std::string, int64> payouts; 453 std::map<std::string, int64> payouts;
284 std::map<std::string, std::string> paidTxids; 454 std::map<std::string, std::string> paidTxids;
455 // Recipients the ledger has written off, and why. Without this the public
456 // round proof goes on advertising an address as owed with "txid": null
457 // forever, while the ledger has already dropped the obligation -- the two
458 // records of the same round permanently disagreeing, with the operator
459 // holding the money.
460 std::map<std::string, std::string> forfeited; // address -> reason
285 461
286 PoolRoundProof() 462 PoolRoundProof()
287 : blockHeight(0), matureAtHeight(0), createdAt(0), feePercent(0.0), 463 : blockHeight(0), matureAtHeight(0), createdAt(0), feePercent(0.0),
@@ -290,6 +466,9 @@ struct PoolRoundProof {
290}; 466};
291static std::vector<PoolRoundProof> gPoolRounds; 467static std::vector<PoolRoundProof> gPoolRounds;
292static std::mutex gPoolRoundsMutex; 468static std::mutex gPoolRoundsMutex;
469static uint64 gPoolRoundsSeq = 0; // guarded by gPoolRoundsMutex
470static std::mutex gPoolRoundsFileMutex;
471static uint64 gPoolRoundsPublishedSeq = 0; // guarded by gPoolRoundsFileMutex
293static const size_t MAX_POOL_ROUNDS = 200; 472static const size_t MAX_POOL_ROUNDS = 200;
294 473
295static std::string PayoutFilePath() 474static std::string PayoutFilePath()
@@ -304,9 +483,302 @@ static std::string PoolRoundsFilePath()
304 return GetAppDir() + "/pool_rounds.json"; 483 return GetAppDir() + "/pool_rounds.json";
305} 484}
306 485
307static std::string PayoutIdForBlock(int blockHeight) 486// ---------------------------------------------------------------------------
487// Atomic, ordered publication of the payout state files
488// ---------------------------------------------------------------------------
489//
490// Every one of these files is a full dump of an in-memory structure, written
491// tmp-then-rename so a crash cannot truncate it. Two things have to be true
492// for that to actually be safe, and only the first was:
493//
494// 1. the rename must be atomic (it is), and
495// 2. exactly one thread at a time may be writing the temp file, and a write
496// must never publish an OLDER image over a newer one.
497//
498// (2) was broken for pending_payouts.json. The event loop wrote it with
499// gPayoutLedgerMutex held, the payout thread wrote it with the mutex
500// released, and nothing serialised the two: both opened the same
501// "<path>.tmp" with independent handles and both renamed it. The interleaved
502// file that lands is either a mixture of two images (LoadPendingPayouts then
503// throws mid-loop and comes up with a nearly empty ledger -- every obligation
504// not yet exploded, gone) or a whole newly-queued round silently overwritten
505// by an older snapshot that predates it.
506//
507// The fix is a per-file write mutex, held only across the I/O and never
508// across a data-structure mutex, plus a version on the snapshot. Serialising
509// happens under the data mutex and stamps a monotonically increasing sequence
510// number; publishing happens with the data mutex released and refuses any
511// blob whose sequence number is not newer than what is already on disk. So
512// writers can queue in any order and finish in any order -- the file only
513// ever moves forwards.
514//
515// The other half of the rule is that no thread waits for I/O while holding
516// gPayoutLedgerMutex, gPayoutExecMutex or gPoolRoundsMutex, which is the
517// contention the merge was right to remove from the event loop. That was
518// claimed here before it was true: pool_status.json is written by the same
519// WriteFileAtomic and RefreshStats held gPayoutLedgerMutex at function scope
520// across the call, so once the writes gained fsync() the event loop was doing
521// two of them every ten seconds with the payout thread's mutex held -- worse
522// than the buffered write the merge replaced. The status file now goes through
523// the same serialise-under-the-lock / publish-outside-it split as the ledgers,
524// and the rule holds for all four files. It is checked, not asserted: see the
525// "no payout mutex is held across a publish" section of the payout-ledger
526// self-test.
527
528// A serialised file image plus the version of the in-memory state it was
529// taken from.
530struct VersionedBlob {
531 std::string blob;
532 uint64 seq = 0;
533};
534
535// Push a file's contents out of the C library's buffer AND out of the kernel's
536// page cache, so the bytes exist on the device before anything points at them.
537static bool SyncFileHandle(FILE* f)
538{
539 if (fflush(f) != 0)
540 return false;
541#ifdef _WIN32
542 return _commit(_fileno(f)) == 0;
543#else
544 return fsync(fileno(f)) == 0;
545#endif
546}
547
548// Make the directory entry that rename() created durable too.
549//
550// A tmp+rename is only atomic with respect to a crash of the process. Against
551// a power cut or a hard VM reset the rename can reach the platter while the
552// data it points at has not, and the file that comes back is short or empty.
553// For pending_payouts.json that is not a corrupt file, it is a silent one:
554// LoadPendingPayouts catches the parse error, logs a single warning, and the
555// node comes up owing nobody anything.
556//
557// Best effort -- a filesystem that does not allow the directory to be opened
558// for this is not a reason to refuse to write the ledger.
559static void SyncDirectoryOf(const std::string& path)
308{ 560{
309 return strprintf("block-%d", blockHeight); 561#ifndef _WIN32
562 size_t slash = path.find_last_of('/');
563 std::string dir = (slash == std::string::npos) ? std::string(".")
564 : path.substr(0, slash);
565 if (dir.empty())
566 dir = "/";
567 int fd = open(dir.c_str(), O_RDONLY);
568 if (fd < 0)
569 return;
570 fsync(fd);
571 close(fd);
572#endif
573}
574
575// Replace `path` with `data` atomically and durably. Caller must hold the
576// write mutex for `path`: the temp name is derived from the target, so two
577// threads in here at once for one path is precisely the race this exists to
578// prevent.
579//
580// Every failure removes the temp file before returning. It used to be left
581// where it fell, so a disk that filled up once left a short, valid-looking
582// "pending_payouts.json.tmp" next to the good ledger for as long as the node
583// ran. That file is a trap: it has the name of a half-finished save, and an
584// operator cleaning up after an incident is exactly the person who will move
585// it into place.
586//
587// Failures print unconditionally rather than through LogPrint(). These three
588// files are written under two different debug categories ("payout" for the
589// ledgers, "pool" for the status file), and a write failure is not a trace of
590// normal operation -- it is the file the operator is about to read not being
591// there. Gating it behind a category that had to be chosen before the failure
592// happened is how "could not replace pool_status.json" went missing.
593static bool WriteFileAtomic(const std::string& path, const std::string& data,
594 const char* what)
595{
596 std::string tmp = path + ".tmp";
597 FILE* f = fopen(tmp.c_str(), "w");
598 if (!f) {
599 printf("[payout] WARNING: could not write %s (%s)\n",
600 tmp.c_str(), what);
601 return false;
602 }
603 if (!data.empty() && fwrite(data.c_str(), 1, data.size(), f) != data.size()) {
604 printf("[payout] WARNING: short write to %s (%s)\n",
605 tmp.c_str(), what);
606 fclose(f);
607 remove(tmp.c_str());
608 return false;
609 }
610 // Before the rename, not after: publishing a name that points at nothing
611 // is the failure this ordering exists to prevent. If the data cannot be
612 // made durable, do not rename -- the previous file is still intact and
613 // still correct, and the next write will try again.
614 if (!SyncFileHandle(f)) {
615 printf("[payout] WARNING: could not flush %s to disk (%s)"
616 " -- keeping the previous file\n", tmp.c_str(), what);
617 fclose(f);
618 remove(tmp.c_str());
619 return false;
620 }
621 // fclose() is where buffered data actually reaches the OS, so a failure
622 // here means the .tmp is short -- renaming it over the real file would
623 // destroy the good copy, which is exactly what these files exist to
624 // prevent.
625 if (fclose(f) != 0) {
626 printf("[payout] WARNING: could not close %s (%s)\n",
627 tmp.c_str(), what);
628 remove(tmp.c_str());
629 return false;
630 }
631#ifdef _WIN32
632 // rename() on Windows will not replace an existing file.
633 remove(path.c_str());
634#endif
635 if (rename(tmp.c_str(), path.c_str()) != 0) {
636 printf("[payout] WARNING: could not replace %s (%s)\n",
637 path.c_str(), what);
638 remove(tmp.c_str());
639 return false;
640 }
641 SyncDirectoryOf(path);
642 return true;
643}
644
645// Paths whose loader threw part-way through, and which must therefore never be
646// written again by this process.
647//
648// All three loaders push entries into the live structure INSIDE the try, so an
649// exception half way down a file leaves a PARTIAL ledger in memory -- the
650// entries before the bad one, and nothing after it. The first publish after
651// that then writes the partial state over the file, and the only record of
652// what was owed is gone. ReconcilePayouts publishes unconditionally at
653// startup, so "the first publish" is a few milliseconds later, before an
654// operator could possibly intervene. a8e2f58 named this failure mode in its
655// own commit message and then only fixed the write side.
656//
657// The file is renamed out of the way instead of being left in place, so the
658// evidence survives even if the operator restarts into the same fault, and so
659// a node that comes up owing nobody anything is visibly missing its ledger
660// rather than quietly appearing to have none.
661static std::set<std::string> gRefusedFiles;
662static std::mutex gRefusedFilesMutex;
663
664static bool FileIsRefused(const std::string& path)
665{
666 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
667 return gRefusedFiles.count(path) > 0;
668}
669
670// Quarantine `path` and refuse to write it for the rest of the process.
671//
672// Prints unconditionally: an operator who has just lost a payout ledger must
673// be told at whatever log settings they were already running with, and the
674// name it was moved to is the only way back to the data.
675static void RefuseFileAfterLoadFailure(const std::string& path, const char* what,
676 const char* why)
677{
678 {
679 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
680 gRefusedFiles.insert(path);
681 }
682 std::string bad = strprintf("%s.bad.%lld", path.c_str(), (long long)GetTime());
683 bool moved = (rename(path.c_str(), bad.c_str()) == 0);
684 printf("[payout] ERROR: %s (%s) did not parse: %s\n"
685 "[payout] ERROR: what loaded before the error is a PARTIAL picture,"
686 " so this file will not be written again by this process --"
687 " restart once the cause is fixed\n"
688 "[payout] ERROR: the file %s\n",
689 path.c_str(), what, why,
690 moved ? strprintf("has been kept as %s", bad.c_str()).c_str()
691 : "could NOT be renamed out of the way -- copy it now");
692}
693
694// Publish a versioned image of some in-memory state to `path`.
695//
696// `fileMutex` serialises writers of `path` and guards `publishedSeq`. A blob
697// older than what is already published is dropped: these files are whole-state
698// dumps, so a higher sequence number is by construction a later state of the
699// same structure and publishing it loses nothing. On a write failure
700// `publishedSeq` is deliberately left alone, so the next (newer) write still
701// gets a chance to land.
702static void PublishVersioned(const std::string& path, const VersionedBlob& v,
703 std::mutex& fileMutex, uint64& publishedSeq,
704 const char* what)
705{
706 // A default-constructed blob is not a snapshot of anything; it is a
707 // VersionedBlob nobody filled in. Today they are already dropped, because
708 // every serialiser stamps ++gSeq and so never produces 0, and 0 fails the
709 // seq <= publishedSeq test below -- but only for as long as publishedSeq
710 // starts at 0 and nobody writes `seq = gSeq++`. One character, and the
711 // ledger file becomes "[]". Say it out loud instead of relying on it.
712 if (!v.seq)
713 return;
714 // A file whose loader threw is not a file this process knows the contents
715 // of. Writing it would replace a full record of what was owed with
716 // whatever happened to parse before the error.
717 if (FileIsRefused(path))
718 return;
719 std::lock_guard<std::mutex> lk(fileMutex);
720 if (v.seq <= publishedSeq)
721 return;
722 if (!WriteFileAtomic(path, v.blob, what))
723 return;
724 publishedSeq = v.seq;
725}
726
727// Identity of a payout round. Used as the pool-round proof id and, more
728// importantly, as the `pool_payout_id` written into wallet transaction
729// metadata and matched by FindRecordedPoolPayout().
730//
731// Derived from the block HASH, not from its height alone. Upstream uses
732// "block-<height>", which names two different blocks identically across a
733// reorg -- and because this id is what the crash-recovery probe matches on, a
734// height-keyed id lets that probe find the *previous* round's transaction and
735// conclude a miner has already been paid when they have not. The height stays
736// in the string for readability only.
737//
738// A zero hash means the entry predates hash keying; those keep the old form so
739// anything already recorded against them still matches.
740static std::string PayoutIdForBlock(int blockHeight, const uint256& blockHash)
741{
742 if (blockHash == 0)
743 return strprintf("block-%d", blockHeight);
744 return strprintf("block-%d-%s", blockHeight,
745 blockHash.ToString().substr(0, 16).c_str());
746}
747
748// Do these two references name the same payout round?
749//
750// The block hash alone does not decide it, and every ledger lookup used to
751// assume it did. Entries grandfathered in from a file written before payouts
752// were keyed by hash all carry hash 0, so under a hash-only comparison the
753// first zero-hash entry answers for every other one: RecordPayoutTxid writes
754// a txid against the wrong round (which is then erased as "fully paid" while
755// the real debt still stands), and ReconcilePayouts drops a live in-flight
756// record because a *different* zero-hash round happened to pay that address.
757//
758// A non-zero hash is unique and decides on its own -- that is the whole point
759// of keying on it. A zero hash falls back to the height, which
760// LoadPendingPayouts supplies for any entry that can name its own maturity.
761// Mixed hash-ness never matches: refusing to match is the safe direction for
762// every caller here, because each of them treats a match as "this debt is
763// already dealt with".
764static bool SamePayoutRound(int aHeight, const uint256& aHash,
765 int bHeight, const uint256& bHash)
766{
767 if (aHash != bHash)
768 return false;
769 if (aHash != 0)
770 return true;
771 return aHeight == bHeight;
772}
773
774// An entry with neither a hash nor a height cannot be told apart from any
775// other entry in the same state, so nothing may be recorded against it: a
776// txid written to the wrong one reports a debt as settled that is still owed.
777// Such an entry is never paid and is logged instead -- refusing to act is the
778// safe failure, and no file this tree has ever written can produce one.
779static bool PayoutRoundIdentifiable(int height, const uint256& hash)
780{
781 return hash != 0 || height != 0;
310} 782}
311 783
312static bool PayoutFullyPaid(const PendingPayout& pp) 784static bool PayoutFullyPaid(const PendingPayout& pp)
@@ -317,15 +789,28 @@ static bool PayoutFullyPaid(const PendingPayout& pp)
317 return true; 789 return true;
318} 790}
319 791
320static void SavePendingPayoutsLocked() 792// Serialise the ledger and stamp it with the version it was taken at. Caller
793// must hold gPayoutLedgerMutex.
794//
795// Split from the file write so no thread holds the ledger mutex across the
796// I/O. Upstream writes the file inside the per-recipient loop with
797// gPayoutLedgerMutex held, while the event loop's FlushMaturePayouts blocks on
798// that same mutex -- one slow fsync per recipient, on the thread the miners'
799// I/O shares. The sequence number is what lets the write happen later, and
800// possibly out of order, without a stale image reaching the disk.
801static VersionedBlob SerializePendingPayoutsLocked()
321{ 802{
322 nlohmann::json arr = nlohmann::json::array(); 803 nlohmann::json arr = nlohmann::json::array();
323 for (const auto& pp : gPendingPayouts) { 804 for (const auto& pp : gPendingPayouts) {
324 nlohmann::json obj; 805 nlohmann::json obj;
325 obj["id"] = pp.id; 806 obj["blockHeight"] = pp.blockHeight;
326 obj["blockHeight"] = pp.blockHeight; 807 obj["blockHash"] = pp.blockHash.ToString();
327 obj["matureAtHeight"] = pp.matureAtHeight; 808 obj["matureAtHeight"] = pp.matureAtHeight;
328 obj["nextAttempt"] = pp.nextAttempt; 809 // No retryAfter: it was never written with anything but zero, and the
810 // retry clock that is actually used is on PayingRecord. Files that
811 // still carry the key load fine (it is ignored), and a build that
812 // expects it defaults an absent one to 0, which means "ready now" --
813 // the same answer it always got.
329 nlohmann::json amounts = nlohmann::json::object(); 814 nlohmann::json amounts = nlohmann::json::object();
330 for (const auto& kv : pp.amounts) 815 for (const auto& kv : pp.amounts)
331 amounts[kv.first] = kv.second; 816 amounts[kv.first] = kv.second;
@@ -336,24 +821,41 @@ static void SavePendingPayoutsLocked()
336 obj["paidTxids"] = paid; 821 obj["paidTxids"] = paid;
337 arr.push_back(obj); 822 arr.push_back(obj);
338 } 823 }
339 std::string path = PayoutFilePath(); 824 VersionedBlob v;
340 std::string tmp = path + ".tmp"; 825 v.blob = arr.dump(2);
341 FILE* f = fopen(tmp.c_str(), "w"); 826 v.seq = ++gPayoutLedgerSeq;
342 if (f) { 827 return v;
343 std::string s = arr.dump(2); 828}
344 fwrite(s.c_str(), 1, s.size(), f); 829
345 if (fclose(f) != 0) { 830// Publish a ledger snapshot. Holds no data mutex; safe -- and intended -- to
346 LogPrint("payout", "[payout] WARNING: could not close %s\n", tmp.c_str()); 831// call with gPayoutLedgerMutex released.
347 return; 832static void PublishPendingPayouts(const VersionedBlob& v)
348 } 833{
349#ifdef _WIN32 834 PublishVersioned(PayoutFilePath(), v, gPayoutLedgerFileMutex,
350 remove(path.c_str()); 835 gPayoutLedgerPublishedSeq, "pending payout ledger");
351#endif 836}
352 if (rename(tmp.c_str(), path.c_str()) != 0) 837
353 LogPrint("payout", "[payout] WARNING: could not replace %s\n", path.c_str()); 838// Parse a persisted block-hash field, refusing to guess. SetHex() reads
354 } else { 839// whatever hex prefix it finds and stops at the first character it does not
355 LogPrint("payout", "[payout] WARNING: could not write %s\n", tmp.c_str()); 840// recognise, so a corrupt value does not fail to parse -- it parses as a
356 } 841// DIFFERENT hash. Both shapes of that are wrong in a dangerous direction: a
842// value with a hex-looking prefix ("deadbeef...") becomes a hash no block
843// ever had, which PayoutBlockStillOnChain reads as an orphaned block and
844// FlushMaturePayouts then VOIDS the debt; a value with no hex prefix at all
845// becomes 0, which is grandfathered past the orphan check on the strength
846// of garbage. A field that is not the 64-digit hex string ToString() wrote
847// is corruption, and corruption is a load failure like any other parse
848// error -- the file is quarantined, not read around.
849static uint256 ParsePayoutBlockHash(const std::string& str)
850{
851 if (str.size() != 64 ||
852 str.find_first_not_of("0123456789abcdefABCDEF") != std::string::npos)
853 throw std::runtime_error(
854 strprintf("blockHash \"%s\" is not a 64-digit hex string",
855 str.c_str()));
856 uint256 h;
857 h.SetHex(str);
858 return h;
357} 859}
358 860
359static void LoadPendingPayouts() 861static void LoadPendingPayouts()
@@ -372,17 +874,27 @@ static void LoadPendingPayouts()
372 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); 874 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
373 for (const auto& obj : arr) { 875 for (const auto& obj : arr) {
374 PendingPayout pp; 876 PendingPayout pp;
375 if (obj.contains("blockHeight"))
376 pp.blockHeight = obj["blockHeight"].get<int>();
377 pp.matureAtHeight = obj["matureAtHeight"].get<int>(); 877 pp.matureAtHeight = obj["matureAtHeight"].get<int>();
378 if (obj.contains("id") && obj["id"].is_string()) 878 // Entries written before payouts were keyed by hash carry neither
379 pp.id = obj["id"].get<std::string>(); 879 // field. Leave blockHash zero and they are grandfathered past the
380 if (pp.blockHeight == 0) 880 // orphan check rather than voided on a guess.
881 if (obj.contains("blockHash") && obj["blockHash"].is_string())
882 pp.blockHash =
883 ParsePayoutBlockHash(obj["blockHash"].get<std::string>());
884 if (obj.contains("blockHeight") && obj["blockHeight"].is_number())
885 pp.blockHeight = obj["blockHeight"].get<int>();
886 // Upstream's tolerance for an entry with no usable height, kept
887 // because it costs nothing and an entry that cannot name its block
888 // is worse than one derived from its maturity. COINBASE_MATURITY
889 // and not COINBASE_SPENDABLE_DEPTH: this reverses whatever the
890 // writer of *that* file used, and only files old enough to lack
891 // blockHeight are affected -- they were written when maturity was
892 // still COINBASE_MATURITY.
893 if (pp.blockHeight == 0 && pp.matureAtHeight > COINBASE_MATURITY)
381 pp.blockHeight = pp.matureAtHeight - COINBASE_MATURITY; 894 pp.blockHeight = pp.matureAtHeight - COINBASE_MATURITY;
382 if (pp.id.empty()) 895 // "retryAfter" is deliberately not read. Files written by the
383 pp.id = PayoutIdForBlock(pp.blockHeight); 896 // merged build carry it, always as 0, and there is nowhere left to
384 if (obj.contains("nextAttempt")) 897 // put it -- the retry clock is PayingRecord's.
385 pp.nextAttempt = obj["nextAttempt"].get<int64>();
386 for (auto it = obj["amounts"].begin(); it != obj["amounts"].end(); ++it) 898 for (auto it = obj["amounts"].begin(); it != obj["amounts"].end(); ++it)
387 pp.amounts[it.key()] = it.value().get<int64>(); 899 pp.amounts[it.key()] = it.value().get<int64>();
388 if (obj.contains("paidTxids") && obj["paidTxids"].is_object()) 900 if (obj.contains("paidTxids") && obj["paidTxids"].is_object())
@@ -393,11 +905,212 @@ static void LoadPendingPayouts()
393 LogPrint("payout", "[payout] loaded %zu pending payout(s) from disk\n", 905 LogPrint("payout", "[payout] loaded %zu pending payout(s) from disk\n",
394 gPendingPayouts.size()); 906 gPendingPayouts.size());
395 } catch (const std::exception& e) { 907 } catch (const std::exception& e) {
396 LogPrint("payout", "[payout] WARNING: failed to parse %s: %s\n", 908 RefuseFileAfterLoadFailure(path, "pending payout ledger", e.what());
397 path.c_str(), e.what()); 909 }
910}
911
912// A payment in flight: one record per (payout entry, address) pair, which is
913// the granularity SendMoney works at and therefore the granularity at which a
914// payment can be half-done when the process dies.
915//
916// This list is persisted. It used to be a memory-only queue, and matured
917// entries were erased from pending_payouts.json *before* being handed to it,
918// so any restart in between lost those payments with no record that anyone
919// was owed anything: a container restart on 2026-07-30 silently dropped
920// ~1,156 BTF that way. Writing the records before removing the ledger entry,
921// and reconciling against the wallet on startup, is what makes a restart
922// survivable.
923struct PayingRecord {
924 // Identity of this SLOT, distinct from the identity of the payment it
925 // describes. ForgetPayment and ReschedulePayment used to find their record
926 // by comparing every field, which is only an identity while no two records
927 // can be equal -- and a restart used to produce byte-identical duplicates
928 // (see FlushMaturePayouts), so the first match absorbed every update and
929 // the second was never touched again.
930 //
931 // Process-local and deliberately not persisted: gPaying is rebuilt from
932 // disk at every start, so an id only ever has to name a slot within one
933 // run, and reading ids back would mean trusting a file to contain no
934 // duplicates. Assigned under gPayoutExecMutex from gPayingNextId.
935 uint64 id = 0;
936 int blockHeight = 0;
937 uint256 blockHash = 0; // zero = predates hash keying, see PendingPayout
938 int matureAtHeight = 0;
939 std::string address;
940 int64 amount = 0;
941 int retries = 0;
942 int deferrals = 0;
943 int64 retryAfter = 0;
944 // Ask the wallet whether this payment already went out, once, before the
945 // first send attempt. Upstream asks on every attempt: FindRecordedPoolPayout
946 // is a linear scan of mapWallet under cs_mapWallet, per recipient, so with
947 // 2000 rotating payout addresses and a retry budget of 1440 that is a
948 // scan per address per minute for a day. The question it answers -- "did a
949 // previous process send this before it died?" -- can only change across a
950 // restart, and every restart re-sets this flag (ReconcilePayouts), so
951 // asking once per record is the whole of the answer.
952 //
953 // Not persisted: a record read back from disk is by definition a restart.
954 bool fProbeWallet = true;
955};
956static std::vector<PayingRecord> gPaying; // guarded by gPayoutExecMutex
957static std::mutex gPayoutExecMutex;
958static uint64 gPayingNextId = 0; // guarded by gPayoutExecMutex
959static uint64 gPayingSeq = 0; // guarded by gPayoutExecMutex
960static std::mutex gPayingFileMutex;
961static uint64 gPayingPublishedSeq = 0; // guarded by gPayingFileMutex
962static std::atomic<bool> gPayoutThreadRunning{false};
963
964// Upstream's gPayoutExecQueue/gPayoutExecMutex/gPayoutThreadRunning are
965// declared above with gPaying: the in-flight list IS the exec queue in this
966// tree, at per-recipient rather than per-round granularity, so a second queue
967// would be a second answer to "who is owed money". Only the dedupe set is new.
968//
969// Ids of rounds currently exploded into gPaying, so a flush cannot queue the
970// same round twice. Guarded by gPayoutExecMutex. Seeded from the restored
971// in-flight list by ReconcilePayouts -- it is memory-only, and leaving it
972// empty at startup was half of why a restart duplicated everything in flight.
973static std::set<std::string> gQueuedPayouts;
974
975// Is this exact payment already in flight? gPaying is keyed on
976// (round, address): the ledger holds one amount per address per round, so two
977// records for that pair are two standing orders to send the same money.
978// Caller must hold gPayoutExecMutex.
979static bool PaymentInFlightLocked(int blockHeight, const uint256& blockHash,
980 const std::string& address)
981{
982 for (const PayingRecord& e : gPaying)
983 if (e.address == address
984 && SamePayoutRound(e.blockHeight, e.blockHash, blockHeight, blockHash))
985 return true;
986 return false;
987}
988
989static std::string PayingFilePath()
990{
991 return GetAppDir() + "/paying_payouts.json";
992}
993
994// Serialise the in-flight list. Caller must hold gPayoutExecMutex.
995//
996// gPayoutExecMutex already serialises every writer of this file, so it never
997// had the interleaving race pending_payouts.json had. It is still split the
998// same way for a different reason: FlushMaturePayouts runs on the event loop
999// and holds gPayoutExecMutex, so writing (and, since this series, fsyncing)
1000// the file under that mutex parks miner I/O behind the payout thread's disk.
1001// The version stamp keeps the file monotonic now that the write is outside
1002// the mutex that orders the writers.
1003static VersionedBlob SerializePayingLocked()
1004{
1005 nlohmann::json arr = nlohmann::json::array();
1006 for (const auto& r : gPaying) {
1007 nlohmann::json obj;
1008 obj["blockHeight"] = r.blockHeight;
1009 obj["blockHash"] = r.blockHash.ToString();
1010 obj["matureAtHeight"] = r.matureAtHeight;
1011 obj["address"] = r.address;
1012 obj["amount"] = r.amount;
1013 obj["retries"] = r.retries;
1014 obj["deferrals"] = r.deferrals;
1015 // The live retry clock, which used not to be written at all: every
1016 // restart handed every backing-off payment a retryAfter of 0 and put
1017 // the whole set into the very next 500 ms batch, so a node restarted
1018 // in a loop retried continuously instead of once a minute.
1019 obj["retryAfter"] = r.retryAfter;
1020 arr.push_back(obj);
1021 }
1022 VersionedBlob v;
1023 v.blob = arr.dump(2);
1024 v.seq = ++gPayingSeq;
1025 return v;
1026}
1027
1028// Publish an in-flight-list snapshot. Call with gPayoutExecMutex released.
1029static void PublishPaying(const VersionedBlob& v)
1030{
1031 PublishVersioned(PayingFilePath(), v, gPayingFileMutex,
1032 gPayingPublishedSeq, "in-flight payment list");
1033}
1034
1035static void LoadPaying()
1036{
1037 std::string path = PayingFilePath();
1038 FILE* f = fopen(path.c_str(), "r");
1039 if (!f) return;
1040 fseek(f, 0, SEEK_END);
1041 long sz = ftell(f);
1042 fseek(f, 0, SEEK_SET);
1043 std::string s(sz, '\0');
1044 fread(&s[0], 1, sz, f);
1045 fclose(f);
1046 try {
1047 nlohmann::json arr = nlohmann::json::parse(s);
1048 // gPaying and gPayingNextId are both annotated as guarded by
1049 // gPayoutExecMutex and this was mutating them without it. Safe today
1050 // -- LoadPaying only runs at startup, before the payout thread is
1051 // spawned -- but "safe because of where it is called from" is a fact
1052 // about the caller, not about this function, and the annotation is
1053 // what every other reader will trust. The mutex is uncontended here by
1054 // construction, so taking it costs nothing.
1055 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
1056 for (const auto& obj : arr) {
1057 PayingRecord r;
1058 r.address = obj["address"].get<std::string>();
1059 r.amount = obj["amount"].get<int64>();
1060 r.matureAtHeight = obj["matureAtHeight"].get<int>();
1061 if (obj.contains("blockHash") && obj["blockHash"].is_string())
1062 r.blockHash =
1063 ParsePayoutBlockHash(obj["blockHash"].get<std::string>());
1064 if (obj.contains("blockHeight") && obj["blockHeight"].is_number())
1065 r.blockHeight = obj["blockHeight"].get<int>();
1066 if (obj.contains("retries") && obj["retries"].is_number())
1067 r.retries = obj["retries"].get<int>();
1068 if (obj.contains("deferrals") && obj["deferrals"].is_number())
1069 r.deferrals = obj["deferrals"].get<int>();
1070 // Absent in files written before this change; 0 means ready now,
1071 // which is exactly the behaviour those files used to get.
1072 if (obj.contains("retryAfter") && obj["retryAfter"].is_number())
1073 r.retryAfter = obj["retryAfter"].get<int64>();
1074 // Collapse duplicates as they are read. A build without the
1075 // ReconcilePayouts seeding below wrote a second (and after another
1076 // restart, a third) byte-identical record for every unpaid
1077 // recipient, so a file written by one of those has to be cleaned
1078 // up here or the pathology is inherited. The retry and deferral
1079 // counts are merged upwards so collapsing does not hand the
1080 // survivor a fresh budget.
1081 PayingRecord* dup = NULL;
1082 for (PayingRecord& e : gPaying)
1083 if (e.address == r.address
1084 && SamePayoutRound(e.blockHeight, e.blockHash,
1085 r.blockHeight, r.blockHash)) {
1086 dup = &e;
1087 break;
1088 }
1089 if (dup) {
1090 dup->retries = std::max(dup->retries, r.retries);
1091 dup->deferrals = std::max(dup->deferrals, r.deferrals);
1092 dup->retryAfter = std::max(dup->retryAfter, r.retryAfter);
1093 LogPrint("payout", "[payout] collapsing duplicate in-flight"
1094 " record for %s to %s -- one payment, one record\n",
1095 FormatMoney(r.amount).c_str(), r.address.c_str());
1096 continue;
1097 }
1098 r.id = ++gPayingNextId;
1099 gPaying.push_back(r);
1100 }
1101 LogPrint("payout", "[payout] loaded %zu in-flight payment(s) from disk\n",
1102 gPaying.size());
1103 } catch (const std::exception& e) {
1104 RefuseFileAfterLoadFailure(path, "in-flight payment list", e.what());
398 } 1105 }
399} 1106}
400 1107
1108// ---------------------------------------------------------------------------
1109// Pool round proofs (upstream cbdd06d). Disjoint from the in-flight ledger
1110// above -- this is the operator's published record of how a round was split,
1111// not an obligation to pay. Both sides added code at this offset; both are
1112// kept.
1113// ---------------------------------------------------------------------------
401static nlohmann::json PoolRoundToJson(const PoolRoundProof& r) 1114static nlohmann::json PoolRoundToJson(const PoolRoundProof& r)
402{ 1115{
403 nlohmann::json j; 1116 nlohmann::json j;
@@ -436,12 +1149,26 @@ static nlohmann::json PoolRoundToJson(const PoolRoundProof& r)
436 p["txid"] = tx->second; 1149 p["txid"] = tx->second;
437 else 1150 else
438 p["txid"] = nullptr; 1151 p["txid"] = nullptr;
1152 // Only emitted when it happened, so the common shape of the file is
1153 // unchanged for anything already parsing it.
1154 std::map<std::string, std::string>::const_iterator fo = r.forfeited.find(kv.first);
1155 if (fo != r.forfeited.end()) {
1156 p["forfeited"] = true;
1157 p["forfeitReason"] = fo->second;
1158 }
439 j["payouts"][kv.first] = p; 1159 j["payouts"][kv.first] = p;
440 } 1160 }
441 return j; 1161 return j;
442} 1162}
443 1163
444static void SavePoolRoundsLocked() 1164// Serialise the published round proofs. Caller must hold gPoolRoundsMutex.
1165//
1166// Like the in-flight list, gPoolRoundsMutex already serialises the writers
1167// (RecordPoolRound on the event loop, MarkPoolRoundPaid on the payout thread),
1168// so this file never had the interleaving race either -- but the event loop
1169// should not be waiting on the payout thread's fsync to publish a round, so
1170// the write moves outside the mutex on the same versioned terms.
1171static VersionedBlob SerializePoolRoundsLocked()
445{ 1172{
446 nlohmann::json root; 1173 nlohmann::json root;
447 root["schema"] = "bitflash-pool-rounds-1"; 1174 root["schema"] = "bitflash-pool-rounds-1";
@@ -450,24 +1177,17 @@ static void SavePoolRoundsLocked()
450 for (const PoolRoundProof& r : gPoolRounds) 1177 for (const PoolRoundProof& r : gPoolRounds)
451 root["rounds"].push_back(PoolRoundToJson(r)); 1178 root["rounds"].push_back(PoolRoundToJson(r));
452 1179
453 std::string path = PoolRoundsFilePath(); 1180 VersionedBlob v;
454 std::string tmp = path + ".tmp"; 1181 v.blob = root.dump(2);
455 FILE* f = fopen(tmp.c_str(), "w"); 1182 v.seq = ++gPoolRoundsSeq;
456 if (!f) { 1183 return v;
457 LogPrint("payout", "[payout] WARNING: could not write %s\n", tmp.c_str()); 1184}
458 return; 1185
459 } 1186// Publish a round-proof snapshot. Call with gPoolRoundsMutex released.
460 std::string s = root.dump(2); 1187static void PublishPoolRounds(const VersionedBlob& v)
461 fwrite(s.c_str(), 1, s.size(), f); 1188{
462 if (fclose(f) != 0) { 1189 PublishVersioned(PoolRoundsFilePath(), v, gPoolRoundsFileMutex,
463 LogPrint("payout", "[payout] WARNING: could not close %s\n", tmp.c_str()); 1190 gPoolRoundsPublishedSeq, "pool round proofs");
464 return;
465 }
466#ifdef _WIN32
467 remove(path.c_str());
468#endif
469 if (rename(tmp.c_str(), path.c_str()) != 0)
470 LogPrint("payout", "[payout] WARNING: could not replace %s\n", path.c_str());
471} 1191}
472 1192
473static void LoadPoolRounds() 1193static void LoadPoolRounds()
@@ -512,6 +1232,9 @@ static void LoadPoolRounds()
512 r.payouts[it.key()] = it.value().value("amountSatoshis", (int64)0); 1232 r.payouts[it.key()] = it.value().value("amountSatoshis", (int64)0);
513 if (it.value().contains("txid") && it.value()["txid"].is_string()) 1233 if (it.value().contains("txid") && it.value()["txid"].is_string())
514 r.paidTxids[it.key()] = it.value()["txid"].get<std::string>(); 1234 r.paidTxids[it.key()] = it.value()["txid"].get<std::string>();
1235 if (it.value().value("forfeited", false))
1236 r.forfeited[it.key()] =
1237 it.value().value("forfeitReason", std::string("unspecified"));
515 } else if (it.value().is_number_integer()) { 1238 } else if (it.value().is_number_integer()) {
516 r.payouts[it.key()] = it.value().get<int64>(); 1239 r.payouts[it.key()] = it.value().get<int64>();
517 } 1240 }
@@ -524,55 +1247,343 @@ static void LoadPoolRounds()
524 LogPrint("payout", "[payout] loaded %zu pool round proof(s) from disk\n", 1247 LogPrint("payout", "[payout] loaded %zu pool round proof(s) from disk\n",
525 gPoolRounds.size()); 1248 gPoolRounds.size());
526 } catch (const std::exception& e) { 1249 } catch (const std::exception& e) {
527 LogPrint("payout", "[payout] WARNING: failed to parse %s: %s\n", 1250 RefuseFileAfterLoadFailure(path, "pool round proofs", e.what());
528 path.c_str(), e.what());
529 } 1251 }
530} 1252}
531 1253
1254// Is the block that funds this payout still on the main chain?
1255//
1256// A zero hash means the entry was written before payouts recorded one, so
1257// there is nothing to check and we assume it is good -- voiding those would
1258// mean refusing to pay miners on the strength of a missing field.
1259static bool PayoutBlockStillOnChain(const uint256& blockHash)
1260{
1261 if (blockHash == 0)
1262 return true;
1263 CRITICAL_BLOCK(cs_main)
1264 {
1265 std::map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(blockHash);
1266 if (mi == mapBlockIndex.end() || mi->second == NULL)
1267 return false;
1268 return mi->second->IsInMainChain();
1269 }
1270 return false;
1271}
1272
1273// The comment every pre-upgrade payout transaction carries, and therefore the
1274// ONLY needle the legacy fallback in FindRecordedPoolPayout may search with.
1275// This format is frozen: change it and every payment those builds made stops
1276// matching, every one of those debts looks unpaid, and reconciliation pays
1277// them all a second time.
1278static std::string LegacyPayoutComment(int blockHeight)
1279{
1280 return strprintf("Pool payout height %d", blockHeight);
1281}
1282
1283// The comment written on new sends. It names the funding block, because the
1284// height alone cannot: a height that is won, lost to a reorg, and won again
1285// yields two rounds whose height-only comments are identical -- and identical
1286// amounts too, whenever the same miner held both rounds. The structured
1287// pool_payout_* metadata already keys the payment by hash for the code; this
1288// puts the same fact where an operator reading the wallet can see it, and
1289// makes a new transaction unmatchable by the legacy needle even if the
1290// structured keys are ever lost.
1291//
1292// A zero hash keeps the legacy form: those records predate hash keying, and
1293// there is no hash to name.
1294static std::string PayoutComment(int blockHeight, const uint256& blockHash)
1295{
1296 if (blockHash == 0)
1297 return LegacyPayoutComment(blockHeight);
1298 return strprintf("Pool payout height %d block %s", blockHeight,
1299 blockHash.ToString().c_str());
1300}
1301
1302// Startup reconciliation.
1303//
1304// This used to be the crash-recovery mechanism: it scanned the wallet with
1305// WalletAlreadyPaid() and then dropped ledger entries that had already been
1306// exploded into in-flight records. Neither job is right any more.
1307//
1308// The ledger is now authoritative and an entry survives until every recipient
1309// has a txid recorded against it (upstream cbdd06d), so dropping ledger
1310// entries here would reintroduce exactly the loss this merge is adopting that
1311// model to prevent. And the wallet scan is now FindRecordedPoolPayout(), done
1312// once per record on its first attempt, which is the same question asked in
1313// the one place that can act on the answer. Running both would be two
1314// authorities for the same debt.
1315//
1316// What is left is the useful part: the in-flight list restored from disk may
1317// contain records the ledger already shows as paid (we died between the send
1318// and the in-flight file being rewritten). The ledger wins.
1319//
1320// And the part that was missing: gQueuedPayouts is memory-only, so it came up
1321// empty while gPaying came up full. FlushMaturePayouts deduped against that
1322// empty set and against the ledger's paidTxids, and against nothing else --
1323// so a round that was exploded but not yet paid got a SECOND record per
1324// unpaid recipient on the first flush after every restart, deterministically.
1325// (Observed live: two byte-identical records for one (block, address).) The
1326// surviving in-flight list is exactly the statement of which rounds are
1327// already exploded, so it is what the set is rebuilt from.
1328static void ReconcilePayouts()
1329{
1330 VersionedBlob payingSnap;
1331 {
1332 std::lock_guard<std::mutex> exec(gPayoutExecMutex);
1333 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
1334
1335 size_t settled = 0;
1336 int64 settledAmount = 0;
1337 for (auto it = gPaying.begin(); it != gPaying.end(); ) {
1338 bool paid = false;
1339 for (const PendingPayout& pp : gPendingPayouts)
1340 if (SamePayoutRound(pp.blockHeight, pp.blockHash,
1341 it->blockHeight, it->blockHash)
1342 && pp.paidTxids.count(it->address)) {
1343 paid = true;
1344 break;
1345 }
1346 if (paid) {
1347 settledAmount += it->amount;
1348 settled++;
1349 it = gPaying.erase(it);
1350 } else {
1351 // Every restored record gets one wallet probe before it is
1352 // allowed to send: this is precisely the window where the
1353 // wallet may hold a transaction the ledger has not recorded.
1354 it->fProbeWallet = true;
1355 ++it;
1356 }
1357 }
1358 if (settled) {
1359 LogPrint("payout", "[payout] %zu in-flight payment(s) totalling %s"
1360 " are already recorded paid in the ledger -- not"
1361 " resending\n",
1362 settled, FormatMoney(settledAmount).c_str());
1363 }
1364
1365 // Rebuild the "already exploded" set from what survived. Without this
1366 // the first flush after a restart re-explodes every round in flight.
1367 gQueuedPayouts.clear();
1368 for (const PayingRecord& r : gPaying)
1369 gQueuedPayouts.insert(PayoutIdForBlock(r.blockHeight, r.blockHash));
1370
1371 payingSnap = SerializePayingLocked();
1372
1373 if (!gPaying.empty()) {
1374 int64 owed = 0;
1375 for (const auto& r : gPaying) owed += r.amount;
1376 LogPrint("payout", "[payout] resuming %zu in-flight payment(s)"
1377 " totalling %s\n",
1378 gPaying.size(), FormatMoney(owed).c_str());
1379 }
1380 }
1381 PublishPaying(payingSnap);
1382}
1383
532static void RecordPoolRound(const PoolRoundProof& round) 1384static void RecordPoolRound(const PoolRoundProof& round)
533{ 1385{
534 std::lock_guard<std::mutex> lk(gPoolRoundsMutex); 1386 VersionedBlob snap;
535 for (PoolRoundProof& existing : gPoolRounds) { 1387 {
536 if (existing.id == round.id) { 1388 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
537 existing = round; 1389 bool replaced = false;
538 SavePoolRoundsLocked(); 1390 for (PoolRoundProof& existing : gPoolRounds) {
539 return; 1391 if (existing.id == round.id) {
1392 existing = round;
1393 replaced = true;
1394 break;
1395 }
1396 }
1397 if (!replaced) {
1398 gPoolRounds.push_back(round);
1399 while (gPoolRounds.size() > MAX_POOL_ROUNDS)
1400 gPoolRounds.erase(gPoolRounds.begin());
540 } 1401 }
1402 snap = SerializePoolRoundsLocked();
541 } 1403 }
542 gPoolRounds.push_back(round); 1404 PublishPoolRounds(snap);
543 while (gPoolRounds.size() > MAX_POOL_ROUNDS)
544 gPoolRounds.erase(gPoolRounds.begin());
545 SavePoolRoundsLocked();
546} 1405}
547 1406
548static void MarkPoolRoundPaid(const std::string& id, 1407static void MarkPoolRoundPaid(const std::string& id,
549 const std::string& address, 1408 const std::string& address,
550 const std::string& txid) 1409 const std::string& txid)
551{ 1410{
552 std::lock_guard<std::mutex> lk(gPoolRoundsMutex); 1411 VersionedBlob snap;
553 for (PoolRoundProof& r : gPoolRounds) { 1412 {
554 if (r.id != id) 1413 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
555 continue; 1414 bool found = false;
556 r.paidTxids[address] = txid; 1415 for (PoolRoundProof& r : gPoolRounds) {
557 SavePoolRoundsLocked(); 1416 if (r.id != id)
558 return; 1417 continue;
1418 r.paidTxids[address] = txid;
1419 found = true;
1420 break;
1421 }
1422 if (!found)
1423 return;
1424 snap = SerializePoolRoundsLocked();
559 } 1425 }
1426 PublishPoolRounds(snap);
560} 1427}
561 1428
562// Payouts ready to execute are moved here under gPayoutExecMutex. 1429// Publish the fact that a recipient was written off, so the round proof and
563// A dedicated thread drains this list so SendMoney (which takes cs_main) 1430// the ledger agree. The ledger erases the amount (it is not owed any more and
564// never runs on the event loop thread and can't stall miner I/O. 1431// must not be reported as sent); the proof keeps the amount visible and marks
565static std::vector<PendingPayout> gPayoutExecQueue; 1432// it forfeited with a reason, because a miner who was promised a share is
566static std::mutex gPayoutExecMutex; 1433// entitled to see what happened to it.
567static std::atomic<bool> gPayoutThreadRunning{false}; 1434static void MarkPoolRoundForfeited(const std::string& id,
1435 const std::string& address,
1436 const std::string& reason)
1437{
1438 VersionedBlob snap;
1439 {
1440 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1441 bool found = false;
1442 for (PoolRoundProof& r : gPoolRounds) {
1443 if (r.id != id)
1444 continue;
1445 r.forfeited[address] = reason;
1446 found = true;
1447 break;
1448 }
1449 if (!found)
1450 return;
1451 snap = SerializePoolRoundsLocked();
1452 }
1453 PublishPoolRounds(snap);
1454}
568 1455
569static std::set<std::string> gQueuedPayouts; 1456// Does this round have a published round proof?
1457//
1458// Existence answers a temporal question the legacy fallback below needs:
1459// which builds could have paid this round. RecordPoolRound writes the proof
1460// when the round is created, and structured payout metadata (upstream
1461// cbdd06d) is an ancestor of round proofs (upstream 83ba822) -- no build
1462// able to write a proof has ever sent a payment that only a comment can
1463// identify. A round with a proof therefore cannot have a comment-only
1464// payment; one that matches it is another round's, colliding on the height.
1465static bool PoolRoundHasProof(const std::string& id)
1466{
1467 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1468 for (const PoolRoundProof& r : gPoolRounds)
1469 if (r.id == id)
1470 return true;
1471 return false;
1472}
570 1473
571static bool FindRecordedPoolPayout(const std::string& id, 1474// Which wallet transaction, if any, already paid this?
572 const std::string& address, 1475//
573 int64 amount, 1476// Two kinds of evidence, in order of quality.
574 std::string& txidRet) 1477//
1478// Structured (pool_payout_id / pool_payout_address / pool_payout_amount,
1479// plus pool_payout_block since this series) is upstream's, extended. The id
1480// is derived from the block hash, but only from the FIRST sixteen hex digits
1481// of it -- and proof-of-work concentrates zeros exactly there, so the id can
1482// name two same-height rounds identically. New sends therefore record the
1483// full hash as pool_payout_block, and a structured match must agree with it
1484// wherever both sides have one; a transaction without the key was sent
1485// before it existed, and for those the id is the best evidence there is.
1486// Every transaction this build sends carries all four keys.
1487//
1488// But NO transaction this pool has ever sent carries it. Every payout in the
1489// existing wallet has only a "comment". So on the first boot after this
1490// upgrade -- precisely the boot where the binary is swapped after an unclean
1491// shutdown with payments in flight -- a structured-only probe finds nothing
1492// and every one of those recipients is paid a second time. The deleted
1493// WalletAlreadyPaid() matched on what the payment WAS, and that evidence is
1494// still there; it just has to be read.
1495//
1496// The legacy fallback is therefore restored, with three constraints that
1497// together stop it answering for a round that is not this one:
1498//
1499// 1. it is only consulted when the structured lookup found nothing;
1500// 2. it ignores any transaction carrying pool_payout_id. That key means this
1501// build wrote the transaction, and this build's transactions are
1502// adjudicated by the exact path -- so a NEW round's payment can never be
1503// offered as evidence for an old one. Only pre-upgrade transactions,
1504// which is exactly the population the fallback exists for, are eligible;
1505// 3. `excluded` holds every txid already credited to some OTHER
1506// (round, address) pair -- from the ledger's paidTxids, which is durable,
1507// and from what this process has already claimed. One legacy transaction
1508// can therefore settle at most one debt, so N transactions can never
1509// satisfy more than N obligations;
1510//
1511// 4. it is not consulted at all for a round that has a round proof
1512// (fAllowLegacy, decided by the caller from PoolRoundHasProof). Proofs
1513// are written when a round is created, and only by builds whose every
1514// payment names its round exactly -- so a comment-only transaction
1515// cannot be a payment for such a round, only a same-height collision
1516// from an older one, and skipping the fallback cannot hide a real
1517// payment: the structured probe above already found any that exist.
1518//
1519// What is left unresolved by construction is a comment-only transaction and
1520// a PROOFLESS round at the same height paying the same address the same
1521// amount: two grandfathered zero-hash twins, or a twin pair from the builds
1522// that keyed the ledger by hash before any metadata reached the wallet. The
1523// comment carries only the height, so nothing ties the transaction to one
1524// twin rather than the other. That needs a reorg.
1525//
1526// This used to say the losing twin's ledger entry is voided by the orphan
1527// check before it can be paid. That is wrong: PayoutBlockStillOnChain()
1528// returns true for a zero hash, and EVERY pre-upgrade entry has a zero hash,
1529// so the orphan check is a no-op for exactly the population this fallback
1530// serves.
1531//
1532// The outcome is still safe, by a different mechanism. SamePayoutRound()
1533// treats two zero-hash entries at the same height as the SAME round, so the
1534// pair is indistinguishable everywhere, not only here: FlushMaturePayouts
1535// explodes one of them and skips the other as already queued,
1536// PaymentInFlightLocked refuses a second record for that (round, address), and
1537// RecordPayoutTxid settles against the first match. The pair therefore
1538// presents as ONE debt and can be paid at most once -- which is the direction
1539// that matters for a fallback whose failure mode is treating one payment as
1540// proof of two.
1541//
1542// The cost of that mechanism, stated plainly: only one of a genuine
1543// same-height pair would ever be paid. It takes a reorg AND both rounds
1544// predating hash keying, and no file this tree has written since can produce
1545// one, so it is a property of history rather than a live risk.
1546//
1547// Where the evidence genuinely runs out the fallback simply does not match,
1548// and the payment is made -- paying a debt we cannot prove was settled is the
1549// correct default for a pool.
1550enum PayoutProbeResult {
1551 PAYOUT_PROBE_NONE = 0, // the wallet holds no record of this payment
1552 PAYOUT_PROBE_STRUCTURED, // matched this build's pool_payout_* metadata
1553 PAYOUT_PROBE_LEGACY // matched a pre-upgrade comment and output
1554};
1555
1556static PayoutProbeResult FindRecordedPoolPayout(const std::string& id,
1557 int blockHeight,
1558 const uint256& blockHash,
1559 const std::string& address,
1560 int64 amount,
1561 bool fAllowLegacy,
1562 const std::set<std::string>& excluded,
1563 std::string& txidRet)
575{ 1564{
1565 // Built before cs_mapWallet is taken: an invalid address has no script to
1566 // compare against, and the caller has already rejected those.
1567 uint160 h160;
1568 bool fHaveScript = AddressToHash160(address, h160);
1569 CScript want;
1570 if (fHaveScript)
1571 want << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
1572 const std::string comment = LegacyPayoutComment(blockHeight);
1573
1574 // Candidate legacy matches, adjudicated after cs_mapWallet is released.
1575 //
1576 // The depth check cannot happen inside the scan: GetDepthInMainChain()
1577 // reads mapBlockIndex and pindexBest, which belong to cs_main, and cs_main
1578 // is taken BEFORE cs_mapWallet everywhere else in this tree
1579 // (CreateTransaction, ConnectBlock). Taking it the other way round here
1580 // would be a lock-order inversion in the payout thread. There is at most a
1581 // handful of these: a candidate has to carry this exact height's comment
1582 // and an output of exactly this amount to this address.
1583 std::vector<uint256> legacyCandidates;
1584
1585 // One pass, not two: this scan runs under cs_mapWallet, which the whole
1586 // node contends on, and the merge is already paying for it once per record.
576 CRITICAL_BLOCK(cs_mapWallet) 1587 CRITICAL_BLOCK(cs_mapWallet)
577 { 1588 {
578 for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); 1589 for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin();
@@ -580,23 +1591,145 @@ static bool FindRecordedPoolPayout(const std::string& id,
580 { 1591 {
581 const CWalletTx& wtx = it->second; 1592 const CWalletTx& wtx = it->second;
582 map<string, string>::const_iterator idIt = wtx.mapValue.find("pool_payout_id"); 1593 map<string, string>::const_iterator idIt = wtx.mapValue.find("pool_payout_id");
583 if (idIt == wtx.mapValue.end() || idIt->second != id) 1594 if (idIt != wtx.mapValue.end()) {
1595 // Written by this build. Adjudicated exactly, and never
1596 // offered to the fallback below.
1597 if (idIt->second != id)
1598 continue;
1599 map<string, string>::const_iterator addrIt = wtx.mapValue.find("pool_payout_address");
1600 if (addrIt == wtx.mapValue.end() || addrIt->second != address)
1601 continue;
1602 map<string, string>::const_iterator amtIt = wtx.mapValue.find("pool_payout_amount");
1603 if (amtIt == wtx.mapValue.end() || amtIt->second != strprintf("%lld", amount))
1604 continue;
1605 // The id alone cannot tell twin rounds at one height apart
1606 // (see above); the full hash can, where the transaction
1607 // recorded one. Where it did not, the id decides, as it
1608 // always has -- refusing those transactions would make every
1609 // payment sent before the key existed look unpaid and pay it
1610 // again.
1611 if (blockHash != 0) {
1612 map<string, string>::const_iterator blkIt =
1613 wtx.mapValue.find("pool_payout_block");
1614 if (blkIt != wtx.mapValue.end()
1615 && blkIt->second != blockHash.ToString())
1616 continue; // a twin round's payment, not this round's
1617 }
1618 txidRet = wtx.GetHash().GetHex();
1619 return PAYOUT_PROBE_STRUCTURED;
1620 }
1621
1622 if (!fAllowLegacy)
1623 continue; // comment-only evidence is inadmissible: rule 4
1624 if (!fHaveScript)
584 continue; 1625 continue;
585 map<string, string>::const_iterator addrIt = wtx.mapValue.find("pool_payout_address"); 1626 // The comment is written only by the payout path, so its presence
586 if (addrIt == wtx.mapValue.end() || addrIt->second != address) 1627 // already identifies the transaction as one of ours.
1628 map<string, string>::const_iterator ci = wtx.mapValue.find("comment");
1629 if (ci == wtx.mapValue.end() || ci->second != comment)
587 continue; 1630 continue;
588 map<string, string>::const_iterator amtIt = wtx.mapValue.find("pool_payout_amount"); 1631 if (excluded.count(wtx.GetHash().GetHex()))
589 if (amtIt == wtx.mapValue.end() || amtIt->second != strprintf("%lld", amount)) 1632 continue; // already credited to a different debt
1633 foreach (const CTxOut& txout, wtx.vout)
1634 if (txout.nValue == amount && txout.scriptPubKey == want) {
1635 legacyCandidates.push_back(it->first);
1636 break;
1637 }
1638 }
1639 }
1640
1641 // A legacy match must have CONFIRMED.
1642 //
1643 // The structured path does not need this and deliberately does not have
1644 // it: this process sent that transaction moments ago, and a send that
1645 // failed leaves nothing behind, because SendMoney calls
1646 // RollbackTransactionSpent, which calls EraseFromWallet.
1647 //
1648 // The legacy path is the opposite case in every respect. Those
1649 // transactions are days or weeks old, so whether they confirmed is a
1650 // settled fact rather than an open question, and a pre-upgrade payout that
1651 // never confirmed is evidence of NON-payment about to be read as payment.
1652 // This chain has had a 280-block reorg (see FlushMaturePayouts): a payout
1653 // invalidated by it is still sitting in mapWallet with its comment intact,
1654 // and the first boot after this upgrade is exactly when it gets matched.
1655 // The cost of being wrong here is a miner who is never paid and no record
1656 // that they were owed.
1657 CRITICAL_BLOCK(cs_main)
1658 CRITICAL_BLOCK(cs_mapWallet)
1659 {
1660 for (const uint256& hash : legacyCandidates) {
1661 map<uint256, CWalletTx>::const_iterator it = mapWallet.find(hash);
1662 if (it == mapWallet.end())
1663 continue; // erased under us; it is not evidence of anything
1664 int depth = it->second.GetDepthInMainChain();
1665 if (depth < 1) {
1666 // Loud: this is the difference between a miner having been
1667 // paid and a miner about to be paid twice, decided on a fact
1668 // that is easy to check and impossible to recover afterwards.
1669 LogPrint("payout", "[payout] wallet tx %s carries the payout"
1670 " comment for height %d and an output of exactly %s to"
1671 " %s, but it is NOT in the main chain (depth %d) --"
1672 " refusing to read it as payment; this recipient will"
1673 " be paid\n",
1674 hash.GetHex().c_str(), blockHeight,
1675 FormatMoney(amount).c_str(), address.c_str(), depth);
590 continue; 1676 continue;
591 txidRet = wtx.GetHash().GetHex(); 1677 }
592 return true; 1678 txidRet = hash.GetHex();
1679 return PAYOUT_PROBE_LEGACY;
593 } 1680 }
594 } 1681 }
595 return false; 1682 return PAYOUT_PROBE_NONE;
596} 1683}
597 1684
1685// txid -> "<round id>|<address>", for payments this run has already credited
1686// to a debt on legacy evidence. Stops one pre-upgrade transaction being read
1687// as proof that two different obligations were met. The ledger's paidTxids
1688// covers the same ground durably; this covers the window before the recovered
1689// txid reaches the ledger, and the case where the ledger entry has already
1690// been retired.
1691static std::map<std::string, std::string> gLegacyClaimedTxids;
1692static std::mutex gClaimedTxidMutex;
1693
1694// Every txid already spoken for by some payment OTHER than this one. Collected
1695// before cs_mapWallet is taken: the ledger mutex must not be acquired under a
1696// wallet lock.
1697static void CollectForeignPayoutTxids(int blockHeight, const uint256& blockHash,
1698 const std::string& address,
1699 const std::string& claimKey,
1700 std::set<std::string>& out)
1701{
1702 {
1703 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1704 for (const PendingPayout& pp : gPendingPayouts) {
1705 bool sameRound = SamePayoutRound(pp.blockHeight, pp.blockHash,
1706 blockHeight, blockHash);
1707 for (const auto& kv : pp.paidTxids)
1708 if (!(sameRound && kv.first == address))
1709 out.insert(kv.second);
1710 }
1711 }
1712 {
1713 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
1714 for (const auto& kv : gLegacyClaimedTxids)
1715 if (kv.second != claimKey)
1716 out.insert(kv.first);
1717 }
1718}
1719
1720static void ClaimLegacyPayoutTxid(const std::string& txid, const std::string& claimKey)
1721{
1722 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
1723 gLegacyClaimedTxids[txid] = claimKey;
1724}
1725
1726// blockHash stays a uint256 and not upstream's display string: it is the key
1727// this fork's payout ledger is built on, and a key that has been through
1728// ToString() is a key that can be compared wrong. It is converted to text only
1729// where the round proof needs text. The other three arguments are upstream's,
1730// carried for the round-proof data.
598static void QueuePayouts(int blockHeight, 1731static void QueuePayouts(int blockHeight,
599 const std::string& blockHash, 1732 const uint256& blockHash,
600 const std::string& coinbaseTxid, 1733 const std::string& coinbaseTxid,
601 int64 coinbaseValue, 1734 int64 coinbaseValue,
602 const std::string& foundBy, 1735 const std::string& foundBy,
@@ -604,11 +1737,11 @@ static void QueuePayouts(int blockHeight,
604 uint64 total) 1737 uint64 total)
605{ 1738{
606 PoolRoundProof round; 1739 PoolRoundProof round;
607 round.id = PayoutIdForBlock(blockHeight); 1740 round.id = PayoutIdForBlock(blockHeight, blockHash);
608 round.blockHeight = blockHeight; 1741 round.blockHeight = blockHeight;
609 round.matureAtHeight = blockHeight + COINBASE_MATURITY; 1742 round.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH;
610 round.createdAt = GetTime(); 1743 round.createdAt = GetTime();
611 round.blockHash = blockHash; 1744 round.blockHash = blockHash.ToString();
612 round.coinbaseTxid = coinbaseTxid; 1745 round.coinbaseTxid = coinbaseTxid;
613 round.coinbaseValue = coinbaseValue; 1746 round.coinbaseValue = coinbaseValue;
614 round.foundBy = foundBy; 1747 round.foundBy = foundBy;
@@ -639,9 +1772,21 @@ static void QueuePayouts(int blockHeight,
639 round.operatorReward = operatorCut; 1772 round.operatorReward = operatorCut;
640 1773
641 PendingPayout pp; 1774 PendingPayout pp;
642 pp.id = round.id;
643 pp.blockHeight = blockHeight; 1775 pp.blockHeight = blockHeight;
644 pp.matureAtHeight = blockHeight + COINBASE_MATURITY; 1776 pp.blockHash = blockHash;
1777 // COINBASE_SPENDABLE_DEPTH, not COINBASE_MATURITY. Paying at depth 100
1778 // means the block's own coinbase is still 20 blocks short of spendable, so
1779 // the payment has to come out of some *other* mature coin -- the pool runs
1780 // a permanent ~19-block float and every payout fails until unrelated coin
1781 // matures. The earlier "96% of 934 payments went through at depth 100"
1782 // measurement that justified the shorter wait was taken while this node was
1783 // alone on a private fork paying itself, so it says nothing about a pool
1784 // competing on the real chain. Waiting the extra 20 blocks makes each
1785 // payout funded by construction, and puts the maturity gate past the
1786 // reorg depth that orphaned 280 blocks of obligations. Upstream uses
1787 // COINBASE_MATURITY here; this is a considered disagreement, not an
1788 // oversight -- see the merge commit message.
1789 pp.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH;
645 1790
646 int skipped = 0; 1791 int skipped = 0;
647 int64 burned = 0; // amount owed to miners with bad/empty addresses 1792 int64 burned = 0; // amount owed to miners with bad/empty addresses
@@ -668,9 +1813,13 @@ static void QueuePayouts(int blockHeight,
668 round.forfeitedBadAddress = burned; 1813 round.forfeitedBadAddress = burned;
669 1814
670 { 1815 {
671 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); 1816 VersionedBlob snap;
672 gPendingPayouts.push_back(pp); 1817 {
673 SavePendingPayoutsLocked(); 1818 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1819 gPendingPayouts.push_back(pp);
1820 snap = SerializePendingPayoutsLocked();
1821 }
1822 PublishPendingPayouts(snap);
674 } 1823 }
675 RecordPoolRound(round); 1824 RecordPoolRound(round);
676 1825
@@ -689,37 +1838,340 @@ static void QueuePayouts(int blockHeight,
689 : ""); 1838 : "");
690} 1839}
691 1840
692// Called from event loop: copy matured entries to the exec queue (fast). 1841// Called from the event loop: explode matured ledger entries into per-recipient
693// The ledger entry stays on disk until every recipient has a txid recorded. 1842// in-flight records (fast; no wallet or disk work beyond one small write).
694// On restart, FindRecordedPoolPayout() checks wallet metadata before sending, 1843//
695// so a crash after SendMoney but before the ledger update does not double-pay. 1844// The ledger entry SURVIVES here. That is upstream's model and it is the
1845// crash-safe one: this fork used to erase the entry as soon as its payments
1846// were handed off, so a death in that window left the debt described by
1847// nothing at all -- a container restart on 2026-07-30 silently dropped
1848// ~1,156 BTF exactly that way. An entry is removed only once every recipient
1849// in it has a txid recorded (PayoutFullyPaid), which the payout thread does.
1850//
1851// gPaying is kept on top of that as the in-flight working set: it carries the
1852// per-recipient retry/deferral state that a per-round `nextAttempt` cannot
1853// express, and it is what the payout thread iterates. It is a cache of work
1854// derived from the ledger, never an independent record of debt -- if it were
1855// lost entirely the ledger would rebuild it on the next flush.
696static void FlushMaturePayouts() 1856static void FlushMaturePayouts()
697{ 1857{
1858 // Which funding blocks are still on the main chain? Asked with no payout
1859 // lock held: PayoutBlockStillOnChain takes cs_main, and cs_main is held by
1860 // ProcessBlock and SendMoney for long stretches. Holding the ledger mutex
1861 // across it would let the event loop park the payout thread behind a chain
1862 // lock it has no interest in. (It is not a deadlock either way -- nothing
1863 // takes the ledger mutex while holding cs_main -- but it is exactly the
1864 // kind of stall this merge is removing from the ledger path.)
1865 std::vector<uint256> orphaned;
1866 {
1867 std::vector<uint256> hashes;
1868 {
1869 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
1870 for (const PendingPayout& pp : gPendingPayouts)
1871 hashes.push_back(pp.blockHash);
1872 }
1873 for (const uint256& h : hashes)
1874 if (!PayoutBlockStillOnChain(h))
1875 orphaned.push_back(h);
1876 }
1877
698 std::vector<PendingPayout> ready; 1878 std::vector<PendingPayout> ready;
1879 VersionedBlob ledgerSnap;
699 { 1880 {
700 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex); 1881 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
701 std::lock_guard<std::mutex> queue(gPayoutExecMutex); 1882
702 int64 now = GetTime(); 1883 // Void obligations whose funding block lost a reorg. The coin that was
1884 // meant to pay them does not exist, so paying anyway spends real-chain
1885 // coin the pool owes to somebody else.
1886 bool voided = false;
1887 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) {
1888 if (std::find(orphaned.begin(), orphaned.end(), it->blockHash) == orphaned.end()) {
1889 ++it;
1890 continue;
1891 }
1892 int64 total = 0;
1893 for (auto& kv : it->amounts) total += kv.second;
1894 // Always: one line per voided entry, never a collapsed summary.
1895 // A reorg voids these in a burst from this one source line.
1896 LogPrintAlways("payout", "[payout] block %d (%s) was orphaned -- voiding %zu"
1897 " payment(s) totalling %s\n",
1898 it->blockHeight, it->blockHash.ToString().substr(0, 14).c_str(),
1899 it->amounts.size(), FormatMoney(total).c_str());
1900 it = gPendingPayouts.erase(it);
1901 voided = true;
1902 }
1903 if (voided) ledgerSnap = SerializePendingPayoutsLocked();
1904
703 for (const PendingPayout& pp : gPendingPayouts) { 1905 for (const PendingPayout& pp : gPendingPayouts) {
704 if (nBestHeight < pp.matureAtHeight) 1906 if (nBestHeight < pp.matureAtHeight)
705 continue; 1907 continue;
706 if (pp.nextAttempt > now)
707 continue;
708 if (PayoutFullyPaid(pp)) 1908 if (PayoutFullyPaid(pp))
709 continue; 1909 continue;
710 if (gQueuedPayouts.count(pp.id)) 1910 // An entry with neither a hash nor a height cannot be told apart
1911 // from another in the same state, so a txid recorded against it
1912 // could settle the wrong debt. Paying it is the unsafe move;
1913 // leaving it standing and saying so is not.
1914 if (!PayoutRoundIdentifiable(pp.blockHeight, pp.blockHash)) {
1915 LogPrint("payout", "[payout] WARNING: ledger entry maturing at"
1916 " height %d names neither a block hash nor a height --"
1917 " refusing to pay %zu recipient(s) that cannot be told"
1918 " apart from another round's\n",
1919 pp.matureAtHeight, pp.amounts.size());
711 continue; 1920 continue;
712 LogPrint("payout", "[payout] block height %d matured -- queuing payout %s (%zu recipients)\n", 1921 }
713 pp.matureAtHeight, pp.id.c_str(), pp.amounts.size());
714 ready.push_back(pp); 1922 ready.push_back(pp);
715 gQueuedPayouts.insert(pp.id);
716 } 1923 }
717 for (auto& pp : ready)
718 gPayoutExecQueue.push_back(pp);
719 } 1924 }
1925 // Outside the ledger mutex: publishing is I/O and the event loop is the
1926 // one thread that must never wait on the payout thread's disk.
1927 PublishPendingPayouts(ledgerSnap);
1928
1929 if (ready.empty())
1930 return;
1931
1932 VersionedBlob payingSnap;
1933 {
1934 std::lock_guard<std::mutex> queue(gPayoutExecMutex);
1935 size_t added = 0;
1936 for (const PendingPayout& pp : ready) {
1937 const std::string id = PayoutIdForBlock(pp.blockHeight, pp.blockHash);
1938 if (gQueuedPayouts.count(id))
1939 continue; // already exploded into gPaying and still in flight
1940 size_t n = 0;
1941 for (const auto& kv : pp.amounts) {
1942 // Upstream's per-recipient dedupe: a recipient already carrying
1943 // a txid is done, even though the entry as a whole is not.
1944 if (pp.paidTxids.count(kv.first))
1945 continue;
1946 // The invariant, checked against the thing it is an invariant
1947 // of: at most one in-flight record per (round, address). The
1948 // round-level set above is an optimisation that can be wrong --
1949 // it is memory only, it is rebuilt from gPaying at startup, and
1950 // a stale `ready` snapshot can race a round finishing --
1951 // whereas gPaying is the list being protected. A second record
1952 // here is a second standing order to send the same money.
1953 if (PaymentInFlightLocked(pp.blockHeight, pp.blockHash, kv.first))
1954 continue;
1955 PayingRecord r;
1956 r.id = ++gPayingNextId;
1957 r.blockHeight = pp.blockHeight;
1958 r.blockHash = pp.blockHash;
1959 r.matureAtHeight = pp.matureAtHeight;
1960 r.address = kv.first;
1961 r.amount = kv.second;
1962 gPaying.push_back(r);
1963 n++;
1964 }
1965 gQueuedPayouts.insert(id);
1966 added += n;
1967 LogPrint("payout", "[payout] block height %d matured -- queuing"
1968 " payout %s (%zu of %zu recipients still owed)\n",
1969 pp.matureAtHeight, id.c_str(), n, pp.amounts.size());
1970 }
1971 if (added) payingSnap = SerializePayingLocked();
1972 }
1973 PublishPaying(payingSnap);
1974}
1975
1976// Settle a payment: it is either done or abandoned, either way it stops being
1977// in flight. Note this only retires the working record -- whether the miner is
1978// still OWED the money is the ledger's business, and the ledger is updated
1979// separately by RecordPayoutTxid().
1980//
1981// Erases by record id, not by comparing fields. Value-matching was only an
1982// identity for as long as no two records could be equal, and the restart bug
1983// above produced records that were equal in every field: the first match then
1984// absorbed every erase and every state update, and the second was never
1985// touched again. The id names the slot the caller took its copy from, so the
1986// update lands where it belongs whatever else is in the list.
1987static void ForgetPayment(const PayingRecord& r)
1988{
1989 VersionedBlob snap;
1990 {
1991 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
1992 for (auto it = gPaying.begin(); it != gPaying.end(); ++it)
1993 if (it->id == r.id) { gPaying.erase(it); break; }
1994
1995 // When the last record for a round is gone the round is no longer in
1996 // flight, so a later flush may explode it again if the ledger still
1997 // shows recipients owed. That is the intended behaviour and it is now
1998 // the only way a stalled payment ever leaves this list: a payment that
1999 // keeps failing is retried forever rather than given up on, so its
2000 // record stays here, and the flush that would re-explode it is stopped
2001 // by PaymentInFlightLocked.
2002 bool stillInFlight = false;
2003 for (const auto& e : gPaying)
2004 if (SamePayoutRound(e.blockHeight, e.blockHash,
2005 r.blockHeight, r.blockHash)) {
2006 stillInFlight = true;
2007 break;
2008 }
2009 if (!stillInFlight)
2010 gQueuedPayouts.erase(PayoutIdForBlock(r.blockHeight, r.blockHash));
2011
2012 snap = SerializePayingLocked();
2013 }
2014 PublishPaying(snap);
2015}
2016
2017// Persist an updated attempt count / backoff for a payment still owed.
2018static void ReschedulePayment(const PayingRecord& r)
2019{
2020 VersionedBlob snap;
2021 {
2022 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
2023 for (auto& e : gPaying)
2024 if (e.id == r.id) {
2025 e.retries = r.retries;
2026 e.deferrals = r.deferrals;
2027 e.retryAfter = r.retryAfter;
2028 // Carry the cleared probe flag back, or every retry re-scans
2029 // the whole wallet -- the cost this merge is specifically
2030 // removing.
2031 e.fProbeWallet = r.fProbeWallet;
2032 break;
2033 }
2034 snap = SerializePayingLocked();
2035 }
2036 PublishPaying(snap);
2037}
2038
2039// Record that a recipient has been paid, in the ledger, which is the only
2040// durable statement that the debt is settled.
2041//
2042// The file write happens with gPayoutLedgerMutex RELEASED. Upstream writes the
2043// file inside the per-recipient loop with the mutex held, so every recipient
2044// costs the event loop a full serialise+fsync it has to wait for in
2045// FlushMaturePayouts. Ordering against the event loop's own writes is the
2046// snapshot version's job, not the mutex's -- see PublishVersioned.
2047static void RecordPayoutTxid(const PayingRecord& r, const std::string& txid)
2048{
2049 VersionedBlob blob;
2050 bool complete = false;
2051 {
2052 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
2053 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ++it) {
2054 if (!SamePayoutRound(it->blockHeight, it->blockHash,
2055 r.blockHeight, r.blockHash))
2056 continue;
2057 it->paidTxids[r.address] = txid;
2058 if (PayoutFullyPaid(*it)) {
2059 LogPrint("payout", "[payout] payout for block %d complete --"
2060 " removing from ledger\n", it->blockHeight);
2061 gPendingPayouts.erase(it);
2062 complete = true;
2063 }
2064 break;
2065 }
2066 blob = SerializePendingPayoutsLocked();
2067 }
2068 PublishPendingPayouts(blob);
2069 // The round proof records a txid per address, so this is per recipient and
2070 // not only on completion.
2071 MarkPoolRoundPaid(PayoutIdForBlock(r.blockHeight, r.blockHash), r.address, txid);
2072 if (complete)
2073 LogPrint("payout", "[payout] block %d fully settled\n", r.blockHeight);
2074}
2075
2076// Write a recipient off.
2077//
2078// Only ever called for a state that is DECIDABLE -- currently one: the ledger
2079// names an address that is not a valid address, so no transaction paying it
2080// can be constructed by this or any other build. It is not called, and must
2081// not be, because a payment has failed some number of times. "The wallet said
2082// no N times" is not a decidable state, it is a description of how long we
2083// have been asking; a debt written off on the strength of it is a debt the
2084// operator keeps, and the arithmetic that decided WHY the wallet said no was
2085// wrong for a year (see PayoutThreadFn).
2086//
2087// The amount is removed from the entry rather than marked paid, so the money
2088// is not reported as sent. Once nothing is left owed the entry retires. The
2089// round proof is told separately, because dropping the obligation from the
2090// ledger while pool_rounds.json goes on publishing the address as owed with
2091// "txid": null leaves the pool's two records of the same round permanently
2092// disagreeing.
2093static void ForfeitPayoutRecipient(const PayingRecord& r, const std::string& reason)
2094{
2095 LogPrintAlways("payout", "[payout] writing off %s owed to %s for block %d: %s\n",
2096 FormatMoney(r.amount).c_str(), r.address.c_str(), r.blockHeight,
2097 reason.c_str());
2098 VersionedBlob blob;
2099 {
2100 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
2101 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ++it) {
2102 if (!SamePayoutRound(it->blockHeight, it->blockHash,
2103 r.blockHeight, r.blockHash))
2104 continue;
2105 it->amounts.erase(r.address);
2106 // PayoutFullyPaid() returns true for an empty amounts map -- there
2107 // is no recipient in it without a txid -- so the
2108 // `amounts.empty()` disjunct that used to be here decided nothing.
2109 if (PayoutFullyPaid(*it))
2110 gPendingPayouts.erase(it);
2111 break;
2112 }
2113 blob = SerializePendingPayoutsLocked();
2114 }
2115 PublishPendingPayouts(blob);
2116 MarkPoolRoundForfeited(PayoutIdForBlock(r.blockHeight, r.blockHash),
2117 r.address, reason);
2118}
2119
2120// What a failed send means for the payment's retry budget.
2121//
2122// A deferral is "the wallet's condition is temporarily wrong": wait, and do
2123// not hold it against this payment. A fault is "the wallet is funded and
2124// unlocked and still would not do it", which is worth logging loudly and
2125// retrying with backoff.
2126//
2127// This used to be decided by re-reading GetBalance() after the failure and
2128// comparing it against `amount + nTransactionFee + CENT`. Every term of that
2129// was wrong:
2130//
2131// * nTransactionFee is 0 in production. It is only ever assigned from the
2132// GUI preferences dialog (gui.cpp), which a headless pool node does not
2133// have, so the whole reserve was the CENT.
2134//
2135// * The fee the reserve is for is GetMinFee(fDiscount=true), a STEP
2136// function of the serialised size: 0 below 10,000 bytes, and
2137// (1 + bytes/1000) * CENT at or above it -- so 11 * CENT at the step.
2138// There is no required-fee value between 1 and 11 CENT. A one-CENT
2139// reserve is therefore never the right number: below the step nothing
2140// needs reserving, and at the step it is short by a factor of eleven.
2141//
2142// * A pool wallet close to broke selects many small inputs BECAUSE it is
2143// close to broke, and ~56 signed P2PKH inputs is 10 KB. The trigger and
2144// the failure are positively correlated: the guess is wrong precisely
2145// when the pool is nearly out of money.
2146//
2147// Concretely, with a balance of amount + 0.05: the pre-send gate passes,
2148// CreateTransaction builds a >10 KB transaction, the fee becomes 0.11,
2149// SelectCoins(amount + 0.11) fails, SendMoney returns false -- and the old
2150// arithmetic said (amount + 0.05) < (amount + 0.01) is false, so "funded",
2151// so a fault. The condition is stable, so it recurred every 60 s, and 1440 of
2152// them erased the debt with the coin still in the operator's wallet.
2153//
2154// So stop guessing. The wallet is asked to say what it did, and this only
2155// classifies the answer. WALLET_SEND_INSUFFICIENT_FUNDS is coin selection
2156// failing to cover amount plus whatever fee that size of transaction
2157// required, whatever that turned out to be -- no reserve to get wrong.
2158enum PayoutFailureClass {
2159 PAYOUT_FAILURE_DEFER = 0, // conditions are wrong; wait for them to change
2160 PAYOUT_FAILURE_FAULT // funded and unlocked, and it still would not go
2161};
2162
2163static PayoutFailureClass ClassifyPayoutSendFailure(WalletSendResult result,
2164 bool fWalletLocked)
2165{
2166 // A locked wallet cannot sign. Nothing about this payment is wrong.
2167 if (fWalletLocked)
2168 return PAYOUT_FAILURE_DEFER;
2169 if (result == WALLET_SEND_INSUFFICIENT_FUNDS)
2170 return PAYOUT_FAILURE_DEFER;
2171 return PAYOUT_FAILURE_FAULT;
720} 2172}
721 2173
722// Payout execution thread: drains gPayoutExecQueue, calls SendMoney. 2174// Payout execution thread: works through gPaying, calls SendMoney.
723// Runs independently of the event loop so slow wallet operations 2175// Runs independently of the event loop so slow wallet operations
724// (SendMoney takes cs_main) never stall miner I/O. 2176// (SendMoney takes cs_main) never stall miner I/O.
725static void PayoutThreadFn(void*) 2177static void PayoutThreadFn(void*)
@@ -728,103 +2180,270 @@ static void PayoutThreadFn(void*)
728 LogPrint("payout", "[payout] payout thread started\n"); 2180 LogPrint("payout", "[payout] payout thread started\n");
729 2181
730 while (!fShutdown && gPoolRunning) { 2182 while (!fShutdown && gPoolRunning) {
731 std::vector<PendingPayout> batch; 2183 std::vector<PayingRecord> batch;
732 { 2184 {
733 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 2185 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
734 batch.swap(gPayoutExecQueue); 2186 int64 now = GetTime();
2187 for (const auto& r : gPaying)
2188 if (r.retryAfter <= now)
2189 batch.push_back(r);
735 } 2190 }
736 2191
737 for (auto& ppQueued : batch) { 2192 for (PayingRecord r : batch) {
738 LogPrint("payout", "[payout] executing payments for block matured at %d" 2193 const std::string id = PayoutIdForBlock(r.blockHeight, r.blockHash);
739 " (%zu miners)\n", ppQueued.matureAtHeight, ppQueued.amounts.size()); 2194
740 bool fAbortBatch = false; 2195 // Re-check here and not only at maturity: a payment can sit in this
741 for (auto& kv : ppQueued.amounts) { 2196 // list for hours across retries, and the reorg that kills its
2197 // funding block can land at any point in that window. Upstream has
2198 // no equivalent -- it decides once, when the entry matures.
2199 if (!PayoutBlockStillOnChain(r.blockHash)) {
2200 LogPrintAlways("payout", "[payout] block %d (%s) orphaned while queued"
2201 " -- dropping %s to %s\n",
2202 r.blockHeight, r.blockHash.ToString().substr(0, 14).c_str(),
2203 FormatMoney(r.amount).c_str(), r.address.c_str());
2204 ForgetPayment(r);
2205 continue;
2206 }
2207
2208 // Upstream's per-recipient dedupe, against the ledger rather than
2209 // against the batch: if a txid is already recorded for this
2210 // address, the money went out and nothing more is owed.
2211 {
742 bool alreadyPaid = false; 2212 bool alreadyPaid = false;
743 { 2213 {
744 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); 2214 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
745 for (const PendingPayout& pp : gPendingPayouts) 2215 for (const PendingPayout& pp : gPendingPayouts)
746 if (pp.id == ppQueued.id && pp.paidTxids.count(kv.first)) 2216 if (SamePayoutRound(pp.blockHeight, pp.blockHash,
2217 r.blockHeight, r.blockHash)
2218 && pp.paidTxids.count(r.address)) {
747 alreadyPaid = true; 2219 alreadyPaid = true;
2220 break;
2221 }
748 } 2222 }
749 if (alreadyPaid) 2223 if (alreadyPaid) {
2224 ForgetPayment(r);
750 continue; 2225 continue;
751
752 uint160 h160;
753 if (!AddressToHash160(kv.first, h160)) {
754 // Should not happen -- bad addresses are filtered in QueuePayouts
755 LogPrint("payout", "[payout] SKIPPED bad address '%s'\n",
756 kv.first.c_str());
757 continue;
758 }
759 CScript sc;
760 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
761 CWalletTx wtx;
762 wtx.mapValue["comment"] = strprintf("Pool payout height %d", ppQueued.matureAtHeight);
763 wtx.mapValue["pool_payout_id"] = ppQueued.id;
764 wtx.mapValue["pool_payout_address"] = kv.first;
765 wtx.mapValue["pool_payout_amount"] = strprintf("%lld", kv.second);
766
767 std::string txid;
768 if (FindRecordedPoolPayout(ppQueued.id, kv.first, kv.second, txid))
769 {
770 LogPrint("payout", "[payout] recovered recorded payout %s to %s"
771 " from wallet tx %s\n",
772 FormatMoney(kv.second).c_str(), kv.first.c_str(), txid.c_str());
773 MarkPoolRoundPaid(ppQueued.id, kv.first, txid);
774 }
775 else if (SendMoney(sc, kv.second, wtx))
776 {
777 txid = wtx.GetHash().GetHex();
778 LogPrint("payout", "[payout] paid %s to %s\n",
779 FormatMoney(kv.second).c_str(), kv.first.c_str());
780 MarkPoolRoundPaid(ppQueued.id, kv.first, txid);
781 }
782 else
783 {
784 LogPrint("payout", "[payout] FAILED to pay %s to %s"
785 " (insufficient funds or wallet locked?)\n",
786 FormatMoney(kv.second).c_str(), kv.first.c_str());
787 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
788 for (PendingPayout& pp : gPendingPayouts)
789 if (pp.id == ppQueued.id)
790 pp.nextAttempt = GetTime() + 60;
791 SavePendingPayoutsLocked();
792 fAbortBatch = true;
793 break;
794 }
795
796 {
797 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
798 for (PendingPayout& pp : gPendingPayouts) {
799 if (pp.id != ppQueued.id)
800 continue;
801 pp.paidTxids[kv.first] = txid;
802 pp.nextAttempt = 0;
803 SavePendingPayoutsLocked();
804 break;
805 }
806 } 2226 }
807 } 2227 }
808 2228
809 { 2229 uint160 h160;
810 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex); 2230 if (!AddressToHash160(r.address, h160)) {
811 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { 2231 // Should not happen -- bad addresses are filtered in
812 if (it->id == ppQueued.id && PayoutFullyPaid(*it)) { 2232 // QueuePayouts. If one is here anyway it is decidable and
813 LogPrint("payout", "[payout] payout %s complete -- removing from ledger\n", 2233 // permanent: there is no transaction that pays a string which
814 it->id.c_str()); 2234 // is not an address, so retrying is not waiting for anything.
815 it = gPendingPayouts.erase(it); 2235 //
816 SavePendingPayoutsLocked(); 2236 // This used to only ForgetPayment(), which drops the in-flight
2237 // record and leaves the ledger entry claiming the debt and
2238 // pool_rounds.json advertising the recipient as unpaid
2239 // forever, with the next flush re-exploding it. Write it off
2240 // properly and say why.
2241 const std::string reason =
2242 strprintf("'%s' is not a valid address, so no transaction"
2243 " can pay it", r.address.c_str());
2244 LogPrintAlways("payout", "[payout] SKIPPED bad address '%s' --"
2245 " %s owed for block %d cannot be paid\n",
2246 r.address.c_str(), FormatMoney(r.amount).c_str(),
2247 r.blockHeight);
2248 ForfeitPayoutRecipient(r, reason);
2249 ForgetPayment(r);
2250 continue;
2251 }
2252 CScript sc;
2253 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
2254
2255 // Did a previous process send this and die before recording it?
2256 // Only asked once per record -- see PayingRecord::fProbeWallet.
2257 if (r.fProbeWallet) {
2258 std::string recovered;
2259 const std::string claimKey = id + "|" + r.address;
2260 std::set<std::string> excluded;
2261 CollectForeignPayoutTxids(r.blockHeight, r.blockHash, r.address,
2262 claimKey, excluded);
2263 // A round with a published proof was created by a build whose
2264 // every payment names its round exactly, so only structured
2265 // evidence can settle it -- a comment match would be another
2266 // round's payment colliding on the height.
2267 const bool fAllowLegacy = !PoolRoundHasProof(id);
2268 r.fProbeWallet = false;
2269 PayoutProbeResult probe = FindRecordedPoolPayout(
2270 id, r.blockHeight, r.blockHash, r.address, r.amount,
2271 fAllowLegacy, excluded, recovered);
2272 if (probe != PAYOUT_PROBE_NONE) {
2273 if (probe == PAYOUT_PROBE_LEGACY) {
2274 // Loud on purpose: this is the upgrade path, where the
2275 // only evidence a payment went out is a comment written
2276 // by a binary that is no longer running.
2277 LogPrint("payout", "[payout] %s to %s was already paid by"
2278 " a pre-upgrade build (wallet tx %s carries the"
2279 " payout comment for height %d and an output of"
2280 " exactly this amount, and no structured payout"
2281 " metadata) -- not resending\n",
2282 FormatMoney(r.amount).c_str(), r.address.c_str(),
2283 recovered.c_str(), r.blockHeight);
2284 ClaimLegacyPayoutTxid(recovered, claimKey);
817 } else { 2285 } else {
818 ++it; 2286 LogPrint("payout", "[payout] recovered recorded payout %s to %s"
2287 " from wallet tx %s -- not resending\n",
2288 FormatMoney(r.amount).c_str(), r.address.c_str(),
2289 recovered.c_str());
819 } 2290 }
2291 RecordPayoutTxid(r, recovered);
2292 ForgetPayment(r);
2293 continue;
820 } 2294 }
821 } 2295 }
822 { 2296
823 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 2297 // Waiting for coin to mature is not a failure, so it must not
824 gQueuedPayouts.erase(ppQueued.id); 2298 // consume the retry budget. Burning a retry every 60s means a
2299 // payment the pool fully intends to honour can hit GIVING UP
2300 // 24h later purely because the subsidy funding it had not
2301 // matured yet -- the miner is then silently never paid.
2302 //
2303 // The comparison includes the fee. It used to be against the
2304 // amount alone, so a balance anywhere between amount and
2305 // amount+fee sailed past this gate, failed in SendMoney, and was
2306 // counted as a send FAILURE rather than a wait -- 1440 of those is
2307 // 24 hours, and the recipient was erased with the money left in
2308 // the operator's wallet and nothing on disk saying it was owed.
2309 //
2310 // nTransactionFee is the part of the fee that can be known before
2311 // the transaction exists, and in production it is 0 -- it is only
2312 // ever assigned from the GUI preferences dialog, which a pool node
2313 // does not have -- so including it here changes nothing on this
2314 // pool and is kept only because a GUI operator can set it. The
2315 // rest is GetMinFee(), a function of the serialised size and
2316 // therefore of how many inputs SelectCoins ends up picking, which
2317 // cannot be predicted from here -- so this gate deliberately does
2318 // NOT try to reserve it. Refusing to attempt a payment the wallet
2319 // could actually have made would be its own way of not paying a
2320 // miner. That residual is handled after the fact, by asking the
2321 // wallet why it refused -- see ClassifyPayoutSendFailure.
2322 int64 nSpendable = GetBalance();
2323 if (nSpendable < r.amount + nTransactionFee) {
2324 r.deferrals++;
2325 r.retryAfter = GetTime() + 60;
2326 // Hourly, not every minute: a dry pool would otherwise
2327 // write one line per payment per minute forever.
2328 if (r.deferrals == 1 || r.deferrals % 60 == 0)
2329 LogPrint("payout", "[payout] deferring %s to %s -- only %s"
2330 " spendable, need %s with fee (waiting %d min so far)\n",
2331 FormatMoney(r.amount).c_str(), r.address.c_str(),
2332 FormatMoney(nSpendable).c_str(),
2333 FormatMoney(r.amount + nTransactionFee).c_str(),
2334 r.deferrals);
2335 ReschedulePayment(r);
2336 continue;
825 } 2337 }
826 if (fAbortBatch) 2338
2339 // Likewise a locked wallet: it cannot sign, so every attempt fails
2340 // for a reason that has nothing to do with this payment being
2341 // payable. Left to the retry budget, an operator who locks the
2342 // wallet overnight comes back to erased debts.
2343 if (IsWalletLocked()) {
2344 r.deferrals++;
2345 r.retryAfter = GetTime() + 60;
2346 if (r.deferrals == 1 || r.deferrals % 60 == 0)
2347 LogPrint("payout", "[payout] deferring %s to %s -- wallet is"
2348 " locked (waiting %d min so far)\n",
2349 FormatMoney(r.amount).c_str(), r.address.c_str(),
2350 r.deferrals);
2351 ReschedulePayment(r);
827 continue; 2352 continue;
2353 }
2354
2355 CWalletTx wtx;
2356 wtx.mapValue["comment"] = PayoutComment(r.blockHeight, r.blockHash);
2357 // Upstream's structured metadata. This is what makes the wallet
2358 // probe above possible at all -- the old comment-only form could
2359 // not distinguish two rounds paying the same address the same
2360 // amount. Keyed by the hash-derived id, so a reorg cannot make one
2361 // round's transaction answer for another's.
2362 wtx.mapValue["pool_payout_id"] = id;
2363 wtx.mapValue["pool_payout_address"] = r.address;
2364 wtx.mapValue["pool_payout_amount"] = strprintf("%lld", r.amount);
2365 // The full hash, not only the id's sixteen-digit prefix: the
2366 // prefix of a proof-of-work hash is where the zeros live, and
2367 // the probe needs to tell twin rounds at one height apart.
2368 if (r.blockHash != 0)
2369 wtx.mapValue["pool_payout_block"] = r.blockHash.ToString();
2370
2371 SendMoneyOutcome outcome;
2372 if (SendMoney(sc, r.amount, wtx, outcome)) {
2373 // The wallet holds the transaction before this returns, so a
2374 // crash between here and the ledger write is recoverable: the
2375 // probe above finds it by id+address+amount and does not
2376 // resend. Ledger first, in-flight record second -- the debt is
2377 // described by at least one of them at every instant.
2378 RecordPayoutTxid(r, wtx.GetHash().GetHex());
2379 ForgetPayment(r);
2380 LogPrintAlways("payout", "[payout] paid %s to %s (tx %s)\n",
2381 FormatMoney(r.amount).c_str(), r.address.c_str(),
2382 wtx.GetHash().ToString().substr(0, 12).c_str());
2383 } else {
2384 // Why did it fail? The answer used to decide whether the
2385 // payment eventually got erased, and it was decided by
2386 // guessing from a balance snapshot -- see
2387 // ClassifyPayoutSendFailure for the arithmetic and what it
2388 // cost. The wallet now says what it did, and nothing here
2389 // erases anything.
2390 //
2391 // The lock state is re-read rather than trusted from before
2392 // the attempt: SendMoney takes cs_main and can block for a
2393 // long time, so it may have moved underneath us. The funding
2394 // answer does not need re-reading -- it is a statement about
2395 // the attempt that was actually made.
2396 bool fLocked = IsWalletLocked();
2397 int64 nAfter = GetBalance();
2398
2399 if (ClassifyPayoutSendFailure(outcome.result, fLocked)
2400 == PAYOUT_FAILURE_DEFER) {
2401 // Conditions are temporarily wrong. Wait.
2402 r.deferrals++;
2403 r.retryAfter = GetTime() + 60;
2404 if (r.deferrals == 1 || r.deferrals % 60 == 0)
2405 LogPrint("payout", "[payout] deferring %s to %s -- send"
2406 " failed with %s (%s spendable; the wallet"
2407 " needed a fee of %s for a transaction this"
2408 " size); waiting %d min so far\n",
2409 FormatMoney(r.amount).c_str(), r.address.c_str(),
2410 fLocked ? "the wallet locked" : "too little coin",
2411 FormatMoney(nAfter).c_str(),
2412 FormatMoney(outcome.nFeeRequired).c_str(),
2413 r.deferrals);
2414 ReschedulePayment(r);
2415 } else {
2416 // Funded, unlocked, block on the main chain, and the
2417 // wallet still would not build, sign or accept the
2418 // transaction. That is worth shouting about; it is not
2419 // grounds to keep the money.
2420 //
2421 // There is deliberately no give-up branch. It used to
2422 // forfeit the debt after 1440 of these, which is not a
2423 // decidable state -- it is a count of how long we have
2424 // been asking, and it was reached by misclassifying a
2425 // funding shortfall as a fault. A debt that is never
2426 // written off cannot be written off wrongly. The payment
2427 // stays in flight, keeps its ledger entry, keeps being
2428 // reported in pool_status.json as owed, and keeps
2429 // retrying once a minute until the operator or the wallet
2430 // makes it possible.
2431 r.retries++;
2432 r.retryAfter = GetTime() + 60;
2433 ReschedulePayment(r);
2434 // Every minute at first, then hourly: a fault that is not
2435 // going away must not turn the log into one line a minute
2436 // forever, and must not go quiet either.
2437 if (r.retries <= 5 || r.retries % 60 == 0)
2438 LogPrintAlways("payout", "[payout] FAILED to pay %s to %s"
2439 " (%s spendable, wallet unlocked, block %d on"
2440 " the main chain) -- attempt %d, retrying in"
2441 " 60s; this debt stands until it is paid\n",
2442 FormatMoney(r.amount).c_str(), r.address.c_str(),
2443 FormatMoney(nAfter).c_str(), r.blockHeight,
2444 r.retries);
2445 }
2446 }
828 } 2447 }
829 2448
830 Sleep(500); // check for new work every 500ms 2449 Sleep(500); // check for new work every 500ms
@@ -877,7 +2496,22 @@ static std::string PoolStatusFilePath()
877 return GetAppDir() + "/pool_status.json"; 2496 return GetAppDir() + "/pool_status.json";
878} 2497}
879 2498
880static void WritePoolStatusJsonLocked() 2499// Serialise the operator status file. Caller must hold gStatsMutex (this reads
2500// gStats / gWorkerStats / gPayoutViews); gPoolRoundsMutex is taken and
2501// released inside.
2502//
2503// Split from the write for the same reason as the three ledgers, and for a
2504// worse instance of the same problem. This used to be called from RefreshStats
2505// with gPayoutLedgerMutex held at function scope, and once the ledger writes
2506// gained fsync() that call became two fsyncs -- one on the file, one on its
2507// directory -- performed by the event loop, the thread that owns every miner's
2508// socket, with the mutex the payout thread needs in its hand. Measured on the
2509// build host (ext4 on LVM on SSD): 3.78 ms per publish, every 10 seconds, and
2510// unbounded on a shared virtual disk under load.
2511//
2512// Nothing here needs the ledger mutex at all: the pending-payout rows were
2513// already copied into gPayoutViews under it, further up RefreshStats.
2514static std::string SerializePoolStatusLocked()
881{ 2515{
882 nlohmann::json j; 2516 nlohmann::json j;
883 j["schema"] = "bitflash-pool-status-1"; 2517 j["schema"] = "bitflash-pool-status-1";
@@ -933,10 +2567,19 @@ static void WritePoolStatusJsonLocked()
933 size_t start = gPoolRounds.size() > 10 ? gPoolRounds.size() - 10 : 0; 2567 size_t start = gPoolRounds.size() > 10 ? gPoolRounds.size() - 10 : 0;
934 for (size_t i = start; i < gPoolRounds.size(); i++) { 2568 for (size_t i = start; i < gPoolRounds.size(); i++) {
935 const PoolRoundProof& r = gPoolRounds[i]; 2569 const PoolRoundProof& r = gPoolRounds[i];
2570 // A forfeited recipient is not "unpaid and pending" -- the ledger
2571 // has already dropped the obligation. Counting it as unpaid is
2572 // what made the round proof and the ledger disagree for good.
936 int unpaid = 0; 2573 int unpaid = 0;
937 for (const auto& kv : r.payouts) 2574 int forfeited = 0;
938 if (r.paidTxids.count(kv.first) == 0) 2575 for (const auto& kv : r.payouts) {
2576 if (r.paidTxids.count(kv.first))
2577 continue;
2578 if (r.forfeited.count(kv.first))
2579 forfeited++;
2580 else
939 unpaid++; 2581 unpaid++;
2582 }
940 j["recentRounds"].push_back({ 2583 j["recentRounds"].push_back({
941 {"id", r.id}, 2584 {"id", r.id},
942 {"blockHeight", r.blockHeight}, 2585 {"blockHeight", r.blockHeight},
@@ -945,30 +2588,34 @@ static void WritePoolStatusJsonLocked()
945 {"totalShares", r.totalShares}, 2588 {"totalShares", r.totalShares},
946 {"payouts", (int)r.payouts.size()}, 2589 {"payouts", (int)r.payouts.size()},
947 {"unpaidPayouts", unpaid}, 2590 {"unpaidPayouts", unpaid},
2591 {"forfeitedPayouts", forfeited},
948 {"coinbaseValueBtf", FormatMoney(r.coinbaseValue)}, 2592 {"coinbaseValueBtf", FormatMoney(r.coinbaseValue)},
949 {"minerRewardBtf", FormatMoney(r.minerReward)} 2593 {"minerRewardBtf", FormatMoney(r.minerReward)}
950 }); 2594 });
951 } 2595 }
952 } 2596 }
953 2597
954 std::string path = PoolStatusFilePath(); 2598 return j.dump(2);
955 std::string tmp = path + ".tmp"; 2599}
956 FILE* f = fopen(tmp.c_str(), "w"); 2600
957 if (!f) { 2601// Publish the operator status file. Call with every payout mutex released.
958 LogPrint("pool", "[pool] WARNING: could not write status file %s\n", tmp.c_str()); 2602//
959 return; 2603// Still no version guard and no write mutex, deliberately, and the split does
960 } 2604// not change the argument: RefreshStats remains the only caller of either
961 std::string s = j.dump(2); 2605// half, in production it runs only on the event loop, and it now calls this
962 fwrite(s.c_str(), 1, s.size(), f); 2606// after leaving the scope that serialised -- so the serialise and the write
963 if (fclose(f) != 0) { 2607// stay strictly ordered on one thread and pool_status.json still has a single
964 LogPrint("pool", "[pool] WARNING: could not close status file %s\n", tmp.c_str()); 2608// writer by construction. It shares WriteFileAtomic so the durability rules
965 return; 2609// stay in one place.
966 } 2610//
967#ifdef _WIN32 2611// "In production" is doing real work in that sentence. The payout-ledger
968 remove(path.c_str()); 2612// self-test also calls RefreshStats, from one thread at a time and with the
969#endif 2613// pool not running, which is why it is still a single writer there too. A
970 if (rename(tmp.c_str(), path.c_str()) != 0) 2614// second concurrent caller anywhere would invalidate this paragraph and want
971 LogPrint("pool", "[pool] WARNING: could not replace status file %s\n", path.c_str()); 2615// the same versioned publish the ledgers use.
2616static void PublishPoolStatus(const std::string& blob)
2617{
2618 WriteFileAtomic(PoolStatusFilePath(), blob, "pool status");
972} 2619}
973 2620
974// --------------------------------------------------------------------------- 2621// ---------------------------------------------------------------------------
@@ -996,9 +2643,35 @@ static const double VARDIFF_MAX = 1e12;
996static const int64 VARDIFF_STALL_SECS = 45; // 3x share target -- reacts faster without false-triggering on normal variance 2643static const int64 VARDIFF_STALL_SECS = 45; // 3x share target -- reacts faster without false-triggering on normal variance
997static const double VARDIFF_STALL_CUT = 4.0; // divide difficulty by this much 2644static const double VARDIFF_STALL_CUT = 4.0; // divide difficulty by this much
998 2645
2646// #7: m->hashRate is an EWMA sampled only when a share arrives, so a miner
2647// that stops hashing keeps its last estimate forever and RefreshStats sums
2648// it into the announced pool total for as long as the connection sits open
2649// -- the published pool hashrate ran several times over ground truth
2650// because of exactly this. MinerReportedHashRate() below applies the
2651// policy: a miner gets a grace of HASHRATE_IDLE_MISSES expected share
2652// intervals (expected interval = expected hashes per share at its
2653// difficulty, divided by the estimate itself) with no share, after which
2654// the figure we report falls off with a half-life of one more such
2655// interval. The interval is clamped to [HASHRATE_MIN_INTERVAL,
2656// HASHRATE_MAX_INTERVAL] so a fast miner at the vardiff floor (expected
2657// interval of milliseconds) is not punished for ordinary share-delivery
2658// jitter, and a very slow miner's grace does not stretch past the point
2659// where its estimate means anything. At the vardiff target of one share
2660// per 15s that is a minute of grace, half the rate reported 75s in, and
2661// nothing reported after about seventeen minutes of silence.
2662static const double HASHRATE_IDLE_MISSES = 4.0; // grace, in expected share intervals
2663static const double HASHRATE_MIN_INTERVAL = 1.0; // seconds; floor for the expected interval
2664static const double HASHRATE_MAX_INTERVAL = 3600.0; // seconds; ceiling for the expected interval
2665// Half-lives past the grace beyond which we report a flat zero. 2^-64 of the
2666// original estimate is not a number anyone needs, and stopping here keeps
2667// pow() from walking down into the denormals for a miner that has been quiet
2668// for hours.
2669static const double HASHRATE_DECAY_CUTOFF = 64.0;
2670
999struct Miner { 2671struct Miner {
1000 btf_socket_t fd; 2672 btf_socket_t fd;
1001 bool authorised = false; 2673 bool authorised = false;
2674 bool subscribed = false; // completed mining.subscribe (#19 census)
1002 std::string address; 2675 std::string address;
1003 std::string worker; 2676 std::string worker;
1004 std::string readBuf; 2677 std::string readBuf;
@@ -1022,11 +2695,80 @@ struct Miner {
1022 int64 lastShareTime = 0; // GetTime() of the previous accepted share 2695 int64 lastShareTime = 0; // GetTime() of the previous accepted share
1023 double hashRate = 0.0; // smoothed H/s estimate 2696 double hashRate = 0.0; // smoothed H/s estimate
1024 2697
2698 // Replay guard for mining.submit (#8): nonces already credited on this
2699 // connection for the job it is currently working. Keyed to the job id in
2700 // seenNoncesJob and cleared by SendJob whenever that id changes, so a
2701 // resubmission under a NEW job is never turned away, while a replayed
2702 // (job_id, nonce) pair under the same job is refused before it can be
2703 // credited twice into roundShareCount -- the figure the payout split
2704 // divides by.
2705 std::set<uint32_t> seenNonces;
2706 std::string seenNoncesJob;
2707
1025 explicit Miner(btf_socket_t f) 2708 explicit Miner(btf_socket_t f)
1026 : fd(f), connectTime(GetTime()), 2709 : fd(f), connectTime(GetTime()),
1027 extranonce2(gNextExtranonce2.fetch_add(1)) {} 2710 extranonce2(gNextExtranonce2.fetch_add(1)) {}
1028}; 2711};
1029 2712
2713// What we publish as a miner's H/s: the last EWMA sample, faded out if shares
2714// have stopped arriving (#7, policy at HASHRATE_IDLE_MISSES).
2715//
2716// This reads m->hashRate and never writes it. That is the whole design. The
2717// first attempt decayed the stored estimate in place from the event loop's
2718// sweep, which put a running accumulator behind a figure that is only ever
2719// displayed, and the accumulator misbehaved in two ways:
2720//
2721// - `expected` was recomputed from the already-decayed estimate on every
2722// pass, so each decay raised `expected`, which raised `grace`, which is
2723// the very thing gating the next decay. `grace` simply chased `silent`
2724// upward and the reported rate settled into a 1/t crawl instead of the
2725// half-life curve the comment above promises -- 291x too high at ten
2726// half-lives past the grace. Issue #7 is about the pool over-reporting;
2727// that under-corrected it for hours.
2728// - Once `expected` reached HASHRATE_MAX_INTERVAL it could not rise any
2729// further, the feedback that had been gating each pass was gone, and the
2730// decay began compounding once a second against the total elapsed
2731// silence -- taking a miner from 0.4% of its baseline to 1e-11 of it in
2732// the next hundred seconds, four hours in, for no reason a reader of
2733// either the code or the announced number could see.
2734//
2735// Computing the value instead of accumulating it removes the failure mode
2736// rather than tuning it: with no stored decay state there is nothing for a
2737// tick rate, a missed pass or a duplicated pass to perturb, and the answer
2738// depends only on how long the miner has actually been quiet. It also keeps
2739// the fade out of the estimator -- the EWMA on the share path still blends
2740// against the value it last computed, not against a faded copy of it, so a
2741// miner that resumes recovers on exactly the curve it would have without any
2742// of this.
2743//
2744// `now` is a parameter rather than a GetTime() call so that one refresh fades
2745// every miner to the same instant, and so the policy is directly testable.
2746static double MinerReportedHashRate(const Miner* m, int64 now)
2747{
2748 // Nothing sampled yet: a fresh connection, or one that has found a single
2749 // share and so has no interval to have been timed over. Nothing to fade,
2750 // and nothing to divide by below.
2751 if (m->hashRate <= 0.0 || m->lastShareTime <= 0) return m->hashRate;
2752
2753 double expected = ExpectedHashesForDifficulty(m->difficulty) / m->hashRate;
2754 // Written so that a NaN would land on the floor rather than sail through:
2755 // the clamp is what guarantees `expected` is finite and strictly positive
2756 // below, whatever the division produced.
2757 if (!(expected > HASHRATE_MIN_INTERVAL)) expected = HASHRATE_MIN_INTERVAL;
2758 if (expected > HASHRATE_MAX_INTERVAL) expected = HASHRATE_MAX_INTERVAL;
2759
2760 double silent = (double)(now - m->lastShareTime);
2761 double grace = HASHRATE_IDLE_MISSES * expected;
2762 // grace is at least HASHRATE_IDLE_MISSES seconds, so a clock that has
2763 // stepped backwards gives a negative `silent`, takes this branch, and
2764 // reports the sample unchanged. Nothing here can scale an estimate up.
2765 if (!(silent > grace)) return m->hashRate;
2766
2767 double halflives = (silent - grace) / expected;
2768 if (halflives >= HASHRATE_DECAY_CUTOFF) return 0.0;
2769 return m->hashRate * pow(0.5, halflives);
2770}
2771
1030static std::vector<Miner*> gMiners; 2772static std::vector<Miner*> gMiners;
1031 2773
1032struct FoundBlock { int height; int64 ts; std::string addr; }; 2774struct FoundBlock { int height; int64 ts; std::string addr; };
@@ -1044,6 +2786,57 @@ static void PushReady(btf_socket_t s)
1044 gReadyQueue.push_back(s); 2786 gReadyQueue.push_back(s);
1045} 2787}
1046 2788
2789// Loopback stratum listener: lets a miner running on the SAME machine reach
2790// this pool over plain TCP (127.0.0.1:8334), skipping the .btf rendezvous
2791// entirely. The rendezvous relay only allows one service registration per
2792// pubkey, so the pool service and the node's own peer-accept fight over the
2793// node identity at the relay -- loopback avoids that fight for local miners.
2794static void LocalAcceptFn(void*)
2795{
2796 SOCKET ls = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
2797 if (ls == INVALID_SOCKET)
2798 return;
2799 int yes = 1;
2800 setsockopt(ls, SOL_SOCKET, SO_REUSEADDR, (char*)&yes, sizeof(yes));
2801 struct sockaddr_in addr;
2802 memset(&addr, 0, sizeof(addr));
2803 addr.sin_family = AF_INET;
2804 addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
2805 addr.sin_port = htons(8334);
2806 if (bind(ls, (struct sockaddr*)&addr, sizeof(addr)) != 0 || listen(ls, 16) != 0)
2807 {
2808 LogPrint("pool", "[pool] loopback stratum bind/listen failed on 127.0.0.1:8334\n");
2809 closesocket(ls);
2810 return;
2811 }
2812 LogPrint("pool", "[pool] loopback stratum listening on 127.0.0.1:8334\n");
2813 while (!fShutdown && gPoolRunning)
2814 {
2815 SOCKET c = accept(ls, NULL, NULL);
2816 if (c == INVALID_SOCKET)
2817 {
2818 if (fShutdown || !gPoolRunning)
2819 break;
2820 continue;
2821 }
2822 // #6: let the OS probe a peer that has gone silent, so a half-open
2823 // session eventually errors out of recv and lands in the event
2824 // loop's existing reap path instead of sitting in gMiners forever.
2825 // This is a backstop, not the mechanism: the default cadence before
2826 // the first probe is hours on most stacks, so the idle timeout in
2827 // PoolEventLoop fires long before keepalive does. Miners arriving
2828 // via a relay ride a loopback socket pair whose remote transport is
2829 // the rendezvous socket owned by the tunnel pump threads; keepalive
2830 // cannot reach across that, which is part of why the idle timeout
2831 // exists.
2832 int keepalive = 1;
2833 setsockopt(c, SOL_SOCKET, SO_KEEPALIVE, (char*)&keepalive, sizeof(keepalive));
2834 LogPrint("pool", "[pool] miner arrived via loopback -- pushing to event loop\n");
2835 PushReady((btf_socket_t)c);
2836 }
2837 closesocket(ls);
2838}
2839
1047// --------------------------------------------------------------------------- 2840// ---------------------------------------------------------------------------
1048// I/O helpers -- event loop thread only 2841// I/O helpers -- event loop thread only
1049// --------------------------------------------------------------------------- 2842// ---------------------------------------------------------------------------
@@ -1068,6 +2861,30 @@ static bool SendLine(btf_socket_t fd, const json& j)
1068static const size_t MAX_STRATUM_LINE = 64 * 1024; 2861static const size_t MAX_STRATUM_LINE = 64 * 1024;
1069static const size_t MAX_POOL_MINERS = 1000; 2862static const size_t MAX_POOL_MINERS = 1000;
1070 2863
2864// #19 instrumentation: the "broadcast to N miner(s)" line counts connected
2865// sockets, which says nothing about how many are actually mining. These split
2866// the connection census into connected / subscribed / authorized / recently
2867// active, so an operator can tell a real workforce from a pile of half-open
2868// sessions at a glance.
2869static const int64 POOL_CENSUS_SECS = 60; // how often to log the breakdown
2870static const int64 POOL_ACTIVE_WINDOW = 300; // a miner is "active" if it
2871 // authorized and was seen within
2872 // this window (one payout-rotate
2873 // cycle)
2874static const int64 POOL_UNAUTH_GRACE_SECS = 60; // a connection still not
2875 // authorized after this long is
2876 // idle, not merely starting up
2877// #6: a Miner used to leave gMiners only when RecvLine reported a clean
2878// disconnect. A half-open session (miner host died, NAT state expired,
2879// cable pulled) sends no FIN and reports no error, so it sat in the table
2880// forever -- authorize outpaced disconnect 2:1 on the live pool, and the
2881// table grew toward MAX_POOL_MINERS, past which real miners are refused.
2882// lastSeen is stamped on every complete line received; a miner that has
2883// said nothing for this long is gone in every way that matters. Ten
2884// minutes is far beyond the vardiff target (a share every 15s) and the
2885// stall watchdog (45s), so no miner that is actually hashing comes close.
2886static const int64 POOL_IDLE_TIMEOUT_SECS = 600;
2887
1071// Non-blocking recv: pull whatever bytes are available now into buf, then 2888// Non-blocking recv: pull whatever bytes are available now into buf, then
1072// return one complete line if one is ready. Three outcomes: 2889// return one complete line if one is ready. Three outcomes:
1073// true -- line is filled; buf may still contain more lines 2890// true -- line is filled; buf may still contain more lines
@@ -1151,6 +2968,17 @@ static json MakeNotifyParams(const StratumJob& job, const CBlock& b,
1151static void SendJob(Miner* m, bool clean) 2968static void SendJob(Miner* m, bool clean)
1152{ 2969{
1153 if (!gHaveJob) return; 2970 if (!gHaveJob) return;
2971 // Every path that hands a miner a new job funnels through here, so this
2972 // is the one place the replay guard needs to follow the job id: nonces
2973 // credited under the previous job mean nothing under the new one. Keying
2974 // the clear to the id (rather than clearing unconditionally) keeps a
2975 // same-job re-notify -- the vardiff retarget and stall watchdog both
2976 // re-send the CURRENT job -- from reopening the duplicate window it
2977 // exists to close.
2978 if (m->seenNoncesJob != gCurrentJob.jobId) {
2979 m->seenNonces.clear();
2980 m->seenNoncesJob = gCurrentJob.jobId;
2981 }
1154 CBlock mb = BuildMinerBlock(gCurrentJob, m->extranonce2); 2982 CBlock mb = BuildMinerBlock(gCurrentJob, m->extranonce2);
1155 // Use this miner's own shareTarget (set by vardiff) in the notify params. 2983 // Use this miner's own shareTarget (set by vardiff) in the notify params.
1156 // The participant miner reads p[3] as its share target, and HandleLine 2984 // The participant miner reads p[3] as its share target, and HandleLine
@@ -1176,6 +3004,65 @@ static void BroadcastJob(bool clean)
1176} 3004}
1177 3005
1178// --------------------------------------------------------------------------- 3006// ---------------------------------------------------------------------------
3007// #19: connection census. "broadcast to N miner(s)" only counts authorized
3008// sockets that got a job; it says nothing about the rest of the connection
3009// table. This logs the full breakdown so an operator can tell a real workforce
3010// apart from half-open / never-authorized / idle sessions, and flags when the
3011// single select() loop is running out of room (issue #26).
3012//
3013// "Out of room" is not the same question on both platforms, and FD_SETSIZE
3014// answers a different one on each. On POSIX it bounds the *value* of any
3015// descriptor handed to FD_SET, so what matters is the highest fd we hold. On
3016// Windows an fd_set is a counted array of SOCKET handles whose numeric values
3017// are arbitrary, so what matters is how many we hold, not how large they are.
3018// FD_SETSIZE comes from <sys/select.h> or winsock2.h; this file must not define
3019// it, because redefining it after those headers changes nothing.
3020// ---------------------------------------------------------------------------
3021static void LogPoolConnectionCensus()
3022{
3023 int connected = 0, subscribed = 0, authorized = 0, active = 0, neverAuthIdle = 0;
3024 int64 now = GetTime();
3025#ifndef _WIN32
3026 int maxfd = 0;
3027#endif
3028 for (Miner* m : gMiners) {
3029 connected++;
3030#ifndef _WIN32
3031 if ((int)m->fd > maxfd) maxfd = (int)m->fd;
3032#endif
3033 if (m->subscribed) subscribed++;
3034 if (m->authorised) {
3035 authorized++;
3036 if (m->lastSeen > 0 && now - m->lastSeen <= POOL_ACTIVE_WINDOW)
3037 active++;
3038 } else if (now - m->connectTime >= POOL_UNAUTH_GRACE_SECS) {
3039 neverAuthIdle++; // past the grace period and still not authorized
3040 }
3041 }
3042
3043#ifndef _WIN32
3044 int headroom = (int)FD_SETSIZE - maxfd;
3045 LogPrint("pool", "[pool] census: connected=%d subscribed=%d authorized=%d"
3046 " active(last %llds)=%d never-auth-idle=%d"
3047 " (cap=%zu, maxfd=%d, FD_SETSIZE=%d)\n",
3048 connected, subscribed, authorized, (long long)POOL_ACTIVE_WINDOW,
3049 active, neverAuthIdle, MAX_POOL_MINERS, maxfd, (int)FD_SETSIZE);
3050#else
3051 int headroom = (int)FD_SETSIZE - connected;
3052 LogPrint("pool", "[pool] census: connected=%d subscribed=%d authorized=%d"
3053 " active(last %llds)=%d never-auth-idle=%d"
3054 " (cap=%zu, FD_SETSIZE=%d sockets)\n",
3055 connected, subscribed, authorized, (long long)POOL_ACTIVE_WINDOW,
3056 active, neverAuthIdle, MAX_POOL_MINERS, (int)FD_SETSIZE);
3057#endif
3058 if (headroom <= 16)
3059 LogPrint("pool", "[pool] WARNING: %d slots left before FD_SETSIZE=%d --"
3060 " the select() loop is about to walk into undefined behaviour"
3061 " (#26). Refuse new miners or rebuild with poll/epoll.\n",
3062 headroom, (int)FD_SETSIZE);
3063}
3064
3065// ---------------------------------------------------------------------------
1179// Stratum message handler -- called for each complete line from a miner 3066// Stratum message handler -- called for each complete line from a miner
1180// Returns false if the miner should be disconnected immediately. 3067// Returns false if the miner should be disconnected immediately.
1181// --------------------------------------------------------------------------- 3068// ---------------------------------------------------------------------------
@@ -1243,6 +3130,7 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1243 // ------------------------------------------------------------------ 3130 // ------------------------------------------------------------------
1244 if (method == "mining.subscribe") { 3131 if (method == "mining.subscribe") {
1245 std::string sid = ToHex(&m->fd, 4); 3132 std::string sid = ToHex(&m->fd, 4);
3133 m->subscribed = true;
1246 LogPrint("worker", "[worker->pool] subscribe from fd=%d sid=%s\n", 3134 LogPrint("worker", "[worker->pool] subscribe from fd=%d sid=%s\n",
1247 (int)m->fd, sid.c_str()); 3135 (int)m->fd, sid.c_str());
1248 reply(json::array({json::array({json::array({"mining.notify",sid})}),sid,4})); 3136 reply(json::array({json::array({json::array({"mining.notify",sid})}),sid,4}));
@@ -1349,6 +3237,22 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1349 } 3237 }
1350 unsigned int nNonce; memcpy(&nNonce, nb.data(), 4); 3238 unsigned int nNonce; memcpy(&nNonce, nb.data(), 4);
1351 3239
3240 // Reject a replayed (job, nonce) before it costs a rebuild and a
3241 // RandomX hash: nothing about the work below distinguishes a second
3242 // submission of a valid share from the first, so without this check
3243 // it passes every test and is credited again -- and roundShareCount
3244 // is what the payout split divides by, so each replay multiplies the
3245 // address's cut of the round. The set is per-connection and cleared
3246 // by SendJob on every job change, so only a same-job repeat is
3247 // refused.
3248 if (m->seenNonces.count(nNonce)) {
3249 LogPrint("worker", "[worker->pool] duplicate share from %s"
3250 " nonce=%u job=%s -- already credited, not counting again\n",
3251 m->address.c_str(), nNonce, submitJobId.c_str());
3252 reply(false, "Duplicate share");
3253 return true;
3254 }
3255
1352 // Rebuild this miner's exact block (unique merkle via extranonce2) 3256 // Rebuild this miner's exact block (unique merkle via extranonce2)
1353 CBlock b = BuildMinerBlock(gCurrentJob, m->extranonce2); 3257 CBlock b = BuildMinerBlock(gCurrentJob, m->extranonce2);
1354 b.nNonce = nNonce; 3258 b.nNonce = nNonce;
@@ -1376,6 +3280,7 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1376 m->roundShares++; 3280 m->roundShares++;
1377 m->totalShares++; 3281 m->totalShares++;
1378 m->vardiffShares++; 3282 m->vardiffShares++;
3283 m->seenNonces.insert(nNonce); // replay guard: credit each nonce once per job
1379 int64 shareTime = GetTime(); 3284 int64 shareTime = GetTime();
1380 m->lastSeen = shareTime; 3285 m->lastSeen = shareTime;
1381 roundShareCount[m->address]++; 3286 roundShareCount[m->address]++;
@@ -1480,7 +3385,11 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1480 if (accepted) { 3385 if (accepted) {
1481 // Snapshot height before RebuildJob() (which increments it) 3386 // Snapshot height before RebuildJob() (which increments it)
1482 int foundHeight = gCurrentJob.height; 3387 int foundHeight = gCurrentJob.height;
1483 QueuePayouts(foundHeight, b.GetHash().GetHex(), b.vtx[0].GetHash().GetHex(), 3388 // b is the block we submitted, untouched by ProcessBlock (which
3389 // took a copy), so its hash is what the payout must be keyed on.
3390 // Passed as a uint256, not hex: the ledger key must stay a real
3391 // hash. The remaining arguments are upstream's round-proof data.
3392 QueuePayouts(foundHeight, b.GetHash(), b.vtx[0].GetHash().GetHex(),
1484 b.vtx[0].vout[0].nValue, m->address, 3393 b.vtx[0].vout[0].nValue, m->address,
1485 roundShareCount, roundShareTotal); 3394 roundShareCount, roundShareTotal);
1486 // Reset round shares 3395 // Reset round shares
@@ -1508,6 +3417,1600 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1508 return true; 3417 return true;
1509} 3418}
1510 3419
3420// ---------------------------------------------------------------------------
3421// payout-ledger self-test
3422//
3423// Regression cover for the payout defects fixed in this series. Everything
3424// here runs against a scratch data directory and an in-memory wallet map; it
3425// never opens wallet.dat, never touches a real datadir, and never sends
3426// anything.
3427// ---------------------------------------------------------------------------
3428
3429static bool PayoutTestReadFile(const std::string& path, std::string& out)
3430{
3431 FILE* f = fopen(path.c_str(), "r");
3432 if (!f) return false;
3433 fseek(f, 0, SEEK_END);
3434 long sz = ftell(f);
3435 fseek(f, 0, SEEK_SET);
3436 out.assign((size_t)(sz > 0 ? sz : 0), '\0');
3437 if (sz > 0 && fread(&out[0], 1, (size_t)sz, f) != (size_t)sz) {
3438 fclose(f);
3439 return false;
3440 }
3441 fclose(f);
3442 return true;
3443}
3444
3445// Delete the scratch data directory and everything in it. A sweep and not a
3446// list of known names: LogPrint binds debug.log to whatever GetAppDir() is
3447// when it first fires, so the test's own log lands in here too, and a
3448// directory left behind in src/ after `make tests` is litter.
3449static void PayoutTestRemoveDir(const std::string& dir)
3450{
3451#ifdef _WIN32
3452 WIN32_FIND_DATAA findData;
3453 HANDLE hFind = FindFirstFileA((dir + "\\*").c_str(), &findData);
3454 if (hFind != INVALID_HANDLE_VALUE) {
3455 do {
3456 std::string name = findData.cFileName;
3457 if (name == "." || name == "..") continue;
3458 DeleteFileA((dir + "\\" + name).c_str());
3459 } while (FindNextFileA(hFind, &findData));
3460 FindClose(hFind);
3461 }
3462 RemoveDirectoryA(dir.c_str());
3463#else
3464 // One level of recursion, because CreateTransaction() opens a Berkeley DB
3465 // environment and BDB puts its logs in a "database" subdirectory. Without
3466 // this the scratch directory cannot be removed and the test leaves one
3467 // behind on every run, which is what d5b3503 was for.
3468 DIR* d = opendir(dir.c_str());
3469 if (d) {
3470 struct dirent* ent;
3471 while ((ent = readdir(d)) != NULL) {
3472 std::string name = ent->d_name;
3473 if (name == "." || name == "..") continue;
3474 const std::string path = dir + "/" + name;
3475 if (unlink(path.c_str()) != 0)
3476 PayoutTestRemoveDir(path);
3477 }
3478 closedir(d);
3479 }
3480 rmdir(dir.c_str());
3481#endif
3482}
3483
3484static bool PayoutTestWriteFile(const std::string& path, const std::string& data)
3485{
3486 FILE* f = fopen(path.c_str(), "w");
3487 if (!f) return false;
3488 bool ok = data.empty() || fwrite(data.c_str(), 1, data.size(), f) == data.size();
3489 return fclose(f) == 0 && ok;
3490}
3491
3492// Wipe every piece of payout state this test touches, so each section starts
3493// from the same place and a "restart" can be simulated by reloading from disk.
3494static void PayoutTestResetState()
3495{
3496 {
3497 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3498 gPendingPayouts.clear();
3499 }
3500 {
3501 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3502 gPaying.clear();
3503 gQueuedPayouts.clear();
3504 }
3505 {
3506 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
3507 gPoolRounds.clear();
3508 }
3509 {
3510 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
3511 gLegacyClaimedTxids.clear();
3512 }
3513 {
3514 // A quarantine is meant to last the life of the process; each section
3515 // here is a fresh process as far as the code under test is concerned.
3516 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
3517 gRefusedFiles.clear();
3518 }
3519}
3520
3521// A BTF address whose hash160 is entirely under the test's control, so the
3522// output script the probe compares against can be built without a wallet.
3523static std::string PayoutTestAddress(unsigned char seed)
3524{
3525 uint160 h = 0;
3526 for (int i = 0; i < 20; i++)
3527 ((unsigned char*)&h)[i] = (unsigned char)(seed + i);
3528 return Hash160ToAddress(h);
3529}
3530
3531static CScript PayoutTestScriptFor(const std::string& address)
3532{
3533 uint160 h160;
3534 CScript sc;
3535 if (AddressToHash160(address, h160))
3536 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
3537 return sc;
3538}
3539
3540// Block index entries fabricated by PayoutTestAddWalletTx, so they can be
3541// freed and unhooked when the test finishes. gPayoutTestChainTip is the end of
3542// that chain: blocks are linked onto it rather than each replacing the tip,
3543// because IsInMainChain() is "pnext is set, or I am the tip".
3544static std::vector<CBlockIndex*> gPayoutTestBlockIndexes;
3545static CBlockIndex* gPayoutTestChainTip = NULL;
3546
3547static void PayoutTestClearBlockIndex()
3548{
3549 CRITICAL_BLOCK(cs_main)
3550 {
3551 for (CBlockIndex* p : gPayoutTestBlockIndexes) {
3552 for (std::map<uint256, CBlockIndex*>::iterator it = mapBlockIndex.begin();
3553 it != mapBlockIndex.end(); ) {
3554 if (it->second == p) mapBlockIndex.erase(it++);
3555 else ++it;
3556 }
3557 delete p;
3558 }
3559 gPayoutTestBlockIndexes.clear();
3560 gPayoutTestChainTip = NULL;
3561 pindexBest = NULL;
3562 }
3563}
3564
3565// Put a transaction in the wallet map that looks exactly like one this pool
3566// sent. `nonce` only exists to keep the hashes distinct.
3567//
3568// fConfirmed decides what GetDepthInMainChain() will answer for it, which is
3569// what the legacy probe now gates on. A confirmed one is given a block index
3570// entry of its own: an empty merkle branch hashes to the transaction itself,
3571// so a block whose merkle root IS that hash accepts it at index 0, and
3572// pointing pindexBest at that block puts it in the main chain one deep. An
3573// unconfirmed one keeps hashBlock 0, which is depth 0 -- the shape a payout
3574// left behind by a reorg has.
3575static std::string PayoutTestAddWalletTx(const std::string& comment,
3576 const std::string& address,
3577 int64 amount,
3578 int nonce,
3579 const std::string& structuredId,
3580 bool fConfirmed = true,
3581 const std::string& structuredBlock = "")
3582{
3583 CWalletTx wtx;
3584 wtx.vin.resize(1);
3585 wtx.vin[0].prevout.SetNull();
3586 wtx.vin[0].scriptSig << nonce;
3587 wtx.vout.resize(1);
3588 wtx.vout[0].nValue = amount;
3589 wtx.vout[0].scriptPubKey = PayoutTestScriptFor(address);
3590 if (!comment.empty())
3591 wtx.mapValue["comment"] = comment;
3592 if (!structuredId.empty()) {
3593 wtx.mapValue["pool_payout_id"] = structuredId;
3594 wtx.mapValue["pool_payout_address"] = address;
3595 wtx.mapValue["pool_payout_amount"] = strprintf("%lld", amount);
3596 if (!structuredBlock.empty())
3597 wtx.mapValue["pool_payout_block"] = structuredBlock;
3598 }
3599 uint256 hash = wtx.GetHash();
3600 if (fConfirmed) {
3601 // A block hash distinct from the transaction's own.
3602 uint256 blockHash = hash;
3603 ((unsigned char*)&blockHash)[0] ^= 0xff;
3604 CRITICAL_BLOCK(cs_main)
3605 {
3606 CBlockIndex* pindex = new CBlockIndex();
3607 pindex->hashMerkleRoot = hash;
3608 // Linked onto the end of the fabricated chain rather than simply
3609 // becoming the new tip: IsInMainChain() is "pnext is set, or I am
3610 // the tip", so a second confirmed transaction would otherwise take
3611 // the first one out of the chain again.
3612 pindex->pprev = gPayoutTestChainTip;
3613 pindex->nHeight = gPayoutTestChainTip ? gPayoutTestChainTip->nHeight + 1 : 1;
3614 if (gPayoutTestChainTip)
3615 gPayoutTestChainTip->pnext = pindex;
3616 gPayoutTestChainTip = pindex;
3617 std::map<uint256, CBlockIndex*>::iterator mi =
3618 mapBlockIndex.insert(std::make_pair(blockHash, pindex)).first;
3619 pindex->phashBlock = &mi->first;
3620 pindexBest = pindex;
3621 gPayoutTestBlockIndexes.push_back(pindex);
3622 }
3623 wtx.hashBlock = blockHash;
3624 wtx.nIndex = 0;
3625 }
3626 CRITICAL_BLOCK(cs_mapWallet)
3627 mapWallet[hash] = wtx;
3628 return hash.GetHex();
3629}
3630
3631// A real key, in the keystore, so a test wallet's outputs are IsMine() and can
3632// actually be signed -- both are needed for CreateTransaction to build the
3633// large transaction the fee schedule is being tested against. Inserted
3634// straight into the key maps rather than through AddKey(), which would also
3635// write it to a wallet.dat.
3636static std::string PayoutTestAddSpendableKey()
3637{
3638 CKey key;
3639 key.MakeNewKey();
3640 std::vector<unsigned char> vchPubKey = key.GetPubKey();
3641 CRITICAL_BLOCK(cs_mapKeys)
3642 {
3643 mapKeys[vchPubKey] = key.GetPrivKey();
3644 mapPubKeys[Hash160(vchPubKey)] = vchPubKey;
3645 }
3646 return Hash160ToAddress(Hash160(vchPubKey));
3647}
3648
3649// Give the wallet `n` spendable outputs of `each`, one transaction apiece --
3650// which is the granularity SelectCoins works at, and therefore the number of
3651// inputs the transaction spending them will carry.
3652//
3653// Deliberately not coinbases: a null prevout would make CWalletTx::GetCredit
3654// return 0 until COINBASE_SPENDABLE_DEPTH, and there is no chain here.
3655static void PayoutTestFundWallet(int n, int64 each, const std::string& address)
3656{
3657 const CScript sc = PayoutTestScriptFor(address);
3658 CRITICAL_BLOCK(cs_mapWallet)
3659 {
3660 for (int i = 0; i < n; i++) {
3661 CWalletTx wtx;
3662 wtx.vin.resize(1);
3663 wtx.vin[0].prevout.hash = (uint64)(i + 1);
3664 wtx.vin[0].prevout.n = 0;
3665 wtx.vout.resize(1);
3666 wtx.vout[0].nValue = each;
3667 wtx.vout[0].scriptPubKey = sc;
3668 mapWallet[wtx.GetHash()] = wtx;
3669 }
3670 }
3671}
3672
3673// Defined below with the event loop; the status-file publication test drives
3674// it directly.
3675static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal);
3676
3677int RunPayoutLedgerSelfTest()
3678{
3679 int nFail = 0;
3680 auto check = [&](bool ok, const char* msg) {
3681 fprintf(stdout, " %s %s\n", ok ? "ok" : "FAIL", msg);
3682 if (!ok) nFail++;
3683 };
3684
3685 fprintf(stdout, "payout-ledger self-test\n");
3686
3687 const std::string savedDataDir = strSetDataDir;
3688 const int savedBestHeight = nBestHeight;
3689
3690 // Scratch data directory in the current working directory. GetAppDir()
3691 // follows strSetDataDir, so this is what PayoutFilePath() and
3692 // PayingFilePath() resolve against for the rest of the test.
3693#ifdef _WIN32
3694 unsigned long pid = (unsigned long)GetCurrentProcessId();
3695#else
3696 unsigned long pid = (unsigned long)getpid();
3697#endif
3698 // Absolute, and the process moves into it.
3699 //
3700 // Both are needed by the CreateTransaction section: Berkeley DB resolves
3701 // its log directory relative to the environment home, so a relative
3702 // datadir makes dbenv.open() fail with DB_RUNRECOVERY; and BDB opens
3703 // db.log relative to the working directory, which is src/ during `make
3704 // tests`. Moving in puts it inside the scratch directory, where it dies
3705 // with everything else.
3706 std::string cwd = ".";
3707 {
3708#ifdef _WIN32
3709 char buf[MAX_PATH];
3710 if (GetCurrentDirectoryA(MAX_PATH, buf) > 0) cwd = buf;
3711#else
3712 char buf[4096];
3713 if (getcwd(buf, sizeof(buf)) != NULL) cwd = buf;
3714#endif
3715 }
3716 std::string dir = cwd + "/" +
3717 strprintf("payout_selftest_%d_%lu", (int)GetTime(), pid);
3718#ifdef _WIN32
3719 CreateDirectoryA(dir.c_str(), NULL);
3720 SetCurrentDirectoryA(dir.c_str());
3721#else
3722 mkdir(dir.c_str(), 0700);
3723 if (chdir(dir.c_str()) != 0) { /* stay put; the paths are absolute anyway */ }
3724#endif
3725 strSetDataDir = dir;
3726 fprintf(stdout, " scratch datadir: %s\n", dir.c_str());
3727
3728 const std::string ledgerPath = PayoutFilePath();
3729 const std::string payingPath = PayingFilePath();
3730 const std::string addrA = PayoutTestAddress(0x11);
3731 const std::string addrB = PayoutTestAddress(0x55);
3732
3733 // -- 1. A restart must not duplicate an in-flight record -----------------
3734 //
3735 // The ledger entry is grandfathered (zero hash) on purpose: it keeps
3736 // PayoutBlockStillOnChain out of the picture, which needs a block index,
3737 // and it exercises the height fallback at the same time.
3738 {
3739 PayoutTestResetState();
3740 remove(ledgerPath.c_str());
3741 remove(payingPath.c_str());
3742
3743 PendingPayout pp;
3744 pp.blockHeight = 4242;
3745 pp.blockHash = 0;
3746 pp.matureAtHeight = 4242;
3747 pp.amounts[addrA] = 10 * COIN;
3748 pp.amounts[addrB] = 20 * COIN;
3749 VersionedBlob snap;
3750 {
3751 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3752 gPendingPayouts.push_back(pp);
3753 snap = SerializePendingPayoutsLocked();
3754 }
3755 PublishPendingPayouts(snap);
3756
3757 nBestHeight = 5000;
3758 FlushMaturePayouts();
3759 size_t afterFirstFlush = 0;
3760 {
3761 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3762 afterFirstFlush = gPaying.size();
3763 }
3764 check(afterFirstFlush == 2,
3765 "a matured round explodes into one record per unpaid recipient");
3766
3767 // Restart: drop everything in memory and come back from the files.
3768 for (int restart = 1; restart <= 2; restart++) {
3769 PayoutTestResetState();
3770 LoadPendingPayouts();
3771 LoadPaying();
3772 ReconcilePayouts();
3773
3774 size_t queued = 0;
3775 {
3776 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3777 queued = gQueuedPayouts.size();
3778 }
3779 if (restart == 1)
3780 check(queued == 1,
3781 "startup rebuilds the exploded-round set from the restored"
3782 " in-flight list");
3783
3784 FlushMaturePayouts();
3785 size_t n = 0;
3786 {
3787 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3788 n = gPaying.size();
3789 }
3790 check(n == 2, restart == 1
3791 ? "a restart does not duplicate an in-flight record"
3792 : "a second restart does not duplicate one either");
3793 }
3794
3795 // The other half of the guard: even with the round-level dedupe set
3796 // deliberately emptied, the per-recipient check against gPaying must
3797 // stop a second standing order for the same money.
3798 {
3799 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3800 gQueuedPayouts.clear();
3801 }
3802 FlushMaturePayouts();
3803 size_t n = 0;
3804 {
3805 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3806 n = gPaying.size();
3807 }
3808 check(n == 2,
3809 "the flush refuses a second record for a (round, address) already"
3810 " in flight even with the round set lost");
3811 }
3812
3813 // -- 2. Duplicates already on disk are collapsed as they are read --------
3814 {
3815 PayoutTestResetState();
3816 nlohmann::json arr = nlohmann::json::array();
3817 for (int i = 0; i < 2; i++) {
3818 nlohmann::json o;
3819 o["blockHeight"] = 4242;
3820 o["blockHash"] = uint256(0).ToString();
3821 o["matureAtHeight"] = 4242;
3822 o["address"] = addrA;
3823 o["amount"] = (int64)(10 * COIN);
3824 o["retries"] = i == 0 ? 7 : 0;
3825 o["deferrals"] = i == 0 ? 0 : 3;
3826 arr.push_back(o);
3827 }
3828 check(PayoutTestWriteFile(payingPath, arr.dump(2)),
3829 "wrote a paying file containing the duplicate the old build made");
3830 LoadPaying();
3831 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3832 check(gPaying.size() == 1,
3833 "loading collapses a duplicate in-flight record from disk");
3834 check(gPaying.size() == 1 && gPaying[0].retries == 7 && gPaying[0].deferrals == 3,
3835 "collapsing merges the attempt counts upwards, not resetting them");
3836 }
3837
3838 // -- 3. State updates address a slot, not a value -----------------------
3839 //
3840 // Build the exact pathology the old code produced -- two records equal in
3841 // every field -- and prove the update lands on the one asked for. Under
3842 // value matching the first record absorbed everything and the second was
3843 // untouchable.
3844 {
3845 PayoutTestResetState();
3846 PayingRecord a;
3847 a.blockHeight = 4242; a.blockHash = 0; a.matureAtHeight = 4242;
3848 a.address = addrA; a.amount = 10 * COIN;
3849 PayingRecord b = a;
3850 {
3851 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3852 a.id = ++gPayingNextId;
3853 b.id = ++gPayingNextId;
3854 gPaying.push_back(a);
3855 gPaying.push_back(b);
3856 }
3857 PayingRecord upd = b;
3858 upd.retries = 5;
3859 upd.retryAfter = 1234567;
3860 ReschedulePayment(upd);
3861 {
3862 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3863 check(gPaying.size() == 2 &&
3864 gPaying[0].retries == 0 && gPaying[1].retries == 5,
3865 "rescheduling updates the record it was given, not the first"
3866 " one that looks like it");
3867 }
3868 ForgetPayment(a);
3869 {
3870 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3871 check(gPaying.size() == 1 && gPaying[0].id == b.id,
3872 "forgetting erases the record it was given, not the first"
3873 " one that looks like it");
3874 }
3875 }
3876
3877 // -- 4. retryAfter survives a restart ------------------------------------
3878 {
3879 PayoutTestResetState();
3880 remove(payingPath.c_str());
3881 PayingRecord r;
3882 r.blockHeight = 77; r.blockHash = 0; r.matureAtHeight = 77;
3883 r.address = addrA; r.amount = 3 * COIN;
3884 r.retryAfter = 1700000000LL;
3885 r.deferrals = 9;
3886 VersionedBlob snap;
3887 {
3888 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3889 r.id = ++gPayingNextId;
3890 gPaying.push_back(r);
3891 snap = SerializePayingLocked();
3892 }
3893 PublishPaying(snap);
3894 PayoutTestResetState();
3895 LoadPaying();
3896 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3897 check(gPaying.size() == 1 && gPaying[0].retryAfter == 1700000000LL &&
3898 gPaying[0].deferrals == 9,
3899 "the retry clock that is actually used survives a restart");
3900 }
3901
3902 // -- 5. The ledger file cannot be published from a stale snapshot --------
3903 {
3904 PayoutTestResetState();
3905 remove(ledgerPath.c_str());
3906
3907 PendingPayout one;
3908 one.blockHeight = 100; one.blockHash = 0; one.matureAtHeight = 100;
3909 one.amounts[addrA] = 1 * COIN;
3910 VersionedBlob older, newer;
3911 {
3912 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3913 gPendingPayouts.push_back(one);
3914 older = SerializePendingPayoutsLocked();
3915 PendingPayout two;
3916 two.blockHeight = 200; two.blockHash = 0; two.matureAtHeight = 200;
3917 two.amounts[addrB] = 2 * COIN;
3918 gPendingPayouts.push_back(two);
3919 newer = SerializePendingPayoutsLocked();
3920 }
3921 check(newer.seq > older.seq,
3922 "a later serialisation carries a strictly higher version");
3923
3924 // The order the race produced: the payout thread's older image
3925 // arriving after the event loop's newer one.
3926 PublishPendingPayouts(newer);
3927 PublishPendingPayouts(older);
3928 std::string onDisk;
3929 check(PayoutTestReadFile(ledgerPath, onDisk) && onDisk == newer.blob,
3930 "an older snapshot cannot overwrite a newer published ledger");
3931
3932 // And the ordinary direction still works.
3933 VersionedBlob newest;
3934 {
3935 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3936 gPendingPayouts.clear();
3937 newest = SerializePendingPayoutsLocked();
3938 }
3939 PublishPendingPayouts(newest);
3940 check(PayoutTestReadFile(ledgerPath, onDisk) && onDisk == newest.blob,
3941 "a newer snapshot does replace the published ledger");
3942 }
3943
3944 // -- 6. Concurrent publishers cannot interleave a file -------------------
3945 //
3946 // Two threads opening and renaming the same "<path>.tmp" over the same
3947 // target was the original defect. This test was mutation-tested and did
3948 // not catch it: with the file mutex removed the suite still went green 2
3949 // runs in 5, and the "never leave it unparseable" assertion fired 0 times
3950 // in 5. Two reasons, both fixed here.
3951 //
3952 // First, every image had the same structure and near enough the same
3953 // length, so an interleaving of two of them was still valid JSON. The
3954 // images now alternate between ONE entry and FIVE THOUSAND, so a write
3955 // that lands on top of another leaves either a truncated array or a small
3956 // image followed by the tail of a large one -- neither of which parses.
3957 //
3958 // Second, and worse, almost nothing was being written. Each thread walked
3959 // a permutation of the images, and PublishVersioned drops any blob that is
3960 // not newer than what is on disk, so only the record highs in each
3961 // permutation ever reached the disk -- on the order of ten writes in
3962 // total, four threads, no contention to speak of. Every publish here is
3963 // stamped from a shared counter at the moment it is made, so every one of
3964 // them is newer than the last and every one of them writes.
3965 //
3966 // And the file is watched while the writers run, not only inspected after
3967 // they finish. "The published file is never observed unparseable" is the
3968 // actual invariant -- rename(2) is atomic, so a reader can only ever see a
3969 // completely written image -- and checking it once at the end samples the
3970 // one moment when a torn file is least likely to still be there.
3971 {
3972 const std::string racePath = dir + "/publish_race_selftest.json";
3973 std::mutex raceMutex;
3974 uint64 racePublished = 0;
3975
3976 std::string tiny, huge;
3977 {
3978 nlohmann::json one = nlohmann::json::array();
3979 nlohmann::json o;
3980 o["image"] = 0;
3981 o["entry"] = 0;
3982 o["address"] = addrA;
3983 o["amount"] = (int64)COIN;
3984 one.push_back(o);
3985 tiny = one.dump(2);
3986
3987 nlohmann::json many = nlohmann::json::array();
3988 for (int k = 0; k < 5000; k++) {
3989 nlohmann::json e;
3990 e["image"] = 1;
3991 e["entry"] = k;
3992 e["address"] = addrA;
3993 e["amount"] = (int64)(k + 1) * COIN;
3994 many.push_back(e);
3995 }
3996 huge = many.dump(2);
3997 }
3998
3999 const int kThreads = 4;
4000 const int kPerThread = 100;
4001 const uint64 kTotal = (uint64)kThreads * kPerThread;
4002
4003 std::atomic<uint64> nextSeq(0);
4004 std::atomic<bool> racing(true);
4005 std::atomic<int> tornReads(0);
4006 std::atomic<int> goodReads(0);
4007
4008 std::thread watcher([&]() {
4009 bool seenOnce = false;
4010 while (racing) {
4011 std::string s;
4012 if (!PayoutTestReadFile(racePath, s)) {
4013 // Only meaningful once the file exists at all.
4014 if (seenOnce) tornReads++;
4015 continue;
4016 }
4017 seenOnce = true;
4018 try {
4019 if (nlohmann::json::parse(s).is_array()) goodReads++;
4020 else tornReads++;
4021 } catch (...) {
4022 tornReads++;
4023 }
4024 }
4025 });
4026
4027 std::vector<std::thread> threads;
4028 for (int t = 0; t < kThreads; t++) {
4029 threads.push_back(std::thread([&]() {
4030 for (int i = 0; i < kPerThread; i++) {
4031 VersionedBlob v;
4032 v.seq = ++nextSeq;
4033 v.blob = (v.seq & 1) ? tiny : huge;
4034 PublishVersioned(racePath, v, raceMutex, racePublished,
4035 "publish race selftest");
4036 }
4037 }));
4038 }
4039 for (std::thread& th : threads)
4040 th.join();
4041 racing = false;
4042 watcher.join();
4043
4044 check(goodReads > 0,
4045 "the race test actually observed the file while it was being"
4046 " written");
4047 check(tornReads == 0,
4048 "four threads publishing the same file never leave it"
4049 " unparseable, at any instant while they are doing it");
4050
4051 // The highest sequence number issued is the one the file must settle
4052 // on: PublishVersioned refuses anything older, so whichever order the
4053 // writes complete in, that image is the last one that can land.
4054 const std::string& expected = (kTotal & 1) ? tiny : huge;
4055 std::string onDisk;
4056 bool read = PayoutTestReadFile(racePath, onDisk);
4057 check(read && onDisk == expected,
4058 "the published file settles on the highest version, whatever the"
4059 " completion order");
4060 remove(racePath.c_str());
4061 remove((racePath + ".tmp").c_str());
4062 }
4063
4064 // -- 7. Grandfathered zero-hash rounds do not answer for each other ------
4065 {
4066 uint256 h1 = 0, h2 = 0;
4067 h1.SetHex("00000000000000000000000000000000000000000000000000000000000000aa");
4068 h2.SetHex("00000000000000000000000000000000000000000000000000000000000000bb");
4069 check(!SamePayoutRound(100, 0, 200, 0),
4070 "two zero-hash entries at different heights are different rounds");
4071 check(SamePayoutRound(100, 0, 100, 0),
4072 "two zero-hash entries at the same height are the same round");
4073 check(SamePayoutRound(100, h1, 200, h1),
4074 "a real hash identifies the round on its own");
4075 check(!SamePayoutRound(100, h1, 100, h2),
4076 "different hashes at one height are different rounds");
4077 check(!SamePayoutRound(100, h1, 100, 0),
4078 "a grandfathered entry never matches a hash-keyed one");
4079 check(!PayoutRoundIdentifiable(0, 0) && PayoutRoundIdentifiable(0, h1) &&
4080 PayoutRoundIdentifiable(7, 0),
4081 "an entry with neither hash nor height is not adjudicable");
4082
4083 // What actually makes a grandfathered same-height pair safe, since the
4084 // comment on FindRecordedPoolPayout used to claim it was the orphan
4085 // check. It is not: the orphan check passes every zero-hash entry, and
4086 // every pre-upgrade entry has a zero hash.
4087 check(PayoutBlockStillOnChain(0),
4088 "the orphan check passes every grandfathered entry, so it cannot"
4089 " be what voids a losing twin");
4090 PayoutTestResetState();
4091 remove(ledgerPath.c_str());
4092 remove(payingPath.c_str());
4093 {
4094 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4095 PendingPayout twin;
4096 twin.blockHeight = 300; twin.blockHash = 0; twin.matureAtHeight = 300;
4097 twin.amounts[addrA] = 1 * COIN;
4098 gPendingPayouts.push_back(twin);
4099 gPendingPayouts.push_back(twin); // the losing side of a reorg
4100 }
4101 nBestHeight = 5000;
4102 FlushMaturePayouts();
4103 {
4104 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
4105 check(gPaying.size() == 1,
4106 "two grandfathered rounds at one height are one round, so the"
4107 " pair presents as one debt and is paid at most once");
4108 }
4109
4110 // End to end: recording a payment against one grandfathered round must
4111 // not settle the other's debt.
4112 PayoutTestResetState();
4113 remove(ledgerPath.c_str());
4114 {
4115 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4116 PendingPayout first;
4117 first.blockHeight = 100; first.blockHash = 0; first.matureAtHeight = 100;
4118 first.amounts[addrA] = 1 * COIN;
4119 PendingPayout second;
4120 second.blockHeight = 200; second.blockHash = 0; second.matureAtHeight = 200;
4121 second.amounts[addrA] = 1 * COIN;
4122 gPendingPayouts.push_back(first);
4123 gPendingPayouts.push_back(second);
4124 }
4125 PayingRecord r;
4126 r.blockHeight = 200; r.blockHash = 0; r.matureAtHeight = 200;
4127 r.address = addrA; r.amount = 1 * COIN;
4128 RecordPayoutTxid(r, "txid-for-the-second-round");
4129 {
4130 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4131 // The second round was fully paid and retired; the first is
4132 // untouched and still owed.
4133 check(gPendingPayouts.size() == 1 &&
4134 gPendingPayouts[0].blockHeight == 100 &&
4135 gPendingPayouts[0].paidTxids.empty(),
4136 "a txid recorded against one zero-hash round leaves the other"
4137 " still owed");
4138 }
4139 }
4140
4141 // -- 8. A payment with only a legacy comment is recognised as paid -------
4142 {
4143 PayoutTestResetState();
4144 const int height = 8080;
4145 const int64 amount = 7 * COIN;
4146 const std::string id = PayoutIdForBlock(height, 0);
4147 const std::string comment = LegacyPayoutComment(height);
4148 std::set<std::string> none;
4149 std::string found;
4150 // No round proof exists for these rounds, so legacy evidence is
4151 // admissible -- computed exactly as the payout thread computes it.
4152 const bool fAllowLegacy = !PoolRoundHasProof(id);
4153
4154 check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, none, found)
4155 == PAYOUT_PROBE_NONE,
4156 "an empty wallet is not evidence that anything was paid");
4157
4158 const std::string legacyTxid =
4159 PayoutTestAddWalletTx(comment, addrA, amount, 1, "");
4160 check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, none, found)
4161 == PAYOUT_PROBE_LEGACY && found == legacyTxid,
4162 "a payment carrying only a pre-upgrade comment is recognised as"
4163 " already paid");
4164
4165 check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found)
4166 == PAYOUT_PROBE_NONE,
4167 "the legacy match requires the output to pay THIS recipient");
4168 check(FindRecordedPoolPayout(id, height, 0, addrA, amount + COIN, fAllowLegacy, none, found)
4169 == PAYOUT_PROBE_NONE,
4170 "the legacy match requires the output to pay exactly this amount");
4171 check(FindRecordedPoolPayout(id, height + 1, 0, addrA, amount, fAllowLegacy, none, found)
4172 == PAYOUT_PROBE_NONE,
4173 "the legacy match requires the comment for this block height");
4174
4175 std::set<std::string> spent;
4176 spent.insert(legacyTxid);
4177 check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, spent, found)
4178 == PAYOUT_PROBE_NONE,
4179 "one legacy transaction cannot settle a second debt");
4180
4181 // A transaction written by THIS build carries pool_payout_id, and must
4182 // never be offered as legacy evidence for a different round -- that is
4183 // exactly what the structured metadata exists to prevent.
4184 const std::string otherId = PayoutIdForBlock(height, 0) + "-other";
4185 PayoutTestAddWalletTx(comment, addrB, amount, 2, otherId);
4186 check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found)
4187 == PAYOUT_PROBE_NONE,
4188 "another round's structured payment is not legacy evidence for"
4189 " this one");
4190
4191 // The structured path still wins where it applies.
4192 const std::string structuredTxid =
4193 PayoutTestAddWalletTx(comment, addrB, amount, 3, id);
4194 check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found)
4195 == PAYOUT_PROBE_STRUCTURED && found == structuredTxid,
4196 "structured payout metadata is matched in preference to a comment");
4197
4198 // A comment-matching transaction that never made it into the chain is
4199 // evidence of NON-payment. This tree has seen a 280-block reorg; a
4200 // pre-upgrade payout invalidated by one is still in mapWallet with its
4201 // comment intact, and the first boot after the upgrade is exactly when
4202 // it would be matched.
4203 CRITICAL_BLOCK(cs_mapWallet)
4204 mapWallet.clear();
4205 PayoutTestClearBlockIndex();
4206 const std::string addrC = PayoutTestAddress(0x33);
4207 PayoutTestAddWalletTx(comment, addrC, amount, 4, "", false);
4208 check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, none, found)
4209 == PAYOUT_PROBE_NONE,
4210 "a pre-upgrade payout that never confirmed is not read as"
4211 " payment");
4212
4213 // The same transaction, confirmed, is.
4214 CRITICAL_BLOCK(cs_mapWallet)
4215 mapWallet.clear();
4216 const std::string confirmedTxid =
4217 PayoutTestAddWalletTx(comment, addrC, amount, 4, "", true);
4218 check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, none, found)
4219 == PAYOUT_PROBE_LEGACY && found == confirmedTxid,
4220 "the same payout, confirmed, is");
4221
4222 // And an unconfirmed one does not shadow a confirmed one that would
4223 // have matched: the probe used to stop at the first comment match.
4224 const std::string alsoConfirmed =
4225 PayoutTestAddWalletTx(comment, addrC, amount, 6, "", true);
4226 PayoutTestAddWalletTx(comment, addrC, amount, 5, "", false);
4227 std::set<std::string> spentFirst;
4228 spentFirst.insert(confirmedTxid);
4229 check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, spentFirst, found)
4230 == PAYOUT_PROBE_LEGACY && found == alsoConfirmed,
4231 "an unconfirmed transaction does not hide a confirmed one behind"
4232 " it");
4233
4234 CRITICAL_BLOCK(cs_mapWallet)
4235 mapWallet.clear();
4236 PayoutTestClearBlockIndex();
4237 }
4238
4239 // -- 9. On-disk compatibility with the live production ledger ------------
4240 {
4241 PayoutTestResetState();
4242 // Exactly the shape of the file on the pool node: amounts, blockHash,
4243 // blockHeight, matureAtHeight, and nothing else.
4244 const char* live =
4245 "[{\"amounts\":{\"%s\":1234500000},"
4246 "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000cd\","
4247 "\"blockHeight\":1000,"
4248 "\"matureAtHeight\":1120}]";
4249 check(PayoutTestWriteFile(ledgerPath, strprintf(live, addrA.c_str())),
4250 "wrote a ledger file in the live production shape");
4251 LoadPendingPayouts();
4252 {
4253 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4254 check(gPendingPayouts.size() == 1 &&
4255 gPendingPayouts[0].blockHeight == 1000 &&
4256 gPendingPayouts[0].matureAtHeight == 1120 &&
4257 gPendingPayouts[0].blockHash != 0 &&
4258 gPendingPayouts[0].amounts.size() == 1 &&
4259 gPendingPayouts[0].amounts[addrA] == 1234500000LL,
4260 "the live production pending_payouts.json still loads");
4261 }
4262
4263 // And the shape the merged build writes, retryAfter included, which
4264 // this build no longer writes but must still read past.
4265 PayoutTestResetState();
4266 const char* merged =
4267 "[{\"amounts\":{\"%s\":100000000},"
4268 "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000ef\","
4269 "\"blockHeight\":2000,\"matureAtHeight\":2120,\"retryAfter\":0,"
4270 "\"paidTxids\":{\"%s\":\"abc123\"}}]";
4271 check(PayoutTestWriteFile(ledgerPath,
4272 strprintf(merged, addrA.c_str(), addrA.c_str())),
4273 "wrote a ledger file in the merged build's shape");
4274 LoadPendingPayouts();
4275 {
4276 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4277 check(gPendingPayouts.size() == 1 &&
4278 gPendingPayouts[0].blockHeight == 2000 &&
4279 gPendingPayouts[0].paidTxids.size() == 1 &&
4280 gPendingPayouts[0].paidTxids[addrA] == "abc123",
4281 "a ledger file carrying retryAfter and paidTxids still loads");
4282 }
4283
4284 // A round trip through this build must not lose anything either.
4285 VersionedBlob snap;
4286 {
4287 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4288 snap = SerializePendingPayoutsLocked();
4289 }
4290 PublishPendingPayouts(snap);
4291 PayoutTestResetState();
4292 LoadPendingPayouts();
4293 {
4294 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4295 check(gPendingPayouts.size() == 1 &&
4296 gPendingPayouts[0].blockHeight == 2000 &&
4297 gPendingPayouts[0].matureAtHeight == 2120 &&
4298 gPendingPayouts[0].paidTxids[addrA] == "abc123",
4299 "what this build writes, this build reads back unchanged");
4300 }
4301 }
4302
4303 // -- 10. A forfeit is published, not just dropped ------------------------
4304 {
4305 PayoutTestResetState();
4306 remove(ledgerPath.c_str());
4307 const std::string roundsPath = dir + "/pool_rounds_selftest.json";
4308 const std::string savedRoundsFile = strPoolRoundsFile;
4309 strPoolRoundsFile = roundsPath;
4310 remove(roundsPath.c_str());
4311
4312 const int height = 606;
4313 PoolRoundProof round;
4314 round.id = PayoutIdForBlock(height, 0);
4315 round.blockHeight = height;
4316 round.matureAtHeight = height;
4317 round.payouts[addrA] = 5 * COIN;
4318 round.payouts[addrB] = 6 * COIN;
4319 RecordPoolRound(round);
4320
4321 {
4322 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4323 PendingPayout pp;
4324 pp.blockHeight = height; pp.blockHash = 0; pp.matureAtHeight = height;
4325 pp.amounts[addrA] = 5 * COIN;
4326 pp.amounts[addrB] = 6 * COIN;
4327 gPendingPayouts.push_back(pp);
4328 }
4329
4330 PayingRecord r;
4331 r.blockHeight = height; r.blockHash = 0; r.matureAtHeight = height;
4332 r.address = addrA; r.amount = 5 * COIN;
4333 ForfeitPayoutRecipient(r, "selftest reason");
4334
4335 {
4336 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4337 check(gPendingPayouts.size() == 1 &&
4338 gPendingPayouts[0].amounts.count(addrA) == 0 &&
4339 gPendingPayouts[0].amounts.count(addrB) == 1,
4340 "a forfeit removes only that recipient from the ledger entry");
4341 }
4342
4343 std::string onDisk;
4344 bool proofOk = false;
4345 if (PayoutTestReadFile(roundsPath, onDisk)) {
4346 try {
4347 nlohmann::json root = nlohmann::json::parse(onDisk);
4348 const nlohmann::json& p = root["rounds"][0]["payouts"];
4349 proofOk = p[addrA].value("forfeited", false) &&
4350 p[addrA].value("forfeitReason", "") == "selftest reason" &&
4351 p[addrA]["txid"].is_null() &&
4352 !p[addrB].contains("forfeited");
4353 } catch (...) {
4354 proofOk = false;
4355 }
4356 }
4357 check(proofOk,
4358 "the published round proof records the forfeit instead of"
4359 " advertising the address as still owed");
4360
4361 // And it survives a reload of the proof file.
4362 {
4363 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
4364 gPoolRounds.clear();
4365 }
4366 LoadPoolRounds();
4367 {
4368 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
4369 check(gPoolRounds.size() == 1 &&
4370 gPoolRounds[0].forfeited.count(addrA) == 1 &&
4371 gPoolRounds[0].forfeited[addrA] == "selftest reason",
4372 "the forfeit is read back from pool_rounds.json");
4373 }
4374
4375 remove(roundsPath.c_str());
4376 remove((roundsPath + ".tmp").c_str());
4377 strPoolRoundsFile = savedRoundsFile;
4378 }
4379
4380 // -- 11. A ledger that failed to load is never written back --------------
4381 //
4382 // The loaders push entries into the live vector inside the try, so an
4383 // exception part way down a file leaves a PARTIAL ledger in memory: the
4384 // entries before the bad one and nothing after it. Publishing that -- and
4385 // ReconcilePayouts publishes unconditionally, milliseconds into startup --
4386 // writes the partial state over the file and destroys the only record of
4387 // what was owed.
4388 {
4389 PayoutTestResetState();
4390 remove(ledgerPath.c_str());
4391
4392 // Two entries, the second with a matureAtHeight that is not a number.
4393 // The first loads; the second throws where it is read.
4394 const std::string corrupt = strprintf(
4395 "[{\"amounts\":{\"%s\":100000000},\"blockHash\":\"%s\","
4396 "\"blockHeight\":10,\"matureAtHeight\":10},"
4397 "{\"amounts\":{\"%s\":900000000},\"blockHash\":\"%s\","
4398 "\"blockHeight\":20,\"matureAtHeight\":\"twenty\"}]",
4399 addrA.c_str(), uint256(0).ToString().c_str(),
4400 addrB.c_str(), uint256(0).ToString().c_str());
4401 check(PayoutTestWriteFile(ledgerPath, corrupt),
4402 "wrote a ledger whose second entry does not parse");
4403
4404 const int64 t0 = GetTime();
4405 LoadPendingPayouts();
4406 const int64 t1 = GetTime();
4407
4408 size_t loaded = 0;
4409 {
4410 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4411 loaded = gPendingPayouts.size();
4412 }
4413 check(loaded == 1,
4414 "a mid-file parse error does leave a partial ledger in memory --"
4415 " which is the reason for everything below");
4416
4417 std::string gone;
4418 check(!PayoutTestReadFile(ledgerPath, gone),
4419 "the file that did not parse is moved out of the way");
4420
4421 // The rename stamps GetTime(); it was taken between t0 and t1.
4422 std::string kept, keptPath;
4423 for (int64 t = t0; t <= t1 && keptPath.empty(); t++) {
4424 const std::string candidate =
4425 strprintf("%s.bad.%lld", ledgerPath.c_str(), (long long)t);
4426 if (PayoutTestReadFile(candidate, kept))
4427 keptPath = candidate;
4428 }
4429 check(!keptPath.empty() && kept == corrupt,
4430 "it is kept, byte for byte, as <path>.bad.<timestamp>");
4431
4432 // The publish that used to destroy it.
4433 VersionedBlob partial;
4434 {
4435 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4436 partial = SerializePendingPayoutsLocked();
4437 }
4438 PublishPendingPayouts(partial);
4439 check(!PayoutTestReadFile(ledgerPath, gone),
4440 "and the partial ledger is not published over it");
4441
4442 // The refusal is per path and lasts the process: a second attempt,
4443 // with more state, is refused too.
4444 {
4445 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4446 PendingPayout pp;
4447 pp.blockHeight = 30; pp.blockHash = 0; pp.matureAtHeight = 30;
4448 pp.amounts[addrA] = 5 * COIN;
4449 gPendingPayouts.push_back(pp);
4450 partial = SerializePendingPayoutsLocked();
4451 }
4452 PublishPendingPayouts(partial);
4453 check(!PayoutTestReadFile(ledgerPath, gone),
4454 "the refusal is not a one-shot -- the path stays unwritable");
4455
4456 if (!keptPath.empty())
4457 remove(keptPath.c_str());
4458 PayoutTestResetState();
4459 remove(ledgerPath.c_str());
4460
4461 // And a file that parses is not quarantined, so the ordinary path is
4462 // still the ordinary path.
4463 check(PayoutTestWriteFile(ledgerPath, "[]"),
4464 "wrote a ledger that parses");
4465 LoadPendingPayouts();
4466 {
4467 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4468 PendingPayout pp;
4469 pp.blockHeight = 40; pp.blockHash = 0; pp.matureAtHeight = 40;
4470 pp.amounts[addrA] = 2 * COIN;
4471 gPendingPayouts.push_back(pp);
4472 partial = SerializePendingPayoutsLocked();
4473 }
4474 PublishPendingPayouts(partial);
4475 std::string written;
4476 check(PayoutTestReadFile(ledgerPath, written) && written == partial.blob,
4477 "a ledger that loaded cleanly is still published normally");
4478 }
4479
4480 // -- 12. A send that failed for want of coin is not a fault --------------
4481 //
4482 // The defect this replaces: the classifier decided "was this a funding
4483 // problem?" by re-reading GetBalance() and allowing amount +
4484 // nTransactionFee + CENT. nTransactionFee is 0 on a headless node, and
4485 // GetMinFee() is a step function -- 0 below 10,000 bytes, 11 * CENT at it
4486 // -- so the allowance was either unnecessary or eleven times too small,
4487 // and the ledger's own retry budget then wrote the debt off after 1440
4488 // repeats of the same stable condition.
4489 //
4490 // This is tested against a real wallet and the real CreateTransaction, not
4491 // against injected values: the honest question is not "does my enum
4492 // switch work" but "does a wallet holding many small coins actually report
4493 // the state I think it does". 60 outputs of 1 BTF give ~180 signed P2PKH
4494 // bytes each, which is where the fee schedule steps. Nothing is committed
4495 // or broadcast -- CreateTransaction only builds.
4496 {
4497 PayoutTestResetState();
4498 CRITICAL_BLOCK(cs_mapWallet)
4499 mapWallet.clear();
4500
4501 // CreateTransaction opens CTxDB("r"), which throws if blkindex.dat is
4502 // not there. Create it in the scratch datadir first.
4503 bool dbOk = true;
4504 try {
4505 CTxDB txdb("cr");
4506 } catch (const std::exception&) {
4507 dbOk = false;
4508 }
4509 check(dbOk, "a scratch block index exists for CreateTransaction to open");
4510
4511 const std::string walletAddr = PayoutTestAddSpendableKey();
4512 const std::string payee = PayoutTestAddress(0x77);
4513 const CScript payeeScript = PayoutTestScriptFor(payee);
4514
4515 // -- the >= 10 KB case: 60 small coins, balance = amount + 0.05 ------
4516 PayoutTestFundWallet(60, 1 * COIN, walletAddr);
4517 const int64 balance = GetBalance();
4518 check(balance == 60 * COIN,
4519 "the test wallet holds 60 separately spendable coins");
4520
4521 const int64 amount = balance - 5 * CENT;
4522 // The pre-send gate in PayoutThreadFn passes: the balance covers the
4523 // amount and the only fee that can be known in advance.
4524 check(balance >= amount + nTransactionFee,
4525 "the pre-send gate would let this payment through");
4526
4527 CWalletTx wtx;
4528 SendMoneyOutcome outcome;
4529 int64 feeRequired = 0;
4530 WalletSendResult result = WALLET_SEND_OK;
4531 bool built = true;
4532 if (dbOk) {
4533 try {
4534 built = CreateTransaction(payeeScript, amount, wtx, feeRequired,
4535 &result);
4536 } catch (const std::exception&) {
4537 built = true; // report as a failure below
4538 }
4539 }
4540 check(dbOk && !built,
4541 "a wallet of many small coins cannot fund amount + the fee its"
4542 " own size demands");
4543 check(dbOk && result == WALLET_SEND_INSUFFICIENT_FUNDS,
4544 "the wallet reports that as a funding shortfall, not as a"
4545 " failure to build");
4546 // GetMinFee(fDiscount) returns 0 below 10,000 bytes and
4547 // (1 + bytes/1000) * CENT at or above, so a required fee of 11 CENT or
4548 // more IS the statement that the transaction crossed 10 KB. Measured
4549 // through the real serialiser rather than asserted.
4550 check(dbOk && feeRequired >= 11 * CENT,
4551 "the transaction it built crossed 10,000 bytes, so the fee"
4552 " stepped from 0 to at least 11 CENT");
4553
4554 // The defect, stated as arithmetic. The old classifier asked whether
4555 // the balance covered amount + nTransactionFee + CENT; it did, so it
4556 // called a permanent funding shortfall a fault and started counting
4557 // towards a write-off.
4558 check(!(balance < amount + nTransactionFee + CENT),
4559 "the reserve this replaces would have judged this send a fault");
4560 check(balance < amount + feeRequired,
4561 "while the fee actually required was more than the balance had");
4562
4563 outcome.result = result;
4564 outcome.nFeeRequired = feeRequired;
4565 check(dbOk && ClassifyPayoutSendFailure(outcome.result, false)
4566 == PAYOUT_FAILURE_DEFER,
4567 "a >= 10 KB send that ran out of coin is deferred, not counted"
4568 " against the payment");
4569
4570 // -- the < 10 KB case: no reserve is needed at all -------------------
4571 //
4572 // The other half of the arithmetic. Below the step the required fee is
4573 // 0, so a reserve of any size is a reason to refuse a payment the
4574 // wallet could have made -- which is its own way of not paying a
4575 // miner.
4576 CRITICAL_BLOCK(cs_mapWallet)
4577 mapWallet.clear();
4578 PayoutTestFundWallet(3, 1 * COIN, walletAddr);
4579 const int64 smallAmount = 3 * COIN - 5 * CENT;
4580 CWalletTx smallTx;
4581 int64 smallFee = -1;
4582 WalletSendResult smallResult = WALLET_SEND_FAILED;
4583 bool smallBuilt = false;
4584 if (dbOk) {
4585 try {
4586 smallBuilt = CreateTransaction(payeeScript, smallAmount, smallTx,
4587 smallFee, &smallResult);
4588 } catch (const std::exception&) {
4589 smallBuilt = false;
4590 }
4591 }
4592 check(dbOk && smallBuilt && smallResult == WALLET_SEND_OK,
4593 "the same balance pays a small transaction with no headroom at"
4594 " all");
4595 check(dbOk && smallFee == 0,
4596 "because below 10,000 bytes the required fee is zero");
4597
4598 // -- and the classification of every other answer --------------------
4599 check(ClassifyPayoutSendFailure(WALLET_SEND_INSUFFICIENT_FUNDS, false)
4600 == PAYOUT_FAILURE_DEFER &&
4601 ClassifyPayoutSendFailure(WALLET_SEND_INSUFFICIENT_FUNDS, true)
4602 == PAYOUT_FAILURE_DEFER &&
4603 ClassifyPayoutSendFailure(WALLET_SEND_FAILED, true)
4604 == PAYOUT_FAILURE_DEFER,
4605 "short of coin or unable to sign is always a deferral");
4606 check(ClassifyPayoutSendFailure(WALLET_SEND_FAILED, false)
4607 == PAYOUT_FAILURE_FAULT,
4608 "funded and unlocked and still refused is a fault");
4609
4610 CRITICAL_BLOCK(cs_mapWallet)
4611 mapWallet.clear();
4612 }
4613
4614 // -- 13. No payout mutex is held across the status-file write ------------
4615 //
4616 // RefreshStats runs on the event loop -- the thread that owns every
4617 // miner's socket -- and used to hold gStatsMutex and gPayoutLedgerMutex at
4618 // function scope across WriteFileAtomic. Once that write gained fsync() on
4619 // the file and on its directory, publishing pool_status.json parked the
4620 // payout thread behind two disk flushes every ten seconds.
4621 //
4622 // Timing assertions would be flaky, so this does not measure anything. It
4623 // makes the write block for as long as the test likes: strPoolStatusFile
4624 // points at a path whose ".tmp" is a FIFO, and open(2) for writing on a
4625 // FIFO does not return until a reader arrives. While RefreshStats is
4626 // parked in there, every mutex it must NOT be holding has to be
4627 // acquirable. Against the previous code this test does not merely fail, it
4628 // cannot pass: the writer is inside fopen() with both mutexes held and
4629 // nothing will release them until the test lets the write finish.
4630#ifndef _WIN32
4631 {
4632 PayoutTestResetState();
4633 const std::string statusPath = dir + "/pool_status_selftest.json";
4634 const std::string statusTmp = statusPath + ".tmp";
4635 const std::string savedStatusFile = strPoolStatusFile;
4636 const int64 savedLastWrite = gLastPoolStatusWrite;
4637 remove(statusPath.c_str());
4638 remove(statusTmp.c_str());
4639 strPoolStatusFile = statusPath;
4640 gLastPoolStatusWrite = 0; // due a publish on the next refresh
4641
4642 // Give the status file something to report, so the ledger mutex is
4643 // genuinely taken on the way through rather than skipped over.
4644 {
4645 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4646 PendingPayout pp;
4647 pp.blockHeight = 909; pp.blockHash = 0; pp.matureAtHeight = 909;
4648 pp.amounts[addrA] = 3 * COIN;
4649 gPendingPayouts.push_back(pp);
4650 }
4651
4652 bool fifoOk = (mkfifo(statusTmp.c_str(), 0600) == 0);
4653 check(fifoOk, "a FIFO stands in for the status file's temp file, so the"
4654 " write can be held open");
4655 if (fifoOk) {
4656 std::atomic<bool> done(false);
4657 std::thread writer([&done]() {
4658 RefreshStats(0, 0);
4659 done = true;
4660 });
4661
4662 // Settle: the work before the write is a handful of vector
4663 // operations, so anything past this point is inside fopen(). The
4664 // !done check below is what actually proves it -- with no reader
4665 // there is no other way for this thread to still be running.
4666 Sleep(500);
4667 bool blocked = !done;
4668 bool gotLedger = false, gotStats = false, gotExec = false,
4669 gotRounds = false;
4670 for (int i = 0; i < 1000 && blocked; i++) {
4671 if (!gotLedger && gPayoutLedgerMutex.try_lock()) {
4672 gotLedger = true; gPayoutLedgerMutex.unlock();
4673 }
4674 if (!gotStats && gStatsMutex.try_lock()) {
4675 gotStats = true; gStatsMutex.unlock();
4676 }
4677 if (!gotExec && gPayoutExecMutex.try_lock()) {
4678 gotExec = true; gPayoutExecMutex.unlock();
4679 }
4680 if (!gotRounds && gPoolRoundsMutex.try_lock()) {
4681 gotRounds = true; gPoolRoundsMutex.unlock();
4682 }
4683 if (gotLedger && gotStats && gotExec && gotRounds)
4684 break;
4685 Sleep(2);
4686 }
4687
4688 check(blocked,
4689 "the status-file write is held open for the duration of this"
4690 " test");
4691 check(!blocked || gotLedger,
4692 "gPayoutLedgerMutex is free while pool_status.json is being"
4693 " written");
4694 check(!blocked || (gotExec && gotRounds),
4695 "gPayoutExecMutex and gPoolRoundsMutex are free while"
4696 " pool_status.json is being written");
4697 check(!blocked || gotStats,
4698 "gStatsMutex is free while pool_status.json is being written");
4699
4700 // Let the write finish. Drain to EOF rather than closing early:
4701 // the writer is still filling the pipe and a closed read end would
4702 // deliver it SIGPIPE.
4703 std::thread reader([&statusTmp]() {
4704 FILE* rf = fopen(statusTmp.c_str(), "r");
4705 if (!rf) return;
4706 char buf[4096];
4707 while (fread(buf, 1, sizeof(buf), rf) > 0) { }
4708 fclose(rf);
4709 });
4710 writer.join();
4711 reader.join();
4712 }
4713 remove(statusTmp.c_str());
4714
4715 // And the ordinary path still produces the file it always did.
4716 gLastPoolStatusWrite = 0;
4717 RefreshStats(0, 0);
4718 std::string onDisk;
4719 bool statusOk = false;
4720 if (PayoutTestReadFile(statusPath, onDisk)) {
4721 try {
4722 nlohmann::json j = nlohmann::json::parse(onDisk);
4723 statusOk = j.value("schema", "") == "bitflash-pool-status-1" &&
4724 j["pendingPayouts"].is_array() &&
4725 j["pendingPayouts"].size() == 1 &&
4726 j["pendingPayouts"][0].value("matureAtHeight", 0) == 909 &&
4727 j["pendingPayouts"][0].value("totalAmountSatoshis",
4728 (int64)0) == 3 * COIN;
4729 } catch (...) {
4730 statusOk = false;
4731 }
4732 }
4733 check(statusOk,
4734 "splitting the serialise from the write leaves pool_status.json"
4735 " saying the same thing");
4736
4737 remove(statusPath.c_str());
4738 remove((statusPath + ".tmp").c_str());
4739 strPoolStatusFile = savedStatusFile;
4740 gLastPoolStatusWrite = savedLastWrite;
4741 }
4742#endif
4743
4744 // -- 14. The payout comment names the funding block ----------------------
4745 //
4746 // Same height, two hashes: the shape a reorg leaves when the pool wins a
4747 // height, loses the block, and wins the height again. The legacy needle is
4748 // frozen at the pre-upgrade format, so pre-upgrade payments keep matching;
4749 // a new-format comment never matches it, so a new send can never be read
4750 // as a pre-upgrade payment for a twin round.
4751 {
4752 PayoutTestResetState();
4753 CRITICAL_BLOCK(cs_mapWallet)
4754 mapWallet.clear();
4755 PayoutTestClearBlockIndex();
4756
4757 const int height = 9090;
4758 const int64 amount = 3 * COIN;
4759 uint256 hashA = 0, hashB = 0;
4760 hashA.SetHex("00000000000000000000000000000000000000000000000000000000000000aa");
4761 hashB.SetHex("00000000000000000000000000000000000000000000000000000000000000bb");
4762
4763 check(LegacyPayoutComment(height) ==
4764 strprintf("Pool payout height %d", height),
4765 "the legacy needle is frozen at the format pre-upgrade builds"
4766 " wrote");
4767 check(PayoutComment(height, 0) == LegacyPayoutComment(height),
4768 "a grandfathered zero-hash record still writes the legacy form");
4769 check(PayoutComment(height, hashA) ==
4770 strprintf("Pool payout height %d block %s", height,
4771 hashA.ToString().c_str()),
4772 "a hash-keyed record's comment names its funding block");
4773 check(PayoutComment(height, hashA) != PayoutComment(height, hashB),
4774 "twin rounds at one height carry distinguishable comments");
4775
4776 // A transaction wearing the NEW comment format and -- hypothetically
4777 // -- no structured metadata must not be matched by the legacy needle:
4778 // the fallback exists for pre-upgrade transactions, and the new format
4779 // proves this is not one.
4780 const std::string idA = PayoutIdForBlock(height, hashA);
4781 const std::string idB = PayoutIdForBlock(height, hashB);
4782 std::set<std::string> none;
4783 std::string found;
4784 // Neither twin has a round proof here (section 15 covers the gate),
4785 // and their ids collide, so one answer serves both.
4786 const bool fAllowLegacy = !PoolRoundHasProof(idA);
4787 PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount, 1, "");
4788 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found)
4789 == PAYOUT_PROBE_NONE,
4790 "a new-format comment is never legacy evidence for a twin round");
4791 check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found)
4792 == PAYOUT_PROBE_NONE,
4793 "nor a match channel for its own round -- structured metadata is");
4794
4795 // Round A's own payment, sent by this build, is found by its
4796 // structured id.
4797 CRITICAL_BLOCK(cs_mapWallet)
4798 mapWallet.clear();
4799 PayoutTestClearBlockIndex();
4800 const std::string paidA =
4801 PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount,
4802 2, idA);
4803 check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found)
4804 == PAYOUT_PROBE_STRUCTURED && found == paidA,
4805 "the winning twin's own payment is found by its structured id");
4806
4807 // The id is derived from the FIRST sixteen hex digits of the hash --
4808 // the end proof-of-work fills with zeros -- so twin hashes that agree
4809 // there collide in id, and a payment carrying no pool_payout_block
4810 // answers for either twin. That is the accepted trade for payments
4811 // sent before the key existed, pinned here so it stays a decision.
4812 check(idA == idB,
4813 "twin hashes differing only past the sixteenth hex digit collide"
4814 " in id -- the reason the full hash must decide");
4815 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found)
4816 == PAYOUT_PROBE_STRUCTURED,
4817 "a payment without the block key answers for a colliding twin --"
4818 " the accepted residual for pre-key transactions");
4819
4820 // With the full hash recorded -- as every send now records it -- the
4821 // twins are told apart and the re-won height is paid.
4822 CRITICAL_BLOCK(cs_mapWallet)
4823 mapWallet.clear();
4824 PayoutTestClearBlockIndex();
4825 const std::string keyedA =
4826 PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount,
4827 3, idA, true, hashA.ToString());
4828 check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found)
4829 == PAYOUT_PROBE_STRUCTURED && found == keyedA,
4830 "a payment carrying the full block hash settles its own round");
4831 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found)
4832 == PAYOUT_PROBE_NONE,
4833 "and is no evidence for the twin, so the re-won height is paid");
4834
4835 CRITICAL_BLOCK(cs_mapWallet)
4836 mapWallet.clear();
4837 PayoutTestClearBlockIndex();
4838 }
4839
4840 // -- 15. A proof-bearing round rejects comment-only evidence -------------
4841 //
4842 // The surviving half of the same-height collision: a pre-upgrade
4843 // transaction carries only "Pool payout height H", so if H is won again
4844 // after a reorg, it matches the new round's (height, address, amount)
4845 // exactly and would settle a debt it never paid. The new round has a
4846 // round proof -- written at creation, and only by builds whose payments
4847 // all name their round -- so for it, comment-only evidence is
4848 // inadmissible and the miner is paid. A round without a proof keeps
4849 // today's bias: the evidence is admissible and doubt resolves to
4850 // "already paid".
4851 {
4852 PayoutTestResetState();
4853 CRITICAL_BLOCK(cs_mapWallet)
4854 mapWallet.clear();
4855 PayoutTestClearBlockIndex();
4856 const std::string roundsPath = dir + "/pool_rounds_gate_selftest.json";
4857 const std::string savedRoundsFile = strPoolRoundsFile;
4858 strPoolRoundsFile = roundsPath;
4859 remove(roundsPath.c_str());
4860
4861 const int height = 7171;
4862 const int64 amount = 4 * COIN;
4863 uint256 hashB = 0;
4864 hashB.SetHex("00000000000000000000000000000000000000000000000000000000000000cc");
4865 const std::string idB = PayoutIdForBlock(height, hashB);
4866 std::set<std::string> none;
4867 std::string found;
4868
4869 // The colliding pre-upgrade payment: confirmed, comment-only, same
4870 // height, same address, same amount -- the strongest wrong evidence
4871 // there is.
4872 PayoutTestAddWalletTx(LegacyPayoutComment(height), addrA, amount, 1, "");
4873
4874 check(!PoolRoundHasProof(idB),
4875 "a round with no recorded proof does not claim one");
4876 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount,
4877 !PoolRoundHasProof(idB), none, found)
4878 == PAYOUT_PROBE_LEGACY,
4879 "without a proof the doubt is genuine and still resolves to"
4880 " already-paid");
4881
4882 PoolRoundProof round;
4883 round.id = idB;
4884 round.blockHeight = height;
4885 round.matureAtHeight = height + COINBASE_SPENDABLE_DEPTH;
4886 round.payouts[addrA] = amount;
4887 RecordPoolRound(round);
4888 check(PoolRoundHasProof(idB),
4889 "recording the round proof marks the round as this lineage's");
4890 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount,
4891 !PoolRoundHasProof(idB), none, found)
4892 == PAYOUT_PROBE_NONE,
4893 "with a proof, a same-height comment-only transaction is not"
4894 " evidence, and the re-won height is paid");
4895
4896 // Structured evidence still is: the round's own payment, recovered.
4897 const std::string paidB =
4898 PayoutTestAddWalletTx(PayoutComment(height, hashB), addrA, amount,
4899 2, idB, true, hashB.ToString());
4900 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount,
4901 !PoolRoundHasProof(idB), none, found)
4902 == PAYOUT_PROBE_STRUCTURED && found == paidB,
4903 "the gate does not blind the probe to the round's own structured"
4904 " payment");
4905
4906 CRITICAL_BLOCK(cs_mapWallet)
4907 mapWallet.clear();
4908 PayoutTestClearBlockIndex();
4909 remove(roundsPath.c_str());
4910 remove((roundsPath + ".tmp").c_str());
4911 strPoolRoundsFile = savedRoundsFile;
4912 }
4913
4914 // -- 16. A malformed block hash is corruption, not a hash ----------------
4915 //
4916 // SetHex() parses whatever hex prefix it finds and stops, so a corrupt
4917 // blockHash did not fail to load -- it loaded as a DIFFERENT hash.
4918 // "deadbeef..." became 0xdeadbeef, a hash no block ever had, which the
4919 // orphan check reads as an orphaned block and the next flush VOIDS the
4920 // debt; a value with no hex prefix became 0, which is grandfathered past
4921 // the orphan check instead. Either way a corrupt file silently changed
4922 // what the pool owed. It is a load failure now, handled like every other
4923 // parse error: quarantine, and never rewrite.
4924 {
4925 PayoutTestResetState();
4926 remove(ledgerPath.c_str());
4927 remove(payingPath.c_str());
4928
4929 const std::string badLedger = strprintf(
4930 "[{\"amounts\":{\"%s\":100000000},"
4931 "\"blockHash\":\"deadbeef oops, not a hash\","
4932 "\"blockHeight\":10,\"matureAtHeight\":10}]", addrA.c_str());
4933 check(PayoutTestWriteFile(ledgerPath, badLedger),
4934 "wrote a ledger whose blockHash is not a 64-digit hex string");
4935 LoadPendingPayouts();
4936 {
4937 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4938 check(gPendingPayouts.empty(),
4939 "no entry is loaded with a guessed hash");
4940 }
4941 std::string gone;
4942 check(!PayoutTestReadFile(ledgerPath, gone),
4943 "the ledger with the corrupt hash is quarantined");
4944
4945 const std::string badPaying = strprintf(
4946 "[{\"address\":\"%s\",\"amount\":100000000,"
4947 "\"blockHash\":\"00000000cafe\",\"blockHeight\":10,"
4948 "\"matureAtHeight\":10}]", addrA.c_str());
4949 check(PayoutTestWriteFile(payingPath, badPaying),
4950 "wrote an in-flight list whose blockHash is truncated");
4951 LoadPaying();
4952 {
4953 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
4954 check(gPaying.empty(),
4955 "no in-flight record is loaded with a guessed hash");
4956 }
4957 check(!PayoutTestReadFile(payingPath, gone),
4958 "the in-flight list with the corrupt hash is quarantined");
4959
4960 // The two spellings every real file uses still load: the 64-digit
4961 // hash of a keyed round, and the 64 zeros this build writes for a
4962 // grandfathered entry.
4963 PayoutTestResetState();
4964 remove(ledgerPath.c_str());
4965 const std::string goodLedger = strprintf(
4966 "[{\"amounts\":{\"%s\":100000000},"
4967 "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000cd\","
4968 "\"blockHeight\":10,\"matureAtHeight\":10},"
4969 "{\"amounts\":{\"%s\":200000000},"
4970 "\"blockHash\":\"%s\","
4971 "\"blockHeight\":20,\"matureAtHeight\":20}]",
4972 addrA.c_str(), addrB.c_str(), uint256(0).ToString().c_str());
4973 check(PayoutTestWriteFile(ledgerPath, goodLedger),
4974 "wrote a ledger in both real hash spellings");
4975 LoadPendingPayouts();
4976 {
4977 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4978 check(gPendingPayouts.size() == 2 &&
4979 gPendingPayouts[0].blockHash != 0 &&
4980 gPendingPayouts[1].blockHash == 0,
4981 "a real hash and a serialised zero hash both still load");
4982 }
4983 remove(ledgerPath.c_str());
4984 }
4985
4986 // -- clean up ------------------------------------------------------------
4987 PayoutTestResetState();
4988 CRITICAL_BLOCK(cs_mapWallet)
4989 mapWallet.clear();
4990 PayoutTestClearBlockIndex();
4991 // Out of the scratch directory before removing it, and out of the key maps
4992 // too: the keys this test generated are process-local, but leaving them in
4993 // the keystore would let a later test think a script was its own.
4994 CRITICAL_BLOCK(cs_mapKeys)
4995 {
4996 mapKeys.clear();
4997 mapPubKeys.clear();
4998 }
4999#ifdef _WIN32
5000 SetCurrentDirectoryA(cwd.c_str());
5001#else
5002 if (chdir(cwd.c_str()) != 0) { /* nothing useful to do about it */ }
5003#endif
5004 PayoutTestRemoveDir(dir);
5005 strSetDataDir = savedDataDir;
5006 nBestHeight = savedBestHeight;
5007
5008 fprintf(stdout, "%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
5009 nFail, nFail == 1 ? "" : "s");
5010 fflush(stdout);
5011 return nFail == 0 ? 0 : 1;
5012}
5013
1511int RunPoolStratumSelfTest() 5014int RunPoolStratumSelfTest()
1512{ 5015{
1513 int nFail = 0; 5016 int nFail = 0;
@@ -1640,6 +5143,82 @@ int RunPoolStratumSelfTest()
1640 strPoolRoundsFile = savedRoundsFile; 5143 strPoolRoundsFile = savedRoundsFile;
1641 gPoolRounds = savedRounds; 5144 gPoolRounds = savedRounds;
1642 5145
5146 // -- hashrate decay (#7) --------------------------------------------------
5147 // The first version of this decayed m->hashRate in place from the event
5148 // loop's sweep, which made the reported figure a function of how the loop
5149 // happened to run rather than of how long the miner had been quiet. These
5150 // pin the properties that fixes: sampling changes nothing, and the answer
5151 // depends only on elapsed silence.
5152 {
5153 // A miner exactly on the vardiff target: 4096 hashes per share at
5154 // difficulty 1.0, one share per 15s, so 4096/15 H/s. Its expected
5155 // interval is 15s and its grace is 60s.
5156 const double base = 4096.0 / 15.0;
5157 const int64 t0 = 1766000000; // any plausible non-zero wall clock
5158 Miner decayed(INVALID_SOCKET);
5159 decayed.difficulty = 1.0;
5160 decayed.hashRate = base;
5161 decayed.lastShareTime = t0;
5162
5163 check(MinerReportedHashRate(&decayed, t0 + 60) == base,
5164 "hashrate decay: a miner inside its grace period reports its full estimate");
5165
5166 // Sampling must not consume the estimate. Twenty passes is one second
5167 // of the event loop's old sweep; a thousand is nearly a minute of it.
5168 double once = MinerReportedHashRate(&decayed, t0 + 61);
5169 bool idempotent = (decayed.hashRate == base);
5170 for (int i = 0; i < 1000 && idempotent; i++)
5171 idempotent = (MinerReportedHashRate(&decayed, t0 + 61) == once) &&
5172 (decayed.hashRate == base);
5173 check(idempotent,
5174 "hashrate decay: repeated sampling returns one value and leaves the estimate alone");
5175
5176 // One second past a 60s grace is 1/15th of a half-life: ~4.5% off, not
5177 // the estimate gone. A miner at the vardiff target overshoots its grace
5178 // roughly 1.8% of the time (e^-4) purely by share-timing variance, and
5179 // must not be zeroed for it.
5180 check(once > 0.94 * base && once < 0.96 * base,
5181 "hashrate decay: one second past grace costs a few percent, not the estimate");
5182 double three = MinerReportedHashRate(&decayed, t0 + 63);
5183 check(three > 0.86 * base && three < 0.88 * base,
5184 "hashrate decay: three seconds past grace costs ~13%");
5185
5186 // One half-life past the grace is exactly half.
5187 double half = MinerReportedHashRate(&decayed, t0 + 75);
5188 check(half > 0.499 * base && half < 0.501 * base,
5189 "hashrate decay: one expected interval past grace halves the estimate");
5190
5191 // A miner that has genuinely stopped ends up reported as nothing --
5192 // the whole point of #7. The old in-place version could not get here:
5193 // its decay fed back into its own grace and settled into a 1/t crawl.
5194 check(MinerReportedHashRate(&decayed, t0 + 3600) == 0.0,
5195 "hashrate decay: a miner silent for an hour is reported as zero");
5196
5197 // A clock that steps backwards must never inflate what we announce.
5198 check(MinerReportedHashRate(&decayed, t0 - 3600) == base,
5199 "hashrate decay: a backwards clock reports the estimate unchanged, never higher");
5200
5201 // Fresh connection, and a connection with one share and so no interval
5202 // to have timed: both must report nothing rather than divide by zero.
5203 Miner fresh(INVALID_SOCKET);
5204 check(MinerReportedHashRate(&fresh, t0) == 0.0,
5205 "hashrate decay: a miner that never submitted a share reports zero");
5206 Miner single(INVALID_SOCKET);
5207 single.lastShareTime = t0;
5208 check(MinerReportedHashRate(&single, t0 + 86400) == 0.0,
5209 "hashrate decay: a miner with no rate sample yet reports zero");
5210
5211 // Nothing in the swept range may produce a NaN, an infinity, a negative
5212 // rate, or a rate above the estimate it started from.
5213 bool finite = true;
5214 for (int64 s = -100; s < 200000 && finite; s += 37) {
5215 double v = MinerReportedHashRate(&decayed, t0 + s);
5216 finite = !std::isnan(v) && !std::isinf(v) && v >= 0.0 && v <= base;
5217 }
5218 check(finite,
5219 "hashrate decay: stays finite, non-negative and non-amplifying across a long silence");
5220 }
5221
1643 gCurrentJob = savedJob; 5222 gCurrentJob = savedJob;
1644 gHaveJob = savedHaveJob; 5223 gHaveJob = savedHaveJob;
1645 5224
@@ -1654,56 +5233,77 @@ int RunPoolStratumSelfTest()
1654// --------------------------------------------------------------------------- 5233// ---------------------------------------------------------------------------
1655static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal) 5234static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal)
1656{ 5235{
1657 std::lock_guard<std::mutex> lk(gStatsMutex); 5236 std::string statusBlob;
1658 5237 bool fPublish = false;
1659 gStats.authorizedMiners = 0; 5238 {
1660 gStats.blocksFound = blocksFoundThisSession; 5239 std::lock_guard<std::mutex> lk(gStatsMutex);
1661 gStats.roundShares = roundShareTotal; 5240
1662 gStats.totalHashRate = 0.0; 5241 gStats.authorizedMiners = 0;
1663 for (Miner* m : gMiners) 5242 gStats.blocksFound = blocksFoundThisSession;
1664 if (m->authorised) gStats.authorizedMiners++; 5243 gStats.roundShares = roundShareTotal;
1665 5244 gStats.totalHashRate = 0.0;
1666 gWorkerStats.clear(); 5245 for (Miner* m : gMiners)
1667 for (Miner* m : gMiners) { 5246 if (m->authorised) gStats.authorizedMiners++;
1668 if (!m->authorised || m->lastSeen <= 0) continue; 5247
1669 PoolWorkerStatView v; 5248 gWorkerStats.clear();
1670 v.address = m->address; 5249 // One timestamp for the whole sweep (#7): every miner is faded to the
1671 v.worker = m->worker; 5250 // same instant, so the per-worker rows and the total they are summed
1672 v.roundShares = m->roundShares; 5251 // into describe one moment rather than a smear across the loop.
1673 v.totalShares = m->totalShares; 5252 int64 nowStats = GetTime();
1674 v.lastSeen = m->lastSeen; 5253 for (Miner* m : gMiners) {
1675 v.hashRate = m->hashRate; 5254 if (!m->authorised || m->lastSeen <= 0) continue;
1676 gWorkerStats.push_back(v); 5255 PoolWorkerStatView v;
1677 gStats.totalHashRate += m->hashRate; 5256 v.address = m->address;
1678 } 5257 v.worker = m->worker;
1679 std::sort(gWorkerStats.begin(), gWorkerStats.end(), 5258 v.roundShares = m->roundShares;
1680 [](const PoolWorkerStatView& a, const PoolWorkerStatView& b) { 5259 v.totalShares = m->totalShares;
1681 return a.roundShares != b.roundShares ? a.roundShares > b.roundShares 5260 v.lastSeen = m->lastSeen;
1682 : a.lastSeen > b.lastSeen; 5261 // Not m->hashRate: a miner that has stopped submitting shares must
1683 }); 5262 // not keep contributing its last sample to the announced total.
5263 v.hashRate = MinerReportedHashRate(m, nowStats);
5264 gWorkerStats.push_back(v);
5265 gStats.totalHashRate += v.hashRate;
5266 }
5267 std::sort(gWorkerStats.begin(), gWorkerStats.end(),
5268 [](const PoolWorkerStatView& a, const PoolWorkerStatView& b) {
5269 return a.roundShares != b.roundShares ? a.roundShares > b.roundShares
5270 : a.lastSeen > b.lastSeen;
5271 });
1684 5272
1685 gPayoutViews.clear(); 5273 // Scoped, not held to the end of the function. At function scope this
1686 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex); 5274 // lock covered the status-file write below, so the event loop did two
1687 for (const PendingPayout& pp : gPendingPayouts) { 5275 // fsyncs holding the mutex the payout thread takes to record that a
1688 PendingPayoutView pv; 5276 // miner has been paid.
1689 pv.matureAtHeight = pp.matureAtHeight; 5277 gPayoutViews.clear();
1690 pv.recipients = 0; 5278 {
1691 pv.totalAmount = 0; 5279 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
1692 for (auto& kv : pp.amounts) { 5280 for (const PendingPayout& pp : gPendingPayouts) {
1693 if (pp.paidTxids.count(kv.first)) 5281 PendingPayoutView pv;
1694 continue; 5282 pv.matureAtHeight = pp.matureAtHeight;
1695 pv.recipients++; 5283 pv.recipients = 0;
1696 pv.totalAmount += kv.second; 5284 pv.totalAmount = 0;
5285 for (auto& kv : pp.amounts) {
5286 if (pp.paidTxids.count(kv.first))
5287 continue;
5288 pv.recipients++;
5289 pv.totalAmount += kv.second;
5290 }
5291 if (pv.recipients > 0)
5292 gPayoutViews.push_back(pv);
5293 }
1697 } 5294 }
1698 if (pv.recipients > 0)
1699 gPayoutViews.push_back(pv);
1700 }
1701 5295
1702 int64 now = GetTime(); 5296 int64 now = GetTime();
1703 if (now - gLastPoolStatusWrite >= 10) { 5297 if (now - gLastPoolStatusWrite >= 10) {
1704 WritePoolStatusJsonLocked(); 5298 statusBlob = SerializePoolStatusLocked();
1705 gLastPoolStatusWrite = now; 5299 gLastPoolStatusWrite = now;
5300 fPublish = true;
5301 }
1706 } 5302 }
5303 // Every payout mutex is released here, and gStatsMutex with them. This is
5304 // where the two fsyncs happen, and no other thread is waiting behind them.
5305 if (fPublish)
5306 PublishPoolStatus(statusBlob);
1707} 5307}
1708 5308
1709// --------------------------------------------------------------------------- 5309// ---------------------------------------------------------------------------
@@ -1714,6 +5314,7 @@ static void PoolEventLoop()
1714 int lastHeight = -1; 5314 int lastHeight = -1;
1715 int64 lastRefresh = GetTime(); 5315 int64 lastRefresh = GetTime();
1716 int64 lastStatsRefresh = 0; 5316 int64 lastStatsRefresh = 0;
5317 int64 lastCensus = 0; // #19 connection census throttle
1717 int blocksFoundThisSession = 0; 5318 int blocksFoundThisSession = 0;
1718 int64 noJobSince = 0; // 0 = have a job 5319 int64 noJobSince = 0; // 0 = have a job
1719 bool warnedNoJob = false; 5320 bool warnedNoJob = false;
@@ -1742,6 +5343,31 @@ static void PoolEventLoop()
1742 sock_close(fd); 5343 sock_close(fd);
1743 continue; 5344 continue;
1744 } 5345 }
5346 // #26: select() + FD_SET goes undefined once we exceed what an
5347 // fd_set can carry, and MAX_POOL_MINERS (1000) alone does not
5348 // prevent it -- a Miner is only removed on clean disconnect, so
5349 // half-open zombies accumulate and the OS can hand out fds past
5350 // the limit long before the count reaches 1000. Refuse rather
5351 // than corrupt the fd_set. Which limit applies is platform
5352 // specific: POSIX bounds the descriptor value, Windows the
5353 // number of SOCKET handles an fd_set can hold.
5354#ifndef _WIN32
5355 if ((int)fd >= (int)FD_SETSIZE) {
5356 LogPrint("worker", "[pool] refusing miner fd=%d -- at/over"
5357 " FD_SETSIZE=%d, select() would be undefined (#26)\n",
5358 (int)fd, (int)FD_SETSIZE);
5359 sock_close(fd);
5360 continue;
5361 }
5362#else
5363 if (gMiners.size() >= (size_t)FD_SETSIZE) {
5364 LogPrint("worker", "[pool] refusing miner -- %zu sockets held,"
5365 " an fd_set carries at most FD_SETSIZE=%d (#26)\n",
5366 gMiners.size(), (int)FD_SETSIZE);
5367 sock_close(fd);
5368 continue;
5369 }
5370#endif
1745 gMiners.push_back(new Miner(fd)); 5371 gMiners.push_back(new Miner(fd));
1746 LogPrint("worker", "[pool] miner fd=%d entered event loop" 5372 LogPrint("worker", "[pool] miner fd=%d entered event loop"
1747 " -- awaiting subscribe/authorize\n", (int)fd); 5373 " -- awaiting subscribe/authorize\n", (int)fd);
@@ -1869,6 +5495,35 @@ static void PoolEventLoop()
1869 } 5495 }
1870 } 5496 }
1871 5497
5498 // 3.6. Idle reap (#6). Mirror the clean-disconnect path exactly --
5499 // log, close, erase from gMiners via std::find, delete -- just with
5500 // a different reason. Shares the miner already earned stay in
5501 // roundShareCount, exactly as they do when a miner cleanly
5502 // disconnects mid-round. connectTime stands in for lastSeen on a
5503 // connection that never completed a line. Erasing inside the loop
5504 // above would invalidate the iteration, so collect then reap.
5505 {
5506 int64 now = GetTime();
5507 std::vector<Miner*> idle;
5508 for (Miner* m : gMiners) {
5509 int64 last = (m->lastSeen > 0) ? m->lastSeen : m->connectTime;
5510 if (now - last > POOL_IDLE_TIMEOUT_SECS)
5511 idle.push_back(m);
5512 }
5513 for (Miner* m : idle) {
5514 LogPrint("worker", "[pool] dropping idle miner %s -- nothing"
5515 " received in over %llds (session shares: %llu,"
5516 " round shares: %llu)\n",
5517 m->worker.empty() ? "(never authorized)" : m->worker.c_str(),
5518 (long long)POOL_IDLE_TIMEOUT_SECS,
5519 (unsigned long long)m->sessionShares,
5520 (unsigned long long)m->roundShares);
5521 sock_close(m->fd);
5522 gMiners.erase(std::find(gMiners.begin(), gMiners.end(), m));
5523 delete m;
5524 }
5525 }
5526
1872 // 4. Select over miner sockets 5527 // 4. Select over miner sockets
1873 if (!gMiners.empty()) { 5528 if (!gMiners.empty()) {
1874 fd_set fds; FD_ZERO(&fds); 5529 fd_set fds; FD_ZERO(&fds);
@@ -1891,6 +5546,11 @@ static void PoolEventLoop()
1891 // terminate the whole node -- not just the pool. Every 5546 // terminate the whole node -- not just the pool. Every
1892 // line arrives from an unauthenticated socket. 5547 // line arrives from an unauthenticated socket.
1893 while (RecvLine(m->fd, m->readBuf, line, disc)) { 5548 while (RecvLine(m->fd, m->readBuf, line, disc)) {
5549 // Any complete line is proof of life (#6): authorize
5550 // and valid shares already stamp lastSeen themselves,
5551 // but a miner that only ever subscribes (or sends
5552 // unrecognised methods) must count as alive too.
5553 m->lastSeen = GetTime();
1894 try { 5554 try {
1895 HandleLine(m, line, roundShareCount, 5555 HandleLine(m, line, roundShareCount,
1896 roundShareTotal, blocksFoundThisSession); 5556 roundShareTotal, blocksFoundThisSession);
@@ -1931,6 +5591,14 @@ static void PoolEventLoop()
1931 RefreshStats(blocksFoundThisSession, roundShareTotal); 5591 RefreshStats(blocksFoundThisSession, roundShareTotal);
1932 lastStatsRefresh = GetTime(); 5592 lastStatsRefresh = GetTime();
1933 } 5593 }
5594
5595 // 6. #19: log the connection census periodically. Cheap (one pass over
5596 // gMiners) and it is the single number an operator reads to judge pool
5597 // health, so it should not be an overstatement.
5598 if (GetTime() - lastCensus >= POOL_CENSUS_SECS) {
5599 LogPoolConnectionCensus();
5600 lastCensus = GetTime();
5601 }
1934 } 5602 }
1935 5603
1936 for (Miner* m : gMiners) { sock_close(m->fd); delete m; } 5604 for (Miner* m : gMiners) { sock_close(m->fd); delete m; }
@@ -1959,10 +5627,20 @@ static void AcceptOneFn(void* arg)
1959 std::string host = relay.substr(0, colon); 5627 std::string host = relay.substr(0, colon);
1960 int port = atoi(relay.substr(colon + 1).c_str()); 5628 int port = atoi(relay.substr(colon + 1).c_str());
1961 5629
1962 LogPrint("pool", "[pool] accept thread started for relay %s\n", relay.c_str()); 5630 LogPrint("pool", "[pool] accept thread started for relay %s%s\n", relay.c_str(),
5631 fBtfRelayPinned ? " (/rvrelay pin)" : "");
5632
5633 // Registration health for this relay. A pin has no fallback, so a pin that
5634 // never registers means the pool is simply unreachable -- and that has to
5635 // be loud. The 2026-07-30 incident went unnoticed because this loop logged
5636 // failures at one flat line and success not at all, so "which relay is this
5637 // pool actually registered at" could not be answered from the log.
5638 int nConsecFail = 0;
5639 int64 nFirstFail = 0;
5640 bool fEverRegistered = false;
1963 5641
1964 while (!fShutdown && gPoolRunning) { 5642 while (!fShutdown && gPoolRunning) {
1965 // Blocks until a miner connects to us at this relay 5643 // Returns as soon as the relay lists us -- registered, not yet paired.
1966 btf::RvSocket rv = btf::RvServiceRegister(host.c_str(), (unsigned short)port, pk); 5644 btf::RvSocket rv = btf::RvServiceRegister(host.c_str(), (unsigned short)port, pk);
1967 5645
1968 if (fShutdown || !gPoolRunning) { 5646 if (fShutdown || !gPoolRunning) {
@@ -1970,12 +5648,53 @@ static void AcceptOneFn(void* arg)
1970 break; 5648 break;
1971 } 5649 }
1972 if (rv == btf::RV_INVALID) { 5650 if (rv == btf::RV_INVALID) {
1973 LogPrint("pool", "[pool] relay %s: RvServiceRegister failed" 5651 nConsecFail++;
1974 " -- backing off 30s before retry\n", relay.c_str()); 5652 if (nFirstFail == 0) nFirstFail = GetTime();
5653 LogPrint("pool", "[pool] relay %s: registration failed (attempt %d,"
5654 " failing for %llds) -- backing off 30s before retry\n",
5655 relay.c_str(), nConsecFail, (long long)(GetTime() - nFirstFail));
5656 if (fBtfRelayPinned && (nConsecFail == 3 || nConsecFail % 10 == 0))
5657 LogPrint("pool", "[pool] WARNING: pinned relay %s refused registration"
5658 " %d times running (%llds); /rvrelay leaves no fallback, so"
5659 " miners cannot reach this pool%s\n",
5660 relay.c_str(), nConsecFail, (long long)(GetTime() - nFirstFail),
5661 fEverRegistered ? "" : " -- it has never registered once");
1975 for (int i = 0; i < 30 && !fShutdown && gPoolRunning; i++) Sleep(1000); 5662 for (int i = 0; i < 30 && !fShutdown && gPoolRunning; i++) Sleep(1000);
1976 continue; 5663 continue;
1977 } 5664 }
1978 5665
5666 // Advertise THIS relay only now. Setting it before the call published a
5667 // meeting node we were not yet listed at, so a miner resolving the
5668 // descriptor inside that window burned its dial on a relay holding no
5669 // registration for us. net.cpp's peer-accept has always ordered it this
5670 // way round; the pool had not.
5671 BtfSetActiveRelay(relay);
5672 if (!fEverRegistered || nConsecFail > 0)
5673 LogPrint("pool", "[pool] registered at relay %s%s%s -- advertising it as"
5674 " the pool's meeting node\n", relay.c_str(),
5675 fBtfRelayPinned ? " (/rvrelay pin)" : "",
5676 nConsecFail > 0 ? ", recovered" : "");
5677 fEverRegistered = true;
5678 nConsecFail = 0;
5679 nFirstFail = 0;
5680
5681 // Block for a miner to actually arrive. RvServiceRegister used to do
5682 // this itself and returned with the relay's 0x01 pairing byte already
5683 // consumed; it no longer does. Without waiting here, BtfServiceWrap
5684 // reads that 0x01 as the first byte of the miner's 32-byte ephemeral
5685 // key -- every subsequent byte is shifted by one, so the X25519
5686 // handshake completes against a garbage shared secret (it has no
5687 // confirmation step to catch it), the tunnel is handed to the event
5688 // loop, and the miner's mining.subscribe never decrypts into a valid
5689 // line. The miner sees the pool close on it; the pool sees a peer that
5690 // never authorized.
5691 if (!btf::RvServiceWaitPaired(rv)) {
5692 btf::RvClose(rv);
5693 LogPrint("pool", "[pool] relay %s: dropped us before any miner dialed"
5694 " -- re-registering\n", relay.c_str());
5695 continue;
5696 }
5697
1979 // Echan handshake -- blocking but fast (local + relay round-trip) 5698 // Echan handshake -- blocking but fast (local + relay round-trip)
1980 btf_socket_t fd = btf::BtfServiceWrap(rv, sk); 5699 btf_socket_t fd = btf::BtfServiceWrap(rv, sk);
1981 if (fd == INVALID_SOCKET) { 5700 if (fd == INVALID_SOCKET) {
@@ -2013,9 +5732,21 @@ static void AcceptLoopFn(void*)
2013 LogPrint("pool", "[pool] .btf identity ready after %ds\n", waitSecs * 2); 5732 LogPrint("pool", "[pool] .btf identity ready after %ds\n", waitSecs * 2);
2014 5733
2015 std::set<std::string> spawned; // relays we've already started a thread for 5734 std::set<std::string> spawned; // relays we've already started a thread for
5735 std::set<std::string> skipped; // discovered relays the pin excluded, logged once each
2016 5736
2017 while (!fShutdown && gPoolRunning) { 5737 while (!fShutdown && gPoolRunning) {
5738 // BtfAllRelays() already honours the pin: with /rvrelay set it returns
5739 // vBtfMeetingRelays alone, so no thread is ever spawned for a
5740 // discovered relay. Name the ones it drops -- a pool registering at a
5741 // stranger's relay is exactly the thing that should not be silent.
2018 vector<string> relays = BtfAllRelays(); 5742 vector<string> relays = BtfAllRelays();
5743 if (fBtfRelayPinned) {
5744 for (const string& r : BtfDiscoveredRelays())
5745 if (std::find(relays.begin(), relays.end(), r) == relays.end() &&
5746 skipped.insert(r).second)
5747 LogPrint("pool", "[pool] not registering at discovered relay %s"
5748 " -- /rvrelay pins the relay list\n", r.c_str());
5749 }
2019 if (relays.empty()) { 5750 if (relays.empty()) {
2020 LogPrint("pool", "[pool] no relays available -- miners cannot connect yet\n"); 5751 LogPrint("pool", "[pool] no relays available -- miners cannot connect yet\n");
2021 } else { 5752 } else {
@@ -2031,8 +5762,9 @@ static void AcceptLoopFn(void*)
2031 relay.c_str()); 5762 relay.c_str());
2032 } else { 5763 } else {
2033 spawned.insert(relay); 5764 spawned.insert(relay);
2034 LogPrint("pool", "[pool] accept thread spawned for relay %s" 5765 LogPrint("pool", "[pool] accept thread spawned for relay %s%s"
2035 " (%zu total)\n", relay.c_str(), spawned.size()); 5766 " (%zu total)\n", relay.c_str(),
5767 fBtfRelayPinned ? " (/rvrelay pin)" : "", spawned.size());
2036 } 5768 }
2037 } 5769 }
2038 } 5770 }
@@ -2062,10 +5794,23 @@ void ThreadRPCServer(void*)
2062 5794
2063 LoadPendingPayouts(); 5795 LoadPendingPayouts();
2064 LoadPoolRounds(); 5796 LoadPoolRounds();
2065 5797 LoadPaying();
2066 _beginthread(AcceptLoopFn, 0, NULL); 5798 // Must run before the payout thread starts, so nothing is resent that the
2067 _beginthread(ThreadBtfPoolAnnouncer, 0, NULL); 5799 // ledger already shows as sent, and every restored in-flight record is
2068 _beginthread(PayoutThreadFn, 0, NULL); 5800 // armed for its one wallet probe before it is allowed to send.
5801 ReconcilePayouts();
5802
5803 // _beginthread cannot throw (it reports -1), but an unchecked failure
5804 // leaves the pool running deaf: nobody accepts, announces, or pays, and
5805 // nothing in the log says why.
5806 if (_beginthread(AcceptLoopFn, 0, NULL) == (uintptr_t)-1)
5807 LogPrint("pool", "[pool] FATAL: accept loop thread failed to spawn -- miners cannot connect\n");
5808 if (_beginthread(LocalAcceptFn, 0, NULL) == (uintptr_t)-1)
5809 LogPrint("pool", "[pool] local accept thread failed to spawn\n");
5810 if (_beginthread(ThreadBtfPoolAnnouncer, 0, NULL) == (uintptr_t)-1)
5811 LogPrint("pool", "[pool] announcer thread failed to spawn -- pool will not be advertised\n");
5812 if (_beginthread(PayoutThreadFn, 0, NULL) == (uintptr_t)-1)
5813 LogPrint("pool", "[pool] FATAL: payout thread failed to spawn -- no one gets paid\n");
2069 5814
2070 PoolEventLoop(); // blocks until gPoolRunning = false or fShutdown 5815 PoolEventLoop(); // blocks until gPoolRunning = false or fShutdown
2071 5816