upleb.uk

Public git repos — served from a NIP-34 GRASP relay at git.upleb.uk

summaryrefslogtreecommitdiff
path: root/src/rpc.cpp
diff options
context:
space:
mode:
authorbitflash-maintainer <noreply@bitflash.invalid>2026-09-05 22:46:52 +0000
committerbitflash-maintainer <noreply@bitflash.invalid>2026-09-05 22:46:52 +0000
commitceae3d076b5ad11ee84729ae43f14e9312896646 (patch)
tree7e9132aa871fdc01cb95268d2f5ea3bb11853ddf /src/rpc.cpp
parentbe63e83d79e7e00e6b8ca889a70e6a19886aa1d8 (diff)
parent66e0b9158d050c46b1ff2509faeee3f4573cac1c (diff)
Bring main onto the line this project actually ships
main had drifted to a history with no ancestor in common with the tree we build and deploy. It sat at be63e83 (2026-08-09) with 28 commits; the shipped line, deploy/soak-2026-08-21, had 383. The two shared nothing -- git merge-base returns no result at all -- so anyone cloning this repository landed on a default branch that was neither current nor related to the running node. The 28 commits on main were audited before this merge: every one is already present in the tree by content, including the two that are hardest to see because they were rewritten afterwards -- the payout reorg-safety work (PayoutBlockStillOnChain, since evolved to re-check on every retry rather than once at maturity) and the nostr relay certificate verification (SSL_set1_host, now with a Windows root-store path). Nothing on main is lost by this merge. Recorded as a merge rather than a force-push: main only ever moves forward, its old history stays reachable as the first parent, and this is revertible. The tree is taken wholesale from the merged branch, so main's content is now exactly what was built and tested. What this brings onto main: - compactblock: BIP152-style compact block relay ported from upstream 1.2.23, primitives and unit test only, with the RAND_bytes failure path closed and the reconstruction allocation bounded. - upstream/main 17741d9: isolated testnet parameters and genesis, optional 2 MB large pages for RandomX. Mainnet genesis verified byte-identical through upstream's BuildGenesisBlock() refactor. - deploy: every image built on bookworm so binaries run on Debian 12, and the container build now honours the RandomX and libsecp256k1 pins it had been silently ignoring since before 1.2.15. Gates at this tree: 15/15 selftests, compactblock's 17 checks, consensus constants unchanged against e4f898c apart from the pre-existing MAX_BLOCK_SIZE_GEN, mainnet genesis intact, money-log check ok.
Diffstat (limited to 'src/rpc.cpp')
-rw-r--r--src/rpc.cpp4490
1 files changed, 4195 insertions, 295 deletions
diff --git a/src/rpc.cpp b/src/rpc.cpp
index 03d1f44..3827cd5 100644
--- a/src/rpc.cpp
+++ b/src/rpc.cpp
@@ -31,10 +31,15 @@
31#include <netinet/in.h> 31#include <netinet/in.h>
32#include <arpa/inet.h> 32#include <arpa/inet.h>
33#include <unistd.h> 33#include <unistd.h>
34#include <fcntl.h>
35#include <dirent.h>
36#else
37#include <io.h>
34#endif 38#endif
35 39
36#include <atomic> 40#include <atomic>
37#include <mutex> 41#include <mutex>
42#include <thread>
38#include <map> 43#include <map>
39#include <set> 44#include <set>
40#include <deque> 45#include <deque>
@@ -110,15 +115,28 @@ static uint256 ShareTargetFromDifficulty(double difficulty)
110{ 115{
111 if (difficulty <= 0.0) difficulty = 1.0; 116 if (difficulty <= 0.0) difficulty = 1.0;
112 static const uint256 diff1 = CBigNum(bnProofOfWorkLimit).getuint256(); 117 static const uint256 diff1 = CBigNum(bnProofOfWorkLimit).getuint256();
118 // Representability floor. shareTarget = diff1 / difficulty, and diff1 is
119 // ~uint256(0) >> 12, i.e. a hair under 2^244 -- so the true target stops
120 // fitting in a uint256 once difficulty reaches 2^-12 (= 1/4096), and the
121 // fixed-point divide below breaks even earlier in a second way: the
122 // (uint64)(difficulty * 65536.0) divisor truncates to zero for any
123 // difficulty under 2^-16. Both thresholds sit far below VARDIFF_MIN
124 // (0.001), so live operation never comes near this, but a caller that
125 // bypasses vardiff (or a future VARDIFF_MIN) must not divide by zero or
126 // hand back getuint256()'s low-256-bit truncation of an oversized
127 // quotient -- a wrapped target is not merely wrong, it is arbitrarily
128 // HARDER than difficulty 1. At these difficulties every hash is meant to
129 // be a share, so the honest answer is the easiest representable target.
130 // (This replaces a dead branch that read `if (t < diff1 && difficulty <
131 // 1.0)`: t = diff1/difficulty, so difficulty < 1.0 implies t > diff1 and
132 // the two conditions could never both hold.)
133 if (difficulty < 1.0 / 4096.0) return ~uint256(0);
113 // shareTarget = diff1 / difficulty 134 // shareTarget = diff1 / difficulty
114 CBigNum bn; bn.setuint256(diff1); 135 CBigNum bn; bn.setuint256(diff1);
115 // Multiply by 2^16 to keep precision, divide, shift back 136 // Multiply by 2^16 to keep precision, divide, shift back
116 bn <<= 16; 137 bn <<= 16;
117 bn /= (uint64)(difficulty * 65536.0); 138 bn /= (uint64)(difficulty * 65536.0);
118 uint256 t = bn.getuint256(); 139 return bn.getuint256();
119 // Cap at all-ones (maximum, easiest possible)
120 if (t < diff1 && difficulty < 1.0) t = ~uint256(0);
121 return t;
122} 140}
123 141
124// Expected number of hashes to find one share at a given Stratum difficulty, 142// Expected number of hashes to find one share at a given Stratum difficulty,
@@ -218,6 +236,29 @@ static bool RebuildJob()
218 // From the pool, so the template's payout key is in wallet.dat before 236 // From the pool, so the template's payout key is in wallet.dat before
219 // any miner is handed work against it. 237 // any miner is handed work against it.
220 vector<unsigned char> vchPubKey = GetKeyFromPool(); 238 vector<unsigned char> vchPubKey = GetKeyFromPool();
239 if (vchPubKey.empty()) {
240 // Empty means the wallet could not produce a key: encrypted and
241 // locked, or key generation itself failed. Serialised, an empty
242 // vector is a zero-length push, so the coinbase would become
243 // `<empty> OP_CHECKSIG` -- a script nobody can satisfy. Every
244 // block mined against such a job burns its whole subsidy, while
245 // QueuePayouts still records the full obligation and the miners
246 // are then paid out of coin that block never contributed.
247 // BitcoinMiner refuses to mine in this state; refuse to hand the
248 // pool's miners work in it too. printf and not LogPrint for the
249 // reason given above the deaf-node refusal, and throttled the
250 // same way -- the no-job watchdog retries every few seconds for
251 // as long as the condition lasts.
252 static int64 nLastSaidNoKey = 0;
253 if (GetTime() - nLastSaidNoKey > 60) {
254 nLastSaidNoKey = GetTime();
255 printf("RebuildJob: no wallet key available for the coinbase "
256 "(wallet locked, or key generation failed -- the "
257 "GetKeyFromPool line above says which) -- not issuing "
258 "jobs\n");
259 }
260 return false;
261 }
221 static std::atomic<uint32_t> sExtra{0}; 262 static std::atomic<uint32_t> sExtra{0};
222 uint32_t extraNonce = ++sExtra; 263 uint32_t extraNonce = ++sExtra;
223 264
@@ -236,35 +277,96 @@ static bool RebuildJob()
236 CBlock block; 277 CBlock block;
237 block.vtx.push_back(txNew); 278 block.vtx.push_back(txNew);
238 279
280 // Fill the block the same way BitcoinMiner() does, because this is the
281 // same job: a block that gets mined and submitted for real.
282 //
283 // It used to differ in two ways, and both were bugs.
284 //
285 // One pass over mapTransactions is not enough. That map is keyed by
286 // hash, so it has no dependency order, and a transaction spending the
287 // still-unconfirmed output of another (change from a recent send, most
288 // commonly) fails ConnectInputs whenever its parent happens to sit
289 // later in map order. A single pass then drops it -- and because map
290 // order does not change between rebuilds, every later rebuild drops it
291 // again at the same point. Deterministic exclusion, not bad luck: if
292 // pool operators mine most of the blocks, a transaction of that shape
293 // can stay unconfirmed for as long as its parent does.
294 //
295 // Diagnosis and fix both come from Vic-Nas, who found it in his fork
296 // of this code and described it exactly. The fork has since been
297 // deleted, so this carries it back rather than losing it.
298 int64 nFees = 0;
239 { 299 {
240 CTxDB txdb("r"); 300 CTxDB txdb("r");
241 CRITICAL_BLOCK(cs_mapTransactions) 301 CRITICAL_BLOCK(cs_mapTransactions)
242 { 302 {
243 map<uint256, CTxIndex> pool; 303 map<uint256, CTxIndex> pool;
244 unsigned int sz = 0; 304 vector<char> vfAlreadyAdded(mapTransactions.size());
245 for (auto& kv : mapTransactions) { 305 bool fFoundSomething = true;
246 CTransaction& tx = kv.second; 306 // The cap is MAX_BLOCK_SIZE_GEN, not MAX_SIZE/2. MAX_SIZE is
247 if (tx.IsCoinBase() || !tx.IsFinal()) continue; 307 // the 32 MB serialisation ceiling, so half of it is 16 MB --
248 int64 nFees = 0; 308 // this builder would happily assemble a template that its own
249 map<uint256, CTxIndex> tmp(pool); 309 // CheckBlock() rejects at the 1 MB consensus cap. This is the
250 if (!tx.ConnectInputs(txdb, tmp, CDiskTxPos(1,1,1), 0, nFees, 310 // pool's job builder, so a block over the cap is not one
251 false, true, tx.GetMinFee(block.vtx.size() < 100))) 311 // wasted solve, it is every miner on the pool working a
252 continue; 312 // template the network will reject.
253 pool = tmp; 313 //
254 block.vtx.push_back(tx); 314 // The inner break used to bound the overshoot to one
255 sz += ::GetSerializeSize(tx, SER_NETWORK); 315 // transaction per pass. One transaction is still too many:
256 // Must stay under the consensus cap CheckBlock() enforces. 316 // nothing caps how big a single transaction may be, so that
257 // This is the pool's job builder, so a block over the cap 317 // bound was worth up to MAX_SIZE. Asking whether each one fits
258 // is not just one wasted solve -- it is every miner on the 318 // before appending it bounds the overshoot at zero instead.
259 // pool working a template the network will reject. The 319 //
260 // transaction is already pushed above, so this can overshoot 320 // Seeded with the header, tx count and coinbase, because
261 // by one; half the cap leaves room for that. 321 // CheckBlock() weighs the whole serialized block.
262 if (sz > MAX_BLOCK_SIZE / 2) break; 322 unsigned int sz = ::GetSerializeSize(block, SER_DISK);
323 // And the sigop budget, which CheckBlock() enforces and this
324 // builder never counted against. Same counting function
325 // CheckBlock() sums, seeded from the coinbase because it sums
326 // over all of vtx.
327 unsigned int nSigOps = block.vtx[0].GetSigOpCount();
328 while (fFoundSomething && sz < MAX_BLOCK_SIZE_GEN)
329 {
330 fFoundSomething = false;
331 unsigned int n = 0;
332 for (map<uint256, CTransaction>::iterator mi = mapTransactions.begin();
333 mi != mapTransactions.end(); ++mi, ++n)
334 {
335 if (vfAlreadyAdded[n])
336 continue;
337 CTransaction& tx = (*mi).second;
338 if (tx.IsCoinBase() || !tx.IsFinal()) continue;
339 // Skip rather than stop: one transaction too big for
340 // the room left does not mean the next is. Ordered
341 // before ConnectInputs because ConnectInputs credits
342 // nFees by reference on success, and a transaction we
343 // then decline would leave the coinbase claiming a fee
344 // this block does not collect.
345 unsigned int nTxSize = ::GetSerializeSize(tx, SER_NETWORK);
346 unsigned int nTxSigOps = tx.GetSigOpCount();
347 if (!BlockTemplateFits(sz, nSigOps, nTxSize, nTxSigOps))
348 continue;
349 map<uint256, CTxIndex> tmp(pool);
350 if (!tx.ConnectInputs(txdb, tmp, CDiskTxPos(1,1,1), 0, nFees,
351 false, true, tx.GetMinFee(block.vtx.size() < 100)))
352 continue;
353 swap(pool, tmp);
354 block.vtx.push_back(tx);
355 sz += nTxSize;
356 nSigOps += nTxSigOps;
357 vfAlreadyAdded[n] = true;
358 fFoundSomething = true;
359 }
263 } 360 }
264 } 361 }
265 } 362 }
266 363
267 block.vtx[0].vout[0].nValue = block.GetBlockValue(pindexPrev->nHeight + 1, 0); 364 // And the fees actually go to the coinbase now. nFees was declared
365 // inside the scan loop, so every transaction reset it and the total was
366 // thrown away, while this line passed a hardcoded 0. A pool operator
367 // collected fees into a block and paid none of them to itself. Solo
368 // mining has always passed nFees here; only this builder did not.
369 block.vtx[0].vout[0].nValue = block.GetBlockValue(pindexPrev->nHeight + 1, nFees);
268 block.hashPrevBlock = pindexPrev->GetBlockHash(); 370 block.hashPrevBlock = pindexPrev->GetBlockHash();
269 block.hashMerkleRoot = block.BuildMerkleTree(); 371 block.hashMerkleRoot = block.BuildMerkleTree();
270 block.nTime = max((unsigned int)(pindexPrev->GetMedianTimePast() + 1), 372 block.nTime = max((unsigned int)(pindexPrev->GetMedianTimePast() + 1),
@@ -297,26 +399,406 @@ struct PendingPayout {
297 // supposed to pay it: a 280-block reorg once left ~14.4k BTF of debt 399 // supposed to pay it: a 280-block reorg once left ~14.4k BTF of debt
298 // owed against blocks that no longer existed, which then drained the 400 // owed against blocks that no longer existed, which then drained the
299 // wallet paying fork-era miners out of real-chain coin. 401 // wallet paying fork-era miners out of real-chain coin.
402 //
403 // Upstream keys these on the string "block-<height>". That is the same
404 // bug this fork already paid for, and height keying additionally lets
405 // FindRecordedPoolPayout match the *previous* round's transaction after a
406 // reorg and silently skip a miner, so their `id` is deliberately dropped.
300 int blockHeight = 0; 407 int blockHeight = 0;
301 // Must be explicitly zeroed: uint256's default constructor is empty 408 // Must be explicitly zeroed: uint256's default constructor is empty
302 // (uint256.h:514), so leaving this unassigned gives every ledger entry 409 // (uint256.h:514), so leaving this unassigned gives every ledger entry
303 // loaded from an older file a garbage hash, which then matches no block 410 // loaded from an older file a garbage hash, which then matches no block
304 // and gets voided as "orphaned". That mistake voided a live payout. 411 // and gets voided as "orphaned". That mistake voided a live payout.
305 uint256 blockHash = 0; // zero = entry predates hash keying 412 uint256 blockHash = 0; // zero = entry predates hash keying
306 int matureAtHeight; 413 int matureAtHeight = 0;
307 std::map<std::string, int64> amounts; // address -> satoshis 414 std::map<std::string, int64> amounts; // address -> satoshis
308 int retries = 0; // exec-thread only: SendMoney attempts so far 415 // From upstream: which recipients already have a transaction. This is what
309 int deferrals = 0; // exec-thread only: waits for spendable coin 416 // lets an entry stay on disk until every address in `amounts` is paid,
310 int64 retryAfter = 0; // exec-thread only: unix time, 0 = ready now 417 // instead of being erased the moment the batch is handed off.
418 std::map<std::string, std::string> paidTxids; // address -> txid
419 // Retry and backoff state does NOT live here. It used to be declared here
420 // as well as on PayingRecord, and the copy on this side was never written:
421 // `retryAfter` was declared, serialised, loaded and tested at the flush
422 // gate, and the only assignment to it anywhere was `= 0`, so the gate could
423 // not fire. `retries` and `deferrals` were the same story. The real state
424 // is per (round, recipient) -- which is the granularity a payment can fail
425 // at -- and it lives on PayingRecord, which is now what persists it.
311}; 426};
312static std::vector<PendingPayout> gPendingPayouts; 427static std::vector<PendingPayout> gPendingPayouts;
428static std::mutex gPayoutLedgerMutex;
429// Bumped on every serialisation, under gPayoutLedgerMutex. See PublishVersioned.
430static uint64 gPayoutLedgerSeq = 0;
431// Serialises writers of pending_payouts.json, and guards the sequence number
432// of the image currently on disk. Never taken while a data mutex is held.
433static std::mutex gPayoutLedgerFileMutex;
434static uint64 gPayoutLedgerPublishedSeq = 0;
435
436struct PoolRoundProof {
437 std::string id;
438 int blockHeight;
439 int matureAtHeight;
440 int64 createdAt;
441 std::string blockHash;
442 std::string coinbaseTxid;
443 std::string foundBy;
444 double feePercent;
445 uint64 totalShares;
446 int64 coinbaseValue;
447 int64 subsidy;
448 int64 minerReward;
449 int64 operatorReward;
450 int skippedDust;
451 int64 forfeitedBadAddress;
452 std::map<std::string, uint64> shares;
453 std::map<std::string, int64> payouts;
454 std::map<std::string, std::string> paidTxids;
455 // Recipients the ledger has written off, and why. Without this the public
456 // round proof goes on advertising an address as owed with "txid": null
457 // forever, while the ledger has already dropped the obligation -- the two
458 // records of the same round permanently disagreeing, with the operator
459 // holding the money.
460 std::map<std::string, std::string> forfeited; // address -> reason
461
462 PoolRoundProof()
463 : blockHeight(0), matureAtHeight(0), createdAt(0), feePercent(0.0),
464 totalShares(0), coinbaseValue(0), subsidy(0), minerReward(0), operatorReward(0),
465 skippedDust(0), forfeitedBadAddress(0) {}
466};
467static std::vector<PoolRoundProof> gPoolRounds;
468static std::mutex gPoolRoundsMutex;
469static uint64 gPoolRoundsSeq = 0; // guarded by gPoolRoundsMutex
470static std::mutex gPoolRoundsFileMutex;
471static uint64 gPoolRoundsPublishedSeq = 0; // guarded by gPoolRoundsFileMutex
472static const size_t MAX_POOL_ROUNDS = 200;
313 473
314static std::string PayoutFilePath() 474static std::string PayoutFilePath()
315{ 475{
316 return GetAppDir() + "/pending_payouts.json"; 476 return GetAppDir() + "/pending_payouts.json";
317} 477}
318 478
319static void SavePendingPayouts() 479static std::string PoolRoundsFilePath()
480{
481 if (!strPoolRoundsFile.empty())
482 return strPoolRoundsFile;
483 return GetAppDir() + "/pool_rounds.json";
484}
485
486// ---------------------------------------------------------------------------
487// Atomic, ordered publication of the payout state files
488// ---------------------------------------------------------------------------
489//
490// Every one of these files is a full dump of an in-memory structure, written
491// tmp-then-rename so a crash cannot truncate it. Two things have to be true
492// for that to actually be safe, and only the first was:
493//
494// 1. the rename must be atomic (it is), and
495// 2. exactly one thread at a time may be writing the temp file, and a write
496// must never publish an OLDER image over a newer one.
497//
498// (2) was broken for pending_payouts.json. The event loop wrote it with
499// gPayoutLedgerMutex held, the payout thread wrote it with the mutex
500// released, and nothing serialised the two: both opened the same
501// "<path>.tmp" with independent handles and both renamed it. The interleaved
502// file that lands is either a mixture of two images (LoadPendingPayouts then
503// throws mid-loop and comes up with a nearly empty ledger -- every obligation
504// not yet exploded, gone) or a whole newly-queued round silently overwritten
505// by an older snapshot that predates it.
506//
507// The fix is a per-file write mutex, held only across the I/O and never
508// across a data-structure mutex, plus a version on the snapshot. Serialising
509// happens under the data mutex and stamps a monotonically increasing sequence
510// number; publishing happens with the data mutex released and refuses any
511// blob whose sequence number is not newer than what is already on disk. So
512// writers can queue in any order and finish in any order -- the file only
513// ever moves forwards.
514//
515// The other half of the rule is that no thread waits for I/O while holding
516// gPayoutLedgerMutex, gPayoutExecMutex or gPoolRoundsMutex, which is the
517// contention the merge was right to remove from the event loop. That was
518// claimed here before it was true: pool_status.json is written by the same
519// WriteFileAtomic and RefreshStats held gPayoutLedgerMutex at function scope
520// across the call, so once the writes gained fsync() the event loop was doing
521// two of them every ten seconds with the payout thread's mutex held -- worse
522// than the buffered write the merge replaced. The status file now goes through
523// the same serialise-under-the-lock / publish-outside-it split as the ledgers,
524// and the rule holds for all four files. It is checked, not asserted: see the
525// "no payout mutex is held across a publish" section of the payout-ledger
526// self-test.
527
528// A serialised file image plus the version of the in-memory state it was
529// taken from.
530struct VersionedBlob {
531 std::string blob;
532 uint64 seq = 0;
533};
534
535// Push a file's contents out of the C library's buffer AND out of the kernel's
536// page cache, so the bytes exist on the device before anything points at them.
537static bool SyncFileHandle(FILE* f)
538{
539 if (fflush(f) != 0)
540 return false;
541#ifdef _WIN32
542 return _commit(_fileno(f)) == 0;
543#else
544 return fsync(fileno(f)) == 0;
545#endif
546}
547
548// Make the directory entry that rename() created durable too.
549//
550// A tmp+rename is only atomic with respect to a crash of the process. Against
551// a power cut or a hard VM reset the rename can reach the platter while the
552// data it points at has not, and the file that comes back is short or empty.
553// For pending_payouts.json that is not a corrupt file, it is a silent one:
554// LoadPendingPayouts catches the parse error, logs a single warning, and the
555// node comes up owing nobody anything.
556//
557// Best effort -- a filesystem that does not allow the directory to be opened
558// for this is not a reason to refuse to write the ledger.
559static void SyncDirectoryOf(const std::string& path)
560{
561#ifndef _WIN32
562 size_t slash = path.find_last_of('/');
563 std::string dir = (slash == std::string::npos) ? std::string(".")
564 : path.substr(0, slash);
565 if (dir.empty())
566 dir = "/";
567 int fd = open(dir.c_str(), O_RDONLY);
568 if (fd < 0)
569 return;
570 fsync(fd);
571 close(fd);
572#endif
573}
574
575// Replace `path` with `data` atomically and durably. Caller must hold the
576// write mutex for `path`: the temp name is derived from the target, so two
577// threads in here at once for one path is precisely the race this exists to
578// prevent.
579//
580// Every failure removes the temp file before returning. It used to be left
581// where it fell, so a disk that filled up once left a short, valid-looking
582// "pending_payouts.json.tmp" next to the good ledger for as long as the node
583// ran. That file is a trap: it has the name of a half-finished save, and an
584// operator cleaning up after an incident is exactly the person who will move
585// it into place.
586//
587// Failures print unconditionally rather than through LogPrint(). These three
588// files are written under two different debug categories ("payout" for the
589// ledgers, "pool" for the status file), and a write failure is not a trace of
590// normal operation -- it is the file the operator is about to read not being
591// there. Gating it behind a category that had to be chosen before the failure
592// happened is how "could not replace pool_status.json" went missing.
593static bool WriteFileAtomic(const std::string& path, const std::string& data,
594 const char* what)
595{
596 std::string tmp = path + ".tmp";
597 FILE* f = fopen(tmp.c_str(), "w");
598 if (!f) {
599 printf("[payout] WARNING: could not write %s (%s)\n",
600 tmp.c_str(), what);
601 return false;
602 }
603 if (!data.empty() && fwrite(data.c_str(), 1, data.size(), f) != data.size()) {
604 printf("[payout] WARNING: short write to %s (%s)\n",
605 tmp.c_str(), what);
606 fclose(f);
607 remove(tmp.c_str());
608 return false;
609 }
610 // Before the rename, not after: publishing a name that points at nothing
611 // is the failure this ordering exists to prevent. If the data cannot be
612 // made durable, do not rename -- the previous file is still intact and
613 // still correct, and the next write will try again.
614 if (!SyncFileHandle(f)) {
615 printf("[payout] WARNING: could not flush %s to disk (%s)"
616 " -- keeping the previous file\n", tmp.c_str(), what);
617 fclose(f);
618 remove(tmp.c_str());
619 return false;
620 }
621 // fclose() is where buffered data actually reaches the OS, so a failure
622 // here means the .tmp is short -- renaming it over the real file would
623 // destroy the good copy, which is exactly what these files exist to
624 // prevent.
625 if (fclose(f) != 0) {
626 printf("[payout] WARNING: could not close %s (%s)\n",
627 tmp.c_str(), what);
628 remove(tmp.c_str());
629 return false;
630 }
631#ifdef _WIN32
632 // rename() on Windows will not replace an existing file.
633 remove(path.c_str());
634#endif
635 if (rename(tmp.c_str(), path.c_str()) != 0) {
636 printf("[payout] WARNING: could not replace %s (%s)\n",
637 path.c_str(), what);
638 remove(tmp.c_str());
639 return false;
640 }
641 SyncDirectoryOf(path);
642 return true;
643}
644
645// Paths whose loader threw part-way through, and which must therefore never be
646// written again by this process.
647//
648// All three loaders push entries into the live structure INSIDE the try, so an
649// exception half way down a file leaves a PARTIAL ledger in memory -- the
650// entries before the bad one, and nothing after it. The first publish after
651// that then writes the partial state over the file, and the only record of
652// what was owed is gone. ReconcilePayouts publishes unconditionally at
653// startup, so "the first publish" is a few milliseconds later, before an
654// operator could possibly intervene. a8e2f58 named this failure mode in its
655// own commit message and then only fixed the write side.
656//
657// The file is renamed out of the way instead of being left in place, so the
658// evidence survives even if the operator restarts into the same fault, and so
659// a node that comes up owing nobody anything is visibly missing its ledger
660// rather than quietly appearing to have none.
661static std::set<std::string> gRefusedFiles;
662static std::mutex gRefusedFilesMutex;
663
664static bool FileIsRefused(const std::string& path)
665{
666 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
667 return gRefusedFiles.count(path) > 0;
668}
669
670// Quarantine `path` and refuse to write it for the rest of the process.
671//
672// Prints unconditionally: an operator who has just lost a payout ledger must
673// be told at whatever log settings they were already running with, and the
674// name it was moved to is the only way back to the data.
675static void RefuseFileAfterLoadFailure(const std::string& path, const char* what,
676 const char* why)
677{
678 {
679 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
680 gRefusedFiles.insert(path);
681 }
682 std::string bad = strprintf("%s.bad.%lld", path.c_str(), (long long)GetTime());
683 bool moved = (rename(path.c_str(), bad.c_str()) == 0);
684 printf("[payout] ERROR: %s (%s) did not parse: %s\n"
685 "[payout] ERROR: what loaded before the error is a PARTIAL picture,"
686 " so this file will not be written again by this process --"
687 " restart once the cause is fixed\n"
688 "[payout] ERROR: the file %s\n",
689 path.c_str(), what, why,
690 moved ? strprintf("has been kept as %s", bad.c_str()).c_str()
691 : "could NOT be renamed out of the way -- copy it now");
692}
693
694// Publish a versioned image of some in-memory state to `path`.
695//
696// `fileMutex` serialises writers of `path` and guards `publishedSeq`. A blob
697// older than what is already published is dropped: these files are whole-state
698// dumps, so a higher sequence number is by construction a later state of the
699// same structure and publishing it loses nothing. On a write failure
700// `publishedSeq` is deliberately left alone, so the next (newer) write still
701// gets a chance to land.
702static void PublishVersioned(const std::string& path, const VersionedBlob& v,
703 std::mutex& fileMutex, uint64& publishedSeq,
704 const char* what)
705{
706 // A default-constructed blob is not a snapshot of anything; it is a
707 // VersionedBlob nobody filled in. Today they are already dropped, because
708 // every serialiser stamps ++gSeq and so never produces 0, and 0 fails the
709 // seq <= publishedSeq test below -- but only for as long as publishedSeq
710 // starts at 0 and nobody writes `seq = gSeq++`. One character, and the
711 // ledger file becomes "[]". Say it out loud instead of relying on it.
712 if (!v.seq)
713 return;
714 // A file whose loader threw is not a file this process knows the contents
715 // of. Writing it would replace a full record of what was owed with
716 // whatever happened to parse before the error.
717 if (FileIsRefused(path))
718 return;
719 std::lock_guard<std::mutex> lk(fileMutex);
720 if (v.seq <= publishedSeq)
721 return;
722 if (!WriteFileAtomic(path, v.blob, what))
723 return;
724 publishedSeq = v.seq;
725}
726
727// Identity of a payout round. Used as the pool-round proof id and, more
728// importantly, as the `pool_payout_id` written into wallet transaction
729// metadata and matched by FindRecordedPoolPayout().
730//
731// Derived from the block HASH, not from its height alone. Upstream uses
732// "block-<height>", which names two different blocks identically across a
733// reorg -- and because this id is what the crash-recovery probe matches on, a
734// height-keyed id lets that probe find the *previous* round's transaction and
735// conclude a miner has already been paid when they have not. The height stays
736// in the string for readability only.
737//
738// A zero hash means the entry predates hash keying; those keep the old form so
739// anything already recorded against them still matches.
740static std::string PayoutIdForBlock(int blockHeight, const uint256& blockHash)
741{
742 if (blockHash == 0)
743 return strprintf("block-%d", blockHeight);
744 return strprintf("block-%d-%s", blockHeight,
745 blockHash.ToString().substr(0, 16).c_str());
746}
747
748// Do these two references name the same payout round?
749//
750// The block hash alone does not decide it, and every ledger lookup used to
751// assume it did. Entries grandfathered in from a file written before payouts
752// were keyed by hash all carry hash 0, so under a hash-only comparison the
753// first zero-hash entry answers for every other one: RecordPayoutTxid writes
754// a txid against the wrong round (which is then erased as "fully paid" while
755// the real debt still stands), and ReconcilePayouts drops a live in-flight
756// record because a *different* zero-hash round happened to pay that address.
757//
758// A non-zero hash is unique and decides on its own -- that is the whole point
759// of keying on it. A zero hash falls back to the height, which
760// LoadPendingPayouts supplies for any entry that can name its own maturity.
761// Mixed hash-ness never matches: refusing to match is the safe direction for
762// every caller here, because each of them treats a match as "this debt is
763// already dealt with".
764static bool SamePayoutRound(int aHeight, const uint256& aHash,
765 int bHeight, const uint256& bHash)
766{
767 if (aHash != bHash)
768 return false;
769 if (aHash != 0)
770 return true;
771 return aHeight == bHeight;
772}
773
774// An entry with neither a hash nor a height cannot be told apart from any
775// other entry in the same state, so nothing may be recorded against it: a
776// txid written to the wrong one reports a debt as settled that is still owed.
777// Such an entry is never paid and is logged instead -- refusing to act is the
778// safe failure, and no file this tree has ever written can produce one.
779static bool PayoutRoundIdentifiable(int height, const uint256& hash)
780{
781 return hash != 0 || height != 0;
782}
783
784static bool PayoutFullyPaid(const PendingPayout& pp)
785{
786 for (const auto& kv : pp.amounts)
787 if (pp.paidTxids.count(kv.first) == 0)
788 return false;
789 return true;
790}
791
792// Serialise the ledger and stamp it with the version it was taken at. Caller
793// must hold gPayoutLedgerMutex.
794//
795// Split from the file write so no thread holds the ledger mutex across the
796// I/O. Upstream writes the file inside the per-recipient loop with
797// gPayoutLedgerMutex held, while the event loop's FlushMaturePayouts blocks on
798// that same mutex -- one slow fsync per recipient, on the thread the miners'
799// I/O shares. The sequence number is what lets the write happen later, and
800// possibly out of order, without a stale image reaching the disk.
801static VersionedBlob SerializePendingPayoutsLocked()
320{ 802{
321 nlohmann::json arr = nlohmann::json::array(); 803 nlohmann::json arr = nlohmann::json::array();
322 for (const auto& pp : gPendingPayouts) { 804 for (const auto& pp : gPendingPayouts) {
@@ -324,21 +806,56 @@ static void SavePendingPayouts()
324 obj["blockHeight"] = pp.blockHeight; 806 obj["blockHeight"] = pp.blockHeight;
325 obj["blockHash"] = pp.blockHash.ToString(); 807 obj["blockHash"] = pp.blockHash.ToString();
326 obj["matureAtHeight"] = pp.matureAtHeight; 808 obj["matureAtHeight"] = pp.matureAtHeight;
809 // No retryAfter: it was never written with anything but zero, and the
810 // retry clock that is actually used is on PayingRecord. Files that
811 // still carry the key load fine (it is ignored), and a build that
812 // expects it defaults an absent one to 0, which means "ready now" --
813 // the same answer it always got.
327 nlohmann::json amounts = nlohmann::json::object(); 814 nlohmann::json amounts = nlohmann::json::object();
328 for (const auto& kv : pp.amounts) 815 for (const auto& kv : pp.amounts)
329 amounts[kv.first] = kv.second; 816 amounts[kv.first] = kv.second;
330 obj["amounts"] = amounts; 817 obj["amounts"] = amounts;
818 nlohmann::json paid = nlohmann::json::object();
819 for (const auto& kv : pp.paidTxids)
820 paid[kv.first] = kv.second;
821 obj["paidTxids"] = paid;
331 arr.push_back(obj); 822 arr.push_back(obj);
332 } 823 }
333 std::string path = PayoutFilePath(); 824 VersionedBlob v;
334 FILE* f = fopen(path.c_str(), "w"); 825 v.blob = arr.dump(2);
335 if (f) { 826 v.seq = ++gPayoutLedgerSeq;
336 std::string s = arr.dump(2); 827 return v;
337 fwrite(s.c_str(), 1, s.size(), f); 828}
338 fclose(f); 829
339 } else { 830// Publish a ledger snapshot. Holds no data mutex; safe -- and intended -- to
340 LogPrint("payout", "[payout] WARNING: could not write %s\n", path.c_str()); 831// call with gPayoutLedgerMutex released.
341 } 832static void PublishPendingPayouts(const VersionedBlob& v)
833{
834 PublishVersioned(PayoutFilePath(), v, gPayoutLedgerFileMutex,
835 gPayoutLedgerPublishedSeq, "pending payout ledger");
836}
837
838// Parse a persisted block-hash field, refusing to guess. SetHex() reads
839// whatever hex prefix it finds and stops at the first character it does not
840// recognise, so a corrupt value does not fail to parse -- it parses as a
841// DIFFERENT hash. Both shapes of that are wrong in a dangerous direction: a
842// value with a hex-looking prefix ("deadbeef...") becomes a hash no block
843// ever had, which PayoutBlockStillOnChain reads as an orphaned block and
844// FlushMaturePayouts then VOIDS the debt; a value with no hex prefix at all
845// becomes 0, which is grandfathered past the orphan check on the strength
846// of garbage. A field that is not the 64-digit hex string ToString() wrote
847// is corruption, and corruption is a load failure like any other parse
848// error -- the file is quarantined, not read around.
849static uint256 ParsePayoutBlockHash(const std::string& str)
850{
851 if (str.size() != 64 ||
852 str.find_first_not_of("0123456789abcdefABCDEF") != std::string::npos)
853 throw std::runtime_error(
854 strprintf("blockHash \"%s\" is not a 64-digit hex string",
855 str.c_str()));
856 uint256 h;
857 h.SetHex(str);
858 return h;
342} 859}
343 860
344static void LoadPendingPayouts() 861static void LoadPendingPayouts()
@@ -354,6 +871,7 @@ static void LoadPendingPayouts()
354 fclose(f); 871 fclose(f);
355 try { 872 try {
356 nlohmann::json arr = nlohmann::json::parse(s); 873 nlohmann::json arr = nlohmann::json::parse(s);
874 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
357 for (const auto& obj : arr) { 875 for (const auto& obj : arr) {
358 PendingPayout pp; 876 PendingPayout pp;
359 pp.matureAtHeight = obj["matureAtHeight"].get<int>(); 877 pp.matureAtHeight = obj["matureAtHeight"].get<int>();
@@ -361,18 +879,33 @@ static void LoadPendingPayouts()
361 // field. Leave blockHash zero and they are grandfathered past the 879 // field. Leave blockHash zero and they are grandfathered past the
362 // orphan check rather than voided on a guess. 880 // orphan check rather than voided on a guess.
363 if (obj.contains("blockHash") && obj["blockHash"].is_string()) 881 if (obj.contains("blockHash") && obj["blockHash"].is_string())
364 pp.blockHash.SetHex(obj["blockHash"].get<std::string>()); 882 pp.blockHash =
883 ParsePayoutBlockHash(obj["blockHash"].get<std::string>());
365 if (obj.contains("blockHeight") && obj["blockHeight"].is_number()) 884 if (obj.contains("blockHeight") && obj["blockHeight"].is_number())
366 pp.blockHeight = obj["blockHeight"].get<int>(); 885 pp.blockHeight = obj["blockHeight"].get<int>();
886 // Upstream's tolerance for an entry with no usable height, kept
887 // because it costs nothing and an entry that cannot name its block
888 // is worse than one derived from its maturity. COINBASE_MATURITY
889 // and not COINBASE_SPENDABLE_DEPTH: this reverses whatever the
890 // writer of *that* file used, and only files old enough to lack
891 // blockHeight are affected -- they were written when maturity was
892 // still COINBASE_MATURITY.
893 if (pp.blockHeight == 0 && pp.matureAtHeight > COINBASE_MATURITY)
894 pp.blockHeight = pp.matureAtHeight - COINBASE_MATURITY;
895 // "retryAfter" is deliberately not read. Files written by the
896 // merged build carry it, always as 0, and there is nowhere left to
897 // put it -- the retry clock is PayingRecord's.
367 for (auto it = obj["amounts"].begin(); it != obj["amounts"].end(); ++it) 898 for (auto it = obj["amounts"].begin(); it != obj["amounts"].end(); ++it)
368 pp.amounts[it.key()] = it.value().get<int64>(); 899 pp.amounts[it.key()] = it.value().get<int64>();
900 if (obj.contains("paidTxids") && obj["paidTxids"].is_object())
901 for (auto it = obj["paidTxids"].begin(); it != obj["paidTxids"].end(); ++it)
902 pp.paidTxids[it.key()] = it.value().get<std::string>();
369 gPendingPayouts.push_back(pp); 903 gPendingPayouts.push_back(pp);
370 } 904 }
371 LogPrint("payout", "[payout] loaded %zu pending payout(s) from disk\n", 905 LogPrint("payout", "[payout] loaded %zu pending payout(s) from disk\n",
372 gPendingPayouts.size()); 906 gPendingPayouts.size());
373 } catch (const std::exception& e) { 907 } catch (const std::exception& e) {
374 LogPrint("payout", "[payout] WARNING: failed to parse %s: %s\n", 908 RefuseFileAfterLoadFailure(path, "pending payout ledger", e.what());
375 path.c_str(), e.what());
376 } 909 }
377} 910}
378 911
@@ -388,6 +921,18 @@ static void LoadPendingPayouts()
388// and reconciling against the wallet on startup, is what makes a restart 921// and reconciling against the wallet on startup, is what makes a restart
389// survivable. 922// survivable.
390struct PayingRecord { 923struct PayingRecord {
924 // Identity of this SLOT, distinct from the identity of the payment it
925 // describes. ForgetPayment and ReschedulePayment used to find their record
926 // by comparing every field, which is only an identity while no two records
927 // can be equal -- and a restart used to produce byte-identical duplicates
928 // (see FlushMaturePayouts), so the first match absorbed every update and
929 // the second was never touched again.
930 //
931 // Process-local and deliberately not persisted: gPaying is rebuilt from
932 // disk at every start, so an id only ever has to name a slot within one
933 // run, and reading ids back would mean trusting a file to contain no
934 // duplicates. Assigned under gPayoutExecMutex from gPayingNextId.
935 uint64 id = 0;
391 int blockHeight = 0; 936 int blockHeight = 0;
392 uint256 blockHash = 0; // zero = predates hash keying, see PendingPayout 937 uint256 blockHash = 0; // zero = predates hash keying, see PendingPayout
393 int matureAtHeight = 0; 938 int matureAtHeight = 0;
@@ -396,18 +941,66 @@ struct PayingRecord {
396 int retries = 0; 941 int retries = 0;
397 int deferrals = 0; 942 int deferrals = 0;
398 int64 retryAfter = 0; 943 int64 retryAfter = 0;
944 // Ask the wallet whether this payment already went out, once, before the
945 // first send attempt. Upstream asks on every attempt: FindRecordedPoolPayout
946 // is a linear scan of mapWallet under cs_mapWallet, per recipient, so with
947 // 2000 rotating payout addresses and a retry budget of 1440 that is a
948 // scan per address per minute for a day. The question it answers -- "did a
949 // previous process send this before it died?" -- can only change across a
950 // restart, and every restart re-sets this flag (ReconcilePayouts), so
951 // asking once per record is the whole of the answer.
952 //
953 // Not persisted: a record read back from disk is by definition a restart.
954 bool fProbeWallet = true;
399}; 955};
400static std::vector<PayingRecord> gPaying; // guarded by gPayoutExecMutex 956static std::vector<PayingRecord> gPaying; // guarded by gPayoutExecMutex
401static std::mutex gPayoutExecMutex; 957static std::mutex gPayoutExecMutex;
958static uint64 gPayingNextId = 0; // guarded by gPayoutExecMutex
959static uint64 gPayingSeq = 0; // guarded by gPayoutExecMutex
960static std::mutex gPayingFileMutex;
961static uint64 gPayingPublishedSeq = 0; // guarded by gPayingFileMutex
402static std::atomic<bool> gPayoutThreadRunning{false}; 962static std::atomic<bool> gPayoutThreadRunning{false};
403 963
964// Upstream's gPayoutExecQueue/gPayoutExecMutex/gPayoutThreadRunning are
965// declared above with gPaying: the in-flight list IS the exec queue in this
966// tree, at per-recipient rather than per-round granularity, so a second queue
967// would be a second answer to "who is owed money". Only the dedupe set is new.
968//
969// Ids of rounds currently exploded into gPaying, so a flush cannot queue the
970// same round twice. Guarded by gPayoutExecMutex. Seeded from the restored
971// in-flight list by ReconcilePayouts -- it is memory-only, and leaving it
972// empty at startup was half of why a restart duplicated everything in flight.
973static std::set<std::string> gQueuedPayouts;
974
975// Is this exact payment already in flight? gPaying is keyed on
976// (round, address): the ledger holds one amount per address per round, so two
977// records for that pair are two standing orders to send the same money.
978// Caller must hold gPayoutExecMutex.
979static bool PaymentInFlightLocked(int blockHeight, const uint256& blockHash,
980 const std::string& address)
981{
982 for (const PayingRecord& e : gPaying)
983 if (e.address == address
984 && SamePayoutRound(e.blockHeight, e.blockHash, blockHeight, blockHash))
985 return true;
986 return false;
987}
988
404static std::string PayingFilePath() 989static std::string PayingFilePath()
405{ 990{
406 return GetAppDir() + "/paying_payouts.json"; 991 return GetAppDir() + "/paying_payouts.json";
407} 992}
408 993
409// Caller must hold gPayoutExecMutex. 994// Serialise the in-flight list. Caller must hold gPayoutExecMutex.
410static void SavePayingLocked() 995//
996// gPayoutExecMutex already serialises every writer of this file, so it never
997// had the interleaving race pending_payouts.json had. It is still split the
998// same way for a different reason: FlushMaturePayouts runs on the event loop
999// and holds gPayoutExecMutex, so writing (and, since this series, fsyncing)
1000// the file under that mutex parks miner I/O behind the payout thread's disk.
1001// The version stamp keeps the file monotonic now that the write is outside
1002// the mutex that orders the writers.
1003static VersionedBlob SerializePayingLocked()
411{ 1004{
412 nlohmann::json arr = nlohmann::json::array(); 1005 nlohmann::json arr = nlohmann::json::array();
413 for (const auto& r : gPaying) { 1006 for (const auto& r : gPaying) {
@@ -419,24 +1012,24 @@ static void SavePayingLocked()
419 obj["amount"] = r.amount; 1012 obj["amount"] = r.amount;
420 obj["retries"] = r.retries; 1013 obj["retries"] = r.retries;
421 obj["deferrals"] = r.deferrals; 1014 obj["deferrals"] = r.deferrals;
1015 // The live retry clock, which used not to be written at all: every
1016 // restart handed every backing-off payment a retryAfter of 0 and put
1017 // the whole set into the very next 500 ms batch, so a node restarted
1018 // in a loop retried continuously instead of once a minute.
1019 obj["retryAfter"] = r.retryAfter;
422 arr.push_back(obj); 1020 arr.push_back(obj);
423 } 1021 }
424 std::string path = PayingFilePath(); 1022 VersionedBlob v;
425 // Write to a temporary file and rename, so a crash mid-write cannot leave 1023 v.blob = arr.dump(2);
426 // a truncated list of who is still owed money. 1024 v.seq = ++gPayingSeq;
427 std::string tmp = path + ".tmp"; 1025 return v;
428 FILE* f = fopen(tmp.c_str(), "w"); 1026}
429 if (!f) { 1027
430 LogPrint("payout", "[payout] WARNING: could not write %s\n", tmp.c_str()); 1028// Publish an in-flight-list snapshot. Call with gPayoutExecMutex released.
431 return; 1029static void PublishPaying(const VersionedBlob& v)
432 } 1030{
433 std::string s = arr.dump(2); 1031 PublishVersioned(PayingFilePath(), v, gPayingFileMutex,
434 fwrite(s.c_str(), 1, s.size(), f); 1032 gPayingPublishedSeq, "in-flight payment list");
435 fflush(f);
436 fclose(f);
437 if (rename(tmp.c_str(), path.c_str()) != 0)
438 LogPrint("payout", "[payout] WARNING: could not rename %s -> %s\n",
439 tmp.c_str(), path.c_str());
440} 1033}
441 1034
442static void LoadPaying() 1035static void LoadPaying()
@@ -452,26 +1045,209 @@ static void LoadPaying()
452 fclose(f); 1045 fclose(f);
453 try { 1046 try {
454 nlohmann::json arr = nlohmann::json::parse(s); 1047 nlohmann::json arr = nlohmann::json::parse(s);
1048 // gPaying and gPayingNextId are both annotated as guarded by
1049 // gPayoutExecMutex and this was mutating them without it. Safe today
1050 // -- LoadPaying only runs at startup, before the payout thread is
1051 // spawned -- but "safe because of where it is called from" is a fact
1052 // about the caller, not about this function, and the annotation is
1053 // what every other reader will trust. The mutex is uncontended here by
1054 // construction, so taking it costs nothing.
1055 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
455 for (const auto& obj : arr) { 1056 for (const auto& obj : arr) {
456 PayingRecord r; 1057 PayingRecord r;
457 r.address = obj["address"].get<std::string>(); 1058 r.address = obj["address"].get<std::string>();
458 r.amount = obj["amount"].get<int64>(); 1059 r.amount = obj["amount"].get<int64>();
459 r.matureAtHeight = obj["matureAtHeight"].get<int>(); 1060 r.matureAtHeight = obj["matureAtHeight"].get<int>();
460 if (obj.contains("blockHash") && obj["blockHash"].is_string()) 1061 if (obj.contains("blockHash") && obj["blockHash"].is_string())
461 r.blockHash.SetHex(obj["blockHash"].get<std::string>()); 1062 r.blockHash =
1063 ParsePayoutBlockHash(obj["blockHash"].get<std::string>());
462 if (obj.contains("blockHeight") && obj["blockHeight"].is_number()) 1064 if (obj.contains("blockHeight") && obj["blockHeight"].is_number())
463 r.blockHeight = obj["blockHeight"].get<int>(); 1065 r.blockHeight = obj["blockHeight"].get<int>();
464 if (obj.contains("retries") && obj["retries"].is_number()) 1066 if (obj.contains("retries") && obj["retries"].is_number())
465 r.retries = obj["retries"].get<int>(); 1067 r.retries = obj["retries"].get<int>();
466 if (obj.contains("deferrals") && obj["deferrals"].is_number()) 1068 if (obj.contains("deferrals") && obj["deferrals"].is_number())
467 r.deferrals = obj["deferrals"].get<int>(); 1069 r.deferrals = obj["deferrals"].get<int>();
1070 // Absent in files written before this change; 0 means ready now,
1071 // which is exactly the behaviour those files used to get.
1072 if (obj.contains("retryAfter") && obj["retryAfter"].is_number())
1073 r.retryAfter = obj["retryAfter"].get<int64>();
1074 // Collapse duplicates as they are read. A build without the
1075 // ReconcilePayouts seeding below wrote a second (and after another
1076 // restart, a third) byte-identical record for every unpaid
1077 // recipient, so a file written by one of those has to be cleaned
1078 // up here or the pathology is inherited. The retry and deferral
1079 // counts are merged upwards so collapsing does not hand the
1080 // survivor a fresh budget.
1081 PayingRecord* dup = NULL;
1082 for (PayingRecord& e : gPaying)
1083 if (e.address == r.address
1084 && SamePayoutRound(e.blockHeight, e.blockHash,
1085 r.blockHeight, r.blockHash)) {
1086 dup = &e;
1087 break;
1088 }
1089 if (dup) {
1090 dup->retries = std::max(dup->retries, r.retries);
1091 dup->deferrals = std::max(dup->deferrals, r.deferrals);
1092 dup->retryAfter = std::max(dup->retryAfter, r.retryAfter);
1093 LogPrint("payout", "[payout] collapsing duplicate in-flight"
1094 " record for %s to %s -- one payment, one record\n",
1095 FormatMoney(r.amount).c_str(), r.address.c_str());
1096 continue;
1097 }
1098 r.id = ++gPayingNextId;
468 gPaying.push_back(r); 1099 gPaying.push_back(r);
469 } 1100 }
470 LogPrint("payout", "[payout] loaded %zu in-flight payment(s) from disk\n", 1101 LogPrint("payout", "[payout] loaded %zu in-flight payment(s) from disk\n",
471 gPaying.size()); 1102 gPaying.size());
472 } catch (const std::exception& e) { 1103 } catch (const std::exception& e) {
473 LogPrint("payout", "[payout] WARNING: failed to parse %s: %s\n", 1104 RefuseFileAfterLoadFailure(path, "in-flight payment list", e.what());
474 path.c_str(), e.what()); 1105 }
1106}
1107
1108// ---------------------------------------------------------------------------
1109// Pool round proofs (upstream cbdd06d). Disjoint from the in-flight ledger
1110// above -- this is the operator's published record of how a round was split,
1111// not an obligation to pay. Both sides added code at this offset; both are
1112// kept.
1113// ---------------------------------------------------------------------------
1114static nlohmann::json PoolRoundToJson(const PoolRoundProof& r)
1115{
1116 nlohmann::json j;
1117 j["id"] = r.id;
1118 j["blockHeight"] = r.blockHeight;
1119 j["matureAtHeight"] = r.matureAtHeight;
1120 j["createdAt"] = r.createdAt;
1121 j["blockHash"] = r.blockHash;
1122 j["coinbaseTxid"] = r.coinbaseTxid;
1123 j["foundBy"] = r.foundBy;
1124 j["feePercent"] = r.feePercent;
1125 j["totalShares"] = r.totalShares;
1126 j["coinbaseValueSatoshis"] = r.coinbaseValue;
1127 j["coinbaseValueBtf"] = FormatMoney(r.coinbaseValue);
1128 j["subsidySatoshis"] = r.subsidy;
1129 j["subsidyBtf"] = FormatMoney(r.subsidy);
1130 j["minerRewardSatoshis"] = r.minerReward;
1131 j["minerRewardBtf"] = FormatMoney(r.minerReward);
1132 j["operatorRewardSatoshis"] = r.operatorReward;
1133 j["operatorRewardBtf"] = FormatMoney(r.operatorReward);
1134 j["skippedDust"] = r.skippedDust;
1135 j["forfeitedBadAddressSatoshis"] = r.forfeitedBadAddress;
1136 j["forfeitedBadAddressBtf"] = FormatMoney(r.forfeitedBadAddress);
1137
1138 j["shares"] = nlohmann::json::object();
1139 for (const auto& kv : r.shares)
1140 j["shares"][kv.first] = kv.second;
1141
1142 j["payouts"] = nlohmann::json::object();
1143 for (const auto& kv : r.payouts) {
1144 nlohmann::json p;
1145 p["amountSatoshis"] = kv.second;
1146 p["amountBtf"] = FormatMoney(kv.second);
1147 std::map<std::string, std::string>::const_iterator tx = r.paidTxids.find(kv.first);
1148 if (tx != r.paidTxids.end())
1149 p["txid"] = tx->second;
1150 else
1151 p["txid"] = nullptr;
1152 // Only emitted when it happened, so the common shape of the file is
1153 // unchanged for anything already parsing it.
1154 std::map<std::string, std::string>::const_iterator fo = r.forfeited.find(kv.first);
1155 if (fo != r.forfeited.end()) {
1156 p["forfeited"] = true;
1157 p["forfeitReason"] = fo->second;
1158 }
1159 j["payouts"][kv.first] = p;
1160 }
1161 return j;
1162}
1163
1164// Serialise the published round proofs. Caller must hold gPoolRoundsMutex.
1165//
1166// Like the in-flight list, gPoolRoundsMutex already serialises the writers
1167// (RecordPoolRound on the event loop, MarkPoolRoundPaid on the payout thread),
1168// so this file never had the interleaving race either -- but the event loop
1169// should not be waiting on the payout thread's fsync to publish a round, so
1170// the write moves outside the mutex on the same versioned terms.
1171static VersionedBlob SerializePoolRoundsLocked()
1172{
1173 nlohmann::json root;
1174 root["schema"] = "bitflash-pool-rounds-1";
1175 root["updatedAt"] = GetTime();
1176 root["rounds"] = nlohmann::json::array();
1177 for (const PoolRoundProof& r : gPoolRounds)
1178 root["rounds"].push_back(PoolRoundToJson(r));
1179
1180 VersionedBlob v;
1181 v.blob = root.dump(2);
1182 v.seq = ++gPoolRoundsSeq;
1183 return v;
1184}
1185
1186// Publish a round-proof snapshot. Call with gPoolRoundsMutex released.
1187static void PublishPoolRounds(const VersionedBlob& v)
1188{
1189 PublishVersioned(PoolRoundsFilePath(), v, gPoolRoundsFileMutex,
1190 gPoolRoundsPublishedSeq, "pool round proofs");
1191}
1192
1193static void LoadPoolRounds()
1194{
1195 std::string path = PoolRoundsFilePath();
1196 FILE* f = fopen(path.c_str(), "r");
1197 if (!f) return;
1198 fseek(f, 0, SEEK_END);
1199 long sz = ftell(f);
1200 fseek(f, 0, SEEK_SET);
1201 std::string s(sz, '\0');
1202 fread(&s[0], 1, sz, f);
1203 fclose(f);
1204
1205 try {
1206 nlohmann::json root = nlohmann::json::parse(s);
1207 nlohmann::json arr = root.is_array() ? root : root["rounds"];
1208 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1209 for (const auto& obj : arr) {
1210 PoolRoundProof r;
1211 if (obj.contains("id") && obj["id"].is_string()) r.id = obj["id"].get<std::string>();
1212 if (obj.contains("blockHeight")) r.blockHeight = obj["blockHeight"].get<int>();
1213 if (obj.contains("matureAtHeight")) r.matureAtHeight = obj["matureAtHeight"].get<int>();
1214 if (obj.contains("createdAt")) r.createdAt = obj["createdAt"].get<int64>();
1215 if (obj.contains("blockHash") && obj["blockHash"].is_string()) r.blockHash = obj["blockHash"].get<std::string>();
1216 if (obj.contains("coinbaseTxid") && obj["coinbaseTxid"].is_string()) r.coinbaseTxid = obj["coinbaseTxid"].get<std::string>();
1217 if (obj.contains("foundBy") && obj["foundBy"].is_string()) r.foundBy = obj["foundBy"].get<std::string>();
1218 if (obj.contains("feePercent")) r.feePercent = obj["feePercent"].get<double>();
1219 if (obj.contains("totalShares")) r.totalShares = obj["totalShares"].get<uint64>();
1220 if (obj.contains("coinbaseValueSatoshis")) r.coinbaseValue = obj["coinbaseValueSatoshis"].get<int64>();
1221 if (obj.contains("subsidySatoshis")) r.subsidy = obj["subsidySatoshis"].get<int64>();
1222 if (obj.contains("minerRewardSatoshis")) r.minerReward = obj["minerRewardSatoshis"].get<int64>();
1223 if (obj.contains("operatorRewardSatoshis")) r.operatorReward = obj["operatorRewardSatoshis"].get<int64>();
1224 if (obj.contains("skippedDust")) r.skippedDust = obj["skippedDust"].get<int>();
1225 if (obj.contains("forfeitedBadAddressSatoshis")) r.forfeitedBadAddress = obj["forfeitedBadAddressSatoshis"].get<int64>();
1226 if (obj.contains("shares") && obj["shares"].is_object())
1227 for (auto it = obj["shares"].begin(); it != obj["shares"].end(); ++it)
1228 r.shares[it.key()] = it.value().get<uint64>();
1229 if (obj.contains("payouts") && obj["payouts"].is_object())
1230 for (auto it = obj["payouts"].begin(); it != obj["payouts"].end(); ++it) {
1231 if (it.value().is_object()) {
1232 r.payouts[it.key()] = it.value().value("amountSatoshis", (int64)0);
1233 if (it.value().contains("txid") && it.value()["txid"].is_string())
1234 r.paidTxids[it.key()] = it.value()["txid"].get<std::string>();
1235 if (it.value().value("forfeited", false))
1236 r.forfeited[it.key()] =
1237 it.value().value("forfeitReason", std::string("unspecified"));
1238 } else if (it.value().is_number_integer()) {
1239 r.payouts[it.key()] = it.value().get<int64>();
1240 }
1241 }
1242 if (!r.id.empty())
1243 gPoolRounds.push_back(r);
1244 }
1245 while (gPoolRounds.size() > MAX_POOL_ROUNDS)
1246 gPoolRounds.erase(gPoolRounds.begin());
1247 LogPrint("payout", "[payout] loaded %zu pool round proof(s) from disk\n",
1248 gPoolRounds.size());
1249 } catch (const std::exception& e) {
1250 RefuseFileAfterLoadFailure(path, "pool round proofs", e.what());
475 } 1251 }
476} 1252}
477 1253
@@ -494,105 +1270,487 @@ static bool PayoutBlockStillOnChain(const uint256& blockHash)
494 return false; 1270 return false;
495} 1271}
496 1272
497static std::string PayoutComment(int blockHeight) 1273// The comment every pre-upgrade payout transaction carries, and therefore the
1274// ONLY needle the legacy fallback in FindRecordedPoolPayout may search with.
1275// This format is frozen: change it and every payment those builds made stops
1276// matching, every one of those debts looks unpaid, and reconciliation pays
1277// them all a second time.
1278static std::string LegacyPayoutComment(int blockHeight)
498{ 1279{
499 return strprintf("Pool payout height %d", blockHeight); 1280 return strprintf("Pool payout height %d", blockHeight);
500} 1281}
501 1282
502// Did we already send this payment before dying? Looks for a wallet 1283// The comment written on new sends. It names the funding block, because the
503// transaction of ours carrying this payout's comment and paying exactly this 1284// height alone cannot: a height that is won, lost to a reorg, and won again
504// amount to this address. 1285// yields two rounds whose height-only comments are identical -- and identical
1286// amounts too, whenever the same miner held both rounds. The structured
1287// pool_payout_* metadata already keys the payment by hash for the code; this
1288// puts the same fact where an operator reading the wallet can see it, and
1289// makes a new transaction unmatchable by the legacy needle even if the
1290// structured keys are ever lost.
505// 1291//
506// Biased towards reporting "already paid" on any doubt: skipping a payment a 1292// A zero hash keeps the legacy form: those records predate hash keying, and
507// miner is owed is visible and recoverable from the log, while paying twice 1293// there is no hash to name.
508// is neither. 1294static std::string PayoutComment(int blockHeight, const uint256& blockHash)
509static bool WalletAlreadyPaid(const PayingRecord& r)
510{ 1295{
511 uint160 h160; 1296 if (blockHash == 0)
512 if (!AddressToHash160(r.address, h160)) 1297 return LegacyPayoutComment(blockHeight);
513 return false; 1298 return strprintf("Pool payout height %d block %s", blockHeight,
514 CScript want; 1299 blockHash.ToString().c_str());
515 want << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; 1300}
516 const std::string comment = PayoutComment(r.blockHeight);
517 1301
518 CRITICAL_BLOCK(cs_mapWallet) 1302// Startup reconciliation.
1303//
1304// This used to be the crash-recovery mechanism: it scanned the wallet with
1305// WalletAlreadyPaid() and then dropped ledger entries that had already been
1306// exploded into in-flight records. Neither job is right any more.
1307//
1308// The ledger is now authoritative and an entry survives until every recipient
1309// has a txid recorded against it (upstream cbdd06d), so dropping ledger
1310// entries here would reintroduce exactly the loss this merge is adopting that
1311// model to prevent. And the wallet scan is now FindRecordedPoolPayout(), done
1312// once per record on its first attempt, which is the same question asked in
1313// the one place that can act on the answer. Running both would be two
1314// authorities for the same debt.
1315//
1316// What is left is the useful part: the in-flight list restored from disk may
1317// contain records the ledger already shows as paid (we died between the send
1318// and the in-flight file being rewritten). The ledger wins.
1319//
1320// And the part that was missing: gQueuedPayouts is memory-only, so it came up
1321// empty while gPaying came up full. FlushMaturePayouts deduped against that
1322// empty set and against the ledger's paidTxids, and against nothing else --
1323// so a round that was exploded but not yet paid got a SECOND record per
1324// unpaid recipient on the first flush after every restart, deterministically.
1325// (Observed live: two byte-identical records for one (block, address).) The
1326// surviving in-flight list is exactly the statement of which rounds are
1327// already exploded, so it is what the set is rebuilt from.
1328static void ReconcilePayouts()
1329{
1330 VersionedBlob payingSnap;
519 { 1331 {
520 for (std::map<uint256, CWalletTx>::iterator it = mapWallet.begin(); 1332 std::lock_guard<std::mutex> exec(gPayoutExecMutex);
521 it != mapWallet.end(); ++it) { 1333 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
522 CWalletTx& wtx = it->second; 1334
523 // The comment is written only by the payout path, so its presence 1335 size_t settled = 0;
524 // already identifies the transaction as one of ours. 1336 int64 settledAmount = 0;
525 std::map<std::string, std::string>::const_iterator ci = 1337 for (auto it = gPaying.begin(); it != gPaying.end(); ) {
526 wtx.mapValue.find("comment"); 1338 bool paid = false;
527 if (ci == wtx.mapValue.end() || ci->second != comment) 1339 for (const PendingPayout& pp : gPendingPayouts)
1340 if (SamePayoutRound(pp.blockHeight, pp.blockHash,
1341 it->blockHeight, it->blockHash)
1342 && pp.paidTxids.count(it->address)) {
1343 paid = true;
1344 break;
1345 }
1346 if (paid) {
1347 settledAmount += it->amount;
1348 settled++;
1349 it = gPaying.erase(it);
1350 } else {
1351 // Every restored record gets one wallet probe before it is
1352 // allowed to send: this is precisely the window where the
1353 // wallet may hold a transaction the ledger has not recorded.
1354 it->fProbeWallet = true;
1355 ++it;
1356 }
1357 }
1358 if (settled) {
1359 LogPrint("payout", "[payout] %zu in-flight payment(s) totalling %s"
1360 " are already recorded paid in the ledger -- not"
1361 " resending\n",
1362 settled, FormatMoney(settledAmount).c_str());
1363 }
1364
1365 // Rebuild the "already exploded" set from what survived. Without this
1366 // the first flush after a restart re-explodes every round in flight.
1367 gQueuedPayouts.clear();
1368 for (const PayingRecord& r : gPaying)
1369 gQueuedPayouts.insert(PayoutIdForBlock(r.blockHeight, r.blockHash));
1370
1371 payingSnap = SerializePayingLocked();
1372
1373 if (!gPaying.empty()) {
1374 int64 owed = 0;
1375 for (const auto& r : gPaying) owed += r.amount;
1376 LogPrint("payout", "[payout] resuming %zu in-flight payment(s)"
1377 " totalling %s\n",
1378 gPaying.size(), FormatMoney(owed).c_str());
1379 }
1380 }
1381 PublishPaying(payingSnap);
1382}
1383
1384static void RecordPoolRound(const PoolRoundProof& round)
1385{
1386 VersionedBlob snap;
1387 {
1388 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1389 bool replaced = false;
1390 for (PoolRoundProof& existing : gPoolRounds) {
1391 if (existing.id == round.id) {
1392 existing = round;
1393 replaced = true;
1394 break;
1395 }
1396 }
1397 if (!replaced) {
1398 gPoolRounds.push_back(round);
1399 while (gPoolRounds.size() > MAX_POOL_ROUNDS)
1400 gPoolRounds.erase(gPoolRounds.begin());
1401 }
1402 snap = SerializePoolRoundsLocked();
1403 }
1404 PublishPoolRounds(snap);
1405}
1406
1407static void MarkPoolRoundPaid(const std::string& id,
1408 const std::string& address,
1409 const std::string& txid)
1410{
1411 VersionedBlob snap;
1412 {
1413 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1414 bool found = false;
1415 for (PoolRoundProof& r : gPoolRounds) {
1416 if (r.id != id)
528 continue; 1417 continue;
529 foreach (const CTxOut& txout, wtx.vout) 1418 r.paidTxids[address] = txid;
530 if (txout.nValue == r.amount && txout.scriptPubKey == want) 1419 found = true;
531 return true; 1420 break;
1421 }
1422 if (!found)
1423 return;
1424 snap = SerializePoolRoundsLocked();
1425 }
1426 PublishPoolRounds(snap);
1427}
1428
1429// Publish the fact that a recipient was written off, so the round proof and
1430// the ledger agree. The ledger erases the amount (it is not owed any more and
1431// must not be reported as sent); the proof keeps the amount visible and marks
1432// it forfeited with a reason, because a miner who was promised a share is
1433// entitled to see what happened to it.
1434static void MarkPoolRoundForfeited(const std::string& id,
1435 const std::string& address,
1436 const std::string& reason)
1437{
1438 VersionedBlob snap;
1439 {
1440 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1441 bool found = false;
1442 for (PoolRoundProof& r : gPoolRounds) {
1443 if (r.id != id)
1444 continue;
1445 r.forfeited[address] = reason;
1446 found = true;
1447 break;
532 } 1448 }
1449 if (!found)
1450 return;
1451 snap = SerializePoolRoundsLocked();
533 } 1452 }
1453 PublishPoolRounds(snap);
1454}
1455
1456// Does this round have a published round proof?
1457//
1458// Existence answers a temporal question the legacy fallback below needs:
1459// which builds could have paid this round. RecordPoolRound writes the proof
1460// when the round is created, and structured payout metadata (upstream
1461// cbdd06d) is an ancestor of round proofs (upstream 83ba822) -- no build
1462// able to write a proof has ever sent a payment that only a comment can
1463// identify. A round with a proof therefore cannot have a comment-only
1464// payment; one that matches it is another round's, colliding on the height.
1465static bool PoolRoundHasProof(const std::string& id)
1466{
1467 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1468 for (const PoolRoundProof& r : gPoolRounds)
1469 if (r.id == id)
1470 return true;
534 return false; 1471 return false;
535} 1472}
536 1473
537// Startup reconciliation. Drops in-flight records the wallet shows were 1474// Which wallet transaction, if any, already paid this?
538// already sent, and drops ledger entries that were already exploded into 1475//
539// records (which happens if we died between writing the records and rewriting 1476// Two kinds of evidence, in order of quality.
540// the ledger -- both files then describe the same debt). 1477//
541static void ReconcilePayouts() 1478// Structured (pool_payout_id / pool_payout_address / pool_payout_amount,
1479// plus pool_payout_block since this series) is upstream's, extended. The id
1480// is derived from the block hash, but only from the FIRST sixteen hex digits
1481// of it -- and proof-of-work concentrates zeros exactly there, so the id can
1482// name two same-height rounds identically. New sends therefore record the
1483// full hash as pool_payout_block, and a structured match must agree with it
1484// wherever both sides have one; a transaction without the key was sent
1485// before it existed, and for those the id is the best evidence there is.
1486// Every transaction this build sends carries all four keys.
1487//
1488// But NO transaction this pool has ever sent carries it. Every payout in the
1489// existing wallet has only a "comment". So on the first boot after this
1490// upgrade -- precisely the boot where the binary is swapped after an unclean
1491// shutdown with payments in flight -- a structured-only probe finds nothing
1492// and every one of those recipients is paid a second time. The deleted
1493// WalletAlreadyPaid() matched on what the payment WAS, and that evidence is
1494// still there; it just has to be read.
1495//
1496// The legacy fallback is therefore restored, with three constraints that
1497// together stop it answering for a round that is not this one:
1498//
1499// 1. it is only consulted when the structured lookup found nothing;
1500// 2. it ignores any transaction carrying pool_payout_id. That key means this
1501// build wrote the transaction, and this build's transactions are
1502// adjudicated by the exact path -- so a NEW round's payment can never be
1503// offered as evidence for an old one. Only pre-upgrade transactions,
1504// which is exactly the population the fallback exists for, are eligible;
1505// 3. `excluded` holds every txid already credited to some OTHER
1506// (round, address) pair -- from the ledger's paidTxids, which is durable,
1507// and from what this process has already claimed. One legacy transaction
1508// can therefore settle at most one debt, so N transactions can never
1509// satisfy more than N obligations;
1510//
1511// 4. it is not consulted at all for a round that has a round proof
1512// (fAllowLegacy, decided by the caller from PoolRoundHasProof). Proofs
1513// are written when a round is created, and only by builds whose every
1514// payment names its round exactly -- so a comment-only transaction
1515// cannot be a payment for such a round, only a same-height collision
1516// from an older one, and skipping the fallback cannot hide a real
1517// payment: the structured probe above already found any that exist.
1518//
1519// What is left unresolved by construction is a comment-only transaction and
1520// a PROOFLESS round at the same height paying the same address the same
1521// amount: two grandfathered zero-hash twins, or a twin pair from the builds
1522// that keyed the ledger by hash before any metadata reached the wallet. The
1523// comment carries only the height, so nothing ties the transaction to one
1524// twin rather than the other. That needs a reorg.
1525//
1526// This used to say the losing twin's ledger entry is voided by the orphan
1527// check before it can be paid. That is wrong: PayoutBlockStillOnChain()
1528// returns true for a zero hash, and EVERY pre-upgrade entry has a zero hash,
1529// so the orphan check is a no-op for exactly the population this fallback
1530// serves.
1531//
1532// The outcome is still safe, by a different mechanism. SamePayoutRound()
1533// treats two zero-hash entries at the same height as the SAME round, so the
1534// pair is indistinguishable everywhere, not only here: FlushMaturePayouts
1535// explodes one of them and skips the other as already queued,
1536// PaymentInFlightLocked refuses a second record for that (round, address), and
1537// RecordPayoutTxid settles against the first match. The pair therefore
1538// presents as ONE debt and can be paid at most once -- which is the direction
1539// that matters for a fallback whose failure mode is treating one payment as
1540// proof of two.
1541//
1542// The cost of that mechanism, stated plainly: only one of a genuine
1543// same-height pair would ever be paid. It takes a reorg AND both rounds
1544// predating hash keying, and no file this tree has written since can produce
1545// one, so it is a property of history rather than a live risk.
1546//
1547// Where the evidence genuinely runs out the fallback simply does not match,
1548// and the payment is made -- paying a debt we cannot prove was settled is the
1549// correct default for a pool.
1550enum PayoutProbeResult {
1551 PAYOUT_PROBE_NONE = 0, // the wallet holds no record of this payment
1552 PAYOUT_PROBE_STRUCTURED, // matched this build's pool_payout_* metadata
1553 PAYOUT_PROBE_LEGACY // matched a pre-upgrade comment and output
1554};
1555
1556static PayoutProbeResult FindRecordedPoolPayout(const std::string& id,
1557 int blockHeight,
1558 const uint256& blockHash,
1559 const std::string& address,
1560 int64 amount,
1561 bool fAllowLegacy,
1562 const std::set<std::string>& excluded,
1563 std::string& txidRet)
542{ 1564{
543 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 1565 // Built before cs_mapWallet is taken: an invalid address has no script to
1566 // compare against, and the caller has already rejected those.
1567 uint160 h160;
1568 bool fHaveScript = AddressToHash160(address, h160);
1569 CScript want;
1570 if (fHaveScript)
1571 want << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
1572 const std::string comment = LegacyPayoutComment(blockHeight);
1573
1574 // Candidate legacy matches, adjudicated after cs_mapWallet is released.
1575 //
1576 // The depth check cannot happen inside the scan: GetDepthInMainChain()
1577 // reads mapBlockIndex and pindexBest, which belong to cs_main, and cs_main
1578 // is taken BEFORE cs_mapWallet everywhere else in this tree
1579 // (CreateTransaction, ConnectBlock). Taking it the other way round here
1580 // would be a lock-order inversion in the payout thread. There is at most a
1581 // handful of these: a candidate has to carry this exact height's comment
1582 // and an output of exactly this amount to this address.
1583 std::vector<uint256> legacyCandidates;
1584
1585 // One pass, not two: this scan runs under cs_mapWallet, which the whole
1586 // node contends on, and the merge is already paying for it once per record.
1587 CRITICAL_BLOCK(cs_mapWallet)
1588 {
1589 for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin();
1590 it != mapWallet.end(); ++it)
1591 {
1592 const CWalletTx& wtx = it->second;
1593 map<string, string>::const_iterator idIt = wtx.mapValue.find("pool_payout_id");
1594 if (idIt != wtx.mapValue.end()) {
1595 // Written by this build. Adjudicated exactly, and never
1596 // offered to the fallback below.
1597 if (idIt->second != id)
1598 continue;
1599 map<string, string>::const_iterator addrIt = wtx.mapValue.find("pool_payout_address");
1600 if (addrIt == wtx.mapValue.end() || addrIt->second != address)
1601 continue;
1602 map<string, string>::const_iterator amtIt = wtx.mapValue.find("pool_payout_amount");
1603 if (amtIt == wtx.mapValue.end() || amtIt->second != strprintf("%lld", amount))
1604 continue;
1605 // The id alone cannot tell twin rounds at one height apart
1606 // (see above); the full hash can, where the transaction
1607 // recorded one. Where it did not, the id decides, as it
1608 // always has -- refusing those transactions would make every
1609 // payment sent before the key existed look unpaid and pay it
1610 // again.
1611 if (blockHash != 0) {
1612 map<string, string>::const_iterator blkIt =
1613 wtx.mapValue.find("pool_payout_block");
1614 if (blkIt != wtx.mapValue.end()
1615 && blkIt->second != blockHash.ToString())
1616 continue; // a twin round's payment, not this round's
1617 }
1618 txidRet = wtx.GetHash().GetHex();
1619 return PAYOUT_PROBE_STRUCTURED;
1620 }
544 1621
545 size_t settled = 0; 1622 if (!fAllowLegacy)
546 int64 settledAmount = 0; 1623 continue; // comment-only evidence is inadmissible: rule 4
547 for (auto it = gPaying.begin(); it != gPaying.end(); ) { 1624 if (!fHaveScript)
548 if (WalletAlreadyPaid(*it)) { 1625 continue;
549 settledAmount += it->amount; 1626 // The comment is written only by the payout path, so its presence
550 settled++; 1627 // already identifies the transaction as one of ours.
551 it = gPaying.erase(it); 1628 map<string, string>::const_iterator ci = wtx.mapValue.find("comment");
552 } else { 1629 if (ci == wtx.mapValue.end() || ci->second != comment)
553 ++it; 1630 continue;
1631 if (excluded.count(wtx.GetHash().GetHex()))
1632 continue; // already credited to a different debt
1633 foreach (const CTxOut& txout, wtx.vout)
1634 if (txout.nValue == amount && txout.scriptPubKey == want) {
1635 legacyCandidates.push_back(it->first);
1636 break;
1637 }
554 } 1638 }
555 } 1639 }
556 if (settled) {
557 LogPrint("payout", "[payout] %zu in-flight payment(s) totalling %s were"
558 " already sent before the last shutdown -- not resending\n",
559 settled, FormatMoney(settledAmount).c_str());
560 SavePayingLocked();
561 }
562 1640
563 size_t dropped = 0; 1641 // A legacy match must have CONFIRMED.
564 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { 1642 //
565 bool exploded = false; 1643 // The structured path does not need this and deliberately does not have
566 for (const auto& r : gPaying) 1644 // it: this process sent that transaction moments ago, and a send that
567 if (r.matureAtHeight == it->matureAtHeight && r.blockHash == it->blockHash) { 1645 // failed leaves nothing behind, because SendMoney calls
568 exploded = true; 1646 // RollbackTransactionSpent, which calls EraseFromWallet.
569 break; 1647 //
1648 // The legacy path is the opposite case in every respect. Those
1649 // transactions are days or weeks old, so whether they confirmed is a
1650 // settled fact rather than an open question, and a pre-upgrade payout that
1651 // never confirmed is evidence of NON-payment about to be read as payment.
1652 // This chain has had a 280-block reorg (see FlushMaturePayouts): a payout
1653 // invalidated by it is still sitting in mapWallet with its comment intact,
1654 // and the first boot after this upgrade is exactly when it gets matched.
1655 // The cost of being wrong here is a miner who is never paid and no record
1656 // that they were owed.
1657 CRITICAL_BLOCK(cs_main)
1658 CRITICAL_BLOCK(cs_mapWallet)
1659 {
1660 for (const uint256& hash : legacyCandidates) {
1661 map<uint256, CWalletTx>::const_iterator it = mapWallet.find(hash);
1662 if (it == mapWallet.end())
1663 continue; // erased under us; it is not evidence of anything
1664 int depth = it->second.GetDepthInMainChain();
1665 if (depth < 1) {
1666 // Loud: this is the difference between a miner having been
1667 // paid and a miner about to be paid twice, decided on a fact
1668 // that is easy to check and impossible to recover afterwards.
1669 LogPrint("payout", "[payout] wallet tx %s carries the payout"
1670 " comment for height %d and an output of exactly %s to"
1671 " %s, but it is NOT in the main chain (depth %d) --"
1672 " refusing to read it as payment; this recipient will"
1673 " be paid\n",
1674 hash.GetHex().c_str(), blockHeight,
1675 FormatMoney(amount).c_str(), address.c_str(), depth);
1676 continue;
570 } 1677 }
571 if (exploded) { 1678 txidRet = hash.GetHex();
572 LogPrint("payout", "[payout] ledger entry for block %d is already" 1679 return PAYOUT_PROBE_LEGACY;
573 " in flight -- dropping the duplicate\n", it->blockHeight);
574 it = gPendingPayouts.erase(it);
575 dropped++;
576 } else {
577 ++it;
578 } 1680 }
579 } 1681 }
580 if (dropped) SavePendingPayouts(); 1682 return PAYOUT_PROBE_NONE;
1683}
581 1684
582 if (!gPaying.empty()) { 1685// txid -> "<round id>|<address>", for payments this run has already credited
583 int64 owed = 0; 1686// to a debt on legacy evidence. Stops one pre-upgrade transaction being read
584 for (const auto& r : gPaying) owed += r.amount; 1687// as proof that two different obligations were met. The ledger's paidTxids
585 LogPrint("payout", "[payout] resuming %zu in-flight payment(s) totalling %s\n", 1688// covers the same ground durably; this covers the window before the recovered
586 gPaying.size(), FormatMoney(owed).c_str()); 1689// txid reaches the ledger, and the case where the ledger entry has already
1690// been retired.
1691static std::map<std::string, std::string> gLegacyClaimedTxids;
1692static std::mutex gClaimedTxidMutex;
1693
1694// Every txid already spoken for by some payment OTHER than this one. Collected
1695// before cs_mapWallet is taken: the ledger mutex must not be acquired under a
1696// wallet lock.
1697static void CollectForeignPayoutTxids(int blockHeight, const uint256& blockHash,
1698 const std::string& address,
1699 const std::string& claimKey,
1700 std::set<std::string>& out)
1701{
1702 {
1703 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1704 for (const PendingPayout& pp : gPendingPayouts) {
1705 bool sameRound = SamePayoutRound(pp.blockHeight, pp.blockHash,
1706 blockHeight, blockHash);
1707 for (const auto& kv : pp.paidTxids)
1708 if (!(sameRound && kv.first == address))
1709 out.insert(kv.second);
1710 }
1711 }
1712 {
1713 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
1714 for (const auto& kv : gLegacyClaimedTxids)
1715 if (kv.second != claimKey)
1716 out.insert(kv.first);
587 } 1717 }
588} 1718}
589 1719
1720static void ClaimLegacyPayoutTxid(const std::string& txid, const std::string& claimKey)
1721{
1722 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
1723 gLegacyClaimedTxids[txid] = claimKey;
1724}
1725
1726// blockHash stays a uint256 and not upstream's display string: it is the key
1727// this fork's payout ledger is built on, and a key that has been through
1728// ToString() is a key that can be compared wrong. It is converted to text only
1729// where the round proof needs text. The other three arguments are upstream's,
1730// carried for the round-proof data.
590static void QueuePayouts(int blockHeight, 1731static void QueuePayouts(int blockHeight,
591 const uint256& blockHash, 1732 const uint256& blockHash,
1733 const std::string& coinbaseTxid,
1734 int64 coinbaseValue,
1735 const std::string& foundBy,
592 const std::map<std::string, uint64>& shareCount, 1736 const std::map<std::string, uint64>& shareCount,
593 uint64 total) 1737 uint64 total)
594{ 1738{
1739 PoolRoundProof round;
1740 round.id = PayoutIdForBlock(blockHeight, blockHash);
1741 round.blockHeight = blockHeight;
1742 round.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH;
1743 round.createdAt = GetTime();
1744 round.blockHash = blockHash.ToString();
1745 round.coinbaseTxid = coinbaseTxid;
1746 round.coinbaseValue = coinbaseValue;
1747 round.foundBy = foundBy;
1748 round.feePercent = dPoolFeePercent;
1749 round.totalShares = total;
1750 round.shares = shareCount;
1751
595 if (total == 0) { 1752 if (total == 0) {
1753 RecordPoolRound(round);
596 LogPrint("payout", "[payout] block %d: zero shares recorded -- nothing to pay out\n", 1754 LogPrint("payout", "[payout] block %d: zero shares recorded -- nothing to pay out\n",
597 blockHeight); 1755 blockHeight);
598 return; 1756 return;
@@ -608,6 +1766,10 @@ static void QueuePayouts(int blockHeight,
608 int64 nSubsidy = 50 * COIN; 1766 int64 nSubsidy = 50 * COIN;
609 nSubsidy >>= (blockHeight / 210000); // halving schedule 1767 nSubsidy >>= (blockHeight / 210000); // halving schedule
610 int64 minerReward = (int64)((double)nSubsidy * (1.0 - feeFrac)); 1768 int64 minerReward = (int64)((double)nSubsidy * (1.0 - feeFrac));
1769 int64 operatorCut = nSubsidy - minerReward;
1770 round.subsidy = nSubsidy;
1771 round.minerReward = minerReward;
1772 round.operatorReward = operatorCut;
611 1773
612 PendingPayout pp; 1774 PendingPayout pp;
613 pp.blockHeight = blockHeight; 1775 pp.blockHeight = blockHeight;
@@ -621,7 +1783,9 @@ static void QueuePayouts(int blockHeight,
621 // alone on a private fork paying itself, so it says nothing about a pool 1783 // alone on a private fork paying itself, so it says nothing about a pool
622 // competing on the real chain. Waiting the extra 20 blocks makes each 1784 // competing on the real chain. Waiting the extra 20 blocks makes each
623 // payout funded by construction, and puts the maturity gate past the 1785 // payout funded by construction, and puts the maturity gate past the
624 // reorg depth that orphaned 280 blocks of obligations. 1786 // reorg depth that orphaned 280 blocks of obligations. Upstream uses
1787 // COINBASE_MATURITY here; this is a considered disagreement, not an
1788 // oversight -- see the merge commit message.
625 pp.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH; 1789 pp.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH;
626 1790
627 int skipped = 0; 1791 int skipped = 0;
@@ -643,12 +1807,22 @@ static void QueuePayouts(int blockHeight,
643 continue; 1807 continue;
644 } 1808 }
645 pp.amounts[kv.first] = amount; 1809 pp.amounts[kv.first] = amount;
1810 round.payouts[kv.first] = amount;
646 } 1811 }
1812 round.skippedDust = skipped;
1813 round.forfeitedBadAddress = burned;
647 1814
648 gPendingPayouts.push_back(pp); 1815 {
649 SavePendingPayouts(); 1816 VersionedBlob snap;
1817 {
1818 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1819 gPendingPayouts.push_back(pp);
1820 snap = SerializePendingPayoutsLocked();
1821 }
1822 PublishPendingPayouts(snap);
1823 }
1824 RecordPoolRound(round);
650 1825
651 int64 operatorCut = nSubsidy - minerReward;
652 LogPrint("payout", 1826 LogPrint("payout",
653 "[payout] block %d: subsidy %s BTF, miners get %s" 1827 "[payout] block %d: subsidy %s BTF, miners get %s"
654 " (fee %.2f%% = %s to operator)," 1828 " (fee %.2f%% = %s to operator),"
@@ -664,101 +1838,337 @@ static void QueuePayouts(int blockHeight,
664 : ""); 1838 : "");
665} 1839}
666 1840
667// Called from event loop: move matured entries into the in-flight list (fast). 1841// Called from the event loop: explode matured ledger entries into per-recipient
1842// in-flight records (fast; no wallet or disk work beyond one small write).
1843//
1844// The ledger entry SURVIVES here. That is upstream's model and it is the
1845// crash-safe one: this fork used to erase the entry as soon as its payments
1846// were handed off, so a death in that window left the debt described by
1847// nothing at all -- a container restart on 2026-07-30 silently dropped
1848// ~1,156 BTF exactly that way. An entry is removed only once every recipient
1849// in it has a txid recorded (PayoutFullyPaid), which the payout thread does.
668// 1850//
669// Ordering matters and is deliberate: the in-flight records are written and 1851// gPaying is kept on top of that as the in-flight working set: it carries the
670// fsync'd to their own file *before* the ledger entry is erased, so the debt 1852// per-recipient retry/deferral state that a per-round `nextAttempt` cannot
671// is always described by at least one of the two files. Dying in the window 1853// express, and it is what the payout thread iterates. It is a cache of work
672// where both describe it is handled at startup by ReconcilePayouts, which 1854// derived from the ledger, never an independent record of debt -- if it were
673// drops the ledger entry it can see is already in flight. 1855// lost entirely the ledger would rebuild it on the next flush.
674static void FlushMaturePayouts() 1856static void FlushMaturePayouts()
675{ 1857{
676 // Void obligations whose funding block lost a reorg. The coin that was 1858 // Which funding blocks are still on the main chain? Asked with no payout
677 // meant to pay them does not exist, so paying anyway spends real-chain 1859 // lock held: PayoutBlockStillOnChain takes cs_main, and cs_main is held by
678 // coin the pool owes to somebody else. 1860 // ProcessBlock and SendMoney for long stretches. Holding the ledger mutex
679 bool voided = false; 1861 // across it would let the event loop park the payout thread behind a chain
680 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { 1862 // lock it has no interest in. (It is not a deadlock either way -- nothing
681 if (PayoutBlockStillOnChain(it->blockHash)) { ++it; continue; } 1863 // takes the ledger mutex while holding cs_main -- but it is exactly the
682 int64 total = 0; 1864 // kind of stall this merge is removing from the ledger path.)
683 for (auto& kv : it->amounts) total += kv.second; 1865 std::vector<uint256> orphaned;
684 LogPrint("payout", "[payout] block %d (%s) was orphaned -- voiding %zu" 1866 {
685 " payment(s) totalling %s\n", 1867 std::vector<uint256> hashes;
686 it->blockHeight, it->blockHash.ToString().substr(0, 14).c_str(),
687 it->amounts.size(), FormatMoney(total).c_str());
688 it = gPendingPayouts.erase(it);
689 voided = true;
690 }
691 if (voided) SavePendingPayouts();
692
693 std::vector<PendingPayout> ready;
694 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) {
695 if (nBestHeight < it->matureAtHeight) { ++it; continue; }
696 LogPrint("payout", "[payout] block height %d matured -- queuing %zu payments\n",
697 it->matureAtHeight, it->amounts.size());
698 ready.push_back(*it); // copy first; erase after save
699 ++it;
700 }
701 if (!ready.empty()) {
702 // Record the individual payments first, and only then drop the ledger
703 // entries that produced them.
704 { 1868 {
705 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 1869 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
706 for (const auto& pp : ready) { 1870 for (const PendingPayout& pp : gPendingPayouts)
707 for (const auto& kv : pp.amounts) { 1871 hashes.push_back(pp.blockHash);
708 PayingRecord r;
709 r.blockHeight = pp.blockHeight;
710 r.blockHash = pp.blockHash;
711 r.matureAtHeight = pp.matureAtHeight;
712 r.address = kv.first;
713 r.amount = kv.second;
714 gPaying.push_back(r);
715 }
716 }
717 SavePayingLocked();
718 } 1872 }
1873 for (const uint256& h : hashes)
1874 if (!PayoutBlockStillOnChain(h))
1875 orphaned.push_back(h);
1876 }
1877
1878 std::vector<PendingPayout> ready;
1879 VersionedBlob ledgerSnap;
1880 {
1881 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
1882
1883 // Void obligations whose funding block lost a reorg. The coin that was
1884 // meant to pay them does not exist, so paying anyway spends real-chain
1885 // coin the pool owes to somebody else.
1886 bool voided = false;
719 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { 1887 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) {
720 if (nBestHeight >= it->matureAtHeight) 1888 if (std::find(orphaned.begin(), orphaned.end(), it->blockHash) == orphaned.end()) {
721 it = gPendingPayouts.erase(it);
722 else
723 ++it; 1889 ++it;
1890 continue;
1891 }
1892 int64 total = 0;
1893 for (auto& kv : it->amounts) total += kv.second;
1894 // Always: one line per voided entry, never a collapsed summary.
1895 // A reorg voids these in a burst from this one source line.
1896 LogPrintAlways("payout", "[payout] block %d (%s) was orphaned -- voiding %zu"
1897 " payment(s) totalling %s\n",
1898 it->blockHeight, it->blockHash.ToString().substr(0, 14).c_str(),
1899 it->amounts.size(), FormatMoney(total).c_str());
1900 it = gPendingPayouts.erase(it);
1901 voided = true;
1902 }
1903 if (voided) ledgerSnap = SerializePendingPayoutsLocked();
1904
1905 for (const PendingPayout& pp : gPendingPayouts) {
1906 if (nBestHeight < pp.matureAtHeight)
1907 continue;
1908 if (PayoutFullyPaid(pp))
1909 continue;
1910 // An entry with neither a hash nor a height cannot be told apart
1911 // from another in the same state, so a txid recorded against it
1912 // could settle the wrong debt. Paying it is the unsafe move;
1913 // leaving it standing and saying so is not.
1914 if (!PayoutRoundIdentifiable(pp.blockHeight, pp.blockHash)) {
1915 LogPrint("payout", "[payout] WARNING: ledger entry maturing at"
1916 " height %d names neither a block hash nor a height --"
1917 " refusing to pay %zu recipient(s) that cannot be told"
1918 " apart from another round's\n",
1919 pp.matureAtHeight, pp.amounts.size());
1920 continue;
1921 }
1922 ready.push_back(pp);
724 } 1923 }
725 SavePendingPayouts();
726 } 1924 }
727} 1925 // Outside the ledger mutex: publishing is I/O and the event loop is the
1926 // one thread that must never wait on the payout thread's disk.
1927 PublishPendingPayouts(ledgerSnap);
728 1928
729// Identity of a payment, for finding its record again after SendMoney. 1929 if (ready.empty())
730// amounts is a map keyed by address, so (block, address) is already unique. 1930 return;
731static bool SamePayment(const PayingRecord& a, const PayingRecord& b) 1931
732{ 1932 VersionedBlob payingSnap;
733 return a.matureAtHeight == b.matureAtHeight 1933 {
734 && a.blockHash == b.blockHash 1934 std::lock_guard<std::mutex> queue(gPayoutExecMutex);
735 && a.address == b.address 1935 size_t added = 0;
736 && a.amount == b.amount; 1936 for (const PendingPayout& pp : ready) {
1937 const std::string id = PayoutIdForBlock(pp.blockHeight, pp.blockHash);
1938 if (gQueuedPayouts.count(id))
1939 continue; // already exploded into gPaying and still in flight
1940 size_t n = 0;
1941 for (const auto& kv : pp.amounts) {
1942 // Upstream's per-recipient dedupe: a recipient already carrying
1943 // a txid is done, even though the entry as a whole is not.
1944 if (pp.paidTxids.count(kv.first))
1945 continue;
1946 // The invariant, checked against the thing it is an invariant
1947 // of: at most one in-flight record per (round, address). The
1948 // round-level set above is an optimisation that can be wrong --
1949 // it is memory only, it is rebuilt from gPaying at startup, and
1950 // a stale `ready` snapshot can race a round finishing --
1951 // whereas gPaying is the list being protected. A second record
1952 // here is a second standing order to send the same money.
1953 if (PaymentInFlightLocked(pp.blockHeight, pp.blockHash, kv.first))
1954 continue;
1955 PayingRecord r;
1956 r.id = ++gPayingNextId;
1957 r.blockHeight = pp.blockHeight;
1958 r.blockHash = pp.blockHash;
1959 r.matureAtHeight = pp.matureAtHeight;
1960 r.address = kv.first;
1961 r.amount = kv.second;
1962 gPaying.push_back(r);
1963 n++;
1964 }
1965 gQueuedPayouts.insert(id);
1966 added += n;
1967 LogPrint("payout", "[payout] block height %d matured -- queuing"
1968 " payout %s (%zu of %zu recipients still owed)\n",
1969 pp.matureAtHeight, id.c_str(), n, pp.amounts.size());
1970 }
1971 if (added) payingSnap = SerializePayingLocked();
1972 }
1973 PublishPaying(payingSnap);
737} 1974}
738 1975
739// Settle a payment: it is either done or abandoned, either way it stops being 1976// Settle a payment: it is either done or abandoned, either way it stops being
740// owed. Removing the record and rewriting the file is what makes the payment 1977// in flight. Note this only retires the working record -- whether the miner is
741// not happen twice after a restart. 1978// still OWED the money is the ledger's business, and the ledger is updated
1979// separately by RecordPayoutTxid().
1980//
1981// Erases by record id, not by comparing fields. Value-matching was only an
1982// identity for as long as no two records could be equal, and the restart bug
1983// above produced records that were equal in every field: the first match then
1984// absorbed every erase and every state update, and the second was never
1985// touched again. The id names the slot the caller took its copy from, so the
1986// update lands where it belongs whatever else is in the list.
742static void ForgetPayment(const PayingRecord& r) 1987static void ForgetPayment(const PayingRecord& r)
743{ 1988{
744 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 1989 VersionedBlob snap;
745 for (auto it = gPaying.begin(); it != gPaying.end(); ++it) 1990 {
746 if (SamePayment(*it, r)) { gPaying.erase(it); break; } 1991 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
747 SavePayingLocked(); 1992 for (auto it = gPaying.begin(); it != gPaying.end(); ++it)
1993 if (it->id == r.id) { gPaying.erase(it); break; }
1994
1995 // When the last record for a round is gone the round is no longer in
1996 // flight, so a later flush may explode it again if the ledger still
1997 // shows recipients owed. That is the intended behaviour and it is now
1998 // the only way a stalled payment ever leaves this list: a payment that
1999 // keeps failing is retried forever rather than given up on, so its
2000 // record stays here, and the flush that would re-explode it is stopped
2001 // by PaymentInFlightLocked.
2002 bool stillInFlight = false;
2003 for (const auto& e : gPaying)
2004 if (SamePayoutRound(e.blockHeight, e.blockHash,
2005 r.blockHeight, r.blockHash)) {
2006 stillInFlight = true;
2007 break;
2008 }
2009 if (!stillInFlight)
2010 gQueuedPayouts.erase(PayoutIdForBlock(r.blockHeight, r.blockHash));
2011
2012 snap = SerializePayingLocked();
2013 }
2014 PublishPaying(snap);
748} 2015}
749 2016
750// Persist an updated attempt count / backoff for a payment still owed. 2017// Persist an updated attempt count / backoff for a payment still owed.
751static void ReschedulePayment(const PayingRecord& r) 2018static void ReschedulePayment(const PayingRecord& r)
752{ 2019{
753 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 2020 VersionedBlob snap;
754 for (auto& e : gPaying) 2021 {
755 if (SamePayment(e, r)) { 2022 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
756 e.retries = r.retries; 2023 for (auto& e : gPaying)
757 e.deferrals = r.deferrals; 2024 if (e.id == r.id) {
758 e.retryAfter = r.retryAfter; 2025 e.retries = r.retries;
2026 e.deferrals = r.deferrals;
2027 e.retryAfter = r.retryAfter;
2028 // Carry the cleared probe flag back, or every retry re-scans
2029 // the whole wallet -- the cost this merge is specifically
2030 // removing.
2031 e.fProbeWallet = r.fProbeWallet;
2032 break;
2033 }
2034 snap = SerializePayingLocked();
2035 }
2036 PublishPaying(snap);
2037}
2038
2039// Record that a recipient has been paid, in the ledger, which is the only
2040// durable statement that the debt is settled.
2041//
2042// The file write happens with gPayoutLedgerMutex RELEASED. Upstream writes the
2043// file inside the per-recipient loop with the mutex held, so every recipient
2044// costs the event loop a full serialise+fsync it has to wait for in
2045// FlushMaturePayouts. Ordering against the event loop's own writes is the
2046// snapshot version's job, not the mutex's -- see PublishVersioned.
2047static void RecordPayoutTxid(const PayingRecord& r, const std::string& txid)
2048{
2049 VersionedBlob blob;
2050 bool complete = false;
2051 {
2052 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
2053 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ++it) {
2054 if (!SamePayoutRound(it->blockHeight, it->blockHash,
2055 r.blockHeight, r.blockHash))
2056 continue;
2057 it->paidTxids[r.address] = txid;
2058 if (PayoutFullyPaid(*it)) {
2059 LogPrint("payout", "[payout] payout for block %d complete --"
2060 " removing from ledger\n", it->blockHeight);
2061 gPendingPayouts.erase(it);
2062 complete = true;
2063 }
759 break; 2064 break;
760 } 2065 }
761 SavePayingLocked(); 2066 blob = SerializePendingPayoutsLocked();
2067 }
2068 PublishPendingPayouts(blob);
2069 // The round proof records a txid per address, so this is per recipient and
2070 // not only on completion.
2071 MarkPoolRoundPaid(PayoutIdForBlock(r.blockHeight, r.blockHash), r.address, txid);
2072 if (complete)
2073 LogPrint("payout", "[payout] block %d fully settled\n", r.blockHeight);
2074}
2075
2076// Write a recipient off.
2077//
2078// Only ever called for a state that is DECIDABLE -- currently one: the ledger
2079// names an address that is not a valid address, so no transaction paying it
2080// can be constructed by this or any other build. It is not called, and must
2081// not be, because a payment has failed some number of times. "The wallet said
2082// no N times" is not a decidable state, it is a description of how long we
2083// have been asking; a debt written off on the strength of it is a debt the
2084// operator keeps, and the arithmetic that decided WHY the wallet said no was
2085// wrong for a year (see PayoutThreadFn).
2086//
2087// The amount is removed from the entry rather than marked paid, so the money
2088// is not reported as sent. Once nothing is left owed the entry retires. The
2089// round proof is told separately, because dropping the obligation from the
2090// ledger while pool_rounds.json goes on publishing the address as owed with
2091// "txid": null leaves the pool's two records of the same round permanently
2092// disagreeing.
2093static void ForfeitPayoutRecipient(const PayingRecord& r, const std::string& reason)
2094{
2095 LogPrintAlways("payout", "[payout] writing off %s owed to %s for block %d: %s\n",
2096 FormatMoney(r.amount).c_str(), r.address.c_str(), r.blockHeight,
2097 reason.c_str());
2098 VersionedBlob blob;
2099 {
2100 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
2101 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ++it) {
2102 if (!SamePayoutRound(it->blockHeight, it->blockHash,
2103 r.blockHeight, r.blockHash))
2104 continue;
2105 it->amounts.erase(r.address);
2106 // PayoutFullyPaid() returns true for an empty amounts map -- there
2107 // is no recipient in it without a txid -- so the
2108 // `amounts.empty()` disjunct that used to be here decided nothing.
2109 if (PayoutFullyPaid(*it))
2110 gPendingPayouts.erase(it);
2111 break;
2112 }
2113 blob = SerializePendingPayoutsLocked();
2114 }
2115 PublishPendingPayouts(blob);
2116 MarkPoolRoundForfeited(PayoutIdForBlock(r.blockHeight, r.blockHash),
2117 r.address, reason);
2118}
2119
2120// What a failed send means for the payment's retry budget.
2121//
2122// A deferral is "the wallet's condition is temporarily wrong": wait, and do
2123// not hold it against this payment. A fault is "the wallet is funded and
2124// unlocked and still would not do it", which is worth logging loudly and
2125// retrying with backoff.
2126//
2127// This used to be decided by re-reading GetBalance() after the failure and
2128// comparing it against `amount + nTransactionFee + CENT`. Every term of that
2129// was wrong:
2130//
2131// * nTransactionFee is 0 in production. It is only ever assigned from the
2132// GUI preferences dialog (gui.cpp), which a headless pool node does not
2133// have, so the whole reserve was the CENT.
2134//
2135// * The fee the reserve is for is GetMinFee(fDiscount=true), a STEP
2136// function of the serialised size: 0 below 10,000 bytes, and
2137// (1 + bytes/1000) * CENT at or above it -- so 11 * CENT at the step.
2138// There is no required-fee value between 1 and 11 CENT. A one-CENT
2139// reserve is therefore never the right number: below the step nothing
2140// needs reserving, and at the step it is short by a factor of eleven.
2141//
2142// * A pool wallet close to broke selects many small inputs BECAUSE it is
2143// close to broke, and ~56 signed P2PKH inputs is 10 KB. The trigger and
2144// the failure are positively correlated: the guess is wrong precisely
2145// when the pool is nearly out of money.
2146//
2147// Concretely, with a balance of amount + 0.05: the pre-send gate passes,
2148// CreateTransaction builds a >10 KB transaction, the fee becomes 0.11,
2149// SelectCoins(amount + 0.11) fails, SendMoney returns false -- and the old
2150// arithmetic said (amount + 0.05) < (amount + 0.01) is false, so "funded",
2151// so a fault. The condition is stable, so it recurred every 60 s, and 1440 of
2152// them erased the debt with the coin still in the operator's wallet.
2153//
2154// So stop guessing. The wallet is asked to say what it did, and this only
2155// classifies the answer. WALLET_SEND_INSUFFICIENT_FUNDS is coin selection
2156// failing to cover amount plus whatever fee that size of transaction
2157// required, whatever that turned out to be -- no reserve to get wrong.
2158enum PayoutFailureClass {
2159 PAYOUT_FAILURE_DEFER = 0, // conditions are wrong; wait for them to change
2160 PAYOUT_FAILURE_FAULT // funded and unlocked, and it still would not go
2161};
2162
2163static PayoutFailureClass ClassifyPayoutSendFailure(WalletSendResult result,
2164 bool fWalletLocked)
2165{
2166 // A locked wallet cannot sign. Nothing about this payment is wrong.
2167 if (fWalletLocked)
2168 return PAYOUT_FAILURE_DEFER;
2169 if (result == WALLET_SEND_INSUFFICIENT_FUNDS)
2170 return PAYOUT_FAILURE_DEFER;
2171 return PAYOUT_FAILURE_FAULT;
762} 2172}
763 2173
764// Payout execution thread: works through gPaying, calls SendMoney. 2174// Payout execution thread: works through gPaying, calls SendMoney.
@@ -780,11 +2190,14 @@ static void PayoutThreadFn(void*)
780 } 2190 }
781 2191
782 for (PayingRecord r : batch) { 2192 for (PayingRecord r : batch) {
2193 const std::string id = PayoutIdForBlock(r.blockHeight, r.blockHash);
2194
783 // Re-check here and not only at maturity: a payment can sit in this 2195 // Re-check here and not only at maturity: a payment can sit in this
784 // list for hours across retries, and the reorg that kills its 2196 // list for hours across retries, and the reorg that kills its
785 // funding block can land at any point in that window. 2197 // funding block can land at any point in that window. Upstream has
2198 // no equivalent -- it decides once, when the entry matures.
786 if (!PayoutBlockStillOnChain(r.blockHash)) { 2199 if (!PayoutBlockStillOnChain(r.blockHash)) {
787 LogPrint("payout", "[payout] block %d (%s) orphaned while queued" 2200 LogPrintAlways("payout", "[payout] block %d (%s) orphaned while queued"
788 " -- dropping %s to %s\n", 2201 " -- dropping %s to %s\n",
789 r.blockHeight, r.blockHash.ToString().substr(0, 14).c_str(), 2202 r.blockHeight, r.blockHash.ToString().substr(0, 14).c_str(),
790 FormatMoney(r.amount).c_str(), r.address.c_str()); 2203 FormatMoney(r.amount).c_str(), r.address.c_str());
@@ -792,72 +2205,244 @@ static void PayoutThreadFn(void*)
792 continue; 2205 continue;
793 } 2206 }
794 2207
2208 // Upstream's per-recipient dedupe, against the ledger rather than
2209 // against the batch: if a txid is already recorded for this
2210 // address, the money went out and nothing more is owed.
2211 {
2212 bool alreadyPaid = false;
2213 {
2214 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
2215 for (const PendingPayout& pp : gPendingPayouts)
2216 if (SamePayoutRound(pp.blockHeight, pp.blockHash,
2217 r.blockHeight, r.blockHash)
2218 && pp.paidTxids.count(r.address)) {
2219 alreadyPaid = true;
2220 break;
2221 }
2222 }
2223 if (alreadyPaid) {
2224 ForgetPayment(r);
2225 continue;
2226 }
2227 }
2228
795 uint160 h160; 2229 uint160 h160;
796 if (!AddressToHash160(r.address, h160)) { 2230 if (!AddressToHash160(r.address, h160)) {
797 // Should not happen -- bad addresses are filtered in QueuePayouts 2231 // Should not happen -- bad addresses are filtered in
798 LogPrint("payout", "[payout] SKIPPED bad address '%s'\n", 2232 // QueuePayouts. If one is here anyway it is decidable and
799 r.address.c_str()); 2233 // permanent: there is no transaction that pays a string which
2234 // is not an address, so retrying is not waiting for anything.
2235 //
2236 // This used to only ForgetPayment(), which drops the in-flight
2237 // record and leaves the ledger entry claiming the debt and
2238 // pool_rounds.json advertising the recipient as unpaid
2239 // forever, with the next flush re-exploding it. Write it off
2240 // properly and say why.
2241 const std::string reason =
2242 strprintf("'%s' is not a valid address, so no transaction"
2243 " can pay it", r.address.c_str());
2244 LogPrintAlways("payout", "[payout] SKIPPED bad address '%s' --"
2245 " %s owed for block %d cannot be paid\n",
2246 r.address.c_str(), FormatMoney(r.amount).c_str(),
2247 r.blockHeight);
2248 ForfeitPayoutRecipient(r, reason);
800 ForgetPayment(r); 2249 ForgetPayment(r);
801 continue; 2250 continue;
802 } 2251 }
803 CScript sc; 2252 CScript sc;
804 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; 2253 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
805 2254
2255 // Did a previous process send this and die before recording it?
2256 // Only asked once per record -- see PayingRecord::fProbeWallet.
2257 if (r.fProbeWallet) {
2258 std::string recovered;
2259 const std::string claimKey = id + "|" + r.address;
2260 std::set<std::string> excluded;
2261 CollectForeignPayoutTxids(r.blockHeight, r.blockHash, r.address,
2262 claimKey, excluded);
2263 // A round with a published proof was created by a build whose
2264 // every payment names its round exactly, so only structured
2265 // evidence can settle it -- a comment match would be another
2266 // round's payment colliding on the height.
2267 const bool fAllowLegacy = !PoolRoundHasProof(id);
2268 r.fProbeWallet = false;
2269 PayoutProbeResult probe = FindRecordedPoolPayout(
2270 id, r.blockHeight, r.blockHash, r.address, r.amount,
2271 fAllowLegacy, excluded, recovered);
2272 if (probe != PAYOUT_PROBE_NONE) {
2273 if (probe == PAYOUT_PROBE_LEGACY) {
2274 // Loud on purpose: this is the upgrade path, where the
2275 // only evidence a payment went out is a comment written
2276 // by a binary that is no longer running.
2277 LogPrint("payout", "[payout] %s to %s was already paid by"
2278 " a pre-upgrade build (wallet tx %s carries the"
2279 " payout comment for height %d and an output of"
2280 " exactly this amount, and no structured payout"
2281 " metadata) -- not resending\n",
2282 FormatMoney(r.amount).c_str(), r.address.c_str(),
2283 recovered.c_str(), r.blockHeight);
2284 ClaimLegacyPayoutTxid(recovered, claimKey);
2285 } else {
2286 LogPrint("payout", "[payout] recovered recorded payout %s to %s"
2287 " from wallet tx %s -- not resending\n",
2288 FormatMoney(r.amount).c_str(), r.address.c_str(),
2289 recovered.c_str());
2290 }
2291 RecordPayoutTxid(r, recovered);
2292 ForgetPayment(r);
2293 continue;
2294 }
2295 }
2296
806 // Waiting for coin to mature is not a failure, so it must not 2297 // Waiting for coin to mature is not a failure, so it must not
807 // consume the retry budget. Burning a retry every 60s means a 2298 // consume the retry budget. Burning a retry every 60s means a
808 // payment the pool fully intends to honour can hit GIVING UP 2299 // payment the pool fully intends to honour can hit GIVING UP
809 // 24h later purely because the subsidy funding it had not 2300 // 24h later purely because the subsidy funding it had not
810 // matured yet -- the miner is then silently never paid. 2301 // matured yet -- the miner is then silently never paid.
2302 //
2303 // The comparison includes the fee. It used to be against the
2304 // amount alone, so a balance anywhere between amount and
2305 // amount+fee sailed past this gate, failed in SendMoney, and was
2306 // counted as a send FAILURE rather than a wait -- 1440 of those is
2307 // 24 hours, and the recipient was erased with the money left in
2308 // the operator's wallet and nothing on disk saying it was owed.
2309 //
2310 // nTransactionFee is the part of the fee that can be known before
2311 // the transaction exists, and in production it is 0 -- it is only
2312 // ever assigned from the GUI preferences dialog, which a pool node
2313 // does not have -- so including it here changes nothing on this
2314 // pool and is kept only because a GUI operator can set it. The
2315 // rest is GetMinFee(), a function of the serialised size and
2316 // therefore of how many inputs SelectCoins ends up picking, which
2317 // cannot be predicted from here -- so this gate deliberately does
2318 // NOT try to reserve it. Refusing to attempt a payment the wallet
2319 // could actually have made would be its own way of not paying a
2320 // miner. That residual is handled after the fact, by asking the
2321 // wallet why it refused -- see ClassifyPayoutSendFailure.
811 int64 nSpendable = GetBalance(); 2322 int64 nSpendable = GetBalance();
812 if (nSpendable < r.amount) { 2323 if (nSpendable < r.amount + nTransactionFee) {
813 r.deferrals++; 2324 r.deferrals++;
814 r.retryAfter = GetTime() + 60; 2325 r.retryAfter = GetTime() + 60;
815 // Hourly, not every minute: a dry pool would otherwise 2326 // Hourly, not every minute: a dry pool would otherwise
816 // write one line per payment per minute forever. 2327 // write one line per payment per minute forever.
817 if (r.deferrals == 1 || r.deferrals % 60 == 0) 2328 if (r.deferrals == 1 || r.deferrals % 60 == 0)
818 LogPrint("payout", "[payout] deferring %s to %s -- only %s" 2329 LogPrint("payout", "[payout] deferring %s to %s -- only %s"
819 " spendable (waiting %d min so far)\n", 2330 " spendable, need %s with fee (waiting %d min so far)\n",
820 FormatMoney(r.amount).c_str(), r.address.c_str(), 2331 FormatMoney(r.amount).c_str(), r.address.c_str(),
821 FormatMoney(nSpendable).c_str(), r.deferrals); 2332 FormatMoney(nSpendable).c_str(),
2333 FormatMoney(r.amount + nTransactionFee).c_str(),
2334 r.deferrals);
822 ReschedulePayment(r); 2335 ReschedulePayment(r);
823 continue; 2336 continue;
824 } 2337 }
825 2338
826 CWalletTx wtx; 2339 // Likewise a locked wallet: it cannot sign, so every attempt fails
827 wtx.mapValue["comment"] = PayoutComment(r.blockHeight); 2340 // for a reason that has nothing to do with this payment being
2341 // payable. Left to the retry budget, an operator who locks the
2342 // wallet overnight comes back to erased debts.
2343 if (IsWalletLocked()) {
2344 r.deferrals++;
2345 r.retryAfter = GetTime() + 60;
2346 if (r.deferrals == 1 || r.deferrals % 60 == 0)
2347 LogPrint("payout", "[payout] deferring %s to %s -- wallet is"
2348 " locked (waiting %d min so far)\n",
2349 FormatMoney(r.amount).c_str(), r.address.c_str(),
2350 r.deferrals);
2351 ReschedulePayment(r);
2352 continue;
2353 }
828 2354
829 if (SendMoney(sc, r.amount, wtx)) { 2355 CWalletTx wtx;
2356 wtx.mapValue["comment"] = PayoutComment(r.blockHeight, r.blockHash);
2357 // Upstream's structured metadata. This is what makes the wallet
2358 // probe above possible at all -- the old comment-only form could
2359 // not distinguish two rounds paying the same address the same
2360 // amount. Keyed by the hash-derived id, so a reorg cannot make one
2361 // round's transaction answer for another's.
2362 wtx.mapValue["pool_payout_id"] = id;
2363 wtx.mapValue["pool_payout_address"] = r.address;
2364 wtx.mapValue["pool_payout_amount"] = strprintf("%lld", r.amount);
2365 // The full hash, not only the id's sixteen-digit prefix: the
2366 // prefix of a proof-of-work hash is where the zeros live, and
2367 // the probe needs to tell twin rounds at one height apart.
2368 if (r.blockHash != 0)
2369 wtx.mapValue["pool_payout_block"] = r.blockHash.ToString();
2370
2371 SendMoneyOutcome outcome;
2372 if (SendMoney(sc, r.amount, wtx, outcome)) {
830 // The wallet holds the transaction before this returns, so a 2373 // The wallet holds the transaction before this returns, so a
831 // crash here is recoverable: ReconcilePayouts finds it by 2374 // crash between here and the ledger write is recoverable: the
832 // comment+address+amount and does not resend. 2375 // probe above finds it by id+address+amount and does not
2376 // resend. Ledger first, in-flight record second -- the debt is
2377 // described by at least one of them at every instant.
2378 RecordPayoutTxid(r, wtx.GetHash().GetHex());
833 ForgetPayment(r); 2379 ForgetPayment(r);
834 LogPrint("payout", "[payout] paid %s to %s (tx %s)\n", 2380 LogPrintAlways("payout", "[payout] paid %s to %s (tx %s)\n",
835 FormatMoney(r.amount).c_str(), r.address.c_str(), 2381 FormatMoney(r.amount).c_str(), r.address.c_str(),
836 wtx.GetHash().ToString().substr(0, 12).c_str()); 2382 wtx.GetHash().ToString().substr(0, 12).c_str());
837 } else if (r.retries < 1440) {
838 // Transient failures (reorg, locked wallet) must not silently
839 // burn the payment -- retry with backoff. Insufficient funds
840 // never lands here; it defers above.
841 r.retries++;
842 r.retryAfter = GetTime() + 60;
843 ReschedulePayment(r);
844 LogPrint("payout", "[payout] FAILED to pay %s to %s"
845 " (%s spendable at the time) -- retry %d/1440 in 60s\n",
846 FormatMoney(r.amount).c_str(), r.address.c_str(),
847 FormatMoney(nSpendable).c_str(), r.retries);
848 } else { 2383 } else {
849 // Reaching here means 1440 attempts that were NOT 2384 // Why did it fail? The answer used to decide whether the
850 // insufficient-funds waits (those defer instead) against a 2385 // payment eventually got erased, and it was decided by
851 // block still on the main chain -- i.e. a real wallet fault, 2386 // guessing from a balance snapshot -- see
852 // not a maturity or reorg question. 2387 // ClassifyPayoutSendFailure for the arithmetic and what it
853 LogPrint("payout", "[payout] GIVING UP on %s to %s after %d" 2388 // cost. The wallet now says what it did, and nothing here
854 " retries -- block %d is still on-chain and %s was" 2389 // erases anything.
855 " spendable, so this is a wallet fault, not maturity" 2390 //
856 " -- payment forfeited\n", 2391 // The lock state is re-read rather than trusted from before
857 FormatMoney(r.amount).c_str(), r.address.c_str(), 2392 // the attempt: SendMoney takes cs_main and can block for a
858 r.retries, r.blockHeight, 2393 // long time, so it may have moved underneath us. The funding
859 FormatMoney(nSpendable).c_str()); 2394 // answer does not need re-reading -- it is a statement about
860 ForgetPayment(r); 2395 // the attempt that was actually made.
2396 bool fLocked = IsWalletLocked();
2397 int64 nAfter = GetBalance();
2398
2399 if (ClassifyPayoutSendFailure(outcome.result, fLocked)
2400 == PAYOUT_FAILURE_DEFER) {
2401 // Conditions are temporarily wrong. Wait.
2402 r.deferrals++;
2403 r.retryAfter = GetTime() + 60;
2404 if (r.deferrals == 1 || r.deferrals % 60 == 0)
2405 LogPrint("payout", "[payout] deferring %s to %s -- send"
2406 " failed with %s (%s spendable; the wallet"
2407 " needed a fee of %s for a transaction this"
2408 " size); waiting %d min so far\n",
2409 FormatMoney(r.amount).c_str(), r.address.c_str(),
2410 fLocked ? "the wallet locked" : "too little coin",
2411 FormatMoney(nAfter).c_str(),
2412 FormatMoney(outcome.nFeeRequired).c_str(),
2413 r.deferrals);
2414 ReschedulePayment(r);
2415 } else {
2416 // Funded, unlocked, block on the main chain, and the
2417 // wallet still would not build, sign or accept the
2418 // transaction. That is worth shouting about; it is not
2419 // grounds to keep the money.
2420 //
2421 // There is deliberately no give-up branch. It used to
2422 // forfeit the debt after 1440 of these, which is not a
2423 // decidable state -- it is a count of how long we have
2424 // been asking, and it was reached by misclassifying a
2425 // funding shortfall as a fault. A debt that is never
2426 // written off cannot be written off wrongly. The payment
2427 // stays in flight, keeps its ledger entry, keeps being
2428 // reported in pool_status.json as owed, and keeps
2429 // retrying once a minute until the operator or the wallet
2430 // makes it possible.
2431 r.retries++;
2432 r.retryAfter = GetTime() + 60;
2433 ReschedulePayment(r);
2434 // Every minute at first, then hourly: a fault that is not
2435 // going away must not turn the log into one line a minute
2436 // forever, and must not go quiet either.
2437 if (r.retries <= 5 || r.retries % 60 == 0)
2438 LogPrintAlways("payout", "[payout] FAILED to pay %s to %s"
2439 " (%s spendable, wallet unlocked, block %d on"
2440 " the main chain) -- attempt %d, retrying in"
2441 " 60s; this debt stands until it is paid\n",
2442 FormatMoney(r.amount).c_str(), r.address.c_str(),
2443 FormatMoney(nAfter).c_str(), r.blockHeight,
2444 r.retries);
2445 }
861 } 2446 }
862 } 2447 }
863 2448
@@ -881,6 +2466,7 @@ static StatsSnapshot gStats;
881static std::vector<PoolWorkerStatView> gWorkerStats; 2466static std::vector<PoolWorkerStatView> gWorkerStats;
882static std::vector<PendingPayoutView> gPayoutViews; 2467static std::vector<PendingPayoutView> gPayoutViews;
883static std::mutex gStatsMutex; 2468static std::mutex gStatsMutex;
2469static int64 gLastPoolStatusWrite = 0;
884 2470
885void GetPoolOperatorStats(int& authorizedMiners, int& blocksFound, uint64& roundShares, double& totalHashRate) 2471void GetPoolOperatorStats(int& authorizedMiners, int& blocksFound, uint64& roundShares, double& totalHashRate)
886{ 2472{
@@ -903,6 +2489,135 @@ void GetPendingPayouts(std::vector<PendingPayoutView>& out)
903 out = gPayoutViews; 2489 out = gPayoutViews;
904} 2490}
905 2491
2492static std::string PoolStatusFilePath()
2493{
2494 if (!strPoolStatusFile.empty())
2495 return strPoolStatusFile;
2496 return GetAppDir() + "/pool_status.json";
2497}
2498
2499// Serialise the operator status file. Caller must hold gStatsMutex (this reads
2500// gStats / gWorkerStats / gPayoutViews); gPoolRoundsMutex is taken and
2501// released inside.
2502//
2503// Split from the write for the same reason as the three ledgers, and for a
2504// worse instance of the same problem. This used to be called from RefreshStats
2505// with gPayoutLedgerMutex held at function scope, and once the ledger writes
2506// gained fsync() that call became two fsyncs -- one on the file, one on its
2507// directory -- performed by the event loop, the thread that owns every miner's
2508// socket, with the mutex the payout thread needs in its hand. Measured on the
2509// build host (ext4 on LVM on SSD): 3.78 ms per publish, every 10 seconds, and
2510// unbounded on a shared virtual disk under load.
2511//
2512// Nothing here needs the ledger mutex at all: the pending-payout rows were
2513// already copied into gPayoutViews under it, further up RefreshStats.
2514static std::string SerializePoolStatusLocked()
2515{
2516 nlohmann::json j;
2517 j["schema"] = "bitflash-pool-status-1";
2518 j["network"] = "bitflash";
2519 j["updatedAt"] = GetTime();
2520
2521 j["operator"] = {
2522 {"name", strPoolName},
2523 {"btfAddress", BtfLocalAddress()},
2524 {"feePercent", dPoolFeePercent},
2525 {"dashboardUrl", strPoolDashboardUrl},
2526 {"roundsFile", "pool_rounds.json"}
2527 };
2528
2529 j["node"] = {
2530 {"height", nBestHeight},
2531 {"poolRunning", (bool)gPoolRunning},
2532 {"mode", nMineMode}
2533 };
2534
2535 j["pool"] = {
2536 {"authorizedMiners", gStats.authorizedMiners},
2537 {"blocksFoundSession", gStats.blocksFound},
2538 {"roundShares", gStats.roundShares},
2539 {"hashRate", gStats.totalHashRate}
2540 };
2541
2542 j["workers"] = nlohmann::json::array();
2543 for (const PoolWorkerStatView& row : gWorkerStats) {
2544 j["workers"].push_back({
2545 {"address", row.address},
2546 {"worker", row.worker},
2547 {"totalShares", row.totalShares},
2548 {"roundShares", row.roundShares},
2549 {"lastSeen", row.lastSeen},
2550 {"hashRate", row.hashRate}
2551 });
2552 }
2553
2554 j["pendingPayouts"] = nlohmann::json::array();
2555 for (const PendingPayoutView& row : gPayoutViews) {
2556 j["pendingPayouts"].push_back({
2557 {"matureAtHeight", row.matureAtHeight},
2558 {"recipients", row.recipients},
2559 {"totalAmountSatoshis", row.totalAmount},
2560 {"totalAmountBtf", FormatMoney(row.totalAmount)}
2561 });
2562 }
2563
2564 j["recentRounds"] = nlohmann::json::array();
2565 {
2566 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
2567 size_t start = gPoolRounds.size() > 10 ? gPoolRounds.size() - 10 : 0;
2568 for (size_t i = start; i < gPoolRounds.size(); i++) {
2569 const PoolRoundProof& r = gPoolRounds[i];
2570 // A forfeited recipient is not "unpaid and pending" -- the ledger
2571 // has already dropped the obligation. Counting it as unpaid is
2572 // what made the round proof and the ledger disagree for good.
2573 int unpaid = 0;
2574 int forfeited = 0;
2575 for (const auto& kv : r.payouts) {
2576 if (r.paidTxids.count(kv.first))
2577 continue;
2578 if (r.forfeited.count(kv.first))
2579 forfeited++;
2580 else
2581 unpaid++;
2582 }
2583 j["recentRounds"].push_back({
2584 {"id", r.id},
2585 {"blockHeight", r.blockHeight},
2586 {"blockHash", r.blockHash},
2587 {"coinbaseTxid", r.coinbaseTxid},
2588 {"totalShares", r.totalShares},
2589 {"payouts", (int)r.payouts.size()},
2590 {"unpaidPayouts", unpaid},
2591 {"forfeitedPayouts", forfeited},
2592 {"coinbaseValueBtf", FormatMoney(r.coinbaseValue)},
2593 {"minerRewardBtf", FormatMoney(r.minerReward)}
2594 });
2595 }
2596 }
2597
2598 return j.dump(2);
2599}
2600
2601// Publish the operator status file. Call with every payout mutex released.
2602//
2603// Still no version guard and no write mutex, deliberately, and the split does
2604// not change the argument: RefreshStats remains the only caller of either
2605// half, in production it runs only on the event loop, and it now calls this
2606// after leaving the scope that serialised -- so the serialise and the write
2607// stay strictly ordered on one thread and pool_status.json still has a single
2608// writer by construction. It shares WriteFileAtomic so the durability rules
2609// stay in one place.
2610//
2611// "In production" is doing real work in that sentence. The payout-ledger
2612// self-test also calls RefreshStats, from one thread at a time and with the
2613// pool not running, which is why it is still a single writer there too. A
2614// second concurrent caller anywhere would invalidate this paragraph and want
2615// the same versioned publish the ledgers use.
2616static void PublishPoolStatus(const std::string& blob)
2617{
2618 WriteFileAtomic(PoolStatusFilePath(), blob, "pool status");
2619}
2620
906// --------------------------------------------------------------------------- 2621// ---------------------------------------------------------------------------
907// Miner state -- event loop thread only 2622// Miner state -- event loop thread only
908// --------------------------------------------------------------------------- 2623// ---------------------------------------------------------------------------
@@ -928,9 +2643,35 @@ static const double VARDIFF_MAX = 1e12;
928static const int64 VARDIFF_STALL_SECS = 45; // 3x share target -- reacts faster without false-triggering on normal variance 2643static const int64 VARDIFF_STALL_SECS = 45; // 3x share target -- reacts faster without false-triggering on normal variance
929static const double VARDIFF_STALL_CUT = 4.0; // divide difficulty by this much 2644static const double VARDIFF_STALL_CUT = 4.0; // divide difficulty by this much
930 2645
2646// #7: m->hashRate is an EWMA sampled only when a share arrives, so a miner
2647// that stops hashing keeps its last estimate forever and RefreshStats sums
2648// it into the announced pool total for as long as the connection sits open
2649// -- the published pool hashrate ran several times over ground truth
2650// because of exactly this. MinerReportedHashRate() below applies the
2651// policy: a miner gets a grace of HASHRATE_IDLE_MISSES expected share
2652// intervals (expected interval = expected hashes per share at its
2653// difficulty, divided by the estimate itself) with no share, after which
2654// the figure we report falls off with a half-life of one more such
2655// interval. The interval is clamped to [HASHRATE_MIN_INTERVAL,
2656// HASHRATE_MAX_INTERVAL] so a fast miner at the vardiff floor (expected
2657// interval of milliseconds) is not punished for ordinary share-delivery
2658// jitter, and a very slow miner's grace does not stretch past the point
2659// where its estimate means anything. At the vardiff target of one share
2660// per 15s that is a minute of grace, half the rate reported 75s in, and
2661// nothing reported after about seventeen minutes of silence.
2662static const double HASHRATE_IDLE_MISSES = 4.0; // grace, in expected share intervals
2663static const double HASHRATE_MIN_INTERVAL = 1.0; // seconds; floor for the expected interval
2664static const double HASHRATE_MAX_INTERVAL = 3600.0; // seconds; ceiling for the expected interval
2665// Half-lives past the grace beyond which we report a flat zero. 2^-64 of the
2666// original estimate is not a number anyone needs, and stopping here keeps
2667// pow() from walking down into the denormals for a miner that has been quiet
2668// for hours.
2669static const double HASHRATE_DECAY_CUTOFF = 64.0;
2670
931struct Miner { 2671struct Miner {
932 btf_socket_t fd; 2672 btf_socket_t fd;
933 bool authorised = false; 2673 bool authorised = false;
2674 bool subscribed = false; // completed mining.subscribe (#19 census)
934 std::string address; 2675 std::string address;
935 std::string worker; 2676 std::string worker;
936 std::string readBuf; 2677 std::string readBuf;
@@ -954,11 +2695,80 @@ struct Miner {
954 int64 lastShareTime = 0; // GetTime() of the previous accepted share 2695 int64 lastShareTime = 0; // GetTime() of the previous accepted share
955 double hashRate = 0.0; // smoothed H/s estimate 2696 double hashRate = 0.0; // smoothed H/s estimate
956 2697
2698 // Replay guard for mining.submit (#8): nonces already credited on this
2699 // connection for the job it is currently working. Keyed to the job id in
2700 // seenNoncesJob and cleared by SendJob whenever that id changes, so a
2701 // resubmission under a NEW job is never turned away, while a replayed
2702 // (job_id, nonce) pair under the same job is refused before it can be
2703 // credited twice into roundShareCount -- the figure the payout split
2704 // divides by.
2705 std::set<uint32_t> seenNonces;
2706 std::string seenNoncesJob;
2707
957 explicit Miner(btf_socket_t f) 2708 explicit Miner(btf_socket_t f)
958 : fd(f), connectTime(GetTime()), 2709 : fd(f), connectTime(GetTime()),
959 extranonce2(gNextExtranonce2.fetch_add(1)) {} 2710 extranonce2(gNextExtranonce2.fetch_add(1)) {}
960}; 2711};
961 2712
2713// What we publish as a miner's H/s: the last EWMA sample, faded out if shares
2714// have stopped arriving (#7, policy at HASHRATE_IDLE_MISSES).
2715//
2716// This reads m->hashRate and never writes it. That is the whole design. The
2717// first attempt decayed the stored estimate in place from the event loop's
2718// sweep, which put a running accumulator behind a figure that is only ever
2719// displayed, and the accumulator misbehaved in two ways:
2720//
2721// - `expected` was recomputed from the already-decayed estimate on every
2722// pass, so each decay raised `expected`, which raised `grace`, which is
2723// the very thing gating the next decay. `grace` simply chased `silent`
2724// upward and the reported rate settled into a 1/t crawl instead of the
2725// half-life curve the comment above promises -- 291x too high at ten
2726// half-lives past the grace. Issue #7 is about the pool over-reporting;
2727// that under-corrected it for hours.
2728// - Once `expected` reached HASHRATE_MAX_INTERVAL it could not rise any
2729// further, the feedback that had been gating each pass was gone, and the
2730// decay began compounding once a second against the total elapsed
2731// silence -- taking a miner from 0.4% of its baseline to 1e-11 of it in
2732// the next hundred seconds, four hours in, for no reason a reader of
2733// either the code or the announced number could see.
2734//
2735// Computing the value instead of accumulating it removes the failure mode
2736// rather than tuning it: with no stored decay state there is nothing for a
2737// tick rate, a missed pass or a duplicated pass to perturb, and the answer
2738// depends only on how long the miner has actually been quiet. It also keeps
2739// the fade out of the estimator -- the EWMA on the share path still blends
2740// against the value it last computed, not against a faded copy of it, so a
2741// miner that resumes recovers on exactly the curve it would have without any
2742// of this.
2743//
2744// `now` is a parameter rather than a GetTime() call so that one refresh fades
2745// every miner to the same instant, and so the policy is directly testable.
2746static double MinerReportedHashRate(const Miner* m, int64 now)
2747{
2748 // Nothing sampled yet: a fresh connection, or one that has found a single
2749 // share and so has no interval to have been timed over. Nothing to fade,
2750 // and nothing to divide by below.
2751 if (m->hashRate <= 0.0 || m->lastShareTime <= 0) return m->hashRate;
2752
2753 double expected = ExpectedHashesForDifficulty(m->difficulty) / m->hashRate;
2754 // Written so that a NaN would land on the floor rather than sail through:
2755 // the clamp is what guarantees `expected` is finite and strictly positive
2756 // below, whatever the division produced.
2757 if (!(expected > HASHRATE_MIN_INTERVAL)) expected = HASHRATE_MIN_INTERVAL;
2758 if (expected > HASHRATE_MAX_INTERVAL) expected = HASHRATE_MAX_INTERVAL;
2759
2760 double silent = (double)(now - m->lastShareTime);
2761 double grace = HASHRATE_IDLE_MISSES * expected;
2762 // grace is at least HASHRATE_IDLE_MISSES seconds, so a clock that has
2763 // stepped backwards gives a negative `silent`, takes this branch, and
2764 // reports the sample unchanged. Nothing here can scale an estimate up.
2765 if (!(silent > grace)) return m->hashRate;
2766
2767 double halflives = (silent - grace) / expected;
2768 if (halflives >= HASHRATE_DECAY_CUTOFF) return 0.0;
2769 return m->hashRate * pow(0.5, halflives);
2770}
2771
962static std::vector<Miner*> gMiners; 2772static std::vector<Miner*> gMiners;
963 2773
964struct FoundBlock { int height; int64 ts; std::string addr; }; 2774struct FoundBlock { int height; int64 ts; std::string addr; };
@@ -1009,6 +2819,18 @@ static void LocalAcceptFn(void*)
1009 break; 2819 break;
1010 continue; 2820 continue;
1011 } 2821 }
2822 // #6: let the OS probe a peer that has gone silent, so a half-open
2823 // session eventually errors out of recv and lands in the event
2824 // loop's existing reap path instead of sitting in gMiners forever.
2825 // This is a backstop, not the mechanism: the default cadence before
2826 // the first probe is hours on most stacks, so the idle timeout in
2827 // PoolEventLoop fires long before keepalive does. Miners arriving
2828 // via a relay ride a loopback socket pair whose remote transport is
2829 // the rendezvous socket owned by the tunnel pump threads; keepalive
2830 // cannot reach across that, which is part of why the idle timeout
2831 // exists.
2832 int keepalive = 1;
2833 setsockopt(c, SOL_SOCKET, SO_KEEPALIVE, (char*)&keepalive, sizeof(keepalive));
1012 LogPrint("pool", "[pool] miner arrived via loopback -- pushing to event loop\n"); 2834 LogPrint("pool", "[pool] miner arrived via loopback -- pushing to event loop\n");
1013 PushReady((btf_socket_t)c); 2835 PushReady((btf_socket_t)c);
1014 } 2836 }
@@ -1039,6 +2861,30 @@ static bool SendLine(btf_socket_t fd, const json& j)
1039static const size_t MAX_STRATUM_LINE = 64 * 1024; 2861static const size_t MAX_STRATUM_LINE = 64 * 1024;
1040static const size_t MAX_POOL_MINERS = 1000; 2862static const size_t MAX_POOL_MINERS = 1000;
1041 2863
2864// #19 instrumentation: the "broadcast to N miner(s)" line counts connected
2865// sockets, which says nothing about how many are actually mining. These split
2866// the connection census into connected / subscribed / authorized / recently
2867// active, so an operator can tell a real workforce from a pile of half-open
2868// sessions at a glance.
2869static const int64 POOL_CENSUS_SECS = 60; // how often to log the breakdown
2870static const int64 POOL_ACTIVE_WINDOW = 300; // a miner is "active" if it
2871 // authorized and was seen within
2872 // this window (one payout-rotate
2873 // cycle)
2874static const int64 POOL_UNAUTH_GRACE_SECS = 60; // a connection still not
2875 // authorized after this long is
2876 // idle, not merely starting up
2877// #6: a Miner used to leave gMiners only when RecvLine reported a clean
2878// disconnect. A half-open session (miner host died, NAT state expired,
2879// cable pulled) sends no FIN and reports no error, so it sat in the table
2880// forever -- authorize outpaced disconnect 2:1 on the live pool, and the
2881// table grew toward MAX_POOL_MINERS, past which real miners are refused.
2882// lastSeen is stamped on every complete line received; a miner that has
2883// said nothing for this long is gone in every way that matters. Ten
2884// minutes is far beyond the vardiff target (a share every 15s) and the
2885// stall watchdog (45s), so no miner that is actually hashing comes close.
2886static const int64 POOL_IDLE_TIMEOUT_SECS = 600;
2887
1042// Non-blocking recv: pull whatever bytes are available now into buf, then 2888// Non-blocking recv: pull whatever bytes are available now into buf, then
1043// return one complete line if one is ready. Three outcomes: 2889// return one complete line if one is ready. Three outcomes:
1044// true -- line is filled; buf may still contain more lines 2890// true -- line is filled; buf may still contain more lines
@@ -1122,6 +2968,17 @@ static json MakeNotifyParams(const StratumJob& job, const CBlock& b,
1122static void SendJob(Miner* m, bool clean) 2968static void SendJob(Miner* m, bool clean)
1123{ 2969{
1124 if (!gHaveJob) return; 2970 if (!gHaveJob) return;
2971 // Every path that hands a miner a new job funnels through here, so this
2972 // is the one place the replay guard needs to follow the job id: nonces
2973 // credited under the previous job mean nothing under the new one. Keying
2974 // the clear to the id (rather than clearing unconditionally) keeps a
2975 // same-job re-notify -- the vardiff retarget and stall watchdog both
2976 // re-send the CURRENT job -- from reopening the duplicate window it
2977 // exists to close.
2978 if (m->seenNoncesJob != gCurrentJob.jobId) {
2979 m->seenNonces.clear();
2980 m->seenNoncesJob = gCurrentJob.jobId;
2981 }
1125 CBlock mb = BuildMinerBlock(gCurrentJob, m->extranonce2); 2982 CBlock mb = BuildMinerBlock(gCurrentJob, m->extranonce2);
1126 // Use this miner's own shareTarget (set by vardiff) in the notify params. 2983 // Use this miner's own shareTarget (set by vardiff) in the notify params.
1127 // The participant miner reads p[3] as its share target, and HandleLine 2984 // The participant miner reads p[3] as its share target, and HandleLine
@@ -1147,6 +3004,65 @@ static void BroadcastJob(bool clean)
1147} 3004}
1148 3005
1149// --------------------------------------------------------------------------- 3006// ---------------------------------------------------------------------------
3007// #19: connection census. "broadcast to N miner(s)" only counts authorized
3008// sockets that got a job; it says nothing about the rest of the connection
3009// table. This logs the full breakdown so an operator can tell a real workforce
3010// apart from half-open / never-authorized / idle sessions, and flags when the
3011// single select() loop is running out of room (issue #26).
3012//
3013// "Out of room" is not the same question on both platforms, and FD_SETSIZE
3014// answers a different one on each. On POSIX it bounds the *value* of any
3015// descriptor handed to FD_SET, so what matters is the highest fd we hold. On
3016// Windows an fd_set is a counted array of SOCKET handles whose numeric values
3017// are arbitrary, so what matters is how many we hold, not how large they are.
3018// FD_SETSIZE comes from <sys/select.h> or winsock2.h; this file must not define
3019// it, because redefining it after those headers changes nothing.
3020// ---------------------------------------------------------------------------
3021static void LogPoolConnectionCensus()
3022{
3023 int connected = 0, subscribed = 0, authorized = 0, active = 0, neverAuthIdle = 0;
3024 int64 now = GetTime();
3025#ifndef _WIN32
3026 int maxfd = 0;
3027#endif
3028 for (Miner* m : gMiners) {
3029 connected++;
3030#ifndef _WIN32
3031 if ((int)m->fd > maxfd) maxfd = (int)m->fd;
3032#endif
3033 if (m->subscribed) subscribed++;
3034 if (m->authorised) {
3035 authorized++;
3036 if (m->lastSeen > 0 && now - m->lastSeen <= POOL_ACTIVE_WINDOW)
3037 active++;
3038 } else if (now - m->connectTime >= POOL_UNAUTH_GRACE_SECS) {
3039 neverAuthIdle++; // past the grace period and still not authorized
3040 }
3041 }
3042
3043#ifndef _WIN32
3044 int headroom = (int)FD_SETSIZE - maxfd;
3045 LogPrint("pool", "[pool] census: connected=%d subscribed=%d authorized=%d"
3046 " active(last %llds)=%d never-auth-idle=%d"
3047 " (cap=%zu, maxfd=%d, FD_SETSIZE=%d)\n",
3048 connected, subscribed, authorized, (long long)POOL_ACTIVE_WINDOW,
3049 active, neverAuthIdle, MAX_POOL_MINERS, maxfd, (int)FD_SETSIZE);
3050#else
3051 int headroom = (int)FD_SETSIZE - connected;
3052 LogPrint("pool", "[pool] census: connected=%d subscribed=%d authorized=%d"
3053 " active(last %llds)=%d never-auth-idle=%d"
3054 " (cap=%zu, FD_SETSIZE=%d sockets)\n",
3055 connected, subscribed, authorized, (long long)POOL_ACTIVE_WINDOW,
3056 active, neverAuthIdle, MAX_POOL_MINERS, (int)FD_SETSIZE);
3057#endif
3058 if (headroom <= 16)
3059 LogPrint("pool", "[pool] WARNING: %d slots left before FD_SETSIZE=%d --"
3060 " the select() loop is about to walk into undefined behaviour"
3061 " (#26). Refuse new miners or rebuild with poll/epoll.\n",
3062 headroom, (int)FD_SETSIZE);
3063}
3064
3065// ---------------------------------------------------------------------------
1150// Stratum message handler -- called for each complete line from a miner 3066// Stratum message handler -- called for each complete line from a miner
1151// Returns false if the miner should be disconnected immediately. 3067// Returns false if the miner should be disconnected immediately.
1152// --------------------------------------------------------------------------- 3068// ---------------------------------------------------------------------------
@@ -1214,6 +3130,7 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1214 // ------------------------------------------------------------------ 3130 // ------------------------------------------------------------------
1215 if (method == "mining.subscribe") { 3131 if (method == "mining.subscribe") {
1216 std::string sid = ToHex(&m->fd, 4); 3132 std::string sid = ToHex(&m->fd, 4);
3133 m->subscribed = true;
1217 LogPrint("worker", "[worker->pool] subscribe from fd=%d sid=%s\n", 3134 LogPrint("worker", "[worker->pool] subscribe from fd=%d sid=%s\n",
1218 (int)m->fd, sid.c_str()); 3135 (int)m->fd, sid.c_str());
1219 reply(json::array({json::array({json::array({"mining.notify",sid})}),sid,4})); 3136 reply(json::array({json::array({json::array({"mining.notify",sid})}),sid,4}));
@@ -1320,6 +3237,22 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1320 } 3237 }
1321 unsigned int nNonce; memcpy(&nNonce, nb.data(), 4); 3238 unsigned int nNonce; memcpy(&nNonce, nb.data(), 4);
1322 3239
3240 // Reject a replayed (job, nonce) before it costs a rebuild and a
3241 // RandomX hash: nothing about the work below distinguishes a second
3242 // submission of a valid share from the first, so without this check
3243 // it passes every test and is credited again -- and roundShareCount
3244 // is what the payout split divides by, so each replay multiplies the
3245 // address's cut of the round. The set is per-connection and cleared
3246 // by SendJob on every job change, so only a same-job repeat is
3247 // refused.
3248 if (m->seenNonces.count(nNonce)) {
3249 LogPrint("worker", "[worker->pool] duplicate share from %s"
3250 " nonce=%u job=%s -- already credited, not counting again\n",
3251 m->address.c_str(), nNonce, submitJobId.c_str());
3252 reply(false, "Duplicate share");
3253 return true;
3254 }
3255
1323 // Rebuild this miner's exact block (unique merkle via extranonce2) 3256 // Rebuild this miner's exact block (unique merkle via extranonce2)
1324 CBlock b = BuildMinerBlock(gCurrentJob, m->extranonce2); 3257 CBlock b = BuildMinerBlock(gCurrentJob, m->extranonce2);
1325 b.nNonce = nNonce; 3258 b.nNonce = nNonce;
@@ -1347,6 +3280,7 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1347 m->roundShares++; 3280 m->roundShares++;
1348 m->totalShares++; 3281 m->totalShares++;
1349 m->vardiffShares++; 3282 m->vardiffShares++;
3283 m->seenNonces.insert(nNonce); // replay guard: credit each nonce once per job
1350 int64 shareTime = GetTime(); 3284 int64 shareTime = GetTime();
1351 m->lastSeen = shareTime; 3285 m->lastSeen = shareTime;
1352 roundShareCount[m->address]++; 3286 roundShareCount[m->address]++;
@@ -1453,7 +3387,11 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1453 int foundHeight = gCurrentJob.height; 3387 int foundHeight = gCurrentJob.height;
1454 // b is the block we submitted, untouched by ProcessBlock (which 3388 // b is the block we submitted, untouched by ProcessBlock (which
1455 // took a copy), so its hash is what the payout must be keyed on. 3389 // took a copy), so its hash is what the payout must be keyed on.
1456 QueuePayouts(foundHeight, b.GetHash(), roundShareCount, roundShareTotal); 3390 // Passed as a uint256, not hex: the ledger key must stay a real
3391 // hash. The remaining arguments are upstream's round-proof data.
3392 QueuePayouts(foundHeight, b.GetHash(), b.vtx[0].GetHash().GetHex(),
3393 b.vtx[0].vout[0].nValue, m->address,
3394 roundShareCount, roundShareTotal);
1457 // Reset round shares 3395 // Reset round shares
1458 roundShareCount.clear(); 3396 roundShareCount.clear();
1459 roundShareTotal = 0; 3397 roundShareTotal = 0;
@@ -1480,49 +3418,1892 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1480} 3418}
1481 3419
1482// --------------------------------------------------------------------------- 3420// ---------------------------------------------------------------------------
1483// Stats refresh -- called every 2s from event loop 3421// payout-ledger self-test
3422//
3423// Regression cover for the payout defects fixed in this series. Everything
3424// here runs against a scratch data directory and an in-memory wallet map; it
3425// never opens wallet.dat, never touches a real datadir, and never sends
3426// anything.
1484// --------------------------------------------------------------------------- 3427// ---------------------------------------------------------------------------
1485static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal) 3428
3429static bool PayoutTestReadFile(const std::string& path, std::string& out)
1486{ 3430{
1487 std::lock_guard<std::mutex> lk(gStatsMutex); 3431 FILE* f = fopen(path.c_str(), "r");
3432 if (!f) return false;
3433 fseek(f, 0, SEEK_END);
3434 long sz = ftell(f);
3435 fseek(f, 0, SEEK_SET);
3436 out.assign((size_t)(sz > 0 ? sz : 0), '\0');
3437 if (sz > 0 && fread(&out[0], 1, (size_t)sz, f) != (size_t)sz) {
3438 fclose(f);
3439 return false;
3440 }
3441 fclose(f);
3442 return true;
3443}
1488 3444
1489 gStats.authorizedMiners = 0; 3445// Delete the scratch data directory and everything in it. A sweep and not a
1490 gStats.blocksFound = blocksFoundThisSession; 3446// list of known names: LogPrint binds debug.log to whatever GetAppDir() is
1491 gStats.roundShares = roundShareTotal; 3447// when it first fires, so the test's own log lands in here too, and a
1492 gStats.totalHashRate = 0.0; 3448// directory left behind in src/ after `make tests` is litter.
1493 for (Miner* m : gMiners) 3449static void PayoutTestRemoveDir(const std::string& dir)
1494 if (m->authorised) gStats.authorizedMiners++; 3450{
3451#ifdef _WIN32
3452 WIN32_FIND_DATAA findData;
3453 HANDLE hFind = FindFirstFileA((dir + "\\*").c_str(), &findData);
3454 if (hFind != INVALID_HANDLE_VALUE) {
3455 do {
3456 std::string name = findData.cFileName;
3457 if (name == "." || name == "..") continue;
3458 DeleteFileA((dir + "\\" + name).c_str());
3459 } while (FindNextFileA(hFind, &findData));
3460 FindClose(hFind);
3461 }
3462 RemoveDirectoryA(dir.c_str());
3463#else
3464 // One level of recursion, because CreateTransaction() opens a Berkeley DB
3465 // environment and BDB puts its logs in a "database" subdirectory. Without
3466 // this the scratch directory cannot be removed and the test leaves one
3467 // behind on every run, which is what d5b3503 was for.
3468 DIR* d = opendir(dir.c_str());
3469 if (d) {
3470 struct dirent* ent;
3471 while ((ent = readdir(d)) != NULL) {
3472 std::string name = ent->d_name;
3473 if (name == "." || name == "..") continue;
3474 const std::string path = dir + "/" + name;
3475 if (unlink(path.c_str()) != 0)
3476 PayoutTestRemoveDir(path);
3477 }
3478 closedir(d);
3479 }
3480 rmdir(dir.c_str());
3481#endif
3482}
1495 3483
1496 gWorkerStats.clear(); 3484static bool PayoutTestWriteFile(const std::string& path, const std::string& data)
1497 for (Miner* m : gMiners) { 3485{
1498 if (!m->authorised || m->lastSeen <= 0) continue; 3486 FILE* f = fopen(path.c_str(), "w");
1499 PoolWorkerStatView v; 3487 if (!f) return false;
1500 v.address = m->address; 3488 bool ok = data.empty() || fwrite(data.c_str(), 1, data.size(), f) == data.size();
1501 v.worker = m->worker; 3489 return fclose(f) == 0 && ok;
1502 v.roundShares = m->roundShares; 3490}
1503 v.totalShares = m->totalShares; 3491
1504 v.lastSeen = m->lastSeen; 3492// Wipe every piece of payout state this test touches, so each section starts
1505 v.hashRate = m->hashRate; 3493// from the same place and a "restart" can be simulated by reloading from disk.
1506 gWorkerStats.push_back(v); 3494static void PayoutTestResetState()
1507 gStats.totalHashRate += m->hashRate; 3495{
1508 } 3496 {
1509 std::sort(gWorkerStats.begin(), gWorkerStats.end(), 3497 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1510 [](const PoolWorkerStatView& a, const PoolWorkerStatView& b) { 3498 gPendingPayouts.clear();
1511 return a.roundShares != b.roundShares ? a.roundShares > b.roundShares 3499 }
1512 : a.lastSeen > b.lastSeen; 3500 {
3501 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3502 gPaying.clear();
3503 gQueuedPayouts.clear();
3504 }
3505 {
3506 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
3507 gPoolRounds.clear();
3508 }
3509 {
3510 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
3511 gLegacyClaimedTxids.clear();
3512 }
3513 {
3514 // A quarantine is meant to last the life of the process; each section
3515 // here is a fresh process as far as the code under test is concerned.
3516 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
3517 gRefusedFiles.clear();
3518 }
3519}
3520
3521// A BTF address whose hash160 is entirely under the test's control, so the
3522// output script the probe compares against can be built without a wallet.
3523static std::string PayoutTestAddress(unsigned char seed)
3524{
3525 uint160 h = 0;
3526 for (int i = 0; i < 20; i++)
3527 ((unsigned char*)&h)[i] = (unsigned char)(seed + i);
3528 return Hash160ToAddress(h);
3529}
3530
3531static CScript PayoutTestScriptFor(const std::string& address)
3532{
3533 uint160 h160;
3534 CScript sc;
3535 if (AddressToHash160(address, h160))
3536 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
3537 return sc;
3538}
3539
3540// Block index entries fabricated by PayoutTestAddWalletTx, so they can be
3541// freed and unhooked when the test finishes. gPayoutTestChainTip is the end of
3542// that chain: blocks are linked onto it rather than each replacing the tip,
3543// because IsInMainChain() is "pnext is set, or I am the tip".
3544static std::vector<CBlockIndex*> gPayoutTestBlockIndexes;
3545static CBlockIndex* gPayoutTestChainTip = NULL;
3546
3547static void PayoutTestClearBlockIndex()
3548{
3549 CRITICAL_BLOCK(cs_main)
3550 {
3551 for (CBlockIndex* p : gPayoutTestBlockIndexes) {
3552 for (std::map<uint256, CBlockIndex*>::iterator it = mapBlockIndex.begin();
3553 it != mapBlockIndex.end(); ) {
3554 if (it->second == p) mapBlockIndex.erase(it++);
3555 else ++it;
3556 }
3557 delete p;
3558 }
3559 gPayoutTestBlockIndexes.clear();
3560 gPayoutTestChainTip = NULL;
3561 pindexBest = NULL;
3562 }
3563}
3564
3565// Put a transaction in the wallet map that looks exactly like one this pool
3566// sent. `nonce` only exists to keep the hashes distinct.
3567//
3568// fConfirmed decides what GetDepthInMainChain() will answer for it, which is
3569// what the legacy probe now gates on. A confirmed one is given a block index
3570// entry of its own: an empty merkle branch hashes to the transaction itself,
3571// so a block whose merkle root IS that hash accepts it at index 0, and
3572// pointing pindexBest at that block puts it in the main chain one deep. An
3573// unconfirmed one keeps hashBlock 0, which is depth 0 -- the shape a payout
3574// left behind by a reorg has.
3575static std::string PayoutTestAddWalletTx(const std::string& comment,
3576 const std::string& address,
3577 int64 amount,
3578 int nonce,
3579 const std::string& structuredId,
3580 bool fConfirmed = true,
3581 const std::string& structuredBlock = "")
3582{
3583 CWalletTx wtx;
3584 wtx.vin.resize(1);
3585 wtx.vin[0].prevout.SetNull();
3586 wtx.vin[0].scriptSig << nonce;
3587 wtx.vout.resize(1);
3588 wtx.vout[0].nValue = amount;
3589 wtx.vout[0].scriptPubKey = PayoutTestScriptFor(address);
3590 if (!comment.empty())
3591 wtx.mapValue["comment"] = comment;
3592 if (!structuredId.empty()) {
3593 wtx.mapValue["pool_payout_id"] = structuredId;
3594 wtx.mapValue["pool_payout_address"] = address;
3595 wtx.mapValue["pool_payout_amount"] = strprintf("%lld", amount);
3596 if (!structuredBlock.empty())
3597 wtx.mapValue["pool_payout_block"] = structuredBlock;
3598 }
3599 uint256 hash = wtx.GetHash();
3600 if (fConfirmed) {
3601 // A block hash distinct from the transaction's own.
3602 uint256 blockHash = hash;
3603 ((unsigned char*)&blockHash)[0] ^= 0xff;
3604 CRITICAL_BLOCK(cs_main)
3605 {
3606 CBlockIndex* pindex = new CBlockIndex();
3607 pindex->hashMerkleRoot = hash;
3608 // Linked onto the end of the fabricated chain rather than simply
3609 // becoming the new tip: IsInMainChain() is "pnext is set, or I am
3610 // the tip", so a second confirmed transaction would otherwise take
3611 // the first one out of the chain again.
3612 pindex->pprev = gPayoutTestChainTip;
3613 pindex->nHeight = gPayoutTestChainTip ? gPayoutTestChainTip->nHeight + 1 : 1;
3614 if (gPayoutTestChainTip)
3615 gPayoutTestChainTip->pnext = pindex;
3616 gPayoutTestChainTip = pindex;
3617 std::map<uint256, CBlockIndex*>::iterator mi =
3618 mapBlockIndex.insert(std::make_pair(blockHash, pindex)).first;
3619 pindex->phashBlock = &mi->first;
3620 pindexBest = pindex;
3621 gPayoutTestBlockIndexes.push_back(pindex);
3622 }
3623 wtx.hashBlock = blockHash;
3624 wtx.nIndex = 0;
3625 }
3626 CRITICAL_BLOCK(cs_mapWallet)
3627 mapWallet[hash] = wtx;
3628 return hash.GetHex();
3629}
3630
3631// A real key, in the keystore, so a test wallet's outputs are IsMine() and can
3632// actually be signed -- both are needed for CreateTransaction to build the
3633// large transaction the fee schedule is being tested against. Inserted
3634// straight into the key maps rather than through AddKey(), which would also
3635// write it to a wallet.dat.
3636static std::string PayoutTestAddSpendableKey()
3637{
3638 CKey key;
3639 key.MakeNewKey();
3640 std::vector<unsigned char> vchPubKey = key.GetPubKey();
3641 CRITICAL_BLOCK(cs_mapKeys)
3642 {
3643 mapKeys[vchPubKey] = key.GetPrivKey();
3644 mapPubKeys[Hash160(vchPubKey)] = vchPubKey;
3645 }
3646 return Hash160ToAddress(Hash160(vchPubKey));
3647}
3648
3649// Give the wallet `n` spendable outputs of `each`, one transaction apiece --
3650// which is the granularity SelectCoins works at, and therefore the number of
3651// inputs the transaction spending them will carry.
3652//
3653// Deliberately not coinbases: a null prevout would make CWalletTx::GetCredit
3654// return 0 until COINBASE_SPENDABLE_DEPTH, and there is no chain here.
3655static void PayoutTestFundWallet(int n, int64 each, const std::string& address)
3656{
3657 const CScript sc = PayoutTestScriptFor(address);
3658 CRITICAL_BLOCK(cs_mapWallet)
3659 {
3660 for (int i = 0; i < n; i++) {
3661 CWalletTx wtx;
3662 wtx.vin.resize(1);
3663 wtx.vin[0].prevout.hash = (uint64)(i + 1);
3664 wtx.vin[0].prevout.n = 0;
3665 wtx.vout.resize(1);
3666 wtx.vout[0].nValue = each;
3667 wtx.vout[0].scriptPubKey = sc;
3668 mapWallet[wtx.GetHash()] = wtx;
3669 }
3670 }
3671}
3672
3673// Defined below with the event loop; the status-file publication test drives
3674// it directly.
3675static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal);
3676
3677int RunPayoutLedgerSelfTest()
3678{
3679 int nFail = 0;
3680 auto check = [&](bool ok, const char* msg) {
3681 fprintf(stdout, " %s %s\n", ok ? "ok" : "FAIL", msg);
3682 if (!ok) nFail++;
3683 };
3684
3685 fprintf(stdout, "payout-ledger self-test\n");
3686
3687 const std::string savedDataDir = strSetDataDir;
3688 const int savedBestHeight = nBestHeight;
3689
3690 // Scratch data directory in the current working directory. GetAppDir()
3691 // follows strSetDataDir, so this is what PayoutFilePath() and
3692 // PayingFilePath() resolve against for the rest of the test.
3693#ifdef _WIN32
3694 unsigned long pid = (unsigned long)GetCurrentProcessId();
3695#else
3696 unsigned long pid = (unsigned long)getpid();
3697#endif
3698 // Absolute, and the process moves into it.
3699 //
3700 // Both are needed by the CreateTransaction section: Berkeley DB resolves
3701 // its log directory relative to the environment home, so a relative
3702 // datadir makes dbenv.open() fail with DB_RUNRECOVERY; and BDB opens
3703 // db.log relative to the working directory, which is src/ during `make
3704 // tests`. Moving in puts it inside the scratch directory, where it dies
3705 // with everything else.
3706 std::string cwd = ".";
3707 {
3708#ifdef _WIN32
3709 char buf[MAX_PATH];
3710 if (GetCurrentDirectoryA(MAX_PATH, buf) > 0) cwd = buf;
3711#else
3712 char buf[4096];
3713 if (getcwd(buf, sizeof(buf)) != NULL) cwd = buf;
3714#endif
3715 }
3716 std::string dir = cwd + "/" +
3717 strprintf("payout_selftest_%d_%lu", (int)GetTime(), pid);
3718#ifdef _WIN32
3719 CreateDirectoryA(dir.c_str(), NULL);
3720 SetCurrentDirectoryA(dir.c_str());
3721#else
3722 mkdir(dir.c_str(), 0700);
3723 if (chdir(dir.c_str()) != 0) { /* stay put; the paths are absolute anyway */ }
3724#endif
3725 strSetDataDir = dir;
3726 fprintf(stdout, " scratch datadir: %s\n", dir.c_str());
3727
3728 const std::string ledgerPath = PayoutFilePath();
3729 const std::string payingPath = PayingFilePath();
3730 const std::string addrA = PayoutTestAddress(0x11);
3731 const std::string addrB = PayoutTestAddress(0x55);
3732
3733 // -- 1. A restart must not duplicate an in-flight record -----------------
3734 //
3735 // The ledger entry is grandfathered (zero hash) on purpose: it keeps
3736 // PayoutBlockStillOnChain out of the picture, which needs a block index,
3737 // and it exercises the height fallback at the same time.
3738 {
3739 PayoutTestResetState();
3740 remove(ledgerPath.c_str());
3741 remove(payingPath.c_str());
3742
3743 PendingPayout pp;
3744 pp.blockHeight = 4242;
3745 pp.blockHash = 0;
3746 pp.matureAtHeight = 4242;
3747 pp.amounts[addrA] = 10 * COIN;
3748 pp.amounts[addrB] = 20 * COIN;
3749 VersionedBlob snap;
3750 {
3751 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3752 gPendingPayouts.push_back(pp);
3753 snap = SerializePendingPayoutsLocked();
3754 }
3755 PublishPendingPayouts(snap);
3756
3757 nBestHeight = 5000;
3758 FlushMaturePayouts();
3759 size_t afterFirstFlush = 0;
3760 {
3761 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3762 afterFirstFlush = gPaying.size();
3763 }
3764 check(afterFirstFlush == 2,
3765 "a matured round explodes into one record per unpaid recipient");
3766
3767 // Restart: drop everything in memory and come back from the files.
3768 for (int restart = 1; restart <= 2; restart++) {
3769 PayoutTestResetState();
3770 LoadPendingPayouts();
3771 LoadPaying();
3772 ReconcilePayouts();
3773
3774 size_t queued = 0;
3775 {
3776 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3777 queued = gQueuedPayouts.size();
3778 }
3779 if (restart == 1)
3780 check(queued == 1,
3781 "startup rebuilds the exploded-round set from the restored"
3782 " in-flight list");
3783
3784 FlushMaturePayouts();
3785 size_t n = 0;
3786 {
3787 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3788 n = gPaying.size();
3789 }
3790 check(n == 2, restart == 1
3791 ? "a restart does not duplicate an in-flight record"
3792 : "a second restart does not duplicate one either");
3793 }
3794
3795 // The other half of the guard: even with the round-level dedupe set
3796 // deliberately emptied, the per-recipient check against gPaying must
3797 // stop a second standing order for the same money.
3798 {
3799 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3800 gQueuedPayouts.clear();
3801 }
3802 FlushMaturePayouts();
3803 size_t n = 0;
3804 {
3805 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3806 n = gPaying.size();
3807 }
3808 check(n == 2,
3809 "the flush refuses a second record for a (round, address) already"
3810 " in flight even with the round set lost");
3811 }
3812
3813 // -- 2. Duplicates already on disk are collapsed as they are read --------
3814 {
3815 PayoutTestResetState();
3816 nlohmann::json arr = nlohmann::json::array();
3817 for (int i = 0; i < 2; i++) {
3818 nlohmann::json o;
3819 o["blockHeight"] = 4242;
3820 o["blockHash"] = uint256(0).ToString();
3821 o["matureAtHeight"] = 4242;
3822 o["address"] = addrA;
3823 o["amount"] = (int64)(10 * COIN);
3824 o["retries"] = i == 0 ? 7 : 0;
3825 o["deferrals"] = i == 0 ? 0 : 3;
3826 arr.push_back(o);
3827 }
3828 check(PayoutTestWriteFile(payingPath, arr.dump(2)),
3829 "wrote a paying file containing the duplicate the old build made");
3830 LoadPaying();
3831 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3832 check(gPaying.size() == 1,
3833 "loading collapses a duplicate in-flight record from disk");
3834 check(gPaying.size() == 1 && gPaying[0].retries == 7 && gPaying[0].deferrals == 3,
3835 "collapsing merges the attempt counts upwards, not resetting them");
3836 }
3837
3838 // -- 3. State updates address a slot, not a value -----------------------
3839 //
3840 // Build the exact pathology the old code produced -- two records equal in
3841 // every field -- and prove the update lands on the one asked for. Under
3842 // value matching the first record absorbed everything and the second was
3843 // untouchable.
3844 {
3845 PayoutTestResetState();
3846 PayingRecord a;
3847 a.blockHeight = 4242; a.blockHash = 0; a.matureAtHeight = 4242;
3848 a.address = addrA; a.amount = 10 * COIN;
3849 PayingRecord b = a;
3850 {
3851 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3852 a.id = ++gPayingNextId;
3853 b.id = ++gPayingNextId;
3854 gPaying.push_back(a);
3855 gPaying.push_back(b);
3856 }
3857 PayingRecord upd = b;
3858 upd.retries = 5;
3859 upd.retryAfter = 1234567;
3860 ReschedulePayment(upd);
3861 {
3862 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3863 check(gPaying.size() == 2 &&
3864 gPaying[0].retries == 0 && gPaying[1].retries == 5,
3865 "rescheduling updates the record it was given, not the first"
3866 " one that looks like it");
3867 }
3868 ForgetPayment(a);
3869 {
3870 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3871 check(gPaying.size() == 1 && gPaying[0].id == b.id,
3872 "forgetting erases the record it was given, not the first"
3873 " one that looks like it");
3874 }
3875 }
3876
3877 // -- 4. retryAfter survives a restart ------------------------------------
3878 {
3879 PayoutTestResetState();
3880 remove(payingPath.c_str());
3881 PayingRecord r;
3882 r.blockHeight = 77; r.blockHash = 0; r.matureAtHeight = 77;
3883 r.address = addrA; r.amount = 3 * COIN;
3884 r.retryAfter = 1700000000LL;
3885 r.deferrals = 9;
3886 VersionedBlob snap;
3887 {
3888 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3889 r.id = ++gPayingNextId;
3890 gPaying.push_back(r);
3891 snap = SerializePayingLocked();
3892 }
3893 PublishPaying(snap);
3894 PayoutTestResetState();
3895 LoadPaying();
3896 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3897 check(gPaying.size() == 1 && gPaying[0].retryAfter == 1700000000LL &&
3898 gPaying[0].deferrals == 9,
3899 "the retry clock that is actually used survives a restart");
3900 }
3901
3902 // -- 5. The ledger file cannot be published from a stale snapshot --------
3903 {
3904 PayoutTestResetState();
3905 remove(ledgerPath.c_str());
3906
3907 PendingPayout one;
3908 one.blockHeight = 100; one.blockHash = 0; one.matureAtHeight = 100;
3909 one.amounts[addrA] = 1 * COIN;
3910 VersionedBlob older, newer;
3911 {
3912 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3913 gPendingPayouts.push_back(one);
3914 older = SerializePendingPayoutsLocked();
3915 PendingPayout two;
3916 two.blockHeight = 200; two.blockHash = 0; two.matureAtHeight = 200;
3917 two.amounts[addrB] = 2 * COIN;
3918 gPendingPayouts.push_back(two);
3919 newer = SerializePendingPayoutsLocked();
3920 }
3921 check(newer.seq > older.seq,
3922 "a later serialisation carries a strictly higher version");
3923
3924 // The order the race produced: the payout thread's older image
3925 // arriving after the event loop's newer one.
3926 PublishPendingPayouts(newer);
3927 PublishPendingPayouts(older);
3928 std::string onDisk;
3929 check(PayoutTestReadFile(ledgerPath, onDisk) && onDisk == newer.blob,
3930 "an older snapshot cannot overwrite a newer published ledger");
3931
3932 // And the ordinary direction still works.
3933 VersionedBlob newest;
3934 {
3935 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3936 gPendingPayouts.clear();
3937 newest = SerializePendingPayoutsLocked();
3938 }
3939 PublishPendingPayouts(newest);
3940 check(PayoutTestReadFile(ledgerPath, onDisk) && onDisk == newest.blob,
3941 "a newer snapshot does replace the published ledger");
3942 }
3943
3944 // -- 6. Concurrent publishers cannot interleave a file -------------------
3945 //
3946 // Two threads opening and renaming the same "<path>.tmp" over the same
3947 // target was the original defect. This test was mutation-tested and did
3948 // not catch it: with the file mutex removed the suite still went green 2
3949 // runs in 5, and the "never leave it unparseable" assertion fired 0 times
3950 // in 5. Two reasons, both fixed here.
3951 //
3952 // First, every image had the same structure and near enough the same
3953 // length, so an interleaving of two of them was still valid JSON. The
3954 // images now alternate between ONE entry and FIVE THOUSAND, so a write
3955 // that lands on top of another leaves either a truncated array or a small
3956 // image followed by the tail of a large one -- neither of which parses.
3957 //
3958 // Second, and worse, almost nothing was being written. Each thread walked
3959 // a permutation of the images, and PublishVersioned drops any blob that is
3960 // not newer than what is on disk, so only the record highs in each
3961 // permutation ever reached the disk -- on the order of ten writes in
3962 // total, four threads, no contention to speak of. Every publish here is
3963 // stamped from a shared counter at the moment it is made, so every one of
3964 // them is newer than the last and every one of them writes.
3965 //
3966 // And the file is watched while the writers run, not only inspected after
3967 // they finish. "The published file is never observed unparseable" is the
3968 // actual invariant -- rename(2) is atomic, so a reader can only ever see a
3969 // completely written image -- and checking it once at the end samples the
3970 // one moment when a torn file is least likely to still be there.
3971 {
3972 const std::string racePath = dir + "/publish_race_selftest.json";
3973 std::mutex raceMutex;
3974 uint64 racePublished = 0;
3975
3976 std::string tiny, huge;
3977 {
3978 nlohmann::json one = nlohmann::json::array();
3979 nlohmann::json o;
3980 o["image"] = 0;
3981 o["entry"] = 0;
3982 o["address"] = addrA;
3983 o["amount"] = (int64)COIN;
3984 one.push_back(o);
3985 tiny = one.dump(2);
3986
3987 nlohmann::json many = nlohmann::json::array();
3988 for (int k = 0; k < 5000; k++) {
3989 nlohmann::json e;
3990 e["image"] = 1;
3991 e["entry"] = k;
3992 e["address"] = addrA;
3993 e["amount"] = (int64)(k + 1) * COIN;
3994 many.push_back(e);
3995 }
3996 huge = many.dump(2);
3997 }
3998
3999 const int kThreads = 4;
4000 const int kPerThread = 100;
4001 const uint64 kTotal = (uint64)kThreads * kPerThread;
4002
4003 std::atomic<uint64> nextSeq(0);
4004 std::atomic<bool> racing(true);
4005 std::atomic<int> tornReads(0);
4006 std::atomic<int> goodReads(0);
4007
4008 std::thread watcher([&]() {
4009 bool seenOnce = false;
4010 while (racing) {
4011 std::string s;
4012 if (!PayoutTestReadFile(racePath, s)) {
4013 // Only meaningful once the file exists at all.
4014 if (seenOnce) tornReads++;
4015 continue;
4016 }
4017 seenOnce = true;
4018 try {
4019 if (nlohmann::json::parse(s).is_array()) goodReads++;
4020 else tornReads++;
4021 } catch (...) {
4022 tornReads++;
4023 }
4024 }
1513 }); 4025 });
1514 4026
1515 // gPendingPayouts is event-loop-only data (no lock needed here since 4027 std::vector<std::thread> threads;
1516 // RefreshStats is always called from the event loop thread). 4028 for (int t = 0; t < kThreads; t++) {
1517 gPayoutViews.clear(); 4029 threads.push_back(std::thread([&]() {
1518 for (const PendingPayout& pp : gPendingPayouts) { 4030 for (int i = 0; i < kPerThread; i++) {
1519 PendingPayoutView pv; 4031 VersionedBlob v;
1520 pv.matureAtHeight = pp.matureAtHeight; 4032 v.seq = ++nextSeq;
1521 pv.recipients = (int)pp.amounts.size(); 4033 v.blob = (v.seq & 1) ? tiny : huge;
1522 pv.totalAmount = 0; 4034 PublishVersioned(racePath, v, raceMutex, racePublished,
1523 for (auto& kv : pp.amounts) pv.totalAmount += kv.second; 4035 "publish race selftest");
1524 gPayoutViews.push_back(pv); 4036 }
4037 }));
4038 }
4039 for (std::thread& th : threads)
4040 th.join();
4041 racing = false;
4042 watcher.join();
4043
4044 check(goodReads > 0,
4045 "the race test actually observed the file while it was being"
4046 " written");
4047 check(tornReads == 0,
4048 "four threads publishing the same file never leave it"
4049 " unparseable, at any instant while they are doing it");
4050
4051 // The highest sequence number issued is the one the file must settle
4052 // on: PublishVersioned refuses anything older, so whichever order the
4053 // writes complete in, that image is the last one that can land.
4054 const std::string& expected = (kTotal & 1) ? tiny : huge;
4055 std::string onDisk;
4056 bool read = PayoutTestReadFile(racePath, onDisk);
4057 check(read && onDisk == expected,
4058 "the published file settles on the highest version, whatever the"
4059 " completion order");
4060 remove(racePath.c_str());
4061 remove((racePath + ".tmp").c_str());
4062 }
4063
4064 // -- 7. Grandfathered zero-hash rounds do not answer for each other ------
4065 {
4066 uint256 h1 = 0, h2 = 0;
4067 h1.SetHex("00000000000000000000000000000000000000000000000000000000000000aa");
4068 h2.SetHex("00000000000000000000000000000000000000000000000000000000000000bb");
4069 check(!SamePayoutRound(100, 0, 200, 0),
4070 "two zero-hash entries at different heights are different rounds");
4071 check(SamePayoutRound(100, 0, 100, 0),
4072 "two zero-hash entries at the same height are the same round");
4073 check(SamePayoutRound(100, h1, 200, h1),
4074 "a real hash identifies the round on its own");
4075 check(!SamePayoutRound(100, h1, 100, h2),
4076 "different hashes at one height are different rounds");
4077 check(!SamePayoutRound(100, h1, 100, 0),
4078 "a grandfathered entry never matches a hash-keyed one");
4079 check(!PayoutRoundIdentifiable(0, 0) && PayoutRoundIdentifiable(0, h1) &&
4080 PayoutRoundIdentifiable(7, 0),
4081 "an entry with neither hash nor height is not adjudicable");
4082
4083 // What actually makes a grandfathered same-height pair safe, since the
4084 // comment on FindRecordedPoolPayout used to claim it was the orphan
4085 // check. It is not: the orphan check passes every zero-hash entry, and
4086 // every pre-upgrade entry has a zero hash.
4087 check(PayoutBlockStillOnChain(0),
4088 "the orphan check passes every grandfathered entry, so it cannot"
4089 " be what voids a losing twin");
4090 PayoutTestResetState();
4091 remove(ledgerPath.c_str());
4092 remove(payingPath.c_str());
4093 {
4094 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4095 PendingPayout twin;
4096 twin.blockHeight = 300; twin.blockHash = 0; twin.matureAtHeight = 300;
4097 twin.amounts[addrA] = 1 * COIN;
4098 gPendingPayouts.push_back(twin);
4099 gPendingPayouts.push_back(twin); // the losing side of a reorg
4100 }
4101 nBestHeight = 5000;
4102 FlushMaturePayouts();
4103 {
4104 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
4105 check(gPaying.size() == 1,
4106 "two grandfathered rounds at one height are one round, so the"
4107 " pair presents as one debt and is paid at most once");
4108 }
4109
4110 // End to end: recording a payment against one grandfathered round must
4111 // not settle the other's debt.
4112 PayoutTestResetState();
4113 remove(ledgerPath.c_str());
4114 {
4115 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4116 PendingPayout first;
4117 first.blockHeight = 100; first.blockHash = 0; first.matureAtHeight = 100;
4118 first.amounts[addrA] = 1 * COIN;
4119 PendingPayout second;
4120 second.blockHeight = 200; second.blockHash = 0; second.matureAtHeight = 200;
4121 second.amounts[addrA] = 1 * COIN;
4122 gPendingPayouts.push_back(first);
4123 gPendingPayouts.push_back(second);
4124 }
4125 PayingRecord r;
4126 r.blockHeight = 200; r.blockHash = 0; r.matureAtHeight = 200;
4127 r.address = addrA; r.amount = 1 * COIN;
4128 RecordPayoutTxid(r, "txid-for-the-second-round");
4129 {
4130 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4131 // The second round was fully paid and retired; the first is
4132 // untouched and still owed.
4133 check(gPendingPayouts.size() == 1 &&
4134 gPendingPayouts[0].blockHeight == 100 &&
4135 gPendingPayouts[0].paidTxids.empty(),
4136 "a txid recorded against one zero-hash round leaves the other"
4137 " still owed");
4138 }
4139 }
4140
4141 // -- 8. A payment with only a legacy comment is recognised as paid -------
4142 {
4143 PayoutTestResetState();
4144 const int height = 8080;
4145 const int64 amount = 7 * COIN;
4146 const std::string id = PayoutIdForBlock(height, 0);
4147 const std::string comment = LegacyPayoutComment(height);
4148 std::set<std::string> none;
4149 std::string found;
4150 // No round proof exists for these rounds, so legacy evidence is
4151 // admissible -- computed exactly as the payout thread computes it.
4152 const bool fAllowLegacy = !PoolRoundHasProof(id);
4153
4154 check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, none, found)
4155 == PAYOUT_PROBE_NONE,
4156 "an empty wallet is not evidence that anything was paid");
4157
4158 const std::string legacyTxid =
4159 PayoutTestAddWalletTx(comment, addrA, amount, 1, "");
4160 check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, none, found)
4161 == PAYOUT_PROBE_LEGACY && found == legacyTxid,
4162 "a payment carrying only a pre-upgrade comment is recognised as"
4163 " already paid");
4164
4165 check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found)
4166 == PAYOUT_PROBE_NONE,
4167 "the legacy match requires the output to pay THIS recipient");
4168 check(FindRecordedPoolPayout(id, height, 0, addrA, amount + COIN, fAllowLegacy, none, found)
4169 == PAYOUT_PROBE_NONE,
4170 "the legacy match requires the output to pay exactly this amount");
4171 check(FindRecordedPoolPayout(id, height + 1, 0, addrA, amount, fAllowLegacy, none, found)
4172 == PAYOUT_PROBE_NONE,
4173 "the legacy match requires the comment for this block height");
4174
4175 std::set<std::string> spent;
4176 spent.insert(legacyTxid);
4177 check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, spent, found)
4178 == PAYOUT_PROBE_NONE,
4179 "one legacy transaction cannot settle a second debt");
4180
4181 // A transaction written by THIS build carries pool_payout_id, and must
4182 // never be offered as legacy evidence for a different round -- that is
4183 // exactly what the structured metadata exists to prevent.
4184 const std::string otherId = PayoutIdForBlock(height, 0) + "-other";
4185 PayoutTestAddWalletTx(comment, addrB, amount, 2, otherId);
4186 check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found)
4187 == PAYOUT_PROBE_NONE,
4188 "another round's structured payment is not legacy evidence for"
4189 " this one");
4190
4191 // The structured path still wins where it applies.
4192 const std::string structuredTxid =
4193 PayoutTestAddWalletTx(comment, addrB, amount, 3, id);
4194 check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found)
4195 == PAYOUT_PROBE_STRUCTURED && found == structuredTxid,
4196 "structured payout metadata is matched in preference to a comment");
4197
4198 // A comment-matching transaction that never made it into the chain is
4199 // evidence of NON-payment. This tree has seen a 280-block reorg; a
4200 // pre-upgrade payout invalidated by one is still in mapWallet with its
4201 // comment intact, and the first boot after the upgrade is exactly when
4202 // it would be matched.
4203 CRITICAL_BLOCK(cs_mapWallet)
4204 mapWallet.clear();
4205 PayoutTestClearBlockIndex();
4206 const std::string addrC = PayoutTestAddress(0x33);
4207 PayoutTestAddWalletTx(comment, addrC, amount, 4, "", false);
4208 check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, none, found)
4209 == PAYOUT_PROBE_NONE,
4210 "a pre-upgrade payout that never confirmed is not read as"
4211 " payment");
4212
4213 // The same transaction, confirmed, is.
4214 CRITICAL_BLOCK(cs_mapWallet)
4215 mapWallet.clear();
4216 const std::string confirmedTxid =
4217 PayoutTestAddWalletTx(comment, addrC, amount, 4, "", true);
4218 check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, none, found)
4219 == PAYOUT_PROBE_LEGACY && found == confirmedTxid,
4220 "the same payout, confirmed, is");
4221
4222 // And an unconfirmed one does not shadow a confirmed one that would
4223 // have matched: the probe used to stop at the first comment match.
4224 const std::string alsoConfirmed =
4225 PayoutTestAddWalletTx(comment, addrC, amount, 6, "", true);
4226 PayoutTestAddWalletTx(comment, addrC, amount, 5, "", false);
4227 std::set<std::string> spentFirst;
4228 spentFirst.insert(confirmedTxid);
4229 check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, spentFirst, found)
4230 == PAYOUT_PROBE_LEGACY && found == alsoConfirmed,
4231 "an unconfirmed transaction does not hide a confirmed one behind"
4232 " it");
4233
4234 CRITICAL_BLOCK(cs_mapWallet)
4235 mapWallet.clear();
4236 PayoutTestClearBlockIndex();
1525 } 4237 }
4238
4239 // -- 9. On-disk compatibility with the live production ledger ------------
4240 {
4241 PayoutTestResetState();
4242 // Exactly the shape of the file on the pool node: amounts, blockHash,
4243 // blockHeight, matureAtHeight, and nothing else.
4244 const char* live =
4245 "[{\"amounts\":{\"%s\":1234500000},"
4246 "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000cd\","
4247 "\"blockHeight\":1000,"
4248 "\"matureAtHeight\":1120}]";
4249 check(PayoutTestWriteFile(ledgerPath, strprintf(live, addrA.c_str())),
4250 "wrote a ledger file in the live production shape");
4251 LoadPendingPayouts();
4252 {
4253 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4254 check(gPendingPayouts.size() == 1 &&
4255 gPendingPayouts[0].blockHeight == 1000 &&
4256 gPendingPayouts[0].matureAtHeight == 1120 &&
4257 gPendingPayouts[0].blockHash != 0 &&
4258 gPendingPayouts[0].amounts.size() == 1 &&
4259 gPendingPayouts[0].amounts[addrA] == 1234500000LL,
4260 "the live production pending_payouts.json still loads");
4261 }
4262
4263 // And the shape the merged build writes, retryAfter included, which
4264 // this build no longer writes but must still read past.
4265 PayoutTestResetState();
4266 const char* merged =
4267 "[{\"amounts\":{\"%s\":100000000},"
4268 "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000ef\","
4269 "\"blockHeight\":2000,\"matureAtHeight\":2120,\"retryAfter\":0,"
4270 "\"paidTxids\":{\"%s\":\"abc123\"}}]";
4271 check(PayoutTestWriteFile(ledgerPath,
4272 strprintf(merged, addrA.c_str(), addrA.c_str())),
4273 "wrote a ledger file in the merged build's shape");
4274 LoadPendingPayouts();
4275 {
4276 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4277 check(gPendingPayouts.size() == 1 &&
4278 gPendingPayouts[0].blockHeight == 2000 &&
4279 gPendingPayouts[0].paidTxids.size() == 1 &&
4280 gPendingPayouts[0].paidTxids[addrA] == "abc123",
4281 "a ledger file carrying retryAfter and paidTxids still loads");
4282 }
4283
4284 // A round trip through this build must not lose anything either.
4285 VersionedBlob snap;
4286 {
4287 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4288 snap = SerializePendingPayoutsLocked();
4289 }
4290 PublishPendingPayouts(snap);
4291 PayoutTestResetState();
4292 LoadPendingPayouts();
4293 {
4294 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4295 check(gPendingPayouts.size() == 1 &&
4296 gPendingPayouts[0].blockHeight == 2000 &&
4297 gPendingPayouts[0].matureAtHeight == 2120 &&
4298 gPendingPayouts[0].paidTxids[addrA] == "abc123",
4299 "what this build writes, this build reads back unchanged");
4300 }
4301 }
4302
4303 // -- 10. A forfeit is published, not just dropped ------------------------
4304 {
4305 PayoutTestResetState();
4306 remove(ledgerPath.c_str());
4307 const std::string roundsPath = dir + "/pool_rounds_selftest.json";
4308 const std::string savedRoundsFile = strPoolRoundsFile;
4309 strPoolRoundsFile = roundsPath;
4310 remove(roundsPath.c_str());
4311
4312 const int height = 606;
4313 PoolRoundProof round;
4314 round.id = PayoutIdForBlock(height, 0);
4315 round.blockHeight = height;
4316 round.matureAtHeight = height;
4317 round.payouts[addrA] = 5 * COIN;
4318 round.payouts[addrB] = 6 * COIN;
4319 RecordPoolRound(round);
4320
4321 {
4322 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4323 PendingPayout pp;
4324 pp.blockHeight = height; pp.blockHash = 0; pp.matureAtHeight = height;
4325 pp.amounts[addrA] = 5 * COIN;
4326 pp.amounts[addrB] = 6 * COIN;
4327 gPendingPayouts.push_back(pp);
4328 }
4329
4330 PayingRecord r;
4331 r.blockHeight = height; r.blockHash = 0; r.matureAtHeight = height;
4332 r.address = addrA; r.amount = 5 * COIN;
4333 ForfeitPayoutRecipient(r, "selftest reason");
4334
4335 {
4336 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4337 check(gPendingPayouts.size() == 1 &&
4338 gPendingPayouts[0].amounts.count(addrA) == 0 &&
4339 gPendingPayouts[0].amounts.count(addrB) == 1,
4340 "a forfeit removes only that recipient from the ledger entry");
4341 }
4342
4343 std::string onDisk;
4344 bool proofOk = false;
4345 if (PayoutTestReadFile(roundsPath, onDisk)) {
4346 try {
4347 nlohmann::json root = nlohmann::json::parse(onDisk);
4348 const nlohmann::json& p = root["rounds"][0]["payouts"];
4349 proofOk = p[addrA].value("forfeited", false) &&
4350 p[addrA].value("forfeitReason", "") == "selftest reason" &&
4351 p[addrA]["txid"].is_null() &&
4352 !p[addrB].contains("forfeited");
4353 } catch (...) {
4354 proofOk = false;
4355 }
4356 }
4357 check(proofOk,
4358 "the published round proof records the forfeit instead of"
4359 " advertising the address as still owed");
4360
4361 // And it survives a reload of the proof file.
4362 {
4363 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
4364 gPoolRounds.clear();
4365 }
4366 LoadPoolRounds();
4367 {
4368 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
4369 check(gPoolRounds.size() == 1 &&
4370 gPoolRounds[0].forfeited.count(addrA) == 1 &&
4371 gPoolRounds[0].forfeited[addrA] == "selftest reason",
4372 "the forfeit is read back from pool_rounds.json");
4373 }
4374
4375 remove(roundsPath.c_str());
4376 remove((roundsPath + ".tmp").c_str());
4377 strPoolRoundsFile = savedRoundsFile;
4378 }
4379
4380 // -- 11. A ledger that failed to load is never written back --------------
4381 //
4382 // The loaders push entries into the live vector inside the try, so an
4383 // exception part way down a file leaves a PARTIAL ledger in memory: the
4384 // entries before the bad one and nothing after it. Publishing that -- and
4385 // ReconcilePayouts publishes unconditionally, milliseconds into startup --
4386 // writes the partial state over the file and destroys the only record of
4387 // what was owed.
4388 {
4389 PayoutTestResetState();
4390 remove(ledgerPath.c_str());
4391
4392 // Two entries, the second with a matureAtHeight that is not a number.
4393 // The first loads; the second throws where it is read.
4394 const std::string corrupt = strprintf(
4395 "[{\"amounts\":{\"%s\":100000000},\"blockHash\":\"%s\","
4396 "\"blockHeight\":10,\"matureAtHeight\":10},"
4397 "{\"amounts\":{\"%s\":900000000},\"blockHash\":\"%s\","
4398 "\"blockHeight\":20,\"matureAtHeight\":\"twenty\"}]",
4399 addrA.c_str(), uint256(0).ToString().c_str(),
4400 addrB.c_str(), uint256(0).ToString().c_str());
4401 check(PayoutTestWriteFile(ledgerPath, corrupt),
4402 "wrote a ledger whose second entry does not parse");
4403
4404 const int64 t0 = GetTime();
4405 LoadPendingPayouts();
4406 const int64 t1 = GetTime();
4407
4408 size_t loaded = 0;
4409 {
4410 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4411 loaded = gPendingPayouts.size();
4412 }
4413 check(loaded == 1,
4414 "a mid-file parse error does leave a partial ledger in memory --"
4415 " which is the reason for everything below");
4416
4417 std::string gone;
4418 check(!PayoutTestReadFile(ledgerPath, gone),
4419 "the file that did not parse is moved out of the way");
4420
4421 // The rename stamps GetTime(); it was taken between t0 and t1.
4422 std::string kept, keptPath;
4423 for (int64 t = t0; t <= t1 && keptPath.empty(); t++) {
4424 const std::string candidate =
4425 strprintf("%s.bad.%lld", ledgerPath.c_str(), (long long)t);
4426 if (PayoutTestReadFile(candidate, kept))
4427 keptPath = candidate;
4428 }
4429 check(!keptPath.empty() && kept == corrupt,
4430 "it is kept, byte for byte, as <path>.bad.<timestamp>");
4431
4432 // The publish that used to destroy it.
4433 VersionedBlob partial;
4434 {
4435 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4436 partial = SerializePendingPayoutsLocked();
4437 }
4438 PublishPendingPayouts(partial);
4439 check(!PayoutTestReadFile(ledgerPath, gone),
4440 "and the partial ledger is not published over it");
4441
4442 // The refusal is per path and lasts the process: a second attempt,
4443 // with more state, is refused too.
4444 {
4445 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4446 PendingPayout pp;
4447 pp.blockHeight = 30; pp.blockHash = 0; pp.matureAtHeight = 30;
4448 pp.amounts[addrA] = 5 * COIN;
4449 gPendingPayouts.push_back(pp);
4450 partial = SerializePendingPayoutsLocked();
4451 }
4452 PublishPendingPayouts(partial);
4453 check(!PayoutTestReadFile(ledgerPath, gone),
4454 "the refusal is not a one-shot -- the path stays unwritable");
4455
4456 if (!keptPath.empty())
4457 remove(keptPath.c_str());
4458 PayoutTestResetState();
4459 remove(ledgerPath.c_str());
4460
4461 // And a file that parses is not quarantined, so the ordinary path is
4462 // still the ordinary path.
4463 check(PayoutTestWriteFile(ledgerPath, "[]"),
4464 "wrote a ledger that parses");
4465 LoadPendingPayouts();
4466 {
4467 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4468 PendingPayout pp;
4469 pp.blockHeight = 40; pp.blockHash = 0; pp.matureAtHeight = 40;
4470 pp.amounts[addrA] = 2 * COIN;
4471 gPendingPayouts.push_back(pp);
4472 partial = SerializePendingPayoutsLocked();
4473 }
4474 PublishPendingPayouts(partial);
4475 std::string written;
4476 check(PayoutTestReadFile(ledgerPath, written) && written == partial.blob,
4477 "a ledger that loaded cleanly is still published normally");
4478 }
4479
4480 // -- 12. A send that failed for want of coin is not a fault --------------
4481 //
4482 // The defect this replaces: the classifier decided "was this a funding
4483 // problem?" by re-reading GetBalance() and allowing amount +
4484 // nTransactionFee + CENT. nTransactionFee is 0 on a headless node, and
4485 // GetMinFee() is a step function -- 0 below 10,000 bytes, 11 * CENT at it
4486 // -- so the allowance was either unnecessary or eleven times too small,
4487 // and the ledger's own retry budget then wrote the debt off after 1440
4488 // repeats of the same stable condition.
4489 //
4490 // This is tested against a real wallet and the real CreateTransaction, not
4491 // against injected values: the honest question is not "does my enum
4492 // switch work" but "does a wallet holding many small coins actually report
4493 // the state I think it does". 60 outputs of 1 BTF give ~180 signed P2PKH
4494 // bytes each, which is where the fee schedule steps. Nothing is committed
4495 // or broadcast -- CreateTransaction only builds.
4496 {
4497 PayoutTestResetState();
4498 CRITICAL_BLOCK(cs_mapWallet)
4499 mapWallet.clear();
4500
4501 // CreateTransaction opens CTxDB("r"), which throws if blkindex.dat is
4502 // not there. Create it in the scratch datadir first.
4503 bool dbOk = true;
4504 try {
4505 CTxDB txdb("cr");
4506 } catch (const std::exception&) {
4507 dbOk = false;
4508 }
4509 check(dbOk, "a scratch block index exists for CreateTransaction to open");
4510
4511 const std::string walletAddr = PayoutTestAddSpendableKey();
4512 const std::string payee = PayoutTestAddress(0x77);
4513 const CScript payeeScript = PayoutTestScriptFor(payee);
4514
4515 // -- the >= 10 KB case: 60 small coins, balance = amount + 0.05 ------
4516 PayoutTestFundWallet(60, 1 * COIN, walletAddr);
4517 const int64 balance = GetBalance();
4518 check(balance == 60 * COIN,
4519 "the test wallet holds 60 separately spendable coins");
4520
4521 const int64 amount = balance - 5 * CENT;
4522 // The pre-send gate in PayoutThreadFn passes: the balance covers the
4523 // amount and the only fee that can be known in advance.
4524 check(balance >= amount + nTransactionFee,
4525 "the pre-send gate would let this payment through");
4526
4527 CWalletTx wtx;
4528 SendMoneyOutcome outcome;
4529 int64 feeRequired = 0;
4530 WalletSendResult result = WALLET_SEND_OK;
4531 bool built = true;
4532 if (dbOk) {
4533 try {
4534 built = CreateTransaction(payeeScript, amount, wtx, feeRequired,
4535 &result);
4536 } catch (const std::exception&) {
4537 built = true; // report as a failure below
4538 }
4539 }
4540 check(dbOk && !built,
4541 "a wallet of many small coins cannot fund amount + the fee its"
4542 " own size demands");
4543 check(dbOk && result == WALLET_SEND_INSUFFICIENT_FUNDS,
4544 "the wallet reports that as a funding shortfall, not as a"
4545 " failure to build");
4546 // GetMinFee(fDiscount) returns 0 below 10,000 bytes and
4547 // (1 + bytes/1000) * CENT at or above, so a required fee of 11 CENT or
4548 // more IS the statement that the transaction crossed 10 KB. Measured
4549 // through the real serialiser rather than asserted.
4550 check(dbOk && feeRequired >= 11 * CENT,
4551 "the transaction it built crossed 10,000 bytes, so the fee"
4552 " stepped from 0 to at least 11 CENT");
4553
4554 // The defect, stated as arithmetic. The old classifier asked whether
4555 // the balance covered amount + nTransactionFee + CENT; it did, so it
4556 // called a permanent funding shortfall a fault and started counting
4557 // towards a write-off.
4558 check(!(balance < amount + nTransactionFee + CENT),
4559 "the reserve this replaces would have judged this send a fault");
4560 check(balance < amount + feeRequired,
4561 "while the fee actually required was more than the balance had");
4562
4563 outcome.result = result;
4564 outcome.nFeeRequired = feeRequired;
4565 check(dbOk && ClassifyPayoutSendFailure(outcome.result, false)
4566 == PAYOUT_FAILURE_DEFER,
4567 "a >= 10 KB send that ran out of coin is deferred, not counted"
4568 " against the payment");
4569
4570 // -- the < 10 KB case: no reserve is needed at all -------------------
4571 //
4572 // The other half of the arithmetic. Below the step the required fee is
4573 // 0, so a reserve of any size is a reason to refuse a payment the
4574 // wallet could have made -- which is its own way of not paying a
4575 // miner.
4576 CRITICAL_BLOCK(cs_mapWallet)
4577 mapWallet.clear();
4578 PayoutTestFundWallet(3, 1 * COIN, walletAddr);
4579 const int64 smallAmount = 3 * COIN - 5 * CENT;
4580 CWalletTx smallTx;
4581 int64 smallFee = -1;
4582 WalletSendResult smallResult = WALLET_SEND_FAILED;
4583 bool smallBuilt = false;
4584 if (dbOk) {
4585 try {
4586 smallBuilt = CreateTransaction(payeeScript, smallAmount, smallTx,
4587 smallFee, &smallResult);
4588 } catch (const std::exception&) {
4589 smallBuilt = false;
4590 }
4591 }
4592 check(dbOk && smallBuilt && smallResult == WALLET_SEND_OK,
4593 "the same balance pays a small transaction with no headroom at"
4594 " all");
4595 check(dbOk && smallFee == 0,
4596 "because below 10,000 bytes the required fee is zero");
4597
4598 // -- and the classification of every other answer --------------------
4599 check(ClassifyPayoutSendFailure(WALLET_SEND_INSUFFICIENT_FUNDS, false)
4600 == PAYOUT_FAILURE_DEFER &&
4601 ClassifyPayoutSendFailure(WALLET_SEND_INSUFFICIENT_FUNDS, true)
4602 == PAYOUT_FAILURE_DEFER &&
4603 ClassifyPayoutSendFailure(WALLET_SEND_FAILED, true)
4604 == PAYOUT_FAILURE_DEFER,
4605 "short of coin or unable to sign is always a deferral");
4606 check(ClassifyPayoutSendFailure(WALLET_SEND_FAILED, false)
4607 == PAYOUT_FAILURE_FAULT,
4608 "funded and unlocked and still refused is a fault");
4609
4610 CRITICAL_BLOCK(cs_mapWallet)
4611 mapWallet.clear();
4612 }
4613
4614 // -- 13. No payout mutex is held across the status-file write ------------
4615 //
4616 // RefreshStats runs on the event loop -- the thread that owns every
4617 // miner's socket -- and used to hold gStatsMutex and gPayoutLedgerMutex at
4618 // function scope across WriteFileAtomic. Once that write gained fsync() on
4619 // the file and on its directory, publishing pool_status.json parked the
4620 // payout thread behind two disk flushes every ten seconds.
4621 //
4622 // Timing assertions would be flaky, so this does not measure anything. It
4623 // makes the write block for as long as the test likes: strPoolStatusFile
4624 // points at a path whose ".tmp" is a FIFO, and open(2) for writing on a
4625 // FIFO does not return until a reader arrives. While RefreshStats is
4626 // parked in there, every mutex it must NOT be holding has to be
4627 // acquirable. Against the previous code this test does not merely fail, it
4628 // cannot pass: the writer is inside fopen() with both mutexes held and
4629 // nothing will release them until the test lets the write finish.
4630#ifndef _WIN32
4631 {
4632 PayoutTestResetState();
4633 const std::string statusPath = dir + "/pool_status_selftest.json";
4634 const std::string statusTmp = statusPath + ".tmp";
4635 const std::string savedStatusFile = strPoolStatusFile;
4636 const int64 savedLastWrite = gLastPoolStatusWrite;
4637 remove(statusPath.c_str());
4638 remove(statusTmp.c_str());
4639 strPoolStatusFile = statusPath;
4640 gLastPoolStatusWrite = 0; // due a publish on the next refresh
4641
4642 // Give the status file something to report, so the ledger mutex is
4643 // genuinely taken on the way through rather than skipped over.
4644 {
4645 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4646 PendingPayout pp;
4647 pp.blockHeight = 909; pp.blockHash = 0; pp.matureAtHeight = 909;
4648 pp.amounts[addrA] = 3 * COIN;
4649 gPendingPayouts.push_back(pp);
4650 }
4651
4652 bool fifoOk = (mkfifo(statusTmp.c_str(), 0600) == 0);
4653 check(fifoOk, "a FIFO stands in for the status file's temp file, so the"
4654 " write can be held open");
4655 if (fifoOk) {
4656 std::atomic<bool> done(false);
4657 std::thread writer([&done]() {
4658 RefreshStats(0, 0);
4659 done = true;
4660 });
4661
4662 // Settle: the work before the write is a handful of vector
4663 // operations, so anything past this point is inside fopen(). The
4664 // !done check below is what actually proves it -- with no reader
4665 // there is no other way for this thread to still be running.
4666 Sleep(500);
4667 bool blocked = !done;
4668 bool gotLedger = false, gotStats = false, gotExec = false,
4669 gotRounds = false;
4670 for (int i = 0; i < 1000 && blocked; i++) {
4671 if (!gotLedger && gPayoutLedgerMutex.try_lock()) {
4672 gotLedger = true; gPayoutLedgerMutex.unlock();
4673 }
4674 if (!gotStats && gStatsMutex.try_lock()) {
4675 gotStats = true; gStatsMutex.unlock();
4676 }
4677 if (!gotExec && gPayoutExecMutex.try_lock()) {
4678 gotExec = true; gPayoutExecMutex.unlock();
4679 }
4680 if (!gotRounds && gPoolRoundsMutex.try_lock()) {
4681 gotRounds = true; gPoolRoundsMutex.unlock();
4682 }
4683 if (gotLedger && gotStats && gotExec && gotRounds)
4684 break;
4685 Sleep(2);
4686 }
4687
4688 check(blocked,
4689 "the status-file write is held open for the duration of this"
4690 " test");
4691 check(!blocked || gotLedger,
4692 "gPayoutLedgerMutex is free while pool_status.json is being"
4693 " written");
4694 check(!blocked || (gotExec && gotRounds),
4695 "gPayoutExecMutex and gPoolRoundsMutex are free while"
4696 " pool_status.json is being written");
4697 check(!blocked || gotStats,
4698 "gStatsMutex is free while pool_status.json is being written");
4699
4700 // Let the write finish. Drain to EOF rather than closing early:
4701 // the writer is still filling the pipe and a closed read end would
4702 // deliver it SIGPIPE.
4703 std::thread reader([&statusTmp]() {
4704 FILE* rf = fopen(statusTmp.c_str(), "r");
4705 if (!rf) return;
4706 char buf[4096];
4707 while (fread(buf, 1, sizeof(buf), rf) > 0) { }
4708 fclose(rf);
4709 });
4710 writer.join();
4711 reader.join();
4712 }
4713 remove(statusTmp.c_str());
4714
4715 // And the ordinary path still produces the file it always did.
4716 gLastPoolStatusWrite = 0;
4717 RefreshStats(0, 0);
4718 std::string onDisk;
4719 bool statusOk = false;
4720 if (PayoutTestReadFile(statusPath, onDisk)) {
4721 try {
4722 nlohmann::json j = nlohmann::json::parse(onDisk);
4723 statusOk = j.value("schema", "") == "bitflash-pool-status-1" &&
4724 j["pendingPayouts"].is_array() &&
4725 j["pendingPayouts"].size() == 1 &&
4726 j["pendingPayouts"][0].value("matureAtHeight", 0) == 909 &&
4727 j["pendingPayouts"][0].value("totalAmountSatoshis",
4728 (int64)0) == 3 * COIN;
4729 } catch (...) {
4730 statusOk = false;
4731 }
4732 }
4733 check(statusOk,
4734 "splitting the serialise from the write leaves pool_status.json"
4735 " saying the same thing");
4736
4737 remove(statusPath.c_str());
4738 remove((statusPath + ".tmp").c_str());
4739 strPoolStatusFile = savedStatusFile;
4740 gLastPoolStatusWrite = savedLastWrite;
4741 }
4742#endif
4743
4744 // -- 14. The payout comment names the funding block ----------------------
4745 //
4746 // Same height, two hashes: the shape a reorg leaves when the pool wins a
4747 // height, loses the block, and wins the height again. The legacy needle is
4748 // frozen at the pre-upgrade format, so pre-upgrade payments keep matching;
4749 // a new-format comment never matches it, so a new send can never be read
4750 // as a pre-upgrade payment for a twin round.
4751 {
4752 PayoutTestResetState();
4753 CRITICAL_BLOCK(cs_mapWallet)
4754 mapWallet.clear();
4755 PayoutTestClearBlockIndex();
4756
4757 const int height = 9090;
4758 const int64 amount = 3 * COIN;
4759 uint256 hashA = 0, hashB = 0;
4760 hashA.SetHex("00000000000000000000000000000000000000000000000000000000000000aa");
4761 hashB.SetHex("00000000000000000000000000000000000000000000000000000000000000bb");
4762
4763 check(LegacyPayoutComment(height) ==
4764 strprintf("Pool payout height %d", height),
4765 "the legacy needle is frozen at the format pre-upgrade builds"
4766 " wrote");
4767 check(PayoutComment(height, 0) == LegacyPayoutComment(height),
4768 "a grandfathered zero-hash record still writes the legacy form");
4769 check(PayoutComment(height, hashA) ==
4770 strprintf("Pool payout height %d block %s", height,
4771 hashA.ToString().c_str()),
4772 "a hash-keyed record's comment names its funding block");
4773 check(PayoutComment(height, hashA) != PayoutComment(height, hashB),
4774 "twin rounds at one height carry distinguishable comments");
4775
4776 // A transaction wearing the NEW comment format and -- hypothetically
4777 // -- no structured metadata must not be matched by the legacy needle:
4778 // the fallback exists for pre-upgrade transactions, and the new format
4779 // proves this is not one.
4780 const std::string idA = PayoutIdForBlock(height, hashA);
4781 const std::string idB = PayoutIdForBlock(height, hashB);
4782 std::set<std::string> none;
4783 std::string found;
4784 // Neither twin has a round proof here (section 15 covers the gate),
4785 // and their ids collide, so one answer serves both.
4786 const bool fAllowLegacy = !PoolRoundHasProof(idA);
4787 PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount, 1, "");
4788 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found)
4789 == PAYOUT_PROBE_NONE,
4790 "a new-format comment is never legacy evidence for a twin round");
4791 check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found)
4792 == PAYOUT_PROBE_NONE,
4793 "nor a match channel for its own round -- structured metadata is");
4794
4795 // Round A's own payment, sent by this build, is found by its
4796 // structured id.
4797 CRITICAL_BLOCK(cs_mapWallet)
4798 mapWallet.clear();
4799 PayoutTestClearBlockIndex();
4800 const std::string paidA =
4801 PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount,
4802 2, idA);
4803 check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found)
4804 == PAYOUT_PROBE_STRUCTURED && found == paidA,
4805 "the winning twin's own payment is found by its structured id");
4806
4807 // The id is derived from the FIRST sixteen hex digits of the hash --
4808 // the end proof-of-work fills with zeros -- so twin hashes that agree
4809 // there collide in id, and a payment carrying no pool_payout_block
4810 // answers for either twin. That is the accepted trade for payments
4811 // sent before the key existed, pinned here so it stays a decision.
4812 check(idA == idB,
4813 "twin hashes differing only past the sixteenth hex digit collide"
4814 " in id -- the reason the full hash must decide");
4815 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found)
4816 == PAYOUT_PROBE_STRUCTURED,
4817 "a payment without the block key answers for a colliding twin --"
4818 " the accepted residual for pre-key transactions");
4819
4820 // With the full hash recorded -- as every send now records it -- the
4821 // twins are told apart and the re-won height is paid.
4822 CRITICAL_BLOCK(cs_mapWallet)
4823 mapWallet.clear();
4824 PayoutTestClearBlockIndex();
4825 const std::string keyedA =
4826 PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount,
4827 3, idA, true, hashA.ToString());
4828 check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found)
4829 == PAYOUT_PROBE_STRUCTURED && found == keyedA,
4830 "a payment carrying the full block hash settles its own round");
4831 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found)
4832 == PAYOUT_PROBE_NONE,
4833 "and is no evidence for the twin, so the re-won height is paid");
4834
4835 CRITICAL_BLOCK(cs_mapWallet)
4836 mapWallet.clear();
4837 PayoutTestClearBlockIndex();
4838 }
4839
4840 // -- 15. A proof-bearing round rejects comment-only evidence -------------
4841 //
4842 // The surviving half of the same-height collision: a pre-upgrade
4843 // transaction carries only "Pool payout height H", so if H is won again
4844 // after a reorg, it matches the new round's (height, address, amount)
4845 // exactly and would settle a debt it never paid. The new round has a
4846 // round proof -- written at creation, and only by builds whose payments
4847 // all name their round -- so for it, comment-only evidence is
4848 // inadmissible and the miner is paid. A round without a proof keeps
4849 // today's bias: the evidence is admissible and doubt resolves to
4850 // "already paid".
4851 {
4852 PayoutTestResetState();
4853 CRITICAL_BLOCK(cs_mapWallet)
4854 mapWallet.clear();
4855 PayoutTestClearBlockIndex();
4856 const std::string roundsPath = dir + "/pool_rounds_gate_selftest.json";
4857 const std::string savedRoundsFile = strPoolRoundsFile;
4858 strPoolRoundsFile = roundsPath;
4859 remove(roundsPath.c_str());
4860
4861 const int height = 7171;
4862 const int64 amount = 4 * COIN;
4863 uint256 hashB = 0;
4864 hashB.SetHex("00000000000000000000000000000000000000000000000000000000000000cc");
4865 const std::string idB = PayoutIdForBlock(height, hashB);
4866 std::set<std::string> none;
4867 std::string found;
4868
4869 // The colliding pre-upgrade payment: confirmed, comment-only, same
4870 // height, same address, same amount -- the strongest wrong evidence
4871 // there is.
4872 PayoutTestAddWalletTx(LegacyPayoutComment(height), addrA, amount, 1, "");
4873
4874 check(!PoolRoundHasProof(idB),
4875 "a round with no recorded proof does not claim one");
4876 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount,
4877 !PoolRoundHasProof(idB), none, found)
4878 == PAYOUT_PROBE_LEGACY,
4879 "without a proof the doubt is genuine and still resolves to"
4880 " already-paid");
4881
4882 PoolRoundProof round;
4883 round.id = idB;
4884 round.blockHeight = height;
4885 round.matureAtHeight = height + COINBASE_SPENDABLE_DEPTH;
4886 round.payouts[addrA] = amount;
4887 RecordPoolRound(round);
4888 check(PoolRoundHasProof(idB),
4889 "recording the round proof marks the round as this lineage's");
4890 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount,
4891 !PoolRoundHasProof(idB), none, found)
4892 == PAYOUT_PROBE_NONE,
4893 "with a proof, a same-height comment-only transaction is not"
4894 " evidence, and the re-won height is paid");
4895
4896 // Structured evidence still is: the round's own payment, recovered.
4897 const std::string paidB =
4898 PayoutTestAddWalletTx(PayoutComment(height, hashB), addrA, amount,
4899 2, idB, true, hashB.ToString());
4900 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount,
4901 !PoolRoundHasProof(idB), none, found)
4902 == PAYOUT_PROBE_STRUCTURED && found == paidB,
4903 "the gate does not blind the probe to the round's own structured"
4904 " payment");
4905
4906 CRITICAL_BLOCK(cs_mapWallet)
4907 mapWallet.clear();
4908 PayoutTestClearBlockIndex();
4909 remove(roundsPath.c_str());
4910 remove((roundsPath + ".tmp").c_str());
4911 strPoolRoundsFile = savedRoundsFile;
4912 }
4913
4914 // -- 16. A malformed block hash is corruption, not a hash ----------------
4915 //
4916 // SetHex() parses whatever hex prefix it finds and stops, so a corrupt
4917 // blockHash did not fail to load -- it loaded as a DIFFERENT hash.
4918 // "deadbeef..." became 0xdeadbeef, a hash no block ever had, which the
4919 // orphan check reads as an orphaned block and the next flush VOIDS the
4920 // debt; a value with no hex prefix became 0, which is grandfathered past
4921 // the orphan check instead. Either way a corrupt file silently changed
4922 // what the pool owed. It is a load failure now, handled like every other
4923 // parse error: quarantine, and never rewrite.
4924 {
4925 PayoutTestResetState();
4926 remove(ledgerPath.c_str());
4927 remove(payingPath.c_str());
4928
4929 const std::string badLedger = strprintf(
4930 "[{\"amounts\":{\"%s\":100000000},"
4931 "\"blockHash\":\"deadbeef oops, not a hash\","
4932 "\"blockHeight\":10,\"matureAtHeight\":10}]", addrA.c_str());
4933 check(PayoutTestWriteFile(ledgerPath, badLedger),
4934 "wrote a ledger whose blockHash is not a 64-digit hex string");
4935 LoadPendingPayouts();
4936 {
4937 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4938 check(gPendingPayouts.empty(),
4939 "no entry is loaded with a guessed hash");
4940 }
4941 std::string gone;
4942 check(!PayoutTestReadFile(ledgerPath, gone),
4943 "the ledger with the corrupt hash is quarantined");
4944
4945 const std::string badPaying = strprintf(
4946 "[{\"address\":\"%s\",\"amount\":100000000,"
4947 "\"blockHash\":\"00000000cafe\",\"blockHeight\":10,"
4948 "\"matureAtHeight\":10}]", addrA.c_str());
4949 check(PayoutTestWriteFile(payingPath, badPaying),
4950 "wrote an in-flight list whose blockHash is truncated");
4951 LoadPaying();
4952 {
4953 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
4954 check(gPaying.empty(),
4955 "no in-flight record is loaded with a guessed hash");
4956 }
4957 check(!PayoutTestReadFile(payingPath, gone),
4958 "the in-flight list with the corrupt hash is quarantined");
4959
4960 // The two spellings every real file uses still load: the 64-digit
4961 // hash of a keyed round, and the 64 zeros this build writes for a
4962 // grandfathered entry.
4963 PayoutTestResetState();
4964 remove(ledgerPath.c_str());
4965 const std::string goodLedger = strprintf(
4966 "[{\"amounts\":{\"%s\":100000000},"
4967 "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000cd\","
4968 "\"blockHeight\":10,\"matureAtHeight\":10},"
4969 "{\"amounts\":{\"%s\":200000000},"
4970 "\"blockHash\":\"%s\","
4971 "\"blockHeight\":20,\"matureAtHeight\":20}]",
4972 addrA.c_str(), addrB.c_str(), uint256(0).ToString().c_str());
4973 check(PayoutTestWriteFile(ledgerPath, goodLedger),
4974 "wrote a ledger in both real hash spellings");
4975 LoadPendingPayouts();
4976 {
4977 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4978 check(gPendingPayouts.size() == 2 &&
4979 gPendingPayouts[0].blockHash != 0 &&
4980 gPendingPayouts[1].blockHash == 0,
4981 "a real hash and a serialised zero hash both still load");
4982 }
4983 remove(ledgerPath.c_str());
4984 }
4985
4986 // -- clean up ------------------------------------------------------------
4987 PayoutTestResetState();
4988 CRITICAL_BLOCK(cs_mapWallet)
4989 mapWallet.clear();
4990 PayoutTestClearBlockIndex();
4991 // Out of the scratch directory before removing it, and out of the key maps
4992 // too: the keys this test generated are process-local, but leaving them in
4993 // the keystore would let a later test think a script was its own.
4994 CRITICAL_BLOCK(cs_mapKeys)
4995 {
4996 mapKeys.clear();
4997 mapPubKeys.clear();
4998 }
4999#ifdef _WIN32
5000 SetCurrentDirectoryA(cwd.c_str());
5001#else
5002 if (chdir(cwd.c_str()) != 0) { /* nothing useful to do about it */ }
5003#endif
5004 PayoutTestRemoveDir(dir);
5005 strSetDataDir = savedDataDir;
5006 nBestHeight = savedBestHeight;
5007
5008 fprintf(stdout, "%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
5009 nFail, nFail == 1 ? "" : "s");
5010 fflush(stdout);
5011 return nFail == 0 ? 0 : 1;
5012}
5013
5014int RunPoolStratumSelfTest()
5015{
5016 int nFail = 0;
5017 auto check = [&](bool ok, const char* msg) {
5018 fprintf(stdout, " %s %s\n", ok ? "ok" : "FAIL", msg);
5019 if (!ok) nFail++;
5020 };
5021
5022 fprintf(stdout, "pool-stratum self-test\n");
5023
5024 StratumJob savedJob = gCurrentJob;
5025 bool savedHaveJob = gHaveJob;
5026
5027 CTransaction coinbase;
5028 coinbase.vin.resize(1);
5029 coinbase.vin[0].prevout.SetNull();
5030 coinbase.vin[0].scriptSig << 1 << (CBigNum)1;
5031 coinbase.vout.resize(1);
5032 coinbase.vout[0].nValue = 50 * COIN;
5033
5034 CBlock block;
5035 block.vtx.push_back(coinbase);
5036 block.nVersion = 1;
5037 block.nBits = 0x1f0fffff;
5038 block.nTime = 1;
5039 block.nNonce = 1;
5040 block.hashMerkleRoot = block.BuildMerkleTree();
5041
5042 gCurrentJob.jobId = "selftest";
5043 gCurrentJob.block = block;
5044 gCurrentJob.target = 0; // accepted share, not a block candidate
5045 gCurrentJob.height = 1;
5046 gHaveJob = true;
5047
5048 Miner miner(INVALID_SOCKET);
5049 miner.authorised = true;
5050 miner.address = "selftest-address";
5051 miner.worker = "worker";
5052 miner.shareTarget = ~uint256(0); // every RandomX hash is a valid share
5053 miner.difficulty = 0.001;
5054 miner.vardiffWindowStart = GetTime();
5055
5056 std::map<std::string, uint64> roundShareCount;
5057 uint64 roundShareTotal = 0;
5058 int blocksFound = 0;
5059
5060 uint256 targetFromNotify;
5061 CBlock minerBlock = BuildMinerBlock(gCurrentJob, miner.extranonce2);
5062 json params = MakeNotifyParams(gCurrentJob, minerBlock, miner.shareTarget, true);
5063 std::vector<unsigned char> targetBytes = FromHex(params[3].get<std::string>());
5064 if (targetBytes.size() == 32)
5065 memcpy(&targetFromNotify, &targetBytes[0], 32);
5066 check(targetFromNotify == miner.shareTarget,
5067 "mining.notify exposes the same share target the pool validates");
5068
5069 json submit = {
5070 {"id", 7},
5071 {"method", "mining.submit"},
5072 {"params", json::array({miner.address, "selftest", ToHex(&minerBlock.nNonce, 4)})}
5073 };
5074 HandleLine(&miner, submit.dump(), roundShareCount, roundShareTotal, blocksFound);
5075
5076 check(roundShareTotal == 1, "fake low-difficulty share increments the round total");
5077 check(roundShareCount[miner.address] == 1, "fake share is credited to the submitting address");
5078 check(miner.sessionShares == 1 && miner.roundShares == 1 && miner.totalShares == 1,
5079 "miner share counters advance together");
5080 check(blocksFound == 0, "share below block target is not counted as a found block");
5081
5082 json stale = {
5083 {"id", 8},
5084 {"method", "mining.submit"},
5085 {"params", json::array({miner.address, "stale", ToHex(&minerBlock.nNonce, 4)})}
5086 };
5087 HandleLine(&miner, stale.dump(), roundShareCount, roundShareTotal, blocksFound);
5088 check(roundShareTotal == 1, "stale submit does not increment shares");
5089
5090 std::vector<PoolRoundProof> savedRounds = gPoolRounds;
5091 std::string savedRoundsFile = strPoolRoundsFile;
5092 std::string proofPath = "pool_rounds_selftest.json";
5093 remove(proofPath.c_str());
5094 remove((proofPath + ".tmp").c_str());
5095 gPoolRounds.clear();
5096 strPoolRoundsFile = proofPath;
5097
5098 PoolRoundProof proof;
5099 proof.id = "selftest-round";
5100 proof.blockHeight = 42;
5101 proof.matureAtHeight = 42 + COINBASE_MATURITY;
5102 proof.createdAt = GetTime();
5103 proof.blockHash = block.GetHash().GetHex();
5104 proof.coinbaseTxid = coinbase.GetHash().GetHex();
5105 proof.foundBy = miner.address;
5106 proof.feePercent = 1.25;
5107 proof.totalShares = 3;
5108 proof.coinbaseValue = 50 * COIN;
5109 proof.subsidy = 50 * COIN;
5110 proof.minerReward = 49 * COIN;
5111 proof.operatorReward = 1 * COIN;
5112 proof.shares[miner.address] = 3;
5113 proof.payouts[miner.address] = 49 * COIN;
5114 RecordPoolRound(proof);
5115 MarkPoolRoundPaid(proof.id, miner.address, "selftest-txid");
5116
5117 bool proofOk = false;
5118 FILE* pf = fopen(proofPath.c_str(), "r");
5119 if (pf) {
5120 fseek(pf, 0, SEEK_END);
5121 long sz = ftell(pf);
5122 fseek(pf, 0, SEEK_SET);
5123 std::string s(sz, '\0');
5124 fread(&s[0], 1, sz, pf);
5125 fclose(pf);
5126 try {
5127 json root = json::parse(s);
5128 proofOk =
5129 root.value("schema", "") == "bitflash-pool-rounds-1" &&
5130 root["rounds"].is_array() &&
5131 root["rounds"].size() == 1 &&
5132 root["rounds"][0].value("id", "") == proof.id &&
5133 root["rounds"][0]["shares"].value(miner.address, (uint64)0) == 3 &&
5134 root["rounds"][0]["payouts"][miner.address].value("txid", "") == "selftest-txid";
5135 } catch (...) {
5136 proofOk = false;
5137 }
5138 }
5139 check(proofOk, "pool round proof ledger is written and paid txid is recorded");
5140
5141 remove(proofPath.c_str());
5142 remove((proofPath + ".tmp").c_str());
5143 strPoolRoundsFile = savedRoundsFile;
5144 gPoolRounds = savedRounds;
5145
5146 // -- hashrate decay (#7) --------------------------------------------------
5147 // The first version of this decayed m->hashRate in place from the event
5148 // loop's sweep, which made the reported figure a function of how the loop
5149 // happened to run rather than of how long the miner had been quiet. These
5150 // pin the properties that fixes: sampling changes nothing, and the answer
5151 // depends only on elapsed silence.
5152 {
5153 // A miner exactly on the vardiff target: 4096 hashes per share at
5154 // difficulty 1.0, one share per 15s, so 4096/15 H/s. Its expected
5155 // interval is 15s and its grace is 60s.
5156 const double base = 4096.0 / 15.0;
5157 const int64 t0 = 1766000000; // any plausible non-zero wall clock
5158 Miner decayed(INVALID_SOCKET);
5159 decayed.difficulty = 1.0;
5160 decayed.hashRate = base;
5161 decayed.lastShareTime = t0;
5162
5163 check(MinerReportedHashRate(&decayed, t0 + 60) == base,
5164 "hashrate decay: a miner inside its grace period reports its full estimate");
5165
5166 // Sampling must not consume the estimate. Twenty passes is one second
5167 // of the event loop's old sweep; a thousand is nearly a minute of it.
5168 double once = MinerReportedHashRate(&decayed, t0 + 61);
5169 bool idempotent = (decayed.hashRate == base);
5170 for (int i = 0; i < 1000 && idempotent; i++)
5171 idempotent = (MinerReportedHashRate(&decayed, t0 + 61) == once) &&
5172 (decayed.hashRate == base);
5173 check(idempotent,
5174 "hashrate decay: repeated sampling returns one value and leaves the estimate alone");
5175
5176 // One second past a 60s grace is 1/15th of a half-life: ~4.5% off, not
5177 // the estimate gone. A miner at the vardiff target overshoots its grace
5178 // roughly 1.8% of the time (e^-4) purely by share-timing variance, and
5179 // must not be zeroed for it.
5180 check(once > 0.94 * base && once < 0.96 * base,
5181 "hashrate decay: one second past grace costs a few percent, not the estimate");
5182 double three = MinerReportedHashRate(&decayed, t0 + 63);
5183 check(three > 0.86 * base && three < 0.88 * base,
5184 "hashrate decay: three seconds past grace costs ~13%");
5185
5186 // One half-life past the grace is exactly half.
5187 double half = MinerReportedHashRate(&decayed, t0 + 75);
5188 check(half > 0.499 * base && half < 0.501 * base,
5189 "hashrate decay: one expected interval past grace halves the estimate");
5190
5191 // A miner that has genuinely stopped ends up reported as nothing --
5192 // the whole point of #7. The old in-place version could not get here:
5193 // its decay fed back into its own grace and settled into a 1/t crawl.
5194 check(MinerReportedHashRate(&decayed, t0 + 3600) == 0.0,
5195 "hashrate decay: a miner silent for an hour is reported as zero");
5196
5197 // A clock that steps backwards must never inflate what we announce.
5198 check(MinerReportedHashRate(&decayed, t0 - 3600) == base,
5199 "hashrate decay: a backwards clock reports the estimate unchanged, never higher");
5200
5201 // Fresh connection, and a connection with one share and so no interval
5202 // to have timed: both must report nothing rather than divide by zero.
5203 Miner fresh(INVALID_SOCKET);
5204 check(MinerReportedHashRate(&fresh, t0) == 0.0,
5205 "hashrate decay: a miner that never submitted a share reports zero");
5206 Miner single(INVALID_SOCKET);
5207 single.lastShareTime = t0;
5208 check(MinerReportedHashRate(&single, t0 + 86400) == 0.0,
5209 "hashrate decay: a miner with no rate sample yet reports zero");
5210
5211 // Nothing in the swept range may produce a NaN, an infinity, a negative
5212 // rate, or a rate above the estimate it started from.
5213 bool finite = true;
5214 for (int64 s = -100; s < 200000 && finite; s += 37) {
5215 double v = MinerReportedHashRate(&decayed, t0 + s);
5216 finite = !std::isnan(v) && !std::isinf(v) && v >= 0.0 && v <= base;
5217 }
5218 check(finite,
5219 "hashrate decay: stays finite, non-negative and non-amplifying across a long silence");
5220 }
5221
5222 gCurrentJob = savedJob;
5223 gHaveJob = savedHaveJob;
5224
5225 fprintf(stdout, "%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
5226 nFail, nFail == 1 ? "" : "s");
5227 fflush(stdout);
5228 return nFail == 0 ? 0 : 1;
5229}
5230
5231// ---------------------------------------------------------------------------
5232// Stats refresh -- called every 2s from event loop
5233// ---------------------------------------------------------------------------
5234static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal)
5235{
5236 std::string statusBlob;
5237 bool fPublish = false;
5238 {
5239 std::lock_guard<std::mutex> lk(gStatsMutex);
5240
5241 gStats.authorizedMiners = 0;
5242 gStats.blocksFound = blocksFoundThisSession;
5243 gStats.roundShares = roundShareTotal;
5244 gStats.totalHashRate = 0.0;
5245 for (Miner* m : gMiners)
5246 if (m->authorised) gStats.authorizedMiners++;
5247
5248 gWorkerStats.clear();
5249 // One timestamp for the whole sweep (#7): every miner is faded to the
5250 // same instant, so the per-worker rows and the total they are summed
5251 // into describe one moment rather than a smear across the loop.
5252 int64 nowStats = GetTime();
5253 for (Miner* m : gMiners) {
5254 if (!m->authorised || m->lastSeen <= 0) continue;
5255 PoolWorkerStatView v;
5256 v.address = m->address;
5257 v.worker = m->worker;
5258 v.roundShares = m->roundShares;
5259 v.totalShares = m->totalShares;
5260 v.lastSeen = m->lastSeen;
5261 // Not m->hashRate: a miner that has stopped submitting shares must
5262 // not keep contributing its last sample to the announced total.
5263 v.hashRate = MinerReportedHashRate(m, nowStats);
5264 gWorkerStats.push_back(v);
5265 gStats.totalHashRate += v.hashRate;
5266 }
5267 std::sort(gWorkerStats.begin(), gWorkerStats.end(),
5268 [](const PoolWorkerStatView& a, const PoolWorkerStatView& b) {
5269 return a.roundShares != b.roundShares ? a.roundShares > b.roundShares
5270 : a.lastSeen > b.lastSeen;
5271 });
5272
5273 // Scoped, not held to the end of the function. At function scope this
5274 // lock covered the status-file write below, so the event loop did two
5275 // fsyncs holding the mutex the payout thread takes to record that a
5276 // miner has been paid.
5277 gPayoutViews.clear();
5278 {
5279 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
5280 for (const PendingPayout& pp : gPendingPayouts) {
5281 PendingPayoutView pv;
5282 pv.matureAtHeight = pp.matureAtHeight;
5283 pv.recipients = 0;
5284 pv.totalAmount = 0;
5285 for (auto& kv : pp.amounts) {
5286 if (pp.paidTxids.count(kv.first))
5287 continue;
5288 pv.recipients++;
5289 pv.totalAmount += kv.second;
5290 }
5291 if (pv.recipients > 0)
5292 gPayoutViews.push_back(pv);
5293 }
5294 }
5295
5296 int64 now = GetTime();
5297 if (now - gLastPoolStatusWrite >= 10) {
5298 statusBlob = SerializePoolStatusLocked();
5299 gLastPoolStatusWrite = now;
5300 fPublish = true;
5301 }
5302 }
5303 // Every payout mutex is released here, and gStatsMutex with them. This is
5304 // where the two fsyncs happen, and no other thread is waiting behind them.
5305 if (fPublish)
5306 PublishPoolStatus(statusBlob);
1526} 5307}
1527 5308
1528// --------------------------------------------------------------------------- 5309// ---------------------------------------------------------------------------
@@ -1533,6 +5314,7 @@ static void PoolEventLoop()
1533 int lastHeight = -1; 5314 int lastHeight = -1;
1534 int64 lastRefresh = GetTime(); 5315 int64 lastRefresh = GetTime();
1535 int64 lastStatsRefresh = 0; 5316 int64 lastStatsRefresh = 0;
5317 int64 lastCensus = 0; // #19 connection census throttle
1536 int blocksFoundThisSession = 0; 5318 int blocksFoundThisSession = 0;
1537 int64 noJobSince = 0; // 0 = have a job 5319 int64 noJobSince = 0; // 0 = have a job
1538 bool warnedNoJob = false; 5320 bool warnedNoJob = false;
@@ -1561,6 +5343,31 @@ static void PoolEventLoop()
1561 sock_close(fd); 5343 sock_close(fd);
1562 continue; 5344 continue;
1563 } 5345 }
5346 // #26: select() + FD_SET goes undefined once we exceed what an
5347 // fd_set can carry, and MAX_POOL_MINERS (1000) alone does not
5348 // prevent it -- a Miner is only removed on clean disconnect, so
5349 // half-open zombies accumulate and the OS can hand out fds past
5350 // the limit long before the count reaches 1000. Refuse rather
5351 // than corrupt the fd_set. Which limit applies is platform
5352 // specific: POSIX bounds the descriptor value, Windows the
5353 // number of SOCKET handles an fd_set can hold.
5354#ifndef _WIN32
5355 if ((int)fd >= (int)FD_SETSIZE) {
5356 LogPrint("worker", "[pool] refusing miner fd=%d -- at/over"
5357 " FD_SETSIZE=%d, select() would be undefined (#26)\n",
5358 (int)fd, (int)FD_SETSIZE);
5359 sock_close(fd);
5360 continue;
5361 }
5362#else
5363 if (gMiners.size() >= (size_t)FD_SETSIZE) {
5364 LogPrint("worker", "[pool] refusing miner -- %zu sockets held,"
5365 " an fd_set carries at most FD_SETSIZE=%d (#26)\n",
5366 gMiners.size(), (int)FD_SETSIZE);
5367 sock_close(fd);
5368 continue;
5369 }
5370#endif
1564 gMiners.push_back(new Miner(fd)); 5371 gMiners.push_back(new Miner(fd));
1565 LogPrint("worker", "[pool] miner fd=%d entered event loop" 5372 LogPrint("worker", "[pool] miner fd=%d entered event loop"
1566 " -- awaiting subscribe/authorize\n", (int)fd); 5373 " -- awaiting subscribe/authorize\n", (int)fd);
@@ -1688,6 +5495,35 @@ static void PoolEventLoop()
1688 } 5495 }
1689 } 5496 }
1690 5497
5498 // 3.6. Idle reap (#6). Mirror the clean-disconnect path exactly --
5499 // log, close, erase from gMiners via std::find, delete -- just with
5500 // a different reason. Shares the miner already earned stay in
5501 // roundShareCount, exactly as they do when a miner cleanly
5502 // disconnects mid-round. connectTime stands in for lastSeen on a
5503 // connection that never completed a line. Erasing inside the loop
5504 // above would invalidate the iteration, so collect then reap.
5505 {
5506 int64 now = GetTime();
5507 std::vector<Miner*> idle;
5508 for (Miner* m : gMiners) {
5509 int64 last = (m->lastSeen > 0) ? m->lastSeen : m->connectTime;
5510 if (now - last > POOL_IDLE_TIMEOUT_SECS)
5511 idle.push_back(m);
5512 }
5513 for (Miner* m : idle) {
5514 LogPrint("worker", "[pool] dropping idle miner %s -- nothing"
5515 " received in over %llds (session shares: %llu,"
5516 " round shares: %llu)\n",
5517 m->worker.empty() ? "(never authorized)" : m->worker.c_str(),
5518 (long long)POOL_IDLE_TIMEOUT_SECS,
5519 (unsigned long long)m->sessionShares,
5520 (unsigned long long)m->roundShares);
5521 sock_close(m->fd);
5522 gMiners.erase(std::find(gMiners.begin(), gMiners.end(), m));
5523 delete m;
5524 }
5525 }
5526
1691 // 4. Select over miner sockets 5527 // 4. Select over miner sockets
1692 if (!gMiners.empty()) { 5528 if (!gMiners.empty()) {
1693 fd_set fds; FD_ZERO(&fds); 5529 fd_set fds; FD_ZERO(&fds);
@@ -1710,6 +5546,11 @@ static void PoolEventLoop()
1710 // terminate the whole node -- not just the pool. Every 5546 // terminate the whole node -- not just the pool. Every
1711 // line arrives from an unauthenticated socket. 5547 // line arrives from an unauthenticated socket.
1712 while (RecvLine(m->fd, m->readBuf, line, disc)) { 5548 while (RecvLine(m->fd, m->readBuf, line, disc)) {
5549 // Any complete line is proof of life (#6): authorize
5550 // and valid shares already stamp lastSeen themselves,
5551 // but a miner that only ever subscribes (or sends
5552 // unrecognised methods) must count as alive too.
5553 m->lastSeen = GetTime();
1713 try { 5554 try {
1714 HandleLine(m, line, roundShareCount, 5555 HandleLine(m, line, roundShareCount,
1715 roundShareTotal, blocksFoundThisSession); 5556 roundShareTotal, blocksFoundThisSession);
@@ -1750,6 +5591,14 @@ static void PoolEventLoop()
1750 RefreshStats(blocksFoundThisSession, roundShareTotal); 5591 RefreshStats(blocksFoundThisSession, roundShareTotal);
1751 lastStatsRefresh = GetTime(); 5592 lastStatsRefresh = GetTime();
1752 } 5593 }
5594
5595 // 6. #19: log the connection census periodically. Cheap (one pass over
5596 // gMiners) and it is the single number an operator reads to judge pool
5597 // health, so it should not be an overstatement.
5598 if (GetTime() - lastCensus >= POOL_CENSUS_SECS) {
5599 LogPoolConnectionCensus();
5600 lastCensus = GetTime();
5601 }
1753 } 5602 }
1754 5603
1755 for (Miner* m : gMiners) { sock_close(m->fd); delete m; } 5604 for (Miner* m : gMiners) { sock_close(m->fd); delete m; }
@@ -1778,13 +5627,19 @@ static void AcceptOneFn(void* arg)
1778 std::string host = relay.substr(0, colon); 5627 std::string host = relay.substr(0, colon);
1779 int port = atoi(relay.substr(colon + 1).c_str()); 5628 int port = atoi(relay.substr(colon + 1).c_str());
1780 5629
1781 LogPrint("pool", "[pool] accept thread started for relay %s\n", relay.c_str()); 5630 LogPrint("pool", "[pool] accept thread started for relay %s%s\n", relay.c_str(),
5631 fBtfRelayPinned ? " (/rvrelay pin)" : "");
5632
5633 // Registration health for this relay. A pin has no fallback, so a pin that
5634 // never registers means the pool is simply unreachable -- and that has to
5635 // be loud. The 2026-07-30 incident went unnoticed because this loop logged
5636 // failures at one flat line and success not at all, so "which relay is this
5637 // pool actually registered at" could not be answered from the log.
5638 int nConsecFail = 0;
5639 int64 nFirstFail = 0;
5640 bool fEverRegistered = false;
1782 5641
1783 while (!fShutdown && gPoolRunning) { 5642 while (!fShutdown && gPoolRunning) {
1784 // Advertise this relay as the pool's meeting node BEFORE registering,
1785 // so that miners which resolve our descriptor find a relay where the
1786 // pool (not the node's peer-accept) holds the registration.
1787 BtfSetActiveRelay(relay);
1788 // Returns as soon as the relay lists us -- registered, not yet paired. 5643 // Returns as soon as the relay lists us -- registered, not yet paired.
1789 btf::RvSocket rv = btf::RvServiceRegister(host.c_str(), (unsigned short)port, pk); 5644 btf::RvSocket rv = btf::RvServiceRegister(host.c_str(), (unsigned short)port, pk);
1790 5645
@@ -1793,12 +5648,36 @@ static void AcceptOneFn(void* arg)
1793 break; 5648 break;
1794 } 5649 }
1795 if (rv == btf::RV_INVALID) { 5650 if (rv == btf::RV_INVALID) {
1796 LogPrint("pool", "[pool] relay %s: RvServiceRegister failed" 5651 nConsecFail++;
1797 " -- backing off 30s before retry\n", relay.c_str()); 5652 if (nFirstFail == 0) nFirstFail = GetTime();
5653 LogPrint("pool", "[pool] relay %s: registration failed (attempt %d,"
5654 " failing for %llds) -- backing off 30s before retry\n",
5655 relay.c_str(), nConsecFail, (long long)(GetTime() - nFirstFail));
5656 if (fBtfRelayPinned && (nConsecFail == 3 || nConsecFail % 10 == 0))
5657 LogPrint("pool", "[pool] WARNING: pinned relay %s refused registration"
5658 " %d times running (%llds); /rvrelay leaves no fallback, so"
5659 " miners cannot reach this pool%s\n",
5660 relay.c_str(), nConsecFail, (long long)(GetTime() - nFirstFail),
5661 fEverRegistered ? "" : " -- it has never registered once");
1798 for (int i = 0; i < 30 && !fShutdown && gPoolRunning; i++) Sleep(1000); 5662 for (int i = 0; i < 30 && !fShutdown && gPoolRunning; i++) Sleep(1000);
1799 continue; 5663 continue;
1800 } 5664 }
1801 5665
5666 // Advertise THIS relay only now. Setting it before the call published a
5667 // meeting node we were not yet listed at, so a miner resolving the
5668 // descriptor inside that window burned its dial on a relay holding no
5669 // registration for us. net.cpp's peer-accept has always ordered it this
5670 // way round; the pool had not.
5671 BtfSetActiveRelay(relay);
5672 if (!fEverRegistered || nConsecFail > 0)
5673 LogPrint("pool", "[pool] registered at relay %s%s%s -- advertising it as"
5674 " the pool's meeting node\n", relay.c_str(),
5675 fBtfRelayPinned ? " (/rvrelay pin)" : "",
5676 nConsecFail > 0 ? ", recovered" : "");
5677 fEverRegistered = true;
5678 nConsecFail = 0;
5679 nFirstFail = 0;
5680
1802 // Block for a miner to actually arrive. RvServiceRegister used to do 5681 // Block for a miner to actually arrive. RvServiceRegister used to do
1803 // this itself and returned with the relay's 0x01 pairing byte already 5682 // this itself and returned with the relay's 0x01 pairing byte already
1804 // consumed; it no longer does. Without waiting here, BtfServiceWrap 5683 // consumed; it no longer does. Without waiting here, BtfServiceWrap
@@ -1853,9 +5732,21 @@ static void AcceptLoopFn(void*)
1853 LogPrint("pool", "[pool] .btf identity ready after %ds\n", waitSecs * 2); 5732 LogPrint("pool", "[pool] .btf identity ready after %ds\n", waitSecs * 2);
1854 5733
1855 std::set<std::string> spawned; // relays we've already started a thread for 5734 std::set<std::string> spawned; // relays we've already started a thread for
5735 std::set<std::string> skipped; // discovered relays the pin excluded, logged once each
1856 5736
1857 while (!fShutdown && gPoolRunning) { 5737 while (!fShutdown && gPoolRunning) {
5738 // BtfAllRelays() already honours the pin: with /rvrelay set it returns
5739 // vBtfMeetingRelays alone, so no thread is ever spawned for a
5740 // discovered relay. Name the ones it drops -- a pool registering at a
5741 // stranger's relay is exactly the thing that should not be silent.
1858 vector<string> relays = BtfAllRelays(); 5742 vector<string> relays = BtfAllRelays();
5743 if (fBtfRelayPinned) {
5744 for (const string& r : BtfDiscoveredRelays())
5745 if (std::find(relays.begin(), relays.end(), r) == relays.end() &&
5746 skipped.insert(r).second)
5747 LogPrint("pool", "[pool] not registering at discovered relay %s"
5748 " -- /rvrelay pins the relay list\n", r.c_str());
5749 }
1859 if (relays.empty()) { 5750 if (relays.empty()) {
1860 LogPrint("pool", "[pool] no relays available -- miners cannot connect yet\n"); 5751 LogPrint("pool", "[pool] no relays available -- miners cannot connect yet\n");
1861 } else { 5752 } else {
@@ -1871,8 +5762,9 @@ static void AcceptLoopFn(void*)
1871 relay.c_str()); 5762 relay.c_str());
1872 } else { 5763 } else {
1873 spawned.insert(relay); 5764 spawned.insert(relay);
1874 LogPrint("pool", "[pool] accept thread spawned for relay %s" 5765 LogPrint("pool", "[pool] accept thread spawned for relay %s%s"
1875 " (%zu total)\n", relay.c_str(), spawned.size()); 5766 " (%zu total)\n", relay.c_str(),
5767 fBtfRelayPinned ? " (/rvrelay pin)" : "", spawned.size());
1876 } 5768 }
1877 } 5769 }
1878 } 5770 }
@@ -1901,16 +5793,24 @@ void ThreadRPCServer(void*)
1901 } 5793 }
1902 5794
1903 LoadPendingPayouts(); 5795 LoadPendingPayouts();
5796 LoadPoolRounds();
1904 LoadPaying(); 5797 LoadPaying();
1905 // Must run before the payout thread starts, so nothing is resent that the 5798 // Must run before the payout thread starts, so nothing is resent that the
1906 // wallet already shows as sent, and nothing is owed twice because we died 5799 // ledger already shows as sent, and every restored in-flight record is
1907 // between recording the payments and rewriting the ledger. 5800 // armed for its one wallet probe before it is allowed to send.
1908 ReconcilePayouts(); 5801 ReconcilePayouts();
1909 5802
1910 _beginthread(AcceptLoopFn, 0, NULL); 5803 // _beginthread cannot throw (it reports -1), but an unchecked failure
1911 _beginthread(LocalAcceptFn, 0, NULL); 5804 // leaves the pool running deaf: nobody accepts, announces, or pays, and
1912 _beginthread(ThreadBtfPoolAnnouncer, 0, NULL); 5805 // nothing in the log says why.
1913 _beginthread(PayoutThreadFn, 0, NULL); 5806 if (_beginthread(AcceptLoopFn, 0, NULL) == (uintptr_t)-1)
5807 LogPrint("pool", "[pool] FATAL: accept loop thread failed to spawn -- miners cannot connect\n");
5808 if (_beginthread(LocalAcceptFn, 0, NULL) == (uintptr_t)-1)
5809 LogPrint("pool", "[pool] local accept thread failed to spawn\n");
5810 if (_beginthread(ThreadBtfPoolAnnouncer, 0, NULL) == (uintptr_t)-1)
5811 LogPrint("pool", "[pool] announcer thread failed to spawn -- pool will not be advertised\n");
5812 if (_beginthread(PayoutThreadFn, 0, NULL) == (uintptr_t)-1)
5813 LogPrint("pool", "[pool] FATAL: payout thread failed to spawn -- no one gets paid\n");
1914 5814
1915 PoolEventLoop(); // blocks until gPoolRunning = false or fShutdown 5815 PoolEventLoop(); // blocks until gPoolRunning = false or fShutdown
1916 5816