upleb.uk

Public git repos — served from a NIP-34 GRASP relay at git.upleb.uk

summaryrefslogtreecommitdiff
path: root/src/rpc.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/rpc.cpp')
-rw-r--r--src/rpc.cpp4490
1 files changed, 4195 insertions, 295 deletions
diff --git a/src/rpc.cpp b/src/rpc.cpp
index 03d1f44..3827cd5 100644
--- a/src/rpc.cpp
+++ b/src/rpc.cpp
@@ -31,10 +31,15 @@
31#include <netinet/in.h> 31#include <netinet/in.h>
32#include <arpa/inet.h> 32#include <arpa/inet.h>
33#include <unistd.h> 33#include <unistd.h>
34#include <fcntl.h>
35#include <dirent.h>
36#else
37#include <io.h>
34#endif 38#endif
35 39
36#include <atomic> 40#include <atomic>
37#include <mutex> 41#include <mutex>
42#include <thread>
38#include <map> 43#include <map>
39#include <set> 44#include <set>
40#include <deque> 45#include <deque>
@@ -110,15 +115,28 @@ static uint256 ShareTargetFromDifficulty(double difficulty)
110{ 115{
111 if (difficulty <= 0.0) difficulty = 1.0; 116 if (difficulty <= 0.0) difficulty = 1.0;
112 static const uint256 diff1 = CBigNum(bnProofOfWorkLimit).getuint256(); 117 static const uint256 diff1 = CBigNum(bnProofOfWorkLimit).getuint256();
118 // Representability floor. shareTarget = diff1 / difficulty, and diff1 is
119 // ~uint256(0) >> 12, i.e. a hair under 2^244 -- so the true target stops
120 // fitting in a uint256 once difficulty reaches 2^-12 (= 1/4096), and the
121 // fixed-point divide below breaks even earlier in a second way: the
122 // (uint64)(difficulty * 65536.0) divisor truncates to zero for any
123 // difficulty under 2^-16. Both thresholds sit far below VARDIFF_MIN
124 // (0.001), so live operation never comes near this, but a caller that
125 // bypasses vardiff (or a future VARDIFF_MIN) must not divide by zero or
126 // hand back getuint256()'s low-256-bit truncation of an oversized
127 // quotient -- a wrapped target is not merely wrong, it is arbitrarily
128 // HARDER than difficulty 1. At these difficulties every hash is meant to
129 // be a share, so the honest answer is the easiest representable target.
130 // (This replaces a dead branch that read `if (t < diff1 && difficulty <
131 // 1.0)`: t = diff1/difficulty, so difficulty < 1.0 implies t > diff1 and
132 // the two conditions could never both hold.)
133 if (difficulty < 1.0 / 4096.0) return ~uint256(0);
113 // shareTarget = diff1 / difficulty 134 // shareTarget = diff1 / difficulty
114 CBigNum bn; bn.setuint256(diff1); 135 CBigNum bn; bn.setuint256(diff1);
115 // Multiply by 2^16 to keep precision, divide, shift back 136 // Multiply by 2^16 to keep precision, divide, shift back
116 bn <<= 16; 137 bn <<= 16;
117 bn /= (uint64)(difficulty * 65536.0); 138 bn /= (uint64)(difficulty * 65536.0);
118 uint256 t = bn.getuint256(); 139 return bn.getuint256();
119 // Cap at all-ones (maximum, easiest possible)
120 if (t < diff1 && difficulty < 1.0) t = ~uint256(0);
121 return t;
122} 140}
123 141
124// Expected number of hashes to find one share at a given Stratum difficulty, 142// Expected number of hashes to find one share at a given Stratum difficulty,
@@ -218,6 +236,29 @@ static bool RebuildJob()
218 // From the pool, so the template's payout key is in wallet.dat before 236 // From the pool, so the template's payout key is in wallet.dat before
219 // any miner is handed work against it. 237 // any miner is handed work against it.
220 vector<unsigned char> vchPubKey = GetKeyFromPool(); 238 vector<unsigned char> vchPubKey = GetKeyFromPool();
239 if (vchPubKey.empty()) {
240 // Empty means the wallet could not produce a key: encrypted and
241 // locked, or key generation itself failed. Serialised, an empty
242 // vector is a zero-length push, so the coinbase would become
243 // `<empty> OP_CHECKSIG` -- a script nobody can satisfy. Every
244 // block mined against such a job burns its whole subsidy, while
245 // QueuePayouts still records the full obligation and the miners
246 // are then paid out of coin that block never contributed.
247 // BitcoinMiner refuses to mine in this state; refuse to hand the
248 // pool's miners work in it too. printf and not LogPrint for the
249 // reason given above the deaf-node refusal, and throttled the
250 // same way -- the no-job watchdog retries every few seconds for
251 // as long as the condition lasts.
252 static int64 nLastSaidNoKey = 0;
253 if (GetTime() - nLastSaidNoKey > 60) {
254 nLastSaidNoKey = GetTime();
255 printf("RebuildJob: no wallet key available for the coinbase "
256 "(wallet locked, or key generation failed -- the "
257 "GetKeyFromPool line above says which) -- not issuing "
258 "jobs\n");
259 }
260 return false;
261 }
221 static std::atomic<uint32_t> sExtra{0}; 262 static std::atomic<uint32_t> sExtra{0};
222 uint32_t extraNonce = ++sExtra; 263 uint32_t extraNonce = ++sExtra;
223 264
@@ -236,35 +277,96 @@ static bool RebuildJob()
236 CBlock block; 277 CBlock block;
237 block.vtx.push_back(txNew); 278 block.vtx.push_back(txNew);
238 279
280 // Fill the block the same way BitcoinMiner() does, because this is the
281 // same job: a block that gets mined and submitted for real.
282 //
283 // It used to differ in two ways, and both were bugs.
284 //
285 // One pass over mapTransactions is not enough. That map is keyed by
286 // hash, so it has no dependency order, and a transaction spending the
287 // still-unconfirmed output of another (change from a recent send, most
288 // commonly) fails ConnectInputs whenever its parent happens to sit
289 // later in map order. A single pass then drops it -- and because map
290 // order does not change between rebuilds, every later rebuild drops it
291 // again at the same point. Deterministic exclusion, not bad luck: if
292 // pool operators mine most of the blocks, a transaction of that shape
293 // can stay unconfirmed for as long as its parent does.
294 //
295 // Diagnosis and fix both come from Vic-Nas, who found it in his fork
296 // of this code and described it exactly. The fork has since been
297 // deleted, so this carries it back rather than losing it.
298 int64 nFees = 0;
239 { 299 {
240 CTxDB txdb("r"); 300 CTxDB txdb("r");
241 CRITICAL_BLOCK(cs_mapTransactions) 301 CRITICAL_BLOCK(cs_mapTransactions)
242 { 302 {
243 map<uint256, CTxIndex> pool; 303 map<uint256, CTxIndex> pool;
244 unsigned int sz = 0; 304 vector<char> vfAlreadyAdded(mapTransactions.size());
245 for (auto& kv : mapTransactions) { 305 bool fFoundSomething = true;
246 CTransaction& tx = kv.second; 306 // The cap is MAX_BLOCK_SIZE_GEN, not MAX_SIZE/2. MAX_SIZE is
247 if (tx.IsCoinBase() || !tx.IsFinal()) continue; 307 // the 32 MB serialisation ceiling, so half of it is 16 MB --
248 int64 nFees = 0; 308 // this builder would happily assemble a template that its own
249 map<uint256, CTxIndex> tmp(pool); 309 // CheckBlock() rejects at the 1 MB consensus cap. This is the
250 if (!tx.ConnectInputs(txdb, tmp, CDiskTxPos(1,1,1), 0, nFees, 310 // pool's job builder, so a block over the cap is not one
251 false, true, tx.GetMinFee(block.vtx.size() < 100))) 311 // wasted solve, it is every miner on the pool working a
252 continue; 312 // template the network will reject.
253 pool = tmp; 313 //
254 block.vtx.push_back(tx); 314 // The inner break used to bound the overshoot to one
255 sz += ::GetSerializeSize(tx, SER_NETWORK); 315 // transaction per pass. One transaction is still too many:
256 // Must stay under the consensus cap CheckBlock() enforces. 316 // nothing caps how big a single transaction may be, so that
257 // This is the pool's job builder, so a block over the cap 317 // bound was worth up to MAX_SIZE. Asking whether each one fits
258 // is not just one wasted solve -- it is every miner on the 318 // before appending it bounds the overshoot at zero instead.
259 // pool working a template the network will reject. The 319 //
260 // transaction is already pushed above, so this can overshoot 320 // Seeded with the header, tx count and coinbase, because
261 // by one; half the cap leaves room for that. 321 // CheckBlock() weighs the whole serialized block.
262 if (sz > MAX_BLOCK_SIZE / 2) break; 322 unsigned int sz = ::GetSerializeSize(block, SER_DISK);
323 // And the sigop budget, which CheckBlock() enforces and this
324 // builder never counted against. Same counting function
325 // CheckBlock() sums, seeded from the coinbase because it sums
326 // over all of vtx.
327 unsigned int nSigOps = block.vtx[0].GetSigOpCount();
328 while (fFoundSomething && sz < MAX_BLOCK_SIZE_GEN)
329 {
330 fFoundSomething = false;
331 unsigned int n = 0;
332 for (map<uint256, CTransaction>::iterator mi = mapTransactions.begin();
333 mi != mapTransactions.end(); ++mi, ++n)
334 {
335 if (vfAlreadyAdded[n])
336 continue;
337 CTransaction& tx = (*mi).second;
338 if (tx.IsCoinBase() || !tx.IsFinal()) continue;
339 // Skip rather than stop: one transaction too big for
340 // the room left does not mean the next is. Ordered
341 // before ConnectInputs because ConnectInputs credits
342 // nFees by reference on success, and a transaction we
343 // then decline would leave the coinbase claiming a fee
344 // this block does not collect.
345 unsigned int nTxSize = ::GetSerializeSize(tx, SER_NETWORK);
346 unsigned int nTxSigOps = tx.GetSigOpCount();
347 if (!BlockTemplateFits(sz, nSigOps, nTxSize, nTxSigOps))
348 continue;
349 map<uint256, CTxIndex> tmp(pool);
350 if (!tx.ConnectInputs(txdb, tmp, CDiskTxPos(1,1,1), 0, nFees,
351 false, true, tx.GetMinFee(block.vtx.size() < 100)))
352 continue;
353 swap(pool, tmp);
354 block.vtx.push_back(tx);
355 sz += nTxSize;
356 nSigOps += nTxSigOps;
357 vfAlreadyAdded[n] = true;
358 fFoundSomething = true;
359 }
263 } 360 }
264 } 361 }
265 } 362 }
266 363
267 block.vtx[0].vout[0].nValue = block.GetBlockValue(pindexPrev->nHeight + 1, 0); 364 // And the fees actually go to the coinbase now. nFees was declared
365 // inside the scan loop, so every transaction reset it and the total was
366 // thrown away, while this line passed a hardcoded 0. A pool operator
367 // collected fees into a block and paid none of them to itself. Solo
368 // mining has always passed nFees here; only this builder did not.
369 block.vtx[0].vout[0].nValue = block.GetBlockValue(pindexPrev->nHeight + 1, nFees);
268 block.hashPrevBlock = pindexPrev->GetBlockHash(); 370 block.hashPrevBlock = pindexPrev->GetBlockHash();
269 block.hashMerkleRoot = block.BuildMerkleTree(); 371 block.hashMerkleRoot = block.BuildMerkleTree();
270 block.nTime = max((unsigned int)(pindexPrev->GetMedianTimePast() + 1), 372 block.nTime = max((unsigned int)(pindexPrev->GetMedianTimePast() + 1),
@@ -297,26 +399,406 @@ struct PendingPayout {
297 // supposed to pay it: a 280-block reorg once left ~14.4k BTF of debt 399 // supposed to pay it: a 280-block reorg once left ~14.4k BTF of debt
298 // owed against blocks that no longer existed, which then drained the 400 // owed against blocks that no longer existed, which then drained the
299 // wallet paying fork-era miners out of real-chain coin. 401 // wallet paying fork-era miners out of real-chain coin.
402 //
403 // Upstream keys these on the string "block-<height>". That is the same
404 // bug this fork already paid for, and height keying additionally lets
405 // FindRecordedPoolPayout match the *previous* round's transaction after a
406 // reorg and silently skip a miner, so their `id` is deliberately dropped.
300 int blockHeight = 0; 407 int blockHeight = 0;
301 // Must be explicitly zeroed: uint256's default constructor is empty 408 // Must be explicitly zeroed: uint256's default constructor is empty
302 // (uint256.h:514), so leaving this unassigned gives every ledger entry 409 // (uint256.h:514), so leaving this unassigned gives every ledger entry
303 // loaded from an older file a garbage hash, which then matches no block 410 // loaded from an older file a garbage hash, which then matches no block
304 // and gets voided as "orphaned". That mistake voided a live payout. 411 // and gets voided as "orphaned". That mistake voided a live payout.
305 uint256 blockHash = 0; // zero = entry predates hash keying 412 uint256 blockHash = 0; // zero = entry predates hash keying
306 int matureAtHeight; 413 int matureAtHeight = 0;
307 std::map<std::string, int64> amounts; // address -> satoshis 414 std::map<std::string, int64> amounts; // address -> satoshis
308 int retries = 0; // exec-thread only: SendMoney attempts so far 415 // From upstream: which recipients already have a transaction. This is what
309 int deferrals = 0; // exec-thread only: waits for spendable coin 416 // lets an entry stay on disk until every address in `amounts` is paid,
310 int64 retryAfter = 0; // exec-thread only: unix time, 0 = ready now 417 // instead of being erased the moment the batch is handed off.
418 std::map<std::string, std::string> paidTxids; // address -> txid
419 // Retry and backoff state does NOT live here. It used to be declared here
420 // as well as on PayingRecord, and the copy on this side was never written:
421 // `retryAfter` was declared, serialised, loaded and tested at the flush
422 // gate, and the only assignment to it anywhere was `= 0`, so the gate could
423 // not fire. `retries` and `deferrals` were the same story. The real state
424 // is per (round, recipient) -- which is the granularity a payment can fail
425 // at -- and it lives on PayingRecord, which is now what persists it.
311}; 426};
312static std::vector<PendingPayout> gPendingPayouts; 427static std::vector<PendingPayout> gPendingPayouts;
428static std::mutex gPayoutLedgerMutex;
429// Bumped on every serialisation, under gPayoutLedgerMutex. See PublishVersioned.
430static uint64 gPayoutLedgerSeq = 0;
431// Serialises writers of pending_payouts.json, and guards the sequence number
432// of the image currently on disk. Never taken while a data mutex is held.
433static std::mutex gPayoutLedgerFileMutex;
434static uint64 gPayoutLedgerPublishedSeq = 0;
435
436struct PoolRoundProof {
437 std::string id;
438 int blockHeight;
439 int matureAtHeight;
440 int64 createdAt;
441 std::string blockHash;
442 std::string coinbaseTxid;
443 std::string foundBy;
444 double feePercent;
445 uint64 totalShares;
446 int64 coinbaseValue;
447 int64 subsidy;
448 int64 minerReward;
449 int64 operatorReward;
450 int skippedDust;
451 int64 forfeitedBadAddress;
452 std::map<std::string, uint64> shares;
453 std::map<std::string, int64> payouts;
454 std::map<std::string, std::string> paidTxids;
455 // Recipients the ledger has written off, and why. Without this the public
456 // round proof goes on advertising an address as owed with "txid": null
457 // forever, while the ledger has already dropped the obligation -- the two
458 // records of the same round permanently disagreeing, with the operator
459 // holding the money.
460 std::map<std::string, std::string> forfeited; // address -> reason
461
462 PoolRoundProof()
463 : blockHeight(0), matureAtHeight(0), createdAt(0), feePercent(0.0),
464 totalShares(0), coinbaseValue(0), subsidy(0), minerReward(0), operatorReward(0),
465 skippedDust(0), forfeitedBadAddress(0) {}
466};
467static std::vector<PoolRoundProof> gPoolRounds;
468static std::mutex gPoolRoundsMutex;
469static uint64 gPoolRoundsSeq = 0; // guarded by gPoolRoundsMutex
470static std::mutex gPoolRoundsFileMutex;
471static uint64 gPoolRoundsPublishedSeq = 0; // guarded by gPoolRoundsFileMutex
472static const size_t MAX_POOL_ROUNDS = 200;
313 473
314static std::string PayoutFilePath() 474static std::string PayoutFilePath()
315{ 475{
316 return GetAppDir() + "/pending_payouts.json"; 476 return GetAppDir() + "/pending_payouts.json";
317} 477}
318 478
319static void SavePendingPayouts() 479static std::string PoolRoundsFilePath()
480{
481 if (!strPoolRoundsFile.empty())
482 return strPoolRoundsFile;
483 return GetAppDir() + "/pool_rounds.json";
484}
485
486// ---------------------------------------------------------------------------
487// Atomic, ordered publication of the payout state files
488// ---------------------------------------------------------------------------
489//
490// Every one of these files is a full dump of an in-memory structure, written
491// tmp-then-rename so a crash cannot truncate it. Two things have to be true
492// for that to actually be safe, and only the first was:
493//
494// 1. the rename must be atomic (it is), and
495// 2. exactly one thread at a time may be writing the temp file, and a write
496// must never publish an OLDER image over a newer one.
497//
498// (2) was broken for pending_payouts.json. The event loop wrote it with
499// gPayoutLedgerMutex held, the payout thread wrote it with the mutex
500// released, and nothing serialised the two: both opened the same
501// "<path>.tmp" with independent handles and both renamed it. The interleaved
502// file that lands is either a mixture of two images (LoadPendingPayouts then
503// throws mid-loop and comes up with a nearly empty ledger -- every obligation
504// not yet exploded, gone) or a whole newly-queued round silently overwritten
505// by an older snapshot that predates it.
506//
507// The fix is a per-file write mutex, held only across the I/O and never
508// across a data-structure mutex, plus a version on the snapshot. Serialising
509// happens under the data mutex and stamps a monotonically increasing sequence
510// number; publishing happens with the data mutex released and refuses any
511// blob whose sequence number is not newer than what is already on disk. So
512// writers can queue in any order and finish in any order -- the file only
513// ever moves forwards.
514//
515// The other half of the rule is that no thread waits for I/O while holding
516// gPayoutLedgerMutex, gPayoutExecMutex or gPoolRoundsMutex, which is the
517// contention the merge was right to remove from the event loop. That was
518// claimed here before it was true: pool_status.json is written by the same
519// WriteFileAtomic and RefreshStats held gPayoutLedgerMutex at function scope
520// across the call, so once the writes gained fsync() the event loop was doing
521// two of them every ten seconds with the payout thread's mutex held -- worse
522// than the buffered write the merge replaced. The status file now goes through
523// the same serialise-under-the-lock / publish-outside-it split as the ledgers,
524// and the rule holds for all four files. It is checked, not asserted: see the
525// "no payout mutex is held across a publish" section of the payout-ledger
526// self-test.
527
528// A serialised file image plus the version of the in-memory state it was
529// taken from.
530struct VersionedBlob {
531 std::string blob;
532 uint64 seq = 0;
533};
534
535// Push a file's contents out of the C library's buffer AND out of the kernel's
536// page cache, so the bytes exist on the device before anything points at them.
537static bool SyncFileHandle(FILE* f)
538{
539 if (fflush(f) != 0)
540 return false;
541#ifdef _WIN32
542 return _commit(_fileno(f)) == 0;
543#else
544 return fsync(fileno(f)) == 0;
545#endif
546}
547
548// Make the directory entry that rename() created durable too.
549//
550// A tmp+rename is only atomic with respect to a crash of the process. Against
551// a power cut or a hard VM reset the rename can reach the platter while the
552// data it points at has not, and the file that comes back is short or empty.
553// For pending_payouts.json that is not a corrupt file, it is a silent one:
554// LoadPendingPayouts catches the parse error, logs a single warning, and the
555// node comes up owing nobody anything.
556//
557// Best effort -- a filesystem that does not allow the directory to be opened
558// for this is not a reason to refuse to write the ledger.
559static void SyncDirectoryOf(const std::string& path)
560{
561#ifndef _WIN32
562 size_t slash = path.find_last_of('/');
563 std::string dir = (slash == std::string::npos) ? std::string(".")
564 : path.substr(0, slash);
565 if (dir.empty())
566 dir = "/";
567 int fd = open(dir.c_str(), O_RDONLY);
568 if (fd < 0)
569 return;
570 fsync(fd);
571 close(fd);
572#endif
573}
574
575// Replace `path` with `data` atomically and durably. Caller must hold the
576// write mutex for `path`: the temp name is derived from the target, so two
577// threads in here at once for one path is precisely the race this exists to
578// prevent.
579//
580// Every failure removes the temp file before returning. It used to be left
581// where it fell, so a disk that filled up once left a short, valid-looking
582// "pending_payouts.json.tmp" next to the good ledger for as long as the node
583// ran. That file is a trap: it has the name of a half-finished save, and an
584// operator cleaning up after an incident is exactly the person who will move
585// it into place.
586//
587// Failures print unconditionally rather than through LogPrint(). These three
588// files are written under two different debug categories ("payout" for the
589// ledgers, "pool" for the status file), and a write failure is not a trace of
590// normal operation -- it is the file the operator is about to read not being
591// there. Gating it behind a category that had to be chosen before the failure
592// happened is how "could not replace pool_status.json" went missing.
593static bool WriteFileAtomic(const std::string& path, const std::string& data,
594 const char* what)
595{
596 std::string tmp = path + ".tmp";
597 FILE* f = fopen(tmp.c_str(), "w");
598 if (!f) {
599 printf("[payout] WARNING: could not write %s (%s)\n",
600 tmp.c_str(), what);
601 return false;
602 }
603 if (!data.empty() && fwrite(data.c_str(), 1, data.size(), f) != data.size()) {
604 printf("[payout] WARNING: short write to %s (%s)\n",
605 tmp.c_str(), what);
606 fclose(f);
607 remove(tmp.c_str());
608 return false;
609 }
610 // Before the rename, not after: publishing a name that points at nothing
611 // is the failure this ordering exists to prevent. If the data cannot be
612 // made durable, do not rename -- the previous file is still intact and
613 // still correct, and the next write will try again.
614 if (!SyncFileHandle(f)) {
615 printf("[payout] WARNING: could not flush %s to disk (%s)"
616 " -- keeping the previous file\n", tmp.c_str(), what);
617 fclose(f);
618 remove(tmp.c_str());
619 return false;
620 }
621 // fclose() is where buffered data actually reaches the OS, so a failure
622 // here means the .tmp is short -- renaming it over the real file would
623 // destroy the good copy, which is exactly what these files exist to
624 // prevent.
625 if (fclose(f) != 0) {
626 printf("[payout] WARNING: could not close %s (%s)\n",
627 tmp.c_str(), what);
628 remove(tmp.c_str());
629 return false;
630 }
631#ifdef _WIN32
632 // rename() on Windows will not replace an existing file.
633 remove(path.c_str());
634#endif
635 if (rename(tmp.c_str(), path.c_str()) != 0) {
636 printf("[payout] WARNING: could not replace %s (%s)\n",
637 path.c_str(), what);
638 remove(tmp.c_str());
639 return false;
640 }
641 SyncDirectoryOf(path);
642 return true;
643}
644
645// Paths whose loader threw part-way through, and which must therefore never be
646// written again by this process.
647//
648// All three loaders push entries into the live structure INSIDE the try, so an
649// exception half way down a file leaves a PARTIAL ledger in memory -- the
650// entries before the bad one, and nothing after it. The first publish after
651// that then writes the partial state over the file, and the only record of
652// what was owed is gone. ReconcilePayouts publishes unconditionally at
653// startup, so "the first publish" is a few milliseconds later, before an
654// operator could possibly intervene. a8e2f58 named this failure mode in its
655// own commit message and then only fixed the write side.
656//
657// The file is renamed out of the way instead of being left in place, so the
658// evidence survives even if the operator restarts into the same fault, and so
659// a node that comes up owing nobody anything is visibly missing its ledger
660// rather than quietly appearing to have none.
661static std::set<std::string> gRefusedFiles;
662static std::mutex gRefusedFilesMutex;
663
664static bool FileIsRefused(const std::string& path)
665{
666 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
667 return gRefusedFiles.count(path) > 0;
668}
669
670// Quarantine `path` and refuse to write it for the rest of the process.
671//
672// Prints unconditionally: an operator who has just lost a payout ledger must
673// be told at whatever log settings they were already running with, and the
674// name it was moved to is the only way back to the data.
675static void RefuseFileAfterLoadFailure(const std::string& path, const char* what,
676 const char* why)
677{
678 {
679 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
680 gRefusedFiles.insert(path);
681 }
682 std::string bad = strprintf("%s.bad.%lld", path.c_str(), (long long)GetTime());
683 bool moved = (rename(path.c_str(), bad.c_str()) == 0);
684 printf("[payout] ERROR: %s (%s) did not parse: %s\n"
685 "[payout] ERROR: what loaded before the error is a PARTIAL picture,"
686 " so this file will not be written again by this process --"
687 " restart once the cause is fixed\n"
688 "[payout] ERROR: the file %s\n",
689 path.c_str(), what, why,
690 moved ? strprintf("has been kept as %s", bad.c_str()).c_str()
691 : "could NOT be renamed out of the way -- copy it now");
692}
693
694// Publish a versioned image of some in-memory state to `path`.
695//
696// `fileMutex` serialises writers of `path` and guards `publishedSeq`. A blob
697// older than what is already published is dropped: these files are whole-state
698// dumps, so a higher sequence number is by construction a later state of the
699// same structure and publishing it loses nothing. On a write failure
700// `publishedSeq` is deliberately left alone, so the next (newer) write still
701// gets a chance to land.
702static void PublishVersioned(const std::string& path, const VersionedBlob& v,
703 std::mutex& fileMutex, uint64& publishedSeq,
704 const char* what)
705{
706 // A default-constructed blob is not a snapshot of anything; it is a
707 // VersionedBlob nobody filled in. Today they are already dropped, because
708 // every serialiser stamps ++gSeq and so never produces 0, and 0 fails the
709 // seq <= publishedSeq test below -- but only for as long as publishedSeq
710 // starts at 0 and nobody writes `seq = gSeq++`. One character, and the
711 // ledger file becomes "[]". Say it out loud instead of relying on it.
712 if (!v.seq)
713 return;
714 // A file whose loader threw is not a file this process knows the contents
715 // of. Writing it would replace a full record of what was owed with
716 // whatever happened to parse before the error.
717 if (FileIsRefused(path))
718 return;
719 std::lock_guard<std::mutex> lk(fileMutex);
720 if (v.seq <= publishedSeq)
721 return;
722 if (!WriteFileAtomic(path, v.blob, what))
723 return;
724 publishedSeq = v.seq;
725}
726
727// Identity of a payout round. Used as the pool-round proof id and, more
728// importantly, as the `pool_payout_id` written into wallet transaction
729// metadata and matched by FindRecordedPoolPayout().
730//
731// Derived from the block HASH, not from its height alone. Upstream uses
732// "block-<height>", which names two different blocks identically across a
733// reorg -- and because this id is what the crash-recovery probe matches on, a
734// height-keyed id lets that probe find the *previous* round's transaction and
735// conclude a miner has already been paid when they have not. The height stays
736// in the string for readability only.
737//
738// A zero hash means the entry predates hash keying; those keep the old form so
739// anything already recorded against them still matches.
740static std::string PayoutIdForBlock(int blockHeight, const uint256& blockHash)
741{
742 if (blockHash == 0)
743 return strprintf("block-%d", blockHeight);
744 return strprintf("block-%d-%s", blockHeight,
745 blockHash.ToString().substr(0, 16).c_str());
746}
747
748// Do these two references name the same payout round?
749//
750// The block hash alone does not decide it, and every ledger lookup used to
751// assume it did. Entries grandfathered in from a file written before payouts
752// were keyed by hash all carry hash 0, so under a hash-only comparison the
753// first zero-hash entry answers for every other one: RecordPayoutTxid writes
754// a txid against the wrong round (which is then erased as "fully paid" while
755// the real debt still stands), and ReconcilePayouts drops a live in-flight
756// record because a *different* zero-hash round happened to pay that address.
757//
758// A non-zero hash is unique and decides on its own -- that is the whole point
759// of keying on it. A zero hash falls back to the height, which
760// LoadPendingPayouts supplies for any entry that can name its own maturity.
761// Mixed hash-ness never matches: refusing to match is the safe direction for
762// every caller here, because each of them treats a match as "this debt is
763// already dealt with".
764static bool SamePayoutRound(int aHeight, const uint256& aHash,
765 int bHeight, const uint256& bHash)
766{
767 if (aHash != bHash)
768 return false;
769 if (aHash != 0)
770 return true;
771 return aHeight == bHeight;
772}
773
774// An entry with neither a hash nor a height cannot be told apart from any
775// other entry in the same state, so nothing may be recorded against it: a
776// txid written to the wrong one reports a debt as settled that is still owed.
777// Such an entry is never paid and is logged instead -- refusing to act is the
778// safe failure, and no file this tree has ever written can produce one.
779static bool PayoutRoundIdentifiable(int height, const uint256& hash)
780{
781 return hash != 0 || height != 0;
782}
783
784static bool PayoutFullyPaid(const PendingPayout& pp)
785{
786 for (const auto& kv : pp.amounts)
787 if (pp.paidTxids.count(kv.first) == 0)
788 return false;
789 return true;
790}
791
792// Serialise the ledger and stamp it with the version it was taken at. Caller
793// must hold gPayoutLedgerMutex.
794//
795// Split from the file write so no thread holds the ledger mutex across the
796// I/O. Upstream writes the file inside the per-recipient loop with
797// gPayoutLedgerMutex held, while the event loop's FlushMaturePayouts blocks on
798// that same mutex -- one slow fsync per recipient, on the thread the miners'
799// I/O shares. The sequence number is what lets the write happen later, and
800// possibly out of order, without a stale image reaching the disk.
801static VersionedBlob SerializePendingPayoutsLocked()
320{ 802{
321 nlohmann::json arr = nlohmann::json::array(); 803 nlohmann::json arr = nlohmann::json::array();
322 for (const auto& pp : gPendingPayouts) { 804 for (const auto& pp : gPendingPayouts) {
@@ -324,21 +806,56 @@ static void SavePendingPayouts()
324 obj["blockHeight"] = pp.blockHeight; 806 obj["blockHeight"] = pp.blockHeight;
325 obj["blockHash"] = pp.blockHash.ToString(); 807 obj["blockHash"] = pp.blockHash.ToString();
326 obj["matureAtHeight"] = pp.matureAtHeight; 808 obj["matureAtHeight"] = pp.matureAtHeight;
809 // No retryAfter: it was never written with anything but zero, and the
810 // retry clock that is actually used is on PayingRecord. Files that
811 // still carry the key load fine (it is ignored), and a build that
812 // expects it defaults an absent one to 0, which means "ready now" --
813 // the same answer it always got.
327 nlohmann::json amounts = nlohmann::json::object(); 814 nlohmann::json amounts = nlohmann::json::object();
328 for (const auto& kv : pp.amounts) 815 for (const auto& kv : pp.amounts)
329 amounts[kv.first] = kv.second; 816 amounts[kv.first] = kv.second;
330 obj["amounts"] = amounts; 817 obj["amounts"] = amounts;
818 nlohmann::json paid = nlohmann::json::object();
819 for (const auto& kv : pp.paidTxids)
820 paid[kv.first] = kv.second;
821 obj["paidTxids"] = paid;
331 arr.push_back(obj); 822 arr.push_back(obj);
332 } 823 }
333 std::string path = PayoutFilePath(); 824 VersionedBlob v;
334 FILE* f = fopen(path.c_str(), "w"); 825 v.blob = arr.dump(2);
335 if (f) { 826 v.seq = ++gPayoutLedgerSeq;
336 std::string s = arr.dump(2); 827 return v;
337 fwrite(s.c_str(), 1, s.size(), f); 828}
338 fclose(f); 829
339 } else { 830// Publish a ledger snapshot. Holds no data mutex; safe -- and intended -- to
340 LogPrint("payout", "[payout] WARNING: could not write %s\n", path.c_str()); 831// call with gPayoutLedgerMutex released.
341 } 832static void PublishPendingPayouts(const VersionedBlob& v)
833{
834 PublishVersioned(PayoutFilePath(), v, gPayoutLedgerFileMutex,
835 gPayoutLedgerPublishedSeq, "pending payout ledger");
836}
837
838// Parse a persisted block-hash field, refusing to guess. SetHex() reads
839// whatever hex prefix it finds and stops at the first character it does not
840// recognise, so a corrupt value does not fail to parse -- it parses as a
841// DIFFERENT hash. Both shapes of that are wrong in a dangerous direction: a
842// value with a hex-looking prefix ("deadbeef...") becomes a hash no block
843// ever had, which PayoutBlockStillOnChain reads as an orphaned block and
844// FlushMaturePayouts then VOIDS the debt; a value with no hex prefix at all
845// becomes 0, which is grandfathered past the orphan check on the strength
846// of garbage. A field that is not the 64-digit hex string ToString() wrote
847// is corruption, and corruption is a load failure like any other parse
848// error -- the file is quarantined, not read around.
849static uint256 ParsePayoutBlockHash(const std::string& str)
850{
851 if (str.size() != 64 ||
852 str.find_first_not_of("0123456789abcdefABCDEF") != std::string::npos)
853 throw std::runtime_error(
854 strprintf("blockHash \"%s\" is not a 64-digit hex string",
855 str.c_str()));
856 uint256 h;
857 h.SetHex(str);
858 return h;
342} 859}
343 860
344static void LoadPendingPayouts() 861static void LoadPendingPayouts()
@@ -354,6 +871,7 @@ static void LoadPendingPayouts()
354 fclose(f); 871 fclose(f);
355 try { 872 try {
356 nlohmann::json arr = nlohmann::json::parse(s); 873 nlohmann::json arr = nlohmann::json::parse(s);
874 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
357 for (const auto& obj : arr) { 875 for (const auto& obj : arr) {
358 PendingPayout pp; 876 PendingPayout pp;
359 pp.matureAtHeight = obj["matureAtHeight"].get<int>(); 877 pp.matureAtHeight = obj["matureAtHeight"].get<int>();
@@ -361,18 +879,33 @@ static void LoadPendingPayouts()
361 // field. Leave blockHash zero and they are grandfathered past the 879 // field. Leave blockHash zero and they are grandfathered past the
362 // orphan check rather than voided on a guess. 880 // orphan check rather than voided on a guess.
363 if (obj.contains("blockHash") && obj["blockHash"].is_string()) 881 if (obj.contains("blockHash") && obj["blockHash"].is_string())
364 pp.blockHash.SetHex(obj["blockHash"].get<std::string>()); 882 pp.blockHash =
883 ParsePayoutBlockHash(obj["blockHash"].get<std::string>());
365 if (obj.contains("blockHeight") && obj["blockHeight"].is_number()) 884 if (obj.contains("blockHeight") && obj["blockHeight"].is_number())
366 pp.blockHeight = obj["blockHeight"].get<int>(); 885 pp.blockHeight = obj["blockHeight"].get<int>();
886 // Upstream's tolerance for an entry with no usable height, kept
887 // because it costs nothing and an entry that cannot name its block
888 // is worse than one derived from its maturity. COINBASE_MATURITY
889 // and not COINBASE_SPENDABLE_DEPTH: this reverses whatever the
890 // writer of *that* file used, and only files old enough to lack
891 // blockHeight are affected -- they were written when maturity was
892 // still COINBASE_MATURITY.
893 if (pp.blockHeight == 0 && pp.matureAtHeight > COINBASE_MATURITY)
894 pp.blockHeight = pp.matureAtHeight - COINBASE_MATURITY;
895 // "retryAfter" is deliberately not read. Files written by the
896 // merged build carry it, always as 0, and there is nowhere left to
897 // put it -- the retry clock is PayingRecord's.
367 for (auto it = obj["amounts"].begin(); it != obj["amounts"].end(); ++it) 898 for (auto it = obj["amounts"].begin(); it != obj["amounts"].end(); ++it)
368 pp.amounts[it.key()] = it.value().get<int64>(); 899 pp.amounts[it.key()] = it.value().get<int64>();
900 if (obj.contains("paidTxids") && obj["paidTxids"].is_object())
901 for (auto it = obj["paidTxids"].begin(); it != obj["paidTxids"].end(); ++it)
902 pp.paidTxids[it.key()] = it.value().get<std::string>();
369 gPendingPayouts.push_back(pp); 903 gPendingPayouts.push_back(pp);
370 } 904 }
371 LogPrint("payout", "[payout] loaded %zu pending payout(s) from disk\n", 905 LogPrint("payout", "[payout] loaded %zu pending payout(s) from disk\n",
372 gPendingPayouts.size()); 906 gPendingPayouts.size());
373 } catch (const std::exception& e) { 907 } catch (const std::exception& e) {
374 LogPrint("payout", "[payout] WARNING: failed to parse %s: %s\n", 908 RefuseFileAfterLoadFailure(path, "pending payout ledger", e.what());
375 path.c_str(), e.what());
376 } 909 }
377} 910}
378 911
@@ -388,6 +921,18 @@ static void LoadPendingPayouts()
388// and reconciling against the wallet on startup, is what makes a restart 921// and reconciling against the wallet on startup, is what makes a restart
389// survivable. 922// survivable.
390struct PayingRecord { 923struct PayingRecord {
924 // Identity of this SLOT, distinct from the identity of the payment it
925 // describes. ForgetPayment and ReschedulePayment used to find their record
926 // by comparing every field, which is only an identity while no two records
927 // can be equal -- and a restart used to produce byte-identical duplicates
928 // (see FlushMaturePayouts), so the first match absorbed every update and
929 // the second was never touched again.
930 //
931 // Process-local and deliberately not persisted: gPaying is rebuilt from
932 // disk at every start, so an id only ever has to name a slot within one
933 // run, and reading ids back would mean trusting a file to contain no
934 // duplicates. Assigned under gPayoutExecMutex from gPayingNextId.
935 uint64 id = 0;
391 int blockHeight = 0; 936 int blockHeight = 0;
392 uint256 blockHash = 0; // zero = predates hash keying, see PendingPayout 937 uint256 blockHash = 0; // zero = predates hash keying, see PendingPayout
393 int matureAtHeight = 0; 938 int matureAtHeight = 0;
@@ -396,18 +941,66 @@ struct PayingRecord {
396 int retries = 0; 941 int retries = 0;
397 int deferrals = 0; 942 int deferrals = 0;
398 int64 retryAfter = 0; 943 int64 retryAfter = 0;
944 // Ask the wallet whether this payment already went out, once, before the
945 // first send attempt. Upstream asks on every attempt: FindRecordedPoolPayout
946 // is a linear scan of mapWallet under cs_mapWallet, per recipient, so with
947 // 2000 rotating payout addresses and a retry budget of 1440 that is a
948 // scan per address per minute for a day. The question it answers -- "did a
949 // previous process send this before it died?" -- can only change across a
950 // restart, and every restart re-sets this flag (ReconcilePayouts), so
951 // asking once per record is the whole of the answer.
952 //
953 // Not persisted: a record read back from disk is by definition a restart.
954 bool fProbeWallet = true;
399}; 955};
400static std::vector<PayingRecord> gPaying; // guarded by gPayoutExecMutex 956static std::vector<PayingRecord> gPaying; // guarded by gPayoutExecMutex
401static std::mutex gPayoutExecMutex; 957static std::mutex gPayoutExecMutex;
958static uint64 gPayingNextId = 0; // guarded by gPayoutExecMutex
959static uint64 gPayingSeq = 0; // guarded by gPayoutExecMutex
960static std::mutex gPayingFileMutex;
961static uint64 gPayingPublishedSeq = 0; // guarded by gPayingFileMutex
402static std::atomic<bool> gPayoutThreadRunning{false}; 962static std::atomic<bool> gPayoutThreadRunning{false};
403 963
964// Upstream's gPayoutExecQueue/gPayoutExecMutex/gPayoutThreadRunning are
965// declared above with gPaying: the in-flight list IS the exec queue in this
966// tree, at per-recipient rather than per-round granularity, so a second queue
967// would be a second answer to "who is owed money". Only the dedupe set is new.
968//
969// Ids of rounds currently exploded into gPaying, so a flush cannot queue the
970// same round twice. Guarded by gPayoutExecMutex. Seeded from the restored
971// in-flight list by ReconcilePayouts -- it is memory-only, and leaving it
972// empty at startup was half of why a restart duplicated everything in flight.
973static std::set<std::string> gQueuedPayouts;
974
975// Is this exact payment already in flight? gPaying is keyed on
976// (round, address): the ledger holds one amount per address per round, so two
977// records for that pair are two standing orders to send the same money.
978// Caller must hold gPayoutExecMutex.
979static bool PaymentInFlightLocked(int blockHeight, const uint256& blockHash,
980 const std::string& address)
981{
982 for (const PayingRecord& e : gPaying)
983 if (e.address == address
984 && SamePayoutRound(e.blockHeight, e.blockHash, blockHeight, blockHash))
985 return true;
986 return false;
987}
988
404static std::string PayingFilePath() 989static std::string PayingFilePath()
405{ 990{
406 return GetAppDir() + "/paying_payouts.json"; 991 return GetAppDir() + "/paying_payouts.json";
407} 992}
408 993
409// Caller must hold gPayoutExecMutex. 994// Serialise the in-flight list. Caller must hold gPayoutExecMutex.
410static void SavePayingLocked() 995//
996// gPayoutExecMutex already serialises every writer of this file, so it never
997// had the interleaving race pending_payouts.json had. It is still split the
998// same way for a different reason: FlushMaturePayouts runs on the event loop
999// and holds gPayoutExecMutex, so writing (and, since this series, fsyncing)
1000// the file under that mutex parks miner I/O behind the payout thread's disk.
1001// The version stamp keeps the file monotonic now that the write is outside
1002// the mutex that orders the writers.
1003static VersionedBlob SerializePayingLocked()
411{ 1004{
412 nlohmann::json arr = nlohmann::json::array(); 1005 nlohmann::json arr = nlohmann::json::array();
413 for (const auto& r : gPaying) { 1006 for (const auto& r : gPaying) {
@@ -419,24 +1012,24 @@ static void SavePayingLocked()
419 obj["amount"] = r.amount; 1012 obj["amount"] = r.amount;
420 obj["retries"] = r.retries; 1013 obj["retries"] = r.retries;
421 obj["deferrals"] = r.deferrals; 1014 obj["deferrals"] = r.deferrals;
1015 // The live retry clock, which used not to be written at all: every
1016 // restart handed every backing-off payment a retryAfter of 0 and put
1017 // the whole set into the very next 500 ms batch, so a node restarted
1018 // in a loop retried continuously instead of once a minute.
1019 obj["retryAfter"] = r.retryAfter;
422 arr.push_back(obj); 1020 arr.push_back(obj);
423 } 1021 }
424 std::string path = PayingFilePath(); 1022 VersionedBlob v;
425 // Write to a temporary file and rename, so a crash mid-write cannot leave 1023 v.blob = arr.dump(2);
426 // a truncated list of who is still owed money. 1024 v.seq = ++gPayingSeq;
427 std::string tmp = path + ".tmp"; 1025 return v;
428 FILE* f = fopen(tmp.c_str(), "w"); 1026}
429 if (!f) { 1027
430 LogPrint("payout", "[payout] WARNING: could not write %s\n", tmp.c_str()); 1028// Publish an in-flight-list snapshot. Call with gPayoutExecMutex released.
431 return; 1029static void PublishPaying(const VersionedBlob& v)
432 } 1030{
433 std::string s = arr.dump(2); 1031 PublishVersioned(PayingFilePath(), v, gPayingFileMutex,
434 fwrite(s.c_str(), 1, s.size(), f); 1032 gPayingPublishedSeq, "in-flight payment list");
435 fflush(f);
436 fclose(f);
437 if (rename(tmp.c_str(), path.c_str()) != 0)
438 LogPrint("payout", "[payout] WARNING: could not rename %s -> %s\n",
439 tmp.c_str(), path.c_str());
440} 1033}
441 1034
442static void LoadPaying() 1035static void LoadPaying()
@@ -452,26 +1045,209 @@ static void LoadPaying()
452 fclose(f); 1045 fclose(f);
453 try { 1046 try {
454 nlohmann::json arr = nlohmann::json::parse(s); 1047 nlohmann::json arr = nlohmann::json::parse(s);
1048 // gPaying and gPayingNextId are both annotated as guarded by
1049 // gPayoutExecMutex and this was mutating them without it. Safe today
1050 // -- LoadPaying only runs at startup, before the payout thread is
1051 // spawned -- but "safe because of where it is called from" is a fact
1052 // about the caller, not about this function, and the annotation is
1053 // what every other reader will trust. The mutex is uncontended here by
1054 // construction, so taking it costs nothing.
1055 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
455 for (const auto& obj : arr) { 1056 for (const auto& obj : arr) {
456 PayingRecord r; 1057 PayingRecord r;
457 r.address = obj["address"].get<std::string>(); 1058 r.address = obj["address"].get<std::string>();
458 r.amount = obj["amount"].get<int64>(); 1059 r.amount = obj["amount"].get<int64>();
459 r.matureAtHeight = obj["matureAtHeight"].get<int>(); 1060 r.matureAtHeight = obj["matureAtHeight"].get<int>();
460 if (obj.contains("blockHash") && obj["blockHash"].is_string()) 1061 if (obj.contains("blockHash") && obj["blockHash"].is_string())
461 r.blockHash.SetHex(obj["blockHash"].get<std::string>()); 1062 r.blockHash =
1063 ParsePayoutBlockHash(obj["blockHash"].get<std::string>());
462 if (obj.contains("blockHeight") && obj["blockHeight"].is_number()) 1064 if (obj.contains("blockHeight") && obj["blockHeight"].is_number())
463 r.blockHeight = obj["blockHeight"].get<int>(); 1065 r.blockHeight = obj["blockHeight"].get<int>();
464 if (obj.contains("retries") && obj["retries"].is_number()) 1066 if (obj.contains("retries") && obj["retries"].is_number())
465 r.retries = obj["retries"].get<int>(); 1067 r.retries = obj["retries"].get<int>();
466 if (obj.contains("deferrals") && obj["deferrals"].is_number()) 1068 if (obj.contains("deferrals") && obj["deferrals"].is_number())
467 r.deferrals = obj["deferrals"].get<int>(); 1069 r.deferrals = obj["deferrals"].get<int>();
1070 // Absent in files written before this change; 0 means ready now,
1071 // which is exactly the behaviour those files used to get.
1072 if (obj.contains("retryAfter") && obj["retryAfter"].is_number())
1073 r.retryAfter = obj["retryAfter"].get<int64>();
1074 // Collapse duplicates as they are read. A build without the
1075 // ReconcilePayouts seeding below wrote a second (and after another
1076 // restart, a third) byte-identical record for every unpaid
1077 // recipient, so a file written by one of those has to be cleaned
1078 // up here or the pathology is inherited. The retry and deferral
1079 // counts are merged upwards so collapsing does not hand the
1080 // survivor a fresh budget.
1081 PayingRecord* dup = NULL;
1082 for (PayingRecord& e : gPaying)
1083 if (e.address == r.address
1084 && SamePayoutRound(e.blockHeight, e.blockHash,
1085 r.blockHeight, r.blockHash)) {
1086 dup = &e;
1087 break;
1088 }
1089 if (dup) {
1090 dup->retries = std::max(dup->retries, r.retries);
1091 dup->deferrals = std::max(dup->deferrals, r.deferrals);
1092 dup->retryAfter = std::max(dup->retryAfter, r.retryAfter);
1093 LogPrint("payout", "[payout] collapsing duplicate in-flight"
1094 " record for %s to %s -- one payment, one record\n",
1095 FormatMoney(r.amount).c_str(), r.address.c_str());
1096 continue;
1097 }
1098 r.id = ++gPayingNextId;
468 gPaying.push_back(r); 1099 gPaying.push_back(r);
469 } 1100 }
470 LogPrint("payout", "[payout] loaded %zu in-flight payment(s) from disk\n", 1101 LogPrint("payout", "[payout] loaded %zu in-flight payment(s) from disk\n",
471 gPaying.size()); 1102 gPaying.size());
472 } catch (const std::exception& e) { 1103 } catch (const std::exception& e) {
473 LogPrint("payout", "[payout] WARNING: failed to parse %s: %s\n", 1104 RefuseFileAfterLoadFailure(path, "in-flight payment list", e.what());
474 path.c_str(), e.what()); 1105 }
1106}
1107
1108// ---------------------------------------------------------------------------
1109// Pool round proofs (upstream cbdd06d). Disjoint from the in-flight ledger
1110// above -- this is the operator's published record of how a round was split,
1111// not an obligation to pay. Both sides added code at this offset; both are
1112// kept.
1113// ---------------------------------------------------------------------------
1114static nlohmann::json PoolRoundToJson(const PoolRoundProof& r)
1115{
1116 nlohmann::json j;
1117 j["id"] = r.id;
1118 j["blockHeight"] = r.blockHeight;
1119 j["matureAtHeight"] = r.matureAtHeight;
1120 j["createdAt"] = r.createdAt;
1121 j["blockHash"] = r.blockHash;
1122 j["coinbaseTxid"] = r.coinbaseTxid;
1123 j["foundBy"] = r.foundBy;
1124 j["feePercent"] = r.feePercent;
1125 j["totalShares"] = r.totalShares;
1126 j["coinbaseValueSatoshis"] = r.coinbaseValue;
1127 j["coinbaseValueBtf"] = FormatMoney(r.coinbaseValue);
1128 j["subsidySatoshis"] = r.subsidy;
1129 j["subsidyBtf"] = FormatMoney(r.subsidy);
1130 j["minerRewardSatoshis"] = r.minerReward;
1131 j["minerRewardBtf"] = FormatMoney(r.minerReward);
1132 j["operatorRewardSatoshis"] = r.operatorReward;
1133 j["operatorRewardBtf"] = FormatMoney(r.operatorReward);
1134 j["skippedDust"] = r.skippedDust;
1135 j["forfeitedBadAddressSatoshis"] = r.forfeitedBadAddress;
1136 j["forfeitedBadAddressBtf"] = FormatMoney(r.forfeitedBadAddress);
1137
1138 j["shares"] = nlohmann::json::object();
1139 for (const auto& kv : r.shares)
1140 j["shares"][kv.first] = kv.second;
1141
1142 j["payouts"] = nlohmann::json::object();
1143 for (const auto& kv : r.payouts) {
1144 nlohmann::json p;
1145 p["amountSatoshis"] = kv.second;
1146 p["amountBtf"] = FormatMoney(kv.second);
1147 std::map<std::string, std::string>::const_iterator tx = r.paidTxids.find(kv.first);
1148 if (tx != r.paidTxids.end())
1149 p["txid"] = tx->second;
1150 else
1151 p["txid"] = nullptr;
1152 // Only emitted when it happened, so the common shape of the file is
1153 // unchanged for anything already parsing it.
1154 std::map<std::string, std::string>::const_iterator fo = r.forfeited.find(kv.first);
1155 if (fo != r.forfeited.end()) {
1156 p["forfeited"] = true;
1157 p["forfeitReason"] = fo->second;
1158 }
1159 j["payouts"][kv.first] = p;
1160 }
1161 return j;
1162}
1163
1164// Serialise the published round proofs. Caller must hold gPoolRoundsMutex.
1165//
1166// Like the in-flight list, gPoolRoundsMutex already serialises the writers
1167// (RecordPoolRound on the event loop, MarkPoolRoundPaid on the payout thread),
1168// so this file never had the interleaving race either -- but the event loop
1169// should not be waiting on the payout thread's fsync to publish a round, so
1170// the write moves outside the mutex on the same versioned terms.
1171static VersionedBlob SerializePoolRoundsLocked()
1172{
1173 nlohmann::json root;
1174 root["schema"] = "bitflash-pool-rounds-1";
1175 root["updatedAt"] = GetTime();
1176 root["rounds"] = nlohmann::json::array();
1177 for (const PoolRoundProof& r : gPoolRounds)
1178 root["rounds"].push_back(PoolRoundToJson(r));
1179
1180 VersionedBlob v;
1181 v.blob = root.dump(2);
1182 v.seq = ++gPoolRoundsSeq;
1183 return v;
1184}
1185
1186// Publish a round-proof snapshot. Call with gPoolRoundsMutex released.
1187static void PublishPoolRounds(const VersionedBlob& v)
1188{
1189 PublishVersioned(PoolRoundsFilePath(), v, gPoolRoundsFileMutex,
1190 gPoolRoundsPublishedSeq, "pool round proofs");
1191}
1192
1193static void LoadPoolRounds()
1194{
1195 std::string path = PoolRoundsFilePath();
1196 FILE* f = fopen(path.c_str(), "r");
1197 if (!f) return;
1198 fseek(f, 0, SEEK_END);
1199 long sz = ftell(f);
1200 fseek(f, 0, SEEK_SET);
1201 std::string s(sz, '\0');
1202 fread(&s[0], 1, sz, f);
1203 fclose(f);
1204
1205 try {
1206 nlohmann::json root = nlohmann::json::parse(s);
1207 nlohmann::json arr = root.is_array() ? root : root["rounds"];
1208 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1209 for (const auto& obj : arr) {
1210 PoolRoundProof r;
1211 if (obj.contains("id") && obj["id"].is_string()) r.id = obj["id"].get<std::string>();
1212 if (obj.contains("blockHeight")) r.blockHeight = obj["blockHeight"].get<int>();
1213 if (obj.contains("matureAtHeight")) r.matureAtHeight = obj["matureAtHeight"].get<int>();
1214 if (obj.contains("createdAt")) r.createdAt = obj["createdAt"].get<int64>();
1215 if (obj.contains("blockHash") && obj["blockHash"].is_string()) r.blockHash = obj["blockHash"].get<std::string>();
1216 if (obj.contains("coinbaseTxid") && obj["coinbaseTxid"].is_string()) r.coinbaseTxid = obj["coinbaseTxid"].get<std::string>();
1217 if (obj.contains("foundBy") && obj["foundBy"].is_string()) r.foundBy = obj["foundBy"].get<std::string>();
1218 if (obj.contains("feePercent")) r.feePercent = obj["feePercent"].get<double>();
1219 if (obj.contains("totalShares")) r.totalShares = obj["totalShares"].get<uint64>();
1220 if (obj.contains("coinbaseValueSatoshis")) r.coinbaseValue = obj["coinbaseValueSatoshis"].get<int64>();
1221 if (obj.contains("subsidySatoshis")) r.subsidy = obj["subsidySatoshis"].get<int64>();
1222 if (obj.contains("minerRewardSatoshis")) r.minerReward = obj["minerRewardSatoshis"].get<int64>();
1223 if (obj.contains("operatorRewardSatoshis")) r.operatorReward = obj["operatorRewardSatoshis"].get<int64>();
1224 if (obj.contains("skippedDust")) r.skippedDust = obj["skippedDust"].get<int>();
1225 if (obj.contains("forfeitedBadAddressSatoshis")) r.forfeitedBadAddress = obj["forfeitedBadAddressSatoshis"].get<int64>();
1226 if (obj.contains("shares") && obj["shares"].is_object())
1227 for (auto it = obj["shares"].begin(); it != obj["shares"].end(); ++it)
1228 r.shares[it.key()] = it.value().get<uint64>();
1229 if (obj.contains("payouts") && obj["payouts"].is_object())
1230 for (auto it = obj["payouts"].begin(); it != obj["payouts"].end(); ++it) {
1231 if (it.value().is_object()) {
1232 r.payouts[it.key()] = it.value().value("amountSatoshis", (int64)0);
1233 if (it.value().contains("txid") && it.value()["txid"].is_string())
1234 r.paidTxids[it.key()] = it.value()["txid"].get<std::string>();
1235 if (it.value().value("forfeited", false))
1236 r.forfeited[it.key()] =
1237 it.value().value("forfeitReason", std::string("unspecified"));
1238 } else if (it.value().is_number_integer()) {
1239 r.payouts[it.key()] = it.value().get<int64>();
1240 }
1241 }
1242 if (!r.id.empty())
1243 gPoolRounds.push_back(r);
1244 }
1245 while (gPoolRounds.size() > MAX_POOL_ROUNDS)
1246 gPoolRounds.erase(gPoolRounds.begin());
1247 LogPrint("payout", "[payout] loaded %zu pool round proof(s) from disk\n",
1248 gPoolRounds.size());
1249 } catch (const std::exception& e) {
1250 RefuseFileAfterLoadFailure(path, "pool round proofs", e.what());
475 } 1251 }
476} 1252}
477 1253
@@ -494,105 +1270,487 @@ static bool PayoutBlockStillOnChain(const uint256& blockHash)
494 return false; 1270 return false;
495} 1271}
496 1272
497static std::string PayoutComment(int blockHeight) 1273// The comment every pre-upgrade payout transaction carries, and therefore the
1274// ONLY needle the legacy fallback in FindRecordedPoolPayout may search with.
1275// This format is frozen: change it and every payment those builds made stops
1276// matching, every one of those debts looks unpaid, and reconciliation pays
1277// them all a second time.
1278static std::string LegacyPayoutComment(int blockHeight)
498{ 1279{
499 return strprintf("Pool payout height %d", blockHeight); 1280 return strprintf("Pool payout height %d", blockHeight);
500} 1281}
501 1282
502// Did we already send this payment before dying? Looks for a wallet 1283// The comment written on new sends. It names the funding block, because the
503// transaction of ours carrying this payout's comment and paying exactly this 1284// height alone cannot: a height that is won, lost to a reorg, and won again
504// amount to this address. 1285// yields two rounds whose height-only comments are identical -- and identical
1286// amounts too, whenever the same miner held both rounds. The structured
1287// pool_payout_* metadata already keys the payment by hash for the code; this
1288// puts the same fact where an operator reading the wallet can see it, and
1289// makes a new transaction unmatchable by the legacy needle even if the
1290// structured keys are ever lost.
505// 1291//
506// Biased towards reporting "already paid" on any doubt: skipping a payment a 1292// A zero hash keeps the legacy form: those records predate hash keying, and
507// miner is owed is visible and recoverable from the log, while paying twice 1293// there is no hash to name.
508// is neither. 1294static std::string PayoutComment(int blockHeight, const uint256& blockHash)
509static bool WalletAlreadyPaid(const PayingRecord& r)
510{ 1295{
511 uint160 h160; 1296 if (blockHash == 0)
512 if (!AddressToHash160(r.address, h160)) 1297 return LegacyPayoutComment(blockHeight);
513 return false; 1298 return strprintf("Pool payout height %d block %s", blockHeight,
514 CScript want; 1299 blockHash.ToString().c_str());
515 want << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; 1300}
516 const std::string comment = PayoutComment(r.blockHeight);
517 1301
518 CRITICAL_BLOCK(cs_mapWallet) 1302// Startup reconciliation.
1303//
1304// This used to be the crash-recovery mechanism: it scanned the wallet with
1305// WalletAlreadyPaid() and then dropped ledger entries that had already been
1306// exploded into in-flight records. Neither job is right any more.
1307//
1308// The ledger is now authoritative and an entry survives until every recipient
1309// has a txid recorded against it (upstream cbdd06d), so dropping ledger
1310// entries here would reintroduce exactly the loss this merge is adopting that
1311// model to prevent. And the wallet scan is now FindRecordedPoolPayout(), done
1312// once per record on its first attempt, which is the same question asked in
1313// the one place that can act on the answer. Running both would be two
1314// authorities for the same debt.
1315//
1316// What is left is the useful part: the in-flight list restored from disk may
1317// contain records the ledger already shows as paid (we died between the send
1318// and the in-flight file being rewritten). The ledger wins.
1319//
1320// And the part that was missing: gQueuedPayouts is memory-only, so it came up
1321// empty while gPaying came up full. FlushMaturePayouts deduped against that
1322// empty set and against the ledger's paidTxids, and against nothing else --
1323// so a round that was exploded but not yet paid got a SECOND record per
1324// unpaid recipient on the first flush after every restart, deterministically.
1325// (Observed live: two byte-identical records for one (block, address).) The
1326// surviving in-flight list is exactly the statement of which rounds are
1327// already exploded, so it is what the set is rebuilt from.
1328static void ReconcilePayouts()
1329{
1330 VersionedBlob payingSnap;
519 { 1331 {
520 for (std::map<uint256, CWalletTx>::iterator it = mapWallet.begin(); 1332 std::lock_guard<std::mutex> exec(gPayoutExecMutex);
521 it != mapWallet.end(); ++it) { 1333 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
522 CWalletTx& wtx = it->second; 1334
523 // The comment is written only by the payout path, so its presence 1335 size_t settled = 0;
524 // already identifies the transaction as one of ours. 1336 int64 settledAmount = 0;
525 std::map<std::string, std::string>::const_iterator ci = 1337 for (auto it = gPaying.begin(); it != gPaying.end(); ) {
526 wtx.mapValue.find("comment"); 1338 bool paid = false;
527 if (ci == wtx.mapValue.end() || ci->second != comment) 1339 for (const PendingPayout& pp : gPendingPayouts)
1340 if (SamePayoutRound(pp.blockHeight, pp.blockHash,
1341 it->blockHeight, it->blockHash)
1342 && pp.paidTxids.count(it->address)) {
1343 paid = true;
1344 break;
1345 }
1346 if (paid) {
1347 settledAmount += it->amount;
1348 settled++;
1349 it = gPaying.erase(it);
1350 } else {
1351 // Every restored record gets one wallet probe before it is
1352 // allowed to send: this is precisely the window where the
1353 // wallet may hold a transaction the ledger has not recorded.
1354 it->fProbeWallet = true;
1355 ++it;
1356 }
1357 }
1358 if (settled) {
1359 LogPrint("payout", "[payout] %zu in-flight payment(s) totalling %s"
1360 " are already recorded paid in the ledger -- not"
1361 " resending\n",
1362 settled, FormatMoney(settledAmount).c_str());
1363 }
1364
1365 // Rebuild the "already exploded" set from what survived. Without this
1366 // the first flush after a restart re-explodes every round in flight.
1367 gQueuedPayouts.clear();
1368 for (const PayingRecord& r : gPaying)
1369 gQueuedPayouts.insert(PayoutIdForBlock(r.blockHeight, r.blockHash));
1370
1371 payingSnap = SerializePayingLocked();
1372
1373 if (!gPaying.empty()) {
1374 int64 owed = 0;
1375 for (const auto& r : gPaying) owed += r.amount;
1376 LogPrint("payout", "[payout] resuming %zu in-flight payment(s)"
1377 " totalling %s\n",
1378 gPaying.size(), FormatMoney(owed).c_str());
1379 }
1380 }
1381 PublishPaying(payingSnap);
1382}
1383
1384static void RecordPoolRound(const PoolRoundProof& round)
1385{
1386 VersionedBlob snap;
1387 {
1388 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1389 bool replaced = false;
1390 for (PoolRoundProof& existing : gPoolRounds) {
1391 if (existing.id == round.id) {
1392 existing = round;
1393 replaced = true;
1394 break;
1395 }
1396 }
1397 if (!replaced) {
1398 gPoolRounds.push_back(round);
1399 while (gPoolRounds.size() > MAX_POOL_ROUNDS)
1400 gPoolRounds.erase(gPoolRounds.begin());
1401 }
1402 snap = SerializePoolRoundsLocked();
1403 }
1404 PublishPoolRounds(snap);
1405}
1406
1407static void MarkPoolRoundPaid(const std::string& id,
1408 const std::string& address,
1409 const std::string& txid)
1410{
1411 VersionedBlob snap;
1412 {
1413 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1414 bool found = false;
1415 for (PoolRoundProof& r : gPoolRounds) {
1416 if (r.id != id)
528 continue; 1417 continue;
529 foreach (const CTxOut& txout, wtx.vout) 1418 r.paidTxids[address] = txid;
530 if (txout.nValue == r.amount && txout.scriptPubKey == want) 1419 found = true;
531 return true; 1420 break;
1421 }
1422 if (!found)
1423 return;
1424 snap = SerializePoolRoundsLocked();
1425 }
1426 PublishPoolRounds(snap);
1427}
1428
1429// Publish the fact that a recipient was written off, so the round proof and
1430// the ledger agree. The ledger erases the amount (it is not owed any more and
1431// must not be reported as sent); the proof keeps the amount visible and marks
1432// it forfeited with a reason, because a miner who was promised a share is
1433// entitled to see what happened to it.
1434static void MarkPoolRoundForfeited(const std::string& id,
1435 const std::string& address,
1436 const std::string& reason)
1437{
1438 VersionedBlob snap;
1439 {
1440 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1441 bool found = false;
1442 for (PoolRoundProof& r : gPoolRounds) {
1443 if (r.id != id)
1444 continue;
1445 r.forfeited[address] = reason;
1446 found = true;
1447 break;
532 } 1448 }
1449 if (!found)
1450 return;
1451 snap = SerializePoolRoundsLocked();
533 } 1452 }
1453 PublishPoolRounds(snap);
1454}
1455
1456// Does this round have a published round proof?
1457//
1458// Existence answers a temporal question the legacy fallback below needs:
1459// which builds could have paid this round. RecordPoolRound writes the proof
1460// when the round is created, and structured payout metadata (upstream
1461// cbdd06d) is an ancestor of round proofs (upstream 83ba822) -- no build
1462// able to write a proof has ever sent a payment that only a comment can
1463// identify. A round with a proof therefore cannot have a comment-only
1464// payment; one that matches it is another round's, colliding on the height.
1465static bool PoolRoundHasProof(const std::string& id)
1466{
1467 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1468 for (const PoolRoundProof& r : gPoolRounds)
1469 if (r.id == id)
1470 return true;
534 return false; 1471 return false;
535} 1472}
536 1473
537// Startup reconciliation. Drops in-flight records the wallet shows were 1474// Which wallet transaction, if any, already paid this?
538// already sent, and drops ledger entries that were already exploded into 1475//
539// records (which happens if we died between writing the records and rewriting 1476// Two kinds of evidence, in order of quality.
540// the ledger -- both files then describe the same debt). 1477//
541static void ReconcilePayouts() 1478// Structured (pool_payout_id / pool_payout_address / pool_payout_amount,
1479// plus pool_payout_block since this series) is upstream's, extended. The id
1480// is derived from the block hash, but only from the FIRST sixteen hex digits
1481// of it -- and proof-of-work concentrates zeros exactly there, so the id can
1482// name two same-height rounds identically. New sends therefore record the
1483// full hash as pool_payout_block, and a structured match must agree with it
1484// wherever both sides have one; a transaction without the key was sent
1485// before it existed, and for those the id is the best evidence there is.
1486// Every transaction this build sends carries all four keys.
1487//
1488// But NO transaction this pool has ever sent carries it. Every payout in the
1489// existing wallet has only a "comment". So on the first boot after this
1490// upgrade -- precisely the boot where the binary is swapped after an unclean
1491// shutdown with payments in flight -- a structured-only probe finds nothing
1492// and every one of those recipients is paid a second time. The deleted
1493// WalletAlreadyPaid() matched on what the payment WAS, and that evidence is
1494// still there; it just has to be read.
1495//
1496// The legacy fallback is therefore restored, with three constraints that
1497// together stop it answering for a round that is not this one:
1498//
1499// 1. it is only consulted when the structured lookup found nothing;
1500// 2. it ignores any transaction carrying pool_payout_id. That key means this
1501// build wrote the transaction, and this build's transactions are
1502// adjudicated by the exact path -- so a NEW round's payment can never be
1503// offered as evidence for an old one. Only pre-upgrade transactions,
1504// which is exactly the population the fallback exists for, are eligible;
1505// 3. `excluded` holds every txid already credited to some OTHER
1506// (round, address) pair -- from the ledger's paidTxids, which is durable,
1507// and from what this process has already claimed. One legacy transaction
1508// can therefore settle at most one debt, so N transactions can never
1509// satisfy more than N obligations;
1510//
1511// 4. it is not consulted at all for a round that has a round proof
1512// (fAllowLegacy, decided by the caller from PoolRoundHasProof). Proofs
1513// are written when a round is created, and only by builds whose every
1514// payment names its round exactly -- so a comment-only transaction
1515// cannot be a payment for such a round, only a same-height collision
1516// from an older one, and skipping the fallback cannot hide a real
1517// payment: the structured probe above already found any that exist.
1518//
1519// What is left unresolved by construction is a comment-only transaction and
1520// a PROOFLESS round at the same height paying the same address the same
1521// amount: two grandfathered zero-hash twins, or a twin pair from the builds
1522// that keyed the ledger by hash before any metadata reached the wallet. The
1523// comment carries only the height, so nothing ties the transaction to one
1524// twin rather than the other. That needs a reorg.
1525//
1526// This used to say the losing twin's ledger entry is voided by the orphan
1527// check before it can be paid. That is wrong: PayoutBlockStillOnChain()
1528// returns true for a zero hash, and EVERY pre-upgrade entry has a zero hash,
1529// so the orphan check is a no-op for exactly the population this fallback
1530// serves.
1531//
1532// The outcome is still safe, by a different mechanism. SamePayoutRound()
1533// treats two zero-hash entries at the same height as the SAME round, so the
1534// pair is indistinguishable everywhere, not only here: FlushMaturePayouts
1535// explodes one of them and skips the other as already queued,
1536// PaymentInFlightLocked refuses a second record for that (round, address), and
1537// RecordPayoutTxid settles against the first match. The pair therefore
1538// presents as ONE debt and can be paid at most once -- which is the direction
1539// that matters for a fallback whose failure mode is treating one payment as
1540// proof of two.
1541//
1542// The cost of that mechanism, stated plainly: only one of a genuine
1543// same-height pair would ever be paid. It takes a reorg AND both rounds
1544// predating hash keying, and no file this tree has written since can produce
1545// one, so it is a property of history rather than a live risk.
1546//
1547// Where the evidence genuinely runs out the fallback simply does not match,
1548// and the payment is made -- paying a debt we cannot prove was settled is the
1549// correct default for a pool.
1550enum PayoutProbeResult {
1551 PAYOUT_PROBE_NONE = 0, // the wallet holds no record of this payment
1552 PAYOUT_PROBE_STRUCTURED, // matched this build's pool_payout_* metadata
1553 PAYOUT_PROBE_LEGACY // matched a pre-upgrade comment and output
1554};
1555
1556static PayoutProbeResult FindRecordedPoolPayout(const std::string& id,
1557 int blockHeight,
1558 const uint256& blockHash,
1559 const std::string& address,
1560 int64 amount,
1561 bool fAllowLegacy,
1562 const std::set<std::string>& excluded,
1563 std::string& txidRet)
542{ 1564{
543 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 1565 // Built before cs_mapWallet is taken: an invalid address has no script to
1566 // compare against, and the caller has already rejected those.
1567 uint160 h160;
1568 bool fHaveScript = AddressToHash160(address, h160);
1569 CScript want;
1570 if (fHaveScript)
1571 want << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
1572 const std::string comment = LegacyPayoutComment(blockHeight);
1573
1574 // Candidate legacy matches, adjudicated after cs_mapWallet is released.
1575 //
1576 // The depth check cannot happen inside the scan: GetDepthInMainChain()
1577 // reads mapBlockIndex and pindexBest, which belong to cs_main, and cs_main
1578 // is taken BEFORE cs_mapWallet everywhere else in this tree
1579 // (CreateTransaction, ConnectBlock). Taking it the other way round here
1580 // would be a lock-order inversion in the payout thread. There is at most a
1581 // handful of these: a candidate has to carry this exact height's comment
1582 // and an output of exactly this amount to this address.
1583 std::vector<uint256> legacyCandidates;
1584
1585 // One pass, not two: this scan runs under cs_mapWallet, which the whole
1586 // node contends on, and the merge is already paying for it once per record.
1587 CRITICAL_BLOCK(cs_mapWallet)
1588 {
1589 for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin();
1590 it != mapWallet.end(); ++it)
1591 {
1592 const CWalletTx& wtx = it->second;
1593 map<string, string>::const_iterator idIt = wtx.mapValue.find("pool_payout_id");
1594 if (idIt != wtx.mapValue.end()) {
1595 // Written by this build. Adjudicated exactly, and never
1596 // offered to the fallback below.
1597 if (idIt->second != id)
1598 continue;
1599 map<string, string>::const_iterator addrIt = wtx.mapValue.find("pool_payout_address");
1600 if (addrIt == wtx.mapValue.end() || addrIt->second != address)
1601 continue;
1602 map<string, string>::const_iterator amtIt = wtx.mapValue.find("pool_payout_amount");
1603 if (amtIt == wtx.mapValue.end() || amtIt->second != strprintf("%lld", amount))
1604 continue;
1605 // The id alone cannot tell twin rounds at one height apart
1606 // (see above); the full hash can, where the transaction
1607 // recorded one. Where it did not, the id decides, as it
1608 // always has -- refusing those transactions would make every
1609 // payment sent before the key existed look unpaid and pay it
1610 // again.
1611 if (blockHash != 0) {
1612 map<string, string>::const_iterator blkIt =
1613 wtx.mapValue.find("pool_payout_block");
1614 if (blkIt != wtx.mapValue.end()
1615 && blkIt->second != blockHash.ToString())
1616 continue; // a twin round's payment, not this round's
1617 }
1618 txidRet = wtx.GetHash().GetHex();
1619 return PAYOUT_PROBE_STRUCTURED;
1620 }
544 1621
545 size_t settled = 0; 1622 if (!fAllowLegacy)
546 int64 settledAmount = 0; 1623 continue; // comment-only evidence is inadmissible: rule 4
547 for (auto it = gPaying.begin(); it != gPaying.end(); ) { 1624 if (!fHaveScript)
548 if (WalletAlreadyPaid(*it)) { 1625 continue;
549 settledAmount += it->amount; 1626 // The comment is written only by the payout path, so its presence
550 settled++; 1627 // already identifies the transaction as one of ours.
551 it = gPaying.erase(it); 1628 map<string, string>::const_iterator ci = wtx.mapValue.find("comment");
552 } else { 1629 if (ci == wtx.mapValue.end() || ci->second != comment)
553 ++it; 1630 continue;
1631 if (excluded.count(wtx.GetHash().GetHex()))
1632 continue; // already credited to a different debt
1633 foreach (const CTxOut& txout, wtx.vout)
1634 if (txout.nValue == amount && txout.scriptPubKey == want) {
1635 legacyCandidates.push_back(it->first);
1636 break;
1637 }
554 } 1638 }
555 } 1639 }
556 if (settled) {
557 LogPrint("payout", "[payout] %zu in-flight payment(s) totalling %s were"
558 " already sent before the last shutdown -- not resending\n",
559 settled, FormatMoney(settledAmount).c_str());
560 SavePayingLocked();
561 }
562 1640
563 size_t dropped = 0; 1641 // A legacy match must have CONFIRMED.
564 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { 1642 //
565 bool exploded = false; 1643 // The structured path does not need this and deliberately does not have
566 for (const auto& r : gPaying) 1644 // it: this process sent that transaction moments ago, and a send that
567 if (r.matureAtHeight == it->matureAtHeight && r.blockHash == it->blockHash) { 1645 // failed leaves nothing behind, because SendMoney calls
568 exploded = true; 1646 // RollbackTransactionSpent, which calls EraseFromWallet.
569 break; 1647 //
1648 // The legacy path is the opposite case in every respect. Those
1649 // transactions are days or weeks old, so whether they confirmed is a
1650 // settled fact rather than an open question, and a pre-upgrade payout that
1651 // never confirmed is evidence of NON-payment about to be read as payment.
1652 // This chain has had a 280-block reorg (see FlushMaturePayouts): a payout
1653 // invalidated by it is still sitting in mapWallet with its comment intact,
1654 // and the first boot after this upgrade is exactly when it gets matched.
1655 // The cost of being wrong here is a miner who is never paid and no record
1656 // that they were owed.
1657 CRITICAL_BLOCK(cs_main)
1658 CRITICAL_BLOCK(cs_mapWallet)
1659 {
1660 for (const uint256& hash : legacyCandidates) {
1661 map<uint256, CWalletTx>::const_iterator it = mapWallet.find(hash);
1662 if (it == mapWallet.end())
1663 continue; // erased under us; it is not evidence of anything
1664 int depth = it->second.GetDepthInMainChain();
1665 if (depth < 1) {
1666 // Loud: this is the difference between a miner having been
1667 // paid and a miner about to be paid twice, decided on a fact
1668 // that is easy to check and impossible to recover afterwards.
1669 LogPrint("payout", "[payout] wallet tx %s carries the payout"
1670 " comment for height %d and an output of exactly %s to"
1671 " %s, but it is NOT in the main chain (depth %d) --"
1672 " refusing to read it as payment; this recipient will"
1673 " be paid\n",
1674 hash.GetHex().c_str(), blockHeight,
1675 FormatMoney(amount).c_str(), address.c_str(), depth);
1676 continue;
570 } 1677 }
571 if (exploded) { 1678 txidRet = hash.GetHex();
572 LogPrint("payout", "[payout] ledger entry for block %d is already" 1679 return PAYOUT_PROBE_LEGACY;
573 " in flight -- dropping the duplicate\n", it->blockHeight);
574 it = gPendingPayouts.erase(it);
575 dropped++;
576 } else {
577 ++it;
578 } 1680 }
579 } 1681 }
580 if (dropped) SavePendingPayouts(); 1682 return PAYOUT_PROBE_NONE;
1683}
581 1684
582 if (!gPaying.empty()) { 1685// txid -> "<round id>|<address>", for payments this run has already credited
583 int64 owed = 0; 1686// to a debt on legacy evidence. Stops one pre-upgrade transaction being read
584 for (const auto& r : gPaying) owed += r.amount; 1687// as proof that two different obligations were met. The ledger's paidTxids
585 LogPrint("payout", "[payout] resuming %zu in-flight payment(s) totalling %s\n", 1688// covers the same ground durably; this covers the window before the recovered
586 gPaying.size(), FormatMoney(owed).c_str()); 1689// txid reaches the ledger, and the case where the ledger entry has already
1690// been retired.
1691static std::map<std::string, std::string> gLegacyClaimedTxids;
1692static std::mutex gClaimedTxidMutex;
1693
1694// Every txid already spoken for by some payment OTHER than this one. Collected
1695// before cs_mapWallet is taken: the ledger mutex must not be acquired under a
1696// wallet lock.
1697static void CollectForeignPayoutTxids(int blockHeight, const uint256& blockHash,
1698 const std::string& address,
1699 const std::string& claimKey,
1700 std::set<std::string>& out)
1701{
1702 {
1703 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1704 for (const PendingPayout& pp : gPendingPayouts) {
1705 bool sameRound = SamePayoutRound(pp.blockHeight, pp.blockHash,
1706 blockHeight, blockHash);
1707 for (const auto& kv : pp.paidTxids)
1708 if (!(sameRound && kv.first == address))
1709 out.insert(kv.second);
1710 }
1711 }
1712 {
1713 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
1714 for (const auto& kv : gLegacyClaimedTxids)
1715 if (kv.second != claimKey)
1716 out.insert(kv.first);
587 } 1717 }
588} 1718}
589 1719
1720static void ClaimLegacyPayoutTxid(const std::string& txid, const std::string& claimKey)
1721{
1722 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
1723 gLegacyClaimedTxids[txid] = claimKey;
1724}
1725
1726// blockHash stays a uint256 and not upstream's display string: it is the key
1727// this fork's payout ledger is built on, and a key that has been through
1728// ToString() is a key that can be compared wrong. It is converted to text only
1729// where the round proof needs text. The other three arguments are upstream's,
1730// carried for the round-proof data.
590static void QueuePayouts(int blockHeight, 1731static void QueuePayouts(int blockHeight,
591 const uint256& blockHash, 1732 const uint256& blockHash,
1733 const std::string& coinbaseTxid,
1734 int64 coinbaseValue,
1735 const std::string& foundBy,
592 const std::map<std::string, uint64>& shareCount, 1736 const std::map<std::string, uint64>& shareCount,
593 uint64 total) 1737 uint64 total)
594{ 1738{
1739 PoolRoundProof round;
1740 round.id = PayoutIdForBlock(blockHeight, blockHash);
1741 round.blockHeight = blockHeight;
1742 round.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH;
1743 round.createdAt = GetTime();
1744 round.blockHash = blockHash.ToString();
1745 round.coinbaseTxid = coinbaseTxid;
1746 round.coinbaseValue = coinbaseValue;
1747 round.foundBy = foundBy;
1748 round.feePercent = dPoolFeePercent;
1749 round.totalShares = total;
1750 round.shares = shareCount;
1751
595 if (total == 0) { 1752 if (total == 0) {
1753 RecordPoolRound(round);
596 LogPrint("payout", "[payout] block %d: zero shares recorded -- nothing to pay out\n", 1754 LogPrint("payout", "[payout] block %d: zero shares recorded -- nothing to pay out\n",
597 blockHeight); 1755 blockHeight);
598 return; 1756 return;
@@ -608,6 +1766,10 @@ static void QueuePayouts(int blockHeight,
608 int64 nSubsidy = 50 * COIN; 1766 int64 nSubsidy = 50 * COIN;
609 nSubsidy >>= (blockHeight / 210000); // halving schedule 1767 nSubsidy >>= (blockHeight / 210000); // halving schedule
610 int64 minerReward = (int64)((double)nSubsidy * (1.0 - feeFrac)); 1768 int64 minerReward = (int64)((double)nSubsidy * (1.0 - feeFrac));
1769 int64 operatorCut = nSubsidy - minerReward;
1770 round.subsidy = nSubsidy;
1771 round.minerReward = minerReward;
1772 round.operatorReward = operatorCut;
611 1773
612 PendingPayout pp; 1774 PendingPayout pp;
613 pp.blockHeight = blockHeight; 1775 pp.blockHeight = blockHeight;
@@ -621,7 +1783,9 @@ static void QueuePayouts(int blockHeight,
621 // alone on a private fork paying itself, so it says nothing about a pool 1783 // alone on a private fork paying itself, so it says nothing about a pool
622 // competing on the real chain. Waiting the extra 20 blocks makes each 1784 // competing on the real chain. Waiting the extra 20 blocks makes each
623 // payout funded by construction, and puts the maturity gate past the 1785 // payout funded by construction, and puts the maturity gate past the
624 // reorg depth that orphaned 280 blocks of obligations. 1786 // reorg depth that orphaned 280 blocks of obligations. Upstream uses
1787 // COINBASE_MATURITY here; this is a considered disagreement, not an
1788 // oversight -- see the merge commit message.
625 pp.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH; 1789 pp.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH;
626 1790
627 int skipped = 0; 1791 int skipped = 0;
@@ -643,12 +1807,22 @@ static void QueuePayouts(int blockHeight,
643 continue; 1807 continue;
644 } 1808 }
645 pp.amounts[kv.first] = amount; 1809 pp.amounts[kv.first] = amount;
1810 round.payouts[kv.first] = amount;
646 } 1811 }
1812 round.skippedDust = skipped;
1813 round.forfeitedBadAddress = burned;
647 1814
648 gPendingPayouts.push_back(pp); 1815 {
649 SavePendingPayouts(); 1816 VersionedBlob snap;
1817 {
1818 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1819 gPendingPayouts.push_back(pp);
1820 snap = SerializePendingPayoutsLocked();
1821 }
1822 PublishPendingPayouts(snap);
1823 }
1824 RecordPoolRound(round);
650 1825
651 int64 operatorCut = nSubsidy - minerReward;
652 LogPrint("payout", 1826 LogPrint("payout",
653 "[payout] block %d: subsidy %s BTF, miners get %s" 1827 "[payout] block %d: subsidy %s BTF, miners get %s"
654 " (fee %.2f%% = %s to operator)," 1828 " (fee %.2f%% = %s to operator),"
@@ -664,101 +1838,337 @@ static void QueuePayouts(int blockHeight,
664 : ""); 1838 : "");
665} 1839}
666 1840
667// Called from event loop: move matured entries into the in-flight list (fast). 1841// Called from the event loop: explode matured ledger entries into per-recipient
1842// in-flight records (fast; no wallet or disk work beyond one small write).
1843//
1844// The ledger entry SURVIVES here. That is upstream's model and it is the
1845// crash-safe one: this fork used to erase the entry as soon as its payments
1846// were handed off, so a death in that window left the debt described by
1847// nothing at all -- a container restart on 2026-07-30 silently dropped
1848// ~1,156 BTF exactly that way. An entry is removed only once every recipient
1849// in it has a txid recorded (PayoutFullyPaid), which the payout thread does.
668// 1850//
669// Ordering matters and is deliberate: the in-flight records are written and 1851// gPaying is kept on top of that as the in-flight working set: it carries the
670// fsync'd to their own file *before* the ledger entry is erased, so the debt 1852// per-recipient retry/deferral state that a per-round `nextAttempt` cannot
671// is always described by at least one of the two files. Dying in the window 1853// express, and it is what the payout thread iterates. It is a cache of work
672// where both describe it is handled at startup by ReconcilePayouts, which 1854// derived from the ledger, never an independent record of debt -- if it were
673// drops the ledger entry it can see is already in flight. 1855// lost entirely the ledger would rebuild it on the next flush.
674static void FlushMaturePayouts() 1856static void FlushMaturePayouts()
675{ 1857{
676 // Void obligations whose funding block lost a reorg. The coin that was 1858 // Which funding blocks are still on the main chain? Asked with no payout
677 // meant to pay them does not exist, so paying anyway spends real-chain 1859 // lock held: PayoutBlockStillOnChain takes cs_main, and cs_main is held by
678 // coin the pool owes to somebody else. 1860 // ProcessBlock and SendMoney for long stretches. Holding the ledger mutex
679 bool voided = false; 1861 // across it would let the event loop park the payout thread behind a chain
680 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { 1862 // lock it has no interest in. (It is not a deadlock either way -- nothing
681 if (PayoutBlockStillOnChain(it->blockHash)) { ++it; continue; } 1863 // takes the ledger mutex while holding cs_main -- but it is exactly the
682 int64 total = 0; 1864 // kind of stall this merge is removing from the ledger path.)
683 for (auto& kv : it->amounts) total += kv.second; 1865 std::vector<uint256> orphaned;
684 LogPrint("payout", "[payout] block %d (%s) was orphaned -- voiding %zu" 1866 {
685 " payment(s) totalling %s\n", 1867 std::vector<uint256> hashes;
686 it->blockHeight, it->blockHash.ToString().substr(0, 14).c_str(),
687 it->amounts.size(), FormatMoney(total).c_str());
688 it = gPendingPayouts.erase(it);
689 voided = true;
690 }
691 if (voided) SavePendingPayouts();
692
693 std::vector<PendingPayout> ready;
694 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) {
695 if (nBestHeight < it->matureAtHeight) { ++it; continue; }
696 LogPrint("payout", "[payout] block height %d matured -- queuing %zu payments\n",
697 it->matureAtHeight, it->amounts.size());
698 ready.push_back(*it); // copy first; erase after save
699 ++it;
700 }
701 if (!ready.empty()) {
702 // Record the individual payments first, and only then drop the ledger
703 // entries that produced them.
704 { 1868 {
705 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 1869 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
706 for (const auto& pp : ready) { 1870 for (const PendingPayout& pp : gPendingPayouts)
707 for (const auto& kv : pp.amounts) { 1871 hashes.push_back(pp.blockHash);
708 PayingRecord r;
709 r.blockHeight = pp.blockHeight;
710 r.blockHash = pp.blockHash;
711 r.matureAtHeight = pp.matureAtHeight;
712 r.address = kv.first;
713 r.amount = kv.second;
714 gPaying.push_back(r);
715 }
716 }
717 SavePayingLocked();
718 } 1872 }
1873 for (const uint256& h : hashes)
1874 if (!PayoutBlockStillOnChain(h))
1875 orphaned.push_back(h);
1876 }
1877
1878 std::vector<PendingPayout> ready;
1879 VersionedBlob ledgerSnap;
1880 {
1881 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
1882
1883 // Void obligations whose funding block lost a reorg. The coin that was
1884 // meant to pay them does not exist, so paying anyway spends real-chain
1885 // coin the pool owes to somebody else.
1886 bool voided = false;
719 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { 1887 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) {
720 if (nBestHeight >= it->matureAtHeight) 1888 if (std::find(orphaned.begin(), orphaned.end(), it->blockHash) == orphaned.end()) {
721 it = gPendingPayouts.erase(it);
722 else
723 ++it; 1889 ++it;
1890 continue;
1891 }
1892 int64 total = 0;
1893 for (auto& kv : it->amounts) total += kv.second;
1894 // Always: one line per voided entry, never a collapsed summary.
1895 // A reorg voids these in a burst from this one source line.
1896 LogPrintAlways("payout", "[payout] block %d (%s) was orphaned -- voiding %zu"
1897 " payment(s) totalling %s\n",
1898 it->blockHeight, it->blockHash.ToString().substr(0, 14).c_str(),
1899 it->amounts.size(), FormatMoney(total).c_str());
1900 it = gPendingPayouts.erase(it);
1901 voided = true;
1902 }
1903 if (voided) ledgerSnap = SerializePendingPayoutsLocked();
1904
1905 for (const PendingPayout& pp : gPendingPayouts) {
1906 if (nBestHeight < pp.matureAtHeight)
1907 continue;
1908 if (PayoutFullyPaid(pp))
1909 continue;
1910 // An entry with neither a hash nor a height cannot be told apart
1911 // from another in the same state, so a txid recorded against it
1912 // could settle the wrong debt. Paying it is the unsafe move;
1913 // leaving it standing and saying so is not.
1914 if (!PayoutRoundIdentifiable(pp.blockHeight, pp.blockHash)) {
1915 LogPrint("payout", "[payout] WARNING: ledger entry maturing at"
1916 " height %d names neither a block hash nor a height --"
1917 " refusing to pay %zu recipient(s) that cannot be told"
1918 " apart from another round's\n",
1919 pp.matureAtHeight, pp.amounts.size());
1920 continue;
1921 }
1922 ready.push_back(pp);
724 } 1923 }
725 SavePendingPayouts();
726 } 1924 }
727} 1925 // Outside the ledger mutex: publishing is I/O and the event loop is the
1926 // one thread that must never wait on the payout thread's disk.
1927 PublishPendingPayouts(ledgerSnap);
728 1928
729// Identity of a payment, for finding its record again after SendMoney. 1929 if (ready.empty())
730// amounts is a map keyed by address, so (block, address) is already unique. 1930 return;
731static bool SamePayment(const PayingRecord& a, const PayingRecord& b) 1931
732{ 1932 VersionedBlob payingSnap;
733 return a.matureAtHeight == b.matureAtHeight 1933 {
734 && a.blockHash == b.blockHash 1934 std::lock_guard<std::mutex> queue(gPayoutExecMutex);
735 && a.address == b.address 1935 size_t added = 0;
736 && a.amount == b.amount; 1936 for (const PendingPayout& pp : ready) {
1937 const std::string id = PayoutIdForBlock(pp.blockHeight, pp.blockHash);
1938 if (gQueuedPayouts.count(id))
1939 continue; // already exploded into gPaying and still in flight
1940 size_t n = 0;
1941 for (const auto& kv : pp.amounts) {
1942 // Upstream's per-recipient dedupe: a recipient already carrying
1943 // a txid is done, even though the entry as a whole is not.
1944 if (pp.paidTxids.count(kv.first))
1945 continue;
1946 // The invariant, checked against the thing it is an invariant
1947 // of: at most one in-flight record per (round, address). The
1948 // round-level set above is an optimisation that can be wrong --
1949 // it is memory only, it is rebuilt from gPaying at startup, and
1950 // a stale `ready` snapshot can race a round finishing --
1951 // whereas gPaying is the list being protected. A second record
1952 // here is a second standing order to send the same money.
1953 if (PaymentInFlightLocked(pp.blockHeight, pp.blockHash, kv.first))
1954 continue;
1955 PayingRecord r;
1956 r.id = ++gPayingNextId;
1957 r.blockHeight = pp.blockHeight;
1958 r.blockHash = pp.blockHash;
1959 r.matureAtHeight = pp.matureAtHeight;
1960 r.address = kv.first;
1961 r.amount = kv.second;
1962 gPaying.push_back(r);
1963 n++;
1964 }
1965 gQueuedPayouts.insert(id);
1966 added += n;
1967 LogPrint("payout", "[payout] block height %d matured -- queuing"
1968 " payout %s (%zu of %zu recipients still owed)\n",
1969 pp.matureAtHeight, id.c_str(), n, pp.amounts.size());
1970 }
1971 if (added) payingSnap = SerializePayingLocked();
1972 }
1973 PublishPaying(payingSnap);
737} 1974}
738 1975
739// Settle a payment: it is either done or abandoned, either way it stops being 1976// Settle a payment: it is either done or abandoned, either way it stops being
740// owed. Removing the record and rewriting the file is what makes the payment 1977// in flight. Note this only retires the working record -- whether the miner is
741// not happen twice after a restart. 1978// still OWED the money is the ledger's business, and the ledger is updated
1979// separately by RecordPayoutTxid().
1980//
1981// Erases by record id, not by comparing fields. Value-matching was only an
1982// identity for as long as no two records could be equal, and the restart bug
1983// above produced records that were equal in every field: the first match then
1984// absorbed every erase and every state update, and the second was never
1985// touched again. The id names the slot the caller took its copy from, so the
1986// update lands where it belongs whatever else is in the list.
742static void ForgetPayment(const PayingRecord& r) 1987static void ForgetPayment(const PayingRecord& r)
743{ 1988{
744 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 1989 VersionedBlob snap;
745 for (auto it = gPaying.begin(); it != gPaying.end(); ++it) 1990 {
746 if (SamePayment(*it, r)) { gPaying.erase(it); break; } 1991 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
747 SavePayingLocked(); 1992 for (auto it = gPaying.begin(); it != gPaying.end(); ++it)
1993 if (it->id == r.id) { gPaying.erase(it); break; }
1994
1995 // When the last record for a round is gone the round is no longer in
1996 // flight, so a later flush may explode it again if the ledger still
1997 // shows recipients owed. That is the intended behaviour and it is now
1998 // the only way a stalled payment ever leaves this list: a payment that
1999 // keeps failing is retried forever rather than given up on, so its
2000 // record stays here, and the flush that would re-explode it is stopped
2001 // by PaymentInFlightLocked.
2002 bool stillInFlight = false;
2003 for (const auto& e : gPaying)
2004 if (SamePayoutRound(e.blockHeight, e.blockHash,
2005 r.blockHeight, r.blockHash)) {
2006 stillInFlight = true;
2007 break;
2008 }
2009 if (!stillInFlight)
2010 gQueuedPayouts.erase(PayoutIdForBlock(r.blockHeight, r.blockHash));
2011
2012 snap = SerializePayingLocked();
2013 }
2014 PublishPaying(snap);
748} 2015}
749 2016
750// Persist an updated attempt count / backoff for a payment still owed. 2017// Persist an updated attempt count / backoff for a payment still owed.
751static void ReschedulePayment(const PayingRecord& r) 2018static void ReschedulePayment(const PayingRecord& r)
752{ 2019{
753 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 2020 VersionedBlob snap;
754 for (auto& e : gPaying) 2021 {
755 if (SamePayment(e, r)) { 2022 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
756 e.retries = r.retries; 2023 for (auto& e : gPaying)
757 e.deferrals = r.deferrals; 2024 if (e.id == r.id) {
758 e.retryAfter = r.retryAfter; 2025 e.retries = r.retries;
2026 e.deferrals = r.deferrals;
2027 e.retryAfter = r.retryAfter;
2028 // Carry the cleared probe flag back, or every retry re-scans
2029 // the whole wallet -- the cost this merge is specifically
2030 // removing.
2031 e.fProbeWallet = r.fProbeWallet;
2032 break;
2033 }
2034 snap = SerializePayingLocked();
2035 }
2036 PublishPaying(snap);
2037}
2038
2039// Record that a recipient has been paid, in the ledger, which is the only
2040// durable statement that the debt is settled.
2041//
2042// The file write happens with gPayoutLedgerMutex RELEASED. Upstream writes the
2043// file inside the per-recipient loop with the mutex held, so every recipient
2044// costs the event loop a full serialise+fsync it has to wait for in
2045// FlushMaturePayouts. Ordering against the event loop's own writes is the
2046// snapshot version's job, not the mutex's -- see PublishVersioned.
2047static void RecordPayoutTxid(const PayingRecord& r, const std::string& txid)
2048{
2049 VersionedBlob blob;
2050 bool complete = false;
2051 {
2052 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
2053 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ++it) {
2054 if (!SamePayoutRound(it->blockHeight, it->blockHash,
2055 r.blockHeight, r.blockHash))
2056 continue;
2057 it->paidTxids[r.address] = txid;
2058 if (PayoutFullyPaid(*it)) {
2059 LogPrint("payout", "[payout] payout for block %d complete --"
2060 " removing from ledger\n", it->blockHeight);
2061 gPendingPayouts.erase(it);
2062 complete = true;
2063 }
759 break; 2064 break;
760 } 2065 }
761 SavePayingLocked(); 2066 blob = SerializePendingPayoutsLocked();
2067 }
2068 PublishPendingPayouts(blob);
2069 // The round proof records a txid per address, so this is per recipient and
2070 // not only on completion.
2071 MarkPoolRoundPaid(PayoutIdForBlock(r.blockHeight, r.blockHash), r.address, txid);
2072 if (complete)
2073 LogPrint("payout", "[payout] block %d fully settled\n", r.blockHeight);
2074}
2075
2076// Write a recipient off.
2077//
2078// Only ever called for a state that is DECIDABLE -- currently one: the ledger
2079// names an address that is not a valid address, so no transaction paying it
2080// can be constructed by this or any other build. It is not called, and must
2081// not be, because a payment has failed some number of times. "The wallet said
2082// no N times" is not a decidable state, it is a description of how long we
2083// have been asking; a debt written off on the strength of it is a debt the
2084// operator keeps, and the arithmetic that decided WHY the wallet said no was
2085// wrong for a year (see PayoutThreadFn).
2086//
2087// The amount is removed from the entry rather than marked paid, so the money
2088// is not reported as sent. Once nothing is left owed the entry retires. The
2089// round proof is told separately, because dropping the obligation from the
2090// ledger while pool_rounds.json goes on publishing the address as owed with
2091// "txid": null leaves the pool's two records of the same round permanently
2092// disagreeing.
2093static void ForfeitPayoutRecipient(const PayingRecord& r, const std::string& reason)
2094{
2095 LogPrintAlways("payout", "[payout] writing off %s owed to %s for block %d: %s\n",
2096 FormatMoney(r.amount).c_str(), r.address.c_str(), r.blockHeight,
2097 reason.c_str());
2098 VersionedBlob blob;
2099 {
2100 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
2101 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ++it) {
2102 if (!SamePayoutRound(it->blockHeight, it->blockHash,
2103 r.blockHeight, r.blockHash))
2104 continue;
2105 it->amounts.erase(r.address);
2106 // PayoutFullyPaid() returns true for an empty amounts map -- there
2107 // is no recipient in it without a txid -- so the
2108 // `amounts.empty()` disjunct that used to be here decided nothing.
2109 if (PayoutFullyPaid(*it))
2110 gPendingPayouts.erase(it);
2111 break;
2112 }
2113 blob = SerializePendingPayoutsLocked();
2114 }
2115 PublishPendingPayouts(blob);
2116 MarkPoolRoundForfeited(PayoutIdForBlock(r.blockHeight, r.blockHash),
2117 r.address, reason);
2118}
2119
2120// What a failed send means for the payment's retry budget.
2121//
2122// A deferral is "the wallet's condition is temporarily wrong": wait, and do
2123// not hold it against this payment. A fault is "the wallet is funded and
2124// unlocked and still would not do it", which is worth logging loudly and
2125// retrying with backoff.
2126//
2127// This used to be decided by re-reading GetBalance() after the failure and
2128// comparing it against `amount + nTransactionFee + CENT`. Every term of that
2129// was wrong:
2130//
2131// * nTransactionFee is 0 in production. It is only ever assigned from the
2132// GUI preferences dialog (gui.cpp), which a headless pool node does not
2133// have, so the whole reserve was the CENT.
2134//
2135// * The fee the reserve is for is GetMinFee(fDiscount=true), a STEP
2136// function of the serialised size: 0 below 10,000 bytes, and
2137// (1 + bytes/1000) * CENT at or above it -- so 11 * CENT at the step.
2138// There is no required-fee value between 1 and 11 CENT. A one-CENT
2139// reserve is therefore never the right number: below the step nothing
2140// needs reserving, and at the step it is short by a factor of eleven.
2141//
2142// * A pool wallet close to broke selects many small inputs BECAUSE it is
2143// close to broke, and ~56 signed P2PKH inputs is 10 KB. The trigger and
2144// the failure are positively correlated: the guess is wrong precisely
2145// when the pool is nearly out of money.
2146//
2147// Concretely, with a balance of amount + 0.05: the pre-send gate passes,
2148// CreateTransaction builds a >10 KB transaction, the fee becomes 0.11,
2149// SelectCoins(amount + 0.11) fails, SendMoney returns false -- and the old
2150// arithmetic said (amount + 0.05) < (amount + 0.01) is false, so "funded",
2151// so a fault. The condition is stable, so it recurred every 60 s, and 1440 of
2152// them erased the debt with the coin still in the operator's wallet.
2153//
2154// So stop guessing. The wallet is asked to say what it did, and this only
2155// classifies the answer. WALLET_SEND_INSUFFICIENT_FUNDS is coin selection
2156// failing to cover amount plus whatever fee that size of transaction
2157// required, whatever that turned out to be -- no reserve to get wrong.
2158enum PayoutFailureClass {
2159 PAYOUT_FAILURE_DEFER = 0, // conditions are wrong; wait for them to change
2160 PAYOUT_FAILURE_FAULT // funded and unlocked, and it still would not go
2161};
2162
2163static PayoutFailureClass ClassifyPayoutSendFailure(WalletSendResult result,
2164 bool fWalletLocked)
2165{
2166 // A locked wallet cannot sign. Nothing about this payment is wrong.
2167 if (fWalletLocked)
2168 return PAYOUT_FAILURE_DEFER;
2169 if (result == WALLET_SEND_INSUFFICIENT_FUNDS)
2170 return PAYOUT_FAILURE_DEFER;
2171 return PAYOUT_FAILURE_FAULT;
762} 2172}
763 2173
764// Payout execution thread: works through gPaying, calls SendMoney. 2174// Payout execution thread: works through gPaying, calls SendMoney.
@@ -780,11 +2190,14 @@ static void PayoutThreadFn(void*)
780 } 2190 }
781 2191
782 for (PayingRecord r : batch) { 2192 for (PayingRecord r : batch) {
2193 const std::string id = PayoutIdForBlock(r.blockHeight, r.blockHash);
2194
783 // Re-check here and not only at maturity: a payment can sit in this 2195 // Re-check here and not only at maturity: a payment can sit in this
784 // list for hours across retries, and the reorg that kills its 2196 // list for hours across retries, and the reorg that kills its
785 // funding block can land at any point in that window. 2197 // funding block can land at any point in that window. Upstream has
2198 // no equivalent -- it decides once, when the entry matures.
786 if (!PayoutBlockStillOnChain(r.blockHash)) { 2199 if (!PayoutBlockStillOnChain(r.blockHash)) {
787 LogPrint("payout", "[payout] block %d (%s) orphaned while queued" 2200 LogPrintAlways("payout", "[payout] block %d (%s) orphaned while queued"
788 " -- dropping %s to %s\n", 2201 " -- dropping %s to %s\n",
789 r.blockHeight, r.blockHash.ToString().substr(0, 14).c_str(), 2202 r.blockHeight, r.blockHash.ToString().substr(0, 14).c_str(),
790 FormatMoney(r.amount).c_str(), r.address.c_str()); 2203 FormatMoney(r.amount).c_str(), r.address.c_str());
@@ -792,72 +2205,244 @@ static void PayoutThreadFn(void*)
792 continue; 2205 continue;
793 } 2206 }
794 2207
2208 // Upstream's per-recipient dedupe, against the ledger rather than
2209 // against the batch: if a txid is already recorded for this
2210 // address, the money went out and nothing more is owed.
2211 {
2212 bool alreadyPaid = false;
2213 {
2214 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
2215 for (const PendingPayout& pp : gPendingPayouts)
2216 if (SamePayoutRound(pp.blockHeight, pp.blockHash,
2217 r.blockHeight, r.blockHash)
2218 && pp.paidTxids.count(r.address)) {
2219 alreadyPaid = true;
2220 break;
2221 }
2222 }
2223 if (alreadyPaid) {
2224 ForgetPayment(r);
2225 continue;
2226 }
2227 }
2228
795 uint160 h160; 2229 uint160 h160;
796 if (!AddressToHash160(r.address, h160)) { 2230 if (!AddressToHash160(r.address, h160)) {
797 // Should not happen -- bad addresses are filtered in QueuePayouts 2231 // Should not happen -- bad addresses are filtered in
798 LogPrint("payout", "[payout] SKIPPED bad address '%s'\n", 2232 // QueuePayouts. If one is here anyway it is decidable and
799 r.address.c_str()); 2233 // permanent: there is no transaction that pays a string which
2234 // is not an address, so retrying is not waiting for anything.
2235 //
2236 // This used to only ForgetPayment(), which drops the in-flight
2237 // record and leaves the ledger entry claiming the debt and
2238 // pool_rounds.json advertising the recipient as unpaid
2239 // forever, with the next flush re-exploding it. Write it off
2240 // properly and say why.
2241 const std::string reason =
2242 strprintf("'%s' is not a valid address, so no transaction"
2243 " can pay it", r.address.c_str());
2244 LogPrintAlways("payout", "[payout] SKIPPED bad address '%s' --"
2245 " %s owed for block %d cannot be paid\n",
2246 r.address.c_str(), FormatMoney(r.amount).c_str(),
2247 r.blockHeight);
2248 ForfeitPayoutRecipient(r, reason);
800 ForgetPayment(r); 2249 ForgetPayment(r);
801 continue; 2250 continue;
802 } 2251 }
803 CScript sc; 2252 CScript sc;
804 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; 2253 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
805 2254
2255 // Did a previous process send this and die before recording it?
2256 // Only asked once per record -- see PayingRecord::fProbeWallet.
2257 if (r.fProbeWallet) {
2258 std::string recovered;
2259 const std::string claimKey = id + "|" + r.address;
2260 std::set<std::string> excluded;
2261 CollectForeignPayoutTxids(r.blockHeight, r.blockHash, r.address,
2262 claimKey, excluded);
2263 // A round with a published proof was created by a build whose
2264 // every payment names its round exactly, so only structured
2265 // evidence can settle it -- a comment match would be another
2266 // round's payment colliding on the height.
2267 const bool fAllowLegacy = !PoolRoundHasProof(id);
2268 r.fProbeWallet = false;
2269 PayoutProbeResult probe = FindRecordedPoolPayout(
2270 id, r.blockHeight, r.blockHash, r.address, r.amount,
2271 fAllowLegacy, excluded, recovered);
2272 if (probe != PAYOUT_PROBE_NONE) {
2273 if (probe == PAYOUT_PROBE_LEGACY) {
2274 // Loud on purpose: this is the upgrade path, where the
2275 // only evidence a payment went out is a comment written
2276 // by a binary that is no longer running.
2277 LogPrint("payout", "[payout] %s to %s was already paid by"
2278 " a pre-upgrade build (wallet tx %s carries the"
2279 " payout comment for height %d and an output of"
2280 " exactly this amount, and no structured payout"
2281 " metadata) -- not resending\n",
2282 FormatMoney(r.amount).c_str(), r.address.c_str(),
2283 recovered.c_str(), r.blockHeight);
2284 ClaimLegacyPayoutTxid(recovered, claimKey);
2285 } else {
2286 LogPrint("payout", "[payout] recovered recorded payout %s to %s"
2287 " from wallet tx %s -- not resending\n",
2288 FormatMoney(r.amount).c_str(), r.address.c_str(),
2289 recovered.c_str());
2290 }
2291 RecordPayoutTxid(r, recovered);
2292 ForgetPayment(r);
2293 continue;
2294 }
2295 }
2296
806 // Waiting for coin to mature is not a failure, so it must not 2297 // Waiting for coin to mature is not a failure, so it must not
807 // consume the retry budget. Burning a retry every 60s means a 2298 // consume the retry budget. Burning a retry every 60s means a
808 // payment the pool fully intends to honour can hit GIVING UP 2299 // payment the pool fully intends to honour can hit GIVING UP
809 // 24h later purely because the subsidy funding it had not 2300 // 24h later purely because the subsidy funding it had not
810 // matured yet -- the miner is then silently never paid. 2301 // matured yet -- the miner is then silently never paid.
2302 //
2303 // The comparison includes the fee. It used to be against the
2304 // amount alone, so a balance anywhere between amount and
2305 // amount+fee sailed past this gate, failed in SendMoney, and was
2306 // counted as a send FAILURE rather than a wait -- 1440 of those is
2307 // 24 hours, and the recipient was erased with the money left in
2308 // the operator's wallet and nothing on disk saying it was owed.
2309 //
2310 // nTransactionFee is the part of the fee that can be known before
2311 // the transaction exists, and in production it is 0 -- it is only
2312 // ever assigned from the GUI preferences dialog, which a pool node
2313 // does not have -- so including it here changes nothing on this
2314 // pool and is kept only because a GUI operator can set it. The
2315 // rest is GetMinFee(), a function of the serialised size and
2316 // therefore of how many inputs SelectCoins ends up picking, which
2317 // cannot be predicted from here -- so this gate deliberately does
2318 // NOT try to reserve it. Refusing to attempt a payment the wallet
2319 // could actually have made would be its own way of not paying a
2320 // miner. That residual is handled after the fact, by asking the
2321 // wallet why it refused -- see ClassifyPayoutSendFailure.
811 int64 nSpendable = GetBalance(); 2322 int64 nSpendable = GetBalance();
812 if (nSpendable < r.amount) { 2323 if (nSpendable < r.amount + nTransactionFee) {
813 r.deferrals++; 2324 r.deferrals++;
814 r.retryAfter = GetTime() + 60; 2325 r.retryAfter = GetTime() + 60;
815 // Hourly, not every minute: a dry pool would otherwise 2326 // Hourly, not every minute: a dry pool would otherwise
816 // write one line per payment per minute forever. 2327 // write one line per payment per minute forever.
817 if (r.deferrals == 1 || r.deferrals % 60 == 0) 2328 if (r.deferrals == 1 || r.deferrals % 60 == 0)
818 LogPrint("payout", "[payout] deferring %s to %s -- only %s" 2329 LogPrint("payout", "[payout] deferring %s to %s -- only %s"
819 " spendable (waiting %d min so far)\n", 2330 " spendable, need %s with fee (waiting %d min so far)\n",
820 FormatMoney(r.amount).c_str(), r.address.c_str(), 2331 FormatMoney(r.amount).c_str(), r.address.c_str(),
821 FormatMoney(nSpendable).c_str(), r.deferrals); 2332 FormatMoney(nSpendable).c_str(),
2333 FormatMoney(r.amount + nTransactionFee).c_str(),
2334 r.deferrals);
822 ReschedulePayment(r); 2335 ReschedulePayment(r);
823 continue; 2336 continue;
824 } 2337 }
825 2338
826 CWalletTx wtx; 2339 // Likewise a locked wallet: it cannot sign, so every attempt fails
827 wtx.mapValue["comment"] = PayoutComment(r.blockHeight); 2340 // for a reason that has nothing to do with this payment being
2341 // payable. Left to the retry budget, an operator who locks the
2342 // wallet overnight comes back to erased debts.
2343 if (IsWalletLocked()) {
2344 r.deferrals++;
2345 r.retryAfter = GetTime() + 60;
2346 if (r.deferrals == 1 || r.deferrals % 60 == 0)
2347 LogPrint("payout", "[payout] deferring %s to %s -- wallet is"
2348 " locked (waiting %d min so far)\n",
2349 FormatMoney(r.amount).c_str(), r.address.c_str(),
2350 r.deferrals);
2351 ReschedulePayment(r);
2352 continue;
2353 }
828 2354
829 if (SendMoney(sc, r.amount, wtx)) { 2355 CWalletTx wtx;
2356 wtx.mapValue["comment"] = PayoutComment(r.blockHeight, r.blockHash);
2357 // Upstream's structured metadata. This is what makes the wallet
2358 // probe above possible at all -- the old comment-only form could
2359 // not distinguish two rounds paying the same address the same
2360 // amount. Keyed by the hash-derived id, so a reorg cannot make one
2361 // round's transaction answer for another's.
2362 wtx.mapValue["pool_payout_id"] = id;
2363 wtx.mapValue["pool_payout_address"] = r.address;
2364 wtx.mapValue["pool_payout_amount"] = strprintf("%lld", r.amount);
2365 // The full hash, not only the id's sixteen-digit prefix: the
2366 // prefix of a proof-of-work hash is where the zeros live, and
2367 // the probe needs to tell twin rounds at one height apart.
2368 if (r.blockHash != 0)
2369 wtx.mapValue["pool_payout_block"] = r.blockHash.ToString();
2370
2371 SendMoneyOutcome outcome;
2372 if (SendMoney(sc, r.amount, wtx, outcome)) {
830 // The wallet holds the transaction before this returns, so a 2373 // The wallet holds the transaction before this returns, so a
831 // crash here is recoverable: ReconcilePayouts finds it by 2374 // crash between here and the ledger write is recoverable: the
832 // comment+address+amount and does not resend. 2375 // probe above finds it by id+address+amount and does not
2376 // resend. Ledger first, in-flight record second -- the debt is
2377 // described by at least one of them at every instant.
2378 RecordPayoutTxid(r, wtx.GetHash().GetHex());
833 ForgetPayment(r); 2379 ForgetPayment(r);
834 LogPrint("payout", "[payout] paid %s to %s (tx %s)\n", 2380 LogPrintAlways("payout", "[payout] paid %s to %s (tx %s)\n",
835 FormatMoney(r.amount).c_str(), r.address.c_str(), 2381 FormatMoney(r.amount).c_str(), r.address.c_str(),
836 wtx.GetHash().ToString().substr(0, 12).c_str()); 2382 wtx.GetHash().ToString().substr(0, 12).c_str());
837 } else if (r.retries < 1440) {
838 // Transient failures (reorg, locked wallet) must not silently
839 // burn the payment -- retry with backoff. Insufficient funds
840 // never lands here; it defers above.
841 r.retries++;
842 r.retryAfter = GetTime() + 60;
843 ReschedulePayment(r);
844 LogPrint("payout", "[payout] FAILED to pay %s to %s"
845 " (%s spendable at the time) -- retry %d/1440 in 60s\n",
846 FormatMoney(r.amount).c_str(), r.address.c_str(),
847 FormatMoney(nSpendable).c_str(), r.retries);
848 } else { 2383 } else {
849 // Reaching here means 1440 attempts that were NOT 2384 // Why did it fail? The answer used to decide whether the
850 // insufficient-funds waits (those defer instead) against a 2385 // payment eventually got erased, and it was decided by
851 // block still on the main chain -- i.e. a real wallet fault, 2386 // guessing from a balance snapshot -- see
852 // not a maturity or reorg question. 2387 // ClassifyPayoutSendFailure for the arithmetic and what it
853 LogPrint("payout", "[payout] GIVING UP on %s to %s after %d" 2388 // cost. The wallet now says what it did, and nothing here
854 " retries -- block %d is still on-chain and %s was" 2389 // erases anything.
855 " spendable, so this is a wallet fault, not maturity" 2390 //
856 " -- payment forfeited\n", 2391 // The lock state is re-read rather than trusted from before
857 FormatMoney(r.amount).c_str(), r.address.c_str(), 2392 // the attempt: SendMoney takes cs_main and can block for a
858 r.retries, r.blockHeight, 2393 // long time, so it may have moved underneath us. The funding
859 FormatMoney(nSpendable).c_str()); 2394 // answer does not need re-reading -- it is a statement about
860 ForgetPayment(r); 2395 // the attempt that was actually made.
2396 bool fLocked = IsWalletLocked();
2397 int64 nAfter = GetBalance();
2398
2399 if (ClassifyPayoutSendFailure(outcome.result, fLocked)
2400 == PAYOUT_FAILURE_DEFER) {
2401 // Conditions are temporarily wrong. Wait.
2402 r.deferrals++;
2403 r.retryAfter = GetTime() + 60;
2404 if (r.deferrals == 1 || r.deferrals % 60 == 0)
2405 LogPrint("payout", "[payout] deferring %s to %s -- send"
2406 " failed with %s (%s spendable; the wallet"
2407 " needed a fee of %s for a transaction this"
2408 " size); waiting %d min so far\n",
2409 FormatMoney(r.amount).c_str(), r.address.c_str(),
2410 fLocked ? "the wallet locked" : "too little coin",
2411 FormatMoney(nAfter).c_str(),
2412 FormatMoney(outcome.nFeeRequired).c_str(),
2413 r.deferrals);
2414 ReschedulePayment(r);
2415 } else {
2416 // Funded, unlocked, block on the main chain, and the
2417 // wallet still would not build, sign or accept the
2418 // transaction. That is worth shouting about; it is not
2419 // grounds to keep the money.
2420 //
2421 // There is deliberately no give-up branch. It used to
2422 // forfeit the debt after 1440 of these, which is not a
2423 // decidable state -- it is a count of how long we have
2424 // been asking, and it was reached by misclassifying a
2425 // funding shortfall as a fault. A debt that is never
2426 // written off cannot be written off wrongly. The payment
2427 // stays in flight, keeps its ledger entry, keeps being
2428 // reported in pool_status.json as owed, and keeps
2429 // retrying once a minute until the operator or the wallet
2430 // makes it possible.
2431 r.retries++;
2432 r.retryAfter = GetTime() + 60;
2433 ReschedulePayment(r);
2434 // Every minute at first, then hourly: a fault that is not
2435 // going away must not turn the log into one line a minute
2436 // forever, and must not go quiet either.
2437 if (r.retries <= 5 || r.retries % 60 == 0)
2438 LogPrintAlways("payout", "[payout] FAILED to pay %s to %s"
2439 " (%s spendable, wallet unlocked, block %d on"
2440 " the main chain) -- attempt %d, retrying in"
2441 " 60s; this debt stands until it is paid\n",
2442 FormatMoney(r.amount).c_str(), r.address.c_str(),
2443 FormatMoney(nAfter).c_str(), r.blockHeight,
2444 r.retries);
2445 }
861 } 2446 }
862 } 2447 }
863 2448
@@ -881,6 +2466,7 @@ static StatsSnapshot gStats;
881static std::vector<PoolWorkerStatView> gWorkerStats; 2466static std::vector<PoolWorkerStatView> gWorkerStats;
882static std::vector<PendingPayoutView> gPayoutViews; 2467static std::vector<PendingPayoutView> gPayoutViews;
883static std::mutex gStatsMutex; 2468static std::mutex gStatsMutex;
2469static int64 gLastPoolStatusWrite = 0;
884 2470
885void GetPoolOperatorStats(int& authorizedMiners, int& blocksFound, uint64& roundShares, double& totalHashRate) 2471void GetPoolOperatorStats(int& authorizedMiners, int& blocksFound, uint64& roundShares, double& totalHashRate)
886{ 2472{
@@ -903,6 +2489,135 @@ void GetPendingPayouts(std::vector<PendingPayoutView>& out)
903 out = gPayoutViews; 2489 out = gPayoutViews;
904} 2490}
905 2491
2492static std::string PoolStatusFilePath()
2493{
2494 if (!strPoolStatusFile.empty())
2495 return strPoolStatusFile;
2496 return GetAppDir() + "/pool_status.json";
2497}
2498
2499// Serialise the operator status file. Caller must hold gStatsMutex (this reads
2500// gStats / gWorkerStats / gPayoutViews); gPoolRoundsMutex is taken and
2501// released inside.
2502//
2503// Split from the write for the same reason as the three ledgers, and for a
2504// worse instance of the same problem. This used to be called from RefreshStats
2505// with gPayoutLedgerMutex held at function scope, and once the ledger writes
2506// gained fsync() that call became two fsyncs -- one on the file, one on its
2507// directory -- performed by the event loop, the thread that owns every miner's
2508// socket, with the mutex the payout thread needs in its hand. Measured on the
2509// build host (ext4 on LVM on SSD): 3.78 ms per publish, every 10 seconds, and
2510// unbounded on a shared virtual disk under load.
2511//
2512// Nothing here needs the ledger mutex at all: the pending-payout rows were
2513// already copied into gPayoutViews under it, further up RefreshStats.
2514static std::string SerializePoolStatusLocked()
2515{
2516 nlohmann::json j;
2517 j["schema"] = "bitflash-pool-status-1";
2518 j["network"] = "bitflash";
2519 j["updatedAt"] = GetTime();
2520
2521 j["operator"] = {
2522 {"name", strPoolName},
2523 {"btfAddress", BtfLocalAddress()},
2524 {"feePercent", dPoolFeePercent},
2525 {"dashboardUrl", strPoolDashboardUrl},
2526 {"roundsFile", "pool_rounds.json"}
2527 };
2528
2529 j["node"] = {
2530 {"height", nBestHeight},
2531 {"poolRunning", (bool)gPoolRunning},
2532 {"mode", nMineMode}
2533 };
2534
2535 j["pool"] = {
2536 {"authorizedMiners", gStats.authorizedMiners},
2537 {"blocksFoundSession", gStats.blocksFound},
2538 {"roundShares", gStats.roundShares},
2539 {"hashRate", gStats.totalHashRate}
2540 };
2541
2542 j["workers"] = nlohmann::json::array();
2543 for (const PoolWorkerStatView& row : gWorkerStats) {
2544 j["workers"].push_back({
2545 {"address", row.address},
2546 {"worker", row.worker},
2547 {"totalShares", row.totalShares},
2548 {"roundShares", row.roundShares},
2549 {"lastSeen", row.lastSeen},
2550 {"hashRate", row.hashRate}
2551 });
2552 }
2553
2554 j["pendingPayouts"] = nlohmann::json::array();
2555 for (const PendingPayoutView& row : gPayoutViews) {
2556 j["pendingPayouts"].push_back({
2557 {"matureAtHeight", row.matureAtHeight},
2558 {"recipients", row.recipients},
2559 {"totalAmountSatoshis", row.totalAmount},
2560 {"totalAmountBtf", FormatMoney(row.totalAmount)}
2561 });
2562 }
2563
2564 j["recentRounds"] = nlohmann::json::array();
2565 {
2566 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
2567 size_t start = gPoolRounds.size() > 10 ? gPoolRounds.size() - 10 : 0;
2568 for (size_t i = start; i < gPoolRounds.size(); i++) {
2569 const PoolRoundProof& r = gPoolRounds[i];
2570 // A forfeited recipient is not "unpaid and pending" -- the ledger
2571 // has already dropped the obligation. Counting it as unpaid is
2572 // what made the round proof and the ledger disagree for good.
2573 int unpaid = 0;
2574 int forfeited = 0;
2575 for (const auto& kv : r.payouts) {
2576 if (r.paidTxids.count(kv.first))
2577 continue;
2578 if (r.forfeited.count(kv.first))
2579 forfeited++;
2580 else
2581 unpaid++;
2582 }
2583 j["recentRounds"].push_back({
2584 {"id", r.id},
2585 {"blockHeight", r.blockHeight},
2586 {"blockHash", r.blockHash},
2587 {"coinbaseTxid", r.coinbaseTxid},
2588 {"totalShares", r.totalShares},
2589 {"payouts", (int)r.payouts.size()},
2590 {"unpaidPayouts", unpaid},
2591 {"forfeitedPayouts", forfeited},
2592 {"coinbaseValueBtf", FormatMoney(r.coinbaseValue)},
2593 {"minerRewardBtf", FormatMoney(r.minerReward)}
2594 });
2595 }
2596 }
2597
2598 return j.dump(2);
2599}
2600
2601// Publish the operator status file. Call with every payout mutex released.
2602//
2603// Still no version guard and no write mutex, deliberately, and the split does
2604// not change the argument: RefreshStats remains the only caller of either
2605// half, in production it runs only on the event loop, and it now calls this
2606// after leaving the scope that serialised -- so the serialise and the write
2607// stay strictly ordered on one thread and pool_status.json still has a single
2608// writer by construction. It shares WriteFileAtomic so the durability rules
2609// stay in one place.
2610//
2611// "In production" is doing real work in that sentence. The payout-ledger
2612// self-test also calls RefreshStats, from one thread at a time and with the
2613// pool not running, which is why it is still a single writer there too. A
2614// second concurrent caller anywhere would invalidate this paragraph and want
2615// the same versioned publish the ledgers use.
2616static void PublishPoolStatus(const std::string& blob)
2617{
2618 WriteFileAtomic(PoolStatusFilePath(), blob, "pool status");
2619}
2620
906// --------------------------------------------------------------------------- 2621// ---------------------------------------------------------------------------
907// Miner state -- event loop thread only 2622// Miner state -- event loop thread only
908// --------------------------------------------------------------------------- 2623// ---------------------------------------------------------------------------
@@ -928,9 +2643,35 @@ static const double VARDIFF_MAX = 1e12;
928static const int64 VARDIFF_STALL_SECS = 45; // 3x share target -- reacts faster without false-triggering on normal variance 2643static const int64 VARDIFF_STALL_SECS = 45; // 3x share target -- reacts faster without false-triggering on normal variance
929static const double VARDIFF_STALL_CUT = 4.0; // divide difficulty by this much 2644static const double VARDIFF_STALL_CUT = 4.0; // divide difficulty by this much
930 2645
2646// #7: m->hashRate is an EWMA sampled only when a share arrives, so a miner
2647// that stops hashing keeps its last estimate forever and RefreshStats sums
2648// it into the announced pool total for as long as the connection sits open
2649// -- the published pool hashrate ran several times over ground truth
2650// because of exactly this. MinerReportedHashRate() below applies the
2651// policy: a miner gets a grace of HASHRATE_IDLE_MISSES expected share
2652// intervals (expected interval = expected hashes per share at its
2653// difficulty, divided by the estimate itself) with no share, after which
2654// the figure we report falls off with a half-life of one more such
2655// interval. The interval is clamped to [HASHRATE_MIN_INTERVAL,
2656// HASHRATE_MAX_INTERVAL] so a fast miner at the vardiff floor (expected
2657// interval of milliseconds) is not punished for ordinary share-delivery
2658// jitter, and a very slow miner's grace does not stretch past the point
2659// where its estimate means anything. At the vardiff target of one share
2660// per 15s that is a minute of grace, half the rate reported 75s in, and
2661// nothing reported after about seventeen minutes of silence.
2662static const double HASHRATE_IDLE_MISSES = 4.0; // grace, in expected share intervals
2663static const double HASHRATE_MIN_INTERVAL = 1.0; // seconds; floor for the expected interval
2664static const double HASHRATE_MAX_INTERVAL = 3600.0; // seconds; ceiling for the expected interval
2665// Half-lives past the grace beyond which we report a flat zero. 2^-64 of the
2666// original estimate is not a number anyone needs, and stopping here keeps
2667// pow() from walking down into the denormals for a miner that has been quiet
2668// for hours.
2669static const double HASHRATE_DECAY_CUTOFF = 64.0;
2670
931struct Miner { 2671struct Miner {
932 btf_socket_t fd; 2672 btf_socket_t fd;
933 bool authorised = false; 2673 bool authorised = false;
2674 bool subscribed = false; // completed mining.subscribe (#19 census)
934 std::string address; 2675 std::string address;
935 std::string worker; 2676 std::string worker;
936 std::string readBuf; 2677 std::string readBuf;
@@ -954,11 +2695,80 @@ struct Miner {
954 int64 lastShareTime = 0; // GetTime() of the previous accepted share 2695 int64 lastShareTime = 0; // GetTime() of the previous accepted share
955 double hashRate = 0.0; // smoothed H/s estimate 2696 double hashRate = 0.0; // smoothed H/s estimate
956 2697
2698 // Replay guard for mining.submit (#8): nonces already credited on this
2699 // connection for the job it is currently working. Keyed to the job id in
2700 // seenNoncesJob and cleared by SendJob whenever that id changes, so a
2701 // resubmission under a NEW job is never turned away, while a replayed
2702 // (job_id, nonce) pair under the same job is refused before it can be
2703 // credited twice into roundShareCount -- the figure the payout split
2704 // divides by.
2705 std::set<uint32_t> seenNonces;
2706 std::string seenNoncesJob;
2707
957 explicit Miner(btf_socket_t f) 2708 explicit Miner(btf_socket_t f)
958 : fd(f), connectTime(GetTime()), 2709 : fd(f), connectTime(GetTime()),
959 extranonce2(gNextExtranonce2.fetch_add(1)) {} 2710 extranonce2(gNextExtranonce2.fetch_add(1)) {}
960}; 2711};
961 2712
2713// What we publish as a miner's H/s: the last EWMA sample, faded out if shares
2714// have stopped arriving (#7, policy at HASHRATE_IDLE_MISSES).
2715//
2716// This reads m->hashRate and never writes it. That is the whole design. The
2717// first attempt decayed the stored estimate in place from the event loop's
2718// sweep, which put a running accumulator behind a figure that is only ever
2719// displayed, and the accumulator misbehaved in two ways:
2720//
2721// - `expected` was recomputed from the already-decayed estimate on every
2722// pass, so each decay raised `expected`, which raised `grace`, which is
2723// the very thing gating the next decay. `grace` simply chased `silent`
2724// upward and the reported rate settled into a 1/t crawl instead of the
2725// half-life curve the comment above promises -- 291x too high at ten
2726// half-lives past the grace. Issue #7 is about the pool over-reporting;
2727// that under-corrected it for hours.
2728// - Once `expected` reached HASHRATE_MAX_INTERVAL it could not rise any
2729// further, the feedback that had been gating each pass was gone, and the
2730// decay began compounding once a second against the total elapsed
2731// silence -- taking a miner from 0.4% of its baseline to 1e-11 of it in
2732// the next hundred seconds, four hours in, for no reason a reader of
2733// either the code or the announced number could see.
2734//
2735// Computing the value instead of accumulating it removes the failure mode
2736// rather than tuning it: with no stored decay state there is nothing for a
2737// tick rate, a missed pass or a duplicated pass to perturb, and the answer
2738// depends only on how long the miner has actually been quiet. It also keeps
2739// the fade out of the estimator -- the EWMA on the share path still blends
2740// against the value it last computed, not against a faded copy of it, so a
2741// miner that resumes recovers on exactly the curve it would have without any
2742// of this.
2743//
2744// `now` is a parameter rather than a GetTime() call so that one refresh fades
2745// every miner to the same instant, and so the policy is directly testable.
2746static double MinerReportedHashRate(const Miner* m, int64 now)
2747{
2748 // Nothing sampled yet: a fresh connection, or one that has found a single
2749 // share and so has no interval to have been timed over. Nothing to fade,
2750 // and nothing to divide by below.
2751 if (m->hashRate <= 0.0 || m->lastShareTime <= 0) return m->hashRate;
2752
2753 double expected = ExpectedHashesForDifficulty(m->difficulty) / m->hashRate;
2754 // Written so that a NaN would land on the floor rather than sail through:
2755 // the clamp is what guarantees `expected` is finite and strictly positive
2756 // below, whatever the division produced.
2757 if (!(expected > HASHRATE_MIN_INTERVAL)) expected = HASHRATE_MIN_INTERVAL;
2758 if (expected > HASHRATE_MAX_INTERVAL) expected = HASHRATE_MAX_INTERVAL;
2759
2760 double silent = (double)(now - m->lastShareTime);
2761 double grace = HASHRATE_IDLE_MISSES * expected;
2762 // grace is at least HASHRATE_IDLE_MISSES seconds, so a clock that has
2763 // stepped backwards gives a negative `silent`, takes this branch, and
2764 // reports the sample unchanged. Nothing here can scale an estimate up.
2765 if (!(silent > grace)) return m->hashRate;
2766
2767 double halflives = (silent - grace) / expected;
2768 if (halflives >= HASHRATE_DECAY_CUTOFF) return 0.0;
2769 return m->hashRate * pow(0.5, halflives);
2770}
2771
962static std::vector<Miner*> gMiners; 2772static std::vector<Miner*> gMiners;
963 2773
964struct FoundBlock { int height; int64 ts; std::string addr; }; 2774struct FoundBlock { int height; int64 ts; std::string addr; };
@@ -1009,6 +2819,18 @@ static void LocalAcceptFn(void*)
1009 break; 2819 break;
1010 continue; 2820 continue;
1011 } 2821 }
2822 // #6: let the OS probe a peer that has gone silent, so a half-open
2823 // session eventually errors out of recv and lands in the event
2824 // loop's existing reap path instead of sitting in gMiners forever.
2825 // This is a backstop, not the mechanism: the default cadence before
2826 // the first probe is hours on most stacks, so the idle timeout in
2827 // PoolEventLoop fires long before keepalive does. Miners arriving
2828 // via a relay ride a loopback socket pair whose remote transport is
2829 // the rendezvous socket owned by the tunnel pump threads; keepalive
2830 // cannot reach across that, which is part of why the idle timeout
2831 // exists.
2832 int keepalive = 1;
2833 setsockopt(c, SOL_SOCKET, SO_KEEPALIVE, (char*)&keepalive, sizeof(keepalive));
1012 LogPrint("pool", "[pool] miner arrived via loopback -- pushing to event loop\n"); 2834 LogPrint("pool", "[pool] miner arrived via loopback -- pushing to event loop\n");
1013 PushReady((btf_socket_t)c); 2835 PushReady((btf_socket_t)c);
1014 } 2836 }
@@ -1039,6 +2861,30 @@ static bool SendLine(btf_socket_t fd, const json& j)
1039static const size_t MAX_STRATUM_LINE = 64 * 1024; 2861static const size_t MAX_STRATUM_LINE = 64 * 1024;
1040static const size_t MAX_POOL_MINERS = 1000; 2862static const size_t MAX_POOL_MINERS = 1000;
1041 2863
2864// #19 instrumentation: the "broadcast to N miner(s)" line counts connected
2865// sockets, which says nothing about how many are actually mining. These split
2866// the connection census into connected / subscribed / authorized / recently
2867// active, so an operator can tell a real workforce from a pile of half-open
2868// sessions at a glance.
2869static const int64 POOL_CENSUS_SECS = 60; // how often to log the breakdown
2870static const int64 POOL_ACTIVE_WINDOW = 300; // a miner is "active" if it
2871 // authorized and was seen within
2872 // this window (one payout-rotate
2873 // cycle)
2874static const int64 POOL_UNAUTH_GRACE_SECS = 60; // a connection still not
2875 // authorized after this long is
2876 // idle, not merely starting up
2877// #6: a Miner used to leave gMiners only when RecvLine reported a clean
2878// disconnect. A half-open session (miner host died, NAT state expired,
2879// cable pulled) sends no FIN and reports no error, so it sat in the table
2880// forever -- authorize outpaced disconnect 2:1 on the live pool, and the
2881// table grew toward MAX_POOL_MINERS, past which real miners are refused.
2882// lastSeen is stamped on every complete line received; a miner that has
2883// said nothing for this long is gone in every way that matters. Ten
2884// minutes is far beyond the vardiff target (a share every 15s) and the
2885// stall watchdog (45s), so no miner that is actually hashing comes close.
2886static const int64 POOL_IDLE_TIMEOUT_SECS = 600;
2887
1042// Non-blocking recv: pull whatever bytes are available now into buf, then 2888// Non-blocking recv: pull whatever bytes are available now into buf, then
1043// return one complete line if one is ready. Three outcomes: 2889// return one complete line if one is ready. Three outcomes:
1044// true -- line is filled; buf may still contain more lines 2890// true -- line is filled; buf may still contain more lines
@@ -1122,6 +2968,17 @@ static json MakeNotifyParams(const StratumJob& job, const CBlock& b,
1122static void SendJob(Miner* m, bool clean) 2968static void SendJob(Miner* m, bool clean)
1123{ 2969{
1124 if (!gHaveJob) return; 2970 if (!gHaveJob) return;
2971 // Every path that hands a miner a new job funnels through here, so this
2972 // is the one place the replay guard needs to follow the job id: nonces
2973 // credited under the previous job mean nothing under the new one. Keying
2974 // the clear to the id (rather than clearing unconditionally) keeps a
2975 // same-job re-notify -- the vardiff retarget and stall watchdog both
2976 // re-send the CURRENT job -- from reopening the duplicate window it
2977 // exists to close.
2978 if (m->seenNoncesJob != gCurrentJob.jobId) {
2979 m->seenNonces.clear();
2980 m->seenNoncesJob = gCurrentJob.jobId;
2981 }
1125 CBlock mb = BuildMinerBlock(gCurrentJob, m->extranonce2); 2982 CBlock mb = BuildMinerBlock(gCurrentJob, m->extranonce2);
1126 // Use this miner's own shareTarget (set by vardiff) in the notify params. 2983 // Use this miner's own shareTarget (set by vardiff) in the notify params.
1127 // The participant miner reads p[3] as its share target, and HandleLine 2984 // The participant miner reads p[3] as its share target, and HandleLine
@@ -1147,6 +3004,65 @@ static void BroadcastJob(bool clean)
1147} 3004}
1148 3005
1149// --------------------------------------------------------------------------- 3006// ---------------------------------------------------------------------------
3007// #19: connection census. "broadcast to N miner(s)" only counts authorized
3008// sockets that got a job; it says nothing about the rest of the connection
3009// table. This logs the full breakdown so an operator can tell a real workforce
3010// apart from half-open / never-authorized / idle sessions, and flags when the
3011// single select() loop is running out of room (issue #26).
3012//
3013// "Out of room" is not the same question on both platforms, and FD_SETSIZE
3014// answers a different one on each. On POSIX it bounds the *value* of any
3015// descriptor handed to FD_SET, so what matters is the highest fd we hold. On
3016// Windows an fd_set is a counted array of SOCKET handles whose numeric values
3017// are arbitrary, so what matters is how many we hold, not how large they are.
3018// FD_SETSIZE comes from <sys/select.h> or winsock2.h; this file must not define
3019// it, because redefining it after those headers changes nothing.
3020// ---------------------------------------------------------------------------
3021static void LogPoolConnectionCensus()
3022{
3023 int connected = 0, subscribed = 0, authorized = 0, active = 0, neverAuthIdle = 0;
3024 int64 now = GetTime();
3025#ifndef _WIN32
3026 int maxfd = 0;
3027#endif
3028 for (Miner* m : gMiners) {
3029 connected++;
3030#ifndef _WIN32
3031 if ((int)m->fd > maxfd) maxfd = (int)m->fd;
3032#endif
3033 if (m->subscribed) subscribed++;
3034 if (m->authorised) {
3035 authorized++;
3036 if (m->lastSeen > 0 && now - m->lastSeen <= POOL_ACTIVE_WINDOW)
3037 active++;
3038 } else if (now - m->connectTime >= POOL_UNAUTH_GRACE_SECS) {
3039 neverAuthIdle++; // past the grace period and still not authorized
3040 }
3041 }
3042
3043#ifndef _WIN32
3044 int headroom = (int)FD_SETSIZE - maxfd;
3045 LogPrint("pool", "[pool] census: connected=%d subscribed=%d authorized=%d"
3046 " active(last %llds)=%d never-auth-idle=%d"
3047 " (cap=%zu, maxfd=%d, FD_SETSIZE=%d)\n",
3048 connected, subscribed, authorized, (long long)POOL_ACTIVE_WINDOW,
3049 active, neverAuthIdle, MAX_POOL_MINERS, maxfd, (int)FD_SETSIZE);
3050#else
3051 int headroom = (int)FD_SETSIZE - connected;
3052 LogPrint("pool", "[pool] census: connected=%d subscribed=%d authorized=%d"
3053 " active(last %llds)=%d never-auth-idle=%d"
3054 " (cap=%zu, FD_SETSIZE=%d sockets)\n",
3055 connected, subscribed, authorized, (long long)POOL_ACTIVE_WINDOW,
3056 active, neverAuthIdle, MAX_POOL_MINERS, (int)FD_SETSIZE);
3057#endif
3058 if (headroom <= 16)
3059 LogPrint("pool", "[pool] WARNING: %d slots left before FD_SETSIZE=%d --"
3060 " the select() loop is about to walk into undefined behaviour"
3061 " (#26). Refuse new miners or rebuild with poll/epoll.\n",
3062 headroom, (int)FD_SETSIZE);
3063}
3064
3065// ---------------------------------------------------------------------------
1150// Stratum message handler -- called for each complete line from a miner 3066// Stratum message handler -- called for each complete line from a miner
1151// Returns false if the miner should be disconnected immediately. 3067// Returns false if the miner should be disconnected immediately.
1152// --------------------------------------------------------------------------- 3068// ---------------------------------------------------------------------------
@@ -1214,6 +3130,7 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1214 // ------------------------------------------------------------------ 3130 // ------------------------------------------------------------------
1215 if (method == "mining.subscribe") { 3131 if (method == "mining.subscribe") {
1216 std::string sid = ToHex(&m->fd, 4); 3132 std::string sid = ToHex(&m->fd, 4);
3133 m->subscribed = true;
1217 LogPrint("worker", "[worker->pool] subscribe from fd=%d sid=%s\n", 3134 LogPrint("worker", "[worker->pool] subscribe from fd=%d sid=%s\n",
1218 (int)m->fd, sid.c_str()); 3135 (int)m->fd, sid.c_str());
1219 reply(json::array({json::array({json::array({"mining.notify",sid})}),sid,4})); 3136 reply(json::array({json::array({json::array({"mining.notify",sid})}),sid,4}));
@@ -1320,6 +3237,22 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1320 } 3237 }
1321 unsigned int nNonce; memcpy(&nNonce, nb.data(), 4); 3238 unsigned int nNonce; memcpy(&nNonce, nb.data(), 4);
1322 3239
3240 // Reject a replayed (job, nonce) before it costs a rebuild and a
3241 // RandomX hash: nothing about the work below distinguishes a second
3242 // submission of a valid share from the first, so without this check
3243 // it passes every test and is credited again -- and roundShareCount
3244 // is what the payout split divides by, so each replay multiplies the
3245 // address's cut of the round. The set is per-connection and cleared
3246 // by SendJob on every job change, so only a same-job repeat is
3247 // refused.
3248 if (m->seenNonces.count(nNonce)) {
3249 LogPrint("worker", "[worker->pool] duplicate share from %s"
3250 " nonce=%u job=%s -- already credited, not counting again\n",
3251 m->address.c_str(), nNonce, submitJobId.c_str());
3252 reply(false, "Duplicate share");
3253 return true;
3254 }
3255
1323 // Rebuild this miner's exact block (unique merkle via extranonce2) 3256 // Rebuild this miner's exact block (unique merkle via extranonce2)
1324 CBlock b = BuildMinerBlock(gCurrentJob, m->extranonce2); 3257 CBlock b = BuildMinerBlock(gCurrentJob, m->extranonce2);
1325 b.nNonce = nNonce; 3258 b.nNonce = nNonce;
@@ -1347,6 +3280,7 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1347 m->roundShares++; 3280 m->roundShares++;
1348 m->totalShares++; 3281 m->totalShares++;
1349 m->vardiffShares++; 3282 m->vardiffShares++;
3283 m->seenNonces.insert(nNonce); // replay guard: credit each nonce once per job
1350 int64 shareTime = GetTime(); 3284 int64 shareTime = GetTime();
1351 m->lastSeen = shareTime; 3285 m->lastSeen = shareTime;
1352 roundShareCount[m->address]++; 3286 roundShareCount[m->address]++;
@@ -1453,7 +3387,11 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1453 int foundHeight = gCurrentJob.height; 3387 int foundHeight = gCurrentJob.height;
1454 // b is the block we submitted, untouched by ProcessBlock (which 3388 // b is the block we submitted, untouched by ProcessBlock (which
1455 // took a copy), so its hash is what the payout must be keyed on. 3389 // took a copy), so its hash is what the payout must be keyed on.
1456 QueuePayouts(foundHeight, b.GetHash(), roundShareCount, roundShareTotal); 3390 // Passed as a uint256, not hex: the ledger key must stay a real
3391 // hash. The remaining arguments are upstream's round-proof data.
3392 QueuePayouts(foundHeight, b.GetHash(), b.vtx[0].GetHash().GetHex(),
3393 b.vtx[0].vout[0].nValue, m->address,
3394 roundShareCount, roundShareTotal);
1457 // Reset round shares 3395 // Reset round shares
1458 roundShareCount.clear(); 3396 roundShareCount.clear();
1459 roundShareTotal = 0; 3397 roundShareTotal = 0;
@@ -1480,49 +3418,1892 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1480} 3418}
1481 3419
1482// --------------------------------------------------------------------------- 3420// ---------------------------------------------------------------------------
1483// Stats refresh -- called every 2s from event loop 3421// payout-ledger self-test
3422//
3423// Regression cover for the payout defects fixed in this series. Everything
3424// here runs against a scratch data directory and an in-memory wallet map; it
3425// never opens wallet.dat, never touches a real datadir, and never sends
3426// anything.
1484// --------------------------------------------------------------------------- 3427// ---------------------------------------------------------------------------
1485static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal) 3428
3429static bool PayoutTestReadFile(const std::string& path, std::string& out)
1486{ 3430{
1487 std::lock_guard<std::mutex> lk(gStatsMutex); 3431 FILE* f = fopen(path.c_str(), "r");
3432 if (!f) return false;
3433 fseek(f, 0, SEEK_END);
3434 long sz = ftell(f);
3435 fseek(f, 0, SEEK_SET);
3436 out.assign((size_t)(sz > 0 ? sz : 0), '\0');
3437 if (sz > 0 && fread(&out[0], 1, (size_t)sz, f) != (size_t)sz) {
3438 fclose(f);
3439 return false;
3440 }
3441 fclose(f);
3442 return true;
3443}
1488 3444
1489 gStats.authorizedMiners = 0; 3445// Delete the scratch data directory and everything in it. A sweep and not a
1490 gStats.blocksFound = blocksFoundThisSession; 3446// list of known names: LogPrint binds debug.log to whatever GetAppDir() is
1491 gStats.roundShares = roundShareTotal; 3447// when it first fires, so the test's own log lands in here too, and a
1492 gStats.totalHashRate = 0.0; 3448// directory left behind in src/ after `make tests` is litter.
1493 for (Miner* m : gMiners) 3449static void PayoutTestRemoveDir(const std::string& dir)
1494 if (m->authorised) gStats.authorizedMiners++; 3450{
3451#ifdef _WIN32
3452 WIN32_FIND_DATAA findData;
3453 HANDLE hFind = FindFirstFileA((dir + "\\*").c_str(), &findData);
3454 if (hFind != INVALID_HANDLE_VALUE) {
3455 do {
3456 std::string name = findData.cFileName;
3457 if (name == "." || name == "..") continue;
3458 DeleteFileA((dir + "\\" + name).c_str());
3459 } while (FindNextFileA(hFind, &findData));
3460 FindClose(hFind);
3461 }
3462 RemoveDirectoryA(dir.c_str());
3463#else
3464 // One level of recursion, because CreateTransaction() opens a Berkeley DB
3465 // environment and BDB puts its logs in a "database" subdirectory. Without
3466 // this the scratch directory cannot be removed and the test leaves one
3467 // behind on every run, which is what d5b3503 was for.
3468 DIR* d = opendir(dir.c_str());
3469 if (d) {
3470 struct dirent* ent;
3471 while ((ent = readdir(d)) != NULL) {
3472 std::string name = ent->d_name;
3473 if (name == "." || name == "..") continue;
3474 const std::string path = dir + "/" + name;
3475 if (unlink(path.c_str()) != 0)
3476 PayoutTestRemoveDir(path);
3477 }
3478 closedir(d);
3479 }
3480 rmdir(dir.c_str());
3481#endif
3482}
1495 3483
1496 gWorkerStats.clear(); 3484static bool PayoutTestWriteFile(const std::string& path, const std::string& data)
1497 for (Miner* m : gMiners) { 3485{
1498 if (!m->authorised || m->lastSeen <= 0) continue; 3486 FILE* f = fopen(path.c_str(), "w");
1499 PoolWorkerStatView v; 3487 if (!f) return false;
1500 v.address = m->address; 3488 bool ok = data.empty() || fwrite(data.c_str(), 1, data.size(), f) == data.size();
1501 v.worker = m->worker; 3489 return fclose(f) == 0 && ok;
1502 v.roundShares = m->roundShares; 3490}
1503 v.totalShares = m->totalShares; 3491
1504 v.lastSeen = m->lastSeen; 3492// Wipe every piece of payout state this test touches, so each section starts
1505 v.hashRate = m->hashRate; 3493// from the same place and a "restart" can be simulated by reloading from disk.
1506 gWorkerStats.push_back(v); 3494static void PayoutTestResetState()
1507 gStats.totalHashRate += m->hashRate; 3495{
1508 } 3496 {
1509 std::sort(gWorkerStats.begin(), gWorkerStats.end(), 3497 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1510 [](const PoolWorkerStatView& a, const PoolWorkerStatView& b) { 3498 gPendingPayouts.clear();
1511 return a.roundShares != b.roundShares ? a.roundShares > b.roundShares 3499 }
1512 : a.lastSeen > b.lastSeen; 3500 {
3501 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3502 gPaying.clear();
3503 gQueuedPayouts.clear();
3504 }
3505 {
3506 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
3507 gPoolRounds.clear();
3508 }
3509 {
3510 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
3511 gLegacyClaimedTxids.clear();
3512 }
3513 {
3514 // A quarantine is meant to last the life of the process; each section
3515 // here is a fresh process as far as the code under test is concerned.
3516 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
3517 gRefusedFiles.clear();
3518 }
3519}
3520
3521// A BTF address whose hash160 is entirely under the test's control, so the
3522// output script the probe compares against can be built without a wallet.
3523static std::string PayoutTestAddress(unsigned char seed)
3524{
3525 uint160 h = 0;
3526 for (int i = 0; i < 20; i++)
3527 ((unsigned char*)&h)[i] = (unsigned char)(seed + i);
3528 return Hash160ToAddress(h);
3529}
3530
3531static CScript PayoutTestScriptFor(const std::string& address)
3532{
3533 uint160 h160;
3534 CScript sc;
3535 if (AddressToHash160(address, h160))
3536 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
3537 return sc;
3538}
3539
3540// Block index entries fabricated by PayoutTestAddWalletTx, so they can be
3541// freed and unhooked when the test finishes. gPayoutTestChainTip is the end of
3542// that chain: blocks are linked onto it rather than each replacing the tip,
3543// because IsInMainChain() is "pnext is set, or I am the tip".
3544static std::vector<CBlockIndex*> gPayoutTestBlockIndexes;
3545static CBlockIndex* gPayoutTestChainTip = NULL;
3546
3547static void PayoutTestClearBlockIndex()
3548{
3549 CRITICAL_BLOCK(cs_main)
3550 {
3551 for (CBlockIndex* p : gPayoutTestBlockIndexes) {
3552 for (std::map<uint256, CBlockIndex*>::iterator it = mapBlockIndex.begin();
3553 it != mapBlockIndex.end(); ) {
3554 if (it->second == p) mapBlockIndex.erase(it++);
3555 else ++it;
3556 }
3557 delete p;
3558 }
3559 gPayoutTestBlockIndexes.clear();
3560 gPayoutTestChainTip = NULL;
3561 pindexBest = NULL;
3562 }
3563}
3564
3565// Put a transaction in the wallet map that looks exactly like one this pool
3566// sent. `nonce` only exists to keep the hashes distinct.
3567//
3568// fConfirmed decides what GetDepthInMainChain() will answer for it, which is
3569// what the legacy probe now gates on. A confirmed one is given a block index
3570// entry of its own: an empty merkle branch hashes to the transaction itself,
3571// so a block whose merkle root IS that hash accepts it at index 0, and
3572// pointing pindexBest at that block puts it in the main chain one deep. An
3573// unconfirmed one keeps hashBlock 0, which is depth 0 -- the shape a payout
3574// left behind by a reorg has.
3575static std::string PayoutTestAddWalletTx(const std::string& comment,
3576 const std::string& address,
3577 int64 amount,
3578 int nonce,
3579 const std::string& structuredId,
3580 bool fConfirmed = true,
3581 const std::string& structuredBlock = "")
3582{
3583 CWalletTx wtx;
3584 wtx.vin.resize(1);
3585 wtx.vin[0].prevout.SetNull();
3586 wtx.vin[0].scriptSig << nonce;
3587 wtx.vout.resize(1);
3588 wtx.vout[0].nValue = amount;
3589 wtx.vout[0].scriptPubKey = PayoutTestScriptFor(address);
3590 if (!comment.empty())
3591 wtx.mapValue["comment"] = comment;
3592 if (!structuredId.empty()) {
3593 wtx.mapValue["pool_payout_id"] = structuredId;
3594 wtx.mapValue["pool_payout_address"] = address;
3595 wtx.mapValue["pool_payout_amount"] = strprintf("%lld", amount);
3596 if (!structuredBlock.empty())
3597 wtx.mapValue["pool_payout_block"] = structuredBlock;
3598 }
3599 uint256 hash = wtx.GetHash();
3600 if (fConfirmed) {
3601 // A block hash distinct from the transaction's own.
3602 uint256 blockHash = hash;
3603 ((unsigned char*)&blockHash)[0] ^= 0xff;
3604 CRITICAL_BLOCK(cs_main)
3605 {
3606 CBlockIndex* pindex = new CBlockIndex();
3607 pindex->hashMerkleRoot = hash;
3608 // Linked onto the end of the fabricated chain rather than simply
3609 // becoming the new tip: IsInMainChain() is "pnext is set, or I am
3610 // the tip", so a second confirmed transaction would otherwise take
3611 // the first one out of the chain again.
3612 pindex->pprev = gPayoutTestChainTip;
3613 pindex->nHeight = gPayoutTestChainTip ? gPayoutTestChainTip->nHeight + 1 : 1;
3614 if (gPayoutTestChainTip)
3615 gPayoutTestChainTip->pnext = pindex;
3616 gPayoutTestChainTip = pindex;
3617 std::map<uint256, CBlockIndex*>::iterator mi =
3618 mapBlockIndex.insert(std::make_pair(blockHash, pindex)).first;
3619 pindex->phashBlock = &mi->first;
3620 pindexBest = pindex;
3621 gPayoutTestBlockIndexes.push_back(pindex);
3622 }
3623 wtx.hashBlock = blockHash;
3624 wtx.nIndex = 0;
3625 }
3626 CRITICAL_BLOCK(cs_mapWallet)
3627 mapWallet[hash] = wtx;
3628 return hash.GetHex();
3629}
3630
3631// A real key, in the keystore, so a test wallet's outputs are IsMine() and can
3632// actually be signed -- both are needed for CreateTransaction to build the
3633// large transaction the fee schedule is being tested against. Inserted
3634// straight into the key maps rather than through AddKey(), which would also
3635// write it to a wallet.dat.
3636static std::string PayoutTestAddSpendableKey()
3637{
3638 CKey key;
3639 key.MakeNewKey();
3640 std::vector<unsigned char> vchPubKey = key.GetPubKey();
3641 CRITICAL_BLOCK(cs_mapKeys)
3642 {
3643 mapKeys[vchPubKey] = key.GetPrivKey();
3644 mapPubKeys[Hash160(vchPubKey)] = vchPubKey;
3645 }
3646 return Hash160ToAddress(Hash160(vchPubKey));
3647}
3648
3649// Give the wallet `n` spendable outputs of `each`, one transaction apiece --
3650// which is the granularity SelectCoins works at, and therefore the number of
3651// inputs the transaction spending them will carry.
3652//
3653// Deliberately not coinbases: a null prevout would make CWalletTx::GetCredit
3654// return 0 until COINBASE_SPENDABLE_DEPTH, and there is no chain here.
3655static void PayoutTestFundWallet(int n, int64 each, const std::string& address)
3656{
3657 const CScript sc = PayoutTestScriptFor(address);
3658 CRITICAL_BLOCK(cs_mapWallet)
3659 {
3660 for (int i = 0; i < n; i++) {
3661 CWalletTx wtx;
3662 wtx.vin.resize(1);
3663 wtx.vin[0].prevout.hash = (uint64)(i + 1);
3664 wtx.vin[0].prevout.n = 0;
3665 wtx.vout.resize(1);
3666 wtx.vout[0].nValue = each;
3667 wtx.vout[0].scriptPubKey = sc;
3668 mapWallet[wtx.GetHash()] = wtx;
3669 }
3670 }
3671}
3672
3673// Defined below with the event loop; the status-file publication test drives
3674// it directly.
3675static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal);
3676
3677int RunPayoutLedgerSelfTest()
3678{
3679 int nFail = 0;
3680 auto check = [&](bool ok, const char* msg) {
3681 fprintf(stdout, " %s %s\n", ok ? "ok" : "FAIL", msg);
3682 if (!ok) nFail++;
3683 };
3684
3685 fprintf(stdout, "payout-ledger self-test\n");
3686
3687 const std::string savedDataDir = strSetDataDir;
3688 const int savedBestHeight = nBestHeight;
3689
3690 // Scratch data directory in the current working directory. GetAppDir()
3691 // follows strSetDataDir, so this is what PayoutFilePath() and
3692 // PayingFilePath() resolve against for the rest of the test.
3693#ifdef _WIN32
3694 unsigned long pid = (unsigned long)GetCurrentProcessId();
3695#else
3696 unsigned long pid = (unsigned long)getpid();
3697#endif
3698 // Absolute, and the process moves into it.
3699 //
3700 // Both are needed by the CreateTransaction section: Berkeley DB resolves
3701 // its log directory relative to the environment home, so a relative
3702 // datadir makes dbenv.open() fail with DB_RUNRECOVERY; and BDB opens
3703 // db.log relative to the working directory, which is src/ during `make
3704 // tests`. Moving in puts it inside the scratch directory, where it dies
3705 // with everything else.
3706 std::string cwd = ".";
3707 {
3708#ifdef _WIN32
3709 char buf[MAX_PATH];
3710 if (GetCurrentDirectoryA(MAX_PATH, buf) > 0) cwd = buf;
3711#else
3712 char buf[4096];
3713 if (getcwd(buf, sizeof(buf)) != NULL) cwd = buf;
3714#endif
3715 }
3716 std::string dir = cwd + "/" +
3717 strprintf("payout_selftest_%d_%lu", (int)GetTime(), pid);
3718#ifdef _WIN32
3719 CreateDirectoryA(dir.c_str(), NULL);
3720 SetCurrentDirectoryA(dir.c_str());
3721#else
3722 mkdir(dir.c_str(), 0700);
3723 if (chdir(dir.c_str()) != 0) { /* stay put; the paths are absolute anyway */ }
3724#endif
3725 strSetDataDir = dir;
3726 fprintf(stdout, " scratch datadir: %s\n", dir.c_str());
3727
3728 const std::string ledgerPath = PayoutFilePath();
3729 const std::string payingPath = PayingFilePath();
3730 const std::string addrA = PayoutTestAddress(0x11);
3731 const std::string addrB = PayoutTestAddress(0x55);
3732
3733 // -- 1. A restart must not duplicate an in-flight record -----------------
3734 //
3735 // The ledger entry is grandfathered (zero hash) on purpose: it keeps
3736 // PayoutBlockStillOnChain out of the picture, which needs a block index,
3737 // and it exercises the height fallback at the same time.
3738 {
3739 PayoutTestResetState();
3740 remove(ledgerPath.c_str());
3741 remove(payingPath.c_str());
3742
3743 PendingPayout pp;
3744 pp.blockHeight = 4242;
3745 pp.blockHash = 0;
3746 pp.matureAtHeight = 4242;
3747 pp.amounts[addrA] = 10 * COIN;
3748 pp.amounts[addrB] = 20 * COIN;
3749 VersionedBlob snap;
3750 {
3751 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3752 gPendingPayouts.push_back(pp);
3753 snap = SerializePendingPayoutsLocked();
3754 }
3755 PublishPendingPayouts(snap);
3756
3757 nBestHeight = 5000;
3758 FlushMaturePayouts();
3759 size_t afterFirstFlush = 0;
3760 {
3761 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3762 afterFirstFlush = gPaying.size();
3763 }
3764 check(afterFirstFlush == 2,
3765 "a matured round explodes into one record per unpaid recipient");
3766
3767 // Restart: drop everything in memory and come back from the files.
3768 for (int restart = 1; restart <= 2; restart++) {
3769 PayoutTestResetState();
3770 LoadPendingPayouts();
3771 LoadPaying();
3772 ReconcilePayouts();
3773
3774 size_t queued = 0;
3775 {
3776 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3777 queued = gQueuedPayouts.size();
3778 }
3779 if (restart == 1)
3780 check(queued == 1,
3781 "startup rebuilds the exploded-round set from the restored"
3782 " in-flight list");
3783
3784 FlushMaturePayouts();
3785 size_t n = 0;
3786 {
3787 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3788 n = gPaying.size();
3789 }
3790 check(n == 2, restart == 1
3791 ? "a restart does not duplicate an in-flight record"
3792 : "a second restart does not duplicate one either");
3793 }
3794
3795 // The other half of the guard: even with the round-level dedupe set
3796 // deliberately emptied, the per-recipient check against gPaying must
3797 // stop a second standing order for the same money.
3798 {
3799 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3800 gQueuedPayouts.clear();
3801 }
3802 FlushMaturePayouts();
3803 size_t n = 0;
3804 {
3805 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3806 n = gPaying.size();
3807 }
3808 check(n == 2,
3809 "the flush refuses a second record for a (round, address) already"
3810 " in flight even with the round set lost");
3811 }
3812
3813 // -- 2. Duplicates already on disk are collapsed as they are read --------
3814 {
3815 PayoutTestResetState();
3816 nlohmann::json arr = nlohmann::json::array();
3817 for (int i = 0; i < 2; i++) {
3818 nlohmann::json o;
3819 o["blockHeight"] = 4242;
3820 o["blockHash"] = uint256(0).ToString();
3821 o["matureAtHeight"] = 4242;
3822 o["address"] = addrA;
3823 o["amount"] = (int64)(10 * COIN);
3824 o["retries"] = i == 0 ? 7 : 0;
3825 o["deferrals"] = i == 0 ? 0 : 3;
3826 arr.push_back(o);
3827 }
3828 check(PayoutTestWriteFile(payingPath, arr.dump(2)),
3829 "wrote a paying file containing the duplicate the old build made");
3830 LoadPaying();
3831 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3832 check(gPaying.size() == 1,
3833 "loading collapses a duplicate in-flight record from disk");
3834 check(gPaying.size() == 1 && gPaying[0].retries == 7 && gPaying[0].deferrals == 3,
3835 "collapsing merges the attempt counts upwards, not resetting them");
3836 }
3837
3838 // -- 3. State updates address a slot, not a value -----------------------
3839 //
3840 // Build the exact pathology the old code produced -- two records equal in
3841 // every field -- and prove the update lands on the one asked for. Under
3842 // value matching the first record absorbed everything and the second was
3843 // untouchable.
3844 {
3845 PayoutTestResetState();
3846 PayingRecord a;
3847 a.blockHeight = 4242; a.blockHash = 0; a.matureAtHeight = 4242;
3848 a.address = addrA; a.amount = 10 * COIN;
3849 PayingRecord b = a;
3850 {
3851 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3852 a.id = ++gPayingNextId;
3853 b.id = ++gPayingNextId;
3854 gPaying.push_back(a);
3855 gPaying.push_back(b);
3856 }
3857 PayingRecord upd = b;
3858 upd.retries = 5;
3859 upd.retryAfter = 1234567;
3860 ReschedulePayment(upd);
3861 {
3862 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3863 check(gPaying.size() == 2 &&
3864 gPaying[0].retries == 0 && gPaying[1].retries == 5,
3865 "rescheduling updates the record it was given, not the first"
3866 " one that looks like it");
3867 }
3868 ForgetPayment(a);
3869 {
3870 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3871 check(gPaying.size() == 1 && gPaying[0].id == b.id,
3872 "forgetting erases the record it was given, not the first"
3873 " one that looks like it");
3874 }
3875 }
3876
3877 // -- 4. retryAfter survives a restart ------------------------------------
3878 {
3879 PayoutTestResetState();
3880 remove(payingPath.c_str());
3881 PayingRecord r;
3882 r.blockHeight = 77; r.blockHash = 0; r.matureAtHeight = 77;
3883 r.address = addrA; r.amount = 3 * COIN;
3884 r.retryAfter = 1700000000LL;
3885 r.deferrals = 9;
3886 VersionedBlob snap;
3887 {
3888 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3889 r.id = ++gPayingNextId;
3890 gPaying.push_back(r);
3891 snap = SerializePayingLocked();
3892 }
3893 PublishPaying(snap);
3894 PayoutTestResetState();
3895 LoadPaying();
3896 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3897 check(gPaying.size() == 1 && gPaying[0].retryAfter == 1700000000LL &&
3898 gPaying[0].deferrals == 9,
3899 "the retry clock that is actually used survives a restart");
3900 }
3901
3902 // -- 5. The ledger file cannot be published from a stale snapshot --------
3903 {
3904 PayoutTestResetState();
3905 remove(ledgerPath.c_str());
3906
3907 PendingPayout one;
3908 one.blockHeight = 100; one.blockHash = 0; one.matureAtHeight = 100;
3909 one.amounts[addrA] = 1 * COIN;
3910 VersionedBlob older, newer;
3911 {
3912 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3913 gPendingPayouts.push_back(one);
3914 older = SerializePendingPayoutsLocked();
3915 PendingPayout two;
3916 two.blockHeight = 200; two.blockHash = 0; two.matureAtHeight = 200;
3917 two.amounts[addrB] = 2 * COIN;
3918 gPendingPayouts.push_back(two);
3919 newer = SerializePendingPayoutsLocked();
3920 }
3921 check(newer.seq > older.seq,
3922 "a later serialisation carries a strictly higher version");
3923
3924 // The order the race produced: the payout thread's older image
3925 // arriving after the event loop's newer one.
3926 PublishPendingPayouts(newer);
3927 PublishPendingPayouts(older);
3928 std::string onDisk;
3929 check(PayoutTestReadFile(ledgerPath, onDisk) && onDisk == newer.blob,
3930 "an older snapshot cannot overwrite a newer published ledger");
3931
3932 // And the ordinary direction still works.
3933 VersionedBlob newest;
3934 {
3935 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3936 gPendingPayouts.clear();
3937 newest = SerializePendingPayoutsLocked();
3938 }
3939 PublishPendingPayouts(newest);
3940 check(PayoutTestReadFile(ledgerPath, onDisk) && onDisk == newest.blob,
3941 "a newer snapshot does replace the published ledger");
3942 }
3943
3944 // -- 6. Concurrent publishers cannot interleave a file -------------------
3945 //
3946 // Two threads opening and renaming the same "<path>.tmp" over the same
3947 // target was the original defect. This test was mutation-tested and did
3948 // not catch it: with the file mutex removed the suite still went green 2
3949 // runs in 5, and the "never leave it unparseable" assertion fired 0 times
3950 // in 5. Two reasons, both fixed here.
3951 //
3952 // First, every image had the same structure and near enough the same
3953 // length, so an interleaving of two of them was still valid JSON. The
3954 // images now alternate between ONE entry and FIVE THOUSAND, so a write
3955 // that lands on top of another leaves either a truncated array or a small
3956 // image followed by the tail of a large one -- neither of which parses.
3957 //
3958 // Second, and worse, almost nothing was being written. Each thread walked
3959 // a permutation of the images, and PublishVersioned drops any blob that is
3960 // not newer than what is on disk, so only the record highs in each
3961 // permutation ever reached the disk -- on the order of ten writes in
3962 // total, four threads, no contention to speak of. Every publish here is
3963 // stamped from a shared counter at the moment it is made, so every one of
3964 // them is newer than the last and every one of them writes.
3965 //
3966 // And the file is watched while the writers run, not only inspected after
3967 // they finish. "The published file is never observed unparseable" is the
3968 // actual invariant -- rename(2) is atomic, so a reader can only ever see a
3969 // completely written image -- and checking it once at the end samples the
3970 // one moment when a torn file is least likely to still be there.
3971 {
3972 const std::string racePath = dir + "/publish_race_selftest.json";
3973 std::mutex raceMutex;
3974 uint64 racePublished = 0;
3975
3976 std::string tiny, huge;
3977 {
3978 nlohmann::json one = nlohmann::json::array();
3979 nlohmann::json o;
3980 o["image"] = 0;
3981 o["entry"] = 0;
3982 o["address"] = addrA;
3983 o["amount"] = (int64)COIN;
3984 one.push_back(o);
3985 tiny = one.dump(2);
3986
3987 nlohmann::json many = nlohmann::json::array();
3988 for (int k = 0; k < 5000; k++) {
3989 nlohmann::json e;
3990 e["image"] = 1;
3991 e["entry"] = k;
3992 e["address"] = addrA;
3993 e["amount"] = (int64)(k + 1) * COIN;
3994 many.push_back(e);
3995 }
3996 huge = many.dump(2);
3997 }
3998
3999 const int kThreads = 4;
4000 const int kPerThread = 100;
4001 const uint64 kTotal = (uint64)kThreads * kPerThread;
4002
4003 std::atomic<uint64> nextSeq(0);
4004 std::atomic<bool> racing(true);
4005 std::atomic<int> tornReads(0);
4006 std::atomic<int> goodReads(0);
4007
4008 std::thread watcher([&]() {
4009 bool seenOnce = false;
4010 while (racing) {
4011 std::string s;
4012 if (!PayoutTestReadFile(racePath, s)) {
4013 // Only meaningful once the file exists at all.
4014 if (seenOnce) tornReads++;
4015 continue;
4016 }
4017 seenOnce = true;
4018 try {
4019 if (nlohmann::json::parse(s).is_array()) goodReads++;
4020 else tornReads++;
4021 } catch (...) {
4022 tornReads++;
4023 }
4024 }
1513 }); 4025 });
1514 4026
1515 // gPendingPayouts is event-loop-only data (no lock needed here since 4027 std::vector<std::thread> threads;
1516 // RefreshStats is always called from the event loop thread). 4028 for (int t = 0; t < kThreads; t++) {
1517 gPayoutViews.clear(); 4029 threads.push_back(std::thread([&]() {
1518 for (const PendingPayout& pp : gPendingPayouts) { 4030 for (int i = 0; i < kPerThread; i++) {
1519 PendingPayoutView pv; 4031 VersionedBlob v;
1520 pv.matureAtHeight = pp.matureAtHeight; 4032 v.seq = ++nextSeq;
1521 pv.recipients = (int)pp.amounts.size(); 4033 v.blob = (v.seq & 1) ? tiny : huge;
1522 pv.totalAmount = 0; 4034 PublishVersioned(racePath, v, raceMutex, racePublished,
1523 for (auto& kv : pp.amounts) pv.totalAmount += kv.second; 4035 "publish race selftest");
1524 gPayoutViews.push_back(pv); 4036 }
4037 }));
4038 }
4039 for (std::thread& th : threads)
4040 th.join();
4041 racing = false;
4042 watcher.join();
4043
4044 check(goodReads > 0,
4045 "the race test actually observed the file while it was being"
4046 " written");
4047 check(tornReads == 0,
4048 "four threads publishing the same file never leave it"
4049 " unparseable, at any instant while they are doing it");
4050
4051 // The highest sequence number issued is the one the file must settle
4052 // on: PublishVersioned refuses anything older, so whichever order the
4053 // writes complete in, that image is the last one that can land.
4054 const std::string& expected = (kTotal & 1) ? tiny : huge;
4055 std::string onDisk;
4056 bool read = PayoutTestReadFile(racePath, onDisk);
4057 check(read && onDisk == expected,
4058 "the published file settles on the highest version, whatever the"
4059 " completion order");
4060 remove(racePath.c_str());
4061 remove((racePath + ".tmp").c_str());
4062 }
4063
4064 // -- 7. Grandfathered zero-hash rounds do not answer for each other ------
4065 {
4066 uint256 h1 = 0, h2 = 0;
4067 h1.SetHex("00000000000000000000000000000000000000000000000000000000000000aa");
4068 h2.SetHex("00000000000000000000000000000000000000000000000000000000000000bb");
4069 check(!SamePayoutRound(100, 0, 200, 0),
4070 "two zero-hash entries at different heights are different rounds");
4071 check(SamePayoutRound(100, 0, 100, 0),
4072 "two zero-hash entries at the same height are the same round");
4073 check(SamePayoutRound(100, h1, 200, h1),
4074 "a real hash identifies the round on its own");
4075 check(!SamePayoutRound(100, h1, 100, h2),
4076 "different hashes at one height are different rounds");
4077 check(!SamePayoutRound(100, h1, 100, 0),
4078 "a grandfathered entry never matches a hash-keyed one");
4079 check(!PayoutRoundIdentifiable(0, 0) && PayoutRoundIdentifiable(0, h1) &&
4080 PayoutRoundIdentifiable(7, 0),
4081 "an entry with neither hash nor height is not adjudicable");
4082
4083 // What actually makes a grandfathered same-height pair safe, since the
4084 // comment on FindRecordedPoolPayout used to claim it was the orphan
4085 // check. It is not: the orphan check passes every zero-hash entry, and
4086 // every pre-upgrade entry has a zero hash.
4087 check(PayoutBlockStillOnChain(0),
4088 "the orphan check passes every grandfathered entry, so it cannot"
4089 " be what voids a losing twin");
4090 PayoutTestResetState();
4091 remove(ledgerPath.c_str());
4092 remove(payingPath.c_str());
4093 {
4094 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4095 PendingPayout twin;
4096 twin.blockHeight = 300; twin.blockHash = 0; twin.matureAtHeight = 300;
4097 twin.amounts[addrA] = 1 * COIN;
4098 gPendingPayouts.push_back(twin);
4099 gPendingPayouts.push_back(twin); // the losing side of a reorg
4100 }
4101 nBestHeight = 5000;
4102 FlushMaturePayouts();
4103 {
4104 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
4105 check(gPaying.size() == 1,
4106 "two grandfathered rounds at one height are one round, so the"
4107 " pair presents as one debt and is paid at most once");
4108 }
4109
4110 // End to end: recording a payment against one grandfathered round must
4111 // not settle the other's debt.
4112 PayoutTestResetState();
4113 remove(ledgerPath.c_str());
4114 {
4115 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4116 PendingPayout first;
4117 first.blockHeight = 100; first.blockHash = 0; first.matureAtHeight = 100;
4118 first.amounts[addrA] = 1 * COIN;
4119 PendingPayout second;
4120 second.blockHeight = 200; second.blockHash = 0; second.matureAtHeight = 200;
4121 second.amounts[addrA] = 1 * COIN;
4122 gPendingPayouts.push_back(first);
4123 gPendingPayouts.push_back(second);
4124 }
4125 PayingRecord r;
4126 r.blockHeight = 200; r.blockHash = 0; r.matureAtHeight = 200;
4127 r.address = addrA; r.amount = 1 * COIN;
4128 RecordPayoutTxid(r, "txid-for-the-second-round");
4129 {
4130 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4131 // The second round was fully paid and retired; the first is
4132 // untouched and still owed.
4133 check(gPendingPayouts.size() == 1 &&
4134 gPendingPayouts[0].blockHeight == 100 &&
4135 gPendingPayouts[0].paidTxids.empty(),
4136 "a txid recorded against one zero-hash round leaves the other"
4137 " still owed");
4138 }
4139 }
4140
4141 // -- 8. A payment with only a legacy comment is recognised as paid -------
4142 {
4143 PayoutTestResetState();
4144 const int height = 8080;
4145 const int64 amount = 7 * COIN;
4146 const std::string id = PayoutIdForBlock(height, 0);
4147 const std::string comment = LegacyPayoutComment(height);
4148 std::set<std::string> none;
4149 std::string found;
4150 // No round proof exists for these rounds, so legacy evidence is
4151 // admissible -- computed exactly as the payout thread computes it.
4152 const bool fAllowLegacy = !PoolRoundHasProof(id);
4153
4154 check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, none, found)
4155 == PAYOUT_PROBE_NONE,
4156 "an empty wallet is not evidence that anything was paid");
4157
4158 const std::string legacyTxid =
4159 PayoutTestAddWalletTx(comment, addrA, amount, 1, "");
4160 check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, none, found)
4161 == PAYOUT_PROBE_LEGACY && found == legacyTxid,
4162 "a payment carrying only a pre-upgrade comment is recognised as"
4163 " already paid");
4164
4165 check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found)
4166 == PAYOUT_PROBE_NONE,
4167 "the legacy match requires the output to pay THIS recipient");
4168 check(FindRecordedPoolPayout(id, height, 0, addrA, amount + COIN, fAllowLegacy, none, found)
4169 == PAYOUT_PROBE_NONE,
4170 "the legacy match requires the output to pay exactly this amount");
4171 check(FindRecordedPoolPayout(id, height + 1, 0, addrA, amount, fAllowLegacy, none, found)
4172 == PAYOUT_PROBE_NONE,
4173 "the legacy match requires the comment for this block height");
4174
4175 std::set<std::string> spent;
4176 spent.insert(legacyTxid);
4177 check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, spent, found)
4178 == PAYOUT_PROBE_NONE,
4179 "one legacy transaction cannot settle a second debt");
4180
4181 // A transaction written by THIS build carries pool_payout_id, and must
4182 // never be offered as legacy evidence for a different round -- that is
4183 // exactly what the structured metadata exists to prevent.
4184 const std::string otherId = PayoutIdForBlock(height, 0) + "-other";
4185 PayoutTestAddWalletTx(comment, addrB, amount, 2, otherId);
4186 check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found)
4187 == PAYOUT_PROBE_NONE,
4188 "another round's structured payment is not legacy evidence for"
4189 " this one");
4190
4191 // The structured path still wins where it applies.
4192 const std::string structuredTxid =
4193 PayoutTestAddWalletTx(comment, addrB, amount, 3, id);
4194 check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found)
4195 == PAYOUT_PROBE_STRUCTURED && found == structuredTxid,
4196 "structured payout metadata is matched in preference to a comment");
4197
4198 // A comment-matching transaction that never made it into the chain is
4199 // evidence of NON-payment. This tree has seen a 280-block reorg; a
4200 // pre-upgrade payout invalidated by one is still in mapWallet with its
4201 // comment intact, and the first boot after the upgrade is exactly when
4202 // it would be matched.
4203 CRITICAL_BLOCK(cs_mapWallet)
4204 mapWallet.clear();
4205 PayoutTestClearBlockIndex();
4206 const std::string addrC = PayoutTestAddress(0x33);
4207 PayoutTestAddWalletTx(comment, addrC, amount, 4, "", false);
4208 check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, none, found)
4209 == PAYOUT_PROBE_NONE,
4210 "a pre-upgrade payout that never confirmed is not read as"
4211 " payment");
4212
4213 // The same transaction, confirmed, is.
4214 CRITICAL_BLOCK(cs_mapWallet)
4215 mapWallet.clear();
4216 const std::string confirmedTxid =
4217 PayoutTestAddWalletTx(comment, addrC, amount, 4, "", true);
4218 check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, none, found)
4219 == PAYOUT_PROBE_LEGACY && found == confirmedTxid,
4220 "the same payout, confirmed, is");
4221
4222 // And an unconfirmed one does not shadow a confirmed one that would
4223 // have matched: the probe used to stop at the first comment match.
4224 const std::string alsoConfirmed =
4225 PayoutTestAddWalletTx(comment, addrC, amount, 6, "", true);
4226 PayoutTestAddWalletTx(comment, addrC, amount, 5, "", false);
4227 std::set<std::string> spentFirst;
4228 spentFirst.insert(confirmedTxid);
4229 check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, spentFirst, found)
4230 == PAYOUT_PROBE_LEGACY && found == alsoConfirmed,
4231 "an unconfirmed transaction does not hide a confirmed one behind"
4232 " it");
4233
4234 CRITICAL_BLOCK(cs_mapWallet)
4235 mapWallet.clear();
4236 PayoutTestClearBlockIndex();
1525 } 4237 }
4238
4239 // -- 9. On-disk compatibility with the live production ledger ------------
4240 {
4241 PayoutTestResetState();
4242 // Exactly the shape of the file on the pool node: amounts, blockHash,
4243 // blockHeight, matureAtHeight, and nothing else.
4244 const char* live =
4245 "[{\"amounts\":{\"%s\":1234500000},"
4246 "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000cd\","
4247 "\"blockHeight\":1000,"
4248 "\"matureAtHeight\":1120}]";
4249 check(PayoutTestWriteFile(ledgerPath, strprintf(live, addrA.c_str())),
4250 "wrote a ledger file in the live production shape");
4251 LoadPendingPayouts();
4252 {
4253 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4254 check(gPendingPayouts.size() == 1 &&
4255 gPendingPayouts[0].blockHeight == 1000 &&
4256 gPendingPayouts[0].matureAtHeight == 1120 &&
4257 gPendingPayouts[0].blockHash != 0 &&
4258 gPendingPayouts[0].amounts.size() == 1 &&
4259 gPendingPayouts[0].amounts[addrA] == 1234500000LL,
4260 "the live production pending_payouts.json still loads");
4261 }
4262
4263 // And the shape the merged build writes, retryAfter included, which
4264 // this build no longer writes but must still read past.
4265 PayoutTestResetState();
4266 const char* merged =
4267 "[{\"amounts\":{\"%s\":100000000},"
4268 "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000ef\","
4269 "\"blockHeight\":2000,\"matureAtHeight\":2120,\"retryAfter\":0,"
4270 "\"paidTxids\":{\"%s\":\"abc123\"}}]";
4271 check(PayoutTestWriteFile(ledgerPath,
4272 strprintf(merged, addrA.c_str(), addrA.c_str())),
4273 "wrote a ledger file in the merged build's shape");
4274 LoadPendingPayouts();
4275 {
4276 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4277 check(gPendingPayouts.size() == 1 &&
4278 gPendingPayouts[0].blockHeight == 2000 &&
4279 gPendingPayouts[0].paidTxids.size() == 1 &&
4280 gPendingPayouts[0].paidTxids[addrA] == "abc123",
4281 "a ledger file carrying retryAfter and paidTxids still loads");
4282 }
4283
4284 // A round trip through this build must not lose anything either.
4285 VersionedBlob snap;
4286 {
4287 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4288 snap = SerializePendingPayoutsLocked();
4289 }
4290 PublishPendingPayouts(snap);
4291 PayoutTestResetState();
4292 LoadPendingPayouts();
4293 {
4294 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4295 check(gPendingPayouts.size() == 1 &&
4296 gPendingPayouts[0].blockHeight == 2000 &&
4297 gPendingPayouts[0].matureAtHeight == 2120 &&
4298 gPendingPayouts[0].paidTxids[addrA] == "abc123",
4299 "what this build writes, this build reads back unchanged");
4300 }
4301 }
4302
4303 // -- 10. A forfeit is published, not just dropped ------------------------
4304 {
4305 PayoutTestResetState();
4306 remove(ledgerPath.c_str());
4307 const std::string roundsPath = dir + "/pool_rounds_selftest.json";
4308 const std::string savedRoundsFile = strPoolRoundsFile;
4309 strPoolRoundsFile = roundsPath;
4310 remove(roundsPath.c_str());
4311
4312 const int height = 606;
4313 PoolRoundProof round;
4314 round.id = PayoutIdForBlock(height, 0);
4315 round.blockHeight = height;
4316 round.matureAtHeight = height;
4317 round.payouts[addrA] = 5 * COIN;
4318 round.payouts[addrB] = 6 * COIN;
4319 RecordPoolRound(round);
4320
4321 {
4322 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4323 PendingPayout pp;
4324 pp.blockHeight = height; pp.blockHash = 0; pp.matureAtHeight = height;
4325 pp.amounts[addrA] = 5 * COIN;
4326 pp.amounts[addrB] = 6 * COIN;
4327 gPendingPayouts.push_back(pp);
4328 }
4329
4330 PayingRecord r;
4331 r.blockHeight = height; r.blockHash = 0; r.matureAtHeight = height;
4332 r.address = addrA; r.amount = 5 * COIN;
4333 ForfeitPayoutRecipient(r, "selftest reason");
4334
4335 {
4336 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4337 check(gPendingPayouts.size() == 1 &&
4338 gPendingPayouts[0].amounts.count(addrA) == 0 &&
4339 gPendingPayouts[0].amounts.count(addrB) == 1,
4340 "a forfeit removes only that recipient from the ledger entry");
4341 }
4342
4343 std::string onDisk;
4344 bool proofOk = false;
4345 if (PayoutTestReadFile(roundsPath, onDisk)) {
4346 try {
4347 nlohmann::json root = nlohmann::json::parse(onDisk);
4348 const nlohmann::json& p = root["rounds"][0]["payouts"];
4349 proofOk = p[addrA].value("forfeited", false) &&
4350 p[addrA].value("forfeitReason", "") == "selftest reason" &&
4351 p[addrA]["txid"].is_null() &&
4352 !p[addrB].contains("forfeited");
4353 } catch (...) {
4354 proofOk = false;
4355 }
4356 }
4357 check(proofOk,
4358 "the published round proof records the forfeit instead of"
4359 " advertising the address as still owed");
4360
4361 // And it survives a reload of the proof file.
4362 {
4363 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
4364 gPoolRounds.clear();
4365 }
4366 LoadPoolRounds();
4367 {
4368 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
4369 check(gPoolRounds.size() == 1 &&
4370 gPoolRounds[0].forfeited.count(addrA) == 1 &&
4371 gPoolRounds[0].forfeited[addrA] == "selftest reason",
4372 "the forfeit is read back from pool_rounds.json");
4373 }
4374
4375 remove(roundsPath.c_str());
4376 remove((roundsPath + ".tmp").c_str());
4377 strPoolRoundsFile = savedRoundsFile;
4378 }
4379
4380 // -- 11. A ledger that failed to load is never written back --------------
4381 //
4382 // The loaders push entries into the live vector inside the try, so an
4383 // exception part way down a file leaves a PARTIAL ledger in memory: the
4384 // entries before the bad one and nothing after it. Publishing that -- and
4385 // ReconcilePayouts publishes unconditionally, milliseconds into startup --
4386 // writes the partial state over the file and destroys the only record of
4387 // what was owed.
4388 {
4389 PayoutTestResetState();
4390 remove(ledgerPath.c_str());
4391
4392 // Two entries, the second with a matureAtHeight that is not a number.
4393 // The first loads; the second throws where it is read.
4394 const std::string corrupt = strprintf(
4395 "[{\"amounts\":{\"%s\":100000000},\"blockHash\":\"%s\","
4396 "\"blockHeight\":10,\"matureAtHeight\":10},"
4397 "{\"amounts\":{\"%s\":900000000},\"blockHash\":\"%s\","
4398 "\"blockHeight\":20,\"matureAtHeight\":\"twenty\"}]",
4399 addrA.c_str(), uint256(0).ToString().c_str(),
4400 addrB.c_str(), uint256(0).ToString().c_str());
4401 check(PayoutTestWriteFile(ledgerPath, corrupt),
4402 "wrote a ledger whose second entry does not parse");
4403
4404 const int64 t0 = GetTime();
4405 LoadPendingPayouts();
4406 const int64 t1 = GetTime();
4407
4408 size_t loaded = 0;
4409 {
4410 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4411 loaded = gPendingPayouts.size();
4412 }
4413 check(loaded == 1,
4414 "a mid-file parse error does leave a partial ledger in memory --"
4415 " which is the reason for everything below");
4416
4417 std::string gone;
4418 check(!PayoutTestReadFile(ledgerPath, gone),
4419 "the file that did not parse is moved out of the way");
4420
4421 // The rename stamps GetTime(); it was taken between t0 and t1.
4422 std::string kept, keptPath;
4423 for (int64 t = t0; t <= t1 && keptPath.empty(); t++) {
4424 const std::string candidate =
4425 strprintf("%s.bad.%lld", ledgerPath.c_str(), (long long)t);
4426 if (PayoutTestReadFile(candidate, kept))
4427 keptPath = candidate;
4428 }
4429 check(!keptPath.empty() && kept == corrupt,
4430 "it is kept, byte for byte, as <path>.bad.<timestamp>");
4431
4432 // The publish that used to destroy it.
4433 VersionedBlob partial;
4434 {
4435 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4436 partial = SerializePendingPayoutsLocked();
4437 }
4438 PublishPendingPayouts(partial);
4439 check(!PayoutTestReadFile(ledgerPath, gone),
4440 "and the partial ledger is not published over it");
4441
4442 // The refusal is per path and lasts the process: a second attempt,
4443 // with more state, is refused too.
4444 {
4445 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4446 PendingPayout pp;
4447 pp.blockHeight = 30; pp.blockHash = 0; pp.matureAtHeight = 30;
4448 pp.amounts[addrA] = 5 * COIN;
4449 gPendingPayouts.push_back(pp);
4450 partial = SerializePendingPayoutsLocked();
4451 }
4452 PublishPendingPayouts(partial);
4453 check(!PayoutTestReadFile(ledgerPath, gone),
4454 "the refusal is not a one-shot -- the path stays unwritable");
4455
4456 if (!keptPath.empty())
4457 remove(keptPath.c_str());
4458 PayoutTestResetState();
4459 remove(ledgerPath.c_str());
4460
4461 // And a file that parses is not quarantined, so the ordinary path is
4462 // still the ordinary path.
4463 check(PayoutTestWriteFile(ledgerPath, "[]"),
4464 "wrote a ledger that parses");
4465 LoadPendingPayouts();
4466 {
4467 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4468 PendingPayout pp;
4469 pp.blockHeight = 40; pp.blockHash = 0; pp.matureAtHeight = 40;
4470 pp.amounts[addrA] = 2 * COIN;
4471 gPendingPayouts.push_back(pp);
4472 partial = SerializePendingPayoutsLocked();
4473 }
4474 PublishPendingPayouts(partial);
4475 std::string written;
4476 check(PayoutTestReadFile(ledgerPath, written) && written == partial.blob,
4477 "a ledger that loaded cleanly is still published normally");
4478 }
4479
4480 // -- 12. A send that failed for want of coin is not a fault --------------
4481 //
4482 // The defect this replaces: the classifier decided "was this a funding
4483 // problem?" by re-reading GetBalance() and allowing amount +
4484 // nTransactionFee + CENT. nTransactionFee is 0 on a headless node, and
4485 // GetMinFee() is a step function -- 0 below 10,000 bytes, 11 * CENT at it
4486 // -- so the allowance was either unnecessary or eleven times too small,
4487 // and the ledger's own retry budget then wrote the debt off after 1440
4488 // repeats of the same stable condition.
4489 //
4490 // This is tested against a real wallet and the real CreateTransaction, not
4491 // against injected values: the honest question is not "does my enum
4492 // switch work" but "does a wallet holding many small coins actually report
4493 // the state I think it does". 60 outputs of 1 BTF give ~180 signed P2PKH
4494 // bytes each, which is where the fee schedule steps. Nothing is committed
4495 // or broadcast -- CreateTransaction only builds.
4496 {
4497 PayoutTestResetState();
4498 CRITICAL_BLOCK(cs_mapWallet)
4499 mapWallet.clear();
4500
4501 // CreateTransaction opens CTxDB("r"), which throws if blkindex.dat is
4502 // not there. Create it in the scratch datadir first.
4503 bool dbOk = true;
4504 try {
4505 CTxDB txdb("cr");
4506 } catch (const std::exception&) {
4507 dbOk = false;
4508 }
4509 check(dbOk, "a scratch block index exists for CreateTransaction to open");
4510
4511 const std::string walletAddr = PayoutTestAddSpendableKey();
4512 const std::string payee = PayoutTestAddress(0x77);
4513 const CScript payeeScript = PayoutTestScriptFor(payee);
4514
4515 // -- the >= 10 KB case: 60 small coins, balance = amount + 0.05 ------
4516 PayoutTestFundWallet(60, 1 * COIN, walletAddr);
4517 const int64 balance = GetBalance();
4518 check(balance == 60 * COIN,
4519 "the test wallet holds 60 separately spendable coins");
4520
4521 const int64 amount = balance - 5 * CENT;
4522 // The pre-send gate in PayoutThreadFn passes: the balance covers the
4523 // amount and the only fee that can be known in advance.
4524 check(balance >= amount + nTransactionFee,
4525 "the pre-send gate would let this payment through");
4526
4527 CWalletTx wtx;
4528 SendMoneyOutcome outcome;
4529 int64 feeRequired = 0;
4530 WalletSendResult result = WALLET_SEND_OK;
4531 bool built = true;
4532 if (dbOk) {
4533 try {
4534 built = CreateTransaction(payeeScript, amount, wtx, feeRequired,
4535 &result);
4536 } catch (const std::exception&) {
4537 built = true; // report as a failure below
4538 }
4539 }
4540 check(dbOk && !built,
4541 "a wallet of many small coins cannot fund amount + the fee its"
4542 " own size demands");
4543 check(dbOk && result == WALLET_SEND_INSUFFICIENT_FUNDS,
4544 "the wallet reports that as a funding shortfall, not as a"
4545 " failure to build");
4546 // GetMinFee(fDiscount) returns 0 below 10,000 bytes and
4547 // (1 + bytes/1000) * CENT at or above, so a required fee of 11 CENT or
4548 // more IS the statement that the transaction crossed 10 KB. Measured
4549 // through the real serialiser rather than asserted.
4550 check(dbOk && feeRequired >= 11 * CENT,
4551 "the transaction it built crossed 10,000 bytes, so the fee"
4552 " stepped from 0 to at least 11 CENT");
4553
4554 // The defect, stated as arithmetic. The old classifier asked whether
4555 // the balance covered amount + nTransactionFee + CENT; it did, so it
4556 // called a permanent funding shortfall a fault and started counting
4557 // towards a write-off.
4558 check(!(balance < amount + nTransactionFee + CENT),
4559 "the reserve this replaces would have judged this send a fault");
4560 check(balance < amount + feeRequired,
4561 "while the fee actually required was more than the balance had");
4562
4563 outcome.result = result;
4564 outcome.nFeeRequired = feeRequired;
4565 check(dbOk && ClassifyPayoutSendFailure(outcome.result, false)
4566 == PAYOUT_FAILURE_DEFER,
4567 "a >= 10 KB send that ran out of coin is deferred, not counted"
4568 " against the payment");
4569
4570 // -- the < 10 KB case: no reserve is needed at all -------------------
4571 //
4572 // The other half of the arithmetic. Below the step the required fee is
4573 // 0, so a reserve of any size is a reason to refuse a payment the
4574 // wallet could have made -- which is its own way of not paying a
4575 // miner.
4576 CRITICAL_BLOCK(cs_mapWallet)
4577 mapWallet.clear();
4578 PayoutTestFundWallet(3, 1 * COIN, walletAddr);
4579 const int64 smallAmount = 3 * COIN - 5 * CENT;
4580 CWalletTx smallTx;
4581 int64 smallFee = -1;
4582 WalletSendResult smallResult = WALLET_SEND_FAILED;
4583 bool smallBuilt = false;
4584 if (dbOk) {
4585 try {
4586 smallBuilt = CreateTransaction(payeeScript, smallAmount, smallTx,
4587 smallFee, &smallResult);
4588 } catch (const std::exception&) {
4589 smallBuilt = false;
4590 }
4591 }
4592 check(dbOk && smallBuilt && smallResult == WALLET_SEND_OK,
4593 "the same balance pays a small transaction with no headroom at"
4594 " all");
4595 check(dbOk && smallFee == 0,
4596 "because below 10,000 bytes the required fee is zero");
4597
4598 // -- and the classification of every other answer --------------------
4599 check(ClassifyPayoutSendFailure(WALLET_SEND_INSUFFICIENT_FUNDS, false)
4600 == PAYOUT_FAILURE_DEFER &&
4601 ClassifyPayoutSendFailure(WALLET_SEND_INSUFFICIENT_FUNDS, true)
4602 == PAYOUT_FAILURE_DEFER &&
4603 ClassifyPayoutSendFailure(WALLET_SEND_FAILED, true)
4604 == PAYOUT_FAILURE_DEFER,
4605 "short of coin or unable to sign is always a deferral");
4606 check(ClassifyPayoutSendFailure(WALLET_SEND_FAILED, false)
4607 == PAYOUT_FAILURE_FAULT,
4608 "funded and unlocked and still refused is a fault");
4609
4610 CRITICAL_BLOCK(cs_mapWallet)
4611 mapWallet.clear();
4612 }
4613
4614 // -- 13. No payout mutex is held across the status-file write ------------
4615 //
4616 // RefreshStats runs on the event loop -- the thread that owns every
4617 // miner's socket -- and used to hold gStatsMutex and gPayoutLedgerMutex at
4618 // function scope across WriteFileAtomic. Once that write gained fsync() on
4619 // the file and on its directory, publishing pool_status.json parked the
4620 // payout thread behind two disk flushes every ten seconds.
4621 //
4622 // Timing assertions would be flaky, so this does not measure anything. It
4623 // makes the write block for as long as the test likes: strPoolStatusFile
4624 // points at a path whose ".tmp" is a FIFO, and open(2) for writing on a
4625 // FIFO does not return until a reader arrives. While RefreshStats is
4626 // parked in there, every mutex it must NOT be holding has to be
4627 // acquirable. Against the previous code this test does not merely fail, it
4628 // cannot pass: the writer is inside fopen() with both mutexes held and
4629 // nothing will release them until the test lets the write finish.
4630#ifndef _WIN32
4631 {
4632 PayoutTestResetState();
4633 const std::string statusPath = dir + "/pool_status_selftest.json";
4634 const std::string statusTmp = statusPath + ".tmp";
4635 const std::string savedStatusFile = strPoolStatusFile;
4636 const int64 savedLastWrite = gLastPoolStatusWrite;
4637 remove(statusPath.c_str());
4638 remove(statusTmp.c_str());
4639 strPoolStatusFile = statusPath;
4640 gLastPoolStatusWrite = 0; // due a publish on the next refresh
4641
4642 // Give the status file something to report, so the ledger mutex is
4643 // genuinely taken on the way through rather than skipped over.
4644 {
4645 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4646 PendingPayout pp;
4647 pp.blockHeight = 909; pp.blockHash = 0; pp.matureAtHeight = 909;
4648 pp.amounts[addrA] = 3 * COIN;
4649 gPendingPayouts.push_back(pp);
4650 }
4651
4652 bool fifoOk = (mkfifo(statusTmp.c_str(), 0600) == 0);
4653 check(fifoOk, "a FIFO stands in for the status file's temp file, so the"
4654 " write can be held open");
4655 if (fifoOk) {
4656 std::atomic<bool> done(false);
4657 std::thread writer([&done]() {
4658 RefreshStats(0, 0);
4659 done = true;
4660 });
4661
4662 // Settle: the work before the write is a handful of vector
4663 // operations, so anything past this point is inside fopen(). The
4664 // !done check below is what actually proves it -- with no reader
4665 // there is no other way for this thread to still be running.
4666 Sleep(500);
4667 bool blocked = !done;
4668 bool gotLedger = false, gotStats = false, gotExec = false,
4669 gotRounds = false;
4670 for (int i = 0; i < 1000 && blocked; i++) {
4671 if (!gotLedger && gPayoutLedgerMutex.try_lock()) {
4672 gotLedger = true; gPayoutLedgerMutex.unlock();
4673 }
4674 if (!gotStats && gStatsMutex.try_lock()) {
4675 gotStats = true; gStatsMutex.unlock();
4676 }
4677 if (!gotExec && gPayoutExecMutex.try_lock()) {
4678 gotExec = true; gPayoutExecMutex.unlock();
4679 }
4680 if (!gotRounds && gPoolRoundsMutex.try_lock()) {
4681 gotRounds = true; gPoolRoundsMutex.unlock();
4682 }
4683 if (gotLedger && gotStats && gotExec && gotRounds)
4684 break;
4685 Sleep(2);
4686 }
4687
4688 check(blocked,
4689 "the status-file write is held open for the duration of this"
4690 " test");
4691 check(!blocked || gotLedger,
4692 "gPayoutLedgerMutex is free while pool_status.json is being"
4693 " written");
4694 check(!blocked || (gotExec && gotRounds),
4695 "gPayoutExecMutex and gPoolRoundsMutex are free while"
4696 " pool_status.json is being written");
4697 check(!blocked || gotStats,
4698 "gStatsMutex is free while pool_status.json is being written");
4699
4700 // Let the write finish. Drain to EOF rather than closing early:
4701 // the writer is still filling the pipe and a closed read end would
4702 // deliver it SIGPIPE.
4703 std::thread reader([&statusTmp]() {
4704 FILE* rf = fopen(statusTmp.c_str(), "r");
4705 if (!rf) return;
4706 char buf[4096];
4707 while (fread(buf, 1, sizeof(buf), rf) > 0) { }
4708 fclose(rf);
4709 });
4710 writer.join();
4711 reader.join();
4712 }
4713 remove(statusTmp.c_str());
4714
4715 // And the ordinary path still produces the file it always did.
4716 gLastPoolStatusWrite = 0;
4717 RefreshStats(0, 0);
4718 std::string onDisk;
4719 bool statusOk = false;
4720 if (PayoutTestReadFile(statusPath, onDisk)) {
4721 try {
4722 nlohmann::json j = nlohmann::json::parse(onDisk);
4723 statusOk = j.value("schema", "") == "bitflash-pool-status-1" &&
4724 j["pendingPayouts"].is_array() &&
4725 j["pendingPayouts"].size() == 1 &&
4726 j["pendingPayouts"][0].value("matureAtHeight", 0) == 909 &&
4727 j["pendingPayouts"][0].value("totalAmountSatoshis",
4728 (int64)0) == 3 * COIN;
4729 } catch (...) {
4730 statusOk = false;
4731 }
4732 }
4733 check(statusOk,
4734 "splitting the serialise from the write leaves pool_status.json"
4735 " saying the same thing");
4736
4737 remove(statusPath.c_str());
4738 remove((statusPath + ".tmp").c_str());
4739 strPoolStatusFile = savedStatusFile;
4740 gLastPoolStatusWrite = savedLastWrite;
4741 }
4742#endif
4743
4744 // -- 14. The payout comment names the funding block ----------------------
4745 //
4746 // Same height, two hashes: the shape a reorg leaves when the pool wins a
4747 // height, loses the block, and wins the height again. The legacy needle is
4748 // frozen at the pre-upgrade format, so pre-upgrade payments keep matching;
4749 // a new-format comment never matches it, so a new send can never be read
4750 // as a pre-upgrade payment for a twin round.
4751 {
4752 PayoutTestResetState();
4753 CRITICAL_BLOCK(cs_mapWallet)
4754 mapWallet.clear();
4755 PayoutTestClearBlockIndex();
4756
4757 const int height = 9090;
4758 const int64 amount = 3 * COIN;
4759 uint256 hashA = 0, hashB = 0;
4760 hashA.SetHex("00000000000000000000000000000000000000000000000000000000000000aa");
4761 hashB.SetHex("00000000000000000000000000000000000000000000000000000000000000bb");
4762
4763 check(LegacyPayoutComment(height) ==
4764 strprintf("Pool payout height %d", height),
4765 "the legacy needle is frozen at the format pre-upgrade builds"
4766 " wrote");
4767 check(PayoutComment(height, 0) == LegacyPayoutComment(height),
4768 "a grandfathered zero-hash record still writes the legacy form");
4769 check(PayoutComment(height, hashA) ==
4770 strprintf("Pool payout height %d block %s", height,
4771 hashA.ToString().c_str()),
4772 "a hash-keyed record's comment names its funding block");
4773 check(PayoutComment(height, hashA) != PayoutComment(height, hashB),
4774 "twin rounds at one height carry distinguishable comments");
4775
4776 // A transaction wearing the NEW comment format and -- hypothetically
4777 // -- no structured metadata must not be matched by the legacy needle:
4778 // the fallback exists for pre-upgrade transactions, and the new format
4779 // proves this is not one.
4780 const std::string idA = PayoutIdForBlock(height, hashA);
4781 const std::string idB = PayoutIdForBlock(height, hashB);
4782 std::set<std::string> none;
4783 std::string found;
4784 // Neither twin has a round proof here (section 15 covers the gate),
4785 // and their ids collide, so one answer serves both.
4786 const bool fAllowLegacy = !PoolRoundHasProof(idA);
4787 PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount, 1, "");
4788 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found)
4789 == PAYOUT_PROBE_NONE,
4790 "a new-format comment is never legacy evidence for a twin round");
4791 check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found)
4792 == PAYOUT_PROBE_NONE,
4793 "nor a match channel for its own round -- structured metadata is");
4794
4795 // Round A's own payment, sent by this build, is found by its
4796 // structured id.
4797 CRITICAL_BLOCK(cs_mapWallet)
4798 mapWallet.clear();
4799 PayoutTestClearBlockIndex();
4800 const std::string paidA =
4801 PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount,
4802 2, idA);
4803 check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found)
4804 == PAYOUT_PROBE_STRUCTURED && found == paidA,
4805 "the winning twin's own payment is found by its structured id");
4806
4807 // The id is derived from the FIRST sixteen hex digits of the hash --
4808 // the end proof-of-work fills with zeros -- so twin hashes that agree
4809 // there collide in id, and a payment carrying no pool_payout_block
4810 // answers for either twin. That is the accepted trade for payments
4811 // sent before the key existed, pinned here so it stays a decision.
4812 check(idA == idB,
4813 "twin hashes differing only past the sixteenth hex digit collide"
4814 " in id -- the reason the full hash must decide");
4815 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found)
4816 == PAYOUT_PROBE_STRUCTURED,
4817 "a payment without the block key answers for a colliding twin --"
4818 " the accepted residual for pre-key transactions");
4819
4820 // With the full hash recorded -- as every send now records it -- the
4821 // twins are told apart and the re-won height is paid.
4822 CRITICAL_BLOCK(cs_mapWallet)
4823 mapWallet.clear();
4824 PayoutTestClearBlockIndex();
4825 const std::string keyedA =
4826 PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount,
4827 3, idA, true, hashA.ToString());
4828 check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found)
4829 == PAYOUT_PROBE_STRUCTURED && found == keyedA,
4830 "a payment carrying the full block hash settles its own round");
4831 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found)
4832 == PAYOUT_PROBE_NONE,
4833 "and is no evidence for the twin, so the re-won height is paid");
4834
4835 CRITICAL_BLOCK(cs_mapWallet)
4836 mapWallet.clear();
4837 PayoutTestClearBlockIndex();
4838 }
4839
4840 // -- 15. A proof-bearing round rejects comment-only evidence -------------
4841 //
4842 // The surviving half of the same-height collision: a pre-upgrade
4843 // transaction carries only "Pool payout height H", so if H is won again
4844 // after a reorg, it matches the new round's (height, address, amount)
4845 // exactly and would settle a debt it never paid. The new round has a
4846 // round proof -- written at creation, and only by builds whose payments
4847 // all name their round -- so for it, comment-only evidence is
4848 // inadmissible and the miner is paid. A round without a proof keeps
4849 // today's bias: the evidence is admissible and doubt resolves to
4850 // "already paid".
4851 {
4852 PayoutTestResetState();
4853 CRITICAL_BLOCK(cs_mapWallet)
4854 mapWallet.clear();
4855 PayoutTestClearBlockIndex();
4856 const std::string roundsPath = dir + "/pool_rounds_gate_selftest.json";
4857 const std::string savedRoundsFile = strPoolRoundsFile;
4858 strPoolRoundsFile = roundsPath;
4859 remove(roundsPath.c_str());
4860
4861 const int height = 7171;
4862 const int64 amount = 4 * COIN;
4863 uint256 hashB = 0;
4864 hashB.SetHex("00000000000000000000000000000000000000000000000000000000000000cc");
4865 const std::string idB = PayoutIdForBlock(height, hashB);
4866 std::set<std::string> none;
4867 std::string found;
4868
4869 // The colliding pre-upgrade payment: confirmed, comment-only, same
4870 // height, same address, same amount -- the strongest wrong evidence
4871 // there is.
4872 PayoutTestAddWalletTx(LegacyPayoutComment(height), addrA, amount, 1, "");
4873
4874 check(!PoolRoundHasProof(idB),
4875 "a round with no recorded proof does not claim one");
4876 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount,
4877 !PoolRoundHasProof(idB), none, found)
4878 == PAYOUT_PROBE_LEGACY,
4879 "without a proof the doubt is genuine and still resolves to"
4880 " already-paid");
4881
4882 PoolRoundProof round;
4883 round.id = idB;
4884 round.blockHeight = height;
4885 round.matureAtHeight = height + COINBASE_SPENDABLE_DEPTH;
4886 round.payouts[addrA] = amount;
4887 RecordPoolRound(round);
4888 check(PoolRoundHasProof(idB),
4889 "recording the round proof marks the round as this lineage's");
4890 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount,
4891 !PoolRoundHasProof(idB), none, found)
4892 == PAYOUT_PROBE_NONE,
4893 "with a proof, a same-height comment-only transaction is not"
4894 " evidence, and the re-won height is paid");
4895
4896 // Structured evidence still is: the round's own payment, recovered.
4897 const std::string paidB =
4898 PayoutTestAddWalletTx(PayoutComment(height, hashB), addrA, amount,
4899 2, idB, true, hashB.ToString());
4900 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount,
4901 !PoolRoundHasProof(idB), none, found)
4902 == PAYOUT_PROBE_STRUCTURED && found == paidB,
4903 "the gate does not blind the probe to the round's own structured"
4904 " payment");
4905
4906 CRITICAL_BLOCK(cs_mapWallet)
4907 mapWallet.clear();
4908 PayoutTestClearBlockIndex();
4909 remove(roundsPath.c_str());
4910 remove((roundsPath + ".tmp").c_str());
4911 strPoolRoundsFile = savedRoundsFile;
4912 }
4913
4914 // -- 16. A malformed block hash is corruption, not a hash ----------------
4915 //
4916 // SetHex() parses whatever hex prefix it finds and stops, so a corrupt
4917 // blockHash did not fail to load -- it loaded as a DIFFERENT hash.
4918 // "deadbeef..." became 0xdeadbeef, a hash no block ever had, which the
4919 // orphan check reads as an orphaned block and the next flush VOIDS the
4920 // debt; a value with no hex prefix became 0, which is grandfathered past
4921 // the orphan check instead. Either way a corrupt file silently changed
4922 // what the pool owed. It is a load failure now, handled like every other
4923 // parse error: quarantine, and never rewrite.
4924 {
4925 PayoutTestResetState();
4926 remove(ledgerPath.c_str());
4927 remove(payingPath.c_str());
4928
4929 const std::string badLedger = strprintf(
4930 "[{\"amounts\":{\"%s\":100000000},"
4931 "\"blockHash\":\"deadbeef oops, not a hash\","
4932 "\"blockHeight\":10,\"matureAtHeight\":10}]", addrA.c_str());
4933 check(PayoutTestWriteFile(ledgerPath, badLedger),
4934 "wrote a ledger whose blockHash is not a 64-digit hex string");
4935 LoadPendingPayouts();
4936 {
4937 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4938 check(gPendingPayouts.empty(),
4939 "no entry is loaded with a guessed hash");
4940 }
4941 std::string gone;
4942 check(!PayoutTestReadFile(ledgerPath, gone),
4943 "the ledger with the corrupt hash is quarantined");
4944
4945 const std::string badPaying = strprintf(
4946 "[{\"address\":\"%s\",\"amount\":100000000,"
4947 "\"blockHash\":\"00000000cafe\",\"blockHeight\":10,"
4948 "\"matureAtHeight\":10}]", addrA.c_str());
4949 check(PayoutTestWriteFile(payingPath, badPaying),
4950 "wrote an in-flight list whose blockHash is truncated");
4951 LoadPaying();
4952 {
4953 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
4954 check(gPaying.empty(),
4955 "no in-flight record is loaded with a guessed hash");
4956 }
4957 check(!PayoutTestReadFile(payingPath, gone),
4958 "the in-flight list with the corrupt hash is quarantined");
4959
4960 // The two spellings every real file uses still load: the 64-digit
4961 // hash of a keyed round, and the 64 zeros this build writes for a
4962 // grandfathered entry.
4963 PayoutTestResetState();
4964 remove(ledgerPath.c_str());
4965 const std::string goodLedger = strprintf(
4966 "[{\"amounts\":{\"%s\":100000000},"
4967 "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000cd\","
4968 "\"blockHeight\":10,\"matureAtHeight\":10},"
4969 "{\"amounts\":{\"%s\":200000000},"
4970 "\"blockHash\":\"%s\","
4971 "\"blockHeight\":20,\"matureAtHeight\":20}]",
4972 addrA.c_str(), addrB.c_str(), uint256(0).ToString().c_str());
4973 check(PayoutTestWriteFile(ledgerPath, goodLedger),
4974 "wrote a ledger in both real hash spellings");
4975 LoadPendingPayouts();
4976 {
4977 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4978 check(gPendingPayouts.size() == 2 &&
4979 gPendingPayouts[0].blockHash != 0 &&
4980 gPendingPayouts[1].blockHash == 0,
4981 "a real hash and a serialised zero hash both still load");
4982 }
4983 remove(ledgerPath.c_str());
4984 }
4985
4986 // -- clean up ------------------------------------------------------------
4987 PayoutTestResetState();
4988 CRITICAL_BLOCK(cs_mapWallet)
4989 mapWallet.clear();
4990 PayoutTestClearBlockIndex();
4991 // Out of the scratch directory before removing it, and out of the key maps
4992 // too: the keys this test generated are process-local, but leaving them in
4993 // the keystore would let a later test think a script was its own.
4994 CRITICAL_BLOCK(cs_mapKeys)
4995 {
4996 mapKeys.clear();
4997 mapPubKeys.clear();
4998 }
4999#ifdef _WIN32
5000 SetCurrentDirectoryA(cwd.c_str());
5001#else
5002 if (chdir(cwd.c_str()) != 0) { /* nothing useful to do about it */ }
5003#endif
5004 PayoutTestRemoveDir(dir);
5005 strSetDataDir = savedDataDir;
5006 nBestHeight = savedBestHeight;
5007
5008 fprintf(stdout, "%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
5009 nFail, nFail == 1 ? "" : "s");
5010 fflush(stdout);
5011 return nFail == 0 ? 0 : 1;
5012}
5013
5014int RunPoolStratumSelfTest()
5015{
5016 int nFail = 0;
5017 auto check = [&](bool ok, const char* msg) {
5018 fprintf(stdout, " %s %s\n", ok ? "ok" : "FAIL", msg);
5019 if (!ok) nFail++;
5020 };
5021
5022 fprintf(stdout, "pool-stratum self-test\n");
5023
5024 StratumJob savedJob = gCurrentJob;
5025 bool savedHaveJob = gHaveJob;
5026
5027 CTransaction coinbase;
5028 coinbase.vin.resize(1);
5029 coinbase.vin[0].prevout.SetNull();
5030 coinbase.vin[0].scriptSig << 1 << (CBigNum)1;
5031 coinbase.vout.resize(1);
5032 coinbase.vout[0].nValue = 50 * COIN;
5033
5034 CBlock block;
5035 block.vtx.push_back(coinbase);
5036 block.nVersion = 1;
5037 block.nBits = 0x1f0fffff;
5038 block.nTime = 1;
5039 block.nNonce = 1;
5040 block.hashMerkleRoot = block.BuildMerkleTree();
5041
5042 gCurrentJob.jobId = "selftest";
5043 gCurrentJob.block = block;
5044 gCurrentJob.target = 0; // accepted share, not a block candidate
5045 gCurrentJob.height = 1;
5046 gHaveJob = true;
5047
5048 Miner miner(INVALID_SOCKET);
5049 miner.authorised = true;
5050 miner.address = "selftest-address";
5051 miner.worker = "worker";
5052 miner.shareTarget = ~uint256(0); // every RandomX hash is a valid share
5053 miner.difficulty = 0.001;
5054 miner.vardiffWindowStart = GetTime();
5055
5056 std::map<std::string, uint64> roundShareCount;
5057 uint64 roundShareTotal = 0;
5058 int blocksFound = 0;
5059
5060 uint256 targetFromNotify;
5061 CBlock minerBlock = BuildMinerBlock(gCurrentJob, miner.extranonce2);
5062 json params = MakeNotifyParams(gCurrentJob, minerBlock, miner.shareTarget, true);
5063 std::vector<unsigned char> targetBytes = FromHex(params[3].get<std::string>());
5064 if (targetBytes.size() == 32)
5065 memcpy(&targetFromNotify, &targetBytes[0], 32);
5066 check(targetFromNotify == miner.shareTarget,
5067 "mining.notify exposes the same share target the pool validates");
5068
5069 json submit = {
5070 {"id", 7},
5071 {"method", "mining.submit"},
5072 {"params", json::array({miner.address, "selftest", ToHex(&minerBlock.nNonce, 4)})}
5073 };
5074 HandleLine(&miner, submit.dump(), roundShareCount, roundShareTotal, blocksFound);
5075
5076 check(roundShareTotal == 1, "fake low-difficulty share increments the round total");
5077 check(roundShareCount[miner.address] == 1, "fake share is credited to the submitting address");
5078 check(miner.sessionShares == 1 && miner.roundShares == 1 && miner.totalShares == 1,
5079 "miner share counters advance together");
5080 check(blocksFound == 0, "share below block target is not counted as a found block");
5081
5082 json stale = {
5083 {"id", 8},
5084 {"method", "mining.submit"},
5085 {"params", json::array({miner.address, "stale", ToHex(&minerBlock.nNonce, 4)})}
5086 };
5087 HandleLine(&miner, stale.dump(), roundShareCount, roundShareTotal, blocksFound);
5088 check(roundShareTotal == 1, "stale submit does not increment shares");
5089
5090 std::vector<PoolRoundProof> savedRounds = gPoolRounds;
5091 std::string savedRoundsFile = strPoolRoundsFile;
5092 std::string proofPath = "pool_rounds_selftest.json";
5093 remove(proofPath.c_str());
5094 remove((proofPath + ".tmp").c_str());
5095 gPoolRounds.clear();
5096 strPoolRoundsFile = proofPath;
5097
5098 PoolRoundProof proof;
5099 proof.id = "selftest-round";
5100 proof.blockHeight = 42;
5101 proof.matureAtHeight = 42 + COINBASE_MATURITY;
5102 proof.createdAt = GetTime();
5103 proof.blockHash = block.GetHash().GetHex();
5104 proof.coinbaseTxid = coinbase.GetHash().GetHex();
5105 proof.foundBy = miner.address;
5106 proof.feePercent = 1.25;
5107 proof.totalShares = 3;
5108 proof.coinbaseValue = 50 * COIN;
5109 proof.subsidy = 50 * COIN;
5110 proof.minerReward = 49 * COIN;
5111 proof.operatorReward = 1 * COIN;
5112 proof.shares[miner.address] = 3;
5113 proof.payouts[miner.address] = 49 * COIN;
5114 RecordPoolRound(proof);
5115 MarkPoolRoundPaid(proof.id, miner.address, "selftest-txid");
5116
5117 bool proofOk = false;
5118 FILE* pf = fopen(proofPath.c_str(), "r");
5119 if (pf) {
5120 fseek(pf, 0, SEEK_END);
5121 long sz = ftell(pf);
5122 fseek(pf, 0, SEEK_SET);
5123 std::string s(sz, '\0');
5124 fread(&s[0], 1, sz, pf);
5125 fclose(pf);
5126 try {
5127 json root = json::parse(s);
5128 proofOk =
5129 root.value("schema", "") == "bitflash-pool-rounds-1" &&
5130 root["rounds"].is_array() &&
5131 root["rounds"].size() == 1 &&
5132 root["rounds"][0].value("id", "") == proof.id &&
5133 root["rounds"][0]["shares"].value(miner.address, (uint64)0) == 3 &&
5134 root["rounds"][0]["payouts"][miner.address].value("txid", "") == "selftest-txid";
5135 } catch (...) {
5136 proofOk = false;
5137 }
5138 }
5139 check(proofOk, "pool round proof ledger is written and paid txid is recorded");
5140
5141 remove(proofPath.c_str());
5142 remove((proofPath + ".tmp").c_str());
5143 strPoolRoundsFile = savedRoundsFile;
5144 gPoolRounds = savedRounds;
5145
5146 // -- hashrate decay (#7) --------------------------------------------------
5147 // The first version of this decayed m->hashRate in place from the event
5148 // loop's sweep, which made the reported figure a function of how the loop
5149 // happened to run rather than of how long the miner had been quiet. These
5150 // pin the properties that fixes: sampling changes nothing, and the answer
5151 // depends only on elapsed silence.
5152 {
5153 // A miner exactly on the vardiff target: 4096 hashes per share at
5154 // difficulty 1.0, one share per 15s, so 4096/15 H/s. Its expected
5155 // interval is 15s and its grace is 60s.
5156 const double base = 4096.0 / 15.0;
5157 const int64 t0 = 1766000000; // any plausible non-zero wall clock
5158 Miner decayed(INVALID_SOCKET);
5159 decayed.difficulty = 1.0;
5160 decayed.hashRate = base;
5161 decayed.lastShareTime = t0;
5162
5163 check(MinerReportedHashRate(&decayed, t0 + 60) == base,
5164 "hashrate decay: a miner inside its grace period reports its full estimate");
5165
5166 // Sampling must not consume the estimate. Twenty passes is one second
5167 // of the event loop's old sweep; a thousand is nearly a minute of it.
5168 double once = MinerReportedHashRate(&decayed, t0 + 61);
5169 bool idempotent = (decayed.hashRate == base);
5170 for (int i = 0; i < 1000 && idempotent; i++)
5171 idempotent = (MinerReportedHashRate(&decayed, t0 + 61) == once) &&
5172 (decayed.hashRate == base);
5173 check(idempotent,
5174 "hashrate decay: repeated sampling returns one value and leaves the estimate alone");
5175
5176 // One second past a 60s grace is 1/15th of a half-life: ~4.5% off, not
5177 // the estimate gone. A miner at the vardiff target overshoots its grace
5178 // roughly 1.8% of the time (e^-4) purely by share-timing variance, and
5179 // must not be zeroed for it.
5180 check(once > 0.94 * base && once < 0.96 * base,
5181 "hashrate decay: one second past grace costs a few percent, not the estimate");
5182 double three = MinerReportedHashRate(&decayed, t0 + 63);
5183 check(three > 0.86 * base && three < 0.88 * base,
5184 "hashrate decay: three seconds past grace costs ~13%");
5185
5186 // One half-life past the grace is exactly half.
5187 double half = MinerReportedHashRate(&decayed, t0 + 75);
5188 check(half > 0.499 * base && half < 0.501 * base,
5189 "hashrate decay: one expected interval past grace halves the estimate");
5190
5191 // A miner that has genuinely stopped ends up reported as nothing --
5192 // the whole point of #7. The old in-place version could not get here:
5193 // its decay fed back into its own grace and settled into a 1/t crawl.
5194 check(MinerReportedHashRate(&decayed, t0 + 3600) == 0.0,
5195 "hashrate decay: a miner silent for an hour is reported as zero");
5196
5197 // A clock that steps backwards must never inflate what we announce.
5198 check(MinerReportedHashRate(&decayed, t0 - 3600) == base,
5199 "hashrate decay: a backwards clock reports the estimate unchanged, never higher");
5200
5201 // Fresh connection, and a connection with one share and so no interval
5202 // to have timed: both must report nothing rather than divide by zero.
5203 Miner fresh(INVALID_SOCKET);
5204 check(MinerReportedHashRate(&fresh, t0) == 0.0,
5205 "hashrate decay: a miner that never submitted a share reports zero");
5206 Miner single(INVALID_SOCKET);
5207 single.lastShareTime = t0;
5208 check(MinerReportedHashRate(&single, t0 + 86400) == 0.0,
5209 "hashrate decay: a miner with no rate sample yet reports zero");
5210
5211 // Nothing in the swept range may produce a NaN, an infinity, a negative
5212 // rate, or a rate above the estimate it started from.
5213 bool finite = true;
5214 for (int64 s = -100; s < 200000 && finite; s += 37) {
5215 double v = MinerReportedHashRate(&decayed, t0 + s);
5216 finite = !std::isnan(v) && !std::isinf(v) && v >= 0.0 && v <= base;
5217 }
5218 check(finite,
5219 "hashrate decay: stays finite, non-negative and non-amplifying across a long silence");
5220 }
5221
5222 gCurrentJob = savedJob;
5223 gHaveJob = savedHaveJob;
5224
5225 fprintf(stdout, "%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
5226 nFail, nFail == 1 ? "" : "s");
5227 fflush(stdout);
5228 return nFail == 0 ? 0 : 1;
5229}
5230
5231// ---------------------------------------------------------------------------
5232// Stats refresh -- called every 2s from event loop
5233// ---------------------------------------------------------------------------
5234static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal)
5235{
5236 std::string statusBlob;
5237 bool fPublish = false;
5238 {
5239 std::lock_guard<std::mutex> lk(gStatsMutex);
5240
5241 gStats.authorizedMiners = 0;
5242 gStats.blocksFound = blocksFoundThisSession;
5243 gStats.roundShares = roundShareTotal;
5244 gStats.totalHashRate = 0.0;
5245 for (Miner* m : gMiners)
5246 if (m->authorised) gStats.authorizedMiners++;
5247
5248 gWorkerStats.clear();
5249 // One timestamp for the whole sweep (#7): every miner is faded to the
5250 // same instant, so the per-worker rows and the total they are summed
5251 // into describe one moment rather than a smear across the loop.
5252 int64 nowStats = GetTime();
5253 for (Miner* m : gMiners) {
5254 if (!m->authorised || m->lastSeen <= 0) continue;
5255 PoolWorkerStatView v;
5256 v.address = m->address;
5257 v.worker = m->worker;
5258 v.roundShares = m->roundShares;
5259 v.totalShares = m->totalShares;
5260 v.lastSeen = m->lastSeen;
5261 // Not m->hashRate: a miner that has stopped submitting shares must
5262 // not keep contributing its last sample to the announced total.
5263 v.hashRate = MinerReportedHashRate(m, nowStats);
5264 gWorkerStats.push_back(v);
5265 gStats.totalHashRate += v.hashRate;
5266 }
5267 std::sort(gWorkerStats.begin(), gWorkerStats.end(),
5268 [](const PoolWorkerStatView& a, const PoolWorkerStatView& b) {
5269 return a.roundShares != b.roundShares ? a.roundShares > b.roundShares
5270 : a.lastSeen > b.lastSeen;
5271 });
5272
5273 // Scoped, not held to the end of the function. At function scope this
5274 // lock covered the status-file write below, so the event loop did two
5275 // fsyncs holding the mutex the payout thread takes to record that a
5276 // miner has been paid.
5277 gPayoutViews.clear();
5278 {
5279 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
5280 for (const PendingPayout& pp : gPendingPayouts) {
5281 PendingPayoutView pv;
5282 pv.matureAtHeight = pp.matureAtHeight;
5283 pv.recipients = 0;
5284 pv.totalAmount = 0;
5285 for (auto& kv : pp.amounts) {
5286 if (pp.paidTxids.count(kv.first))
5287 continue;
5288 pv.recipients++;
5289 pv.totalAmount += kv.second;
5290 }
5291 if (pv.recipients > 0)
5292 gPayoutViews.push_back(pv);
5293 }
5294 }
5295
5296 int64 now = GetTime();
5297 if (now - gLastPoolStatusWrite >= 10) {
5298 statusBlob = SerializePoolStatusLocked();
5299 gLastPoolStatusWrite = now;
5300 fPublish = true;
5301 }
5302 }
5303 // Every payout mutex is released here, and gStatsMutex with them. This is
5304 // where the two fsyncs happen, and no other thread is waiting behind them.
5305 if (fPublish)
5306 PublishPoolStatus(statusBlob);
1526} 5307}
1527 5308
1528// --------------------------------------------------------------------------- 5309// ---------------------------------------------------------------------------
@@ -1533,6 +5314,7 @@ static void PoolEventLoop()
1533 int lastHeight = -1; 5314 int lastHeight = -1;
1534 int64 lastRefresh = GetTime(); 5315 int64 lastRefresh = GetTime();
1535 int64 lastStatsRefresh = 0; 5316 int64 lastStatsRefresh = 0;
5317 int64 lastCensus = 0; // #19 connection census throttle
1536 int blocksFoundThisSession = 0; 5318 int blocksFoundThisSession = 0;
1537 int64 noJobSince = 0; // 0 = have a job 5319 int64 noJobSince = 0; // 0 = have a job
1538 bool warnedNoJob = false; 5320 bool warnedNoJob = false;
@@ -1561,6 +5343,31 @@ static void PoolEventLoop()
1561 sock_close(fd); 5343 sock_close(fd);
1562 continue; 5344 continue;
1563 } 5345 }
5346 // #26: select() + FD_SET goes undefined once we exceed what an
5347 // fd_set can carry, and MAX_POOL_MINERS (1000) alone does not
5348 // prevent it -- a Miner is only removed on clean disconnect, so
5349 // half-open zombies accumulate and the OS can hand out fds past
5350 // the limit long before the count reaches 1000. Refuse rather
5351 // than corrupt the fd_set. Which limit applies is platform
5352 // specific: POSIX bounds the descriptor value, Windows the
5353 // number of SOCKET handles an fd_set can hold.
5354#ifndef _WIN32
5355 if ((int)fd >= (int)FD_SETSIZE) {
5356 LogPrint("worker", "[pool] refusing miner fd=%d -- at/over"
5357 " FD_SETSIZE=%d, select() would be undefined (#26)\n",
5358 (int)fd, (int)FD_SETSIZE);
5359 sock_close(fd);
5360 continue;
5361 }
5362#else
5363 if (gMiners.size() >= (size_t)FD_SETSIZE) {
5364 LogPrint("worker", "[pool] refusing miner -- %zu sockets held,"
5365 " an fd_set carries at most FD_SETSIZE=%d (#26)\n",
5366 gMiners.size(), (int)FD_SETSIZE);
5367 sock_close(fd);
5368 continue;
5369 }
5370#endif
1564 gMiners.push_back(new Miner(fd)); 5371 gMiners.push_back(new Miner(fd));
1565 LogPrint("worker", "[pool] miner fd=%d entered event loop" 5372 LogPrint("worker", "[pool] miner fd=%d entered event loop"
1566 " -- awaiting subscribe/authorize\n", (int)fd); 5373 " -- awaiting subscribe/authorize\n", (int)fd);
@@ -1688,6 +5495,35 @@ static void PoolEventLoop()
1688 } 5495 }
1689 } 5496 }
1690 5497
5498 // 3.6. Idle reap (#6). Mirror the clean-disconnect path exactly --
5499 // log, close, erase from gMiners via std::find, delete -- just with
5500 // a different reason. Shares the miner already earned stay in
5501 // roundShareCount, exactly as they do when a miner cleanly
5502 // disconnects mid-round. connectTime stands in for lastSeen on a
5503 // connection that never completed a line. Erasing inside the loop
5504 // above would invalidate the iteration, so collect then reap.
5505 {
5506 int64 now = GetTime();
5507 std::vector<Miner*> idle;
5508 for (Miner* m : gMiners) {
5509 int64 last = (m->lastSeen > 0) ? m->lastSeen : m->connectTime;
5510 if (now - last > POOL_IDLE_TIMEOUT_SECS)
5511 idle.push_back(m);
5512 }
5513 for (Miner* m : idle) {
5514 LogPrint("worker", "[pool] dropping idle miner %s -- nothing"
5515 " received in over %llds (session shares: %llu,"
5516 " round shares: %llu)\n",
5517 m->worker.empty() ? "(never authorized)" : m->worker.c_str(),
5518 (long long)POOL_IDLE_TIMEOUT_SECS,
5519 (unsigned long long)m->sessionShares,
5520 (unsigned long long)m->roundShares);
5521 sock_close(m->fd);
5522 gMiners.erase(std::find(gMiners.begin(), gMiners.end(), m));
5523 delete m;
5524 }
5525 }
5526
1691 // 4. Select over miner sockets 5527 // 4. Select over miner sockets
1692 if (!gMiners.empty()) { 5528 if (!gMiners.empty()) {
1693 fd_set fds; FD_ZERO(&fds); 5529 fd_set fds; FD_ZERO(&fds);
@@ -1710,6 +5546,11 @@ static void PoolEventLoop()
1710 // terminate the whole node -- not just the pool. Every 5546 // terminate the whole node -- not just the pool. Every
1711 // line arrives from an unauthenticated socket. 5547 // line arrives from an unauthenticated socket.
1712 while (RecvLine(m->fd, m->readBuf, line, disc)) { 5548 while (RecvLine(m->fd, m->readBuf, line, disc)) {
5549 // Any complete line is proof of life (#6): authorize
5550 // and valid shares already stamp lastSeen themselves,
5551 // but a miner that only ever subscribes (or sends
5552 // unrecognised methods) must count as alive too.
5553 m->lastSeen = GetTime();
1713 try { 5554 try {
1714 HandleLine(m, line, roundShareCount, 5555 HandleLine(m, line, roundShareCount,
1715 roundShareTotal, blocksFoundThisSession); 5556 roundShareTotal, blocksFoundThisSession);
@@ -1750,6 +5591,14 @@ static void PoolEventLoop()
1750 RefreshStats(blocksFoundThisSession, roundShareTotal); 5591 RefreshStats(blocksFoundThisSession, roundShareTotal);
1751 lastStatsRefresh = GetTime(); 5592 lastStatsRefresh = GetTime();
1752 } 5593 }
5594
5595 // 6. #19: log the connection census periodically. Cheap (one pass over
5596 // gMiners) and it is the single number an operator reads to judge pool
5597 // health, so it should not be an overstatement.
5598 if (GetTime() - lastCensus >= POOL_CENSUS_SECS) {
5599 LogPoolConnectionCensus();
5600 lastCensus = GetTime();
5601 }
1753 } 5602 }
1754 5603
1755 for (Miner* m : gMiners) { sock_close(m->fd); delete m; } 5604 for (Miner* m : gMiners) { sock_close(m->fd); delete m; }
@@ -1778,13 +5627,19 @@ static void AcceptOneFn(void* arg)
1778 std::string host = relay.substr(0, colon); 5627 std::string host = relay.substr(0, colon);
1779 int port = atoi(relay.substr(colon + 1).c_str()); 5628 int port = atoi(relay.substr(colon + 1).c_str());
1780 5629
1781 LogPrint("pool", "[pool] accept thread started for relay %s\n", relay.c_str()); 5630 LogPrint("pool", "[pool] accept thread started for relay %s%s\n", relay.c_str(),
5631 fBtfRelayPinned ? " (/rvrelay pin)" : "");
5632
5633 // Registration health for this relay. A pin has no fallback, so a pin that
5634 // never registers means the pool is simply unreachable -- and that has to
5635 // be loud. The 2026-07-30 incident went unnoticed because this loop logged
5636 // failures at one flat line and success not at all, so "which relay is this
5637 // pool actually registered at" could not be answered from the log.
5638 int nConsecFail = 0;
5639 int64 nFirstFail = 0;
5640 bool fEverRegistered = false;
1782 5641
1783 while (!fShutdown && gPoolRunning) { 5642 while (!fShutdown && gPoolRunning) {
1784 // Advertise this relay as the pool's meeting node BEFORE registering,
1785 // so that miners which resolve our descriptor find a relay where the
1786 // pool (not the node's peer-accept) holds the registration.
1787 BtfSetActiveRelay(relay);
1788 // Returns as soon as the relay lists us -- registered, not yet paired. 5643 // Returns as soon as the relay lists us -- registered, not yet paired.
1789 btf::RvSocket rv = btf::RvServiceRegister(host.c_str(), (unsigned short)port, pk); 5644 btf::RvSocket rv = btf::RvServiceRegister(host.c_str(), (unsigned short)port, pk);
1790 5645
@@ -1793,12 +5648,36 @@ static void AcceptOneFn(void* arg)
1793 break; 5648 break;
1794 } 5649 }
1795 if (rv == btf::RV_INVALID) { 5650 if (rv == btf::RV_INVALID) {
1796 LogPrint("pool", "[pool] relay %s: RvServiceRegister failed" 5651 nConsecFail++;
1797 " -- backing off 30s before retry\n", relay.c_str()); 5652 if (nFirstFail == 0) nFirstFail = GetTime();
5653 LogPrint("pool", "[pool] relay %s: registration failed (attempt %d,"
5654 " failing for %llds) -- backing off 30s before retry\n",
5655 relay.c_str(), nConsecFail, (long long)(GetTime() - nFirstFail));
5656 if (fBtfRelayPinned && (nConsecFail == 3 || nConsecFail % 10 == 0))
5657 LogPrint("pool", "[pool] WARNING: pinned relay %s refused registration"
5658 " %d times running (%llds); /rvrelay leaves no fallback, so"
5659 " miners cannot reach this pool%s\n",
5660 relay.c_str(), nConsecFail, (long long)(GetTime() - nFirstFail),
5661 fEverRegistered ? "" : " -- it has never registered once");
1798 for (int i = 0; i < 30 && !fShutdown && gPoolRunning; i++) Sleep(1000); 5662 for (int i = 0; i < 30 && !fShutdown && gPoolRunning; i++) Sleep(1000);
1799 continue; 5663 continue;
1800 } 5664 }
1801 5665
5666 // Advertise THIS relay only now. Setting it before the call published a
5667 // meeting node we were not yet listed at, so a miner resolving the
5668 // descriptor inside that window burned its dial on a relay holding no
5669 // registration for us. net.cpp's peer-accept has always ordered it this
5670 // way round; the pool had not.
5671 BtfSetActiveRelay(relay);
5672 if (!fEverRegistered || nConsecFail > 0)
5673 LogPrint("pool", "[pool] registered at relay %s%s%s -- advertising it as"
5674 " the pool's meeting node\n", relay.c_str(),
5675 fBtfRelayPinned ? " (/rvrelay pin)" : "",
5676 nConsecFail > 0 ? ", recovered" : "");
5677 fEverRegistered = true;
5678 nConsecFail = 0;
5679 nFirstFail = 0;
5680
1802 // Block for a miner to actually arrive. RvServiceRegister used to do 5681 // Block for a miner to actually arrive. RvServiceRegister used to do
1803 // this itself and returned with the relay's 0x01 pairing byte already 5682 // this itself and returned with the relay's 0x01 pairing byte already
1804 // consumed; it no longer does. Without waiting here, BtfServiceWrap 5683 // consumed; it no longer does. Without waiting here, BtfServiceWrap
@@ -1853,9 +5732,21 @@ static void AcceptLoopFn(void*)
1853 LogPrint("pool", "[pool] .btf identity ready after %ds\n", waitSecs * 2); 5732 LogPrint("pool", "[pool] .btf identity ready after %ds\n", waitSecs * 2);
1854 5733
1855 std::set<std::string> spawned; // relays we've already started a thread for 5734 std::set<std::string> spawned; // relays we've already started a thread for
5735 std::set<std::string> skipped; // discovered relays the pin excluded, logged once each
1856 5736
1857 while (!fShutdown && gPoolRunning) { 5737 while (!fShutdown && gPoolRunning) {
5738 // BtfAllRelays() already honours the pin: with /rvrelay set it returns
5739 // vBtfMeetingRelays alone, so no thread is ever spawned for a
5740 // discovered relay. Name the ones it drops -- a pool registering at a
5741 // stranger's relay is exactly the thing that should not be silent.
1858 vector<string> relays = BtfAllRelays(); 5742 vector<string> relays = BtfAllRelays();
5743 if (fBtfRelayPinned) {
5744 for (const string& r : BtfDiscoveredRelays())
5745 if (std::find(relays.begin(), relays.end(), r) == relays.end() &&
5746 skipped.insert(r).second)
5747 LogPrint("pool", "[pool] not registering at discovered relay %s"
5748 " -- /rvrelay pins the relay list\n", r.c_str());
5749 }
1859 if (relays.empty()) { 5750 if (relays.empty()) {
1860 LogPrint("pool", "[pool] no relays available -- miners cannot connect yet\n"); 5751 LogPrint("pool", "[pool] no relays available -- miners cannot connect yet\n");
1861 } else { 5752 } else {
@@ -1871,8 +5762,9 @@ static void AcceptLoopFn(void*)
1871 relay.c_str()); 5762 relay.c_str());
1872 } else { 5763 } else {
1873 spawned.insert(relay); 5764 spawned.insert(relay);
1874 LogPrint("pool", "[pool] accept thread spawned for relay %s" 5765 LogPrint("pool", "[pool] accept thread spawned for relay %s%s"
1875 " (%zu total)\n", relay.c_str(), spawned.size()); 5766 " (%zu total)\n", relay.c_str(),
5767 fBtfRelayPinned ? " (/rvrelay pin)" : "", spawned.size());
1876 } 5768 }
1877 } 5769 }
1878 } 5770 }
@@ -1901,16 +5793,24 @@ void ThreadRPCServer(void*)
1901 } 5793 }
1902 5794
1903 LoadPendingPayouts(); 5795 LoadPendingPayouts();
5796 LoadPoolRounds();
1904 LoadPaying(); 5797 LoadPaying();
1905 // Must run before the payout thread starts, so nothing is resent that the 5798 // Must run before the payout thread starts, so nothing is resent that the
1906 // wallet already shows as sent, and nothing is owed twice because we died 5799 // ledger already shows as sent, and every restored in-flight record is
1907 // between recording the payments and rewriting the ledger. 5800 // armed for its one wallet probe before it is allowed to send.
1908 ReconcilePayouts(); 5801 ReconcilePayouts();
1909 5802
1910 _beginthread(AcceptLoopFn, 0, NULL); 5803 // _beginthread cannot throw (it reports -1), but an unchecked failure
1911 _beginthread(LocalAcceptFn, 0, NULL); 5804 // leaves the pool running deaf: nobody accepts, announces, or pays, and
1912 _beginthread(ThreadBtfPoolAnnouncer, 0, NULL); 5805 // nothing in the log says why.
1913 _beginthread(PayoutThreadFn, 0, NULL); 5806 if (_beginthread(AcceptLoopFn, 0, NULL) == (uintptr_t)-1)
5807 LogPrint("pool", "[pool] FATAL: accept loop thread failed to spawn -- miners cannot connect\n");
5808 if (_beginthread(LocalAcceptFn, 0, NULL) == (uintptr_t)-1)
5809 LogPrint("pool", "[pool] local accept thread failed to spawn\n");
5810 if (_beginthread(ThreadBtfPoolAnnouncer, 0, NULL) == (uintptr_t)-1)
5811 LogPrint("pool", "[pool] announcer thread failed to spawn -- pool will not be advertised\n");
5812 if (_beginthread(PayoutThreadFn, 0, NULL) == (uintptr_t)-1)
5813 LogPrint("pool", "[pool] FATAL: payout thread failed to spawn -- no one gets paid\n");
1914 5814
1915 PoolEventLoop(); // blocks until gPoolRunning = false or fShutdown 5815 PoolEventLoop(); // blocks until gPoolRunning = false or fShutdown
1916 5816