diff options
| author | bitflash-maintainer <noreply@bitflash.invalid> | 2026-09-05 22:46:52 +0000 |
|---|---|---|
| committer | bitflash-maintainer <noreply@bitflash.invalid> | 2026-09-05 22:46:52 +0000 |
| commit | ceae3d076b5ad11ee84729ae43f14e9312896646 (patch) | |
| tree | 7e9132aa871fdc01cb95268d2f5ea3bb11853ddf /src/rpc.cpp | |
| parent | be63e83d79e7e00e6b8ca889a70e6a19886aa1d8 (diff) | |
| parent | 66e0b9158d050c46b1ff2509faeee3f4573cac1c (diff) | |
Bring main onto the line this project actually ships
main had drifted to a history with no ancestor in common with the tree we
build and deploy. It sat at be63e83 (2026-08-09) with 28 commits; the shipped
line, deploy/soak-2026-08-21, had 383. The two shared nothing -- git
merge-base returns no result at all -- so anyone cloning this repository
landed on a default branch that was neither current nor related to the
running node.
The 28 commits on main were audited before this merge: every one is already
present in the tree by content, including the two that are hardest to see
because they were rewritten afterwards -- the payout reorg-safety work
(PayoutBlockStillOnChain, since evolved to re-check on every retry rather
than once at maturity) and the nostr relay certificate verification
(SSL_set1_host, now with a Windows root-store path). Nothing on main is lost
by this merge.
Recorded as a merge rather than a force-push: main only ever moves forward,
its old history stays reachable as the first parent, and this is revertible.
The tree is taken wholesale from the merged branch, so main's content is now
exactly what was built and tested.
What this brings onto main:
- compactblock: BIP152-style compact block relay ported from upstream 1.2.23,
primitives and unit test only, with the RAND_bytes failure path closed and
the reconstruction allocation bounded.
- upstream/main 17741d9: isolated testnet parameters and genesis, optional
2 MB large pages for RandomX. Mainnet genesis verified byte-identical
through upstream's BuildGenesisBlock() refactor.
- deploy: every image built on bookworm so binaries run on Debian 12, and the
container build now honours the RandomX and libsecp256k1 pins it had been
silently ignoring since before 1.2.15.
Gates at this tree: 15/15 selftests, compactblock's 17 checks, consensus
constants unchanged against e4f898c apart from the pre-existing
MAX_BLOCK_SIZE_GEN, mainnet genesis intact, money-log check ok.
Diffstat (limited to 'src/rpc.cpp')
| -rw-r--r-- | src/rpc.cpp | 4490 |
1 files changed, 4195 insertions, 295 deletions
diff --git a/src/rpc.cpp b/src/rpc.cpp index 03d1f44..3827cd5 100644 --- a/src/rpc.cpp +++ b/src/rpc.cpp | |||
| @@ -31,10 +31,15 @@ | |||
| 31 | #include <netinet/in.h> | 31 | #include <netinet/in.h> |
| 32 | #include <arpa/inet.h> | 32 | #include <arpa/inet.h> |
| 33 | #include <unistd.h> | 33 | #include <unistd.h> |
| 34 | #include <fcntl.h> | ||
| 35 | #include <dirent.h> | ||
| 36 | #else | ||
| 37 | #include <io.h> | ||
| 34 | #endif | 38 | #endif |
| 35 | 39 | ||
| 36 | #include <atomic> | 40 | #include <atomic> |
| 37 | #include <mutex> | 41 | #include <mutex> |
| 42 | #include <thread> | ||
| 38 | #include <map> | 43 | #include <map> |
| 39 | #include <set> | 44 | #include <set> |
| 40 | #include <deque> | 45 | #include <deque> |
| @@ -110,15 +115,28 @@ static uint256 ShareTargetFromDifficulty(double difficulty) | |||
| 110 | { | 115 | { |
| 111 | if (difficulty <= 0.0) difficulty = 1.0; | 116 | if (difficulty <= 0.0) difficulty = 1.0; |
| 112 | static const uint256 diff1 = CBigNum(bnProofOfWorkLimit).getuint256(); | 117 | static const uint256 diff1 = CBigNum(bnProofOfWorkLimit).getuint256(); |
| 118 | // Representability floor. shareTarget = diff1 / difficulty, and diff1 is | ||
| 119 | // ~uint256(0) >> 12, i.e. a hair under 2^244 -- so the true target stops | ||
| 120 | // fitting in a uint256 once difficulty reaches 2^-12 (= 1/4096), and the | ||
| 121 | // fixed-point divide below breaks even earlier in a second way: the | ||
| 122 | // (uint64)(difficulty * 65536.0) divisor truncates to zero for any | ||
| 123 | // difficulty under 2^-16. Both thresholds sit far below VARDIFF_MIN | ||
| 124 | // (0.001), so live operation never comes near this, but a caller that | ||
| 125 | // bypasses vardiff (or a future VARDIFF_MIN) must not divide by zero or | ||
| 126 | // hand back getuint256()'s low-256-bit truncation of an oversized | ||
| 127 | // quotient -- a wrapped target is not merely wrong, it is arbitrarily | ||
| 128 | // HARDER than difficulty 1. At these difficulties every hash is meant to | ||
| 129 | // be a share, so the honest answer is the easiest representable target. | ||
| 130 | // (This replaces a dead branch that read `if (t < diff1 && difficulty < | ||
| 131 | // 1.0)`: t = diff1/difficulty, so difficulty < 1.0 implies t > diff1 and | ||
| 132 | // the two conditions could never both hold.) | ||
| 133 | if (difficulty < 1.0 / 4096.0) return ~uint256(0); | ||
| 113 | // shareTarget = diff1 / difficulty | 134 | // shareTarget = diff1 / difficulty |
| 114 | CBigNum bn; bn.setuint256(diff1); | 135 | CBigNum bn; bn.setuint256(diff1); |
| 115 | // Multiply by 2^16 to keep precision, divide, shift back | 136 | // Multiply by 2^16 to keep precision, divide, shift back |
| 116 | bn <<= 16; | 137 | bn <<= 16; |
| 117 | bn /= (uint64)(difficulty * 65536.0); | 138 | bn /= (uint64)(difficulty * 65536.0); |
| 118 | uint256 t = bn.getuint256(); | 139 | return bn.getuint256(); |
| 119 | // Cap at all-ones (maximum, easiest possible) | ||
| 120 | if (t < diff1 && difficulty < 1.0) t = ~uint256(0); | ||
| 121 | return t; | ||
| 122 | } | 140 | } |
| 123 | 141 | ||
| 124 | // Expected number of hashes to find one share at a given Stratum difficulty, | 142 | // Expected number of hashes to find one share at a given Stratum difficulty, |
| @@ -218,6 +236,29 @@ static bool RebuildJob() | |||
| 218 | // From the pool, so the template's payout key is in wallet.dat before | 236 | // From the pool, so the template's payout key is in wallet.dat before |
| 219 | // any miner is handed work against it. | 237 | // any miner is handed work against it. |
| 220 | vector<unsigned char> vchPubKey = GetKeyFromPool(); | 238 | vector<unsigned char> vchPubKey = GetKeyFromPool(); |
| 239 | if (vchPubKey.empty()) { | ||
| 240 | // Empty means the wallet could not produce a key: encrypted and | ||
| 241 | // locked, or key generation itself failed. Serialised, an empty | ||
| 242 | // vector is a zero-length push, so the coinbase would become | ||
| 243 | // `<empty> OP_CHECKSIG` -- a script nobody can satisfy. Every | ||
| 244 | // block mined against such a job burns its whole subsidy, while | ||
| 245 | // QueuePayouts still records the full obligation and the miners | ||
| 246 | // are then paid out of coin that block never contributed. | ||
| 247 | // BitcoinMiner refuses to mine in this state; refuse to hand the | ||
| 248 | // pool's miners work in it too. printf and not LogPrint for the | ||
| 249 | // reason given above the deaf-node refusal, and throttled the | ||
| 250 | // same way -- the no-job watchdog retries every few seconds for | ||
| 251 | // as long as the condition lasts. | ||
| 252 | static int64 nLastSaidNoKey = 0; | ||
| 253 | if (GetTime() - nLastSaidNoKey > 60) { | ||
| 254 | nLastSaidNoKey = GetTime(); | ||
| 255 | printf("RebuildJob: no wallet key available for the coinbase " | ||
| 256 | "(wallet locked, or key generation failed -- the " | ||
| 257 | "GetKeyFromPool line above says which) -- not issuing " | ||
| 258 | "jobs\n"); | ||
| 259 | } | ||
| 260 | return false; | ||
| 261 | } | ||
| 221 | static std::atomic<uint32_t> sExtra{0}; | 262 | static std::atomic<uint32_t> sExtra{0}; |
| 222 | uint32_t extraNonce = ++sExtra; | 263 | uint32_t extraNonce = ++sExtra; |
| 223 | 264 | ||
| @@ -236,35 +277,96 @@ static bool RebuildJob() | |||
| 236 | CBlock block; | 277 | CBlock block; |
| 237 | block.vtx.push_back(txNew); | 278 | block.vtx.push_back(txNew); |
| 238 | 279 | ||
| 280 | // Fill the block the same way BitcoinMiner() does, because this is the | ||
| 281 | // same job: a block that gets mined and submitted for real. | ||
| 282 | // | ||
| 283 | // It used to differ in two ways, and both were bugs. | ||
| 284 | // | ||
| 285 | // One pass over mapTransactions is not enough. That map is keyed by | ||
| 286 | // hash, so it has no dependency order, and a transaction spending the | ||
| 287 | // still-unconfirmed output of another (change from a recent send, most | ||
| 288 | // commonly) fails ConnectInputs whenever its parent happens to sit | ||
| 289 | // later in map order. A single pass then drops it -- and because map | ||
| 290 | // order does not change between rebuilds, every later rebuild drops it | ||
| 291 | // again at the same point. Deterministic exclusion, not bad luck: if | ||
| 292 | // pool operators mine most of the blocks, a transaction of that shape | ||
| 293 | // can stay unconfirmed for as long as its parent does. | ||
| 294 | // | ||
| 295 | // Diagnosis and fix both come from Vic-Nas, who found it in his fork | ||
| 296 | // of this code and described it exactly. The fork has since been | ||
| 297 | // deleted, so this carries it back rather than losing it. | ||
| 298 | int64 nFees = 0; | ||
| 239 | { | 299 | { |
| 240 | CTxDB txdb("r"); | 300 | CTxDB txdb("r"); |
| 241 | CRITICAL_BLOCK(cs_mapTransactions) | 301 | CRITICAL_BLOCK(cs_mapTransactions) |
| 242 | { | 302 | { |
| 243 | map<uint256, CTxIndex> pool; | 303 | map<uint256, CTxIndex> pool; |
| 244 | unsigned int sz = 0; | 304 | vector<char> vfAlreadyAdded(mapTransactions.size()); |
| 245 | for (auto& kv : mapTransactions) { | 305 | bool fFoundSomething = true; |
| 246 | CTransaction& tx = kv.second; | 306 | // The cap is MAX_BLOCK_SIZE_GEN, not MAX_SIZE/2. MAX_SIZE is |
| 247 | if (tx.IsCoinBase() || !tx.IsFinal()) continue; | 307 | // the 32 MB serialisation ceiling, so half of it is 16 MB -- |
| 248 | int64 nFees = 0; | 308 | // this builder would happily assemble a template that its own |
| 249 | map<uint256, CTxIndex> tmp(pool); | 309 | // CheckBlock() rejects at the 1 MB consensus cap. This is the |
| 250 | if (!tx.ConnectInputs(txdb, tmp, CDiskTxPos(1,1,1), 0, nFees, | 310 | // pool's job builder, so a block over the cap is not one |
| 251 | false, true, tx.GetMinFee(block.vtx.size() < 100))) | 311 | // wasted solve, it is every miner on the pool working a |
| 252 | continue; | 312 | // template the network will reject. |
| 253 | pool = tmp; | 313 | // |
| 254 | block.vtx.push_back(tx); | 314 | // The inner break used to bound the overshoot to one |
| 255 | sz += ::GetSerializeSize(tx, SER_NETWORK); | 315 | // transaction per pass. One transaction is still too many: |
| 256 | // Must stay under the consensus cap CheckBlock() enforces. | 316 | // nothing caps how big a single transaction may be, so that |
| 257 | // This is the pool's job builder, so a block over the cap | 317 | // bound was worth up to MAX_SIZE. Asking whether each one fits |
| 258 | // is not just one wasted solve -- it is every miner on the | 318 | // before appending it bounds the overshoot at zero instead. |
| 259 | // pool working a template the network will reject. The | 319 | // |
| 260 | // transaction is already pushed above, so this can overshoot | 320 | // Seeded with the header, tx count and coinbase, because |
| 261 | // by one; half the cap leaves room for that. | 321 | // CheckBlock() weighs the whole serialized block. |
| 262 | if (sz > MAX_BLOCK_SIZE / 2) break; | 322 | unsigned int sz = ::GetSerializeSize(block, SER_DISK); |
| 323 | // And the sigop budget, which CheckBlock() enforces and this | ||
| 324 | // builder never counted against. Same counting function | ||
| 325 | // CheckBlock() sums, seeded from the coinbase because it sums | ||
| 326 | // over all of vtx. | ||
| 327 | unsigned int nSigOps = block.vtx[0].GetSigOpCount(); | ||
| 328 | while (fFoundSomething && sz < MAX_BLOCK_SIZE_GEN) | ||
| 329 | { | ||
| 330 | fFoundSomething = false; | ||
| 331 | unsigned int n = 0; | ||
| 332 | for (map<uint256, CTransaction>::iterator mi = mapTransactions.begin(); | ||
| 333 | mi != mapTransactions.end(); ++mi, ++n) | ||
| 334 | { | ||
| 335 | if (vfAlreadyAdded[n]) | ||
| 336 | continue; | ||
| 337 | CTransaction& tx = (*mi).second; | ||
| 338 | if (tx.IsCoinBase() || !tx.IsFinal()) continue; | ||
| 339 | // Skip rather than stop: one transaction too big for | ||
| 340 | // the room left does not mean the next is. Ordered | ||
| 341 | // before ConnectInputs because ConnectInputs credits | ||
| 342 | // nFees by reference on success, and a transaction we | ||
| 343 | // then decline would leave the coinbase claiming a fee | ||
| 344 | // this block does not collect. | ||
| 345 | unsigned int nTxSize = ::GetSerializeSize(tx, SER_NETWORK); | ||
| 346 | unsigned int nTxSigOps = tx.GetSigOpCount(); | ||
| 347 | if (!BlockTemplateFits(sz, nSigOps, nTxSize, nTxSigOps)) | ||
| 348 | continue; | ||
| 349 | map<uint256, CTxIndex> tmp(pool); | ||
| 350 | if (!tx.ConnectInputs(txdb, tmp, CDiskTxPos(1,1,1), 0, nFees, | ||
| 351 | false, true, tx.GetMinFee(block.vtx.size() < 100))) | ||
| 352 | continue; | ||
| 353 | swap(pool, tmp); | ||
| 354 | block.vtx.push_back(tx); | ||
| 355 | sz += nTxSize; | ||
| 356 | nSigOps += nTxSigOps; | ||
| 357 | vfAlreadyAdded[n] = true; | ||
| 358 | fFoundSomething = true; | ||
| 359 | } | ||
| 263 | } | 360 | } |
| 264 | } | 361 | } |
| 265 | } | 362 | } |
| 266 | 363 | ||
| 267 | block.vtx[0].vout[0].nValue = block.GetBlockValue(pindexPrev->nHeight + 1, 0); | 364 | // And the fees actually go to the coinbase now. nFees was declared |
| 365 | // inside the scan loop, so every transaction reset it and the total was | ||
| 366 | // thrown away, while this line passed a hardcoded 0. A pool operator | ||
| 367 | // collected fees into a block and paid none of them to itself. Solo | ||
| 368 | // mining has always passed nFees here; only this builder did not. | ||
| 369 | block.vtx[0].vout[0].nValue = block.GetBlockValue(pindexPrev->nHeight + 1, nFees); | ||
| 268 | block.hashPrevBlock = pindexPrev->GetBlockHash(); | 370 | block.hashPrevBlock = pindexPrev->GetBlockHash(); |
| 269 | block.hashMerkleRoot = block.BuildMerkleTree(); | 371 | block.hashMerkleRoot = block.BuildMerkleTree(); |
| 270 | block.nTime = max((unsigned int)(pindexPrev->GetMedianTimePast() + 1), | 372 | block.nTime = max((unsigned int)(pindexPrev->GetMedianTimePast() + 1), |
| @@ -297,26 +399,406 @@ struct PendingPayout { | |||
| 297 | // supposed to pay it: a 280-block reorg once left ~14.4k BTF of debt | 399 | // supposed to pay it: a 280-block reorg once left ~14.4k BTF of debt |
| 298 | // owed against blocks that no longer existed, which then drained the | 400 | // owed against blocks that no longer existed, which then drained the |
| 299 | // wallet paying fork-era miners out of real-chain coin. | 401 | // wallet paying fork-era miners out of real-chain coin. |
| 402 | // | ||
| 403 | // Upstream keys these on the string "block-<height>". That is the same | ||
| 404 | // bug this fork already paid for, and height keying additionally lets | ||
| 405 | // FindRecordedPoolPayout match the *previous* round's transaction after a | ||
| 406 | // reorg and silently skip a miner, so their `id` is deliberately dropped. | ||
| 300 | int blockHeight = 0; | 407 | int blockHeight = 0; |
| 301 | // Must be explicitly zeroed: uint256's default constructor is empty | 408 | // Must be explicitly zeroed: uint256's default constructor is empty |
| 302 | // (uint256.h:514), so leaving this unassigned gives every ledger entry | 409 | // (uint256.h:514), so leaving this unassigned gives every ledger entry |
| 303 | // loaded from an older file a garbage hash, which then matches no block | 410 | // loaded from an older file a garbage hash, which then matches no block |
| 304 | // and gets voided as "orphaned". That mistake voided a live payout. | 411 | // and gets voided as "orphaned". That mistake voided a live payout. |
| 305 | uint256 blockHash = 0; // zero = entry predates hash keying | 412 | uint256 blockHash = 0; // zero = entry predates hash keying |
| 306 | int matureAtHeight; | 413 | int matureAtHeight = 0; |
| 307 | std::map<std::string, int64> amounts; // address -> satoshis | 414 | std::map<std::string, int64> amounts; // address -> satoshis |
| 308 | int retries = 0; // exec-thread only: SendMoney attempts so far | 415 | // From upstream: which recipients already have a transaction. This is what |
| 309 | int deferrals = 0; // exec-thread only: waits for spendable coin | 416 | // lets an entry stay on disk until every address in `amounts` is paid, |
| 310 | int64 retryAfter = 0; // exec-thread only: unix time, 0 = ready now | 417 | // instead of being erased the moment the batch is handed off. |
| 418 | std::map<std::string, std::string> paidTxids; // address -> txid | ||
| 419 | // Retry and backoff state does NOT live here. It used to be declared here | ||
| 420 | // as well as on PayingRecord, and the copy on this side was never written: | ||
| 421 | // `retryAfter` was declared, serialised, loaded and tested at the flush | ||
| 422 | // gate, and the only assignment to it anywhere was `= 0`, so the gate could | ||
| 423 | // not fire. `retries` and `deferrals` were the same story. The real state | ||
| 424 | // is per (round, recipient) -- which is the granularity a payment can fail | ||
| 425 | // at -- and it lives on PayingRecord, which is now what persists it. | ||
| 311 | }; | 426 | }; |
| 312 | static std::vector<PendingPayout> gPendingPayouts; | 427 | static std::vector<PendingPayout> gPendingPayouts; |
| 428 | static std::mutex gPayoutLedgerMutex; | ||
| 429 | // Bumped on every serialisation, under gPayoutLedgerMutex. See PublishVersioned. | ||
| 430 | static uint64 gPayoutLedgerSeq = 0; | ||
| 431 | // Serialises writers of pending_payouts.json, and guards the sequence number | ||
| 432 | // of the image currently on disk. Never taken while a data mutex is held. | ||
| 433 | static std::mutex gPayoutLedgerFileMutex; | ||
| 434 | static uint64 gPayoutLedgerPublishedSeq = 0; | ||
| 435 | |||
| 436 | struct PoolRoundProof { | ||
| 437 | std::string id; | ||
| 438 | int blockHeight; | ||
| 439 | int matureAtHeight; | ||
| 440 | int64 createdAt; | ||
| 441 | std::string blockHash; | ||
| 442 | std::string coinbaseTxid; | ||
| 443 | std::string foundBy; | ||
| 444 | double feePercent; | ||
| 445 | uint64 totalShares; | ||
| 446 | int64 coinbaseValue; | ||
| 447 | int64 subsidy; | ||
| 448 | int64 minerReward; | ||
| 449 | int64 operatorReward; | ||
| 450 | int skippedDust; | ||
| 451 | int64 forfeitedBadAddress; | ||
| 452 | std::map<std::string, uint64> shares; | ||
| 453 | std::map<std::string, int64> payouts; | ||
| 454 | std::map<std::string, std::string> paidTxids; | ||
| 455 | // Recipients the ledger has written off, and why. Without this the public | ||
| 456 | // round proof goes on advertising an address as owed with "txid": null | ||
| 457 | // forever, while the ledger has already dropped the obligation -- the two | ||
| 458 | // records of the same round permanently disagreeing, with the operator | ||
| 459 | // holding the money. | ||
| 460 | std::map<std::string, std::string> forfeited; // address -> reason | ||
| 461 | |||
| 462 | PoolRoundProof() | ||
| 463 | : blockHeight(0), matureAtHeight(0), createdAt(0), feePercent(0.0), | ||
| 464 | totalShares(0), coinbaseValue(0), subsidy(0), minerReward(0), operatorReward(0), | ||
| 465 | skippedDust(0), forfeitedBadAddress(0) {} | ||
| 466 | }; | ||
| 467 | static std::vector<PoolRoundProof> gPoolRounds; | ||
| 468 | static std::mutex gPoolRoundsMutex; | ||
| 469 | static uint64 gPoolRoundsSeq = 0; // guarded by gPoolRoundsMutex | ||
| 470 | static std::mutex gPoolRoundsFileMutex; | ||
| 471 | static uint64 gPoolRoundsPublishedSeq = 0; // guarded by gPoolRoundsFileMutex | ||
| 472 | static const size_t MAX_POOL_ROUNDS = 200; | ||
| 313 | 473 | ||
| 314 | static std::string PayoutFilePath() | 474 | static std::string PayoutFilePath() |
| 315 | { | 475 | { |
| 316 | return GetAppDir() + "/pending_payouts.json"; | 476 | return GetAppDir() + "/pending_payouts.json"; |
| 317 | } | 477 | } |
| 318 | 478 | ||
| 319 | static void SavePendingPayouts() | 479 | static std::string PoolRoundsFilePath() |
| 480 | { | ||
| 481 | if (!strPoolRoundsFile.empty()) | ||
| 482 | return strPoolRoundsFile; | ||
| 483 | return GetAppDir() + "/pool_rounds.json"; | ||
| 484 | } | ||
| 485 | |||
| 486 | // --------------------------------------------------------------------------- | ||
| 487 | // Atomic, ordered publication of the payout state files | ||
| 488 | // --------------------------------------------------------------------------- | ||
| 489 | // | ||
| 490 | // Every one of these files is a full dump of an in-memory structure, written | ||
| 491 | // tmp-then-rename so a crash cannot truncate it. Two things have to be true | ||
| 492 | // for that to actually be safe, and only the first was: | ||
| 493 | // | ||
| 494 | // 1. the rename must be atomic (it is), and | ||
| 495 | // 2. exactly one thread at a time may be writing the temp file, and a write | ||
| 496 | // must never publish an OLDER image over a newer one. | ||
| 497 | // | ||
| 498 | // (2) was broken for pending_payouts.json. The event loop wrote it with | ||
| 499 | // gPayoutLedgerMutex held, the payout thread wrote it with the mutex | ||
| 500 | // released, and nothing serialised the two: both opened the same | ||
| 501 | // "<path>.tmp" with independent handles and both renamed it. The interleaved | ||
| 502 | // file that lands is either a mixture of two images (LoadPendingPayouts then | ||
| 503 | // throws mid-loop and comes up with a nearly empty ledger -- every obligation | ||
| 504 | // not yet exploded, gone) or a whole newly-queued round silently overwritten | ||
| 505 | // by an older snapshot that predates it. | ||
| 506 | // | ||
| 507 | // The fix is a per-file write mutex, held only across the I/O and never | ||
| 508 | // across a data-structure mutex, plus a version on the snapshot. Serialising | ||
| 509 | // happens under the data mutex and stamps a monotonically increasing sequence | ||
| 510 | // number; publishing happens with the data mutex released and refuses any | ||
| 511 | // blob whose sequence number is not newer than what is already on disk. So | ||
| 512 | // writers can queue in any order and finish in any order -- the file only | ||
| 513 | // ever moves forwards. | ||
| 514 | // | ||
| 515 | // The other half of the rule is that no thread waits for I/O while holding | ||
| 516 | // gPayoutLedgerMutex, gPayoutExecMutex or gPoolRoundsMutex, which is the | ||
| 517 | // contention the merge was right to remove from the event loop. That was | ||
| 518 | // claimed here before it was true: pool_status.json is written by the same | ||
| 519 | // WriteFileAtomic and RefreshStats held gPayoutLedgerMutex at function scope | ||
| 520 | // across the call, so once the writes gained fsync() the event loop was doing | ||
| 521 | // two of them every ten seconds with the payout thread's mutex held -- worse | ||
| 522 | // than the buffered write the merge replaced. The status file now goes through | ||
| 523 | // the same serialise-under-the-lock / publish-outside-it split as the ledgers, | ||
| 524 | // and the rule holds for all four files. It is checked, not asserted: see the | ||
| 525 | // "no payout mutex is held across a publish" section of the payout-ledger | ||
| 526 | // self-test. | ||
| 527 | |||
| 528 | // A serialised file image plus the version of the in-memory state it was | ||
| 529 | // taken from. | ||
| 530 | struct VersionedBlob { | ||
| 531 | std::string blob; | ||
| 532 | uint64 seq = 0; | ||
| 533 | }; | ||
| 534 | |||
| 535 | // Push a file's contents out of the C library's buffer AND out of the kernel's | ||
| 536 | // page cache, so the bytes exist on the device before anything points at them. | ||
| 537 | static bool SyncFileHandle(FILE* f) | ||
| 538 | { | ||
| 539 | if (fflush(f) != 0) | ||
| 540 | return false; | ||
| 541 | #ifdef _WIN32 | ||
| 542 | return _commit(_fileno(f)) == 0; | ||
| 543 | #else | ||
| 544 | return fsync(fileno(f)) == 0; | ||
| 545 | #endif | ||
| 546 | } | ||
| 547 | |||
| 548 | // Make the directory entry that rename() created durable too. | ||
| 549 | // | ||
| 550 | // A tmp+rename is only atomic with respect to a crash of the process. Against | ||
| 551 | // a power cut or a hard VM reset the rename can reach the platter while the | ||
| 552 | // data it points at has not, and the file that comes back is short or empty. | ||
| 553 | // For pending_payouts.json that is not a corrupt file, it is a silent one: | ||
| 554 | // LoadPendingPayouts catches the parse error, logs a single warning, and the | ||
| 555 | // node comes up owing nobody anything. | ||
| 556 | // | ||
| 557 | // Best effort -- a filesystem that does not allow the directory to be opened | ||
| 558 | // for this is not a reason to refuse to write the ledger. | ||
| 559 | static void SyncDirectoryOf(const std::string& path) | ||
| 560 | { | ||
| 561 | #ifndef _WIN32 | ||
| 562 | size_t slash = path.find_last_of('/'); | ||
| 563 | std::string dir = (slash == std::string::npos) ? std::string(".") | ||
| 564 | : path.substr(0, slash); | ||
| 565 | if (dir.empty()) | ||
| 566 | dir = "/"; | ||
| 567 | int fd = open(dir.c_str(), O_RDONLY); | ||
| 568 | if (fd < 0) | ||
| 569 | return; | ||
| 570 | fsync(fd); | ||
| 571 | close(fd); | ||
| 572 | #endif | ||
| 573 | } | ||
| 574 | |||
| 575 | // Replace `path` with `data` atomically and durably. Caller must hold the | ||
| 576 | // write mutex for `path`: the temp name is derived from the target, so two | ||
| 577 | // threads in here at once for one path is precisely the race this exists to | ||
| 578 | // prevent. | ||
| 579 | // | ||
| 580 | // Every failure removes the temp file before returning. It used to be left | ||
| 581 | // where it fell, so a disk that filled up once left a short, valid-looking | ||
| 582 | // "pending_payouts.json.tmp" next to the good ledger for as long as the node | ||
| 583 | // ran. That file is a trap: it has the name of a half-finished save, and an | ||
| 584 | // operator cleaning up after an incident is exactly the person who will move | ||
| 585 | // it into place. | ||
| 586 | // | ||
| 587 | // Failures print unconditionally rather than through LogPrint(). These three | ||
| 588 | // files are written under two different debug categories ("payout" for the | ||
| 589 | // ledgers, "pool" for the status file), and a write failure is not a trace of | ||
| 590 | // normal operation -- it is the file the operator is about to read not being | ||
| 591 | // there. Gating it behind a category that had to be chosen before the failure | ||
| 592 | // happened is how "could not replace pool_status.json" went missing. | ||
| 593 | static bool WriteFileAtomic(const std::string& path, const std::string& data, | ||
| 594 | const char* what) | ||
| 595 | { | ||
| 596 | std::string tmp = path + ".tmp"; | ||
| 597 | FILE* f = fopen(tmp.c_str(), "w"); | ||
| 598 | if (!f) { | ||
| 599 | printf("[payout] WARNING: could not write %s (%s)\n", | ||
| 600 | tmp.c_str(), what); | ||
| 601 | return false; | ||
| 602 | } | ||
| 603 | if (!data.empty() && fwrite(data.c_str(), 1, data.size(), f) != data.size()) { | ||
| 604 | printf("[payout] WARNING: short write to %s (%s)\n", | ||
| 605 | tmp.c_str(), what); | ||
| 606 | fclose(f); | ||
| 607 | remove(tmp.c_str()); | ||
| 608 | return false; | ||
| 609 | } | ||
| 610 | // Before the rename, not after: publishing a name that points at nothing | ||
| 611 | // is the failure this ordering exists to prevent. If the data cannot be | ||
| 612 | // made durable, do not rename -- the previous file is still intact and | ||
| 613 | // still correct, and the next write will try again. | ||
| 614 | if (!SyncFileHandle(f)) { | ||
| 615 | printf("[payout] WARNING: could not flush %s to disk (%s)" | ||
| 616 | " -- keeping the previous file\n", tmp.c_str(), what); | ||
| 617 | fclose(f); | ||
| 618 | remove(tmp.c_str()); | ||
| 619 | return false; | ||
| 620 | } | ||
| 621 | // fclose() is where buffered data actually reaches the OS, so a failure | ||
| 622 | // here means the .tmp is short -- renaming it over the real file would | ||
| 623 | // destroy the good copy, which is exactly what these files exist to | ||
| 624 | // prevent. | ||
| 625 | if (fclose(f) != 0) { | ||
| 626 | printf("[payout] WARNING: could not close %s (%s)\n", | ||
| 627 | tmp.c_str(), what); | ||
| 628 | remove(tmp.c_str()); | ||
| 629 | return false; | ||
| 630 | } | ||
| 631 | #ifdef _WIN32 | ||
| 632 | // rename() on Windows will not replace an existing file. | ||
| 633 | remove(path.c_str()); | ||
| 634 | #endif | ||
| 635 | if (rename(tmp.c_str(), path.c_str()) != 0) { | ||
| 636 | printf("[payout] WARNING: could not replace %s (%s)\n", | ||
| 637 | path.c_str(), what); | ||
| 638 | remove(tmp.c_str()); | ||
| 639 | return false; | ||
| 640 | } | ||
| 641 | SyncDirectoryOf(path); | ||
| 642 | return true; | ||
| 643 | } | ||
| 644 | |||
| 645 | // Paths whose loader threw part-way through, and which must therefore never be | ||
| 646 | // written again by this process. | ||
| 647 | // | ||
| 648 | // All three loaders push entries into the live structure INSIDE the try, so an | ||
| 649 | // exception half way down a file leaves a PARTIAL ledger in memory -- the | ||
| 650 | // entries before the bad one, and nothing after it. The first publish after | ||
| 651 | // that then writes the partial state over the file, and the only record of | ||
| 652 | // what was owed is gone. ReconcilePayouts publishes unconditionally at | ||
| 653 | // startup, so "the first publish" is a few milliseconds later, before an | ||
| 654 | // operator could possibly intervene. a8e2f58 named this failure mode in its | ||
| 655 | // own commit message and then only fixed the write side. | ||
| 656 | // | ||
| 657 | // The file is renamed out of the way instead of being left in place, so the | ||
| 658 | // evidence survives even if the operator restarts into the same fault, and so | ||
| 659 | // a node that comes up owing nobody anything is visibly missing its ledger | ||
| 660 | // rather than quietly appearing to have none. | ||
| 661 | static std::set<std::string> gRefusedFiles; | ||
| 662 | static std::mutex gRefusedFilesMutex; | ||
| 663 | |||
| 664 | static bool FileIsRefused(const std::string& path) | ||
| 665 | { | ||
| 666 | std::lock_guard<std::mutex> lk(gRefusedFilesMutex); | ||
| 667 | return gRefusedFiles.count(path) > 0; | ||
| 668 | } | ||
| 669 | |||
| 670 | // Quarantine `path` and refuse to write it for the rest of the process. | ||
| 671 | // | ||
| 672 | // Prints unconditionally: an operator who has just lost a payout ledger must | ||
| 673 | // be told at whatever log settings they were already running with, and the | ||
| 674 | // name it was moved to is the only way back to the data. | ||
| 675 | static void RefuseFileAfterLoadFailure(const std::string& path, const char* what, | ||
| 676 | const char* why) | ||
| 677 | { | ||
| 678 | { | ||
| 679 | std::lock_guard<std::mutex> lk(gRefusedFilesMutex); | ||
| 680 | gRefusedFiles.insert(path); | ||
| 681 | } | ||
| 682 | std::string bad = strprintf("%s.bad.%lld", path.c_str(), (long long)GetTime()); | ||
| 683 | bool moved = (rename(path.c_str(), bad.c_str()) == 0); | ||
| 684 | printf("[payout] ERROR: %s (%s) did not parse: %s\n" | ||
| 685 | "[payout] ERROR: what loaded before the error is a PARTIAL picture," | ||
| 686 | " so this file will not be written again by this process --" | ||
| 687 | " restart once the cause is fixed\n" | ||
| 688 | "[payout] ERROR: the file %s\n", | ||
| 689 | path.c_str(), what, why, | ||
| 690 | moved ? strprintf("has been kept as %s", bad.c_str()).c_str() | ||
| 691 | : "could NOT be renamed out of the way -- copy it now"); | ||
| 692 | } | ||
| 693 | |||
| 694 | // Publish a versioned image of some in-memory state to `path`. | ||
| 695 | // | ||
| 696 | // `fileMutex` serialises writers of `path` and guards `publishedSeq`. A blob | ||
| 697 | // older than what is already published is dropped: these files are whole-state | ||
| 698 | // dumps, so a higher sequence number is by construction a later state of the | ||
| 699 | // same structure and publishing it loses nothing. On a write failure | ||
| 700 | // `publishedSeq` is deliberately left alone, so the next (newer) write still | ||
| 701 | // gets a chance to land. | ||
| 702 | static void PublishVersioned(const std::string& path, const VersionedBlob& v, | ||
| 703 | std::mutex& fileMutex, uint64& publishedSeq, | ||
| 704 | const char* what) | ||
| 705 | { | ||
| 706 | // A default-constructed blob is not a snapshot of anything; it is a | ||
| 707 | // VersionedBlob nobody filled in. Today they are already dropped, because | ||
| 708 | // every serialiser stamps ++gSeq and so never produces 0, and 0 fails the | ||
| 709 | // seq <= publishedSeq test below -- but only for as long as publishedSeq | ||
| 710 | // starts at 0 and nobody writes `seq = gSeq++`. One character, and the | ||
| 711 | // ledger file becomes "[]". Say it out loud instead of relying on it. | ||
| 712 | if (!v.seq) | ||
| 713 | return; | ||
| 714 | // A file whose loader threw is not a file this process knows the contents | ||
| 715 | // of. Writing it would replace a full record of what was owed with | ||
| 716 | // whatever happened to parse before the error. | ||
| 717 | if (FileIsRefused(path)) | ||
| 718 | return; | ||
| 719 | std::lock_guard<std::mutex> lk(fileMutex); | ||
| 720 | if (v.seq <= publishedSeq) | ||
| 721 | return; | ||
| 722 | if (!WriteFileAtomic(path, v.blob, what)) | ||
| 723 | return; | ||
| 724 | publishedSeq = v.seq; | ||
| 725 | } | ||
| 726 | |||
| 727 | // Identity of a payout round. Used as the pool-round proof id and, more | ||
| 728 | // importantly, as the `pool_payout_id` written into wallet transaction | ||
| 729 | // metadata and matched by FindRecordedPoolPayout(). | ||
| 730 | // | ||
| 731 | // Derived from the block HASH, not from its height alone. Upstream uses | ||
| 732 | // "block-<height>", which names two different blocks identically across a | ||
| 733 | // reorg -- and because this id is what the crash-recovery probe matches on, a | ||
| 734 | // height-keyed id lets that probe find the *previous* round's transaction and | ||
| 735 | // conclude a miner has already been paid when they have not. The height stays | ||
| 736 | // in the string for readability only. | ||
| 737 | // | ||
| 738 | // A zero hash means the entry predates hash keying; those keep the old form so | ||
| 739 | // anything already recorded against them still matches. | ||
| 740 | static std::string PayoutIdForBlock(int blockHeight, const uint256& blockHash) | ||
| 741 | { | ||
| 742 | if (blockHash == 0) | ||
| 743 | return strprintf("block-%d", blockHeight); | ||
| 744 | return strprintf("block-%d-%s", blockHeight, | ||
| 745 | blockHash.ToString().substr(0, 16).c_str()); | ||
| 746 | } | ||
| 747 | |||
| 748 | // Do these two references name the same payout round? | ||
| 749 | // | ||
| 750 | // The block hash alone does not decide it, and every ledger lookup used to | ||
| 751 | // assume it did. Entries grandfathered in from a file written before payouts | ||
| 752 | // were keyed by hash all carry hash 0, so under a hash-only comparison the | ||
| 753 | // first zero-hash entry answers for every other one: RecordPayoutTxid writes | ||
| 754 | // a txid against the wrong round (which is then erased as "fully paid" while | ||
| 755 | // the real debt still stands), and ReconcilePayouts drops a live in-flight | ||
| 756 | // record because a *different* zero-hash round happened to pay that address. | ||
| 757 | // | ||
| 758 | // A non-zero hash is unique and decides on its own -- that is the whole point | ||
| 759 | // of keying on it. A zero hash falls back to the height, which | ||
| 760 | // LoadPendingPayouts supplies for any entry that can name its own maturity. | ||
| 761 | // Mixed hash-ness never matches: refusing to match is the safe direction for | ||
| 762 | // every caller here, because each of them treats a match as "this debt is | ||
| 763 | // already dealt with". | ||
| 764 | static bool SamePayoutRound(int aHeight, const uint256& aHash, | ||
| 765 | int bHeight, const uint256& bHash) | ||
| 766 | { | ||
| 767 | if (aHash != bHash) | ||
| 768 | return false; | ||
| 769 | if (aHash != 0) | ||
| 770 | return true; | ||
| 771 | return aHeight == bHeight; | ||
| 772 | } | ||
| 773 | |||
| 774 | // An entry with neither a hash nor a height cannot be told apart from any | ||
| 775 | // other entry in the same state, so nothing may be recorded against it: a | ||
| 776 | // txid written to the wrong one reports a debt as settled that is still owed. | ||
| 777 | // Such an entry is never paid and is logged instead -- refusing to act is the | ||
| 778 | // safe failure, and no file this tree has ever written can produce one. | ||
| 779 | static bool PayoutRoundIdentifiable(int height, const uint256& hash) | ||
| 780 | { | ||
| 781 | return hash != 0 || height != 0; | ||
| 782 | } | ||
| 783 | |||
| 784 | static bool PayoutFullyPaid(const PendingPayout& pp) | ||
| 785 | { | ||
| 786 | for (const auto& kv : pp.amounts) | ||
| 787 | if (pp.paidTxids.count(kv.first) == 0) | ||
| 788 | return false; | ||
| 789 | return true; | ||
| 790 | } | ||
| 791 | |||
| 792 | // Serialise the ledger and stamp it with the version it was taken at. Caller | ||
| 793 | // must hold gPayoutLedgerMutex. | ||
| 794 | // | ||
| 795 | // Split from the file write so no thread holds the ledger mutex across the | ||
| 796 | // I/O. Upstream writes the file inside the per-recipient loop with | ||
| 797 | // gPayoutLedgerMutex held, while the event loop's FlushMaturePayouts blocks on | ||
| 798 | // that same mutex -- one slow fsync per recipient, on the thread the miners' | ||
| 799 | // I/O shares. The sequence number is what lets the write happen later, and | ||
| 800 | // possibly out of order, without a stale image reaching the disk. | ||
| 801 | static VersionedBlob SerializePendingPayoutsLocked() | ||
| 320 | { | 802 | { |
| 321 | nlohmann::json arr = nlohmann::json::array(); | 803 | nlohmann::json arr = nlohmann::json::array(); |
| 322 | for (const auto& pp : gPendingPayouts) { | 804 | for (const auto& pp : gPendingPayouts) { |
| @@ -324,21 +806,56 @@ static void SavePendingPayouts() | |||
| 324 | obj["blockHeight"] = pp.blockHeight; | 806 | obj["blockHeight"] = pp.blockHeight; |
| 325 | obj["blockHash"] = pp.blockHash.ToString(); | 807 | obj["blockHash"] = pp.blockHash.ToString(); |
| 326 | obj["matureAtHeight"] = pp.matureAtHeight; | 808 | obj["matureAtHeight"] = pp.matureAtHeight; |
| 809 | // No retryAfter: it was never written with anything but zero, and the | ||
| 810 | // retry clock that is actually used is on PayingRecord. Files that | ||
| 811 | // still carry the key load fine (it is ignored), and a build that | ||
| 812 | // expects it defaults an absent one to 0, which means "ready now" -- | ||
| 813 | // the same answer it always got. | ||
| 327 | nlohmann::json amounts = nlohmann::json::object(); | 814 | nlohmann::json amounts = nlohmann::json::object(); |
| 328 | for (const auto& kv : pp.amounts) | 815 | for (const auto& kv : pp.amounts) |
| 329 | amounts[kv.first] = kv.second; | 816 | amounts[kv.first] = kv.second; |
| 330 | obj["amounts"] = amounts; | 817 | obj["amounts"] = amounts; |
| 818 | nlohmann::json paid = nlohmann::json::object(); | ||
| 819 | for (const auto& kv : pp.paidTxids) | ||
| 820 | paid[kv.first] = kv.second; | ||
| 821 | obj["paidTxids"] = paid; | ||
| 331 | arr.push_back(obj); | 822 | arr.push_back(obj); |
| 332 | } | 823 | } |
| 333 | std::string path = PayoutFilePath(); | 824 | VersionedBlob v; |
| 334 | FILE* f = fopen(path.c_str(), "w"); | 825 | v.blob = arr.dump(2); |
| 335 | if (f) { | 826 | v.seq = ++gPayoutLedgerSeq; |
| 336 | std::string s = arr.dump(2); | 827 | return v; |
| 337 | fwrite(s.c_str(), 1, s.size(), f); | 828 | } |
| 338 | fclose(f); | 829 | |
| 339 | } else { | 830 | // Publish a ledger snapshot. Holds no data mutex; safe -- and intended -- to |
| 340 | LogPrint("payout", "[payout] WARNING: could not write %s\n", path.c_str()); | 831 | // call with gPayoutLedgerMutex released. |
| 341 | } | 832 | static void PublishPendingPayouts(const VersionedBlob& v) |
| 833 | { | ||
| 834 | PublishVersioned(PayoutFilePath(), v, gPayoutLedgerFileMutex, | ||
| 835 | gPayoutLedgerPublishedSeq, "pending payout ledger"); | ||
| 836 | } | ||
| 837 | |||
| 838 | // Parse a persisted block-hash field, refusing to guess. SetHex() reads | ||
| 839 | // whatever hex prefix it finds and stops at the first character it does not | ||
| 840 | // recognise, so a corrupt value does not fail to parse -- it parses as a | ||
| 841 | // DIFFERENT hash. Both shapes of that are wrong in a dangerous direction: a | ||
| 842 | // value with a hex-looking prefix ("deadbeef...") becomes a hash no block | ||
| 843 | // ever had, which PayoutBlockStillOnChain reads as an orphaned block and | ||
| 844 | // FlushMaturePayouts then VOIDS the debt; a value with no hex prefix at all | ||
| 845 | // becomes 0, which is grandfathered past the orphan check on the strength | ||
| 846 | // of garbage. A field that is not the 64-digit hex string ToString() wrote | ||
| 847 | // is corruption, and corruption is a load failure like any other parse | ||
| 848 | // error -- the file is quarantined, not read around. | ||
| 849 | static uint256 ParsePayoutBlockHash(const std::string& str) | ||
| 850 | { | ||
| 851 | if (str.size() != 64 || | ||
| 852 | str.find_first_not_of("0123456789abcdefABCDEF") != std::string::npos) | ||
| 853 | throw std::runtime_error( | ||
| 854 | strprintf("blockHash \"%s\" is not a 64-digit hex string", | ||
| 855 | str.c_str())); | ||
| 856 | uint256 h; | ||
| 857 | h.SetHex(str); | ||
| 858 | return h; | ||
| 342 | } | 859 | } |
| 343 | 860 | ||
| 344 | static void LoadPendingPayouts() | 861 | static void LoadPendingPayouts() |
| @@ -354,6 +871,7 @@ static void LoadPendingPayouts() | |||
| 354 | fclose(f); | 871 | fclose(f); |
| 355 | try { | 872 | try { |
| 356 | nlohmann::json arr = nlohmann::json::parse(s); | 873 | nlohmann::json arr = nlohmann::json::parse(s); |
| 874 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 357 | for (const auto& obj : arr) { | 875 | for (const auto& obj : arr) { |
| 358 | PendingPayout pp; | 876 | PendingPayout pp; |
| 359 | pp.matureAtHeight = obj["matureAtHeight"].get<int>(); | 877 | pp.matureAtHeight = obj["matureAtHeight"].get<int>(); |
| @@ -361,18 +879,33 @@ static void LoadPendingPayouts() | |||
| 361 | // field. Leave blockHash zero and they are grandfathered past the | 879 | // field. Leave blockHash zero and they are grandfathered past the |
| 362 | // orphan check rather than voided on a guess. | 880 | // orphan check rather than voided on a guess. |
| 363 | if (obj.contains("blockHash") && obj["blockHash"].is_string()) | 881 | if (obj.contains("blockHash") && obj["blockHash"].is_string()) |
| 364 | pp.blockHash.SetHex(obj["blockHash"].get<std::string>()); | 882 | pp.blockHash = |
| 883 | ParsePayoutBlockHash(obj["blockHash"].get<std::string>()); | ||
| 365 | if (obj.contains("blockHeight") && obj["blockHeight"].is_number()) | 884 | if (obj.contains("blockHeight") && obj["blockHeight"].is_number()) |
| 366 | pp.blockHeight = obj["blockHeight"].get<int>(); | 885 | pp.blockHeight = obj["blockHeight"].get<int>(); |
| 886 | // Upstream's tolerance for an entry with no usable height, kept | ||
| 887 | // because it costs nothing and an entry that cannot name its block | ||
| 888 | // is worse than one derived from its maturity. COINBASE_MATURITY | ||
| 889 | // and not COINBASE_SPENDABLE_DEPTH: this reverses whatever the | ||
| 890 | // writer of *that* file used, and only files old enough to lack | ||
| 891 | // blockHeight are affected -- they were written when maturity was | ||
| 892 | // still COINBASE_MATURITY. | ||
| 893 | if (pp.blockHeight == 0 && pp.matureAtHeight > COINBASE_MATURITY) | ||
| 894 | pp.blockHeight = pp.matureAtHeight - COINBASE_MATURITY; | ||
| 895 | // "retryAfter" is deliberately not read. Files written by the | ||
| 896 | // merged build carry it, always as 0, and there is nowhere left to | ||
| 897 | // put it -- the retry clock is PayingRecord's. | ||
| 367 | for (auto it = obj["amounts"].begin(); it != obj["amounts"].end(); ++it) | 898 | for (auto it = obj["amounts"].begin(); it != obj["amounts"].end(); ++it) |
| 368 | pp.amounts[it.key()] = it.value().get<int64>(); | 899 | pp.amounts[it.key()] = it.value().get<int64>(); |
| 900 | if (obj.contains("paidTxids") && obj["paidTxids"].is_object()) | ||
| 901 | for (auto it = obj["paidTxids"].begin(); it != obj["paidTxids"].end(); ++it) | ||
| 902 | pp.paidTxids[it.key()] = it.value().get<std::string>(); | ||
| 369 | gPendingPayouts.push_back(pp); | 903 | gPendingPayouts.push_back(pp); |
| 370 | } | 904 | } |
| 371 | LogPrint("payout", "[payout] loaded %zu pending payout(s) from disk\n", | 905 | LogPrint("payout", "[payout] loaded %zu pending payout(s) from disk\n", |
| 372 | gPendingPayouts.size()); | 906 | gPendingPayouts.size()); |
| 373 | } catch (const std::exception& e) { | 907 | } catch (const std::exception& e) { |
| 374 | LogPrint("payout", "[payout] WARNING: failed to parse %s: %s\n", | 908 | RefuseFileAfterLoadFailure(path, "pending payout ledger", e.what()); |
| 375 | path.c_str(), e.what()); | ||
| 376 | } | 909 | } |
| 377 | } | 910 | } |
| 378 | 911 | ||
| @@ -388,6 +921,18 @@ static void LoadPendingPayouts() | |||
| 388 | // and reconciling against the wallet on startup, is what makes a restart | 921 | // and reconciling against the wallet on startup, is what makes a restart |
| 389 | // survivable. | 922 | // survivable. |
| 390 | struct PayingRecord { | 923 | struct PayingRecord { |
| 924 | // Identity of this SLOT, distinct from the identity of the payment it | ||
| 925 | // describes. ForgetPayment and ReschedulePayment used to find their record | ||
| 926 | // by comparing every field, which is only an identity while no two records | ||
| 927 | // can be equal -- and a restart used to produce byte-identical duplicates | ||
| 928 | // (see FlushMaturePayouts), so the first match absorbed every update and | ||
| 929 | // the second was never touched again. | ||
| 930 | // | ||
| 931 | // Process-local and deliberately not persisted: gPaying is rebuilt from | ||
| 932 | // disk at every start, so an id only ever has to name a slot within one | ||
| 933 | // run, and reading ids back would mean trusting a file to contain no | ||
| 934 | // duplicates. Assigned under gPayoutExecMutex from gPayingNextId. | ||
| 935 | uint64 id = 0; | ||
| 391 | int blockHeight = 0; | 936 | int blockHeight = 0; |
| 392 | uint256 blockHash = 0; // zero = predates hash keying, see PendingPayout | 937 | uint256 blockHash = 0; // zero = predates hash keying, see PendingPayout |
| 393 | int matureAtHeight = 0; | 938 | int matureAtHeight = 0; |
| @@ -396,18 +941,66 @@ struct PayingRecord { | |||
| 396 | int retries = 0; | 941 | int retries = 0; |
| 397 | int deferrals = 0; | 942 | int deferrals = 0; |
| 398 | int64 retryAfter = 0; | 943 | int64 retryAfter = 0; |
| 944 | // Ask the wallet whether this payment already went out, once, before the | ||
| 945 | // first send attempt. Upstream asks on every attempt: FindRecordedPoolPayout | ||
| 946 | // is a linear scan of mapWallet under cs_mapWallet, per recipient, so with | ||
| 947 | // 2000 rotating payout addresses and a retry budget of 1440 that is a | ||
| 948 | // scan per address per minute for a day. The question it answers -- "did a | ||
| 949 | // previous process send this before it died?" -- can only change across a | ||
| 950 | // restart, and every restart re-sets this flag (ReconcilePayouts), so | ||
| 951 | // asking once per record is the whole of the answer. | ||
| 952 | // | ||
| 953 | // Not persisted: a record read back from disk is by definition a restart. | ||
| 954 | bool fProbeWallet = true; | ||
| 399 | }; | 955 | }; |
| 400 | static std::vector<PayingRecord> gPaying; // guarded by gPayoutExecMutex | 956 | static std::vector<PayingRecord> gPaying; // guarded by gPayoutExecMutex |
| 401 | static std::mutex gPayoutExecMutex; | 957 | static std::mutex gPayoutExecMutex; |
| 958 | static uint64 gPayingNextId = 0; // guarded by gPayoutExecMutex | ||
| 959 | static uint64 gPayingSeq = 0; // guarded by gPayoutExecMutex | ||
| 960 | static std::mutex gPayingFileMutex; | ||
| 961 | static uint64 gPayingPublishedSeq = 0; // guarded by gPayingFileMutex | ||
| 402 | static std::atomic<bool> gPayoutThreadRunning{false}; | 962 | static std::atomic<bool> gPayoutThreadRunning{false}; |
| 403 | 963 | ||
| 964 | // Upstream's gPayoutExecQueue/gPayoutExecMutex/gPayoutThreadRunning are | ||
| 965 | // declared above with gPaying: the in-flight list IS the exec queue in this | ||
| 966 | // tree, at per-recipient rather than per-round granularity, so a second queue | ||
| 967 | // would be a second answer to "who is owed money". Only the dedupe set is new. | ||
| 968 | // | ||
| 969 | // Ids of rounds currently exploded into gPaying, so a flush cannot queue the | ||
| 970 | // same round twice. Guarded by gPayoutExecMutex. Seeded from the restored | ||
| 971 | // in-flight list by ReconcilePayouts -- it is memory-only, and leaving it | ||
| 972 | // empty at startup was half of why a restart duplicated everything in flight. | ||
| 973 | static std::set<std::string> gQueuedPayouts; | ||
| 974 | |||
| 975 | // Is this exact payment already in flight? gPaying is keyed on | ||
| 976 | // (round, address): the ledger holds one amount per address per round, so two | ||
| 977 | // records for that pair are two standing orders to send the same money. | ||
| 978 | // Caller must hold gPayoutExecMutex. | ||
| 979 | static bool PaymentInFlightLocked(int blockHeight, const uint256& blockHash, | ||
| 980 | const std::string& address) | ||
| 981 | { | ||
| 982 | for (const PayingRecord& e : gPaying) | ||
| 983 | if (e.address == address | ||
| 984 | && SamePayoutRound(e.blockHeight, e.blockHash, blockHeight, blockHash)) | ||
| 985 | return true; | ||
| 986 | return false; | ||
| 987 | } | ||
| 988 | |||
| 404 | static std::string PayingFilePath() | 989 | static std::string PayingFilePath() |
| 405 | { | 990 | { |
| 406 | return GetAppDir() + "/paying_payouts.json"; | 991 | return GetAppDir() + "/paying_payouts.json"; |
| 407 | } | 992 | } |
| 408 | 993 | ||
| 409 | // Caller must hold gPayoutExecMutex. | 994 | // Serialise the in-flight list. Caller must hold gPayoutExecMutex. |
| 410 | static void SavePayingLocked() | 995 | // |
| 996 | // gPayoutExecMutex already serialises every writer of this file, so it never | ||
| 997 | // had the interleaving race pending_payouts.json had. It is still split the | ||
| 998 | // same way for a different reason: FlushMaturePayouts runs on the event loop | ||
| 999 | // and holds gPayoutExecMutex, so writing (and, since this series, fsyncing) | ||
| 1000 | // the file under that mutex parks miner I/O behind the payout thread's disk. | ||
| 1001 | // The version stamp keeps the file monotonic now that the write is outside | ||
| 1002 | // the mutex that orders the writers. | ||
| 1003 | static VersionedBlob SerializePayingLocked() | ||
| 411 | { | 1004 | { |
| 412 | nlohmann::json arr = nlohmann::json::array(); | 1005 | nlohmann::json arr = nlohmann::json::array(); |
| 413 | for (const auto& r : gPaying) { | 1006 | for (const auto& r : gPaying) { |
| @@ -419,24 +1012,24 @@ static void SavePayingLocked() | |||
| 419 | obj["amount"] = r.amount; | 1012 | obj["amount"] = r.amount; |
| 420 | obj["retries"] = r.retries; | 1013 | obj["retries"] = r.retries; |
| 421 | obj["deferrals"] = r.deferrals; | 1014 | obj["deferrals"] = r.deferrals; |
| 1015 | // The live retry clock, which used not to be written at all: every | ||
| 1016 | // restart handed every backing-off payment a retryAfter of 0 and put | ||
| 1017 | // the whole set into the very next 500 ms batch, so a node restarted | ||
| 1018 | // in a loop retried continuously instead of once a minute. | ||
| 1019 | obj["retryAfter"] = r.retryAfter; | ||
| 422 | arr.push_back(obj); | 1020 | arr.push_back(obj); |
| 423 | } | 1021 | } |
| 424 | std::string path = PayingFilePath(); | 1022 | VersionedBlob v; |
| 425 | // Write to a temporary file and rename, so a crash mid-write cannot leave | 1023 | v.blob = arr.dump(2); |
| 426 | // a truncated list of who is still owed money. | 1024 | v.seq = ++gPayingSeq; |
| 427 | std::string tmp = path + ".tmp"; | 1025 | return v; |
| 428 | FILE* f = fopen(tmp.c_str(), "w"); | 1026 | } |
| 429 | if (!f) { | 1027 | |
| 430 | LogPrint("payout", "[payout] WARNING: could not write %s\n", tmp.c_str()); | 1028 | // Publish an in-flight-list snapshot. Call with gPayoutExecMutex released. |
| 431 | return; | 1029 | static void PublishPaying(const VersionedBlob& v) |
| 432 | } | 1030 | { |
| 433 | std::string s = arr.dump(2); | 1031 | PublishVersioned(PayingFilePath(), v, gPayingFileMutex, |
| 434 | fwrite(s.c_str(), 1, s.size(), f); | 1032 | gPayingPublishedSeq, "in-flight payment list"); |
| 435 | fflush(f); | ||
| 436 | fclose(f); | ||
| 437 | if (rename(tmp.c_str(), path.c_str()) != 0) | ||
| 438 | LogPrint("payout", "[payout] WARNING: could not rename %s -> %s\n", | ||
| 439 | tmp.c_str(), path.c_str()); | ||
| 440 | } | 1033 | } |
| 441 | 1034 | ||
| 442 | static void LoadPaying() | 1035 | static void LoadPaying() |
| @@ -452,26 +1045,209 @@ static void LoadPaying() | |||
| 452 | fclose(f); | 1045 | fclose(f); |
| 453 | try { | 1046 | try { |
| 454 | nlohmann::json arr = nlohmann::json::parse(s); | 1047 | nlohmann::json arr = nlohmann::json::parse(s); |
| 1048 | // gPaying and gPayingNextId are both annotated as guarded by | ||
| 1049 | // gPayoutExecMutex and this was mutating them without it. Safe today | ||
| 1050 | // -- LoadPaying only runs at startup, before the payout thread is | ||
| 1051 | // spawned -- but "safe because of where it is called from" is a fact | ||
| 1052 | // about the caller, not about this function, and the annotation is | ||
| 1053 | // what every other reader will trust. The mutex is uncontended here by | ||
| 1054 | // construction, so taking it costs nothing. | ||
| 1055 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 455 | for (const auto& obj : arr) { | 1056 | for (const auto& obj : arr) { |
| 456 | PayingRecord r; | 1057 | PayingRecord r; |
| 457 | r.address = obj["address"].get<std::string>(); | 1058 | r.address = obj["address"].get<std::string>(); |
| 458 | r.amount = obj["amount"].get<int64>(); | 1059 | r.amount = obj["amount"].get<int64>(); |
| 459 | r.matureAtHeight = obj["matureAtHeight"].get<int>(); | 1060 | r.matureAtHeight = obj["matureAtHeight"].get<int>(); |
| 460 | if (obj.contains("blockHash") && obj["blockHash"].is_string()) | 1061 | if (obj.contains("blockHash") && obj["blockHash"].is_string()) |
| 461 | r.blockHash.SetHex(obj["blockHash"].get<std::string>()); | 1062 | r.blockHash = |
| 1063 | ParsePayoutBlockHash(obj["blockHash"].get<std::string>()); | ||
| 462 | if (obj.contains("blockHeight") && obj["blockHeight"].is_number()) | 1064 | if (obj.contains("blockHeight") && obj["blockHeight"].is_number()) |
| 463 | r.blockHeight = obj["blockHeight"].get<int>(); | 1065 | r.blockHeight = obj["blockHeight"].get<int>(); |
| 464 | if (obj.contains("retries") && obj["retries"].is_number()) | 1066 | if (obj.contains("retries") && obj["retries"].is_number()) |
| 465 | r.retries = obj["retries"].get<int>(); | 1067 | r.retries = obj["retries"].get<int>(); |
| 466 | if (obj.contains("deferrals") && obj["deferrals"].is_number()) | 1068 | if (obj.contains("deferrals") && obj["deferrals"].is_number()) |
| 467 | r.deferrals = obj["deferrals"].get<int>(); | 1069 | r.deferrals = obj["deferrals"].get<int>(); |
| 1070 | // Absent in files written before this change; 0 means ready now, | ||
| 1071 | // which is exactly the behaviour those files used to get. | ||
| 1072 | if (obj.contains("retryAfter") && obj["retryAfter"].is_number()) | ||
| 1073 | r.retryAfter = obj["retryAfter"].get<int64>(); | ||
| 1074 | // Collapse duplicates as they are read. A build without the | ||
| 1075 | // ReconcilePayouts seeding below wrote a second (and after another | ||
| 1076 | // restart, a third) byte-identical record for every unpaid | ||
| 1077 | // recipient, so a file written by one of those has to be cleaned | ||
| 1078 | // up here or the pathology is inherited. The retry and deferral | ||
| 1079 | // counts are merged upwards so collapsing does not hand the | ||
| 1080 | // survivor a fresh budget. | ||
| 1081 | PayingRecord* dup = NULL; | ||
| 1082 | for (PayingRecord& e : gPaying) | ||
| 1083 | if (e.address == r.address | ||
| 1084 | && SamePayoutRound(e.blockHeight, e.blockHash, | ||
| 1085 | r.blockHeight, r.blockHash)) { | ||
| 1086 | dup = &e; | ||
| 1087 | break; | ||
| 1088 | } | ||
| 1089 | if (dup) { | ||
| 1090 | dup->retries = std::max(dup->retries, r.retries); | ||
| 1091 | dup->deferrals = std::max(dup->deferrals, r.deferrals); | ||
| 1092 | dup->retryAfter = std::max(dup->retryAfter, r.retryAfter); | ||
| 1093 | LogPrint("payout", "[payout] collapsing duplicate in-flight" | ||
| 1094 | " record for %s to %s -- one payment, one record\n", | ||
| 1095 | FormatMoney(r.amount).c_str(), r.address.c_str()); | ||
| 1096 | continue; | ||
| 1097 | } | ||
| 1098 | r.id = ++gPayingNextId; | ||
| 468 | gPaying.push_back(r); | 1099 | gPaying.push_back(r); |
| 469 | } | 1100 | } |
| 470 | LogPrint("payout", "[payout] loaded %zu in-flight payment(s) from disk\n", | 1101 | LogPrint("payout", "[payout] loaded %zu in-flight payment(s) from disk\n", |
| 471 | gPaying.size()); | 1102 | gPaying.size()); |
| 472 | } catch (const std::exception& e) { | 1103 | } catch (const std::exception& e) { |
| 473 | LogPrint("payout", "[payout] WARNING: failed to parse %s: %s\n", | 1104 | RefuseFileAfterLoadFailure(path, "in-flight payment list", e.what()); |
| 474 | path.c_str(), e.what()); | 1105 | } |
| 1106 | } | ||
| 1107 | |||
| 1108 | // --------------------------------------------------------------------------- | ||
| 1109 | // Pool round proofs (upstream cbdd06d). Disjoint from the in-flight ledger | ||
| 1110 | // above -- this is the operator's published record of how a round was split, | ||
| 1111 | // not an obligation to pay. Both sides added code at this offset; both are | ||
| 1112 | // kept. | ||
| 1113 | // --------------------------------------------------------------------------- | ||
| 1114 | static nlohmann::json PoolRoundToJson(const PoolRoundProof& r) | ||
| 1115 | { | ||
| 1116 | nlohmann::json j; | ||
| 1117 | j["id"] = r.id; | ||
| 1118 | j["blockHeight"] = r.blockHeight; | ||
| 1119 | j["matureAtHeight"] = r.matureAtHeight; | ||
| 1120 | j["createdAt"] = r.createdAt; | ||
| 1121 | j["blockHash"] = r.blockHash; | ||
| 1122 | j["coinbaseTxid"] = r.coinbaseTxid; | ||
| 1123 | j["foundBy"] = r.foundBy; | ||
| 1124 | j["feePercent"] = r.feePercent; | ||
| 1125 | j["totalShares"] = r.totalShares; | ||
| 1126 | j["coinbaseValueSatoshis"] = r.coinbaseValue; | ||
| 1127 | j["coinbaseValueBtf"] = FormatMoney(r.coinbaseValue); | ||
| 1128 | j["subsidySatoshis"] = r.subsidy; | ||
| 1129 | j["subsidyBtf"] = FormatMoney(r.subsidy); | ||
| 1130 | j["minerRewardSatoshis"] = r.minerReward; | ||
| 1131 | j["minerRewardBtf"] = FormatMoney(r.minerReward); | ||
| 1132 | j["operatorRewardSatoshis"] = r.operatorReward; | ||
| 1133 | j["operatorRewardBtf"] = FormatMoney(r.operatorReward); | ||
| 1134 | j["skippedDust"] = r.skippedDust; | ||
| 1135 | j["forfeitedBadAddressSatoshis"] = r.forfeitedBadAddress; | ||
| 1136 | j["forfeitedBadAddressBtf"] = FormatMoney(r.forfeitedBadAddress); | ||
| 1137 | |||
| 1138 | j["shares"] = nlohmann::json::object(); | ||
| 1139 | for (const auto& kv : r.shares) | ||
| 1140 | j["shares"][kv.first] = kv.second; | ||
| 1141 | |||
| 1142 | j["payouts"] = nlohmann::json::object(); | ||
| 1143 | for (const auto& kv : r.payouts) { | ||
| 1144 | nlohmann::json p; | ||
| 1145 | p["amountSatoshis"] = kv.second; | ||
| 1146 | p["amountBtf"] = FormatMoney(kv.second); | ||
| 1147 | std::map<std::string, std::string>::const_iterator tx = r.paidTxids.find(kv.first); | ||
| 1148 | if (tx != r.paidTxids.end()) | ||
| 1149 | p["txid"] = tx->second; | ||
| 1150 | else | ||
| 1151 | p["txid"] = nullptr; | ||
| 1152 | // Only emitted when it happened, so the common shape of the file is | ||
| 1153 | // unchanged for anything already parsing it. | ||
| 1154 | std::map<std::string, std::string>::const_iterator fo = r.forfeited.find(kv.first); | ||
| 1155 | if (fo != r.forfeited.end()) { | ||
| 1156 | p["forfeited"] = true; | ||
| 1157 | p["forfeitReason"] = fo->second; | ||
| 1158 | } | ||
| 1159 | j["payouts"][kv.first] = p; | ||
| 1160 | } | ||
| 1161 | return j; | ||
| 1162 | } | ||
| 1163 | |||
| 1164 | // Serialise the published round proofs. Caller must hold gPoolRoundsMutex. | ||
| 1165 | // | ||
| 1166 | // Like the in-flight list, gPoolRoundsMutex already serialises the writers | ||
| 1167 | // (RecordPoolRound on the event loop, MarkPoolRoundPaid on the payout thread), | ||
| 1168 | // so this file never had the interleaving race either -- but the event loop | ||
| 1169 | // should not be waiting on the payout thread's fsync to publish a round, so | ||
| 1170 | // the write moves outside the mutex on the same versioned terms. | ||
| 1171 | static VersionedBlob SerializePoolRoundsLocked() | ||
| 1172 | { | ||
| 1173 | nlohmann::json root; | ||
| 1174 | root["schema"] = "bitflash-pool-rounds-1"; | ||
| 1175 | root["updatedAt"] = GetTime(); | ||
| 1176 | root["rounds"] = nlohmann::json::array(); | ||
| 1177 | for (const PoolRoundProof& r : gPoolRounds) | ||
| 1178 | root["rounds"].push_back(PoolRoundToJson(r)); | ||
| 1179 | |||
| 1180 | VersionedBlob v; | ||
| 1181 | v.blob = root.dump(2); | ||
| 1182 | v.seq = ++gPoolRoundsSeq; | ||
| 1183 | return v; | ||
| 1184 | } | ||
| 1185 | |||
| 1186 | // Publish a round-proof snapshot. Call with gPoolRoundsMutex released. | ||
| 1187 | static void PublishPoolRounds(const VersionedBlob& v) | ||
| 1188 | { | ||
| 1189 | PublishVersioned(PoolRoundsFilePath(), v, gPoolRoundsFileMutex, | ||
| 1190 | gPoolRoundsPublishedSeq, "pool round proofs"); | ||
| 1191 | } | ||
| 1192 | |||
| 1193 | static void LoadPoolRounds() | ||
| 1194 | { | ||
| 1195 | std::string path = PoolRoundsFilePath(); | ||
| 1196 | FILE* f = fopen(path.c_str(), "r"); | ||
| 1197 | if (!f) return; | ||
| 1198 | fseek(f, 0, SEEK_END); | ||
| 1199 | long sz = ftell(f); | ||
| 1200 | fseek(f, 0, SEEK_SET); | ||
| 1201 | std::string s(sz, '\0'); | ||
| 1202 | fread(&s[0], 1, sz, f); | ||
| 1203 | fclose(f); | ||
| 1204 | |||
| 1205 | try { | ||
| 1206 | nlohmann::json root = nlohmann::json::parse(s); | ||
| 1207 | nlohmann::json arr = root.is_array() ? root : root["rounds"]; | ||
| 1208 | std::lock_guard<std::mutex> lk(gPoolRoundsMutex); | ||
| 1209 | for (const auto& obj : arr) { | ||
| 1210 | PoolRoundProof r; | ||
| 1211 | if (obj.contains("id") && obj["id"].is_string()) r.id = obj["id"].get<std::string>(); | ||
| 1212 | if (obj.contains("blockHeight")) r.blockHeight = obj["blockHeight"].get<int>(); | ||
| 1213 | if (obj.contains("matureAtHeight")) r.matureAtHeight = obj["matureAtHeight"].get<int>(); | ||
| 1214 | if (obj.contains("createdAt")) r.createdAt = obj["createdAt"].get<int64>(); | ||
| 1215 | if (obj.contains("blockHash") && obj["blockHash"].is_string()) r.blockHash = obj["blockHash"].get<std::string>(); | ||
| 1216 | if (obj.contains("coinbaseTxid") && obj["coinbaseTxid"].is_string()) r.coinbaseTxid = obj["coinbaseTxid"].get<std::string>(); | ||
| 1217 | if (obj.contains("foundBy") && obj["foundBy"].is_string()) r.foundBy = obj["foundBy"].get<std::string>(); | ||
| 1218 | if (obj.contains("feePercent")) r.feePercent = obj["feePercent"].get<double>(); | ||
| 1219 | if (obj.contains("totalShares")) r.totalShares = obj["totalShares"].get<uint64>(); | ||
| 1220 | if (obj.contains("coinbaseValueSatoshis")) r.coinbaseValue = obj["coinbaseValueSatoshis"].get<int64>(); | ||
| 1221 | if (obj.contains("subsidySatoshis")) r.subsidy = obj["subsidySatoshis"].get<int64>(); | ||
| 1222 | if (obj.contains("minerRewardSatoshis")) r.minerReward = obj["minerRewardSatoshis"].get<int64>(); | ||
| 1223 | if (obj.contains("operatorRewardSatoshis")) r.operatorReward = obj["operatorRewardSatoshis"].get<int64>(); | ||
| 1224 | if (obj.contains("skippedDust")) r.skippedDust = obj["skippedDust"].get<int>(); | ||
| 1225 | if (obj.contains("forfeitedBadAddressSatoshis")) r.forfeitedBadAddress = obj["forfeitedBadAddressSatoshis"].get<int64>(); | ||
| 1226 | if (obj.contains("shares") && obj["shares"].is_object()) | ||
| 1227 | for (auto it = obj["shares"].begin(); it != obj["shares"].end(); ++it) | ||
| 1228 | r.shares[it.key()] = it.value().get<uint64>(); | ||
| 1229 | if (obj.contains("payouts") && obj["payouts"].is_object()) | ||
| 1230 | for (auto it = obj["payouts"].begin(); it != obj["payouts"].end(); ++it) { | ||
| 1231 | if (it.value().is_object()) { | ||
| 1232 | r.payouts[it.key()] = it.value().value("amountSatoshis", (int64)0); | ||
| 1233 | if (it.value().contains("txid") && it.value()["txid"].is_string()) | ||
| 1234 | r.paidTxids[it.key()] = it.value()["txid"].get<std::string>(); | ||
| 1235 | if (it.value().value("forfeited", false)) | ||
| 1236 | r.forfeited[it.key()] = | ||
| 1237 | it.value().value("forfeitReason", std::string("unspecified")); | ||
| 1238 | } else if (it.value().is_number_integer()) { | ||
| 1239 | r.payouts[it.key()] = it.value().get<int64>(); | ||
| 1240 | } | ||
| 1241 | } | ||
| 1242 | if (!r.id.empty()) | ||
| 1243 | gPoolRounds.push_back(r); | ||
| 1244 | } | ||
| 1245 | while (gPoolRounds.size() > MAX_POOL_ROUNDS) | ||
| 1246 | gPoolRounds.erase(gPoolRounds.begin()); | ||
| 1247 | LogPrint("payout", "[payout] loaded %zu pool round proof(s) from disk\n", | ||
| 1248 | gPoolRounds.size()); | ||
| 1249 | } catch (const std::exception& e) { | ||
| 1250 | RefuseFileAfterLoadFailure(path, "pool round proofs", e.what()); | ||
| 475 | } | 1251 | } |
| 476 | } | 1252 | } |
| 477 | 1253 | ||
| @@ -494,105 +1270,487 @@ static bool PayoutBlockStillOnChain(const uint256& blockHash) | |||
| 494 | return false; | 1270 | return false; |
| 495 | } | 1271 | } |
| 496 | 1272 | ||
| 497 | static std::string PayoutComment(int blockHeight) | 1273 | // The comment every pre-upgrade payout transaction carries, and therefore the |
| 1274 | // ONLY needle the legacy fallback in FindRecordedPoolPayout may search with. | ||
| 1275 | // This format is frozen: change it and every payment those builds made stops | ||
| 1276 | // matching, every one of those debts looks unpaid, and reconciliation pays | ||
| 1277 | // them all a second time. | ||
| 1278 | static std::string LegacyPayoutComment(int blockHeight) | ||
| 498 | { | 1279 | { |
| 499 | return strprintf("Pool payout height %d", blockHeight); | 1280 | return strprintf("Pool payout height %d", blockHeight); |
| 500 | } | 1281 | } |
| 501 | 1282 | ||
| 502 | // Did we already send this payment before dying? Looks for a wallet | 1283 | // The comment written on new sends. It names the funding block, because the |
| 503 | // transaction of ours carrying this payout's comment and paying exactly this | 1284 | // height alone cannot: a height that is won, lost to a reorg, and won again |
| 504 | // amount to this address. | 1285 | // yields two rounds whose height-only comments are identical -- and identical |
| 1286 | // amounts too, whenever the same miner held both rounds. The structured | ||
| 1287 | // pool_payout_* metadata already keys the payment by hash for the code; this | ||
| 1288 | // puts the same fact where an operator reading the wallet can see it, and | ||
| 1289 | // makes a new transaction unmatchable by the legacy needle even if the | ||
| 1290 | // structured keys are ever lost. | ||
| 505 | // | 1291 | // |
| 506 | // Biased towards reporting "already paid" on any doubt: skipping a payment a | 1292 | // A zero hash keeps the legacy form: those records predate hash keying, and |
| 507 | // miner is owed is visible and recoverable from the log, while paying twice | 1293 | // there is no hash to name. |
| 508 | // is neither. | 1294 | static std::string PayoutComment(int blockHeight, const uint256& blockHash) |
| 509 | static bool WalletAlreadyPaid(const PayingRecord& r) | ||
| 510 | { | 1295 | { |
| 511 | uint160 h160; | 1296 | if (blockHash == 0) |
| 512 | if (!AddressToHash160(r.address, h160)) | 1297 | return LegacyPayoutComment(blockHeight); |
| 513 | return false; | 1298 | return strprintf("Pool payout height %d block %s", blockHeight, |
| 514 | CScript want; | 1299 | blockHash.ToString().c_str()); |
| 515 | want << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; | 1300 | } |
| 516 | const std::string comment = PayoutComment(r.blockHeight); | ||
| 517 | 1301 | ||
| 518 | CRITICAL_BLOCK(cs_mapWallet) | 1302 | // Startup reconciliation. |
| 1303 | // | ||
| 1304 | // This used to be the crash-recovery mechanism: it scanned the wallet with | ||
| 1305 | // WalletAlreadyPaid() and then dropped ledger entries that had already been | ||
| 1306 | // exploded into in-flight records. Neither job is right any more. | ||
| 1307 | // | ||
| 1308 | // The ledger is now authoritative and an entry survives until every recipient | ||
| 1309 | // has a txid recorded against it (upstream cbdd06d), so dropping ledger | ||
| 1310 | // entries here would reintroduce exactly the loss this merge is adopting that | ||
| 1311 | // model to prevent. And the wallet scan is now FindRecordedPoolPayout(), done | ||
| 1312 | // once per record on its first attempt, which is the same question asked in | ||
| 1313 | // the one place that can act on the answer. Running both would be two | ||
| 1314 | // authorities for the same debt. | ||
| 1315 | // | ||
| 1316 | // What is left is the useful part: the in-flight list restored from disk may | ||
| 1317 | // contain records the ledger already shows as paid (we died between the send | ||
| 1318 | // and the in-flight file being rewritten). The ledger wins. | ||
| 1319 | // | ||
| 1320 | // And the part that was missing: gQueuedPayouts is memory-only, so it came up | ||
| 1321 | // empty while gPaying came up full. FlushMaturePayouts deduped against that | ||
| 1322 | // empty set and against the ledger's paidTxids, and against nothing else -- | ||
| 1323 | // so a round that was exploded but not yet paid got a SECOND record per | ||
| 1324 | // unpaid recipient on the first flush after every restart, deterministically. | ||
| 1325 | // (Observed live: two byte-identical records for one (block, address).) The | ||
| 1326 | // surviving in-flight list is exactly the statement of which rounds are | ||
| 1327 | // already exploded, so it is what the set is rebuilt from. | ||
| 1328 | static void ReconcilePayouts() | ||
| 1329 | { | ||
| 1330 | VersionedBlob payingSnap; | ||
| 519 | { | 1331 | { |
| 520 | for (std::map<uint256, CWalletTx>::iterator it = mapWallet.begin(); | 1332 | std::lock_guard<std::mutex> exec(gPayoutExecMutex); |
| 521 | it != mapWallet.end(); ++it) { | 1333 | std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex); |
| 522 | CWalletTx& wtx = it->second; | 1334 | |
| 523 | // The comment is written only by the payout path, so its presence | 1335 | size_t settled = 0; |
| 524 | // already identifies the transaction as one of ours. | 1336 | int64 settledAmount = 0; |
| 525 | std::map<std::string, std::string>::const_iterator ci = | 1337 | for (auto it = gPaying.begin(); it != gPaying.end(); ) { |
| 526 | wtx.mapValue.find("comment"); | 1338 | bool paid = false; |
| 527 | if (ci == wtx.mapValue.end() || ci->second != comment) | 1339 | for (const PendingPayout& pp : gPendingPayouts) |
| 1340 | if (SamePayoutRound(pp.blockHeight, pp.blockHash, | ||
| 1341 | it->blockHeight, it->blockHash) | ||
| 1342 | && pp.paidTxids.count(it->address)) { | ||
| 1343 | paid = true; | ||
| 1344 | break; | ||
| 1345 | } | ||
| 1346 | if (paid) { | ||
| 1347 | settledAmount += it->amount; | ||
| 1348 | settled++; | ||
| 1349 | it = gPaying.erase(it); | ||
| 1350 | } else { | ||
| 1351 | // Every restored record gets one wallet probe before it is | ||
| 1352 | // allowed to send: this is precisely the window where the | ||
| 1353 | // wallet may hold a transaction the ledger has not recorded. | ||
| 1354 | it->fProbeWallet = true; | ||
| 1355 | ++it; | ||
| 1356 | } | ||
| 1357 | } | ||
| 1358 | if (settled) { | ||
| 1359 | LogPrint("payout", "[payout] %zu in-flight payment(s) totalling %s" | ||
| 1360 | " are already recorded paid in the ledger -- not" | ||
| 1361 | " resending\n", | ||
| 1362 | settled, FormatMoney(settledAmount).c_str()); | ||
| 1363 | } | ||
| 1364 | |||
| 1365 | // Rebuild the "already exploded" set from what survived. Without this | ||
| 1366 | // the first flush after a restart re-explodes every round in flight. | ||
| 1367 | gQueuedPayouts.clear(); | ||
| 1368 | for (const PayingRecord& r : gPaying) | ||
| 1369 | gQueuedPayouts.insert(PayoutIdForBlock(r.blockHeight, r.blockHash)); | ||
| 1370 | |||
| 1371 | payingSnap = SerializePayingLocked(); | ||
| 1372 | |||
| 1373 | if (!gPaying.empty()) { | ||
| 1374 | int64 owed = 0; | ||
| 1375 | for (const auto& r : gPaying) owed += r.amount; | ||
| 1376 | LogPrint("payout", "[payout] resuming %zu in-flight payment(s)" | ||
| 1377 | " totalling %s\n", | ||
| 1378 | gPaying.size(), FormatMoney(owed).c_str()); | ||
| 1379 | } | ||
| 1380 | } | ||
| 1381 | PublishPaying(payingSnap); | ||
| 1382 | } | ||
| 1383 | |||
| 1384 | static void RecordPoolRound(const PoolRoundProof& round) | ||
| 1385 | { | ||
| 1386 | VersionedBlob snap; | ||
| 1387 | { | ||
| 1388 | std::lock_guard<std::mutex> lk(gPoolRoundsMutex); | ||
| 1389 | bool replaced = false; | ||
| 1390 | for (PoolRoundProof& existing : gPoolRounds) { | ||
| 1391 | if (existing.id == round.id) { | ||
| 1392 | existing = round; | ||
| 1393 | replaced = true; | ||
| 1394 | break; | ||
| 1395 | } | ||
| 1396 | } | ||
| 1397 | if (!replaced) { | ||
| 1398 | gPoolRounds.push_back(round); | ||
| 1399 | while (gPoolRounds.size() > MAX_POOL_ROUNDS) | ||
| 1400 | gPoolRounds.erase(gPoolRounds.begin()); | ||
| 1401 | } | ||
| 1402 | snap = SerializePoolRoundsLocked(); | ||
| 1403 | } | ||
| 1404 | PublishPoolRounds(snap); | ||
| 1405 | } | ||
| 1406 | |||
| 1407 | static void MarkPoolRoundPaid(const std::string& id, | ||
| 1408 | const std::string& address, | ||
| 1409 | const std::string& txid) | ||
| 1410 | { | ||
| 1411 | VersionedBlob snap; | ||
| 1412 | { | ||
| 1413 | std::lock_guard<std::mutex> lk(gPoolRoundsMutex); | ||
| 1414 | bool found = false; | ||
| 1415 | for (PoolRoundProof& r : gPoolRounds) { | ||
| 1416 | if (r.id != id) | ||
| 528 | continue; | 1417 | continue; |
| 529 | foreach (const CTxOut& txout, wtx.vout) | 1418 | r.paidTxids[address] = txid; |
| 530 | if (txout.nValue == r.amount && txout.scriptPubKey == want) | 1419 | found = true; |
| 531 | return true; | 1420 | break; |
| 1421 | } | ||
| 1422 | if (!found) | ||
| 1423 | return; | ||
| 1424 | snap = SerializePoolRoundsLocked(); | ||
| 1425 | } | ||
| 1426 | PublishPoolRounds(snap); | ||
| 1427 | } | ||
| 1428 | |||
| 1429 | // Publish the fact that a recipient was written off, so the round proof and | ||
| 1430 | // the ledger agree. The ledger erases the amount (it is not owed any more and | ||
| 1431 | // must not be reported as sent); the proof keeps the amount visible and marks | ||
| 1432 | // it forfeited with a reason, because a miner who was promised a share is | ||
| 1433 | // entitled to see what happened to it. | ||
| 1434 | static void MarkPoolRoundForfeited(const std::string& id, | ||
| 1435 | const std::string& address, | ||
| 1436 | const std::string& reason) | ||
| 1437 | { | ||
| 1438 | VersionedBlob snap; | ||
| 1439 | { | ||
| 1440 | std::lock_guard<std::mutex> lk(gPoolRoundsMutex); | ||
| 1441 | bool found = false; | ||
| 1442 | for (PoolRoundProof& r : gPoolRounds) { | ||
| 1443 | if (r.id != id) | ||
| 1444 | continue; | ||
| 1445 | r.forfeited[address] = reason; | ||
| 1446 | found = true; | ||
| 1447 | break; | ||
| 532 | } | 1448 | } |
| 1449 | if (!found) | ||
| 1450 | return; | ||
| 1451 | snap = SerializePoolRoundsLocked(); | ||
| 533 | } | 1452 | } |
| 1453 | PublishPoolRounds(snap); | ||
| 1454 | } | ||
| 1455 | |||
| 1456 | // Does this round have a published round proof? | ||
| 1457 | // | ||
| 1458 | // Existence answers a temporal question the legacy fallback below needs: | ||
| 1459 | // which builds could have paid this round. RecordPoolRound writes the proof | ||
| 1460 | // when the round is created, and structured payout metadata (upstream | ||
| 1461 | // cbdd06d) is an ancestor of round proofs (upstream 83ba822) -- no build | ||
| 1462 | // able to write a proof has ever sent a payment that only a comment can | ||
| 1463 | // identify. A round with a proof therefore cannot have a comment-only | ||
| 1464 | // payment; one that matches it is another round's, colliding on the height. | ||
| 1465 | static bool PoolRoundHasProof(const std::string& id) | ||
| 1466 | { | ||
| 1467 | std::lock_guard<std::mutex> lk(gPoolRoundsMutex); | ||
| 1468 | for (const PoolRoundProof& r : gPoolRounds) | ||
| 1469 | if (r.id == id) | ||
| 1470 | return true; | ||
| 534 | return false; | 1471 | return false; |
| 535 | } | 1472 | } |
| 536 | 1473 | ||
| 537 | // Startup reconciliation. Drops in-flight records the wallet shows were | 1474 | // Which wallet transaction, if any, already paid this? |
| 538 | // already sent, and drops ledger entries that were already exploded into | 1475 | // |
| 539 | // records (which happens if we died between writing the records and rewriting | 1476 | // Two kinds of evidence, in order of quality. |
| 540 | // the ledger -- both files then describe the same debt). | 1477 | // |
| 541 | static void ReconcilePayouts() | 1478 | // Structured (pool_payout_id / pool_payout_address / pool_payout_amount, |
| 1479 | // plus pool_payout_block since this series) is upstream's, extended. The id | ||
| 1480 | // is derived from the block hash, but only from the FIRST sixteen hex digits | ||
| 1481 | // of it -- and proof-of-work concentrates zeros exactly there, so the id can | ||
| 1482 | // name two same-height rounds identically. New sends therefore record the | ||
| 1483 | // full hash as pool_payout_block, and a structured match must agree with it | ||
| 1484 | // wherever both sides have one; a transaction without the key was sent | ||
| 1485 | // before it existed, and for those the id is the best evidence there is. | ||
| 1486 | // Every transaction this build sends carries all four keys. | ||
| 1487 | // | ||
| 1488 | // But NO transaction this pool has ever sent carries it. Every payout in the | ||
| 1489 | // existing wallet has only a "comment". So on the first boot after this | ||
| 1490 | // upgrade -- precisely the boot where the binary is swapped after an unclean | ||
| 1491 | // shutdown with payments in flight -- a structured-only probe finds nothing | ||
| 1492 | // and every one of those recipients is paid a second time. The deleted | ||
| 1493 | // WalletAlreadyPaid() matched on what the payment WAS, and that evidence is | ||
| 1494 | // still there; it just has to be read. | ||
| 1495 | // | ||
| 1496 | // The legacy fallback is therefore restored, with three constraints that | ||
| 1497 | // together stop it answering for a round that is not this one: | ||
| 1498 | // | ||
| 1499 | // 1. it is only consulted when the structured lookup found nothing; | ||
| 1500 | // 2. it ignores any transaction carrying pool_payout_id. That key means this | ||
| 1501 | // build wrote the transaction, and this build's transactions are | ||
| 1502 | // adjudicated by the exact path -- so a NEW round's payment can never be | ||
| 1503 | // offered as evidence for an old one. Only pre-upgrade transactions, | ||
| 1504 | // which is exactly the population the fallback exists for, are eligible; | ||
| 1505 | // 3. `excluded` holds every txid already credited to some OTHER | ||
| 1506 | // (round, address) pair -- from the ledger's paidTxids, which is durable, | ||
| 1507 | // and from what this process has already claimed. One legacy transaction | ||
| 1508 | // can therefore settle at most one debt, so N transactions can never | ||
| 1509 | // satisfy more than N obligations; | ||
| 1510 | // | ||
| 1511 | // 4. it is not consulted at all for a round that has a round proof | ||
| 1512 | // (fAllowLegacy, decided by the caller from PoolRoundHasProof). Proofs | ||
| 1513 | // are written when a round is created, and only by builds whose every | ||
| 1514 | // payment names its round exactly -- so a comment-only transaction | ||
| 1515 | // cannot be a payment for such a round, only a same-height collision | ||
| 1516 | // from an older one, and skipping the fallback cannot hide a real | ||
| 1517 | // payment: the structured probe above already found any that exist. | ||
| 1518 | // | ||
| 1519 | // What is left unresolved by construction is a comment-only transaction and | ||
| 1520 | // a PROOFLESS round at the same height paying the same address the same | ||
| 1521 | // amount: two grandfathered zero-hash twins, or a twin pair from the builds | ||
| 1522 | // that keyed the ledger by hash before any metadata reached the wallet. The | ||
| 1523 | // comment carries only the height, so nothing ties the transaction to one | ||
| 1524 | // twin rather than the other. That needs a reorg. | ||
| 1525 | // | ||
| 1526 | // This used to say the losing twin's ledger entry is voided by the orphan | ||
| 1527 | // check before it can be paid. That is wrong: PayoutBlockStillOnChain() | ||
| 1528 | // returns true for a zero hash, and EVERY pre-upgrade entry has a zero hash, | ||
| 1529 | // so the orphan check is a no-op for exactly the population this fallback | ||
| 1530 | // serves. | ||
| 1531 | // | ||
| 1532 | // The outcome is still safe, by a different mechanism. SamePayoutRound() | ||
| 1533 | // treats two zero-hash entries at the same height as the SAME round, so the | ||
| 1534 | // pair is indistinguishable everywhere, not only here: FlushMaturePayouts | ||
| 1535 | // explodes one of them and skips the other as already queued, | ||
| 1536 | // PaymentInFlightLocked refuses a second record for that (round, address), and | ||
| 1537 | // RecordPayoutTxid settles against the first match. The pair therefore | ||
| 1538 | // presents as ONE debt and can be paid at most once -- which is the direction | ||
| 1539 | // that matters for a fallback whose failure mode is treating one payment as | ||
| 1540 | // proof of two. | ||
| 1541 | // | ||
| 1542 | // The cost of that mechanism, stated plainly: only one of a genuine | ||
| 1543 | // same-height pair would ever be paid. It takes a reorg AND both rounds | ||
| 1544 | // predating hash keying, and no file this tree has written since can produce | ||
| 1545 | // one, so it is a property of history rather than a live risk. | ||
| 1546 | // | ||
| 1547 | // Where the evidence genuinely runs out the fallback simply does not match, | ||
| 1548 | // and the payment is made -- paying a debt we cannot prove was settled is the | ||
| 1549 | // correct default for a pool. | ||
| 1550 | enum PayoutProbeResult { | ||
| 1551 | PAYOUT_PROBE_NONE = 0, // the wallet holds no record of this payment | ||
| 1552 | PAYOUT_PROBE_STRUCTURED, // matched this build's pool_payout_* metadata | ||
| 1553 | PAYOUT_PROBE_LEGACY // matched a pre-upgrade comment and output | ||
| 1554 | }; | ||
| 1555 | |||
| 1556 | static PayoutProbeResult FindRecordedPoolPayout(const std::string& id, | ||
| 1557 | int blockHeight, | ||
| 1558 | const uint256& blockHash, | ||
| 1559 | const std::string& address, | ||
| 1560 | int64 amount, | ||
| 1561 | bool fAllowLegacy, | ||
| 1562 | const std::set<std::string>& excluded, | ||
| 1563 | std::string& txidRet) | ||
| 542 | { | 1564 | { |
| 543 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | 1565 | // Built before cs_mapWallet is taken: an invalid address has no script to |
| 1566 | // compare against, and the caller has already rejected those. | ||
| 1567 | uint160 h160; | ||
| 1568 | bool fHaveScript = AddressToHash160(address, h160); | ||
| 1569 | CScript want; | ||
| 1570 | if (fHaveScript) | ||
| 1571 | want << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; | ||
| 1572 | const std::string comment = LegacyPayoutComment(blockHeight); | ||
| 1573 | |||
| 1574 | // Candidate legacy matches, adjudicated after cs_mapWallet is released. | ||
| 1575 | // | ||
| 1576 | // The depth check cannot happen inside the scan: GetDepthInMainChain() | ||
| 1577 | // reads mapBlockIndex and pindexBest, which belong to cs_main, and cs_main | ||
| 1578 | // is taken BEFORE cs_mapWallet everywhere else in this tree | ||
| 1579 | // (CreateTransaction, ConnectBlock). Taking it the other way round here | ||
| 1580 | // would be a lock-order inversion in the payout thread. There is at most a | ||
| 1581 | // handful of these: a candidate has to carry this exact height's comment | ||
| 1582 | // and an output of exactly this amount to this address. | ||
| 1583 | std::vector<uint256> legacyCandidates; | ||
| 1584 | |||
| 1585 | // One pass, not two: this scan runs under cs_mapWallet, which the whole | ||
| 1586 | // node contends on, and the merge is already paying for it once per record. | ||
| 1587 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 1588 | { | ||
| 1589 | for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin(); | ||
| 1590 | it != mapWallet.end(); ++it) | ||
| 1591 | { | ||
| 1592 | const CWalletTx& wtx = it->second; | ||
| 1593 | map<string, string>::const_iterator idIt = wtx.mapValue.find("pool_payout_id"); | ||
| 1594 | if (idIt != wtx.mapValue.end()) { | ||
| 1595 | // Written by this build. Adjudicated exactly, and never | ||
| 1596 | // offered to the fallback below. | ||
| 1597 | if (idIt->second != id) | ||
| 1598 | continue; | ||
| 1599 | map<string, string>::const_iterator addrIt = wtx.mapValue.find("pool_payout_address"); | ||
| 1600 | if (addrIt == wtx.mapValue.end() || addrIt->second != address) | ||
| 1601 | continue; | ||
| 1602 | map<string, string>::const_iterator amtIt = wtx.mapValue.find("pool_payout_amount"); | ||
| 1603 | if (amtIt == wtx.mapValue.end() || amtIt->second != strprintf("%lld", amount)) | ||
| 1604 | continue; | ||
| 1605 | // The id alone cannot tell twin rounds at one height apart | ||
| 1606 | // (see above); the full hash can, where the transaction | ||
| 1607 | // recorded one. Where it did not, the id decides, as it | ||
| 1608 | // always has -- refusing those transactions would make every | ||
| 1609 | // payment sent before the key existed look unpaid and pay it | ||
| 1610 | // again. | ||
| 1611 | if (blockHash != 0) { | ||
| 1612 | map<string, string>::const_iterator blkIt = | ||
| 1613 | wtx.mapValue.find("pool_payout_block"); | ||
| 1614 | if (blkIt != wtx.mapValue.end() | ||
| 1615 | && blkIt->second != blockHash.ToString()) | ||
| 1616 | continue; // a twin round's payment, not this round's | ||
| 1617 | } | ||
| 1618 | txidRet = wtx.GetHash().GetHex(); | ||
| 1619 | return PAYOUT_PROBE_STRUCTURED; | ||
| 1620 | } | ||
| 544 | 1621 | ||
| 545 | size_t settled = 0; | 1622 | if (!fAllowLegacy) |
| 546 | int64 settledAmount = 0; | 1623 | continue; // comment-only evidence is inadmissible: rule 4 |
| 547 | for (auto it = gPaying.begin(); it != gPaying.end(); ) { | 1624 | if (!fHaveScript) |
| 548 | if (WalletAlreadyPaid(*it)) { | 1625 | continue; |
| 549 | settledAmount += it->amount; | 1626 | // The comment is written only by the payout path, so its presence |
| 550 | settled++; | 1627 | // already identifies the transaction as one of ours. |
| 551 | it = gPaying.erase(it); | 1628 | map<string, string>::const_iterator ci = wtx.mapValue.find("comment"); |
| 552 | } else { | 1629 | if (ci == wtx.mapValue.end() || ci->second != comment) |
| 553 | ++it; | 1630 | continue; |
| 1631 | if (excluded.count(wtx.GetHash().GetHex())) | ||
| 1632 | continue; // already credited to a different debt | ||
| 1633 | foreach (const CTxOut& txout, wtx.vout) | ||
| 1634 | if (txout.nValue == amount && txout.scriptPubKey == want) { | ||
| 1635 | legacyCandidates.push_back(it->first); | ||
| 1636 | break; | ||
| 1637 | } | ||
| 554 | } | 1638 | } |
| 555 | } | 1639 | } |
| 556 | if (settled) { | ||
| 557 | LogPrint("payout", "[payout] %zu in-flight payment(s) totalling %s were" | ||
| 558 | " already sent before the last shutdown -- not resending\n", | ||
| 559 | settled, FormatMoney(settledAmount).c_str()); | ||
| 560 | SavePayingLocked(); | ||
| 561 | } | ||
| 562 | 1640 | ||
| 563 | size_t dropped = 0; | 1641 | // A legacy match must have CONFIRMED. |
| 564 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { | 1642 | // |
| 565 | bool exploded = false; | 1643 | // The structured path does not need this and deliberately does not have |
| 566 | for (const auto& r : gPaying) | 1644 | // it: this process sent that transaction moments ago, and a send that |
| 567 | if (r.matureAtHeight == it->matureAtHeight && r.blockHash == it->blockHash) { | 1645 | // failed leaves nothing behind, because SendMoney calls |
| 568 | exploded = true; | 1646 | // RollbackTransactionSpent, which calls EraseFromWallet. |
| 569 | break; | 1647 | // |
| 1648 | // The legacy path is the opposite case in every respect. Those | ||
| 1649 | // transactions are days or weeks old, so whether they confirmed is a | ||
| 1650 | // settled fact rather than an open question, and a pre-upgrade payout that | ||
| 1651 | // never confirmed is evidence of NON-payment about to be read as payment. | ||
| 1652 | // This chain has had a 280-block reorg (see FlushMaturePayouts): a payout | ||
| 1653 | // invalidated by it is still sitting in mapWallet with its comment intact, | ||
| 1654 | // and the first boot after this upgrade is exactly when it gets matched. | ||
| 1655 | // The cost of being wrong here is a miner who is never paid and no record | ||
| 1656 | // that they were owed. | ||
| 1657 | CRITICAL_BLOCK(cs_main) | ||
| 1658 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 1659 | { | ||
| 1660 | for (const uint256& hash : legacyCandidates) { | ||
| 1661 | map<uint256, CWalletTx>::const_iterator it = mapWallet.find(hash); | ||
| 1662 | if (it == mapWallet.end()) | ||
| 1663 | continue; // erased under us; it is not evidence of anything | ||
| 1664 | int depth = it->second.GetDepthInMainChain(); | ||
| 1665 | if (depth < 1) { | ||
| 1666 | // Loud: this is the difference between a miner having been | ||
| 1667 | // paid and a miner about to be paid twice, decided on a fact | ||
| 1668 | // that is easy to check and impossible to recover afterwards. | ||
| 1669 | LogPrint("payout", "[payout] wallet tx %s carries the payout" | ||
| 1670 | " comment for height %d and an output of exactly %s to" | ||
| 1671 | " %s, but it is NOT in the main chain (depth %d) --" | ||
| 1672 | " refusing to read it as payment; this recipient will" | ||
| 1673 | " be paid\n", | ||
| 1674 | hash.GetHex().c_str(), blockHeight, | ||
| 1675 | FormatMoney(amount).c_str(), address.c_str(), depth); | ||
| 1676 | continue; | ||
| 570 | } | 1677 | } |
| 571 | if (exploded) { | 1678 | txidRet = hash.GetHex(); |
| 572 | LogPrint("payout", "[payout] ledger entry for block %d is already" | 1679 | return PAYOUT_PROBE_LEGACY; |
| 573 | " in flight -- dropping the duplicate\n", it->blockHeight); | ||
| 574 | it = gPendingPayouts.erase(it); | ||
| 575 | dropped++; | ||
| 576 | } else { | ||
| 577 | ++it; | ||
| 578 | } | 1680 | } |
| 579 | } | 1681 | } |
| 580 | if (dropped) SavePendingPayouts(); | 1682 | return PAYOUT_PROBE_NONE; |
| 1683 | } | ||
| 581 | 1684 | ||
| 582 | if (!gPaying.empty()) { | 1685 | // txid -> "<round id>|<address>", for payments this run has already credited |
| 583 | int64 owed = 0; | 1686 | // to a debt on legacy evidence. Stops one pre-upgrade transaction being read |
| 584 | for (const auto& r : gPaying) owed += r.amount; | 1687 | // as proof that two different obligations were met. The ledger's paidTxids |
| 585 | LogPrint("payout", "[payout] resuming %zu in-flight payment(s) totalling %s\n", | 1688 | // covers the same ground durably; this covers the window before the recovered |
| 586 | gPaying.size(), FormatMoney(owed).c_str()); | 1689 | // txid reaches the ledger, and the case where the ledger entry has already |
| 1690 | // been retired. | ||
| 1691 | static std::map<std::string, std::string> gLegacyClaimedTxids; | ||
| 1692 | static std::mutex gClaimedTxidMutex; | ||
| 1693 | |||
| 1694 | // Every txid already spoken for by some payment OTHER than this one. Collected | ||
| 1695 | // before cs_mapWallet is taken: the ledger mutex must not be acquired under a | ||
| 1696 | // wallet lock. | ||
| 1697 | static void CollectForeignPayoutTxids(int blockHeight, const uint256& blockHash, | ||
| 1698 | const std::string& address, | ||
| 1699 | const std::string& claimKey, | ||
| 1700 | std::set<std::string>& out) | ||
| 1701 | { | ||
| 1702 | { | ||
| 1703 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 1704 | for (const PendingPayout& pp : gPendingPayouts) { | ||
| 1705 | bool sameRound = SamePayoutRound(pp.blockHeight, pp.blockHash, | ||
| 1706 | blockHeight, blockHash); | ||
| 1707 | for (const auto& kv : pp.paidTxids) | ||
| 1708 | if (!(sameRound && kv.first == address)) | ||
| 1709 | out.insert(kv.second); | ||
| 1710 | } | ||
| 1711 | } | ||
| 1712 | { | ||
| 1713 | std::lock_guard<std::mutex> lk(gClaimedTxidMutex); | ||
| 1714 | for (const auto& kv : gLegacyClaimedTxids) | ||
| 1715 | if (kv.second != claimKey) | ||
| 1716 | out.insert(kv.first); | ||
| 587 | } | 1717 | } |
| 588 | } | 1718 | } |
| 589 | 1719 | ||
| 1720 | static void ClaimLegacyPayoutTxid(const std::string& txid, const std::string& claimKey) | ||
| 1721 | { | ||
| 1722 | std::lock_guard<std::mutex> lk(gClaimedTxidMutex); | ||
| 1723 | gLegacyClaimedTxids[txid] = claimKey; | ||
| 1724 | } | ||
| 1725 | |||
| 1726 | // blockHash stays a uint256 and not upstream's display string: it is the key | ||
| 1727 | // this fork's payout ledger is built on, and a key that has been through | ||
| 1728 | // ToString() is a key that can be compared wrong. It is converted to text only | ||
| 1729 | // where the round proof needs text. The other three arguments are upstream's, | ||
| 1730 | // carried for the round-proof data. | ||
| 590 | static void QueuePayouts(int blockHeight, | 1731 | static void QueuePayouts(int blockHeight, |
| 591 | const uint256& blockHash, | 1732 | const uint256& blockHash, |
| 1733 | const std::string& coinbaseTxid, | ||
| 1734 | int64 coinbaseValue, | ||
| 1735 | const std::string& foundBy, | ||
| 592 | const std::map<std::string, uint64>& shareCount, | 1736 | const std::map<std::string, uint64>& shareCount, |
| 593 | uint64 total) | 1737 | uint64 total) |
| 594 | { | 1738 | { |
| 1739 | PoolRoundProof round; | ||
| 1740 | round.id = PayoutIdForBlock(blockHeight, blockHash); | ||
| 1741 | round.blockHeight = blockHeight; | ||
| 1742 | round.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH; | ||
| 1743 | round.createdAt = GetTime(); | ||
| 1744 | round.blockHash = blockHash.ToString(); | ||
| 1745 | round.coinbaseTxid = coinbaseTxid; | ||
| 1746 | round.coinbaseValue = coinbaseValue; | ||
| 1747 | round.foundBy = foundBy; | ||
| 1748 | round.feePercent = dPoolFeePercent; | ||
| 1749 | round.totalShares = total; | ||
| 1750 | round.shares = shareCount; | ||
| 1751 | |||
| 595 | if (total == 0) { | 1752 | if (total == 0) { |
| 1753 | RecordPoolRound(round); | ||
| 596 | LogPrint("payout", "[payout] block %d: zero shares recorded -- nothing to pay out\n", | 1754 | LogPrint("payout", "[payout] block %d: zero shares recorded -- nothing to pay out\n", |
| 597 | blockHeight); | 1755 | blockHeight); |
| 598 | return; | 1756 | return; |
| @@ -608,6 +1766,10 @@ static void QueuePayouts(int blockHeight, | |||
| 608 | int64 nSubsidy = 50 * COIN; | 1766 | int64 nSubsidy = 50 * COIN; |
| 609 | nSubsidy >>= (blockHeight / 210000); // halving schedule | 1767 | nSubsidy >>= (blockHeight / 210000); // halving schedule |
| 610 | int64 minerReward = (int64)((double)nSubsidy * (1.0 - feeFrac)); | 1768 | int64 minerReward = (int64)((double)nSubsidy * (1.0 - feeFrac)); |
| 1769 | int64 operatorCut = nSubsidy - minerReward; | ||
| 1770 | round.subsidy = nSubsidy; | ||
| 1771 | round.minerReward = minerReward; | ||
| 1772 | round.operatorReward = operatorCut; | ||
| 611 | 1773 | ||
| 612 | PendingPayout pp; | 1774 | PendingPayout pp; |
| 613 | pp.blockHeight = blockHeight; | 1775 | pp.blockHeight = blockHeight; |
| @@ -621,7 +1783,9 @@ static void QueuePayouts(int blockHeight, | |||
| 621 | // alone on a private fork paying itself, so it says nothing about a pool | 1783 | // alone on a private fork paying itself, so it says nothing about a pool |
| 622 | // competing on the real chain. Waiting the extra 20 blocks makes each | 1784 | // competing on the real chain. Waiting the extra 20 blocks makes each |
| 623 | // payout funded by construction, and puts the maturity gate past the | 1785 | // payout funded by construction, and puts the maturity gate past the |
| 624 | // reorg depth that orphaned 280 blocks of obligations. | 1786 | // reorg depth that orphaned 280 blocks of obligations. Upstream uses |
| 1787 | // COINBASE_MATURITY here; this is a considered disagreement, not an | ||
| 1788 | // oversight -- see the merge commit message. | ||
| 625 | pp.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH; | 1789 | pp.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH; |
| 626 | 1790 | ||
| 627 | int skipped = 0; | 1791 | int skipped = 0; |
| @@ -643,12 +1807,22 @@ static void QueuePayouts(int blockHeight, | |||
| 643 | continue; | 1807 | continue; |
| 644 | } | 1808 | } |
| 645 | pp.amounts[kv.first] = amount; | 1809 | pp.amounts[kv.first] = amount; |
| 1810 | round.payouts[kv.first] = amount; | ||
| 646 | } | 1811 | } |
| 1812 | round.skippedDust = skipped; | ||
| 1813 | round.forfeitedBadAddress = burned; | ||
| 647 | 1814 | ||
| 648 | gPendingPayouts.push_back(pp); | 1815 | { |
| 649 | SavePendingPayouts(); | 1816 | VersionedBlob snap; |
| 1817 | { | ||
| 1818 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 1819 | gPendingPayouts.push_back(pp); | ||
| 1820 | snap = SerializePendingPayoutsLocked(); | ||
| 1821 | } | ||
| 1822 | PublishPendingPayouts(snap); | ||
| 1823 | } | ||
| 1824 | RecordPoolRound(round); | ||
| 650 | 1825 | ||
| 651 | int64 operatorCut = nSubsidy - minerReward; | ||
| 652 | LogPrint("payout", | 1826 | LogPrint("payout", |
| 653 | "[payout] block %d: subsidy %s BTF, miners get %s" | 1827 | "[payout] block %d: subsidy %s BTF, miners get %s" |
| 654 | " (fee %.2f%% = %s to operator)," | 1828 | " (fee %.2f%% = %s to operator)," |
| @@ -664,101 +1838,337 @@ static void QueuePayouts(int blockHeight, | |||
| 664 | : ""); | 1838 | : ""); |
| 665 | } | 1839 | } |
| 666 | 1840 | ||
| 667 | // Called from event loop: move matured entries into the in-flight list (fast). | 1841 | // Called from the event loop: explode matured ledger entries into per-recipient |
| 1842 | // in-flight records (fast; no wallet or disk work beyond one small write). | ||
| 1843 | // | ||
| 1844 | // The ledger entry SURVIVES here. That is upstream's model and it is the | ||
| 1845 | // crash-safe one: this fork used to erase the entry as soon as its payments | ||
| 1846 | // were handed off, so a death in that window left the debt described by | ||
| 1847 | // nothing at all -- a container restart on 2026-07-30 silently dropped | ||
| 1848 | // ~1,156 BTF exactly that way. An entry is removed only once every recipient | ||
| 1849 | // in it has a txid recorded (PayoutFullyPaid), which the payout thread does. | ||
| 668 | // | 1850 | // |
| 669 | // Ordering matters and is deliberate: the in-flight records are written and | 1851 | // gPaying is kept on top of that as the in-flight working set: it carries the |
| 670 | // fsync'd to their own file *before* the ledger entry is erased, so the debt | 1852 | // per-recipient retry/deferral state that a per-round `nextAttempt` cannot |
| 671 | // is always described by at least one of the two files. Dying in the window | 1853 | // express, and it is what the payout thread iterates. It is a cache of work |
| 672 | // where both describe it is handled at startup by ReconcilePayouts, which | 1854 | // derived from the ledger, never an independent record of debt -- if it were |
| 673 | // drops the ledger entry it can see is already in flight. | 1855 | // lost entirely the ledger would rebuild it on the next flush. |
| 674 | static void FlushMaturePayouts() | 1856 | static void FlushMaturePayouts() |
| 675 | { | 1857 | { |
| 676 | // Void obligations whose funding block lost a reorg. The coin that was | 1858 | // Which funding blocks are still on the main chain? Asked with no payout |
| 677 | // meant to pay them does not exist, so paying anyway spends real-chain | 1859 | // lock held: PayoutBlockStillOnChain takes cs_main, and cs_main is held by |
| 678 | // coin the pool owes to somebody else. | 1860 | // ProcessBlock and SendMoney for long stretches. Holding the ledger mutex |
| 679 | bool voided = false; | 1861 | // across it would let the event loop park the payout thread behind a chain |
| 680 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { | 1862 | // lock it has no interest in. (It is not a deadlock either way -- nothing |
| 681 | if (PayoutBlockStillOnChain(it->blockHash)) { ++it; continue; } | 1863 | // takes the ledger mutex while holding cs_main -- but it is exactly the |
| 682 | int64 total = 0; | 1864 | // kind of stall this merge is removing from the ledger path.) |
| 683 | for (auto& kv : it->amounts) total += kv.second; | 1865 | std::vector<uint256> orphaned; |
| 684 | LogPrint("payout", "[payout] block %d (%s) was orphaned -- voiding %zu" | 1866 | { |
| 685 | " payment(s) totalling %s\n", | 1867 | std::vector<uint256> hashes; |
| 686 | it->blockHeight, it->blockHash.ToString().substr(0, 14).c_str(), | ||
| 687 | it->amounts.size(), FormatMoney(total).c_str()); | ||
| 688 | it = gPendingPayouts.erase(it); | ||
| 689 | voided = true; | ||
| 690 | } | ||
| 691 | if (voided) SavePendingPayouts(); | ||
| 692 | |||
| 693 | std::vector<PendingPayout> ready; | ||
| 694 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { | ||
| 695 | if (nBestHeight < it->matureAtHeight) { ++it; continue; } | ||
| 696 | LogPrint("payout", "[payout] block height %d matured -- queuing %zu payments\n", | ||
| 697 | it->matureAtHeight, it->amounts.size()); | ||
| 698 | ready.push_back(*it); // copy first; erase after save | ||
| 699 | ++it; | ||
| 700 | } | ||
| 701 | if (!ready.empty()) { | ||
| 702 | // Record the individual payments first, and only then drop the ledger | ||
| 703 | // entries that produced them. | ||
| 704 | { | 1868 | { |
| 705 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | 1869 | std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex); |
| 706 | for (const auto& pp : ready) { | 1870 | for (const PendingPayout& pp : gPendingPayouts) |
| 707 | for (const auto& kv : pp.amounts) { | 1871 | hashes.push_back(pp.blockHash); |
| 708 | PayingRecord r; | ||
| 709 | r.blockHeight = pp.blockHeight; | ||
| 710 | r.blockHash = pp.blockHash; | ||
| 711 | r.matureAtHeight = pp.matureAtHeight; | ||
| 712 | r.address = kv.first; | ||
| 713 | r.amount = kv.second; | ||
| 714 | gPaying.push_back(r); | ||
| 715 | } | ||
| 716 | } | ||
| 717 | SavePayingLocked(); | ||
| 718 | } | 1872 | } |
| 1873 | for (const uint256& h : hashes) | ||
| 1874 | if (!PayoutBlockStillOnChain(h)) | ||
| 1875 | orphaned.push_back(h); | ||
| 1876 | } | ||
| 1877 | |||
| 1878 | std::vector<PendingPayout> ready; | ||
| 1879 | VersionedBlob ledgerSnap; | ||
| 1880 | { | ||
| 1881 | std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex); | ||
| 1882 | |||
| 1883 | // Void obligations whose funding block lost a reorg. The coin that was | ||
| 1884 | // meant to pay them does not exist, so paying anyway spends real-chain | ||
| 1885 | // coin the pool owes to somebody else. | ||
| 1886 | bool voided = false; | ||
| 719 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { | 1887 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { |
| 720 | if (nBestHeight >= it->matureAtHeight) | 1888 | if (std::find(orphaned.begin(), orphaned.end(), it->blockHash) == orphaned.end()) { |
| 721 | it = gPendingPayouts.erase(it); | ||
| 722 | else | ||
| 723 | ++it; | 1889 | ++it; |
| 1890 | continue; | ||
| 1891 | } | ||
| 1892 | int64 total = 0; | ||
| 1893 | for (auto& kv : it->amounts) total += kv.second; | ||
| 1894 | // Always: one line per voided entry, never a collapsed summary. | ||
| 1895 | // A reorg voids these in a burst from this one source line. | ||
| 1896 | LogPrintAlways("payout", "[payout] block %d (%s) was orphaned -- voiding %zu" | ||
| 1897 | " payment(s) totalling %s\n", | ||
| 1898 | it->blockHeight, it->blockHash.ToString().substr(0, 14).c_str(), | ||
| 1899 | it->amounts.size(), FormatMoney(total).c_str()); | ||
| 1900 | it = gPendingPayouts.erase(it); | ||
| 1901 | voided = true; | ||
| 1902 | } | ||
| 1903 | if (voided) ledgerSnap = SerializePendingPayoutsLocked(); | ||
| 1904 | |||
| 1905 | for (const PendingPayout& pp : gPendingPayouts) { | ||
| 1906 | if (nBestHeight < pp.matureAtHeight) | ||
| 1907 | continue; | ||
| 1908 | if (PayoutFullyPaid(pp)) | ||
| 1909 | continue; | ||
| 1910 | // An entry with neither a hash nor a height cannot be told apart | ||
| 1911 | // from another in the same state, so a txid recorded against it | ||
| 1912 | // could settle the wrong debt. Paying it is the unsafe move; | ||
| 1913 | // leaving it standing and saying so is not. | ||
| 1914 | if (!PayoutRoundIdentifiable(pp.blockHeight, pp.blockHash)) { | ||
| 1915 | LogPrint("payout", "[payout] WARNING: ledger entry maturing at" | ||
| 1916 | " height %d names neither a block hash nor a height --" | ||
| 1917 | " refusing to pay %zu recipient(s) that cannot be told" | ||
| 1918 | " apart from another round's\n", | ||
| 1919 | pp.matureAtHeight, pp.amounts.size()); | ||
| 1920 | continue; | ||
| 1921 | } | ||
| 1922 | ready.push_back(pp); | ||
| 724 | } | 1923 | } |
| 725 | SavePendingPayouts(); | ||
| 726 | } | 1924 | } |
| 727 | } | 1925 | // Outside the ledger mutex: publishing is I/O and the event loop is the |
| 1926 | // one thread that must never wait on the payout thread's disk. | ||
| 1927 | PublishPendingPayouts(ledgerSnap); | ||
| 728 | 1928 | ||
| 729 | // Identity of a payment, for finding its record again after SendMoney. | 1929 | if (ready.empty()) |
| 730 | // amounts is a map keyed by address, so (block, address) is already unique. | 1930 | return; |
| 731 | static bool SamePayment(const PayingRecord& a, const PayingRecord& b) | 1931 | |
| 732 | { | 1932 | VersionedBlob payingSnap; |
| 733 | return a.matureAtHeight == b.matureAtHeight | 1933 | { |
| 734 | && a.blockHash == b.blockHash | 1934 | std::lock_guard<std::mutex> queue(gPayoutExecMutex); |
| 735 | && a.address == b.address | 1935 | size_t added = 0; |
| 736 | && a.amount == b.amount; | 1936 | for (const PendingPayout& pp : ready) { |
| 1937 | const std::string id = PayoutIdForBlock(pp.blockHeight, pp.blockHash); | ||
| 1938 | if (gQueuedPayouts.count(id)) | ||
| 1939 | continue; // already exploded into gPaying and still in flight | ||
| 1940 | size_t n = 0; | ||
| 1941 | for (const auto& kv : pp.amounts) { | ||
| 1942 | // Upstream's per-recipient dedupe: a recipient already carrying | ||
| 1943 | // a txid is done, even though the entry as a whole is not. | ||
| 1944 | if (pp.paidTxids.count(kv.first)) | ||
| 1945 | continue; | ||
| 1946 | // The invariant, checked against the thing it is an invariant | ||
| 1947 | // of: at most one in-flight record per (round, address). The | ||
| 1948 | // round-level set above is an optimisation that can be wrong -- | ||
| 1949 | // it is memory only, it is rebuilt from gPaying at startup, and | ||
| 1950 | // a stale `ready` snapshot can race a round finishing -- | ||
| 1951 | // whereas gPaying is the list being protected. A second record | ||
| 1952 | // here is a second standing order to send the same money. | ||
| 1953 | if (PaymentInFlightLocked(pp.blockHeight, pp.blockHash, kv.first)) | ||
| 1954 | continue; | ||
| 1955 | PayingRecord r; | ||
| 1956 | r.id = ++gPayingNextId; | ||
| 1957 | r.blockHeight = pp.blockHeight; | ||
| 1958 | r.blockHash = pp.blockHash; | ||
| 1959 | r.matureAtHeight = pp.matureAtHeight; | ||
| 1960 | r.address = kv.first; | ||
| 1961 | r.amount = kv.second; | ||
| 1962 | gPaying.push_back(r); | ||
| 1963 | n++; | ||
| 1964 | } | ||
| 1965 | gQueuedPayouts.insert(id); | ||
| 1966 | added += n; | ||
| 1967 | LogPrint("payout", "[payout] block height %d matured -- queuing" | ||
| 1968 | " payout %s (%zu of %zu recipients still owed)\n", | ||
| 1969 | pp.matureAtHeight, id.c_str(), n, pp.amounts.size()); | ||
| 1970 | } | ||
| 1971 | if (added) payingSnap = SerializePayingLocked(); | ||
| 1972 | } | ||
| 1973 | PublishPaying(payingSnap); | ||
| 737 | } | 1974 | } |
| 738 | 1975 | ||
| 739 | // Settle a payment: it is either done or abandoned, either way it stops being | 1976 | // Settle a payment: it is either done or abandoned, either way it stops being |
| 740 | // owed. Removing the record and rewriting the file is what makes the payment | 1977 | // in flight. Note this only retires the working record -- whether the miner is |
| 741 | // not happen twice after a restart. | 1978 | // still OWED the money is the ledger's business, and the ledger is updated |
| 1979 | // separately by RecordPayoutTxid(). | ||
| 1980 | // | ||
| 1981 | // Erases by record id, not by comparing fields. Value-matching was only an | ||
| 1982 | // identity for as long as no two records could be equal, and the restart bug | ||
| 1983 | // above produced records that were equal in every field: the first match then | ||
| 1984 | // absorbed every erase and every state update, and the second was never | ||
| 1985 | // touched again. The id names the slot the caller took its copy from, so the | ||
| 1986 | // update lands where it belongs whatever else is in the list. | ||
| 742 | static void ForgetPayment(const PayingRecord& r) | 1987 | static void ForgetPayment(const PayingRecord& r) |
| 743 | { | 1988 | { |
| 744 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | 1989 | VersionedBlob snap; |
| 745 | for (auto it = gPaying.begin(); it != gPaying.end(); ++it) | 1990 | { |
| 746 | if (SamePayment(*it, r)) { gPaying.erase(it); break; } | 1991 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); |
| 747 | SavePayingLocked(); | 1992 | for (auto it = gPaying.begin(); it != gPaying.end(); ++it) |
| 1993 | if (it->id == r.id) { gPaying.erase(it); break; } | ||
| 1994 | |||
| 1995 | // When the last record for a round is gone the round is no longer in | ||
| 1996 | // flight, so a later flush may explode it again if the ledger still | ||
| 1997 | // shows recipients owed. That is the intended behaviour and it is now | ||
| 1998 | // the only way a stalled payment ever leaves this list: a payment that | ||
| 1999 | // keeps failing is retried forever rather than given up on, so its | ||
| 2000 | // record stays here, and the flush that would re-explode it is stopped | ||
| 2001 | // by PaymentInFlightLocked. | ||
| 2002 | bool stillInFlight = false; | ||
| 2003 | for (const auto& e : gPaying) | ||
| 2004 | if (SamePayoutRound(e.blockHeight, e.blockHash, | ||
| 2005 | r.blockHeight, r.blockHash)) { | ||
| 2006 | stillInFlight = true; | ||
| 2007 | break; | ||
| 2008 | } | ||
| 2009 | if (!stillInFlight) | ||
| 2010 | gQueuedPayouts.erase(PayoutIdForBlock(r.blockHeight, r.blockHash)); | ||
| 2011 | |||
| 2012 | snap = SerializePayingLocked(); | ||
| 2013 | } | ||
| 2014 | PublishPaying(snap); | ||
| 748 | } | 2015 | } |
| 749 | 2016 | ||
| 750 | // Persist an updated attempt count / backoff for a payment still owed. | 2017 | // Persist an updated attempt count / backoff for a payment still owed. |
| 751 | static void ReschedulePayment(const PayingRecord& r) | 2018 | static void ReschedulePayment(const PayingRecord& r) |
| 752 | { | 2019 | { |
| 753 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | 2020 | VersionedBlob snap; |
| 754 | for (auto& e : gPaying) | 2021 | { |
| 755 | if (SamePayment(e, r)) { | 2022 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); |
| 756 | e.retries = r.retries; | 2023 | for (auto& e : gPaying) |
| 757 | e.deferrals = r.deferrals; | 2024 | if (e.id == r.id) { |
| 758 | e.retryAfter = r.retryAfter; | 2025 | e.retries = r.retries; |
| 2026 | e.deferrals = r.deferrals; | ||
| 2027 | e.retryAfter = r.retryAfter; | ||
| 2028 | // Carry the cleared probe flag back, or every retry re-scans | ||
| 2029 | // the whole wallet -- the cost this merge is specifically | ||
| 2030 | // removing. | ||
| 2031 | e.fProbeWallet = r.fProbeWallet; | ||
| 2032 | break; | ||
| 2033 | } | ||
| 2034 | snap = SerializePayingLocked(); | ||
| 2035 | } | ||
| 2036 | PublishPaying(snap); | ||
| 2037 | } | ||
| 2038 | |||
| 2039 | // Record that a recipient has been paid, in the ledger, which is the only | ||
| 2040 | // durable statement that the debt is settled. | ||
| 2041 | // | ||
| 2042 | // The file write happens with gPayoutLedgerMutex RELEASED. Upstream writes the | ||
| 2043 | // file inside the per-recipient loop with the mutex held, so every recipient | ||
| 2044 | // costs the event loop a full serialise+fsync it has to wait for in | ||
| 2045 | // FlushMaturePayouts. Ordering against the event loop's own writes is the | ||
| 2046 | // snapshot version's job, not the mutex's -- see PublishVersioned. | ||
| 2047 | static void RecordPayoutTxid(const PayingRecord& r, const std::string& txid) | ||
| 2048 | { | ||
| 2049 | VersionedBlob blob; | ||
| 2050 | bool complete = false; | ||
| 2051 | { | ||
| 2052 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 2053 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ++it) { | ||
| 2054 | if (!SamePayoutRound(it->blockHeight, it->blockHash, | ||
| 2055 | r.blockHeight, r.blockHash)) | ||
| 2056 | continue; | ||
| 2057 | it->paidTxids[r.address] = txid; | ||
| 2058 | if (PayoutFullyPaid(*it)) { | ||
| 2059 | LogPrint("payout", "[payout] payout for block %d complete --" | ||
| 2060 | " removing from ledger\n", it->blockHeight); | ||
| 2061 | gPendingPayouts.erase(it); | ||
| 2062 | complete = true; | ||
| 2063 | } | ||
| 759 | break; | 2064 | break; |
| 760 | } | 2065 | } |
| 761 | SavePayingLocked(); | 2066 | blob = SerializePendingPayoutsLocked(); |
| 2067 | } | ||
| 2068 | PublishPendingPayouts(blob); | ||
| 2069 | // The round proof records a txid per address, so this is per recipient and | ||
| 2070 | // not only on completion. | ||
| 2071 | MarkPoolRoundPaid(PayoutIdForBlock(r.blockHeight, r.blockHash), r.address, txid); | ||
| 2072 | if (complete) | ||
| 2073 | LogPrint("payout", "[payout] block %d fully settled\n", r.blockHeight); | ||
| 2074 | } | ||
| 2075 | |||
| 2076 | // Write a recipient off. | ||
| 2077 | // | ||
| 2078 | // Only ever called for a state that is DECIDABLE -- currently one: the ledger | ||
| 2079 | // names an address that is not a valid address, so no transaction paying it | ||
| 2080 | // can be constructed by this or any other build. It is not called, and must | ||
| 2081 | // not be, because a payment has failed some number of times. "The wallet said | ||
| 2082 | // no N times" is not a decidable state, it is a description of how long we | ||
| 2083 | // have been asking; a debt written off on the strength of it is a debt the | ||
| 2084 | // operator keeps, and the arithmetic that decided WHY the wallet said no was | ||
| 2085 | // wrong for a year (see PayoutThreadFn). | ||
| 2086 | // | ||
| 2087 | // The amount is removed from the entry rather than marked paid, so the money | ||
| 2088 | // is not reported as sent. Once nothing is left owed the entry retires. The | ||
| 2089 | // round proof is told separately, because dropping the obligation from the | ||
| 2090 | // ledger while pool_rounds.json goes on publishing the address as owed with | ||
| 2091 | // "txid": null leaves the pool's two records of the same round permanently | ||
| 2092 | // disagreeing. | ||
| 2093 | static void ForfeitPayoutRecipient(const PayingRecord& r, const std::string& reason) | ||
| 2094 | { | ||
| 2095 | LogPrintAlways("payout", "[payout] writing off %s owed to %s for block %d: %s\n", | ||
| 2096 | FormatMoney(r.amount).c_str(), r.address.c_str(), r.blockHeight, | ||
| 2097 | reason.c_str()); | ||
| 2098 | VersionedBlob blob; | ||
| 2099 | { | ||
| 2100 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 2101 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ++it) { | ||
| 2102 | if (!SamePayoutRound(it->blockHeight, it->blockHash, | ||
| 2103 | r.blockHeight, r.blockHash)) | ||
| 2104 | continue; | ||
| 2105 | it->amounts.erase(r.address); | ||
| 2106 | // PayoutFullyPaid() returns true for an empty amounts map -- there | ||
| 2107 | // is no recipient in it without a txid -- so the | ||
| 2108 | // `amounts.empty()` disjunct that used to be here decided nothing. | ||
| 2109 | if (PayoutFullyPaid(*it)) | ||
| 2110 | gPendingPayouts.erase(it); | ||
| 2111 | break; | ||
| 2112 | } | ||
| 2113 | blob = SerializePendingPayoutsLocked(); | ||
| 2114 | } | ||
| 2115 | PublishPendingPayouts(blob); | ||
| 2116 | MarkPoolRoundForfeited(PayoutIdForBlock(r.blockHeight, r.blockHash), | ||
| 2117 | r.address, reason); | ||
| 2118 | } | ||
| 2119 | |||
| 2120 | // What a failed send means for the payment's retry budget. | ||
| 2121 | // | ||
| 2122 | // A deferral is "the wallet's condition is temporarily wrong": wait, and do | ||
| 2123 | // not hold it against this payment. A fault is "the wallet is funded and | ||
| 2124 | // unlocked and still would not do it", which is worth logging loudly and | ||
| 2125 | // retrying with backoff. | ||
| 2126 | // | ||
| 2127 | // This used to be decided by re-reading GetBalance() after the failure and | ||
| 2128 | // comparing it against `amount + nTransactionFee + CENT`. Every term of that | ||
| 2129 | // was wrong: | ||
| 2130 | // | ||
| 2131 | // * nTransactionFee is 0 in production. It is only ever assigned from the | ||
| 2132 | // GUI preferences dialog (gui.cpp), which a headless pool node does not | ||
| 2133 | // have, so the whole reserve was the CENT. | ||
| 2134 | // | ||
| 2135 | // * The fee the reserve is for is GetMinFee(fDiscount=true), a STEP | ||
| 2136 | // function of the serialised size: 0 below 10,000 bytes, and | ||
| 2137 | // (1 + bytes/1000) * CENT at or above it -- so 11 * CENT at the step. | ||
| 2138 | // There is no required-fee value between 1 and 11 CENT. A one-CENT | ||
| 2139 | // reserve is therefore never the right number: below the step nothing | ||
| 2140 | // needs reserving, and at the step it is short by a factor of eleven. | ||
| 2141 | // | ||
| 2142 | // * A pool wallet close to broke selects many small inputs BECAUSE it is | ||
| 2143 | // close to broke, and ~56 signed P2PKH inputs is 10 KB. The trigger and | ||
| 2144 | // the failure are positively correlated: the guess is wrong precisely | ||
| 2145 | // when the pool is nearly out of money. | ||
| 2146 | // | ||
| 2147 | // Concretely, with a balance of amount + 0.05: the pre-send gate passes, | ||
| 2148 | // CreateTransaction builds a >10 KB transaction, the fee becomes 0.11, | ||
| 2149 | // SelectCoins(amount + 0.11) fails, SendMoney returns false -- and the old | ||
| 2150 | // arithmetic said (amount + 0.05) < (amount + 0.01) is false, so "funded", | ||
| 2151 | // so a fault. The condition is stable, so it recurred every 60 s, and 1440 of | ||
| 2152 | // them erased the debt with the coin still in the operator's wallet. | ||
| 2153 | // | ||
| 2154 | // So stop guessing. The wallet is asked to say what it did, and this only | ||
| 2155 | // classifies the answer. WALLET_SEND_INSUFFICIENT_FUNDS is coin selection | ||
| 2156 | // failing to cover amount plus whatever fee that size of transaction | ||
| 2157 | // required, whatever that turned out to be -- no reserve to get wrong. | ||
| 2158 | enum PayoutFailureClass { | ||
| 2159 | PAYOUT_FAILURE_DEFER = 0, // conditions are wrong; wait for them to change | ||
| 2160 | PAYOUT_FAILURE_FAULT // funded and unlocked, and it still would not go | ||
| 2161 | }; | ||
| 2162 | |||
| 2163 | static PayoutFailureClass ClassifyPayoutSendFailure(WalletSendResult result, | ||
| 2164 | bool fWalletLocked) | ||
| 2165 | { | ||
| 2166 | // A locked wallet cannot sign. Nothing about this payment is wrong. | ||
| 2167 | if (fWalletLocked) | ||
| 2168 | return PAYOUT_FAILURE_DEFER; | ||
| 2169 | if (result == WALLET_SEND_INSUFFICIENT_FUNDS) | ||
| 2170 | return PAYOUT_FAILURE_DEFER; | ||
| 2171 | return PAYOUT_FAILURE_FAULT; | ||
| 762 | } | 2172 | } |
| 763 | 2173 | ||
| 764 | // Payout execution thread: works through gPaying, calls SendMoney. | 2174 | // Payout execution thread: works through gPaying, calls SendMoney. |
| @@ -780,11 +2190,14 @@ static void PayoutThreadFn(void*) | |||
| 780 | } | 2190 | } |
| 781 | 2191 | ||
| 782 | for (PayingRecord r : batch) { | 2192 | for (PayingRecord r : batch) { |
| 2193 | const std::string id = PayoutIdForBlock(r.blockHeight, r.blockHash); | ||
| 2194 | |||
| 783 | // Re-check here and not only at maturity: a payment can sit in this | 2195 | // Re-check here and not only at maturity: a payment can sit in this |
| 784 | // list for hours across retries, and the reorg that kills its | 2196 | // list for hours across retries, and the reorg that kills its |
| 785 | // funding block can land at any point in that window. | 2197 | // funding block can land at any point in that window. Upstream has |
| 2198 | // no equivalent -- it decides once, when the entry matures. | ||
| 786 | if (!PayoutBlockStillOnChain(r.blockHash)) { | 2199 | if (!PayoutBlockStillOnChain(r.blockHash)) { |
| 787 | LogPrint("payout", "[payout] block %d (%s) orphaned while queued" | 2200 | LogPrintAlways("payout", "[payout] block %d (%s) orphaned while queued" |
| 788 | " -- dropping %s to %s\n", | 2201 | " -- dropping %s to %s\n", |
| 789 | r.blockHeight, r.blockHash.ToString().substr(0, 14).c_str(), | 2202 | r.blockHeight, r.blockHash.ToString().substr(0, 14).c_str(), |
| 790 | FormatMoney(r.amount).c_str(), r.address.c_str()); | 2203 | FormatMoney(r.amount).c_str(), r.address.c_str()); |
| @@ -792,72 +2205,244 @@ static void PayoutThreadFn(void*) | |||
| 792 | continue; | 2205 | continue; |
| 793 | } | 2206 | } |
| 794 | 2207 | ||
| 2208 | // Upstream's per-recipient dedupe, against the ledger rather than | ||
| 2209 | // against the batch: if a txid is already recorded for this | ||
| 2210 | // address, the money went out and nothing more is owed. | ||
| 2211 | { | ||
| 2212 | bool alreadyPaid = false; | ||
| 2213 | { | ||
| 2214 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 2215 | for (const PendingPayout& pp : gPendingPayouts) | ||
| 2216 | if (SamePayoutRound(pp.blockHeight, pp.blockHash, | ||
| 2217 | r.blockHeight, r.blockHash) | ||
| 2218 | && pp.paidTxids.count(r.address)) { | ||
| 2219 | alreadyPaid = true; | ||
| 2220 | break; | ||
| 2221 | } | ||
| 2222 | } | ||
| 2223 | if (alreadyPaid) { | ||
| 2224 | ForgetPayment(r); | ||
| 2225 | continue; | ||
| 2226 | } | ||
| 2227 | } | ||
| 2228 | |||
| 795 | uint160 h160; | 2229 | uint160 h160; |
| 796 | if (!AddressToHash160(r.address, h160)) { | 2230 | if (!AddressToHash160(r.address, h160)) { |
| 797 | // Should not happen -- bad addresses are filtered in QueuePayouts | 2231 | // Should not happen -- bad addresses are filtered in |
| 798 | LogPrint("payout", "[payout] SKIPPED bad address '%s'\n", | 2232 | // QueuePayouts. If one is here anyway it is decidable and |
| 799 | r.address.c_str()); | 2233 | // permanent: there is no transaction that pays a string which |
| 2234 | // is not an address, so retrying is not waiting for anything. | ||
| 2235 | // | ||
| 2236 | // This used to only ForgetPayment(), which drops the in-flight | ||
| 2237 | // record and leaves the ledger entry claiming the debt and | ||
| 2238 | // pool_rounds.json advertising the recipient as unpaid | ||
| 2239 | // forever, with the next flush re-exploding it. Write it off | ||
| 2240 | // properly and say why. | ||
| 2241 | const std::string reason = | ||
| 2242 | strprintf("'%s' is not a valid address, so no transaction" | ||
| 2243 | " can pay it", r.address.c_str()); | ||
| 2244 | LogPrintAlways("payout", "[payout] SKIPPED bad address '%s' --" | ||
| 2245 | " %s owed for block %d cannot be paid\n", | ||
| 2246 | r.address.c_str(), FormatMoney(r.amount).c_str(), | ||
| 2247 | r.blockHeight); | ||
| 2248 | ForfeitPayoutRecipient(r, reason); | ||
| 800 | ForgetPayment(r); | 2249 | ForgetPayment(r); |
| 801 | continue; | 2250 | continue; |
| 802 | } | 2251 | } |
| 803 | CScript sc; | 2252 | CScript sc; |
| 804 | sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; | 2253 | sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; |
| 805 | 2254 | ||
| 2255 | // Did a previous process send this and die before recording it? | ||
| 2256 | // Only asked once per record -- see PayingRecord::fProbeWallet. | ||
| 2257 | if (r.fProbeWallet) { | ||
| 2258 | std::string recovered; | ||
| 2259 | const std::string claimKey = id + "|" + r.address; | ||
| 2260 | std::set<std::string> excluded; | ||
| 2261 | CollectForeignPayoutTxids(r.blockHeight, r.blockHash, r.address, | ||
| 2262 | claimKey, excluded); | ||
| 2263 | // A round with a published proof was created by a build whose | ||
| 2264 | // every payment names its round exactly, so only structured | ||
| 2265 | // evidence can settle it -- a comment match would be another | ||
| 2266 | // round's payment colliding on the height. | ||
| 2267 | const bool fAllowLegacy = !PoolRoundHasProof(id); | ||
| 2268 | r.fProbeWallet = false; | ||
| 2269 | PayoutProbeResult probe = FindRecordedPoolPayout( | ||
| 2270 | id, r.blockHeight, r.blockHash, r.address, r.amount, | ||
| 2271 | fAllowLegacy, excluded, recovered); | ||
| 2272 | if (probe != PAYOUT_PROBE_NONE) { | ||
| 2273 | if (probe == PAYOUT_PROBE_LEGACY) { | ||
| 2274 | // Loud on purpose: this is the upgrade path, where the | ||
| 2275 | // only evidence a payment went out is a comment written | ||
| 2276 | // by a binary that is no longer running. | ||
| 2277 | LogPrint("payout", "[payout] %s to %s was already paid by" | ||
| 2278 | " a pre-upgrade build (wallet tx %s carries the" | ||
| 2279 | " payout comment for height %d and an output of" | ||
| 2280 | " exactly this amount, and no structured payout" | ||
| 2281 | " metadata) -- not resending\n", | ||
| 2282 | FormatMoney(r.amount).c_str(), r.address.c_str(), | ||
| 2283 | recovered.c_str(), r.blockHeight); | ||
| 2284 | ClaimLegacyPayoutTxid(recovered, claimKey); | ||
| 2285 | } else { | ||
| 2286 | LogPrint("payout", "[payout] recovered recorded payout %s to %s" | ||
| 2287 | " from wallet tx %s -- not resending\n", | ||
| 2288 | FormatMoney(r.amount).c_str(), r.address.c_str(), | ||
| 2289 | recovered.c_str()); | ||
| 2290 | } | ||
| 2291 | RecordPayoutTxid(r, recovered); | ||
| 2292 | ForgetPayment(r); | ||
| 2293 | continue; | ||
| 2294 | } | ||
| 2295 | } | ||
| 2296 | |||
| 806 | // Waiting for coin to mature is not a failure, so it must not | 2297 | // Waiting for coin to mature is not a failure, so it must not |
| 807 | // consume the retry budget. Burning a retry every 60s means a | 2298 | // consume the retry budget. Burning a retry every 60s means a |
| 808 | // payment the pool fully intends to honour can hit GIVING UP | 2299 | // payment the pool fully intends to honour can hit GIVING UP |
| 809 | // 24h later purely because the subsidy funding it had not | 2300 | // 24h later purely because the subsidy funding it had not |
| 810 | // matured yet -- the miner is then silently never paid. | 2301 | // matured yet -- the miner is then silently never paid. |
| 2302 | // | ||
| 2303 | // The comparison includes the fee. It used to be against the | ||
| 2304 | // amount alone, so a balance anywhere between amount and | ||
| 2305 | // amount+fee sailed past this gate, failed in SendMoney, and was | ||
| 2306 | // counted as a send FAILURE rather than a wait -- 1440 of those is | ||
| 2307 | // 24 hours, and the recipient was erased with the money left in | ||
| 2308 | // the operator's wallet and nothing on disk saying it was owed. | ||
| 2309 | // | ||
| 2310 | // nTransactionFee is the part of the fee that can be known before | ||
| 2311 | // the transaction exists, and in production it is 0 -- it is only | ||
| 2312 | // ever assigned from the GUI preferences dialog, which a pool node | ||
| 2313 | // does not have -- so including it here changes nothing on this | ||
| 2314 | // pool and is kept only because a GUI operator can set it. The | ||
| 2315 | // rest is GetMinFee(), a function of the serialised size and | ||
| 2316 | // therefore of how many inputs SelectCoins ends up picking, which | ||
| 2317 | // cannot be predicted from here -- so this gate deliberately does | ||
| 2318 | // NOT try to reserve it. Refusing to attempt a payment the wallet | ||
| 2319 | // could actually have made would be its own way of not paying a | ||
| 2320 | // miner. That residual is handled after the fact, by asking the | ||
| 2321 | // wallet why it refused -- see ClassifyPayoutSendFailure. | ||
| 811 | int64 nSpendable = GetBalance(); | 2322 | int64 nSpendable = GetBalance(); |
| 812 | if (nSpendable < r.amount) { | 2323 | if (nSpendable < r.amount + nTransactionFee) { |
| 813 | r.deferrals++; | 2324 | r.deferrals++; |
| 814 | r.retryAfter = GetTime() + 60; | 2325 | r.retryAfter = GetTime() + 60; |
| 815 | // Hourly, not every minute: a dry pool would otherwise | 2326 | // Hourly, not every minute: a dry pool would otherwise |
| 816 | // write one line per payment per minute forever. | 2327 | // write one line per payment per minute forever. |
| 817 | if (r.deferrals == 1 || r.deferrals % 60 == 0) | 2328 | if (r.deferrals == 1 || r.deferrals % 60 == 0) |
| 818 | LogPrint("payout", "[payout] deferring %s to %s -- only %s" | 2329 | LogPrint("payout", "[payout] deferring %s to %s -- only %s" |
| 819 | " spendable (waiting %d min so far)\n", | 2330 | " spendable, need %s with fee (waiting %d min so far)\n", |
| 820 | FormatMoney(r.amount).c_str(), r.address.c_str(), | 2331 | FormatMoney(r.amount).c_str(), r.address.c_str(), |
| 821 | FormatMoney(nSpendable).c_str(), r.deferrals); | 2332 | FormatMoney(nSpendable).c_str(), |
| 2333 | FormatMoney(r.amount + nTransactionFee).c_str(), | ||
| 2334 | r.deferrals); | ||
| 822 | ReschedulePayment(r); | 2335 | ReschedulePayment(r); |
| 823 | continue; | 2336 | continue; |
| 824 | } | 2337 | } |
| 825 | 2338 | ||
| 826 | CWalletTx wtx; | 2339 | // Likewise a locked wallet: it cannot sign, so every attempt fails |
| 827 | wtx.mapValue["comment"] = PayoutComment(r.blockHeight); | 2340 | // for a reason that has nothing to do with this payment being |
| 2341 | // payable. Left to the retry budget, an operator who locks the | ||
| 2342 | // wallet overnight comes back to erased debts. | ||
| 2343 | if (IsWalletLocked()) { | ||
| 2344 | r.deferrals++; | ||
| 2345 | r.retryAfter = GetTime() + 60; | ||
| 2346 | if (r.deferrals == 1 || r.deferrals % 60 == 0) | ||
| 2347 | LogPrint("payout", "[payout] deferring %s to %s -- wallet is" | ||
| 2348 | " locked (waiting %d min so far)\n", | ||
| 2349 | FormatMoney(r.amount).c_str(), r.address.c_str(), | ||
| 2350 | r.deferrals); | ||
| 2351 | ReschedulePayment(r); | ||
| 2352 | continue; | ||
| 2353 | } | ||
| 828 | 2354 | ||
| 829 | if (SendMoney(sc, r.amount, wtx)) { | 2355 | CWalletTx wtx; |
| 2356 | wtx.mapValue["comment"] = PayoutComment(r.blockHeight, r.blockHash); | ||
| 2357 | // Upstream's structured metadata. This is what makes the wallet | ||
| 2358 | // probe above possible at all -- the old comment-only form could | ||
| 2359 | // not distinguish two rounds paying the same address the same | ||
| 2360 | // amount. Keyed by the hash-derived id, so a reorg cannot make one | ||
| 2361 | // round's transaction answer for another's. | ||
| 2362 | wtx.mapValue["pool_payout_id"] = id; | ||
| 2363 | wtx.mapValue["pool_payout_address"] = r.address; | ||
| 2364 | wtx.mapValue["pool_payout_amount"] = strprintf("%lld", r.amount); | ||
| 2365 | // The full hash, not only the id's sixteen-digit prefix: the | ||
| 2366 | // prefix of a proof-of-work hash is where the zeros live, and | ||
| 2367 | // the probe needs to tell twin rounds at one height apart. | ||
| 2368 | if (r.blockHash != 0) | ||
| 2369 | wtx.mapValue["pool_payout_block"] = r.blockHash.ToString(); | ||
| 2370 | |||
| 2371 | SendMoneyOutcome outcome; | ||
| 2372 | if (SendMoney(sc, r.amount, wtx, outcome)) { | ||
| 830 | // The wallet holds the transaction before this returns, so a | 2373 | // The wallet holds the transaction before this returns, so a |
| 831 | // crash here is recoverable: ReconcilePayouts finds it by | 2374 | // crash between here and the ledger write is recoverable: the |
| 832 | // comment+address+amount and does not resend. | 2375 | // probe above finds it by id+address+amount and does not |
| 2376 | // resend. Ledger first, in-flight record second -- the debt is | ||
| 2377 | // described by at least one of them at every instant. | ||
| 2378 | RecordPayoutTxid(r, wtx.GetHash().GetHex()); | ||
| 833 | ForgetPayment(r); | 2379 | ForgetPayment(r); |
| 834 | LogPrint("payout", "[payout] paid %s to %s (tx %s)\n", | 2380 | LogPrintAlways("payout", "[payout] paid %s to %s (tx %s)\n", |
| 835 | FormatMoney(r.amount).c_str(), r.address.c_str(), | 2381 | FormatMoney(r.amount).c_str(), r.address.c_str(), |
| 836 | wtx.GetHash().ToString().substr(0, 12).c_str()); | 2382 | wtx.GetHash().ToString().substr(0, 12).c_str()); |
| 837 | } else if (r.retries < 1440) { | ||
| 838 | // Transient failures (reorg, locked wallet) must not silently | ||
| 839 | // burn the payment -- retry with backoff. Insufficient funds | ||
| 840 | // never lands here; it defers above. | ||
| 841 | r.retries++; | ||
| 842 | r.retryAfter = GetTime() + 60; | ||
| 843 | ReschedulePayment(r); | ||
| 844 | LogPrint("payout", "[payout] FAILED to pay %s to %s" | ||
| 845 | " (%s spendable at the time) -- retry %d/1440 in 60s\n", | ||
| 846 | FormatMoney(r.amount).c_str(), r.address.c_str(), | ||
| 847 | FormatMoney(nSpendable).c_str(), r.retries); | ||
| 848 | } else { | 2383 | } else { |
| 849 | // Reaching here means 1440 attempts that were NOT | 2384 | // Why did it fail? The answer used to decide whether the |
| 850 | // insufficient-funds waits (those defer instead) against a | 2385 | // payment eventually got erased, and it was decided by |
| 851 | // block still on the main chain -- i.e. a real wallet fault, | 2386 | // guessing from a balance snapshot -- see |
| 852 | // not a maturity or reorg question. | 2387 | // ClassifyPayoutSendFailure for the arithmetic and what it |
| 853 | LogPrint("payout", "[payout] GIVING UP on %s to %s after %d" | 2388 | // cost. The wallet now says what it did, and nothing here |
| 854 | " retries -- block %d is still on-chain and %s was" | 2389 | // erases anything. |
| 855 | " spendable, so this is a wallet fault, not maturity" | 2390 | // |
| 856 | " -- payment forfeited\n", | 2391 | // The lock state is re-read rather than trusted from before |
| 857 | FormatMoney(r.amount).c_str(), r.address.c_str(), | 2392 | // the attempt: SendMoney takes cs_main and can block for a |
| 858 | r.retries, r.blockHeight, | 2393 | // long time, so it may have moved underneath us. The funding |
| 859 | FormatMoney(nSpendable).c_str()); | 2394 | // answer does not need re-reading -- it is a statement about |
| 860 | ForgetPayment(r); | 2395 | // the attempt that was actually made. |
| 2396 | bool fLocked = IsWalletLocked(); | ||
| 2397 | int64 nAfter = GetBalance(); | ||
| 2398 | |||
| 2399 | if (ClassifyPayoutSendFailure(outcome.result, fLocked) | ||
| 2400 | == PAYOUT_FAILURE_DEFER) { | ||
| 2401 | // Conditions are temporarily wrong. Wait. | ||
| 2402 | r.deferrals++; | ||
| 2403 | r.retryAfter = GetTime() + 60; | ||
| 2404 | if (r.deferrals == 1 || r.deferrals % 60 == 0) | ||
| 2405 | LogPrint("payout", "[payout] deferring %s to %s -- send" | ||
| 2406 | " failed with %s (%s spendable; the wallet" | ||
| 2407 | " needed a fee of %s for a transaction this" | ||
| 2408 | " size); waiting %d min so far\n", | ||
| 2409 | FormatMoney(r.amount).c_str(), r.address.c_str(), | ||
| 2410 | fLocked ? "the wallet locked" : "too little coin", | ||
| 2411 | FormatMoney(nAfter).c_str(), | ||
| 2412 | FormatMoney(outcome.nFeeRequired).c_str(), | ||
| 2413 | r.deferrals); | ||
| 2414 | ReschedulePayment(r); | ||
| 2415 | } else { | ||
| 2416 | // Funded, unlocked, block on the main chain, and the | ||
| 2417 | // wallet still would not build, sign or accept the | ||
| 2418 | // transaction. That is worth shouting about; it is not | ||
| 2419 | // grounds to keep the money. | ||
| 2420 | // | ||
| 2421 | // There is deliberately no give-up branch. It used to | ||
| 2422 | // forfeit the debt after 1440 of these, which is not a | ||
| 2423 | // decidable state -- it is a count of how long we have | ||
| 2424 | // been asking, and it was reached by misclassifying a | ||
| 2425 | // funding shortfall as a fault. A debt that is never | ||
| 2426 | // written off cannot be written off wrongly. The payment | ||
| 2427 | // stays in flight, keeps its ledger entry, keeps being | ||
| 2428 | // reported in pool_status.json as owed, and keeps | ||
| 2429 | // retrying once a minute until the operator or the wallet | ||
| 2430 | // makes it possible. | ||
| 2431 | r.retries++; | ||
| 2432 | r.retryAfter = GetTime() + 60; | ||
| 2433 | ReschedulePayment(r); | ||
| 2434 | // Every minute at first, then hourly: a fault that is not | ||
| 2435 | // going away must not turn the log into one line a minute | ||
| 2436 | // forever, and must not go quiet either. | ||
| 2437 | if (r.retries <= 5 || r.retries % 60 == 0) | ||
| 2438 | LogPrintAlways("payout", "[payout] FAILED to pay %s to %s" | ||
| 2439 | " (%s spendable, wallet unlocked, block %d on" | ||
| 2440 | " the main chain) -- attempt %d, retrying in" | ||
| 2441 | " 60s; this debt stands until it is paid\n", | ||
| 2442 | FormatMoney(r.amount).c_str(), r.address.c_str(), | ||
| 2443 | FormatMoney(nAfter).c_str(), r.blockHeight, | ||
| 2444 | r.retries); | ||
| 2445 | } | ||
| 861 | } | 2446 | } |
| 862 | } | 2447 | } |
| 863 | 2448 | ||
| @@ -881,6 +2466,7 @@ static StatsSnapshot gStats; | |||
| 881 | static std::vector<PoolWorkerStatView> gWorkerStats; | 2466 | static std::vector<PoolWorkerStatView> gWorkerStats; |
| 882 | static std::vector<PendingPayoutView> gPayoutViews; | 2467 | static std::vector<PendingPayoutView> gPayoutViews; |
| 883 | static std::mutex gStatsMutex; | 2468 | static std::mutex gStatsMutex; |
| 2469 | static int64 gLastPoolStatusWrite = 0; | ||
| 884 | 2470 | ||
| 885 | void GetPoolOperatorStats(int& authorizedMiners, int& blocksFound, uint64& roundShares, double& totalHashRate) | 2471 | void GetPoolOperatorStats(int& authorizedMiners, int& blocksFound, uint64& roundShares, double& totalHashRate) |
| 886 | { | 2472 | { |
| @@ -903,6 +2489,135 @@ void GetPendingPayouts(std::vector<PendingPayoutView>& out) | |||
| 903 | out = gPayoutViews; | 2489 | out = gPayoutViews; |
| 904 | } | 2490 | } |
| 905 | 2491 | ||
| 2492 | static std::string PoolStatusFilePath() | ||
| 2493 | { | ||
| 2494 | if (!strPoolStatusFile.empty()) | ||
| 2495 | return strPoolStatusFile; | ||
| 2496 | return GetAppDir() + "/pool_status.json"; | ||
| 2497 | } | ||
| 2498 | |||
| 2499 | // Serialise the operator status file. Caller must hold gStatsMutex (this reads | ||
| 2500 | // gStats / gWorkerStats / gPayoutViews); gPoolRoundsMutex is taken and | ||
| 2501 | // released inside. | ||
| 2502 | // | ||
| 2503 | // Split from the write for the same reason as the three ledgers, and for a | ||
| 2504 | // worse instance of the same problem. This used to be called from RefreshStats | ||
| 2505 | // with gPayoutLedgerMutex held at function scope, and once the ledger writes | ||
| 2506 | // gained fsync() that call became two fsyncs -- one on the file, one on its | ||
| 2507 | // directory -- performed by the event loop, the thread that owns every miner's | ||
| 2508 | // socket, with the mutex the payout thread needs in its hand. Measured on the | ||
| 2509 | // build host (ext4 on LVM on SSD): 3.78 ms per publish, every 10 seconds, and | ||
| 2510 | // unbounded on a shared virtual disk under load. | ||
| 2511 | // | ||
| 2512 | // Nothing here needs the ledger mutex at all: the pending-payout rows were | ||
| 2513 | // already copied into gPayoutViews under it, further up RefreshStats. | ||
| 2514 | static std::string SerializePoolStatusLocked() | ||
| 2515 | { | ||
| 2516 | nlohmann::json j; | ||
| 2517 | j["schema"] = "bitflash-pool-status-1"; | ||
| 2518 | j["network"] = "bitflash"; | ||
| 2519 | j["updatedAt"] = GetTime(); | ||
| 2520 | |||
| 2521 | j["operator"] = { | ||
| 2522 | {"name", strPoolName}, | ||
| 2523 | {"btfAddress", BtfLocalAddress()}, | ||
| 2524 | {"feePercent", dPoolFeePercent}, | ||
| 2525 | {"dashboardUrl", strPoolDashboardUrl}, | ||
| 2526 | {"roundsFile", "pool_rounds.json"} | ||
| 2527 | }; | ||
| 2528 | |||
| 2529 | j["node"] = { | ||
| 2530 | {"height", nBestHeight}, | ||
| 2531 | {"poolRunning", (bool)gPoolRunning}, | ||
| 2532 | {"mode", nMineMode} | ||
| 2533 | }; | ||
| 2534 | |||
| 2535 | j["pool"] = { | ||
| 2536 | {"authorizedMiners", gStats.authorizedMiners}, | ||
| 2537 | {"blocksFoundSession", gStats.blocksFound}, | ||
| 2538 | {"roundShares", gStats.roundShares}, | ||
| 2539 | {"hashRate", gStats.totalHashRate} | ||
| 2540 | }; | ||
| 2541 | |||
| 2542 | j["workers"] = nlohmann::json::array(); | ||
| 2543 | for (const PoolWorkerStatView& row : gWorkerStats) { | ||
| 2544 | j["workers"].push_back({ | ||
| 2545 | {"address", row.address}, | ||
| 2546 | {"worker", row.worker}, | ||
| 2547 | {"totalShares", row.totalShares}, | ||
| 2548 | {"roundShares", row.roundShares}, | ||
| 2549 | {"lastSeen", row.lastSeen}, | ||
| 2550 | {"hashRate", row.hashRate} | ||
| 2551 | }); | ||
| 2552 | } | ||
| 2553 | |||
| 2554 | j["pendingPayouts"] = nlohmann::json::array(); | ||
| 2555 | for (const PendingPayoutView& row : gPayoutViews) { | ||
| 2556 | j["pendingPayouts"].push_back({ | ||
| 2557 | {"matureAtHeight", row.matureAtHeight}, | ||
| 2558 | {"recipients", row.recipients}, | ||
| 2559 | {"totalAmountSatoshis", row.totalAmount}, | ||
| 2560 | {"totalAmountBtf", FormatMoney(row.totalAmount)} | ||
| 2561 | }); | ||
| 2562 | } | ||
| 2563 | |||
| 2564 | j["recentRounds"] = nlohmann::json::array(); | ||
| 2565 | { | ||
| 2566 | std::lock_guard<std::mutex> lk(gPoolRoundsMutex); | ||
| 2567 | size_t start = gPoolRounds.size() > 10 ? gPoolRounds.size() - 10 : 0; | ||
| 2568 | for (size_t i = start; i < gPoolRounds.size(); i++) { | ||
| 2569 | const PoolRoundProof& r = gPoolRounds[i]; | ||
| 2570 | // A forfeited recipient is not "unpaid and pending" -- the ledger | ||
| 2571 | // has already dropped the obligation. Counting it as unpaid is | ||
| 2572 | // what made the round proof and the ledger disagree for good. | ||
| 2573 | int unpaid = 0; | ||
| 2574 | int forfeited = 0; | ||
| 2575 | for (const auto& kv : r.payouts) { | ||
| 2576 | if (r.paidTxids.count(kv.first)) | ||
| 2577 | continue; | ||
| 2578 | if (r.forfeited.count(kv.first)) | ||
| 2579 | forfeited++; | ||
| 2580 | else | ||
| 2581 | unpaid++; | ||
| 2582 | } | ||
| 2583 | j["recentRounds"].push_back({ | ||
| 2584 | {"id", r.id}, | ||
| 2585 | {"blockHeight", r.blockHeight}, | ||
| 2586 | {"blockHash", r.blockHash}, | ||
| 2587 | {"coinbaseTxid", r.coinbaseTxid}, | ||
| 2588 | {"totalShares", r.totalShares}, | ||
| 2589 | {"payouts", (int)r.payouts.size()}, | ||
| 2590 | {"unpaidPayouts", unpaid}, | ||
| 2591 | {"forfeitedPayouts", forfeited}, | ||
| 2592 | {"coinbaseValueBtf", FormatMoney(r.coinbaseValue)}, | ||
| 2593 | {"minerRewardBtf", FormatMoney(r.minerReward)} | ||
| 2594 | }); | ||
| 2595 | } | ||
| 2596 | } | ||
| 2597 | |||
| 2598 | return j.dump(2); | ||
| 2599 | } | ||
| 2600 | |||
| 2601 | // Publish the operator status file. Call with every payout mutex released. | ||
| 2602 | // | ||
| 2603 | // Still no version guard and no write mutex, deliberately, and the split does | ||
| 2604 | // not change the argument: RefreshStats remains the only caller of either | ||
| 2605 | // half, in production it runs only on the event loop, and it now calls this | ||
| 2606 | // after leaving the scope that serialised -- so the serialise and the write | ||
| 2607 | // stay strictly ordered on one thread and pool_status.json still has a single | ||
| 2608 | // writer by construction. It shares WriteFileAtomic so the durability rules | ||
| 2609 | // stay in one place. | ||
| 2610 | // | ||
| 2611 | // "In production" is doing real work in that sentence. The payout-ledger | ||
| 2612 | // self-test also calls RefreshStats, from one thread at a time and with the | ||
| 2613 | // pool not running, which is why it is still a single writer there too. A | ||
| 2614 | // second concurrent caller anywhere would invalidate this paragraph and want | ||
| 2615 | // the same versioned publish the ledgers use. | ||
| 2616 | static void PublishPoolStatus(const std::string& blob) | ||
| 2617 | { | ||
| 2618 | WriteFileAtomic(PoolStatusFilePath(), blob, "pool status"); | ||
| 2619 | } | ||
| 2620 | |||
| 906 | // --------------------------------------------------------------------------- | 2621 | // --------------------------------------------------------------------------- |
| 907 | // Miner state -- event loop thread only | 2622 | // Miner state -- event loop thread only |
| 908 | // --------------------------------------------------------------------------- | 2623 | // --------------------------------------------------------------------------- |
| @@ -928,9 +2643,35 @@ static const double VARDIFF_MAX = 1e12; | |||
| 928 | static const int64 VARDIFF_STALL_SECS = 45; // 3x share target -- reacts faster without false-triggering on normal variance | 2643 | static const int64 VARDIFF_STALL_SECS = 45; // 3x share target -- reacts faster without false-triggering on normal variance |
| 929 | static const double VARDIFF_STALL_CUT = 4.0; // divide difficulty by this much | 2644 | static const double VARDIFF_STALL_CUT = 4.0; // divide difficulty by this much |
| 930 | 2645 | ||
| 2646 | // #7: m->hashRate is an EWMA sampled only when a share arrives, so a miner | ||
| 2647 | // that stops hashing keeps its last estimate forever and RefreshStats sums | ||
| 2648 | // it into the announced pool total for as long as the connection sits open | ||
| 2649 | // -- the published pool hashrate ran several times over ground truth | ||
| 2650 | // because of exactly this. MinerReportedHashRate() below applies the | ||
| 2651 | // policy: a miner gets a grace of HASHRATE_IDLE_MISSES expected share | ||
| 2652 | // intervals (expected interval = expected hashes per share at its | ||
| 2653 | // difficulty, divided by the estimate itself) with no share, after which | ||
| 2654 | // the figure we report falls off with a half-life of one more such | ||
| 2655 | // interval. The interval is clamped to [HASHRATE_MIN_INTERVAL, | ||
| 2656 | // HASHRATE_MAX_INTERVAL] so a fast miner at the vardiff floor (expected | ||
| 2657 | // interval of milliseconds) is not punished for ordinary share-delivery | ||
| 2658 | // jitter, and a very slow miner's grace does not stretch past the point | ||
| 2659 | // where its estimate means anything. At the vardiff target of one share | ||
| 2660 | // per 15s that is a minute of grace, half the rate reported 75s in, and | ||
| 2661 | // nothing reported after about seventeen minutes of silence. | ||
| 2662 | static const double HASHRATE_IDLE_MISSES = 4.0; // grace, in expected share intervals | ||
| 2663 | static const double HASHRATE_MIN_INTERVAL = 1.0; // seconds; floor for the expected interval | ||
| 2664 | static const double HASHRATE_MAX_INTERVAL = 3600.0; // seconds; ceiling for the expected interval | ||
| 2665 | // Half-lives past the grace beyond which we report a flat zero. 2^-64 of the | ||
| 2666 | // original estimate is not a number anyone needs, and stopping here keeps | ||
| 2667 | // pow() from walking down into the denormals for a miner that has been quiet | ||
| 2668 | // for hours. | ||
| 2669 | static const double HASHRATE_DECAY_CUTOFF = 64.0; | ||
| 2670 | |||
| 931 | struct Miner { | 2671 | struct Miner { |
| 932 | btf_socket_t fd; | 2672 | btf_socket_t fd; |
| 933 | bool authorised = false; | 2673 | bool authorised = false; |
| 2674 | bool subscribed = false; // completed mining.subscribe (#19 census) | ||
| 934 | std::string address; | 2675 | std::string address; |
| 935 | std::string worker; | 2676 | std::string worker; |
| 936 | std::string readBuf; | 2677 | std::string readBuf; |
| @@ -954,11 +2695,80 @@ struct Miner { | |||
| 954 | int64 lastShareTime = 0; // GetTime() of the previous accepted share | 2695 | int64 lastShareTime = 0; // GetTime() of the previous accepted share |
| 955 | double hashRate = 0.0; // smoothed H/s estimate | 2696 | double hashRate = 0.0; // smoothed H/s estimate |
| 956 | 2697 | ||
| 2698 | // Replay guard for mining.submit (#8): nonces already credited on this | ||
| 2699 | // connection for the job it is currently working. Keyed to the job id in | ||
| 2700 | // seenNoncesJob and cleared by SendJob whenever that id changes, so a | ||
| 2701 | // resubmission under a NEW job is never turned away, while a replayed | ||
| 2702 | // (job_id, nonce) pair under the same job is refused before it can be | ||
| 2703 | // credited twice into roundShareCount -- the figure the payout split | ||
| 2704 | // divides by. | ||
| 2705 | std::set<uint32_t> seenNonces; | ||
| 2706 | std::string seenNoncesJob; | ||
| 2707 | |||
| 957 | explicit Miner(btf_socket_t f) | 2708 | explicit Miner(btf_socket_t f) |
| 958 | : fd(f), connectTime(GetTime()), | 2709 | : fd(f), connectTime(GetTime()), |
| 959 | extranonce2(gNextExtranonce2.fetch_add(1)) {} | 2710 | extranonce2(gNextExtranonce2.fetch_add(1)) {} |
| 960 | }; | 2711 | }; |
| 961 | 2712 | ||
| 2713 | // What we publish as a miner's H/s: the last EWMA sample, faded out if shares | ||
| 2714 | // have stopped arriving (#7, policy at HASHRATE_IDLE_MISSES). | ||
| 2715 | // | ||
| 2716 | // This reads m->hashRate and never writes it. That is the whole design. The | ||
| 2717 | // first attempt decayed the stored estimate in place from the event loop's | ||
| 2718 | // sweep, which put a running accumulator behind a figure that is only ever | ||
| 2719 | // displayed, and the accumulator misbehaved in two ways: | ||
| 2720 | // | ||
| 2721 | // - `expected` was recomputed from the already-decayed estimate on every | ||
| 2722 | // pass, so each decay raised `expected`, which raised `grace`, which is | ||
| 2723 | // the very thing gating the next decay. `grace` simply chased `silent` | ||
| 2724 | // upward and the reported rate settled into a 1/t crawl instead of the | ||
| 2725 | // half-life curve the comment above promises -- 291x too high at ten | ||
| 2726 | // half-lives past the grace. Issue #7 is about the pool over-reporting; | ||
| 2727 | // that under-corrected it for hours. | ||
| 2728 | // - Once `expected` reached HASHRATE_MAX_INTERVAL it could not rise any | ||
| 2729 | // further, the feedback that had been gating each pass was gone, and the | ||
| 2730 | // decay began compounding once a second against the total elapsed | ||
| 2731 | // silence -- taking a miner from 0.4% of its baseline to 1e-11 of it in | ||
| 2732 | // the next hundred seconds, four hours in, for no reason a reader of | ||
| 2733 | // either the code or the announced number could see. | ||
| 2734 | // | ||
| 2735 | // Computing the value instead of accumulating it removes the failure mode | ||
| 2736 | // rather than tuning it: with no stored decay state there is nothing for a | ||
| 2737 | // tick rate, a missed pass or a duplicated pass to perturb, and the answer | ||
| 2738 | // depends only on how long the miner has actually been quiet. It also keeps | ||
| 2739 | // the fade out of the estimator -- the EWMA on the share path still blends | ||
| 2740 | // against the value it last computed, not against a faded copy of it, so a | ||
| 2741 | // miner that resumes recovers on exactly the curve it would have without any | ||
| 2742 | // of this. | ||
| 2743 | // | ||
| 2744 | // `now` is a parameter rather than a GetTime() call so that one refresh fades | ||
| 2745 | // every miner to the same instant, and so the policy is directly testable. | ||
| 2746 | static double MinerReportedHashRate(const Miner* m, int64 now) | ||
| 2747 | { | ||
| 2748 | // Nothing sampled yet: a fresh connection, or one that has found a single | ||
| 2749 | // share and so has no interval to have been timed over. Nothing to fade, | ||
| 2750 | // and nothing to divide by below. | ||
| 2751 | if (m->hashRate <= 0.0 || m->lastShareTime <= 0) return m->hashRate; | ||
| 2752 | |||
| 2753 | double expected = ExpectedHashesForDifficulty(m->difficulty) / m->hashRate; | ||
| 2754 | // Written so that a NaN would land on the floor rather than sail through: | ||
| 2755 | // the clamp is what guarantees `expected` is finite and strictly positive | ||
| 2756 | // below, whatever the division produced. | ||
| 2757 | if (!(expected > HASHRATE_MIN_INTERVAL)) expected = HASHRATE_MIN_INTERVAL; | ||
| 2758 | if (expected > HASHRATE_MAX_INTERVAL) expected = HASHRATE_MAX_INTERVAL; | ||
| 2759 | |||
| 2760 | double silent = (double)(now - m->lastShareTime); | ||
| 2761 | double grace = HASHRATE_IDLE_MISSES * expected; | ||
| 2762 | // grace is at least HASHRATE_IDLE_MISSES seconds, so a clock that has | ||
| 2763 | // stepped backwards gives a negative `silent`, takes this branch, and | ||
| 2764 | // reports the sample unchanged. Nothing here can scale an estimate up. | ||
| 2765 | if (!(silent > grace)) return m->hashRate; | ||
| 2766 | |||
| 2767 | double halflives = (silent - grace) / expected; | ||
| 2768 | if (halflives >= HASHRATE_DECAY_CUTOFF) return 0.0; | ||
| 2769 | return m->hashRate * pow(0.5, halflives); | ||
| 2770 | } | ||
| 2771 | |||
| 962 | static std::vector<Miner*> gMiners; | 2772 | static std::vector<Miner*> gMiners; |
| 963 | 2773 | ||
| 964 | struct FoundBlock { int height; int64 ts; std::string addr; }; | 2774 | struct FoundBlock { int height; int64 ts; std::string addr; }; |
| @@ -1009,6 +2819,18 @@ static void LocalAcceptFn(void*) | |||
| 1009 | break; | 2819 | break; |
| 1010 | continue; | 2820 | continue; |
| 1011 | } | 2821 | } |
| 2822 | // #6: let the OS probe a peer that has gone silent, so a half-open | ||
| 2823 | // session eventually errors out of recv and lands in the event | ||
| 2824 | // loop's existing reap path instead of sitting in gMiners forever. | ||
| 2825 | // This is a backstop, not the mechanism: the default cadence before | ||
| 2826 | // the first probe is hours on most stacks, so the idle timeout in | ||
| 2827 | // PoolEventLoop fires long before keepalive does. Miners arriving | ||
| 2828 | // via a relay ride a loopback socket pair whose remote transport is | ||
| 2829 | // the rendezvous socket owned by the tunnel pump threads; keepalive | ||
| 2830 | // cannot reach across that, which is part of why the idle timeout | ||
| 2831 | // exists. | ||
| 2832 | int keepalive = 1; | ||
| 2833 | setsockopt(c, SOL_SOCKET, SO_KEEPALIVE, (char*)&keepalive, sizeof(keepalive)); | ||
| 1012 | LogPrint("pool", "[pool] miner arrived via loopback -- pushing to event loop\n"); | 2834 | LogPrint("pool", "[pool] miner arrived via loopback -- pushing to event loop\n"); |
| 1013 | PushReady((btf_socket_t)c); | 2835 | PushReady((btf_socket_t)c); |
| 1014 | } | 2836 | } |
| @@ -1039,6 +2861,30 @@ static bool SendLine(btf_socket_t fd, const json& j) | |||
| 1039 | static const size_t MAX_STRATUM_LINE = 64 * 1024; | 2861 | static const size_t MAX_STRATUM_LINE = 64 * 1024; |
| 1040 | static const size_t MAX_POOL_MINERS = 1000; | 2862 | static const size_t MAX_POOL_MINERS = 1000; |
| 1041 | 2863 | ||
| 2864 | // #19 instrumentation: the "broadcast to N miner(s)" line counts connected | ||
| 2865 | // sockets, which says nothing about how many are actually mining. These split | ||
| 2866 | // the connection census into connected / subscribed / authorized / recently | ||
| 2867 | // active, so an operator can tell a real workforce from a pile of half-open | ||
| 2868 | // sessions at a glance. | ||
| 2869 | static const int64 POOL_CENSUS_SECS = 60; // how often to log the breakdown | ||
| 2870 | static const int64 POOL_ACTIVE_WINDOW = 300; // a miner is "active" if it | ||
| 2871 | // authorized and was seen within | ||
| 2872 | // this window (one payout-rotate | ||
| 2873 | // cycle) | ||
| 2874 | static const int64 POOL_UNAUTH_GRACE_SECS = 60; // a connection still not | ||
| 2875 | // authorized after this long is | ||
| 2876 | // idle, not merely starting up | ||
| 2877 | // #6: a Miner used to leave gMiners only when RecvLine reported a clean | ||
| 2878 | // disconnect. A half-open session (miner host died, NAT state expired, | ||
| 2879 | // cable pulled) sends no FIN and reports no error, so it sat in the table | ||
| 2880 | // forever -- authorize outpaced disconnect 2:1 on the live pool, and the | ||
| 2881 | // table grew toward MAX_POOL_MINERS, past which real miners are refused. | ||
| 2882 | // lastSeen is stamped on every complete line received; a miner that has | ||
| 2883 | // said nothing for this long is gone in every way that matters. Ten | ||
| 2884 | // minutes is far beyond the vardiff target (a share every 15s) and the | ||
| 2885 | // stall watchdog (45s), so no miner that is actually hashing comes close. | ||
| 2886 | static const int64 POOL_IDLE_TIMEOUT_SECS = 600; | ||
| 2887 | |||
| 1042 | // Non-blocking recv: pull whatever bytes are available now into buf, then | 2888 | // Non-blocking recv: pull whatever bytes are available now into buf, then |
| 1043 | // return one complete line if one is ready. Three outcomes: | 2889 | // return one complete line if one is ready. Three outcomes: |
| 1044 | // true -- line is filled; buf may still contain more lines | 2890 | // true -- line is filled; buf may still contain more lines |
| @@ -1122,6 +2968,17 @@ static json MakeNotifyParams(const StratumJob& job, const CBlock& b, | |||
| 1122 | static void SendJob(Miner* m, bool clean) | 2968 | static void SendJob(Miner* m, bool clean) |
| 1123 | { | 2969 | { |
| 1124 | if (!gHaveJob) return; | 2970 | if (!gHaveJob) return; |
| 2971 | // Every path that hands a miner a new job funnels through here, so this | ||
| 2972 | // is the one place the replay guard needs to follow the job id: nonces | ||
| 2973 | // credited under the previous job mean nothing under the new one. Keying | ||
| 2974 | // the clear to the id (rather than clearing unconditionally) keeps a | ||
| 2975 | // same-job re-notify -- the vardiff retarget and stall watchdog both | ||
| 2976 | // re-send the CURRENT job -- from reopening the duplicate window it | ||
| 2977 | // exists to close. | ||
| 2978 | if (m->seenNoncesJob != gCurrentJob.jobId) { | ||
| 2979 | m->seenNonces.clear(); | ||
| 2980 | m->seenNoncesJob = gCurrentJob.jobId; | ||
| 2981 | } | ||
| 1125 | CBlock mb = BuildMinerBlock(gCurrentJob, m->extranonce2); | 2982 | CBlock mb = BuildMinerBlock(gCurrentJob, m->extranonce2); |
| 1126 | // Use this miner's own shareTarget (set by vardiff) in the notify params. | 2983 | // Use this miner's own shareTarget (set by vardiff) in the notify params. |
| 1127 | // The participant miner reads p[3] as its share target, and HandleLine | 2984 | // The participant miner reads p[3] as its share target, and HandleLine |
| @@ -1147,6 +3004,65 @@ static void BroadcastJob(bool clean) | |||
| 1147 | } | 3004 | } |
| 1148 | 3005 | ||
| 1149 | // --------------------------------------------------------------------------- | 3006 | // --------------------------------------------------------------------------- |
| 3007 | // #19: connection census. "broadcast to N miner(s)" only counts authorized | ||
| 3008 | // sockets that got a job; it says nothing about the rest of the connection | ||
| 3009 | // table. This logs the full breakdown so an operator can tell a real workforce | ||
| 3010 | // apart from half-open / never-authorized / idle sessions, and flags when the | ||
| 3011 | // single select() loop is running out of room (issue #26). | ||
| 3012 | // | ||
| 3013 | // "Out of room" is not the same question on both platforms, and FD_SETSIZE | ||
| 3014 | // answers a different one on each. On POSIX it bounds the *value* of any | ||
| 3015 | // descriptor handed to FD_SET, so what matters is the highest fd we hold. On | ||
| 3016 | // Windows an fd_set is a counted array of SOCKET handles whose numeric values | ||
| 3017 | // are arbitrary, so what matters is how many we hold, not how large they are. | ||
| 3018 | // FD_SETSIZE comes from <sys/select.h> or winsock2.h; this file must not define | ||
| 3019 | // it, because redefining it after those headers changes nothing. | ||
| 3020 | // --------------------------------------------------------------------------- | ||
| 3021 | static void LogPoolConnectionCensus() | ||
| 3022 | { | ||
| 3023 | int connected = 0, subscribed = 0, authorized = 0, active = 0, neverAuthIdle = 0; | ||
| 3024 | int64 now = GetTime(); | ||
| 3025 | #ifndef _WIN32 | ||
| 3026 | int maxfd = 0; | ||
| 3027 | #endif | ||
| 3028 | for (Miner* m : gMiners) { | ||
| 3029 | connected++; | ||
| 3030 | #ifndef _WIN32 | ||
| 3031 | if ((int)m->fd > maxfd) maxfd = (int)m->fd; | ||
| 3032 | #endif | ||
| 3033 | if (m->subscribed) subscribed++; | ||
| 3034 | if (m->authorised) { | ||
| 3035 | authorized++; | ||
| 3036 | if (m->lastSeen > 0 && now - m->lastSeen <= POOL_ACTIVE_WINDOW) | ||
| 3037 | active++; | ||
| 3038 | } else if (now - m->connectTime >= POOL_UNAUTH_GRACE_SECS) { | ||
| 3039 | neverAuthIdle++; // past the grace period and still not authorized | ||
| 3040 | } | ||
| 3041 | } | ||
| 3042 | |||
| 3043 | #ifndef _WIN32 | ||
| 3044 | int headroom = (int)FD_SETSIZE - maxfd; | ||
| 3045 | LogPrint("pool", "[pool] census: connected=%d subscribed=%d authorized=%d" | ||
| 3046 | " active(last %llds)=%d never-auth-idle=%d" | ||
| 3047 | " (cap=%zu, maxfd=%d, FD_SETSIZE=%d)\n", | ||
| 3048 | connected, subscribed, authorized, (long long)POOL_ACTIVE_WINDOW, | ||
| 3049 | active, neverAuthIdle, MAX_POOL_MINERS, maxfd, (int)FD_SETSIZE); | ||
| 3050 | #else | ||
| 3051 | int headroom = (int)FD_SETSIZE - connected; | ||
| 3052 | LogPrint("pool", "[pool] census: connected=%d subscribed=%d authorized=%d" | ||
| 3053 | " active(last %llds)=%d never-auth-idle=%d" | ||
| 3054 | " (cap=%zu, FD_SETSIZE=%d sockets)\n", | ||
| 3055 | connected, subscribed, authorized, (long long)POOL_ACTIVE_WINDOW, | ||
| 3056 | active, neverAuthIdle, MAX_POOL_MINERS, (int)FD_SETSIZE); | ||
| 3057 | #endif | ||
| 3058 | if (headroom <= 16) | ||
| 3059 | LogPrint("pool", "[pool] WARNING: %d slots left before FD_SETSIZE=%d --" | ||
| 3060 | " the select() loop is about to walk into undefined behaviour" | ||
| 3061 | " (#26). Refuse new miners or rebuild with poll/epoll.\n", | ||
| 3062 | headroom, (int)FD_SETSIZE); | ||
| 3063 | } | ||
| 3064 | |||
| 3065 | // --------------------------------------------------------------------------- | ||
| 1150 | // Stratum message handler -- called for each complete line from a miner | 3066 | // Stratum message handler -- called for each complete line from a miner |
| 1151 | // Returns false if the miner should be disconnected immediately. | 3067 | // Returns false if the miner should be disconnected immediately. |
| 1152 | // --------------------------------------------------------------------------- | 3068 | // --------------------------------------------------------------------------- |
| @@ -1214,6 +3130,7 @@ static bool HandleLine(Miner* m, const std::string& rawLine, | |||
| 1214 | // ------------------------------------------------------------------ | 3130 | // ------------------------------------------------------------------ |
| 1215 | if (method == "mining.subscribe") { | 3131 | if (method == "mining.subscribe") { |
| 1216 | std::string sid = ToHex(&m->fd, 4); | 3132 | std::string sid = ToHex(&m->fd, 4); |
| 3133 | m->subscribed = true; | ||
| 1217 | LogPrint("worker", "[worker->pool] subscribe from fd=%d sid=%s\n", | 3134 | LogPrint("worker", "[worker->pool] subscribe from fd=%d sid=%s\n", |
| 1218 | (int)m->fd, sid.c_str()); | 3135 | (int)m->fd, sid.c_str()); |
| 1219 | reply(json::array({json::array({json::array({"mining.notify",sid})}),sid,4})); | 3136 | reply(json::array({json::array({json::array({"mining.notify",sid})}),sid,4})); |
| @@ -1320,6 +3237,22 @@ static bool HandleLine(Miner* m, const std::string& rawLine, | |||
| 1320 | } | 3237 | } |
| 1321 | unsigned int nNonce; memcpy(&nNonce, nb.data(), 4); | 3238 | unsigned int nNonce; memcpy(&nNonce, nb.data(), 4); |
| 1322 | 3239 | ||
| 3240 | // Reject a replayed (job, nonce) before it costs a rebuild and a | ||
| 3241 | // RandomX hash: nothing about the work below distinguishes a second | ||
| 3242 | // submission of a valid share from the first, so without this check | ||
| 3243 | // it passes every test and is credited again -- and roundShareCount | ||
| 3244 | // is what the payout split divides by, so each replay multiplies the | ||
| 3245 | // address's cut of the round. The set is per-connection and cleared | ||
| 3246 | // by SendJob on every job change, so only a same-job repeat is | ||
| 3247 | // refused. | ||
| 3248 | if (m->seenNonces.count(nNonce)) { | ||
| 3249 | LogPrint("worker", "[worker->pool] duplicate share from %s" | ||
| 3250 | " nonce=%u job=%s -- already credited, not counting again\n", | ||
| 3251 | m->address.c_str(), nNonce, submitJobId.c_str()); | ||
| 3252 | reply(false, "Duplicate share"); | ||
| 3253 | return true; | ||
| 3254 | } | ||
| 3255 | |||
| 1323 | // Rebuild this miner's exact block (unique merkle via extranonce2) | 3256 | // Rebuild this miner's exact block (unique merkle via extranonce2) |
| 1324 | CBlock b = BuildMinerBlock(gCurrentJob, m->extranonce2); | 3257 | CBlock b = BuildMinerBlock(gCurrentJob, m->extranonce2); |
| 1325 | b.nNonce = nNonce; | 3258 | b.nNonce = nNonce; |
| @@ -1347,6 +3280,7 @@ static bool HandleLine(Miner* m, const std::string& rawLine, | |||
| 1347 | m->roundShares++; | 3280 | m->roundShares++; |
| 1348 | m->totalShares++; | 3281 | m->totalShares++; |
| 1349 | m->vardiffShares++; | 3282 | m->vardiffShares++; |
| 3283 | m->seenNonces.insert(nNonce); // replay guard: credit each nonce once per job | ||
| 1350 | int64 shareTime = GetTime(); | 3284 | int64 shareTime = GetTime(); |
| 1351 | m->lastSeen = shareTime; | 3285 | m->lastSeen = shareTime; |
| 1352 | roundShareCount[m->address]++; | 3286 | roundShareCount[m->address]++; |
| @@ -1453,7 +3387,11 @@ static bool HandleLine(Miner* m, const std::string& rawLine, | |||
| 1453 | int foundHeight = gCurrentJob.height; | 3387 | int foundHeight = gCurrentJob.height; |
| 1454 | // b is the block we submitted, untouched by ProcessBlock (which | 3388 | // b is the block we submitted, untouched by ProcessBlock (which |
| 1455 | // took a copy), so its hash is what the payout must be keyed on. | 3389 | // took a copy), so its hash is what the payout must be keyed on. |
| 1456 | QueuePayouts(foundHeight, b.GetHash(), roundShareCount, roundShareTotal); | 3390 | // Passed as a uint256, not hex: the ledger key must stay a real |
| 3391 | // hash. The remaining arguments are upstream's round-proof data. | ||
| 3392 | QueuePayouts(foundHeight, b.GetHash(), b.vtx[0].GetHash().GetHex(), | ||
| 3393 | b.vtx[0].vout[0].nValue, m->address, | ||
| 3394 | roundShareCount, roundShareTotal); | ||
| 1457 | // Reset round shares | 3395 | // Reset round shares |
| 1458 | roundShareCount.clear(); | 3396 | roundShareCount.clear(); |
| 1459 | roundShareTotal = 0; | 3397 | roundShareTotal = 0; |
| @@ -1480,49 +3418,1892 @@ static bool HandleLine(Miner* m, const std::string& rawLine, | |||
| 1480 | } | 3418 | } |
| 1481 | 3419 | ||
| 1482 | // --------------------------------------------------------------------------- | 3420 | // --------------------------------------------------------------------------- |
| 1483 | // Stats refresh -- called every 2s from event loop | 3421 | // payout-ledger self-test |
| 3422 | // | ||
| 3423 | // Regression cover for the payout defects fixed in this series. Everything | ||
| 3424 | // here runs against a scratch data directory and an in-memory wallet map; it | ||
| 3425 | // never opens wallet.dat, never touches a real datadir, and never sends | ||
| 3426 | // anything. | ||
| 1484 | // --------------------------------------------------------------------------- | 3427 | // --------------------------------------------------------------------------- |
| 1485 | static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal) | 3428 | |
| 3429 | static bool PayoutTestReadFile(const std::string& path, std::string& out) | ||
| 1486 | { | 3430 | { |
| 1487 | std::lock_guard<std::mutex> lk(gStatsMutex); | 3431 | FILE* f = fopen(path.c_str(), "r"); |
| 3432 | if (!f) return false; | ||
| 3433 | fseek(f, 0, SEEK_END); | ||
| 3434 | long sz = ftell(f); | ||
| 3435 | fseek(f, 0, SEEK_SET); | ||
| 3436 | out.assign((size_t)(sz > 0 ? sz : 0), '\0'); | ||
| 3437 | if (sz > 0 && fread(&out[0], 1, (size_t)sz, f) != (size_t)sz) { | ||
| 3438 | fclose(f); | ||
| 3439 | return false; | ||
| 3440 | } | ||
| 3441 | fclose(f); | ||
| 3442 | return true; | ||
| 3443 | } | ||
| 1488 | 3444 | ||
| 1489 | gStats.authorizedMiners = 0; | 3445 | // Delete the scratch data directory and everything in it. A sweep and not a |
| 1490 | gStats.blocksFound = blocksFoundThisSession; | 3446 | // list of known names: LogPrint binds debug.log to whatever GetAppDir() is |
| 1491 | gStats.roundShares = roundShareTotal; | 3447 | // when it first fires, so the test's own log lands in here too, and a |
| 1492 | gStats.totalHashRate = 0.0; | 3448 | // directory left behind in src/ after `make tests` is litter. |
| 1493 | for (Miner* m : gMiners) | 3449 | static void PayoutTestRemoveDir(const std::string& dir) |
| 1494 | if (m->authorised) gStats.authorizedMiners++; | 3450 | { |
| 3451 | #ifdef _WIN32 | ||
| 3452 | WIN32_FIND_DATAA findData; | ||
| 3453 | HANDLE hFind = FindFirstFileA((dir + "\\*").c_str(), &findData); | ||
| 3454 | if (hFind != INVALID_HANDLE_VALUE) { | ||
| 3455 | do { | ||
| 3456 | std::string name = findData.cFileName; | ||
| 3457 | if (name == "." || name == "..") continue; | ||
| 3458 | DeleteFileA((dir + "\\" + name).c_str()); | ||
| 3459 | } while (FindNextFileA(hFind, &findData)); | ||
| 3460 | FindClose(hFind); | ||
| 3461 | } | ||
| 3462 | RemoveDirectoryA(dir.c_str()); | ||
| 3463 | #else | ||
| 3464 | // One level of recursion, because CreateTransaction() opens a Berkeley DB | ||
| 3465 | // environment and BDB puts its logs in a "database" subdirectory. Without | ||
| 3466 | // this the scratch directory cannot be removed and the test leaves one | ||
| 3467 | // behind on every run, which is what d5b3503 was for. | ||
| 3468 | DIR* d = opendir(dir.c_str()); | ||
| 3469 | if (d) { | ||
| 3470 | struct dirent* ent; | ||
| 3471 | while ((ent = readdir(d)) != NULL) { | ||
| 3472 | std::string name = ent->d_name; | ||
| 3473 | if (name == "." || name == "..") continue; | ||
| 3474 | const std::string path = dir + "/" + name; | ||
| 3475 | if (unlink(path.c_str()) != 0) | ||
| 3476 | PayoutTestRemoveDir(path); | ||
| 3477 | } | ||
| 3478 | closedir(d); | ||
| 3479 | } | ||
| 3480 | rmdir(dir.c_str()); | ||
| 3481 | #endif | ||
| 3482 | } | ||
| 1495 | 3483 | ||
| 1496 | gWorkerStats.clear(); | 3484 | static bool PayoutTestWriteFile(const std::string& path, const std::string& data) |
| 1497 | for (Miner* m : gMiners) { | 3485 | { |
| 1498 | if (!m->authorised || m->lastSeen <= 0) continue; | 3486 | FILE* f = fopen(path.c_str(), "w"); |
| 1499 | PoolWorkerStatView v; | 3487 | if (!f) return false; |
| 1500 | v.address = m->address; | 3488 | bool ok = data.empty() || fwrite(data.c_str(), 1, data.size(), f) == data.size(); |
| 1501 | v.worker = m->worker; | 3489 | return fclose(f) == 0 && ok; |
| 1502 | v.roundShares = m->roundShares; | 3490 | } |
| 1503 | v.totalShares = m->totalShares; | 3491 | |
| 1504 | v.lastSeen = m->lastSeen; | 3492 | // Wipe every piece of payout state this test touches, so each section starts |
| 1505 | v.hashRate = m->hashRate; | 3493 | // from the same place and a "restart" can be simulated by reloading from disk. |
| 1506 | gWorkerStats.push_back(v); | 3494 | static void PayoutTestResetState() |
| 1507 | gStats.totalHashRate += m->hashRate; | 3495 | { |
| 1508 | } | 3496 | { |
| 1509 | std::sort(gWorkerStats.begin(), gWorkerStats.end(), | 3497 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); |
| 1510 | [](const PoolWorkerStatView& a, const PoolWorkerStatView& b) { | 3498 | gPendingPayouts.clear(); |
| 1511 | return a.roundShares != b.roundShares ? a.roundShares > b.roundShares | 3499 | } |
| 1512 | : a.lastSeen > b.lastSeen; | 3500 | { |
| 3501 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 3502 | gPaying.clear(); | ||
| 3503 | gQueuedPayouts.clear(); | ||
| 3504 | } | ||
| 3505 | { | ||
| 3506 | std::lock_guard<std::mutex> lk(gPoolRoundsMutex); | ||
| 3507 | gPoolRounds.clear(); | ||
| 3508 | } | ||
| 3509 | { | ||
| 3510 | std::lock_guard<std::mutex> lk(gClaimedTxidMutex); | ||
| 3511 | gLegacyClaimedTxids.clear(); | ||
| 3512 | } | ||
| 3513 | { | ||
| 3514 | // A quarantine is meant to last the life of the process; each section | ||
| 3515 | // here is a fresh process as far as the code under test is concerned. | ||
| 3516 | std::lock_guard<std::mutex> lk(gRefusedFilesMutex); | ||
| 3517 | gRefusedFiles.clear(); | ||
| 3518 | } | ||
| 3519 | } | ||
| 3520 | |||
| 3521 | // A BTF address whose hash160 is entirely under the test's control, so the | ||
| 3522 | // output script the probe compares against can be built without a wallet. | ||
| 3523 | static std::string PayoutTestAddress(unsigned char seed) | ||
| 3524 | { | ||
| 3525 | uint160 h = 0; | ||
| 3526 | for (int i = 0; i < 20; i++) | ||
| 3527 | ((unsigned char*)&h)[i] = (unsigned char)(seed + i); | ||
| 3528 | return Hash160ToAddress(h); | ||
| 3529 | } | ||
| 3530 | |||
| 3531 | static CScript PayoutTestScriptFor(const std::string& address) | ||
| 3532 | { | ||
| 3533 | uint160 h160; | ||
| 3534 | CScript sc; | ||
| 3535 | if (AddressToHash160(address, h160)) | ||
| 3536 | sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; | ||
| 3537 | return sc; | ||
| 3538 | } | ||
| 3539 | |||
| 3540 | // Block index entries fabricated by PayoutTestAddWalletTx, so they can be | ||
| 3541 | // freed and unhooked when the test finishes. gPayoutTestChainTip is the end of | ||
| 3542 | // that chain: blocks are linked onto it rather than each replacing the tip, | ||
| 3543 | // because IsInMainChain() is "pnext is set, or I am the tip". | ||
| 3544 | static std::vector<CBlockIndex*> gPayoutTestBlockIndexes; | ||
| 3545 | static CBlockIndex* gPayoutTestChainTip = NULL; | ||
| 3546 | |||
| 3547 | static void PayoutTestClearBlockIndex() | ||
| 3548 | { | ||
| 3549 | CRITICAL_BLOCK(cs_main) | ||
| 3550 | { | ||
| 3551 | for (CBlockIndex* p : gPayoutTestBlockIndexes) { | ||
| 3552 | for (std::map<uint256, CBlockIndex*>::iterator it = mapBlockIndex.begin(); | ||
| 3553 | it != mapBlockIndex.end(); ) { | ||
| 3554 | if (it->second == p) mapBlockIndex.erase(it++); | ||
| 3555 | else ++it; | ||
| 3556 | } | ||
| 3557 | delete p; | ||
| 3558 | } | ||
| 3559 | gPayoutTestBlockIndexes.clear(); | ||
| 3560 | gPayoutTestChainTip = NULL; | ||
| 3561 | pindexBest = NULL; | ||
| 3562 | } | ||
| 3563 | } | ||
| 3564 | |||
| 3565 | // Put a transaction in the wallet map that looks exactly like one this pool | ||
| 3566 | // sent. `nonce` only exists to keep the hashes distinct. | ||
| 3567 | // | ||
| 3568 | // fConfirmed decides what GetDepthInMainChain() will answer for it, which is | ||
| 3569 | // what the legacy probe now gates on. A confirmed one is given a block index | ||
| 3570 | // entry of its own: an empty merkle branch hashes to the transaction itself, | ||
| 3571 | // so a block whose merkle root IS that hash accepts it at index 0, and | ||
| 3572 | // pointing pindexBest at that block puts it in the main chain one deep. An | ||
| 3573 | // unconfirmed one keeps hashBlock 0, which is depth 0 -- the shape a payout | ||
| 3574 | // left behind by a reorg has. | ||
| 3575 | static std::string PayoutTestAddWalletTx(const std::string& comment, | ||
| 3576 | const std::string& address, | ||
| 3577 | int64 amount, | ||
| 3578 | int nonce, | ||
| 3579 | const std::string& structuredId, | ||
| 3580 | bool fConfirmed = true, | ||
| 3581 | const std::string& structuredBlock = "") | ||
| 3582 | { | ||
| 3583 | CWalletTx wtx; | ||
| 3584 | wtx.vin.resize(1); | ||
| 3585 | wtx.vin[0].prevout.SetNull(); | ||
| 3586 | wtx.vin[0].scriptSig << nonce; | ||
| 3587 | wtx.vout.resize(1); | ||
| 3588 | wtx.vout[0].nValue = amount; | ||
| 3589 | wtx.vout[0].scriptPubKey = PayoutTestScriptFor(address); | ||
| 3590 | if (!comment.empty()) | ||
| 3591 | wtx.mapValue["comment"] = comment; | ||
| 3592 | if (!structuredId.empty()) { | ||
| 3593 | wtx.mapValue["pool_payout_id"] = structuredId; | ||
| 3594 | wtx.mapValue["pool_payout_address"] = address; | ||
| 3595 | wtx.mapValue["pool_payout_amount"] = strprintf("%lld", amount); | ||
| 3596 | if (!structuredBlock.empty()) | ||
| 3597 | wtx.mapValue["pool_payout_block"] = structuredBlock; | ||
| 3598 | } | ||
| 3599 | uint256 hash = wtx.GetHash(); | ||
| 3600 | if (fConfirmed) { | ||
| 3601 | // A block hash distinct from the transaction's own. | ||
| 3602 | uint256 blockHash = hash; | ||
| 3603 | ((unsigned char*)&blockHash)[0] ^= 0xff; | ||
| 3604 | CRITICAL_BLOCK(cs_main) | ||
| 3605 | { | ||
| 3606 | CBlockIndex* pindex = new CBlockIndex(); | ||
| 3607 | pindex->hashMerkleRoot = hash; | ||
| 3608 | // Linked onto the end of the fabricated chain rather than simply | ||
| 3609 | // becoming the new tip: IsInMainChain() is "pnext is set, or I am | ||
| 3610 | // the tip", so a second confirmed transaction would otherwise take | ||
| 3611 | // the first one out of the chain again. | ||
| 3612 | pindex->pprev = gPayoutTestChainTip; | ||
| 3613 | pindex->nHeight = gPayoutTestChainTip ? gPayoutTestChainTip->nHeight + 1 : 1; | ||
| 3614 | if (gPayoutTestChainTip) | ||
| 3615 | gPayoutTestChainTip->pnext = pindex; | ||
| 3616 | gPayoutTestChainTip = pindex; | ||
| 3617 | std::map<uint256, CBlockIndex*>::iterator mi = | ||
| 3618 | mapBlockIndex.insert(std::make_pair(blockHash, pindex)).first; | ||
| 3619 | pindex->phashBlock = &mi->first; | ||
| 3620 | pindexBest = pindex; | ||
| 3621 | gPayoutTestBlockIndexes.push_back(pindex); | ||
| 3622 | } | ||
| 3623 | wtx.hashBlock = blockHash; | ||
| 3624 | wtx.nIndex = 0; | ||
| 3625 | } | ||
| 3626 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 3627 | mapWallet[hash] = wtx; | ||
| 3628 | return hash.GetHex(); | ||
| 3629 | } | ||
| 3630 | |||
| 3631 | // A real key, in the keystore, so a test wallet's outputs are IsMine() and can | ||
| 3632 | // actually be signed -- both are needed for CreateTransaction to build the | ||
| 3633 | // large transaction the fee schedule is being tested against. Inserted | ||
| 3634 | // straight into the key maps rather than through AddKey(), which would also | ||
| 3635 | // write it to a wallet.dat. | ||
| 3636 | static std::string PayoutTestAddSpendableKey() | ||
| 3637 | { | ||
| 3638 | CKey key; | ||
| 3639 | key.MakeNewKey(); | ||
| 3640 | std::vector<unsigned char> vchPubKey = key.GetPubKey(); | ||
| 3641 | CRITICAL_BLOCK(cs_mapKeys) | ||
| 3642 | { | ||
| 3643 | mapKeys[vchPubKey] = key.GetPrivKey(); | ||
| 3644 | mapPubKeys[Hash160(vchPubKey)] = vchPubKey; | ||
| 3645 | } | ||
| 3646 | return Hash160ToAddress(Hash160(vchPubKey)); | ||
| 3647 | } | ||
| 3648 | |||
| 3649 | // Give the wallet `n` spendable outputs of `each`, one transaction apiece -- | ||
| 3650 | // which is the granularity SelectCoins works at, and therefore the number of | ||
| 3651 | // inputs the transaction spending them will carry. | ||
| 3652 | // | ||
| 3653 | // Deliberately not coinbases: a null prevout would make CWalletTx::GetCredit | ||
| 3654 | // return 0 until COINBASE_SPENDABLE_DEPTH, and there is no chain here. | ||
| 3655 | static void PayoutTestFundWallet(int n, int64 each, const std::string& address) | ||
| 3656 | { | ||
| 3657 | const CScript sc = PayoutTestScriptFor(address); | ||
| 3658 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 3659 | { | ||
| 3660 | for (int i = 0; i < n; i++) { | ||
| 3661 | CWalletTx wtx; | ||
| 3662 | wtx.vin.resize(1); | ||
| 3663 | wtx.vin[0].prevout.hash = (uint64)(i + 1); | ||
| 3664 | wtx.vin[0].prevout.n = 0; | ||
| 3665 | wtx.vout.resize(1); | ||
| 3666 | wtx.vout[0].nValue = each; | ||
| 3667 | wtx.vout[0].scriptPubKey = sc; | ||
| 3668 | mapWallet[wtx.GetHash()] = wtx; | ||
| 3669 | } | ||
| 3670 | } | ||
| 3671 | } | ||
| 3672 | |||
| 3673 | // Defined below with the event loop; the status-file publication test drives | ||
| 3674 | // it directly. | ||
| 3675 | static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal); | ||
| 3676 | |||
| 3677 | int RunPayoutLedgerSelfTest() | ||
| 3678 | { | ||
| 3679 | int nFail = 0; | ||
| 3680 | auto check = [&](bool ok, const char* msg) { | ||
| 3681 | fprintf(stdout, " %s %s\n", ok ? "ok" : "FAIL", msg); | ||
| 3682 | if (!ok) nFail++; | ||
| 3683 | }; | ||
| 3684 | |||
| 3685 | fprintf(stdout, "payout-ledger self-test\n"); | ||
| 3686 | |||
| 3687 | const std::string savedDataDir = strSetDataDir; | ||
| 3688 | const int savedBestHeight = nBestHeight; | ||
| 3689 | |||
| 3690 | // Scratch data directory in the current working directory. GetAppDir() | ||
| 3691 | // follows strSetDataDir, so this is what PayoutFilePath() and | ||
| 3692 | // PayingFilePath() resolve against for the rest of the test. | ||
| 3693 | #ifdef _WIN32 | ||
| 3694 | unsigned long pid = (unsigned long)GetCurrentProcessId(); | ||
| 3695 | #else | ||
| 3696 | unsigned long pid = (unsigned long)getpid(); | ||
| 3697 | #endif | ||
| 3698 | // Absolute, and the process moves into it. | ||
| 3699 | // | ||
| 3700 | // Both are needed by the CreateTransaction section: Berkeley DB resolves | ||
| 3701 | // its log directory relative to the environment home, so a relative | ||
| 3702 | // datadir makes dbenv.open() fail with DB_RUNRECOVERY; and BDB opens | ||
| 3703 | // db.log relative to the working directory, which is src/ during `make | ||
| 3704 | // tests`. Moving in puts it inside the scratch directory, where it dies | ||
| 3705 | // with everything else. | ||
| 3706 | std::string cwd = "."; | ||
| 3707 | { | ||
| 3708 | #ifdef _WIN32 | ||
| 3709 | char buf[MAX_PATH]; | ||
| 3710 | if (GetCurrentDirectoryA(MAX_PATH, buf) > 0) cwd = buf; | ||
| 3711 | #else | ||
| 3712 | char buf[4096]; | ||
| 3713 | if (getcwd(buf, sizeof(buf)) != NULL) cwd = buf; | ||
| 3714 | #endif | ||
| 3715 | } | ||
| 3716 | std::string dir = cwd + "/" + | ||
| 3717 | strprintf("payout_selftest_%d_%lu", (int)GetTime(), pid); | ||
| 3718 | #ifdef _WIN32 | ||
| 3719 | CreateDirectoryA(dir.c_str(), NULL); | ||
| 3720 | SetCurrentDirectoryA(dir.c_str()); | ||
| 3721 | #else | ||
| 3722 | mkdir(dir.c_str(), 0700); | ||
| 3723 | if (chdir(dir.c_str()) != 0) { /* stay put; the paths are absolute anyway */ } | ||
| 3724 | #endif | ||
| 3725 | strSetDataDir = dir; | ||
| 3726 | fprintf(stdout, " scratch datadir: %s\n", dir.c_str()); | ||
| 3727 | |||
| 3728 | const std::string ledgerPath = PayoutFilePath(); | ||
| 3729 | const std::string payingPath = PayingFilePath(); | ||
| 3730 | const std::string addrA = PayoutTestAddress(0x11); | ||
| 3731 | const std::string addrB = PayoutTestAddress(0x55); | ||
| 3732 | |||
| 3733 | // -- 1. A restart must not duplicate an in-flight record ----------------- | ||
| 3734 | // | ||
| 3735 | // The ledger entry is grandfathered (zero hash) on purpose: it keeps | ||
| 3736 | // PayoutBlockStillOnChain out of the picture, which needs a block index, | ||
| 3737 | // and it exercises the height fallback at the same time. | ||
| 3738 | { | ||
| 3739 | PayoutTestResetState(); | ||
| 3740 | remove(ledgerPath.c_str()); | ||
| 3741 | remove(payingPath.c_str()); | ||
| 3742 | |||
| 3743 | PendingPayout pp; | ||
| 3744 | pp.blockHeight = 4242; | ||
| 3745 | pp.blockHash = 0; | ||
| 3746 | pp.matureAtHeight = 4242; | ||
| 3747 | pp.amounts[addrA] = 10 * COIN; | ||
| 3748 | pp.amounts[addrB] = 20 * COIN; | ||
| 3749 | VersionedBlob snap; | ||
| 3750 | { | ||
| 3751 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 3752 | gPendingPayouts.push_back(pp); | ||
| 3753 | snap = SerializePendingPayoutsLocked(); | ||
| 3754 | } | ||
| 3755 | PublishPendingPayouts(snap); | ||
| 3756 | |||
| 3757 | nBestHeight = 5000; | ||
| 3758 | FlushMaturePayouts(); | ||
| 3759 | size_t afterFirstFlush = 0; | ||
| 3760 | { | ||
| 3761 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 3762 | afterFirstFlush = gPaying.size(); | ||
| 3763 | } | ||
| 3764 | check(afterFirstFlush == 2, | ||
| 3765 | "a matured round explodes into one record per unpaid recipient"); | ||
| 3766 | |||
| 3767 | // Restart: drop everything in memory and come back from the files. | ||
| 3768 | for (int restart = 1; restart <= 2; restart++) { | ||
| 3769 | PayoutTestResetState(); | ||
| 3770 | LoadPendingPayouts(); | ||
| 3771 | LoadPaying(); | ||
| 3772 | ReconcilePayouts(); | ||
| 3773 | |||
| 3774 | size_t queued = 0; | ||
| 3775 | { | ||
| 3776 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 3777 | queued = gQueuedPayouts.size(); | ||
| 3778 | } | ||
| 3779 | if (restart == 1) | ||
| 3780 | check(queued == 1, | ||
| 3781 | "startup rebuilds the exploded-round set from the restored" | ||
| 3782 | " in-flight list"); | ||
| 3783 | |||
| 3784 | FlushMaturePayouts(); | ||
| 3785 | size_t n = 0; | ||
| 3786 | { | ||
| 3787 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 3788 | n = gPaying.size(); | ||
| 3789 | } | ||
| 3790 | check(n == 2, restart == 1 | ||
| 3791 | ? "a restart does not duplicate an in-flight record" | ||
| 3792 | : "a second restart does not duplicate one either"); | ||
| 3793 | } | ||
| 3794 | |||
| 3795 | // The other half of the guard: even with the round-level dedupe set | ||
| 3796 | // deliberately emptied, the per-recipient check against gPaying must | ||
| 3797 | // stop a second standing order for the same money. | ||
| 3798 | { | ||
| 3799 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 3800 | gQueuedPayouts.clear(); | ||
| 3801 | } | ||
| 3802 | FlushMaturePayouts(); | ||
| 3803 | size_t n = 0; | ||
| 3804 | { | ||
| 3805 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 3806 | n = gPaying.size(); | ||
| 3807 | } | ||
| 3808 | check(n == 2, | ||
| 3809 | "the flush refuses a second record for a (round, address) already" | ||
| 3810 | " in flight even with the round set lost"); | ||
| 3811 | } | ||
| 3812 | |||
| 3813 | // -- 2. Duplicates already on disk are collapsed as they are read -------- | ||
| 3814 | { | ||
| 3815 | PayoutTestResetState(); | ||
| 3816 | nlohmann::json arr = nlohmann::json::array(); | ||
| 3817 | for (int i = 0; i < 2; i++) { | ||
| 3818 | nlohmann::json o; | ||
| 3819 | o["blockHeight"] = 4242; | ||
| 3820 | o["blockHash"] = uint256(0).ToString(); | ||
| 3821 | o["matureAtHeight"] = 4242; | ||
| 3822 | o["address"] = addrA; | ||
| 3823 | o["amount"] = (int64)(10 * COIN); | ||
| 3824 | o["retries"] = i == 0 ? 7 : 0; | ||
| 3825 | o["deferrals"] = i == 0 ? 0 : 3; | ||
| 3826 | arr.push_back(o); | ||
| 3827 | } | ||
| 3828 | check(PayoutTestWriteFile(payingPath, arr.dump(2)), | ||
| 3829 | "wrote a paying file containing the duplicate the old build made"); | ||
| 3830 | LoadPaying(); | ||
| 3831 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 3832 | check(gPaying.size() == 1, | ||
| 3833 | "loading collapses a duplicate in-flight record from disk"); | ||
| 3834 | check(gPaying.size() == 1 && gPaying[0].retries == 7 && gPaying[0].deferrals == 3, | ||
| 3835 | "collapsing merges the attempt counts upwards, not resetting them"); | ||
| 3836 | } | ||
| 3837 | |||
| 3838 | // -- 3. State updates address a slot, not a value ----------------------- | ||
| 3839 | // | ||
| 3840 | // Build the exact pathology the old code produced -- two records equal in | ||
| 3841 | // every field -- and prove the update lands on the one asked for. Under | ||
| 3842 | // value matching the first record absorbed everything and the second was | ||
| 3843 | // untouchable. | ||
| 3844 | { | ||
| 3845 | PayoutTestResetState(); | ||
| 3846 | PayingRecord a; | ||
| 3847 | a.blockHeight = 4242; a.blockHash = 0; a.matureAtHeight = 4242; | ||
| 3848 | a.address = addrA; a.amount = 10 * COIN; | ||
| 3849 | PayingRecord b = a; | ||
| 3850 | { | ||
| 3851 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 3852 | a.id = ++gPayingNextId; | ||
| 3853 | b.id = ++gPayingNextId; | ||
| 3854 | gPaying.push_back(a); | ||
| 3855 | gPaying.push_back(b); | ||
| 3856 | } | ||
| 3857 | PayingRecord upd = b; | ||
| 3858 | upd.retries = 5; | ||
| 3859 | upd.retryAfter = 1234567; | ||
| 3860 | ReschedulePayment(upd); | ||
| 3861 | { | ||
| 3862 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 3863 | check(gPaying.size() == 2 && | ||
| 3864 | gPaying[0].retries == 0 && gPaying[1].retries == 5, | ||
| 3865 | "rescheduling updates the record it was given, not the first" | ||
| 3866 | " one that looks like it"); | ||
| 3867 | } | ||
| 3868 | ForgetPayment(a); | ||
| 3869 | { | ||
| 3870 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 3871 | check(gPaying.size() == 1 && gPaying[0].id == b.id, | ||
| 3872 | "forgetting erases the record it was given, not the first" | ||
| 3873 | " one that looks like it"); | ||
| 3874 | } | ||
| 3875 | } | ||
| 3876 | |||
| 3877 | // -- 4. retryAfter survives a restart ------------------------------------ | ||
| 3878 | { | ||
| 3879 | PayoutTestResetState(); | ||
| 3880 | remove(payingPath.c_str()); | ||
| 3881 | PayingRecord r; | ||
| 3882 | r.blockHeight = 77; r.blockHash = 0; r.matureAtHeight = 77; | ||
| 3883 | r.address = addrA; r.amount = 3 * COIN; | ||
| 3884 | r.retryAfter = 1700000000LL; | ||
| 3885 | r.deferrals = 9; | ||
| 3886 | VersionedBlob snap; | ||
| 3887 | { | ||
| 3888 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 3889 | r.id = ++gPayingNextId; | ||
| 3890 | gPaying.push_back(r); | ||
| 3891 | snap = SerializePayingLocked(); | ||
| 3892 | } | ||
| 3893 | PublishPaying(snap); | ||
| 3894 | PayoutTestResetState(); | ||
| 3895 | LoadPaying(); | ||
| 3896 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 3897 | check(gPaying.size() == 1 && gPaying[0].retryAfter == 1700000000LL && | ||
| 3898 | gPaying[0].deferrals == 9, | ||
| 3899 | "the retry clock that is actually used survives a restart"); | ||
| 3900 | } | ||
| 3901 | |||
| 3902 | // -- 5. The ledger file cannot be published from a stale snapshot -------- | ||
| 3903 | { | ||
| 3904 | PayoutTestResetState(); | ||
| 3905 | remove(ledgerPath.c_str()); | ||
| 3906 | |||
| 3907 | PendingPayout one; | ||
| 3908 | one.blockHeight = 100; one.blockHash = 0; one.matureAtHeight = 100; | ||
| 3909 | one.amounts[addrA] = 1 * COIN; | ||
| 3910 | VersionedBlob older, newer; | ||
| 3911 | { | ||
| 3912 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 3913 | gPendingPayouts.push_back(one); | ||
| 3914 | older = SerializePendingPayoutsLocked(); | ||
| 3915 | PendingPayout two; | ||
| 3916 | two.blockHeight = 200; two.blockHash = 0; two.matureAtHeight = 200; | ||
| 3917 | two.amounts[addrB] = 2 * COIN; | ||
| 3918 | gPendingPayouts.push_back(two); | ||
| 3919 | newer = SerializePendingPayoutsLocked(); | ||
| 3920 | } | ||
| 3921 | check(newer.seq > older.seq, | ||
| 3922 | "a later serialisation carries a strictly higher version"); | ||
| 3923 | |||
| 3924 | // The order the race produced: the payout thread's older image | ||
| 3925 | // arriving after the event loop's newer one. | ||
| 3926 | PublishPendingPayouts(newer); | ||
| 3927 | PublishPendingPayouts(older); | ||
| 3928 | std::string onDisk; | ||
| 3929 | check(PayoutTestReadFile(ledgerPath, onDisk) && onDisk == newer.blob, | ||
| 3930 | "an older snapshot cannot overwrite a newer published ledger"); | ||
| 3931 | |||
| 3932 | // And the ordinary direction still works. | ||
| 3933 | VersionedBlob newest; | ||
| 3934 | { | ||
| 3935 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 3936 | gPendingPayouts.clear(); | ||
| 3937 | newest = SerializePendingPayoutsLocked(); | ||
| 3938 | } | ||
| 3939 | PublishPendingPayouts(newest); | ||
| 3940 | check(PayoutTestReadFile(ledgerPath, onDisk) && onDisk == newest.blob, | ||
| 3941 | "a newer snapshot does replace the published ledger"); | ||
| 3942 | } | ||
| 3943 | |||
| 3944 | // -- 6. Concurrent publishers cannot interleave a file ------------------- | ||
| 3945 | // | ||
| 3946 | // Two threads opening and renaming the same "<path>.tmp" over the same | ||
| 3947 | // target was the original defect. This test was mutation-tested and did | ||
| 3948 | // not catch it: with the file mutex removed the suite still went green 2 | ||
| 3949 | // runs in 5, and the "never leave it unparseable" assertion fired 0 times | ||
| 3950 | // in 5. Two reasons, both fixed here. | ||
| 3951 | // | ||
| 3952 | // First, every image had the same structure and near enough the same | ||
| 3953 | // length, so an interleaving of two of them was still valid JSON. The | ||
| 3954 | // images now alternate between ONE entry and FIVE THOUSAND, so a write | ||
| 3955 | // that lands on top of another leaves either a truncated array or a small | ||
| 3956 | // image followed by the tail of a large one -- neither of which parses. | ||
| 3957 | // | ||
| 3958 | // Second, and worse, almost nothing was being written. Each thread walked | ||
| 3959 | // a permutation of the images, and PublishVersioned drops any blob that is | ||
| 3960 | // not newer than what is on disk, so only the record highs in each | ||
| 3961 | // permutation ever reached the disk -- on the order of ten writes in | ||
| 3962 | // total, four threads, no contention to speak of. Every publish here is | ||
| 3963 | // stamped from a shared counter at the moment it is made, so every one of | ||
| 3964 | // them is newer than the last and every one of them writes. | ||
| 3965 | // | ||
| 3966 | // And the file is watched while the writers run, not only inspected after | ||
| 3967 | // they finish. "The published file is never observed unparseable" is the | ||
| 3968 | // actual invariant -- rename(2) is atomic, so a reader can only ever see a | ||
| 3969 | // completely written image -- and checking it once at the end samples the | ||
| 3970 | // one moment when a torn file is least likely to still be there. | ||
| 3971 | { | ||
| 3972 | const std::string racePath = dir + "/publish_race_selftest.json"; | ||
| 3973 | std::mutex raceMutex; | ||
| 3974 | uint64 racePublished = 0; | ||
| 3975 | |||
| 3976 | std::string tiny, huge; | ||
| 3977 | { | ||
| 3978 | nlohmann::json one = nlohmann::json::array(); | ||
| 3979 | nlohmann::json o; | ||
| 3980 | o["image"] = 0; | ||
| 3981 | o["entry"] = 0; | ||
| 3982 | o["address"] = addrA; | ||
| 3983 | o["amount"] = (int64)COIN; | ||
| 3984 | one.push_back(o); | ||
| 3985 | tiny = one.dump(2); | ||
| 3986 | |||
| 3987 | nlohmann::json many = nlohmann::json::array(); | ||
| 3988 | for (int k = 0; k < 5000; k++) { | ||
| 3989 | nlohmann::json e; | ||
| 3990 | e["image"] = 1; | ||
| 3991 | e["entry"] = k; | ||
| 3992 | e["address"] = addrA; | ||
| 3993 | e["amount"] = (int64)(k + 1) * COIN; | ||
| 3994 | many.push_back(e); | ||
| 3995 | } | ||
| 3996 | huge = many.dump(2); | ||
| 3997 | } | ||
| 3998 | |||
| 3999 | const int kThreads = 4; | ||
| 4000 | const int kPerThread = 100; | ||
| 4001 | const uint64 kTotal = (uint64)kThreads * kPerThread; | ||
| 4002 | |||
| 4003 | std::atomic<uint64> nextSeq(0); | ||
| 4004 | std::atomic<bool> racing(true); | ||
| 4005 | std::atomic<int> tornReads(0); | ||
| 4006 | std::atomic<int> goodReads(0); | ||
| 4007 | |||
| 4008 | std::thread watcher([&]() { | ||
| 4009 | bool seenOnce = false; | ||
| 4010 | while (racing) { | ||
| 4011 | std::string s; | ||
| 4012 | if (!PayoutTestReadFile(racePath, s)) { | ||
| 4013 | // Only meaningful once the file exists at all. | ||
| 4014 | if (seenOnce) tornReads++; | ||
| 4015 | continue; | ||
| 4016 | } | ||
| 4017 | seenOnce = true; | ||
| 4018 | try { | ||
| 4019 | if (nlohmann::json::parse(s).is_array()) goodReads++; | ||
| 4020 | else tornReads++; | ||
| 4021 | } catch (...) { | ||
| 4022 | tornReads++; | ||
| 4023 | } | ||
| 4024 | } | ||
| 1513 | }); | 4025 | }); |
| 1514 | 4026 | ||
| 1515 | // gPendingPayouts is event-loop-only data (no lock needed here since | 4027 | std::vector<std::thread> threads; |
| 1516 | // RefreshStats is always called from the event loop thread). | 4028 | for (int t = 0; t < kThreads; t++) { |
| 1517 | gPayoutViews.clear(); | 4029 | threads.push_back(std::thread([&]() { |
| 1518 | for (const PendingPayout& pp : gPendingPayouts) { | 4030 | for (int i = 0; i < kPerThread; i++) { |
| 1519 | PendingPayoutView pv; | 4031 | VersionedBlob v; |
| 1520 | pv.matureAtHeight = pp.matureAtHeight; | 4032 | v.seq = ++nextSeq; |
| 1521 | pv.recipients = (int)pp.amounts.size(); | 4033 | v.blob = (v.seq & 1) ? tiny : huge; |
| 1522 | pv.totalAmount = 0; | 4034 | PublishVersioned(racePath, v, raceMutex, racePublished, |
| 1523 | for (auto& kv : pp.amounts) pv.totalAmount += kv.second; | 4035 | "publish race selftest"); |
| 1524 | gPayoutViews.push_back(pv); | 4036 | } |
| 4037 | })); | ||
| 4038 | } | ||
| 4039 | for (std::thread& th : threads) | ||
| 4040 | th.join(); | ||
| 4041 | racing = false; | ||
| 4042 | watcher.join(); | ||
| 4043 | |||
| 4044 | check(goodReads > 0, | ||
| 4045 | "the race test actually observed the file while it was being" | ||
| 4046 | " written"); | ||
| 4047 | check(tornReads == 0, | ||
| 4048 | "four threads publishing the same file never leave it" | ||
| 4049 | " unparseable, at any instant while they are doing it"); | ||
| 4050 | |||
| 4051 | // The highest sequence number issued is the one the file must settle | ||
| 4052 | // on: PublishVersioned refuses anything older, so whichever order the | ||
| 4053 | // writes complete in, that image is the last one that can land. | ||
| 4054 | const std::string& expected = (kTotal & 1) ? tiny : huge; | ||
| 4055 | std::string onDisk; | ||
| 4056 | bool read = PayoutTestReadFile(racePath, onDisk); | ||
| 4057 | check(read && onDisk == expected, | ||
| 4058 | "the published file settles on the highest version, whatever the" | ||
| 4059 | " completion order"); | ||
| 4060 | remove(racePath.c_str()); | ||
| 4061 | remove((racePath + ".tmp").c_str()); | ||
| 4062 | } | ||
| 4063 | |||
| 4064 | // -- 7. Grandfathered zero-hash rounds do not answer for each other ------ | ||
| 4065 | { | ||
| 4066 | uint256 h1 = 0, h2 = 0; | ||
| 4067 | h1.SetHex("00000000000000000000000000000000000000000000000000000000000000aa"); | ||
| 4068 | h2.SetHex("00000000000000000000000000000000000000000000000000000000000000bb"); | ||
| 4069 | check(!SamePayoutRound(100, 0, 200, 0), | ||
| 4070 | "two zero-hash entries at different heights are different rounds"); | ||
| 4071 | check(SamePayoutRound(100, 0, 100, 0), | ||
| 4072 | "two zero-hash entries at the same height are the same round"); | ||
| 4073 | check(SamePayoutRound(100, h1, 200, h1), | ||
| 4074 | "a real hash identifies the round on its own"); | ||
| 4075 | check(!SamePayoutRound(100, h1, 100, h2), | ||
| 4076 | "different hashes at one height are different rounds"); | ||
| 4077 | check(!SamePayoutRound(100, h1, 100, 0), | ||
| 4078 | "a grandfathered entry never matches a hash-keyed one"); | ||
| 4079 | check(!PayoutRoundIdentifiable(0, 0) && PayoutRoundIdentifiable(0, h1) && | ||
| 4080 | PayoutRoundIdentifiable(7, 0), | ||
| 4081 | "an entry with neither hash nor height is not adjudicable"); | ||
| 4082 | |||
| 4083 | // What actually makes a grandfathered same-height pair safe, since the | ||
| 4084 | // comment on FindRecordedPoolPayout used to claim it was the orphan | ||
| 4085 | // check. It is not: the orphan check passes every zero-hash entry, and | ||
| 4086 | // every pre-upgrade entry has a zero hash. | ||
| 4087 | check(PayoutBlockStillOnChain(0), | ||
| 4088 | "the orphan check passes every grandfathered entry, so it cannot" | ||
| 4089 | " be what voids a losing twin"); | ||
| 4090 | PayoutTestResetState(); | ||
| 4091 | remove(ledgerPath.c_str()); | ||
| 4092 | remove(payingPath.c_str()); | ||
| 4093 | { | ||
| 4094 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4095 | PendingPayout twin; | ||
| 4096 | twin.blockHeight = 300; twin.blockHash = 0; twin.matureAtHeight = 300; | ||
| 4097 | twin.amounts[addrA] = 1 * COIN; | ||
| 4098 | gPendingPayouts.push_back(twin); | ||
| 4099 | gPendingPayouts.push_back(twin); // the losing side of a reorg | ||
| 4100 | } | ||
| 4101 | nBestHeight = 5000; | ||
| 4102 | FlushMaturePayouts(); | ||
| 4103 | { | ||
| 4104 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 4105 | check(gPaying.size() == 1, | ||
| 4106 | "two grandfathered rounds at one height are one round, so the" | ||
| 4107 | " pair presents as one debt and is paid at most once"); | ||
| 4108 | } | ||
| 4109 | |||
| 4110 | // End to end: recording a payment against one grandfathered round must | ||
| 4111 | // not settle the other's debt. | ||
| 4112 | PayoutTestResetState(); | ||
| 4113 | remove(ledgerPath.c_str()); | ||
| 4114 | { | ||
| 4115 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4116 | PendingPayout first; | ||
| 4117 | first.blockHeight = 100; first.blockHash = 0; first.matureAtHeight = 100; | ||
| 4118 | first.amounts[addrA] = 1 * COIN; | ||
| 4119 | PendingPayout second; | ||
| 4120 | second.blockHeight = 200; second.blockHash = 0; second.matureAtHeight = 200; | ||
| 4121 | second.amounts[addrA] = 1 * COIN; | ||
| 4122 | gPendingPayouts.push_back(first); | ||
| 4123 | gPendingPayouts.push_back(second); | ||
| 4124 | } | ||
| 4125 | PayingRecord r; | ||
| 4126 | r.blockHeight = 200; r.blockHash = 0; r.matureAtHeight = 200; | ||
| 4127 | r.address = addrA; r.amount = 1 * COIN; | ||
| 4128 | RecordPayoutTxid(r, "txid-for-the-second-round"); | ||
| 4129 | { | ||
| 4130 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4131 | // The second round was fully paid and retired; the first is | ||
| 4132 | // untouched and still owed. | ||
| 4133 | check(gPendingPayouts.size() == 1 && | ||
| 4134 | gPendingPayouts[0].blockHeight == 100 && | ||
| 4135 | gPendingPayouts[0].paidTxids.empty(), | ||
| 4136 | "a txid recorded against one zero-hash round leaves the other" | ||
| 4137 | " still owed"); | ||
| 4138 | } | ||
| 4139 | } | ||
| 4140 | |||
| 4141 | // -- 8. A payment with only a legacy comment is recognised as paid ------- | ||
| 4142 | { | ||
| 4143 | PayoutTestResetState(); | ||
| 4144 | const int height = 8080; | ||
| 4145 | const int64 amount = 7 * COIN; | ||
| 4146 | const std::string id = PayoutIdForBlock(height, 0); | ||
| 4147 | const std::string comment = LegacyPayoutComment(height); | ||
| 4148 | std::set<std::string> none; | ||
| 4149 | std::string found; | ||
| 4150 | // No round proof exists for these rounds, so legacy evidence is | ||
| 4151 | // admissible -- computed exactly as the payout thread computes it. | ||
| 4152 | const bool fAllowLegacy = !PoolRoundHasProof(id); | ||
| 4153 | |||
| 4154 | check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, none, found) | ||
| 4155 | == PAYOUT_PROBE_NONE, | ||
| 4156 | "an empty wallet is not evidence that anything was paid"); | ||
| 4157 | |||
| 4158 | const std::string legacyTxid = | ||
| 4159 | PayoutTestAddWalletTx(comment, addrA, amount, 1, ""); | ||
| 4160 | check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, none, found) | ||
| 4161 | == PAYOUT_PROBE_LEGACY && found == legacyTxid, | ||
| 4162 | "a payment carrying only a pre-upgrade comment is recognised as" | ||
| 4163 | " already paid"); | ||
| 4164 | |||
| 4165 | check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found) | ||
| 4166 | == PAYOUT_PROBE_NONE, | ||
| 4167 | "the legacy match requires the output to pay THIS recipient"); | ||
| 4168 | check(FindRecordedPoolPayout(id, height, 0, addrA, amount + COIN, fAllowLegacy, none, found) | ||
| 4169 | == PAYOUT_PROBE_NONE, | ||
| 4170 | "the legacy match requires the output to pay exactly this amount"); | ||
| 4171 | check(FindRecordedPoolPayout(id, height + 1, 0, addrA, amount, fAllowLegacy, none, found) | ||
| 4172 | == PAYOUT_PROBE_NONE, | ||
| 4173 | "the legacy match requires the comment for this block height"); | ||
| 4174 | |||
| 4175 | std::set<std::string> spent; | ||
| 4176 | spent.insert(legacyTxid); | ||
| 4177 | check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, spent, found) | ||
| 4178 | == PAYOUT_PROBE_NONE, | ||
| 4179 | "one legacy transaction cannot settle a second debt"); | ||
| 4180 | |||
| 4181 | // A transaction written by THIS build carries pool_payout_id, and must | ||
| 4182 | // never be offered as legacy evidence for a different round -- that is | ||
| 4183 | // exactly what the structured metadata exists to prevent. | ||
| 4184 | const std::string otherId = PayoutIdForBlock(height, 0) + "-other"; | ||
| 4185 | PayoutTestAddWalletTx(comment, addrB, amount, 2, otherId); | ||
| 4186 | check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found) | ||
| 4187 | == PAYOUT_PROBE_NONE, | ||
| 4188 | "another round's structured payment is not legacy evidence for" | ||
| 4189 | " this one"); | ||
| 4190 | |||
| 4191 | // The structured path still wins where it applies. | ||
| 4192 | const std::string structuredTxid = | ||
| 4193 | PayoutTestAddWalletTx(comment, addrB, amount, 3, id); | ||
| 4194 | check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found) | ||
| 4195 | == PAYOUT_PROBE_STRUCTURED && found == structuredTxid, | ||
| 4196 | "structured payout metadata is matched in preference to a comment"); | ||
| 4197 | |||
| 4198 | // A comment-matching transaction that never made it into the chain is | ||
| 4199 | // evidence of NON-payment. This tree has seen a 280-block reorg; a | ||
| 4200 | // pre-upgrade payout invalidated by one is still in mapWallet with its | ||
| 4201 | // comment intact, and the first boot after the upgrade is exactly when | ||
| 4202 | // it would be matched. | ||
| 4203 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 4204 | mapWallet.clear(); | ||
| 4205 | PayoutTestClearBlockIndex(); | ||
| 4206 | const std::string addrC = PayoutTestAddress(0x33); | ||
| 4207 | PayoutTestAddWalletTx(comment, addrC, amount, 4, "", false); | ||
| 4208 | check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, none, found) | ||
| 4209 | == PAYOUT_PROBE_NONE, | ||
| 4210 | "a pre-upgrade payout that never confirmed is not read as" | ||
| 4211 | " payment"); | ||
| 4212 | |||
| 4213 | // The same transaction, confirmed, is. | ||
| 4214 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 4215 | mapWallet.clear(); | ||
| 4216 | const std::string confirmedTxid = | ||
| 4217 | PayoutTestAddWalletTx(comment, addrC, amount, 4, "", true); | ||
| 4218 | check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, none, found) | ||
| 4219 | == PAYOUT_PROBE_LEGACY && found == confirmedTxid, | ||
| 4220 | "the same payout, confirmed, is"); | ||
| 4221 | |||
| 4222 | // And an unconfirmed one does not shadow a confirmed one that would | ||
| 4223 | // have matched: the probe used to stop at the first comment match. | ||
| 4224 | const std::string alsoConfirmed = | ||
| 4225 | PayoutTestAddWalletTx(comment, addrC, amount, 6, "", true); | ||
| 4226 | PayoutTestAddWalletTx(comment, addrC, amount, 5, "", false); | ||
| 4227 | std::set<std::string> spentFirst; | ||
| 4228 | spentFirst.insert(confirmedTxid); | ||
| 4229 | check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, spentFirst, found) | ||
| 4230 | == PAYOUT_PROBE_LEGACY && found == alsoConfirmed, | ||
| 4231 | "an unconfirmed transaction does not hide a confirmed one behind" | ||
| 4232 | " it"); | ||
| 4233 | |||
| 4234 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 4235 | mapWallet.clear(); | ||
| 4236 | PayoutTestClearBlockIndex(); | ||
| 1525 | } | 4237 | } |
| 4238 | |||
| 4239 | // -- 9. On-disk compatibility with the live production ledger ------------ | ||
| 4240 | { | ||
| 4241 | PayoutTestResetState(); | ||
| 4242 | // Exactly the shape of the file on the pool node: amounts, blockHash, | ||
| 4243 | // blockHeight, matureAtHeight, and nothing else. | ||
| 4244 | const char* live = | ||
| 4245 | "[{\"amounts\":{\"%s\":1234500000}," | ||
| 4246 | "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000cd\"," | ||
| 4247 | "\"blockHeight\":1000," | ||
| 4248 | "\"matureAtHeight\":1120}]"; | ||
| 4249 | check(PayoutTestWriteFile(ledgerPath, strprintf(live, addrA.c_str())), | ||
| 4250 | "wrote a ledger file in the live production shape"); | ||
| 4251 | LoadPendingPayouts(); | ||
| 4252 | { | ||
| 4253 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4254 | check(gPendingPayouts.size() == 1 && | ||
| 4255 | gPendingPayouts[0].blockHeight == 1000 && | ||
| 4256 | gPendingPayouts[0].matureAtHeight == 1120 && | ||
| 4257 | gPendingPayouts[0].blockHash != 0 && | ||
| 4258 | gPendingPayouts[0].amounts.size() == 1 && | ||
| 4259 | gPendingPayouts[0].amounts[addrA] == 1234500000LL, | ||
| 4260 | "the live production pending_payouts.json still loads"); | ||
| 4261 | } | ||
| 4262 | |||
| 4263 | // And the shape the merged build writes, retryAfter included, which | ||
| 4264 | // this build no longer writes but must still read past. | ||
| 4265 | PayoutTestResetState(); | ||
| 4266 | const char* merged = | ||
| 4267 | "[{\"amounts\":{\"%s\":100000000}," | ||
| 4268 | "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000ef\"," | ||
| 4269 | "\"blockHeight\":2000,\"matureAtHeight\":2120,\"retryAfter\":0," | ||
| 4270 | "\"paidTxids\":{\"%s\":\"abc123\"}}]"; | ||
| 4271 | check(PayoutTestWriteFile(ledgerPath, | ||
| 4272 | strprintf(merged, addrA.c_str(), addrA.c_str())), | ||
| 4273 | "wrote a ledger file in the merged build's shape"); | ||
| 4274 | LoadPendingPayouts(); | ||
| 4275 | { | ||
| 4276 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4277 | check(gPendingPayouts.size() == 1 && | ||
| 4278 | gPendingPayouts[0].blockHeight == 2000 && | ||
| 4279 | gPendingPayouts[0].paidTxids.size() == 1 && | ||
| 4280 | gPendingPayouts[0].paidTxids[addrA] == "abc123", | ||
| 4281 | "a ledger file carrying retryAfter and paidTxids still loads"); | ||
| 4282 | } | ||
| 4283 | |||
| 4284 | // A round trip through this build must not lose anything either. | ||
| 4285 | VersionedBlob snap; | ||
| 4286 | { | ||
| 4287 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4288 | snap = SerializePendingPayoutsLocked(); | ||
| 4289 | } | ||
| 4290 | PublishPendingPayouts(snap); | ||
| 4291 | PayoutTestResetState(); | ||
| 4292 | LoadPendingPayouts(); | ||
| 4293 | { | ||
| 4294 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4295 | check(gPendingPayouts.size() == 1 && | ||
| 4296 | gPendingPayouts[0].blockHeight == 2000 && | ||
| 4297 | gPendingPayouts[0].matureAtHeight == 2120 && | ||
| 4298 | gPendingPayouts[0].paidTxids[addrA] == "abc123", | ||
| 4299 | "what this build writes, this build reads back unchanged"); | ||
| 4300 | } | ||
| 4301 | } | ||
| 4302 | |||
| 4303 | // -- 10. A forfeit is published, not just dropped ------------------------ | ||
| 4304 | { | ||
| 4305 | PayoutTestResetState(); | ||
| 4306 | remove(ledgerPath.c_str()); | ||
| 4307 | const std::string roundsPath = dir + "/pool_rounds_selftest.json"; | ||
| 4308 | const std::string savedRoundsFile = strPoolRoundsFile; | ||
| 4309 | strPoolRoundsFile = roundsPath; | ||
| 4310 | remove(roundsPath.c_str()); | ||
| 4311 | |||
| 4312 | const int height = 606; | ||
| 4313 | PoolRoundProof round; | ||
| 4314 | round.id = PayoutIdForBlock(height, 0); | ||
| 4315 | round.blockHeight = height; | ||
| 4316 | round.matureAtHeight = height; | ||
| 4317 | round.payouts[addrA] = 5 * COIN; | ||
| 4318 | round.payouts[addrB] = 6 * COIN; | ||
| 4319 | RecordPoolRound(round); | ||
| 4320 | |||
| 4321 | { | ||
| 4322 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4323 | PendingPayout pp; | ||
| 4324 | pp.blockHeight = height; pp.blockHash = 0; pp.matureAtHeight = height; | ||
| 4325 | pp.amounts[addrA] = 5 * COIN; | ||
| 4326 | pp.amounts[addrB] = 6 * COIN; | ||
| 4327 | gPendingPayouts.push_back(pp); | ||
| 4328 | } | ||
| 4329 | |||
| 4330 | PayingRecord r; | ||
| 4331 | r.blockHeight = height; r.blockHash = 0; r.matureAtHeight = height; | ||
| 4332 | r.address = addrA; r.amount = 5 * COIN; | ||
| 4333 | ForfeitPayoutRecipient(r, "selftest reason"); | ||
| 4334 | |||
| 4335 | { | ||
| 4336 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4337 | check(gPendingPayouts.size() == 1 && | ||
| 4338 | gPendingPayouts[0].amounts.count(addrA) == 0 && | ||
| 4339 | gPendingPayouts[0].amounts.count(addrB) == 1, | ||
| 4340 | "a forfeit removes only that recipient from the ledger entry"); | ||
| 4341 | } | ||
| 4342 | |||
| 4343 | std::string onDisk; | ||
| 4344 | bool proofOk = false; | ||
| 4345 | if (PayoutTestReadFile(roundsPath, onDisk)) { | ||
| 4346 | try { | ||
| 4347 | nlohmann::json root = nlohmann::json::parse(onDisk); | ||
| 4348 | const nlohmann::json& p = root["rounds"][0]["payouts"]; | ||
| 4349 | proofOk = p[addrA].value("forfeited", false) && | ||
| 4350 | p[addrA].value("forfeitReason", "") == "selftest reason" && | ||
| 4351 | p[addrA]["txid"].is_null() && | ||
| 4352 | !p[addrB].contains("forfeited"); | ||
| 4353 | } catch (...) { | ||
| 4354 | proofOk = false; | ||
| 4355 | } | ||
| 4356 | } | ||
| 4357 | check(proofOk, | ||
| 4358 | "the published round proof records the forfeit instead of" | ||
| 4359 | " advertising the address as still owed"); | ||
| 4360 | |||
| 4361 | // And it survives a reload of the proof file. | ||
| 4362 | { | ||
| 4363 | std::lock_guard<std::mutex> lk(gPoolRoundsMutex); | ||
| 4364 | gPoolRounds.clear(); | ||
| 4365 | } | ||
| 4366 | LoadPoolRounds(); | ||
| 4367 | { | ||
| 4368 | std::lock_guard<std::mutex> lk(gPoolRoundsMutex); | ||
| 4369 | check(gPoolRounds.size() == 1 && | ||
| 4370 | gPoolRounds[0].forfeited.count(addrA) == 1 && | ||
| 4371 | gPoolRounds[0].forfeited[addrA] == "selftest reason", | ||
| 4372 | "the forfeit is read back from pool_rounds.json"); | ||
| 4373 | } | ||
| 4374 | |||
| 4375 | remove(roundsPath.c_str()); | ||
| 4376 | remove((roundsPath + ".tmp").c_str()); | ||
| 4377 | strPoolRoundsFile = savedRoundsFile; | ||
| 4378 | } | ||
| 4379 | |||
| 4380 | // -- 11. A ledger that failed to load is never written back -------------- | ||
| 4381 | // | ||
| 4382 | // The loaders push entries into the live vector inside the try, so an | ||
| 4383 | // exception part way down a file leaves a PARTIAL ledger in memory: the | ||
| 4384 | // entries before the bad one and nothing after it. Publishing that -- and | ||
| 4385 | // ReconcilePayouts publishes unconditionally, milliseconds into startup -- | ||
| 4386 | // writes the partial state over the file and destroys the only record of | ||
| 4387 | // what was owed. | ||
| 4388 | { | ||
| 4389 | PayoutTestResetState(); | ||
| 4390 | remove(ledgerPath.c_str()); | ||
| 4391 | |||
| 4392 | // Two entries, the second with a matureAtHeight that is not a number. | ||
| 4393 | // The first loads; the second throws where it is read. | ||
| 4394 | const std::string corrupt = strprintf( | ||
| 4395 | "[{\"amounts\":{\"%s\":100000000},\"blockHash\":\"%s\"," | ||
| 4396 | "\"blockHeight\":10,\"matureAtHeight\":10}," | ||
| 4397 | "{\"amounts\":{\"%s\":900000000},\"blockHash\":\"%s\"," | ||
| 4398 | "\"blockHeight\":20,\"matureAtHeight\":\"twenty\"}]", | ||
| 4399 | addrA.c_str(), uint256(0).ToString().c_str(), | ||
| 4400 | addrB.c_str(), uint256(0).ToString().c_str()); | ||
| 4401 | check(PayoutTestWriteFile(ledgerPath, corrupt), | ||
| 4402 | "wrote a ledger whose second entry does not parse"); | ||
| 4403 | |||
| 4404 | const int64 t0 = GetTime(); | ||
| 4405 | LoadPendingPayouts(); | ||
| 4406 | const int64 t1 = GetTime(); | ||
| 4407 | |||
| 4408 | size_t loaded = 0; | ||
| 4409 | { | ||
| 4410 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4411 | loaded = gPendingPayouts.size(); | ||
| 4412 | } | ||
| 4413 | check(loaded == 1, | ||
| 4414 | "a mid-file parse error does leave a partial ledger in memory --" | ||
| 4415 | " which is the reason for everything below"); | ||
| 4416 | |||
| 4417 | std::string gone; | ||
| 4418 | check(!PayoutTestReadFile(ledgerPath, gone), | ||
| 4419 | "the file that did not parse is moved out of the way"); | ||
| 4420 | |||
| 4421 | // The rename stamps GetTime(); it was taken between t0 and t1. | ||
| 4422 | std::string kept, keptPath; | ||
| 4423 | for (int64 t = t0; t <= t1 && keptPath.empty(); t++) { | ||
| 4424 | const std::string candidate = | ||
| 4425 | strprintf("%s.bad.%lld", ledgerPath.c_str(), (long long)t); | ||
| 4426 | if (PayoutTestReadFile(candidate, kept)) | ||
| 4427 | keptPath = candidate; | ||
| 4428 | } | ||
| 4429 | check(!keptPath.empty() && kept == corrupt, | ||
| 4430 | "it is kept, byte for byte, as <path>.bad.<timestamp>"); | ||
| 4431 | |||
| 4432 | // The publish that used to destroy it. | ||
| 4433 | VersionedBlob partial; | ||
| 4434 | { | ||
| 4435 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4436 | partial = SerializePendingPayoutsLocked(); | ||
| 4437 | } | ||
| 4438 | PublishPendingPayouts(partial); | ||
| 4439 | check(!PayoutTestReadFile(ledgerPath, gone), | ||
| 4440 | "and the partial ledger is not published over it"); | ||
| 4441 | |||
| 4442 | // The refusal is per path and lasts the process: a second attempt, | ||
| 4443 | // with more state, is refused too. | ||
| 4444 | { | ||
| 4445 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4446 | PendingPayout pp; | ||
| 4447 | pp.blockHeight = 30; pp.blockHash = 0; pp.matureAtHeight = 30; | ||
| 4448 | pp.amounts[addrA] = 5 * COIN; | ||
| 4449 | gPendingPayouts.push_back(pp); | ||
| 4450 | partial = SerializePendingPayoutsLocked(); | ||
| 4451 | } | ||
| 4452 | PublishPendingPayouts(partial); | ||
| 4453 | check(!PayoutTestReadFile(ledgerPath, gone), | ||
| 4454 | "the refusal is not a one-shot -- the path stays unwritable"); | ||
| 4455 | |||
| 4456 | if (!keptPath.empty()) | ||
| 4457 | remove(keptPath.c_str()); | ||
| 4458 | PayoutTestResetState(); | ||
| 4459 | remove(ledgerPath.c_str()); | ||
| 4460 | |||
| 4461 | // And a file that parses is not quarantined, so the ordinary path is | ||
| 4462 | // still the ordinary path. | ||
| 4463 | check(PayoutTestWriteFile(ledgerPath, "[]"), | ||
| 4464 | "wrote a ledger that parses"); | ||
| 4465 | LoadPendingPayouts(); | ||
| 4466 | { | ||
| 4467 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4468 | PendingPayout pp; | ||
| 4469 | pp.blockHeight = 40; pp.blockHash = 0; pp.matureAtHeight = 40; | ||
| 4470 | pp.amounts[addrA] = 2 * COIN; | ||
| 4471 | gPendingPayouts.push_back(pp); | ||
| 4472 | partial = SerializePendingPayoutsLocked(); | ||
| 4473 | } | ||
| 4474 | PublishPendingPayouts(partial); | ||
| 4475 | std::string written; | ||
| 4476 | check(PayoutTestReadFile(ledgerPath, written) && written == partial.blob, | ||
| 4477 | "a ledger that loaded cleanly is still published normally"); | ||
| 4478 | } | ||
| 4479 | |||
| 4480 | // -- 12. A send that failed for want of coin is not a fault -------------- | ||
| 4481 | // | ||
| 4482 | // The defect this replaces: the classifier decided "was this a funding | ||
| 4483 | // problem?" by re-reading GetBalance() and allowing amount + | ||
| 4484 | // nTransactionFee + CENT. nTransactionFee is 0 on a headless node, and | ||
| 4485 | // GetMinFee() is a step function -- 0 below 10,000 bytes, 11 * CENT at it | ||
| 4486 | // -- so the allowance was either unnecessary or eleven times too small, | ||
| 4487 | // and the ledger's own retry budget then wrote the debt off after 1440 | ||
| 4488 | // repeats of the same stable condition. | ||
| 4489 | // | ||
| 4490 | // This is tested against a real wallet and the real CreateTransaction, not | ||
| 4491 | // against injected values: the honest question is not "does my enum | ||
| 4492 | // switch work" but "does a wallet holding many small coins actually report | ||
| 4493 | // the state I think it does". 60 outputs of 1 BTF give ~180 signed P2PKH | ||
| 4494 | // bytes each, which is where the fee schedule steps. Nothing is committed | ||
| 4495 | // or broadcast -- CreateTransaction only builds. | ||
| 4496 | { | ||
| 4497 | PayoutTestResetState(); | ||
| 4498 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 4499 | mapWallet.clear(); | ||
| 4500 | |||
| 4501 | // CreateTransaction opens CTxDB("r"), which throws if blkindex.dat is | ||
| 4502 | // not there. Create it in the scratch datadir first. | ||
| 4503 | bool dbOk = true; | ||
| 4504 | try { | ||
| 4505 | CTxDB txdb("cr"); | ||
| 4506 | } catch (const std::exception&) { | ||
| 4507 | dbOk = false; | ||
| 4508 | } | ||
| 4509 | check(dbOk, "a scratch block index exists for CreateTransaction to open"); | ||
| 4510 | |||
| 4511 | const std::string walletAddr = PayoutTestAddSpendableKey(); | ||
| 4512 | const std::string payee = PayoutTestAddress(0x77); | ||
| 4513 | const CScript payeeScript = PayoutTestScriptFor(payee); | ||
| 4514 | |||
| 4515 | // -- the >= 10 KB case: 60 small coins, balance = amount + 0.05 ------ | ||
| 4516 | PayoutTestFundWallet(60, 1 * COIN, walletAddr); | ||
| 4517 | const int64 balance = GetBalance(); | ||
| 4518 | check(balance == 60 * COIN, | ||
| 4519 | "the test wallet holds 60 separately spendable coins"); | ||
| 4520 | |||
| 4521 | const int64 amount = balance - 5 * CENT; | ||
| 4522 | // The pre-send gate in PayoutThreadFn passes: the balance covers the | ||
| 4523 | // amount and the only fee that can be known in advance. | ||
| 4524 | check(balance >= amount + nTransactionFee, | ||
| 4525 | "the pre-send gate would let this payment through"); | ||
| 4526 | |||
| 4527 | CWalletTx wtx; | ||
| 4528 | SendMoneyOutcome outcome; | ||
| 4529 | int64 feeRequired = 0; | ||
| 4530 | WalletSendResult result = WALLET_SEND_OK; | ||
| 4531 | bool built = true; | ||
| 4532 | if (dbOk) { | ||
| 4533 | try { | ||
| 4534 | built = CreateTransaction(payeeScript, amount, wtx, feeRequired, | ||
| 4535 | &result); | ||
| 4536 | } catch (const std::exception&) { | ||
| 4537 | built = true; // report as a failure below | ||
| 4538 | } | ||
| 4539 | } | ||
| 4540 | check(dbOk && !built, | ||
| 4541 | "a wallet of many small coins cannot fund amount + the fee its" | ||
| 4542 | " own size demands"); | ||
| 4543 | check(dbOk && result == WALLET_SEND_INSUFFICIENT_FUNDS, | ||
| 4544 | "the wallet reports that as a funding shortfall, not as a" | ||
| 4545 | " failure to build"); | ||
| 4546 | // GetMinFee(fDiscount) returns 0 below 10,000 bytes and | ||
| 4547 | // (1 + bytes/1000) * CENT at or above, so a required fee of 11 CENT or | ||
| 4548 | // more IS the statement that the transaction crossed 10 KB. Measured | ||
| 4549 | // through the real serialiser rather than asserted. | ||
| 4550 | check(dbOk && feeRequired >= 11 * CENT, | ||
| 4551 | "the transaction it built crossed 10,000 bytes, so the fee" | ||
| 4552 | " stepped from 0 to at least 11 CENT"); | ||
| 4553 | |||
| 4554 | // The defect, stated as arithmetic. The old classifier asked whether | ||
| 4555 | // the balance covered amount + nTransactionFee + CENT; it did, so it | ||
| 4556 | // called a permanent funding shortfall a fault and started counting | ||
| 4557 | // towards a write-off. | ||
| 4558 | check(!(balance < amount + nTransactionFee + CENT), | ||
| 4559 | "the reserve this replaces would have judged this send a fault"); | ||
| 4560 | check(balance < amount + feeRequired, | ||
| 4561 | "while the fee actually required was more than the balance had"); | ||
| 4562 | |||
| 4563 | outcome.result = result; | ||
| 4564 | outcome.nFeeRequired = feeRequired; | ||
| 4565 | check(dbOk && ClassifyPayoutSendFailure(outcome.result, false) | ||
| 4566 | == PAYOUT_FAILURE_DEFER, | ||
| 4567 | "a >= 10 KB send that ran out of coin is deferred, not counted" | ||
| 4568 | " against the payment"); | ||
| 4569 | |||
| 4570 | // -- the < 10 KB case: no reserve is needed at all ------------------- | ||
| 4571 | // | ||
| 4572 | // The other half of the arithmetic. Below the step the required fee is | ||
| 4573 | // 0, so a reserve of any size is a reason to refuse a payment the | ||
| 4574 | // wallet could have made -- which is its own way of not paying a | ||
| 4575 | // miner. | ||
| 4576 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 4577 | mapWallet.clear(); | ||
| 4578 | PayoutTestFundWallet(3, 1 * COIN, walletAddr); | ||
| 4579 | const int64 smallAmount = 3 * COIN - 5 * CENT; | ||
| 4580 | CWalletTx smallTx; | ||
| 4581 | int64 smallFee = -1; | ||
| 4582 | WalletSendResult smallResult = WALLET_SEND_FAILED; | ||
| 4583 | bool smallBuilt = false; | ||
| 4584 | if (dbOk) { | ||
| 4585 | try { | ||
| 4586 | smallBuilt = CreateTransaction(payeeScript, smallAmount, smallTx, | ||
| 4587 | smallFee, &smallResult); | ||
| 4588 | } catch (const std::exception&) { | ||
| 4589 | smallBuilt = false; | ||
| 4590 | } | ||
| 4591 | } | ||
| 4592 | check(dbOk && smallBuilt && smallResult == WALLET_SEND_OK, | ||
| 4593 | "the same balance pays a small transaction with no headroom at" | ||
| 4594 | " all"); | ||
| 4595 | check(dbOk && smallFee == 0, | ||
| 4596 | "because below 10,000 bytes the required fee is zero"); | ||
| 4597 | |||
| 4598 | // -- and the classification of every other answer -------------------- | ||
| 4599 | check(ClassifyPayoutSendFailure(WALLET_SEND_INSUFFICIENT_FUNDS, false) | ||
| 4600 | == PAYOUT_FAILURE_DEFER && | ||
| 4601 | ClassifyPayoutSendFailure(WALLET_SEND_INSUFFICIENT_FUNDS, true) | ||
| 4602 | == PAYOUT_FAILURE_DEFER && | ||
| 4603 | ClassifyPayoutSendFailure(WALLET_SEND_FAILED, true) | ||
| 4604 | == PAYOUT_FAILURE_DEFER, | ||
| 4605 | "short of coin or unable to sign is always a deferral"); | ||
| 4606 | check(ClassifyPayoutSendFailure(WALLET_SEND_FAILED, false) | ||
| 4607 | == PAYOUT_FAILURE_FAULT, | ||
| 4608 | "funded and unlocked and still refused is a fault"); | ||
| 4609 | |||
| 4610 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 4611 | mapWallet.clear(); | ||
| 4612 | } | ||
| 4613 | |||
| 4614 | // -- 13. No payout mutex is held across the status-file write ------------ | ||
| 4615 | // | ||
| 4616 | // RefreshStats runs on the event loop -- the thread that owns every | ||
| 4617 | // miner's socket -- and used to hold gStatsMutex and gPayoutLedgerMutex at | ||
| 4618 | // function scope across WriteFileAtomic. Once that write gained fsync() on | ||
| 4619 | // the file and on its directory, publishing pool_status.json parked the | ||
| 4620 | // payout thread behind two disk flushes every ten seconds. | ||
| 4621 | // | ||
| 4622 | // Timing assertions would be flaky, so this does not measure anything. It | ||
| 4623 | // makes the write block for as long as the test likes: strPoolStatusFile | ||
| 4624 | // points at a path whose ".tmp" is a FIFO, and open(2) for writing on a | ||
| 4625 | // FIFO does not return until a reader arrives. While RefreshStats is | ||
| 4626 | // parked in there, every mutex it must NOT be holding has to be | ||
| 4627 | // acquirable. Against the previous code this test does not merely fail, it | ||
| 4628 | // cannot pass: the writer is inside fopen() with both mutexes held and | ||
| 4629 | // nothing will release them until the test lets the write finish. | ||
| 4630 | #ifndef _WIN32 | ||
| 4631 | { | ||
| 4632 | PayoutTestResetState(); | ||
| 4633 | const std::string statusPath = dir + "/pool_status_selftest.json"; | ||
| 4634 | const std::string statusTmp = statusPath + ".tmp"; | ||
| 4635 | const std::string savedStatusFile = strPoolStatusFile; | ||
| 4636 | const int64 savedLastWrite = gLastPoolStatusWrite; | ||
| 4637 | remove(statusPath.c_str()); | ||
| 4638 | remove(statusTmp.c_str()); | ||
| 4639 | strPoolStatusFile = statusPath; | ||
| 4640 | gLastPoolStatusWrite = 0; // due a publish on the next refresh | ||
| 4641 | |||
| 4642 | // Give the status file something to report, so the ledger mutex is | ||
| 4643 | // genuinely taken on the way through rather than skipped over. | ||
| 4644 | { | ||
| 4645 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4646 | PendingPayout pp; | ||
| 4647 | pp.blockHeight = 909; pp.blockHash = 0; pp.matureAtHeight = 909; | ||
| 4648 | pp.amounts[addrA] = 3 * COIN; | ||
| 4649 | gPendingPayouts.push_back(pp); | ||
| 4650 | } | ||
| 4651 | |||
| 4652 | bool fifoOk = (mkfifo(statusTmp.c_str(), 0600) == 0); | ||
| 4653 | check(fifoOk, "a FIFO stands in for the status file's temp file, so the" | ||
| 4654 | " write can be held open"); | ||
| 4655 | if (fifoOk) { | ||
| 4656 | std::atomic<bool> done(false); | ||
| 4657 | std::thread writer([&done]() { | ||
| 4658 | RefreshStats(0, 0); | ||
| 4659 | done = true; | ||
| 4660 | }); | ||
| 4661 | |||
| 4662 | // Settle: the work before the write is a handful of vector | ||
| 4663 | // operations, so anything past this point is inside fopen(). The | ||
| 4664 | // !done check below is what actually proves it -- with no reader | ||
| 4665 | // there is no other way for this thread to still be running. | ||
| 4666 | Sleep(500); | ||
| 4667 | bool blocked = !done; | ||
| 4668 | bool gotLedger = false, gotStats = false, gotExec = false, | ||
| 4669 | gotRounds = false; | ||
| 4670 | for (int i = 0; i < 1000 && blocked; i++) { | ||
| 4671 | if (!gotLedger && gPayoutLedgerMutex.try_lock()) { | ||
| 4672 | gotLedger = true; gPayoutLedgerMutex.unlock(); | ||
| 4673 | } | ||
| 4674 | if (!gotStats && gStatsMutex.try_lock()) { | ||
| 4675 | gotStats = true; gStatsMutex.unlock(); | ||
| 4676 | } | ||
| 4677 | if (!gotExec && gPayoutExecMutex.try_lock()) { | ||
| 4678 | gotExec = true; gPayoutExecMutex.unlock(); | ||
| 4679 | } | ||
| 4680 | if (!gotRounds && gPoolRoundsMutex.try_lock()) { | ||
| 4681 | gotRounds = true; gPoolRoundsMutex.unlock(); | ||
| 4682 | } | ||
| 4683 | if (gotLedger && gotStats && gotExec && gotRounds) | ||
| 4684 | break; | ||
| 4685 | Sleep(2); | ||
| 4686 | } | ||
| 4687 | |||
| 4688 | check(blocked, | ||
| 4689 | "the status-file write is held open for the duration of this" | ||
| 4690 | " test"); | ||
| 4691 | check(!blocked || gotLedger, | ||
| 4692 | "gPayoutLedgerMutex is free while pool_status.json is being" | ||
| 4693 | " written"); | ||
| 4694 | check(!blocked || (gotExec && gotRounds), | ||
| 4695 | "gPayoutExecMutex and gPoolRoundsMutex are free while" | ||
| 4696 | " pool_status.json is being written"); | ||
| 4697 | check(!blocked || gotStats, | ||
| 4698 | "gStatsMutex is free while pool_status.json is being written"); | ||
| 4699 | |||
| 4700 | // Let the write finish. Drain to EOF rather than closing early: | ||
| 4701 | // the writer is still filling the pipe and a closed read end would | ||
| 4702 | // deliver it SIGPIPE. | ||
| 4703 | std::thread reader([&statusTmp]() { | ||
| 4704 | FILE* rf = fopen(statusTmp.c_str(), "r"); | ||
| 4705 | if (!rf) return; | ||
| 4706 | char buf[4096]; | ||
| 4707 | while (fread(buf, 1, sizeof(buf), rf) > 0) { } | ||
| 4708 | fclose(rf); | ||
| 4709 | }); | ||
| 4710 | writer.join(); | ||
| 4711 | reader.join(); | ||
| 4712 | } | ||
| 4713 | remove(statusTmp.c_str()); | ||
| 4714 | |||
| 4715 | // And the ordinary path still produces the file it always did. | ||
| 4716 | gLastPoolStatusWrite = 0; | ||
| 4717 | RefreshStats(0, 0); | ||
| 4718 | std::string onDisk; | ||
| 4719 | bool statusOk = false; | ||
| 4720 | if (PayoutTestReadFile(statusPath, onDisk)) { | ||
| 4721 | try { | ||
| 4722 | nlohmann::json j = nlohmann::json::parse(onDisk); | ||
| 4723 | statusOk = j.value("schema", "") == "bitflash-pool-status-1" && | ||
| 4724 | j["pendingPayouts"].is_array() && | ||
| 4725 | j["pendingPayouts"].size() == 1 && | ||
| 4726 | j["pendingPayouts"][0].value("matureAtHeight", 0) == 909 && | ||
| 4727 | j["pendingPayouts"][0].value("totalAmountSatoshis", | ||
| 4728 | (int64)0) == 3 * COIN; | ||
| 4729 | } catch (...) { | ||
| 4730 | statusOk = false; | ||
| 4731 | } | ||
| 4732 | } | ||
| 4733 | check(statusOk, | ||
| 4734 | "splitting the serialise from the write leaves pool_status.json" | ||
| 4735 | " saying the same thing"); | ||
| 4736 | |||
| 4737 | remove(statusPath.c_str()); | ||
| 4738 | remove((statusPath + ".tmp").c_str()); | ||
| 4739 | strPoolStatusFile = savedStatusFile; | ||
| 4740 | gLastPoolStatusWrite = savedLastWrite; | ||
| 4741 | } | ||
| 4742 | #endif | ||
| 4743 | |||
| 4744 | // -- 14. The payout comment names the funding block ---------------------- | ||
| 4745 | // | ||
| 4746 | // Same height, two hashes: the shape a reorg leaves when the pool wins a | ||
| 4747 | // height, loses the block, and wins the height again. The legacy needle is | ||
| 4748 | // frozen at the pre-upgrade format, so pre-upgrade payments keep matching; | ||
| 4749 | // a new-format comment never matches it, so a new send can never be read | ||
| 4750 | // as a pre-upgrade payment for a twin round. | ||
| 4751 | { | ||
| 4752 | PayoutTestResetState(); | ||
| 4753 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 4754 | mapWallet.clear(); | ||
| 4755 | PayoutTestClearBlockIndex(); | ||
| 4756 | |||
| 4757 | const int height = 9090; | ||
| 4758 | const int64 amount = 3 * COIN; | ||
| 4759 | uint256 hashA = 0, hashB = 0; | ||
| 4760 | hashA.SetHex("00000000000000000000000000000000000000000000000000000000000000aa"); | ||
| 4761 | hashB.SetHex("00000000000000000000000000000000000000000000000000000000000000bb"); | ||
| 4762 | |||
| 4763 | check(LegacyPayoutComment(height) == | ||
| 4764 | strprintf("Pool payout height %d", height), | ||
| 4765 | "the legacy needle is frozen at the format pre-upgrade builds" | ||
| 4766 | " wrote"); | ||
| 4767 | check(PayoutComment(height, 0) == LegacyPayoutComment(height), | ||
| 4768 | "a grandfathered zero-hash record still writes the legacy form"); | ||
| 4769 | check(PayoutComment(height, hashA) == | ||
| 4770 | strprintf("Pool payout height %d block %s", height, | ||
| 4771 | hashA.ToString().c_str()), | ||
| 4772 | "a hash-keyed record's comment names its funding block"); | ||
| 4773 | check(PayoutComment(height, hashA) != PayoutComment(height, hashB), | ||
| 4774 | "twin rounds at one height carry distinguishable comments"); | ||
| 4775 | |||
| 4776 | // A transaction wearing the NEW comment format and -- hypothetically | ||
| 4777 | // -- no structured metadata must not be matched by the legacy needle: | ||
| 4778 | // the fallback exists for pre-upgrade transactions, and the new format | ||
| 4779 | // proves this is not one. | ||
| 4780 | const std::string idA = PayoutIdForBlock(height, hashA); | ||
| 4781 | const std::string idB = PayoutIdForBlock(height, hashB); | ||
| 4782 | std::set<std::string> none; | ||
| 4783 | std::string found; | ||
| 4784 | // Neither twin has a round proof here (section 15 covers the gate), | ||
| 4785 | // and their ids collide, so one answer serves both. | ||
| 4786 | const bool fAllowLegacy = !PoolRoundHasProof(idA); | ||
| 4787 | PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount, 1, ""); | ||
| 4788 | check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found) | ||
| 4789 | == PAYOUT_PROBE_NONE, | ||
| 4790 | "a new-format comment is never legacy evidence for a twin round"); | ||
| 4791 | check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found) | ||
| 4792 | == PAYOUT_PROBE_NONE, | ||
| 4793 | "nor a match channel for its own round -- structured metadata is"); | ||
| 4794 | |||
| 4795 | // Round A's own payment, sent by this build, is found by its | ||
| 4796 | // structured id. | ||
| 4797 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 4798 | mapWallet.clear(); | ||
| 4799 | PayoutTestClearBlockIndex(); | ||
| 4800 | const std::string paidA = | ||
| 4801 | PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount, | ||
| 4802 | 2, idA); | ||
| 4803 | check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found) | ||
| 4804 | == PAYOUT_PROBE_STRUCTURED && found == paidA, | ||
| 4805 | "the winning twin's own payment is found by its structured id"); | ||
| 4806 | |||
| 4807 | // The id is derived from the FIRST sixteen hex digits of the hash -- | ||
| 4808 | // the end proof-of-work fills with zeros -- so twin hashes that agree | ||
| 4809 | // there collide in id, and a payment carrying no pool_payout_block | ||
| 4810 | // answers for either twin. That is the accepted trade for payments | ||
| 4811 | // sent before the key existed, pinned here so it stays a decision. | ||
| 4812 | check(idA == idB, | ||
| 4813 | "twin hashes differing only past the sixteenth hex digit collide" | ||
| 4814 | " in id -- the reason the full hash must decide"); | ||
| 4815 | check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found) | ||
| 4816 | == PAYOUT_PROBE_STRUCTURED, | ||
| 4817 | "a payment without the block key answers for a colliding twin --" | ||
| 4818 | " the accepted residual for pre-key transactions"); | ||
| 4819 | |||
| 4820 | // With the full hash recorded -- as every send now records it -- the | ||
| 4821 | // twins are told apart and the re-won height is paid. | ||
| 4822 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 4823 | mapWallet.clear(); | ||
| 4824 | PayoutTestClearBlockIndex(); | ||
| 4825 | const std::string keyedA = | ||
| 4826 | PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount, | ||
| 4827 | 3, idA, true, hashA.ToString()); | ||
| 4828 | check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found) | ||
| 4829 | == PAYOUT_PROBE_STRUCTURED && found == keyedA, | ||
| 4830 | "a payment carrying the full block hash settles its own round"); | ||
| 4831 | check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found) | ||
| 4832 | == PAYOUT_PROBE_NONE, | ||
| 4833 | "and is no evidence for the twin, so the re-won height is paid"); | ||
| 4834 | |||
| 4835 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 4836 | mapWallet.clear(); | ||
| 4837 | PayoutTestClearBlockIndex(); | ||
| 4838 | } | ||
| 4839 | |||
| 4840 | // -- 15. A proof-bearing round rejects comment-only evidence ------------- | ||
| 4841 | // | ||
| 4842 | // The surviving half of the same-height collision: a pre-upgrade | ||
| 4843 | // transaction carries only "Pool payout height H", so if H is won again | ||
| 4844 | // after a reorg, it matches the new round's (height, address, amount) | ||
| 4845 | // exactly and would settle a debt it never paid. The new round has a | ||
| 4846 | // round proof -- written at creation, and only by builds whose payments | ||
| 4847 | // all name their round -- so for it, comment-only evidence is | ||
| 4848 | // inadmissible and the miner is paid. A round without a proof keeps | ||
| 4849 | // today's bias: the evidence is admissible and doubt resolves to | ||
| 4850 | // "already paid". | ||
| 4851 | { | ||
| 4852 | PayoutTestResetState(); | ||
| 4853 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 4854 | mapWallet.clear(); | ||
| 4855 | PayoutTestClearBlockIndex(); | ||
| 4856 | const std::string roundsPath = dir + "/pool_rounds_gate_selftest.json"; | ||
| 4857 | const std::string savedRoundsFile = strPoolRoundsFile; | ||
| 4858 | strPoolRoundsFile = roundsPath; | ||
| 4859 | remove(roundsPath.c_str()); | ||
| 4860 | |||
| 4861 | const int height = 7171; | ||
| 4862 | const int64 amount = 4 * COIN; | ||
| 4863 | uint256 hashB = 0; | ||
| 4864 | hashB.SetHex("00000000000000000000000000000000000000000000000000000000000000cc"); | ||
| 4865 | const std::string idB = PayoutIdForBlock(height, hashB); | ||
| 4866 | std::set<std::string> none; | ||
| 4867 | std::string found; | ||
| 4868 | |||
| 4869 | // The colliding pre-upgrade payment: confirmed, comment-only, same | ||
| 4870 | // height, same address, same amount -- the strongest wrong evidence | ||
| 4871 | // there is. | ||
| 4872 | PayoutTestAddWalletTx(LegacyPayoutComment(height), addrA, amount, 1, ""); | ||
| 4873 | |||
| 4874 | check(!PoolRoundHasProof(idB), | ||
| 4875 | "a round with no recorded proof does not claim one"); | ||
| 4876 | check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, | ||
| 4877 | !PoolRoundHasProof(idB), none, found) | ||
| 4878 | == PAYOUT_PROBE_LEGACY, | ||
| 4879 | "without a proof the doubt is genuine and still resolves to" | ||
| 4880 | " already-paid"); | ||
| 4881 | |||
| 4882 | PoolRoundProof round; | ||
| 4883 | round.id = idB; | ||
| 4884 | round.blockHeight = height; | ||
| 4885 | round.matureAtHeight = height + COINBASE_SPENDABLE_DEPTH; | ||
| 4886 | round.payouts[addrA] = amount; | ||
| 4887 | RecordPoolRound(round); | ||
| 4888 | check(PoolRoundHasProof(idB), | ||
| 4889 | "recording the round proof marks the round as this lineage's"); | ||
| 4890 | check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, | ||
| 4891 | !PoolRoundHasProof(idB), none, found) | ||
| 4892 | == PAYOUT_PROBE_NONE, | ||
| 4893 | "with a proof, a same-height comment-only transaction is not" | ||
| 4894 | " evidence, and the re-won height is paid"); | ||
| 4895 | |||
| 4896 | // Structured evidence still is: the round's own payment, recovered. | ||
| 4897 | const std::string paidB = | ||
| 4898 | PayoutTestAddWalletTx(PayoutComment(height, hashB), addrA, amount, | ||
| 4899 | 2, idB, true, hashB.ToString()); | ||
| 4900 | check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, | ||
| 4901 | !PoolRoundHasProof(idB), none, found) | ||
| 4902 | == PAYOUT_PROBE_STRUCTURED && found == paidB, | ||
| 4903 | "the gate does not blind the probe to the round's own structured" | ||
| 4904 | " payment"); | ||
| 4905 | |||
| 4906 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 4907 | mapWallet.clear(); | ||
| 4908 | PayoutTestClearBlockIndex(); | ||
| 4909 | remove(roundsPath.c_str()); | ||
| 4910 | remove((roundsPath + ".tmp").c_str()); | ||
| 4911 | strPoolRoundsFile = savedRoundsFile; | ||
| 4912 | } | ||
| 4913 | |||
| 4914 | // -- 16. A malformed block hash is corruption, not a hash ---------------- | ||
| 4915 | // | ||
| 4916 | // SetHex() parses whatever hex prefix it finds and stops, so a corrupt | ||
| 4917 | // blockHash did not fail to load -- it loaded as a DIFFERENT hash. | ||
| 4918 | // "deadbeef..." became 0xdeadbeef, a hash no block ever had, which the | ||
| 4919 | // orphan check reads as an orphaned block and the next flush VOIDS the | ||
| 4920 | // debt; a value with no hex prefix became 0, which is grandfathered past | ||
| 4921 | // the orphan check instead. Either way a corrupt file silently changed | ||
| 4922 | // what the pool owed. It is a load failure now, handled like every other | ||
| 4923 | // parse error: quarantine, and never rewrite. | ||
| 4924 | { | ||
| 4925 | PayoutTestResetState(); | ||
| 4926 | remove(ledgerPath.c_str()); | ||
| 4927 | remove(payingPath.c_str()); | ||
| 4928 | |||
| 4929 | const std::string badLedger = strprintf( | ||
| 4930 | "[{\"amounts\":{\"%s\":100000000}," | ||
| 4931 | "\"blockHash\":\"deadbeef oops, not a hash\"," | ||
| 4932 | "\"blockHeight\":10,\"matureAtHeight\":10}]", addrA.c_str()); | ||
| 4933 | check(PayoutTestWriteFile(ledgerPath, badLedger), | ||
| 4934 | "wrote a ledger whose blockHash is not a 64-digit hex string"); | ||
| 4935 | LoadPendingPayouts(); | ||
| 4936 | { | ||
| 4937 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4938 | check(gPendingPayouts.empty(), | ||
| 4939 | "no entry is loaded with a guessed hash"); | ||
| 4940 | } | ||
| 4941 | std::string gone; | ||
| 4942 | check(!PayoutTestReadFile(ledgerPath, gone), | ||
| 4943 | "the ledger with the corrupt hash is quarantined"); | ||
| 4944 | |||
| 4945 | const std::string badPaying = strprintf( | ||
| 4946 | "[{\"address\":\"%s\",\"amount\":100000000," | ||
| 4947 | "\"blockHash\":\"00000000cafe\",\"blockHeight\":10," | ||
| 4948 | "\"matureAtHeight\":10}]", addrA.c_str()); | ||
| 4949 | check(PayoutTestWriteFile(payingPath, badPaying), | ||
| 4950 | "wrote an in-flight list whose blockHash is truncated"); | ||
| 4951 | LoadPaying(); | ||
| 4952 | { | ||
| 4953 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 4954 | check(gPaying.empty(), | ||
| 4955 | "no in-flight record is loaded with a guessed hash"); | ||
| 4956 | } | ||
| 4957 | check(!PayoutTestReadFile(payingPath, gone), | ||
| 4958 | "the in-flight list with the corrupt hash is quarantined"); | ||
| 4959 | |||
| 4960 | // The two spellings every real file uses still load: the 64-digit | ||
| 4961 | // hash of a keyed round, and the 64 zeros this build writes for a | ||
| 4962 | // grandfathered entry. | ||
| 4963 | PayoutTestResetState(); | ||
| 4964 | remove(ledgerPath.c_str()); | ||
| 4965 | const std::string goodLedger = strprintf( | ||
| 4966 | "[{\"amounts\":{\"%s\":100000000}," | ||
| 4967 | "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000cd\"," | ||
| 4968 | "\"blockHeight\":10,\"matureAtHeight\":10}," | ||
| 4969 | "{\"amounts\":{\"%s\":200000000}," | ||
| 4970 | "\"blockHash\":\"%s\"," | ||
| 4971 | "\"blockHeight\":20,\"matureAtHeight\":20}]", | ||
| 4972 | addrA.c_str(), addrB.c_str(), uint256(0).ToString().c_str()); | ||
| 4973 | check(PayoutTestWriteFile(ledgerPath, goodLedger), | ||
| 4974 | "wrote a ledger in both real hash spellings"); | ||
| 4975 | LoadPendingPayouts(); | ||
| 4976 | { | ||
| 4977 | std::lock_guard<std::mutex> lk(gPayoutLedgerMutex); | ||
| 4978 | check(gPendingPayouts.size() == 2 && | ||
| 4979 | gPendingPayouts[0].blockHash != 0 && | ||
| 4980 | gPendingPayouts[1].blockHash == 0, | ||
| 4981 | "a real hash and a serialised zero hash both still load"); | ||
| 4982 | } | ||
| 4983 | remove(ledgerPath.c_str()); | ||
| 4984 | } | ||
| 4985 | |||
| 4986 | // -- clean up ------------------------------------------------------------ | ||
| 4987 | PayoutTestResetState(); | ||
| 4988 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 4989 | mapWallet.clear(); | ||
| 4990 | PayoutTestClearBlockIndex(); | ||
| 4991 | // Out of the scratch directory before removing it, and out of the key maps | ||
| 4992 | // too: the keys this test generated are process-local, but leaving them in | ||
| 4993 | // the keystore would let a later test think a script was its own. | ||
| 4994 | CRITICAL_BLOCK(cs_mapKeys) | ||
| 4995 | { | ||
| 4996 | mapKeys.clear(); | ||
| 4997 | mapPubKeys.clear(); | ||
| 4998 | } | ||
| 4999 | #ifdef _WIN32 | ||
| 5000 | SetCurrentDirectoryA(cwd.c_str()); | ||
| 5001 | #else | ||
| 5002 | if (chdir(cwd.c_str()) != 0) { /* nothing useful to do about it */ } | ||
| 5003 | #endif | ||
| 5004 | PayoutTestRemoveDir(dir); | ||
| 5005 | strSetDataDir = savedDataDir; | ||
| 5006 | nBestHeight = savedBestHeight; | ||
| 5007 | |||
| 5008 | fprintf(stdout, "%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED", | ||
| 5009 | nFail, nFail == 1 ? "" : "s"); | ||
| 5010 | fflush(stdout); | ||
| 5011 | return nFail == 0 ? 0 : 1; | ||
| 5012 | } | ||
| 5013 | |||
| 5014 | int RunPoolStratumSelfTest() | ||
| 5015 | { | ||
| 5016 | int nFail = 0; | ||
| 5017 | auto check = [&](bool ok, const char* msg) { | ||
| 5018 | fprintf(stdout, " %s %s\n", ok ? "ok" : "FAIL", msg); | ||
| 5019 | if (!ok) nFail++; | ||
| 5020 | }; | ||
| 5021 | |||
| 5022 | fprintf(stdout, "pool-stratum self-test\n"); | ||
| 5023 | |||
| 5024 | StratumJob savedJob = gCurrentJob; | ||
| 5025 | bool savedHaveJob = gHaveJob; | ||
| 5026 | |||
| 5027 | CTransaction coinbase; | ||
| 5028 | coinbase.vin.resize(1); | ||
| 5029 | coinbase.vin[0].prevout.SetNull(); | ||
| 5030 | coinbase.vin[0].scriptSig << 1 << (CBigNum)1; | ||
| 5031 | coinbase.vout.resize(1); | ||
| 5032 | coinbase.vout[0].nValue = 50 * COIN; | ||
| 5033 | |||
| 5034 | CBlock block; | ||
| 5035 | block.vtx.push_back(coinbase); | ||
| 5036 | block.nVersion = 1; | ||
| 5037 | block.nBits = 0x1f0fffff; | ||
| 5038 | block.nTime = 1; | ||
| 5039 | block.nNonce = 1; | ||
| 5040 | block.hashMerkleRoot = block.BuildMerkleTree(); | ||
| 5041 | |||
| 5042 | gCurrentJob.jobId = "selftest"; | ||
| 5043 | gCurrentJob.block = block; | ||
| 5044 | gCurrentJob.target = 0; // accepted share, not a block candidate | ||
| 5045 | gCurrentJob.height = 1; | ||
| 5046 | gHaveJob = true; | ||
| 5047 | |||
| 5048 | Miner miner(INVALID_SOCKET); | ||
| 5049 | miner.authorised = true; | ||
| 5050 | miner.address = "selftest-address"; | ||
| 5051 | miner.worker = "worker"; | ||
| 5052 | miner.shareTarget = ~uint256(0); // every RandomX hash is a valid share | ||
| 5053 | miner.difficulty = 0.001; | ||
| 5054 | miner.vardiffWindowStart = GetTime(); | ||
| 5055 | |||
| 5056 | std::map<std::string, uint64> roundShareCount; | ||
| 5057 | uint64 roundShareTotal = 0; | ||
| 5058 | int blocksFound = 0; | ||
| 5059 | |||
| 5060 | uint256 targetFromNotify; | ||
| 5061 | CBlock minerBlock = BuildMinerBlock(gCurrentJob, miner.extranonce2); | ||
| 5062 | json params = MakeNotifyParams(gCurrentJob, minerBlock, miner.shareTarget, true); | ||
| 5063 | std::vector<unsigned char> targetBytes = FromHex(params[3].get<std::string>()); | ||
| 5064 | if (targetBytes.size() == 32) | ||
| 5065 | memcpy(&targetFromNotify, &targetBytes[0], 32); | ||
| 5066 | check(targetFromNotify == miner.shareTarget, | ||
| 5067 | "mining.notify exposes the same share target the pool validates"); | ||
| 5068 | |||
| 5069 | json submit = { | ||
| 5070 | {"id", 7}, | ||
| 5071 | {"method", "mining.submit"}, | ||
| 5072 | {"params", json::array({miner.address, "selftest", ToHex(&minerBlock.nNonce, 4)})} | ||
| 5073 | }; | ||
| 5074 | HandleLine(&miner, submit.dump(), roundShareCount, roundShareTotal, blocksFound); | ||
| 5075 | |||
| 5076 | check(roundShareTotal == 1, "fake low-difficulty share increments the round total"); | ||
| 5077 | check(roundShareCount[miner.address] == 1, "fake share is credited to the submitting address"); | ||
| 5078 | check(miner.sessionShares == 1 && miner.roundShares == 1 && miner.totalShares == 1, | ||
| 5079 | "miner share counters advance together"); | ||
| 5080 | check(blocksFound == 0, "share below block target is not counted as a found block"); | ||
| 5081 | |||
| 5082 | json stale = { | ||
| 5083 | {"id", 8}, | ||
| 5084 | {"method", "mining.submit"}, | ||
| 5085 | {"params", json::array({miner.address, "stale", ToHex(&minerBlock.nNonce, 4)})} | ||
| 5086 | }; | ||
| 5087 | HandleLine(&miner, stale.dump(), roundShareCount, roundShareTotal, blocksFound); | ||
| 5088 | check(roundShareTotal == 1, "stale submit does not increment shares"); | ||
| 5089 | |||
| 5090 | std::vector<PoolRoundProof> savedRounds = gPoolRounds; | ||
| 5091 | std::string savedRoundsFile = strPoolRoundsFile; | ||
| 5092 | std::string proofPath = "pool_rounds_selftest.json"; | ||
| 5093 | remove(proofPath.c_str()); | ||
| 5094 | remove((proofPath + ".tmp").c_str()); | ||
| 5095 | gPoolRounds.clear(); | ||
| 5096 | strPoolRoundsFile = proofPath; | ||
| 5097 | |||
| 5098 | PoolRoundProof proof; | ||
| 5099 | proof.id = "selftest-round"; | ||
| 5100 | proof.blockHeight = 42; | ||
| 5101 | proof.matureAtHeight = 42 + COINBASE_MATURITY; | ||
| 5102 | proof.createdAt = GetTime(); | ||
| 5103 | proof.blockHash = block.GetHash().GetHex(); | ||
| 5104 | proof.coinbaseTxid = coinbase.GetHash().GetHex(); | ||
| 5105 | proof.foundBy = miner.address; | ||
| 5106 | proof.feePercent = 1.25; | ||
| 5107 | proof.totalShares = 3; | ||
| 5108 | proof.coinbaseValue = 50 * COIN; | ||
| 5109 | proof.subsidy = 50 * COIN; | ||
| 5110 | proof.minerReward = 49 * COIN; | ||
| 5111 | proof.operatorReward = 1 * COIN; | ||
| 5112 | proof.shares[miner.address] = 3; | ||
| 5113 | proof.payouts[miner.address] = 49 * COIN; | ||
| 5114 | RecordPoolRound(proof); | ||
| 5115 | MarkPoolRoundPaid(proof.id, miner.address, "selftest-txid"); | ||
| 5116 | |||
| 5117 | bool proofOk = false; | ||
| 5118 | FILE* pf = fopen(proofPath.c_str(), "r"); | ||
| 5119 | if (pf) { | ||
| 5120 | fseek(pf, 0, SEEK_END); | ||
| 5121 | long sz = ftell(pf); | ||
| 5122 | fseek(pf, 0, SEEK_SET); | ||
| 5123 | std::string s(sz, '\0'); | ||
| 5124 | fread(&s[0], 1, sz, pf); | ||
| 5125 | fclose(pf); | ||
| 5126 | try { | ||
| 5127 | json root = json::parse(s); | ||
| 5128 | proofOk = | ||
| 5129 | root.value("schema", "") == "bitflash-pool-rounds-1" && | ||
| 5130 | root["rounds"].is_array() && | ||
| 5131 | root["rounds"].size() == 1 && | ||
| 5132 | root["rounds"][0].value("id", "") == proof.id && | ||
| 5133 | root["rounds"][0]["shares"].value(miner.address, (uint64)0) == 3 && | ||
| 5134 | root["rounds"][0]["payouts"][miner.address].value("txid", "") == "selftest-txid"; | ||
| 5135 | } catch (...) { | ||
| 5136 | proofOk = false; | ||
| 5137 | } | ||
| 5138 | } | ||
| 5139 | check(proofOk, "pool round proof ledger is written and paid txid is recorded"); | ||
| 5140 | |||
| 5141 | remove(proofPath.c_str()); | ||
| 5142 | remove((proofPath + ".tmp").c_str()); | ||
| 5143 | strPoolRoundsFile = savedRoundsFile; | ||
| 5144 | gPoolRounds = savedRounds; | ||
| 5145 | |||
| 5146 | // -- hashrate decay (#7) -------------------------------------------------- | ||
| 5147 | // The first version of this decayed m->hashRate in place from the event | ||
| 5148 | // loop's sweep, which made the reported figure a function of how the loop | ||
| 5149 | // happened to run rather than of how long the miner had been quiet. These | ||
| 5150 | // pin the properties that fixes: sampling changes nothing, and the answer | ||
| 5151 | // depends only on elapsed silence. | ||
| 5152 | { | ||
| 5153 | // A miner exactly on the vardiff target: 4096 hashes per share at | ||
| 5154 | // difficulty 1.0, one share per 15s, so 4096/15 H/s. Its expected | ||
| 5155 | // interval is 15s and its grace is 60s. | ||
| 5156 | const double base = 4096.0 / 15.0; | ||
| 5157 | const int64 t0 = 1766000000; // any plausible non-zero wall clock | ||
| 5158 | Miner decayed(INVALID_SOCKET); | ||
| 5159 | decayed.difficulty = 1.0; | ||
| 5160 | decayed.hashRate = base; | ||
| 5161 | decayed.lastShareTime = t0; | ||
| 5162 | |||
| 5163 | check(MinerReportedHashRate(&decayed, t0 + 60) == base, | ||
| 5164 | "hashrate decay: a miner inside its grace period reports its full estimate"); | ||
| 5165 | |||
| 5166 | // Sampling must not consume the estimate. Twenty passes is one second | ||
| 5167 | // of the event loop's old sweep; a thousand is nearly a minute of it. | ||
| 5168 | double once = MinerReportedHashRate(&decayed, t0 + 61); | ||
| 5169 | bool idempotent = (decayed.hashRate == base); | ||
| 5170 | for (int i = 0; i < 1000 && idempotent; i++) | ||
| 5171 | idempotent = (MinerReportedHashRate(&decayed, t0 + 61) == once) && | ||
| 5172 | (decayed.hashRate == base); | ||
| 5173 | check(idempotent, | ||
| 5174 | "hashrate decay: repeated sampling returns one value and leaves the estimate alone"); | ||
| 5175 | |||
| 5176 | // One second past a 60s grace is 1/15th of a half-life: ~4.5% off, not | ||
| 5177 | // the estimate gone. A miner at the vardiff target overshoots its grace | ||
| 5178 | // roughly 1.8% of the time (e^-4) purely by share-timing variance, and | ||
| 5179 | // must not be zeroed for it. | ||
| 5180 | check(once > 0.94 * base && once < 0.96 * base, | ||
| 5181 | "hashrate decay: one second past grace costs a few percent, not the estimate"); | ||
| 5182 | double three = MinerReportedHashRate(&decayed, t0 + 63); | ||
| 5183 | check(three > 0.86 * base && three < 0.88 * base, | ||
| 5184 | "hashrate decay: three seconds past grace costs ~13%"); | ||
| 5185 | |||
| 5186 | // One half-life past the grace is exactly half. | ||
| 5187 | double half = MinerReportedHashRate(&decayed, t0 + 75); | ||
| 5188 | check(half > 0.499 * base && half < 0.501 * base, | ||
| 5189 | "hashrate decay: one expected interval past grace halves the estimate"); | ||
| 5190 | |||
| 5191 | // A miner that has genuinely stopped ends up reported as nothing -- | ||
| 5192 | // the whole point of #7. The old in-place version could not get here: | ||
| 5193 | // its decay fed back into its own grace and settled into a 1/t crawl. | ||
| 5194 | check(MinerReportedHashRate(&decayed, t0 + 3600) == 0.0, | ||
| 5195 | "hashrate decay: a miner silent for an hour is reported as zero"); | ||
| 5196 | |||
| 5197 | // A clock that steps backwards must never inflate what we announce. | ||
| 5198 | check(MinerReportedHashRate(&decayed, t0 - 3600) == base, | ||
| 5199 | "hashrate decay: a backwards clock reports the estimate unchanged, never higher"); | ||
| 5200 | |||
| 5201 | // Fresh connection, and a connection with one share and so no interval | ||
| 5202 | // to have timed: both must report nothing rather than divide by zero. | ||
| 5203 | Miner fresh(INVALID_SOCKET); | ||
| 5204 | check(MinerReportedHashRate(&fresh, t0) == 0.0, | ||
| 5205 | "hashrate decay: a miner that never submitted a share reports zero"); | ||
| 5206 | Miner single(INVALID_SOCKET); | ||
| 5207 | single.lastShareTime = t0; | ||
| 5208 | check(MinerReportedHashRate(&single, t0 + 86400) == 0.0, | ||
| 5209 | "hashrate decay: a miner with no rate sample yet reports zero"); | ||
| 5210 | |||
| 5211 | // Nothing in the swept range may produce a NaN, an infinity, a negative | ||
| 5212 | // rate, or a rate above the estimate it started from. | ||
| 5213 | bool finite = true; | ||
| 5214 | for (int64 s = -100; s < 200000 && finite; s += 37) { | ||
| 5215 | double v = MinerReportedHashRate(&decayed, t0 + s); | ||
| 5216 | finite = !std::isnan(v) && !std::isinf(v) && v >= 0.0 && v <= base; | ||
| 5217 | } | ||
| 5218 | check(finite, | ||
| 5219 | "hashrate decay: stays finite, non-negative and non-amplifying across a long silence"); | ||
| 5220 | } | ||
| 5221 | |||
| 5222 | gCurrentJob = savedJob; | ||
| 5223 | gHaveJob = savedHaveJob; | ||
| 5224 | |||
| 5225 | fprintf(stdout, "%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED", | ||
| 5226 | nFail, nFail == 1 ? "" : "s"); | ||
| 5227 | fflush(stdout); | ||
| 5228 | return nFail == 0 ? 0 : 1; | ||
| 5229 | } | ||
| 5230 | |||
| 5231 | // --------------------------------------------------------------------------- | ||
| 5232 | // Stats refresh -- called every 2s from event loop | ||
| 5233 | // --------------------------------------------------------------------------- | ||
| 5234 | static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal) | ||
| 5235 | { | ||
| 5236 | std::string statusBlob; | ||
| 5237 | bool fPublish = false; | ||
| 5238 | { | ||
| 5239 | std::lock_guard<std::mutex> lk(gStatsMutex); | ||
| 5240 | |||
| 5241 | gStats.authorizedMiners = 0; | ||
| 5242 | gStats.blocksFound = blocksFoundThisSession; | ||
| 5243 | gStats.roundShares = roundShareTotal; | ||
| 5244 | gStats.totalHashRate = 0.0; | ||
| 5245 | for (Miner* m : gMiners) | ||
| 5246 | if (m->authorised) gStats.authorizedMiners++; | ||
| 5247 | |||
| 5248 | gWorkerStats.clear(); | ||
| 5249 | // One timestamp for the whole sweep (#7): every miner is faded to the | ||
| 5250 | // same instant, so the per-worker rows and the total they are summed | ||
| 5251 | // into describe one moment rather than a smear across the loop. | ||
| 5252 | int64 nowStats = GetTime(); | ||
| 5253 | for (Miner* m : gMiners) { | ||
| 5254 | if (!m->authorised || m->lastSeen <= 0) continue; | ||
| 5255 | PoolWorkerStatView v; | ||
| 5256 | v.address = m->address; | ||
| 5257 | v.worker = m->worker; | ||
| 5258 | v.roundShares = m->roundShares; | ||
| 5259 | v.totalShares = m->totalShares; | ||
| 5260 | v.lastSeen = m->lastSeen; | ||
| 5261 | // Not m->hashRate: a miner that has stopped submitting shares must | ||
| 5262 | // not keep contributing its last sample to the announced total. | ||
| 5263 | v.hashRate = MinerReportedHashRate(m, nowStats); | ||
| 5264 | gWorkerStats.push_back(v); | ||
| 5265 | gStats.totalHashRate += v.hashRate; | ||
| 5266 | } | ||
| 5267 | std::sort(gWorkerStats.begin(), gWorkerStats.end(), | ||
| 5268 | [](const PoolWorkerStatView& a, const PoolWorkerStatView& b) { | ||
| 5269 | return a.roundShares != b.roundShares ? a.roundShares > b.roundShares | ||
| 5270 | : a.lastSeen > b.lastSeen; | ||
| 5271 | }); | ||
| 5272 | |||
| 5273 | // Scoped, not held to the end of the function. At function scope this | ||
| 5274 | // lock covered the status-file write below, so the event loop did two | ||
| 5275 | // fsyncs holding the mutex the payout thread takes to record that a | ||
| 5276 | // miner has been paid. | ||
| 5277 | gPayoutViews.clear(); | ||
| 5278 | { | ||
| 5279 | std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex); | ||
| 5280 | for (const PendingPayout& pp : gPendingPayouts) { | ||
| 5281 | PendingPayoutView pv; | ||
| 5282 | pv.matureAtHeight = pp.matureAtHeight; | ||
| 5283 | pv.recipients = 0; | ||
| 5284 | pv.totalAmount = 0; | ||
| 5285 | for (auto& kv : pp.amounts) { | ||
| 5286 | if (pp.paidTxids.count(kv.first)) | ||
| 5287 | continue; | ||
| 5288 | pv.recipients++; | ||
| 5289 | pv.totalAmount += kv.second; | ||
| 5290 | } | ||
| 5291 | if (pv.recipients > 0) | ||
| 5292 | gPayoutViews.push_back(pv); | ||
| 5293 | } | ||
| 5294 | } | ||
| 5295 | |||
| 5296 | int64 now = GetTime(); | ||
| 5297 | if (now - gLastPoolStatusWrite >= 10) { | ||
| 5298 | statusBlob = SerializePoolStatusLocked(); | ||
| 5299 | gLastPoolStatusWrite = now; | ||
| 5300 | fPublish = true; | ||
| 5301 | } | ||
| 5302 | } | ||
| 5303 | // Every payout mutex is released here, and gStatsMutex with them. This is | ||
| 5304 | // where the two fsyncs happen, and no other thread is waiting behind them. | ||
| 5305 | if (fPublish) | ||
| 5306 | PublishPoolStatus(statusBlob); | ||
| 1526 | } | 5307 | } |
| 1527 | 5308 | ||
| 1528 | // --------------------------------------------------------------------------- | 5309 | // --------------------------------------------------------------------------- |
| @@ -1533,6 +5314,7 @@ static void PoolEventLoop() | |||
| 1533 | int lastHeight = -1; | 5314 | int lastHeight = -1; |
| 1534 | int64 lastRefresh = GetTime(); | 5315 | int64 lastRefresh = GetTime(); |
| 1535 | int64 lastStatsRefresh = 0; | 5316 | int64 lastStatsRefresh = 0; |
| 5317 | int64 lastCensus = 0; // #19 connection census throttle | ||
| 1536 | int blocksFoundThisSession = 0; | 5318 | int blocksFoundThisSession = 0; |
| 1537 | int64 noJobSince = 0; // 0 = have a job | 5319 | int64 noJobSince = 0; // 0 = have a job |
| 1538 | bool warnedNoJob = false; | 5320 | bool warnedNoJob = false; |
| @@ -1561,6 +5343,31 @@ static void PoolEventLoop() | |||
| 1561 | sock_close(fd); | 5343 | sock_close(fd); |
| 1562 | continue; | 5344 | continue; |
| 1563 | } | 5345 | } |
| 5346 | // #26: select() + FD_SET goes undefined once we exceed what an | ||
| 5347 | // fd_set can carry, and MAX_POOL_MINERS (1000) alone does not | ||
| 5348 | // prevent it -- a Miner is only removed on clean disconnect, so | ||
| 5349 | // half-open zombies accumulate and the OS can hand out fds past | ||
| 5350 | // the limit long before the count reaches 1000. Refuse rather | ||
| 5351 | // than corrupt the fd_set. Which limit applies is platform | ||
| 5352 | // specific: POSIX bounds the descriptor value, Windows the | ||
| 5353 | // number of SOCKET handles an fd_set can hold. | ||
| 5354 | #ifndef _WIN32 | ||
| 5355 | if ((int)fd >= (int)FD_SETSIZE) { | ||
| 5356 | LogPrint("worker", "[pool] refusing miner fd=%d -- at/over" | ||
| 5357 | " FD_SETSIZE=%d, select() would be undefined (#26)\n", | ||
| 5358 | (int)fd, (int)FD_SETSIZE); | ||
| 5359 | sock_close(fd); | ||
| 5360 | continue; | ||
| 5361 | } | ||
| 5362 | #else | ||
| 5363 | if (gMiners.size() >= (size_t)FD_SETSIZE) { | ||
| 5364 | LogPrint("worker", "[pool] refusing miner -- %zu sockets held," | ||
| 5365 | " an fd_set carries at most FD_SETSIZE=%d (#26)\n", | ||
| 5366 | gMiners.size(), (int)FD_SETSIZE); | ||
| 5367 | sock_close(fd); | ||
| 5368 | continue; | ||
| 5369 | } | ||
| 5370 | #endif | ||
| 1564 | gMiners.push_back(new Miner(fd)); | 5371 | gMiners.push_back(new Miner(fd)); |
| 1565 | LogPrint("worker", "[pool] miner fd=%d entered event loop" | 5372 | LogPrint("worker", "[pool] miner fd=%d entered event loop" |
| 1566 | " -- awaiting subscribe/authorize\n", (int)fd); | 5373 | " -- awaiting subscribe/authorize\n", (int)fd); |
| @@ -1688,6 +5495,35 @@ static void PoolEventLoop() | |||
| 1688 | } | 5495 | } |
| 1689 | } | 5496 | } |
| 1690 | 5497 | ||
| 5498 | // 3.6. Idle reap (#6). Mirror the clean-disconnect path exactly -- | ||
| 5499 | // log, close, erase from gMiners via std::find, delete -- just with | ||
| 5500 | // a different reason. Shares the miner already earned stay in | ||
| 5501 | // roundShareCount, exactly as they do when a miner cleanly | ||
| 5502 | // disconnects mid-round. connectTime stands in for lastSeen on a | ||
| 5503 | // connection that never completed a line. Erasing inside the loop | ||
| 5504 | // above would invalidate the iteration, so collect then reap. | ||
| 5505 | { | ||
| 5506 | int64 now = GetTime(); | ||
| 5507 | std::vector<Miner*> idle; | ||
| 5508 | for (Miner* m : gMiners) { | ||
| 5509 | int64 last = (m->lastSeen > 0) ? m->lastSeen : m->connectTime; | ||
| 5510 | if (now - last > POOL_IDLE_TIMEOUT_SECS) | ||
| 5511 | idle.push_back(m); | ||
| 5512 | } | ||
| 5513 | for (Miner* m : idle) { | ||
| 5514 | LogPrint("worker", "[pool] dropping idle miner %s -- nothing" | ||
| 5515 | " received in over %llds (session shares: %llu," | ||
| 5516 | " round shares: %llu)\n", | ||
| 5517 | m->worker.empty() ? "(never authorized)" : m->worker.c_str(), | ||
| 5518 | (long long)POOL_IDLE_TIMEOUT_SECS, | ||
| 5519 | (unsigned long long)m->sessionShares, | ||
| 5520 | (unsigned long long)m->roundShares); | ||
| 5521 | sock_close(m->fd); | ||
| 5522 | gMiners.erase(std::find(gMiners.begin(), gMiners.end(), m)); | ||
| 5523 | delete m; | ||
| 5524 | } | ||
| 5525 | } | ||
| 5526 | |||
| 1691 | // 4. Select over miner sockets | 5527 | // 4. Select over miner sockets |
| 1692 | if (!gMiners.empty()) { | 5528 | if (!gMiners.empty()) { |
| 1693 | fd_set fds; FD_ZERO(&fds); | 5529 | fd_set fds; FD_ZERO(&fds); |
| @@ -1710,6 +5546,11 @@ static void PoolEventLoop() | |||
| 1710 | // terminate the whole node -- not just the pool. Every | 5546 | // terminate the whole node -- not just the pool. Every |
| 1711 | // line arrives from an unauthenticated socket. | 5547 | // line arrives from an unauthenticated socket. |
| 1712 | while (RecvLine(m->fd, m->readBuf, line, disc)) { | 5548 | while (RecvLine(m->fd, m->readBuf, line, disc)) { |
| 5549 | // Any complete line is proof of life (#6): authorize | ||
| 5550 | // and valid shares already stamp lastSeen themselves, | ||
| 5551 | // but a miner that only ever subscribes (or sends | ||
| 5552 | // unrecognised methods) must count as alive too. | ||
| 5553 | m->lastSeen = GetTime(); | ||
| 1713 | try { | 5554 | try { |
| 1714 | HandleLine(m, line, roundShareCount, | 5555 | HandleLine(m, line, roundShareCount, |
| 1715 | roundShareTotal, blocksFoundThisSession); | 5556 | roundShareTotal, blocksFoundThisSession); |
| @@ -1750,6 +5591,14 @@ static void PoolEventLoop() | |||
| 1750 | RefreshStats(blocksFoundThisSession, roundShareTotal); | 5591 | RefreshStats(blocksFoundThisSession, roundShareTotal); |
| 1751 | lastStatsRefresh = GetTime(); | 5592 | lastStatsRefresh = GetTime(); |
| 1752 | } | 5593 | } |
| 5594 | |||
| 5595 | // 6. #19: log the connection census periodically. Cheap (one pass over | ||
| 5596 | // gMiners) and it is the single number an operator reads to judge pool | ||
| 5597 | // health, so it should not be an overstatement. | ||
| 5598 | if (GetTime() - lastCensus >= POOL_CENSUS_SECS) { | ||
| 5599 | LogPoolConnectionCensus(); | ||
| 5600 | lastCensus = GetTime(); | ||
| 5601 | } | ||
| 1753 | } | 5602 | } |
| 1754 | 5603 | ||
| 1755 | for (Miner* m : gMiners) { sock_close(m->fd); delete m; } | 5604 | for (Miner* m : gMiners) { sock_close(m->fd); delete m; } |
| @@ -1778,13 +5627,19 @@ static void AcceptOneFn(void* arg) | |||
| 1778 | std::string host = relay.substr(0, colon); | 5627 | std::string host = relay.substr(0, colon); |
| 1779 | int port = atoi(relay.substr(colon + 1).c_str()); | 5628 | int port = atoi(relay.substr(colon + 1).c_str()); |
| 1780 | 5629 | ||
| 1781 | LogPrint("pool", "[pool] accept thread started for relay %s\n", relay.c_str()); | 5630 | LogPrint("pool", "[pool] accept thread started for relay %s%s\n", relay.c_str(), |
| 5631 | fBtfRelayPinned ? " (/rvrelay pin)" : ""); | ||
| 5632 | |||
| 5633 | // Registration health for this relay. A pin has no fallback, so a pin that | ||
| 5634 | // never registers means the pool is simply unreachable -- and that has to | ||
| 5635 | // be loud. The 2026-07-30 incident went unnoticed because this loop logged | ||
| 5636 | // failures at one flat line and success not at all, so "which relay is this | ||
| 5637 | // pool actually registered at" could not be answered from the log. | ||
| 5638 | int nConsecFail = 0; | ||
| 5639 | int64 nFirstFail = 0; | ||
| 5640 | bool fEverRegistered = false; | ||
| 1782 | 5641 | ||
| 1783 | while (!fShutdown && gPoolRunning) { | 5642 | while (!fShutdown && gPoolRunning) { |
| 1784 | // Advertise this relay as the pool's meeting node BEFORE registering, | ||
| 1785 | // so that miners which resolve our descriptor find a relay where the | ||
| 1786 | // pool (not the node's peer-accept) holds the registration. | ||
| 1787 | BtfSetActiveRelay(relay); | ||
| 1788 | // Returns as soon as the relay lists us -- registered, not yet paired. | 5643 | // Returns as soon as the relay lists us -- registered, not yet paired. |
| 1789 | btf::RvSocket rv = btf::RvServiceRegister(host.c_str(), (unsigned short)port, pk); | 5644 | btf::RvSocket rv = btf::RvServiceRegister(host.c_str(), (unsigned short)port, pk); |
| 1790 | 5645 | ||
| @@ -1793,12 +5648,36 @@ static void AcceptOneFn(void* arg) | |||
| 1793 | break; | 5648 | break; |
| 1794 | } | 5649 | } |
| 1795 | if (rv == btf::RV_INVALID) { | 5650 | if (rv == btf::RV_INVALID) { |
| 1796 | LogPrint("pool", "[pool] relay %s: RvServiceRegister failed" | 5651 | nConsecFail++; |
| 1797 | " -- backing off 30s before retry\n", relay.c_str()); | 5652 | if (nFirstFail == 0) nFirstFail = GetTime(); |
| 5653 | LogPrint("pool", "[pool] relay %s: registration failed (attempt %d," | ||
| 5654 | " failing for %llds) -- backing off 30s before retry\n", | ||
| 5655 | relay.c_str(), nConsecFail, (long long)(GetTime() - nFirstFail)); | ||
| 5656 | if (fBtfRelayPinned && (nConsecFail == 3 || nConsecFail % 10 == 0)) | ||
| 5657 | LogPrint("pool", "[pool] WARNING: pinned relay %s refused registration" | ||
| 5658 | " %d times running (%llds); /rvrelay leaves no fallback, so" | ||
| 5659 | " miners cannot reach this pool%s\n", | ||
| 5660 | relay.c_str(), nConsecFail, (long long)(GetTime() - nFirstFail), | ||
| 5661 | fEverRegistered ? "" : " -- it has never registered once"); | ||
| 1798 | for (int i = 0; i < 30 && !fShutdown && gPoolRunning; i++) Sleep(1000); | 5662 | for (int i = 0; i < 30 && !fShutdown && gPoolRunning; i++) Sleep(1000); |
| 1799 | continue; | 5663 | continue; |
| 1800 | } | 5664 | } |
| 1801 | 5665 | ||
| 5666 | // Advertise THIS relay only now. Setting it before the call published a | ||
| 5667 | // meeting node we were not yet listed at, so a miner resolving the | ||
| 5668 | // descriptor inside that window burned its dial on a relay holding no | ||
| 5669 | // registration for us. net.cpp's peer-accept has always ordered it this | ||
| 5670 | // way round; the pool had not. | ||
| 5671 | BtfSetActiveRelay(relay); | ||
| 5672 | if (!fEverRegistered || nConsecFail > 0) | ||
| 5673 | LogPrint("pool", "[pool] registered at relay %s%s%s -- advertising it as" | ||
| 5674 | " the pool's meeting node\n", relay.c_str(), | ||
| 5675 | fBtfRelayPinned ? " (/rvrelay pin)" : "", | ||
| 5676 | nConsecFail > 0 ? ", recovered" : ""); | ||
| 5677 | fEverRegistered = true; | ||
| 5678 | nConsecFail = 0; | ||
| 5679 | nFirstFail = 0; | ||
| 5680 | |||
| 1802 | // Block for a miner to actually arrive. RvServiceRegister used to do | 5681 | // Block for a miner to actually arrive. RvServiceRegister used to do |
| 1803 | // this itself and returned with the relay's 0x01 pairing byte already | 5682 | // this itself and returned with the relay's 0x01 pairing byte already |
| 1804 | // consumed; it no longer does. Without waiting here, BtfServiceWrap | 5683 | // consumed; it no longer does. Without waiting here, BtfServiceWrap |
| @@ -1853,9 +5732,21 @@ static void AcceptLoopFn(void*) | |||
| 1853 | LogPrint("pool", "[pool] .btf identity ready after %ds\n", waitSecs * 2); | 5732 | LogPrint("pool", "[pool] .btf identity ready after %ds\n", waitSecs * 2); |
| 1854 | 5733 | ||
| 1855 | std::set<std::string> spawned; // relays we've already started a thread for | 5734 | std::set<std::string> spawned; // relays we've already started a thread for |
| 5735 | std::set<std::string> skipped; // discovered relays the pin excluded, logged once each | ||
| 1856 | 5736 | ||
| 1857 | while (!fShutdown && gPoolRunning) { | 5737 | while (!fShutdown && gPoolRunning) { |
| 5738 | // BtfAllRelays() already honours the pin: with /rvrelay set it returns | ||
| 5739 | // vBtfMeetingRelays alone, so no thread is ever spawned for a | ||
| 5740 | // discovered relay. Name the ones it drops -- a pool registering at a | ||
| 5741 | // stranger's relay is exactly the thing that should not be silent. | ||
| 1858 | vector<string> relays = BtfAllRelays(); | 5742 | vector<string> relays = BtfAllRelays(); |
| 5743 | if (fBtfRelayPinned) { | ||
| 5744 | for (const string& r : BtfDiscoveredRelays()) | ||
| 5745 | if (std::find(relays.begin(), relays.end(), r) == relays.end() && | ||
| 5746 | skipped.insert(r).second) | ||
| 5747 | LogPrint("pool", "[pool] not registering at discovered relay %s" | ||
| 5748 | " -- /rvrelay pins the relay list\n", r.c_str()); | ||
| 5749 | } | ||
| 1859 | if (relays.empty()) { | 5750 | if (relays.empty()) { |
| 1860 | LogPrint("pool", "[pool] no relays available -- miners cannot connect yet\n"); | 5751 | LogPrint("pool", "[pool] no relays available -- miners cannot connect yet\n"); |
| 1861 | } else { | 5752 | } else { |
| @@ -1871,8 +5762,9 @@ static void AcceptLoopFn(void*) | |||
| 1871 | relay.c_str()); | 5762 | relay.c_str()); |
| 1872 | } else { | 5763 | } else { |
| 1873 | spawned.insert(relay); | 5764 | spawned.insert(relay); |
| 1874 | LogPrint("pool", "[pool] accept thread spawned for relay %s" | 5765 | LogPrint("pool", "[pool] accept thread spawned for relay %s%s" |
| 1875 | " (%zu total)\n", relay.c_str(), spawned.size()); | 5766 | " (%zu total)\n", relay.c_str(), |
| 5767 | fBtfRelayPinned ? " (/rvrelay pin)" : "", spawned.size()); | ||
| 1876 | } | 5768 | } |
| 1877 | } | 5769 | } |
| 1878 | } | 5770 | } |
| @@ -1901,16 +5793,24 @@ void ThreadRPCServer(void*) | |||
| 1901 | } | 5793 | } |
| 1902 | 5794 | ||
| 1903 | LoadPendingPayouts(); | 5795 | LoadPendingPayouts(); |
| 5796 | LoadPoolRounds(); | ||
| 1904 | LoadPaying(); | 5797 | LoadPaying(); |
| 1905 | // Must run before the payout thread starts, so nothing is resent that the | 5798 | // Must run before the payout thread starts, so nothing is resent that the |
| 1906 | // wallet already shows as sent, and nothing is owed twice because we died | 5799 | // ledger already shows as sent, and every restored in-flight record is |
| 1907 | // between recording the payments and rewriting the ledger. | 5800 | // armed for its one wallet probe before it is allowed to send. |
| 1908 | ReconcilePayouts(); | 5801 | ReconcilePayouts(); |
| 1909 | 5802 | ||
| 1910 | _beginthread(AcceptLoopFn, 0, NULL); | 5803 | // _beginthread cannot throw (it reports -1), but an unchecked failure |
| 1911 | _beginthread(LocalAcceptFn, 0, NULL); | 5804 | // leaves the pool running deaf: nobody accepts, announces, or pays, and |
| 1912 | _beginthread(ThreadBtfPoolAnnouncer, 0, NULL); | 5805 | // nothing in the log says why. |
| 1913 | _beginthread(PayoutThreadFn, 0, NULL); | 5806 | if (_beginthread(AcceptLoopFn, 0, NULL) == (uintptr_t)-1) |
| 5807 | LogPrint("pool", "[pool] FATAL: accept loop thread failed to spawn -- miners cannot connect\n"); | ||
| 5808 | if (_beginthread(LocalAcceptFn, 0, NULL) == (uintptr_t)-1) | ||
| 5809 | LogPrint("pool", "[pool] local accept thread failed to spawn\n"); | ||
| 5810 | if (_beginthread(ThreadBtfPoolAnnouncer, 0, NULL) == (uintptr_t)-1) | ||
| 5811 | LogPrint("pool", "[pool] announcer thread failed to spawn -- pool will not be advertised\n"); | ||
| 5812 | if (_beginthread(PayoutThreadFn, 0, NULL) == (uintptr_t)-1) | ||
| 5813 | LogPrint("pool", "[pool] FATAL: payout thread failed to spawn -- no one gets paid\n"); | ||
| 1914 | 5814 | ||
| 1915 | PoolEventLoop(); // blocks until gPoolRunning = false or fShutdown | 5815 | PoolEventLoop(); // blocks until gPoolRunning = false or fShutdown |
| 1916 | 5816 | ||