diff options
| author | bitflash-maintainer <noreply@bitflash.invalid> | 2026-08-09 13:09:01 +0000 |
|---|---|---|
| committer | bitflash-maintainer <noreply@bitflash.invalid> | 2026-08-19 22:48:26 +0000 |
| commit | ec1627c68d3b6abdbf84b2a816d98bdd4ce113c9 (patch) | |
| tree | 3b06f1828c109ea20de183609a966c94bc4ae28f /src | |
| parent | b75ddafb68ba60902a96b1f11f722b991f21d549 (diff) | |
payout: make in-flight payments survive a restart
The exec queue was memory-only, and FlushMaturePayouts erased a matured entry
from pending_payouts.json *before* handing it over. Any restart in that window
lost those payments with no record that anyone was owed anything. It was not
hypothetical: a container restart on 2026-07-30 silently dropped a backlog of
~1,156 BTF, and there was nothing left to reconstruct it from.
Re-queueing blindly would have double-paid whoever had already been sent to,
so the payment needs to be idempotent rather than merely persistent.
In-flight work is now a persisted list of per-payment records
(paying_payouts.json), one per (block, address) -- the granularity SendMoney
works at, and therefore the granularity at which a payment can be half-done
when the process dies. Records are written, via a temp file and rename so a
crash cannot truncate them, *before* the ledger entry is erased, so the debt
is always described by at least one of the two files. Each attempt updates the
record (retries, deferrals, backoff); settling it removes the record and
rewrites the file, which is what stops a restart from paying twice.
On startup ReconcilePayouts runs before the payout thread:
- a record whose payment the wallet already shows -- matched on the payout
comment plus the exact amount and destination -- is dropped, not resent.
SendMoney commits the transaction to the wallet before it returns, so a
crash immediately after sending is recoverable.
- a ledger entry that has already been exploded into records is dropped,
which is the crash-in-the-window case where both files describe one debt.
The wallet check is deliberately biased towards "already paid": skipping a
payment is visible in the log and recoverable, paying twice is neither.
Verified live: sampling paying_payouts.json during a payout burst caught 3
records / 18.56 BTF mid-flight, cleared once paid, and the paid lines now
carry the txid.
(cherry picked from commit d0248f2e65dbe745f2e9b1d09e1f53a4354d3640)
Diffstat (limited to 'src')
| -rw-r--r-- | src/rpc.cpp | 462 |
1 files changed, 338 insertions, 124 deletions
diff --git a/src/rpc.cpp b/src/rpc.cpp index 11020d8..ae94289 100644 --- a/src/rpc.cpp +++ b/src/rpc.cpp | |||
| @@ -315,25 +315,117 @@ static void LoadPendingPayouts() | |||
| 315 | } | 315 | } |
| 316 | } | 316 | } |
| 317 | 317 | ||
| 318 | // Payouts ready to execute are moved here under gPayoutExecMutex. | 318 | // A payment in flight: one record per (payout entry, address) pair, which is |
| 319 | // A dedicated thread drains this list so SendMoney (which takes cs_main) | 319 | // the granularity SendMoney works at and therefore the granularity at which a |
| 320 | // never runs on the event loop thread and can't stall miner I/O. | 320 | // payment can be half-done when the process dies. |
| 321 | static std::vector<PendingPayout> gPayoutExecQueue; | 321 | // |
| 322 | static std::mutex gPayoutExecMutex; | 322 | // This list is persisted. It used to be a memory-only queue, and matured |
| 323 | static std::atomic<bool> gPayoutThreadRunning{false}; | 323 | // entries were erased from pending_payouts.json *before* being handed to it, |
| 324 | // so any restart in between lost those payments with no record that anyone | ||
| 325 | // was owed anything: a container restart on 2026-07-30 silently dropped | ||
| 326 | // ~1,156 BTF that way. Writing the records before removing the ledger entry, | ||
| 327 | // and reconciling against the wallet on startup, is what makes a restart | ||
| 328 | // survivable. | ||
| 329 | struct PayingRecord { | ||
| 330 | int blockHeight = 0; | ||
| 331 | uint256 blockHash = 0; // zero = predates hash keying, see PendingPayout | ||
| 332 | int matureAtHeight = 0; | ||
| 333 | std::string address; | ||
| 334 | int64 amount = 0; | ||
| 335 | int retries = 0; | ||
| 336 | int deferrals = 0; | ||
| 337 | int64 retryAfter = 0; | ||
| 338 | }; | ||
| 339 | static std::vector<PayingRecord> gPaying; // guarded by gPayoutExecMutex | ||
| 340 | static std::mutex gPayoutExecMutex; | ||
| 341 | static std::atomic<bool> gPayoutThreadRunning{false}; | ||
| 342 | |||
| 343 | static std::string PayingFilePath() | ||
| 344 | { | ||
| 345 | return GetAppDir() + "/paying_payouts.json"; | ||
| 346 | } | ||
| 347 | |||
| 348 | // Caller must hold gPayoutExecMutex. | ||
| 349 | static void SavePayingLocked() | ||
| 350 | { | ||
| 351 | nlohmann::json arr = nlohmann::json::array(); | ||
| 352 | for (const auto& r : gPaying) { | ||
| 353 | nlohmann::json obj; | ||
| 354 | obj["blockHeight"] = r.blockHeight; | ||
| 355 | obj["blockHash"] = r.blockHash.ToString(); | ||
| 356 | obj["matureAtHeight"] = r.matureAtHeight; | ||
| 357 | obj["address"] = r.address; | ||
| 358 | obj["amount"] = r.amount; | ||
| 359 | obj["retries"] = r.retries; | ||
| 360 | obj["deferrals"] = r.deferrals; | ||
| 361 | arr.push_back(obj); | ||
| 362 | } | ||
| 363 | std::string path = PayingFilePath(); | ||
| 364 | // Write to a temporary file and rename, so a crash mid-write cannot leave | ||
| 365 | // a truncated list of who is still owed money. | ||
| 366 | std::string tmp = path + ".tmp"; | ||
| 367 | FILE* f = fopen(tmp.c_str(), "w"); | ||
| 368 | if (!f) { | ||
| 369 | LogPrint("payout", "[payout] WARNING: could not write %s\n", tmp.c_str()); | ||
| 370 | return; | ||
| 371 | } | ||
| 372 | std::string s = arr.dump(2); | ||
| 373 | fwrite(s.c_str(), 1, s.size(), f); | ||
| 374 | fflush(f); | ||
| 375 | fclose(f); | ||
| 376 | if (rename(tmp.c_str(), path.c_str()) != 0) | ||
| 377 | LogPrint("payout", "[payout] WARNING: could not rename %s -> %s\n", | ||
| 378 | tmp.c_str(), path.c_str()); | ||
| 379 | } | ||
| 380 | |||
| 381 | static void LoadPaying() | ||
| 382 | { | ||
| 383 | std::string path = PayingFilePath(); | ||
| 384 | FILE* f = fopen(path.c_str(), "r"); | ||
| 385 | if (!f) return; | ||
| 386 | fseek(f, 0, SEEK_END); | ||
| 387 | long sz = ftell(f); | ||
| 388 | fseek(f, 0, SEEK_SET); | ||
| 389 | std::string s(sz, '\0'); | ||
| 390 | fread(&s[0], 1, sz, f); | ||
| 391 | fclose(f); | ||
| 392 | try { | ||
| 393 | nlohmann::json arr = nlohmann::json::parse(s); | ||
| 394 | for (const auto& obj : arr) { | ||
| 395 | PayingRecord r; | ||
| 396 | r.address = obj["address"].get<std::string>(); | ||
| 397 | r.amount = obj["amount"].get<int64>(); | ||
| 398 | r.matureAtHeight = obj["matureAtHeight"].get<int>(); | ||
| 399 | if (obj.contains("blockHash") && obj["blockHash"].is_string()) | ||
| 400 | r.blockHash.SetHex(obj["blockHash"].get<std::string>()); | ||
| 401 | if (obj.contains("blockHeight") && obj["blockHeight"].is_number()) | ||
| 402 | r.blockHeight = obj["blockHeight"].get<int>(); | ||
| 403 | if (obj.contains("retries") && obj["retries"].is_number()) | ||
| 404 | r.retries = obj["retries"].get<int>(); | ||
| 405 | if (obj.contains("deferrals") && obj["deferrals"].is_number()) | ||
| 406 | r.deferrals = obj["deferrals"].get<int>(); | ||
| 407 | gPaying.push_back(r); | ||
| 408 | } | ||
| 409 | LogPrint("payout", "[payout] loaded %zu in-flight payment(s) from disk\n", | ||
| 410 | gPaying.size()); | ||
| 411 | } catch (const std::exception& e) { | ||
| 412 | LogPrint("payout", "[payout] WARNING: failed to parse %s: %s\n", | ||
| 413 | path.c_str(), e.what()); | ||
| 414 | } | ||
| 415 | } | ||
| 324 | 416 | ||
| 325 | // Is the block that funds this payout still on the main chain? | 417 | // Is the block that funds this payout still on the main chain? |
| 326 | // | 418 | // |
| 327 | // A zero hash means the entry was written before payouts recorded one, so | 419 | // A zero hash means the entry was written before payouts recorded one, so |
| 328 | // there is nothing to check and we assume it is good -- voiding those would | 420 | // there is nothing to check and we assume it is good -- voiding those would |
| 329 | // mean refusing to pay miners on the strength of a missing field. | 421 | // mean refusing to pay miners on the strength of a missing field. |
| 330 | static bool PayoutBlockStillOnChain(const PendingPayout& pp) | 422 | static bool PayoutBlockStillOnChain(const uint256& blockHash) |
| 331 | { | 423 | { |
| 332 | if (pp.blockHash == 0) | 424 | if (blockHash == 0) |
| 333 | return true; | 425 | return true; |
| 334 | CRITICAL_BLOCK(cs_main) | 426 | CRITICAL_BLOCK(cs_main) |
| 335 | { | 427 | { |
| 336 | std::map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(pp.blockHash); | 428 | std::map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(blockHash); |
| 337 | if (mi == mapBlockIndex.end() || mi->second == NULL) | 429 | if (mi == mapBlockIndex.end() || mi->second == NULL) |
| 338 | return false; | 430 | return false; |
| 339 | return mi->second->IsInMainChain(); | 431 | return mi->second->IsInMainChain(); |
| @@ -341,6 +433,99 @@ static bool PayoutBlockStillOnChain(const PendingPayout& pp) | |||
| 341 | return false; | 433 | return false; |
| 342 | } | 434 | } |
| 343 | 435 | ||
| 436 | static std::string PayoutComment(int blockHeight) | ||
| 437 | { | ||
| 438 | return strprintf("Pool payout height %d", blockHeight); | ||
| 439 | } | ||
| 440 | |||
| 441 | // Did we already send this payment before dying? Looks for a wallet | ||
| 442 | // transaction of ours carrying this payout's comment and paying exactly this | ||
| 443 | // amount to this address. | ||
| 444 | // | ||
| 445 | // Biased towards reporting "already paid" on any doubt: skipping a payment a | ||
| 446 | // miner is owed is visible and recoverable from the log, while paying twice | ||
| 447 | // is neither. | ||
| 448 | static bool WalletAlreadyPaid(const PayingRecord& r) | ||
| 449 | { | ||
| 450 | uint160 h160; | ||
| 451 | if (!AddressToHash160(r.address, h160)) | ||
| 452 | return false; | ||
| 453 | CScript want; | ||
| 454 | want << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; | ||
| 455 | const std::string comment = PayoutComment(r.blockHeight); | ||
| 456 | |||
| 457 | CRITICAL_BLOCK(cs_mapWallet) | ||
| 458 | { | ||
| 459 | for (std::map<uint256, CWalletTx>::iterator it = mapWallet.begin(); | ||
| 460 | it != mapWallet.end(); ++it) { | ||
| 461 | CWalletTx& wtx = it->second; | ||
| 462 | // The comment is written only by the payout path, so its presence | ||
| 463 | // already identifies the transaction as one of ours. | ||
| 464 | std::map<std::string, std::string>::const_iterator ci = | ||
| 465 | wtx.mapValue.find("comment"); | ||
| 466 | if (ci == wtx.mapValue.end() || ci->second != comment) | ||
| 467 | continue; | ||
| 468 | foreach (const CTxOut& txout, wtx.vout) | ||
| 469 | if (txout.nValue == r.amount && txout.scriptPubKey == want) | ||
| 470 | return true; | ||
| 471 | } | ||
| 472 | } | ||
| 473 | return false; | ||
| 474 | } | ||
| 475 | |||
| 476 | // Startup reconciliation. Drops in-flight records the wallet shows were | ||
| 477 | // already sent, and drops ledger entries that were already exploded into | ||
| 478 | // records (which happens if we died between writing the records and rewriting | ||
| 479 | // the ledger -- both files then describe the same debt). | ||
| 480 | static void ReconcilePayouts() | ||
| 481 | { | ||
| 482 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 483 | |||
| 484 | size_t settled = 0; | ||
| 485 | int64 settledAmount = 0; | ||
| 486 | for (auto it = gPaying.begin(); it != gPaying.end(); ) { | ||
| 487 | if (WalletAlreadyPaid(*it)) { | ||
| 488 | settledAmount += it->amount; | ||
| 489 | settled++; | ||
| 490 | it = gPaying.erase(it); | ||
| 491 | } else { | ||
| 492 | ++it; | ||
| 493 | } | ||
| 494 | } | ||
| 495 | if (settled) { | ||
| 496 | LogPrint("payout", "[payout] %zu in-flight payment(s) totalling %s were" | ||
| 497 | " already sent before the last shutdown -- not resending\n", | ||
| 498 | settled, FormatMoney(settledAmount).c_str()); | ||
| 499 | SavePayingLocked(); | ||
| 500 | } | ||
| 501 | |||
| 502 | size_t dropped = 0; | ||
| 503 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { | ||
| 504 | bool exploded = false; | ||
| 505 | for (const auto& r : gPaying) | ||
| 506 | if (r.matureAtHeight == it->matureAtHeight && r.blockHash == it->blockHash) { | ||
| 507 | exploded = true; | ||
| 508 | break; | ||
| 509 | } | ||
| 510 | if (exploded) { | ||
| 511 | LogPrint("payout", "[payout] ledger entry for block %d is already" | ||
| 512 | " in flight -- dropping the duplicate\n", it->blockHeight); | ||
| 513 | it = gPendingPayouts.erase(it); | ||
| 514 | dropped++; | ||
| 515 | } else { | ||
| 516 | ++it; | ||
| 517 | } | ||
| 518 | } | ||
| 519 | if (dropped) SavePendingPayouts(); | ||
| 520 | |||
| 521 | if (!gPaying.empty()) { | ||
| 522 | int64 owed = 0; | ||
| 523 | for (const auto& r : gPaying) owed += r.amount; | ||
| 524 | LogPrint("payout", "[payout] resuming %zu in-flight payment(s) totalling %s\n", | ||
| 525 | gPaying.size(), FormatMoney(owed).c_str()); | ||
| 526 | } | ||
| 527 | } | ||
| 528 | |||
| 344 | static void QueuePayouts(int blockHeight, | 529 | static void QueuePayouts(int blockHeight, |
| 345 | const uint256& blockHash, | 530 | const uint256& blockHash, |
| 346 | const std::map<std::string, uint64>& shareCount, | 531 | const std::map<std::string, uint64>& shareCount, |
| @@ -418,19 +603,13 @@ static void QueuePayouts(int blockHeight, | |||
| 418 | : ""); | 603 | : ""); |
| 419 | } | 604 | } |
| 420 | 605 | ||
| 421 | // Called from event loop: move matured entries to the exec queue (fast). | 606 | // Called from event loop: move matured entries into the in-flight list (fast). |
| 422 | // NOT crash-safe, despite the ordering below looking like it is. | ||
| 423 | // | 607 | // |
| 424 | // A matured payout is erased from gPendingPayouts, the file is rewritten | 608 | // Ordering matters and is deliberate: the in-flight records are written and |
| 425 | // without it, and only then does it reach the exec queue -- which lives in | 609 | // fsync'd to their own file *before* the ledger entry is erased, so the debt |
| 426 | // memory alone. SavePendingPayouts writes gPendingPayouts and nothing else; | 610 | // is always described by at least one of the two files. Dying in the window |
| 427 | // there is no exec-queue section of the file to recover from. Die anywhere | 611 | // where both describe it is handled at startup by ReconcilePayouts, which |
| 428 | // between the rewrite and SendMoney and those miners are simply never paid, | 612 | // drops the ledger entry it can see is already in flight. |
| 429 | // with no record that they were owed anything. | ||
| 430 | // | ||
| 431 | // Making this safe needs the payment to be idempotent -- record the txid and | ||
| 432 | // check the wallet for it on restart -- rather than a reordering, since | ||
| 433 | // re-queueing blindly would double-pay whoever was already sent to. | ||
| 434 | static void FlushMaturePayouts() | 613 | static void FlushMaturePayouts() |
| 435 | { | 614 | { |
| 436 | // Void obligations whose funding block lost a reorg. The coin that was | 615 | // Void obligations whose funding block lost a reorg. The coin that was |
| @@ -438,7 +617,7 @@ static void FlushMaturePayouts() | |||
| 438 | // coin the pool owes to somebody else. | 617 | // coin the pool owes to somebody else. |
| 439 | bool voided = false; | 618 | bool voided = false; |
| 440 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { | 619 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { |
| 441 | if (PayoutBlockStillOnChain(*it)) { ++it; continue; } | 620 | if (PayoutBlockStillOnChain(it->blockHash)) { ++it; continue; } |
| 442 | int64 total = 0; | 621 | int64 total = 0; |
| 443 | for (auto& kv : it->amounts) total += kv.second; | 622 | for (auto& kv : it->amounts) total += kv.second; |
| 444 | LogPrint("payout", "[payout] block %d (%s) was orphaned -- voiding %zu" | 623 | LogPrint("payout", "[payout] block %d (%s) was orphaned -- voiding %zu" |
| @@ -459,8 +638,23 @@ static void FlushMaturePayouts() | |||
| 459 | ++it; | 638 | ++it; |
| 460 | } | 639 | } |
| 461 | if (!ready.empty()) { | 640 | if (!ready.empty()) { |
| 462 | // Erase from pending then save before handing off to exec thread, | 641 | // Record the individual payments first, and only then drop the ledger |
| 463 | // so the file never contains an entry that is already being sent. | 642 | // entries that produced them. |
| 643 | { | ||
| 644 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 645 | for (const auto& pp : ready) { | ||
| 646 | for (const auto& kv : pp.amounts) { | ||
| 647 | PayingRecord r; | ||
| 648 | r.blockHeight = pp.blockHeight; | ||
| 649 | r.blockHash = pp.blockHash; | ||
| 650 | r.matureAtHeight = pp.matureAtHeight; | ||
| 651 | r.address = kv.first; | ||
| 652 | r.amount = kv.second; | ||
| 653 | gPaying.push_back(r); | ||
| 654 | } | ||
| 655 | } | ||
| 656 | SavePayingLocked(); | ||
| 657 | } | ||
| 464 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { | 658 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { |
| 465 | if (nBestHeight >= it->matureAtHeight) | 659 | if (nBestHeight >= it->matureAtHeight) |
| 466 | it = gPendingPayouts.erase(it); | 660 | it = gPendingPayouts.erase(it); |
| @@ -468,13 +662,45 @@ static void FlushMaturePayouts() | |||
| 468 | ++it; | 662 | ++it; |
| 469 | } | 663 | } |
| 470 | SavePendingPayouts(); | 664 | SavePendingPayouts(); |
| 471 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 472 | for (auto& pp : ready) | ||
| 473 | gPayoutExecQueue.push_back(std::move(pp)); | ||
| 474 | } | 665 | } |
| 475 | } | 666 | } |
| 476 | 667 | ||
| 477 | // Payout execution thread: drains gPayoutExecQueue, calls SendMoney. | 668 | // Identity of a payment, for finding its record again after SendMoney. |
| 669 | // amounts is a map keyed by address, so (block, address) is already unique. | ||
| 670 | static bool SamePayment(const PayingRecord& a, const PayingRecord& b) | ||
| 671 | { | ||
| 672 | return a.matureAtHeight == b.matureAtHeight | ||
| 673 | && a.blockHash == b.blockHash | ||
| 674 | && a.address == b.address | ||
| 675 | && a.amount == b.amount; | ||
| 676 | } | ||
| 677 | |||
| 678 | // Settle a payment: it is either done or abandoned, either way it stops being | ||
| 679 | // owed. Removing the record and rewriting the file is what makes the payment | ||
| 680 | // not happen twice after a restart. | ||
| 681 | static void ForgetPayment(const PayingRecord& r) | ||
| 682 | { | ||
| 683 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 684 | for (auto it = gPaying.begin(); it != gPaying.end(); ++it) | ||
| 685 | if (SamePayment(*it, r)) { gPaying.erase(it); break; } | ||
| 686 | SavePayingLocked(); | ||
| 687 | } | ||
| 688 | |||
| 689 | // Persist an updated attempt count / backoff for a payment still owed. | ||
| 690 | static void ReschedulePayment(const PayingRecord& r) | ||
| 691 | { | ||
| 692 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 693 | for (auto& e : gPaying) | ||
| 694 | if (SamePayment(e, r)) { | ||
| 695 | e.retries = r.retries; | ||
| 696 | e.deferrals = r.deferrals; | ||
| 697 | e.retryAfter = r.retryAfter; | ||
| 698 | break; | ||
| 699 | } | ||
| 700 | SavePayingLocked(); | ||
| 701 | } | ||
| 702 | |||
| 703 | // Payout execution thread: works through gPaying, calls SendMoney. | ||
| 478 | // Runs independently of the event loop so slow wallet operations | 704 | // Runs independently of the event loop so slow wallet operations |
| 479 | // (SendMoney takes cs_main) never stall miner I/O. | 705 | // (SendMoney takes cs_main) never stall miner I/O. |
| 480 | static void PayoutThreadFn(void*) | 706 | static void PayoutThreadFn(void*) |
| @@ -483,111 +709,94 @@ static void PayoutThreadFn(void*) | |||
| 483 | LogPrint("payout", "[payout] payout thread started\n"); | 709 | LogPrint("payout", "[payout] payout thread started\n"); |
| 484 | 710 | ||
| 485 | while (!fShutdown && gPoolRunning) { | 711 | while (!fShutdown && gPoolRunning) { |
| 486 | std::vector<PendingPayout> batch; | 712 | std::vector<PayingRecord> batch; |
| 487 | { | 713 | { |
| 488 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | 714 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); |
| 489 | batch.swap(gPayoutExecQueue); | 715 | int64 now = GetTime(); |
| 716 | for (const auto& r : gPaying) | ||
| 717 | if (r.retryAfter <= now) | ||
| 718 | batch.push_back(r); | ||
| 490 | } | 719 | } |
| 491 | 720 | ||
| 492 | for (auto& pp : batch) { | 721 | for (PayingRecord r : batch) { |
| 493 | if (pp.retryAfter > GetTime()) { | 722 | // Re-check here and not only at maturity: a payment can sit in this |
| 494 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | 723 | // list for hours across retries, and the reorg that kills its |
| 495 | gPayoutExecQueue.push_back(pp); | ||
| 496 | continue; | ||
| 497 | } | ||
| 498 | // Re-check here and not only at maturity: an entry can sit in this | ||
| 499 | // queue for hours across retries, and the reorg that kills its | ||
| 500 | // funding block can land at any point in that window. | 724 | // funding block can land at any point in that window. |
| 501 | if (!PayoutBlockStillOnChain(pp)) { | 725 | if (!PayoutBlockStillOnChain(r.blockHash)) { |
| 502 | int64 total = 0; | ||
| 503 | for (auto& kv : pp.amounts) total += kv.second; | ||
| 504 | LogPrint("payout", "[payout] block %d (%s) orphaned while queued" | 726 | LogPrint("payout", "[payout] block %d (%s) orphaned while queued" |
| 505 | " -- dropping %zu payment(s) totalling %s\n", | 727 | " -- dropping %s to %s\n", |
| 506 | pp.blockHeight, pp.blockHash.ToString().substr(0, 14).c_str(), | 728 | r.blockHeight, r.blockHash.ToString().substr(0, 14).c_str(), |
| 507 | pp.amounts.size(), FormatMoney(total).c_str()); | 729 | FormatMoney(r.amount).c_str(), r.address.c_str()); |
| 730 | ForgetPayment(r); | ||
| 508 | continue; | 731 | continue; |
| 509 | } | 732 | } |
| 510 | LogPrint("payout", "[payout] executing payments for block matured at %d" | ||
| 511 | " (%zu miners)\n", pp.matureAtHeight, pp.amounts.size()); | ||
| 512 | for (auto& kv : pp.amounts) { | ||
| 513 | uint160 h160; | ||
| 514 | if (!AddressToHash160(kv.first, h160)) { | ||
| 515 | // Should not happen -- bad addresses are filtered in QueuePayouts | ||
| 516 | LogPrint("payout", "[payout] SKIPPED bad address '%s'\n", | ||
| 517 | kv.first.c_str()); | ||
| 518 | continue; | ||
| 519 | } | ||
| 520 | CScript sc; | ||
| 521 | sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; | ||
| 522 | CWalletTx wtx; | ||
| 523 | wtx.mapValue["comment"] = strprintf("Pool payout height %d", | ||
| 524 | pp.blockHeight); | ||
| 525 | |||
| 526 | // Waiting for coin to mature is not a failure, so it must not | ||
| 527 | // consume the retry budget. Burning a retry every 60s means a | ||
| 528 | // payment the pool fully intends to honour can hit GIVING UP | ||
| 529 | // 24h later purely because the subsidy funding it had not | ||
| 530 | // matured yet -- the miner is then silently never paid. | ||
| 531 | int64 nSpendable = GetBalance(); | ||
| 532 | if (nSpendable < kv.second) { | ||
| 533 | PendingPayout dp; | ||
| 534 | dp.blockHeight = pp.blockHeight; | ||
| 535 | dp.blockHash = pp.blockHash; | ||
| 536 | dp.matureAtHeight = pp.matureAtHeight; | ||
| 537 | dp.amounts[kv.first] = kv.second; | ||
| 538 | dp.retries = pp.retries; // unchanged on purpose | ||
| 539 | dp.deferrals = pp.deferrals + 1; | ||
| 540 | dp.retryAfter = GetTime() + 60; | ||
| 541 | // Hourly, not every minute: a dry pool would otherwise | ||
| 542 | // write one line per payment per minute forever. | ||
| 543 | if (dp.deferrals == 1 || dp.deferrals % 60 == 0) | ||
| 544 | LogPrint("payout", "[payout] deferring %s to %s -- only %s" | ||
| 545 | " spendable (waiting %d min so far)\n", | ||
| 546 | FormatMoney(kv.second).c_str(), kv.first.c_str(), | ||
| 547 | FormatMoney(nSpendable).c_str(), dp.deferrals); | ||
| 548 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 549 | gPayoutExecQueue.push_back(std::move(dp)); | ||
| 550 | continue; | ||
| 551 | } | ||
| 552 | 733 | ||
| 553 | if (SendMoney(sc, kv.second, wtx)) { | 734 | uint160 h160; |
| 554 | LogPrint("payout", "[payout] paid %s to %s\n", | 735 | if (!AddressToHash160(r.address, h160)) { |
| 555 | FormatMoney(kv.second).c_str(), kv.first.c_str()); | 736 | // Should not happen -- bad addresses are filtered in QueuePayouts |
| 556 | } else if (pp.retries < 1440) { | 737 | LogPrint("payout", "[payout] SKIPPED bad address '%s'\n", |
| 557 | // Transient failures (immature coinbase, reorg, locked | 738 | r.address.c_str()); |
| 558 | // wallet) must not silently burn the payment -- requeue | 739 | ForgetPayment(r); |
| 559 | // with backoff. Permanent cases (orphaned coinbase) give | 740 | continue; |
| 560 | // up loudly after 1440 tries (~24 h). | 741 | } |
| 561 | PendingPayout rp; | 742 | CScript sc; |
| 562 | rp.blockHeight = pp.blockHeight; | 743 | sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; |
| 563 | rp.blockHash = pp.blockHash; | 744 | |
| 564 | rp.matureAtHeight = pp.matureAtHeight; | 745 | // Waiting for coin to mature is not a failure, so it must not |
| 565 | rp.amounts[kv.first] = kv.second; | 746 | // consume the retry budget. Burning a retry every 60s means a |
| 566 | rp.retries = pp.retries + 1; | 747 | // payment the pool fully intends to honour can hit GIVING UP |
| 567 | rp.deferrals = pp.deferrals; | 748 | // 24h later purely because the subsidy funding it had not |
| 568 | rp.retryAfter = GetTime() + 60; | 749 | // matured yet -- the miner is then silently never paid. |
| 569 | { | 750 | int64 nSpendable = GetBalance(); |
| 570 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | 751 | if (nSpendable < r.amount) { |
| 571 | gPayoutExecQueue.push_back(std::move(rp)); | 752 | r.deferrals++; |
| 572 | } | 753 | r.retryAfter = GetTime() + 60; |
| 573 | LogPrint("payout", "[payout] FAILED to pay %s to %s" | 754 | // Hourly, not every minute: a dry pool would otherwise |
| 574 | " (%s spendable at the time) -- retry %d/1440 in 60s\n", | 755 | // write one line per payment per minute forever. |
| 575 | FormatMoney(kv.second).c_str(), kv.first.c_str(), | 756 | if (r.deferrals == 1 || r.deferrals % 60 == 0) |
| 576 | FormatMoney(nSpendable).c_str(), | 757 | LogPrint("payout", "[payout] deferring %s to %s -- only %s" |
| 577 | pp.retries + 1); | 758 | " spendable (waiting %d min so far)\n", |
| 578 | } else { | 759 | FormatMoney(r.amount).c_str(), r.address.c_str(), |
| 579 | // Reaching here now means 1440 attempts that were NOT | 760 | FormatMoney(nSpendable).c_str(), r.deferrals); |
| 580 | // insufficient-funds waits (those defer instead) against a | 761 | ReschedulePayment(r); |
| 581 | // block still on the main chain -- i.e. a real wallet fault, | 762 | continue; |
| 582 | // not a maturity or reorg question. | 763 | } |
| 583 | LogPrint("payout", "[payout] GIVING UP on %s to %s after %d" | 764 | |
| 584 | " retries -- block %d is still on-chain and %s was" | 765 | CWalletTx wtx; |
| 585 | " spendable, so this is a wallet fault, not maturity" | 766 | wtx.mapValue["comment"] = PayoutComment(r.blockHeight); |
| 586 | " -- payment forfeited\n", | 767 | |
| 587 | FormatMoney(kv.second).c_str(), kv.first.c_str(), | 768 | if (SendMoney(sc, r.amount, wtx)) { |
| 588 | pp.retries, pp.blockHeight, | 769 | // The wallet holds the transaction before this returns, so a |
| 589 | FormatMoney(nSpendable).c_str()); | 770 | // crash here is recoverable: ReconcilePayouts finds it by |
| 590 | } | 771 | // comment+address+amount and does not resend. |
| 772 | ForgetPayment(r); | ||
| 773 | LogPrint("payout", "[payout] paid %s to %s (tx %s)\n", | ||
| 774 | FormatMoney(r.amount).c_str(), r.address.c_str(), | ||
| 775 | wtx.GetHash().ToString().substr(0, 12).c_str()); | ||
| 776 | } else if (r.retries < 1440) { | ||
| 777 | // Transient failures (reorg, locked wallet) must not silently | ||
| 778 | // burn the payment -- retry with backoff. Insufficient funds | ||
| 779 | // never lands here; it defers above. | ||
| 780 | r.retries++; | ||
| 781 | r.retryAfter = GetTime() + 60; | ||
| 782 | ReschedulePayment(r); | ||
| 783 | LogPrint("payout", "[payout] FAILED to pay %s to %s" | ||
| 784 | " (%s spendable at the time) -- retry %d/1440 in 60s\n", | ||
| 785 | FormatMoney(r.amount).c_str(), r.address.c_str(), | ||
| 786 | FormatMoney(nSpendable).c_str(), r.retries); | ||
| 787 | } else { | ||
| 788 | // Reaching here means 1440 attempts that were NOT | ||
| 789 | // insufficient-funds waits (those defer instead) against a | ||
| 790 | // block still on the main chain -- i.e. a real wallet fault, | ||
| 791 | // not a maturity or reorg question. | ||
| 792 | LogPrint("payout", "[payout] GIVING UP on %s to %s after %d" | ||
| 793 | " retries -- block %d is still on-chain and %s was" | ||
| 794 | " spendable, so this is a wallet fault, not maturity" | ||
| 795 | " -- payment forfeited\n", | ||
| 796 | FormatMoney(r.amount).c_str(), r.address.c_str(), | ||
| 797 | r.retries, r.blockHeight, | ||
| 798 | FormatMoney(nSpendable).c_str()); | ||
| 799 | ForgetPayment(r); | ||
| 591 | } | 800 | } |
| 592 | } | 801 | } |
| 593 | 802 | ||
| @@ -1631,6 +1840,11 @@ void ThreadRPCServer(void*) | |||
| 1631 | } | 1840 | } |
| 1632 | 1841 | ||
| 1633 | LoadPendingPayouts(); | 1842 | LoadPendingPayouts(); |
| 1843 | LoadPaying(); | ||
| 1844 | // Must run before the payout thread starts, so nothing is resent that the | ||
| 1845 | // wallet already shows as sent, and nothing is owed twice because we died | ||
| 1846 | // between recording the payments and rewriting the ledger. | ||
| 1847 | ReconcilePayouts(); | ||
| 1634 | 1848 | ||
| 1635 | _beginthread(AcceptLoopFn, 0, NULL); | 1849 | _beginthread(AcceptLoopFn, 0, NULL); |
| 1636 | _beginthread(LocalAcceptFn, 0, NULL); | 1850 | _beginthread(LocalAcceptFn, 0, NULL); |