diff options
Diffstat (limited to 'src/rpc.cpp')
| -rw-r--r-- | src/rpc.cpp | 138 |
1 files changed, 126 insertions, 12 deletions
diff --git a/src/rpc.cpp b/src/rpc.cpp index 24b1c88..b12dc2f 100644 --- a/src/rpc.cpp +++ b/src/rpc.cpp | |||
| @@ -230,9 +230,18 @@ static bool RebuildJob() | |||
| 230 | // Payouts -- event loop thread only | 230 | // Payouts -- event loop thread only |
| 231 | // --------------------------------------------------------------------------- | 231 | // --------------------------------------------------------------------------- |
| 232 | struct PendingPayout { | 232 | struct PendingPayout { |
| 233 | // The block that funds this payout, identified by hash and not by height | ||
| 234 | // alone. A reorg replaces the block at a height without changing the | ||
| 235 | // height, so a height-keyed obligation outlives the coinbase that was | ||
| 236 | // supposed to pay it: a 280-block reorg once left ~14.4k BTF of debt | ||
| 237 | // owed against blocks that no longer existed, which then drained the | ||
| 238 | // wallet paying fork-era miners out of real-chain coin. | ||
| 239 | int blockHeight = 0; | ||
| 240 | uint256 blockHash; // zero = entry predates hash keying | ||
| 233 | int matureAtHeight; | 241 | int matureAtHeight; |
| 234 | std::map<std::string, int64> amounts; // address -> satoshis | 242 | std::map<std::string, int64> amounts; // address -> satoshis |
| 235 | int retries = 0; // exec-thread only: SendMoney attempts so far | 243 | int retries = 0; // exec-thread only: SendMoney attempts so far |
| 244 | int deferrals = 0; // exec-thread only: waits for spendable coin | ||
| 236 | int64 retryAfter = 0; // exec-thread only: unix time, 0 = ready now | 245 | int64 retryAfter = 0; // exec-thread only: unix time, 0 = ready now |
| 237 | }; | 246 | }; |
| 238 | static std::vector<PendingPayout> gPendingPayouts; | 247 | static std::vector<PendingPayout> gPendingPayouts; |
| @@ -247,6 +256,8 @@ static void SavePendingPayouts() | |||
| 247 | nlohmann::json arr = nlohmann::json::array(); | 256 | nlohmann::json arr = nlohmann::json::array(); |
| 248 | for (const auto& pp : gPendingPayouts) { | 257 | for (const auto& pp : gPendingPayouts) { |
| 249 | nlohmann::json obj; | 258 | nlohmann::json obj; |
| 259 | obj["blockHeight"] = pp.blockHeight; | ||
| 260 | obj["blockHash"] = pp.blockHash.ToString(); | ||
| 250 | obj["matureAtHeight"] = pp.matureAtHeight; | 261 | obj["matureAtHeight"] = pp.matureAtHeight; |
| 251 | nlohmann::json amounts = nlohmann::json::object(); | 262 | nlohmann::json amounts = nlohmann::json::object(); |
| 252 | for (const auto& kv : pp.amounts) | 263 | for (const auto& kv : pp.amounts) |
| @@ -281,6 +292,13 @@ static void LoadPendingPayouts() | |||
| 281 | for (const auto& obj : arr) { | 292 | for (const auto& obj : arr) { |
| 282 | PendingPayout pp; | 293 | PendingPayout pp; |
| 283 | pp.matureAtHeight = obj["matureAtHeight"].get<int>(); | 294 | pp.matureAtHeight = obj["matureAtHeight"].get<int>(); |
| 295 | // Entries written before payouts were keyed by hash carry neither | ||
| 296 | // field. Leave blockHash zero and they are grandfathered past the | ||
| 297 | // orphan check rather than voided on a guess. | ||
| 298 | if (obj.contains("blockHash") && obj["blockHash"].is_string()) | ||
| 299 | pp.blockHash.SetHex(obj["blockHash"].get<std::string>()); | ||
| 300 | if (obj.contains("blockHeight") && obj["blockHeight"].is_number()) | ||
| 301 | pp.blockHeight = obj["blockHeight"].get<int>(); | ||
| 284 | for (auto it = obj["amounts"].begin(); it != obj["amounts"].end(); ++it) | 302 | for (auto it = obj["amounts"].begin(); it != obj["amounts"].end(); ++it) |
| 285 | pp.amounts[it.key()] = it.value().get<int64>(); | 303 | pp.amounts[it.key()] = it.value().get<int64>(); |
| 286 | gPendingPayouts.push_back(pp); | 304 | gPendingPayouts.push_back(pp); |
| @@ -300,7 +318,27 @@ static std::vector<PendingPayout> gPayoutExecQueue; | |||
| 300 | static std::mutex gPayoutExecMutex; | 318 | static std::mutex gPayoutExecMutex; |
| 301 | static std::atomic<bool> gPayoutThreadRunning{false}; | 319 | static std::atomic<bool> gPayoutThreadRunning{false}; |
| 302 | 320 | ||
| 321 | // Is the block that funds this payout still on the main chain? | ||
| 322 | // | ||
| 323 | // A zero hash means the entry was written before payouts recorded one, so | ||
| 324 | // there is nothing to check and we assume it is good -- voiding those would | ||
| 325 | // mean refusing to pay miners on the strength of a missing field. | ||
| 326 | static bool PayoutBlockStillOnChain(const PendingPayout& pp) | ||
| 327 | { | ||
| 328 | if (pp.blockHash == 0) | ||
| 329 | return true; | ||
| 330 | CRITICAL_BLOCK(cs_main) | ||
| 331 | { | ||
| 332 | std::map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(pp.blockHash); | ||
| 333 | if (mi == mapBlockIndex.end() || mi->second == NULL) | ||
| 334 | return false; | ||
| 335 | return mi->second->IsInMainChain(); | ||
| 336 | } | ||
| 337 | return false; | ||
| 338 | } | ||
| 339 | |||
| 303 | static void QueuePayouts(int blockHeight, | 340 | static void QueuePayouts(int blockHeight, |
| 341 | const uint256& blockHash, | ||
| 304 | const std::map<std::string, uint64>& shareCount, | 342 | const std::map<std::string, uint64>& shareCount, |
| 305 | uint64 total) | 343 | uint64 total) |
| 306 | { | 344 | { |
| @@ -322,12 +360,19 @@ static void QueuePayouts(int blockHeight, | |||
| 322 | int64 minerReward = (int64)((double)nSubsidy * (1.0 - feeFrac)); | 360 | int64 minerReward = (int64)((double)nSubsidy * (1.0 - feeFrac)); |
| 323 | 361 | ||
| 324 | PendingPayout pp; | 362 | PendingPayout pp; |
| 325 | // Deliberately COINBASE_MATURITY, not COINBASE_SPENDABLE_DEPTH: a payout is | 363 | pp.blockHeight = blockHeight; |
| 326 | // funded from whatever coin is already mature, not from its own block's | 364 | pp.blockHash = blockHash; |
| 327 | // coinbase, so waiting the extra 20 blocks delays every payment without | 365 | // COINBASE_SPENDABLE_DEPTH, not COINBASE_MATURITY. Paying at depth 100 |
| 328 | // making one more likely to succeed. Measured over 934 payments: 96% went | 366 | // means the block's own coinbase is still 20 blocks short of spendable, so |
| 329 | // through at depth 100, the moment they were queued. | 367 | // the payment has to come out of some *other* mature coin -- the pool runs |
| 330 | pp.matureAtHeight = blockHeight + COINBASE_MATURITY; | 368 | // a permanent ~19-block float and every payout fails until unrelated coin |
| 369 | // matures. The earlier "96% of 934 payments went through at depth 100" | ||
| 370 | // measurement that justified the shorter wait was taken while this node was | ||
| 371 | // alone on a private fork paying itself, so it says nothing about a pool | ||
| 372 | // competing on the real chain. Waiting the extra 20 blocks makes each | ||
| 373 | // payout funded by construction, and puts the maturity gate past the | ||
| 374 | // reorg depth that orphaned 280 blocks of obligations. | ||
| 375 | pp.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH; | ||
| 331 | 376 | ||
| 332 | int skipped = 0; | 377 | int skipped = 0; |
| 333 | int64 burned = 0; // amount owed to miners with bad/empty addresses | 378 | int64 burned = 0; // amount owed to miners with bad/empty addresses |
| @@ -384,6 +429,23 @@ static void QueuePayouts(int blockHeight, | |||
| 384 | // re-queueing blindly would double-pay whoever was already sent to. | 429 | // re-queueing blindly would double-pay whoever was already sent to. |
| 385 | static void FlushMaturePayouts() | 430 | static void FlushMaturePayouts() |
| 386 | { | 431 | { |
| 432 | // Void obligations whose funding block lost a reorg. The coin that was | ||
| 433 | // meant to pay them does not exist, so paying anyway spends real-chain | ||
| 434 | // coin the pool owes to somebody else. | ||
| 435 | bool voided = false; | ||
| 436 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { | ||
| 437 | if (PayoutBlockStillOnChain(*it)) { ++it; continue; } | ||
| 438 | int64 total = 0; | ||
| 439 | for (auto& kv : it->amounts) total += kv.second; | ||
| 440 | LogPrint("payout", "[payout] block %d (%s) was orphaned -- voiding %zu" | ||
| 441 | " payment(s) totalling %s\n", | ||
| 442 | it->blockHeight, it->blockHash.ToString().substr(0, 14).c_str(), | ||
| 443 | it->amounts.size(), FormatMoney(total).c_str()); | ||
| 444 | it = gPendingPayouts.erase(it); | ||
| 445 | voided = true; | ||
| 446 | } | ||
| 447 | if (voided) SavePendingPayouts(); | ||
| 448 | |||
| 387 | std::vector<PendingPayout> ready; | 449 | std::vector<PendingPayout> ready; |
| 388 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { | 450 | for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { |
| 389 | if (nBestHeight < it->matureAtHeight) { ++it; continue; } | 451 | if (nBestHeight < it->matureAtHeight) { ++it; continue; } |
| @@ -429,6 +491,18 @@ static void PayoutThreadFn(void*) | |||
| 429 | gPayoutExecQueue.push_back(pp); | 491 | gPayoutExecQueue.push_back(pp); |
| 430 | continue; | 492 | continue; |
| 431 | } | 493 | } |
| 494 | // Re-check here and not only at maturity: an entry can sit in this | ||
| 495 | // queue for hours across retries, and the reorg that kills its | ||
| 496 | // funding block can land at any point in that window. | ||
| 497 | if (!PayoutBlockStillOnChain(pp)) { | ||
| 498 | int64 total = 0; | ||
| 499 | for (auto& kv : pp.amounts) total += kv.second; | ||
| 500 | LogPrint("payout", "[payout] block %d (%s) orphaned while queued" | ||
| 501 | " -- dropping %zu payment(s) totalling %s\n", | ||
| 502 | pp.blockHeight, pp.blockHash.ToString().substr(0, 14).c_str(), | ||
| 503 | pp.amounts.size(), FormatMoney(total).c_str()); | ||
| 504 | continue; | ||
| 505 | } | ||
| 432 | LogPrint("payout", "[payout] executing payments for block matured at %d" | 506 | LogPrint("payout", "[payout] executing payments for block matured at %d" |
| 433 | " (%zu miners)\n", pp.matureAtHeight, pp.amounts.size()); | 507 | " (%zu miners)\n", pp.matureAtHeight, pp.amounts.size()); |
| 434 | for (auto& kv : pp.amounts) { | 508 | for (auto& kv : pp.amounts) { |
| @@ -443,7 +517,35 @@ static void PayoutThreadFn(void*) | |||
| 443 | sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; | 517 | sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; |
| 444 | CWalletTx wtx; | 518 | CWalletTx wtx; |
| 445 | wtx.mapValue["comment"] = strprintf("Pool payout height %d", | 519 | wtx.mapValue["comment"] = strprintf("Pool payout height %d", |
| 446 | pp.matureAtHeight); | 520 | pp.blockHeight); |
| 521 | |||
| 522 | // Waiting for coin to mature is not a failure, so it must not | ||
| 523 | // consume the retry budget. Burning a retry every 60s means a | ||
| 524 | // payment the pool fully intends to honour can hit GIVING UP | ||
| 525 | // 24h later purely because the subsidy funding it had not | ||
| 526 | // matured yet -- the miner is then silently never paid. | ||
| 527 | int64 nSpendable = GetBalance(); | ||
| 528 | if (nSpendable < kv.second) { | ||
| 529 | PendingPayout dp; | ||
| 530 | dp.blockHeight = pp.blockHeight; | ||
| 531 | dp.blockHash = pp.blockHash; | ||
| 532 | dp.matureAtHeight = pp.matureAtHeight; | ||
| 533 | dp.amounts[kv.first] = kv.second; | ||
| 534 | dp.retries = pp.retries; // unchanged on purpose | ||
| 535 | dp.deferrals = pp.deferrals + 1; | ||
| 536 | dp.retryAfter = GetTime() + 60; | ||
| 537 | // Hourly, not every minute: a dry pool would otherwise | ||
| 538 | // write one line per payment per minute forever. | ||
| 539 | if (dp.deferrals == 1 || dp.deferrals % 60 == 0) | ||
| 540 | LogPrint("payout", "[payout] deferring %s to %s -- only %s" | ||
| 541 | " spendable (waiting %d min so far)\n", | ||
| 542 | FormatMoney(kv.second).c_str(), kv.first.c_str(), | ||
| 543 | FormatMoney(nSpendable).c_str(), dp.deferrals); | ||
| 544 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | ||
| 545 | gPayoutExecQueue.push_back(std::move(dp)); | ||
| 546 | continue; | ||
| 547 | } | ||
| 548 | |||
| 447 | if (SendMoney(sc, kv.second, wtx)) { | 549 | if (SendMoney(sc, kv.second, wtx)) { |
| 448 | LogPrint("payout", "[payout] paid %s to %s\n", | 550 | LogPrint("payout", "[payout] paid %s to %s\n", |
| 449 | FormatMoney(kv.second).c_str(), kv.first.c_str()); | 551 | FormatMoney(kv.second).c_str(), kv.first.c_str()); |
| @@ -453,24 +555,34 @@ static void PayoutThreadFn(void*) | |||
| 453 | // with backoff. Permanent cases (orphaned coinbase) give | 555 | // with backoff. Permanent cases (orphaned coinbase) give |
| 454 | // up loudly after 1440 tries (~24 h). | 556 | // up loudly after 1440 tries (~24 h). |
| 455 | PendingPayout rp; | 557 | PendingPayout rp; |
| 558 | rp.blockHeight = pp.blockHeight; | ||
| 559 | rp.blockHash = pp.blockHash; | ||
| 456 | rp.matureAtHeight = pp.matureAtHeight; | 560 | rp.matureAtHeight = pp.matureAtHeight; |
| 457 | rp.amounts[kv.first] = kv.second; | 561 | rp.amounts[kv.first] = kv.second; |
| 458 | rp.retries = pp.retries + 1; | 562 | rp.retries = pp.retries + 1; |
| 563 | rp.deferrals = pp.deferrals; | ||
| 459 | rp.retryAfter = GetTime() + 60; | 564 | rp.retryAfter = GetTime() + 60; |
| 460 | { | 565 | { |
| 461 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); | 566 | std::lock_guard<std::mutex> lk(gPayoutExecMutex); |
| 462 | gPayoutExecQueue.push_back(std::move(rp)); | 567 | gPayoutExecQueue.push_back(std::move(rp)); |
| 463 | } | 568 | } |
| 464 | LogPrint("payout", "[payout] FAILED to pay %s to %s" | 569 | LogPrint("payout", "[payout] FAILED to pay %s to %s" |
| 465 | " -- retry %d/1440 in 60s\n", | 570 | " (%s spendable at the time) -- retry %d/1440 in 60s\n", |
| 466 | FormatMoney(kv.second).c_str(), kv.first.c_str(), | 571 | FormatMoney(kv.second).c_str(), kv.first.c_str(), |
| 572 | FormatMoney(nSpendable).c_str(), | ||
| 467 | pp.retries + 1); | 573 | pp.retries + 1); |
| 468 | } else { | 574 | } else { |
| 575 | // Reaching here now means 1440 attempts that were NOT | ||
| 576 | // insufficient-funds waits (those defer instead) against a | ||
| 577 | // block still on the main chain -- i.e. a real wallet fault, | ||
| 578 | // not a maturity or reorg question. | ||
| 469 | LogPrint("payout", "[payout] GIVING UP on %s to %s after %d" | 579 | LogPrint("payout", "[payout] GIVING UP on %s to %s after %d" |
| 470 | " retries (orphaned coinbase or persistent wallet" | 580 | " retries -- block %d is still on-chain and %s was" |
| 471 | " error?) -- payment forfeited\n", | 581 | " spendable, so this is a wallet fault, not maturity" |
| 582 | " -- payment forfeited\n", | ||
| 472 | FormatMoney(kv.second).c_str(), kv.first.c_str(), | 583 | FormatMoney(kv.second).c_str(), kv.first.c_str(), |
| 473 | pp.retries); | 584 | pp.retries, pp.blockHeight, |
| 585 | FormatMoney(nSpendable).c_str()); | ||
| 474 | } | 586 | } |
| 475 | } | 587 | } |
| 476 | } | 588 | } |
| @@ -1065,7 +1177,9 @@ static bool HandleLine(Miner* m, const std::string& rawLine, | |||
| 1065 | if (accepted) { | 1177 | if (accepted) { |
| 1066 | // Snapshot height before RebuildJob() (which increments it) | 1178 | // Snapshot height before RebuildJob() (which increments it) |
| 1067 | int foundHeight = gCurrentJob.height; | 1179 | int foundHeight = gCurrentJob.height; |
| 1068 | QueuePayouts(foundHeight, roundShareCount, roundShareTotal); | 1180 | // b is the block we submitted, untouched by ProcessBlock (which |
| 1181 | // took a copy), so its hash is what the payout must be keyed on. | ||
| 1182 | QueuePayouts(foundHeight, b.GetHash(), roundShareCount, roundShareTotal); | ||
| 1069 | // Reset round shares | 1183 | // Reset round shares |
| 1070 | roundShareCount.clear(); | 1184 | roundShareCount.clear(); |
| 1071 | roundShareTotal = 0; | 1185 | roundShareTotal = 0; |