2 files changed, 327 insertions, 2 deletions

diff --git a/src/main.cpp b/src/main.cpp
index dded83a..681673c 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -65,6 +65,137 @@ CBlockIndex* pindexBest = NULL;
 map<uint256, CBlock*> mapOrphanBlocks;
 multimap<uint256, CBlock*> mapOrphanBlocksByPrev;
 
+// Block bodies we have sent a getdata for and not yet received, so the
+// failover pass in SendMessages can tell "requested and being served" from
+// "requested and gone quiet" and re-issue the request to a different peer
+// (#33). Keyed by block hash; setAsked is the peers already tried since the
+// round last restarted, so the retry rotates instead of hammering the
+// announcer that is not serving. Guarded by cs_main, which both the message
+// handlers and SendMessages already hold.
+//
+// Capped small on purpose. At the tip there is one block in flight; during
+// initial download there are legitimately hundreds outstanding to one peer,
+// and rotating those would duplicate the whole download to every other peer.
+//
+// The cap is a shared resource and nothing on the path from inv to insertion
+// can tell a real block hash from 32 random bytes -- AlreadyHave only asks
+// whether we already hold the block, and there is no header to check work
+// against. So the table does not refuse a new hash when it is full; it makes
+// room. See MakeRoomInBlockFetch for the rule, which is what stops one peer
+// filling all eight slots and switching failover off for everybody.
+//
+// The initialisers matter: the insertion path uses operator[] and then tests
+// nTimeFirstAsked == 0 to detect a fresh entry. That worked on
+// value-initialisation alone, but only for as long as nobody gave this struct
+// a constructor.
+struct CBlockFetch
+{
+    int64 nTimeFirstAsked = 0;  // when the first getdata for this block went out
+    int64 nTimeAsked      = 0;  // when the latest getdata went out
+    CAddress addrAnnounced;     // peer whose announcement put this hash here
+    set<CAddress> setAsked;     // peers tried since the round last restarted
+};
+map<uint256, CBlockFetch> mapBlockFetch;
+static const int BLOCK_FETCH_TRACK_MAX = 8;
+// A block nobody has served for this long is not a fetch problem: the hash may
+// have been announced and never existed. Drop the entry rather than getdata
+// forever; a re-announcement recreates it.
+static const int BLOCK_FETCH_GIVEUP_SECS = 10 * 60;
+
+// Failover firing at all is worth saying out loud: it means peers are
+// announcing blocks they then do not serve, which is the pathology of #33 and
+// the thing nobody could otherwise see. Naming the peer on every re-request is
+// a different matter. A peer can drive that line as often as it likes by
+// announcing hashes it will not serve, and what it prints is a peer address,
+// which is the one thing the .btf transport otherwise takes trouble not to
+// spread around. So the per-peer detail sits behind the net category and the
+// unconditional line is a count over a window no peer can shorten: the first
+// failover after a quiet spell prints at once, because the change from "blocks
+// arrive" to "blocks are announced and not served" is the event worth seeing,
+// and everything after it is summarised once a minute.
+static const int BLOCK_FETCH_REPORT_SECS = 60;
+static int64 nBlockFetchWindowStart = 0;
+static int   nBlockFetchFailovers   = 0;
+
+// How stale our own best block may be before we stop calling this the tip.
+// Blocks target two minutes, so this is thirty intervals: far longer than any
+// stall worth failing over from -- the one in #33 ran sixteen minutes -- and
+// far shorter than catching up, which starts hours or days back.
+static const int BLOCK_FETCH_TIP_AGE_SECS = 60 * 60;
+
+// Whether this node is tracking the tip rather than still catching up.
+//
+// Measured off the age of our own best block, not off any height a peer told
+// us. nStartingHeight is optional in the version message and every node
+// released so far sends the short form, so it is -1 for essentially every
+// peer; and where it is filled in later it is filled in from the height of a
+// block that peer served us, which during catch-up tracks our own height and
+// would report "at the tip" for the whole download. Block timestamps are the
+// one clock here that peers cannot move: CheckBlock refuses anything more than
+// two hours ahead and AcceptBlock refuses anything at or before the previous
+// median, so the only way to make this read "at the tip" falsely is to mine,
+// and all that buys is the 15s spacing this code had unconditionally anyway.
+bool IsBlockFetchNearTip()
+{
+    return pindexBest && (int64)pindexBest->nTime > GetTime() - BLOCK_FETCH_TIP_AGE_SECS;
+}
+
+// Drop one entry so a hash announced by addrNew can be tracked.
+//
+// Refusing the insertion instead was a remote off-switch for the whole
+// feature. An inv carrying eight hashes that never existed filled every slot
+// -- nothing between inv and here can tell those from real blocks -- and no
+// genuinely new block was tracked again until the give-up timer expired ten
+// minutes later, re-armable for one message per ten minutes from a single
+// connection.
+//
+// So evict instead, by occupancy: a peer may only take a slot from a peer
+// holding strictly more slots than itself, and otherwise recycles one of its
+// own. That is what keeps eviction from becoming the same attack in a new
+// shape. A peer that has filled the table is by definition the heaviest
+// holder, so its own entries are the only ones it can displace, while a peer
+// announcing its first block holds nothing and therefore always evicts from
+// the heaviest -- a genuine announcement is never refused a slot, whatever the
+// attacker holds or however many connections it holds it from. The one case
+// that costs a real fetch is eight distinct peers holding one entry each,
+// where a ninth takes the oldest; that is a busy healthy node, not an attack,
+// and losing the oldest of eight beats refusing the ninth.
+static void MakeRoomInBlockFetch(const CAddress& addrNew)
+{
+    if (mapBlockFetch.empty())
+        return;
+
+    map<CAddress, int> mapHeld;
+    for (map<uint256, CBlockFetch>::iterator mi = mapBlockFetch.begin(); mi != mapBlockFetch.end(); ++mi)
+        mapHeld[(*mi).second.addrAnnounced]++;
+
+    int nMine = mapHeld.count(addrNew) ? mapHeld[addrNew] : 0;
+
+    int nMax = 0;
+    CAddress addrVictim;
+    for (map<CAddress, int>::iterator mi = mapHeld.begin(); mi != mapHeld.end(); ++mi)
+        if ((*mi).second > nMax)
+        {
+            nMax = (*mi).second;
+            addrVictim = (*mi).first;
+        }
+
+    // Nobody is holding more than this peer, so it recycles its own oldest
+    // rather than taking a slot off an equal.
+    if (nMax <= nMine)
+        addrVictim = addrNew;
+
+    map<uint256, CBlockFetch>::iterator miVictim = mapBlockFetch.end();
+    for (map<uint256, CBlockFetch>::iterator mi = mapBlockFetch.begin(); mi != mapBlockFetch.end(); ++mi)
+        if ((*mi).second.addrAnnounced == addrVictim &&
+            (miVictim == mapBlockFetch.end() ||
+             (*mi).second.nTimeFirstAsked < (*miVictim).second.nTimeFirstAsked))
+            miVictim = mi;
+
+    if (miVictim != mapBlockFetch.end())
+        mapBlockFetch.erase(miVictim);
+}
+
 map<uint256, CDataStream*> mapOrphanTransactions;
 multimap<uint256, CDataStream*> mapOrphanTransactionsByPrev;
 
@@ -3292,10 +3423,56 @@ bool ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv)
         CInv inv(MSG_BLOCK, pblock->GetHash());
         pfrom->AddInventoryKnown(inv);
 
+        // GetHash() covers the 80-byte header and nothing else, so the hash
+        // coming out right does not make this the block we asked for. Every
+        // peer that saw X announced holds X's header, and can answer our
+        // getdata with that header bolted onto a transaction list of its own
+        // invention -- the reply still hashes to X. hashMerkleRoot is the only
+        // thing binding the header to the body, so check it here, before the
+        // mapBlockFetch entry is touched and before CheckBlock spends a
+        // RandomX hash on a body already known to be forged.
+        //
+        // There is no honest way to arrive here. Peer traffic rides the .btf
+        // channel, which is XChaCha20-Poly1305, so a body that arrives at all
+        // arrived intact: a mismatch is the peer's own construction, not line
+        // noise. Drop the peer and leave the fetch outstanding so the failover
+        // sweep asks somebody else for the real X, rather than recording X as
+        // delivered on the strength of a header anyone could copy.
+        if (pblock->hashMerkleRoot != pblock->BuildMerkleTree())
+        {
+            if (LogAcceptsCategory("net"))
+                printf("peer %s served a forged body for %s\n",
+                       pfrom->addr.ToString().c_str(), inv.ToString().c_str());
+            // One line per connection: ProcessMessages keeps draining the
+            // receive buffer after fDisconnect is set, so a peer that packs
+            // many forged bodies into one buffer must not get one log line
+            // for each of them.
+            if (!pfrom->fDisconnect)
+                printf("block fetch: a peer answered the request for %s with a body that is not "
+                       "that block -- dropping it\n", inv.hash.ToString().substr(0,14).c_str());
+            pfrom->fDisconnect = true;
+            return error("ProcessMessage() : forged block body for %s",
+                         inv.hash.ToString().substr(0,14).c_str());
+        }
+
         if (ProcessBlock(pfrom, pblock.release()))
         {
             mapAlreadyAskedFor.erase(inv);
 
+            // The body arrived and validated, so the fetch really is over:
+            // the block is now in mapBlockIndex, or parked in mapOrphanBlocks,
+            // and either way AlreadyHave answers for it from here on.
+            //
+            // This has to stay inside the success branch, next to the erase
+            // above. When validation rejects a body nothing is stored,
+            // AlreadyHave stays false, and the mapAskFor entry was already
+            // consumed when the getdata went out -- so erasing the tracking
+            // here would leave the block requested from nobody at all until
+            // some peer happened to re-announce it. That is exactly the stall
+            // of #33, and erasing before ProcessBlock made it reachable on
+            // demand.
+            mapBlockFetch.erase(inv.hash);
+
             // A peer that just handed us a block is demonstrably at least at
             // that block's height. Without this, nStartingHeight is a snapshot
             // taken once during the version handshake and never touched again,
@@ -3420,12 +3597,119 @@ bool SendMessages(CNode* pto)
             const CInv& inv = (*pto->mapAskFor.begin()).second;
             if (LogAcceptsCategory("net")) printf("sending getdata: %s\n", inv.ToString().c_str());
             if (!AlreadyHave(txdb, inv))
+            {
                 vAskFor.push_back(inv);
+                // Blocks get their delivery tracked so the failover pass
+                // below can chase the ones that go quiet.
+                if (inv.type == MSG_BLOCK)
+                {
+                    // Full, and this is a hash we are not already tracking:
+                    // make room rather than drop it on the floor, or one peer
+                    // announcing eight hashes turns failover off for everyone.
+                    if (!mapBlockFetch.count(inv.hash) &&
+                        mapBlockFetch.size() >= BLOCK_FETCH_TRACK_MAX)
+                        MakeRoomInBlockFetch(pto->addr);
+
+                    if (mapBlockFetch.count(inv.hash) ||
+                        mapBlockFetch.size() < BLOCK_FETCH_TRACK_MAX)
+                    {
+                        CBlockFetch& fetch = mapBlockFetch[inv.hash];
+                        if (fetch.nTimeFirstAsked == 0)
+                        {
+                            fetch.nTimeFirstAsked = GetTime();
+                            // Who is answerable for this hash existing. Set
+                            // once, on creation: the failover sweep asks other
+                            // peers later, and none of them chose to announce
+                            // it, so none of them should be charged for it.
+                            fetch.addrAnnounced = pto->addr;
+                        }
+                        fetch.nTimeAsked = GetTime();
+                        fetch.setAsked.insert(pto->addr);
+                    }
+                }
+            }
             pto->mapAskFor.erase(pto->mapAskFor.begin());
         }
         if (!vAskFor.empty())
             pto->PushMessage("getdata", vAskFor);
 
+
+        //
+        // Message: getdata (block fetch failover)
+        //
+        // A requested block that has not arrived in BLOCK_FETCH_RETRY_SECS is
+        // not still in transit -- a body is at most 1 MB and somebody
+        // demonstrably has it -- so ask somebody else instead of waiting on
+        // whoever was asked first (#33). Each peer the message handler visits
+        // takes at most one turn per block per round; once every connected
+        // peer has had its turn and a further interval passes with nobody
+        // new to ask, the round restarts. A peer that declared itself a
+        // client cannot serve block bodies and is no use as a target.
+        if (!pto->fClient)
+        {
+            vector<CInv> vFailover;
+            int64 nNowSecs = GetTime();
+            for (map<uint256, CBlockFetch>::iterator mi = mapBlockFetch.begin(); mi != mapBlockFetch.end();)
+            {
+                CBlockFetch& fetch = (*mi).second;
+                CInv inv(MSG_BLOCK, (*mi).first);
+                if (AlreadyHave(txdb, inv))
+                {
+                    // Arrived by some other road -- our own miner, an orphan
+                    // parked in mapOrphanBlocks. Nothing left to chase.
+                    mapBlockFetch.erase(mi++);
+                    continue;
+                }
+                if (nNowSecs - fetch.nTimeFirstAsked > BLOCK_FETCH_GIVEUP_SECS)
+                {
+                    printf("block fetch: giving up on %s -- no peer served it in %d s\n",
+                           inv.ToString().c_str(), (int)(nNowSecs - fetch.nTimeFirstAsked));
+                    mapBlockFetch.erase(mi++);
+                    continue;
+                }
+                if (nNowSecs - fetch.nTimeAsked >= BLOCK_FETCH_RETRY_SECS)
+                {
+                    // The handler loop visits every peer within ~100ms, so a
+                    // request stale by two full intervals means nobody outside
+                    // setAsked was there to ask. Start the round over.
+                    if (nNowSecs - fetch.nTimeAsked >= 2 * BLOCK_FETCH_RETRY_SECS)
+                        fetch.setAsked.clear();
+                    if (!fetch.setAsked.count(pto->addr))
+                    {
+                        // Which block, how long, and which peer we are trying
+                        // next. Behind the category because it names a peer
+                        // and a peer decides how often it is written.
+                        if (LogAcceptsCategory("net"))
+                            printf("block fetch failover: %s unanswered for %d s, re-requesting from %s\n",
+                                   inv.ToString().c_str(), (int)(nNowSecs - fetch.nTimeAsked),
+                                   pto->addr.ToString().c_str());
+
+                        // The unconditional half: no addresses, and at most
+                        // one line per BLOCK_FETCH_REPORT_SECS however hard a
+                        // peer tries. See the counters for the reasoning.
+                        nBlockFetchFailovers++;
+                        if (nNowSecs - nBlockFetchWindowStart >= BLOCK_FETCH_REPORT_SECS)
+                        {
+                            printf("block fetch: %d block request%s went unanswered and %s been "
+                                   "re-asked of other peers (%d tracked)\n",
+                                   nBlockFetchFailovers, nBlockFetchFailovers == 1 ? "" : "s",
+                                   nBlockFetchFailovers == 1 ? "has" : "have",
+                                   (int)mapBlockFetch.size());
+                            nBlockFetchWindowStart = nNowSecs;
+                            nBlockFetchFailovers   = 0;
+                        }
+
+                        vFailover.push_back(inv);
+                        fetch.nTimeAsked = nNowSecs;
+                        fetch.setAsked.insert(pto->addr);
+                    }
+                }
+                ++mi;
+            }
+            if (!vFailover.empty())
+                pto->PushMessage("getdata", vFailover);
+        }
+
     }
     return true;
 }
diff --git a/src/net.h b/src/net.h
index 600922a..fc48d0b 100644
--- a/src/net.h
+++ b/src/net.h
@@ -23,6 +23,12 @@ enum
 
 
 
+// Defined in main.cpp. True while this node is tracking the tip rather than
+// still catching up, measured off the age of our own best block -- see there
+// for why that and not a peer-reported height. Declared here because net.h is
+// parsed before main.h and CNode::AskFor needs it.
+bool IsBlockFetchNearTip();
+
 bool GetMyExternalIP(unsigned int& ipRet);
 CNode* ConnectNodeBtf(const string& strBtfAddr);
 CNode* ConnectNodeBtfResolved(const string& strBtfAddr, const string& strMeeting,
@@ -89,6 +95,15 @@ static const int          BTF_RECV_TIMEOUT_SECS    = 30 * 60;
 static const int          BTF_SEND_STALL_SECS      = 10 * 60;
 static const int          BTF_HANDSHAKE_GRACE_SECS = 60;
 
+// How long a requested block gets before it is asked for again, in seconds.
+// Used twice: CNode::AskFor spaces the queued per-announcer block retries this
+// far apart instead of the two minutes transactions get, and the failover
+// sweep in main.cpp re-issues a getdata to a different peer once a sent
+// request has gone unanswered this long. A block body is at most 1 MB and
+// somebody demonstrably has it -- they announced it -- so on this timescale
+// "not here yet" means "not coming from that peer", not "still in transit".
+static const int          BLOCK_FETCH_RETRY_SECS   = 15;
+
 // A serialized object cannot be larger than MAX_SIZE, so no valid network
 // message needs the old 256 MB header allowance. Keep the wire cap aligned with
 // what the deserializer can accept. How long a peer may sit on a partial
@@ -795,8 +810,34 @@ public:
         static int64 nLastTime;
         nLastTime = nNow = max(nNow, ++nLastTime);
 
-        // Each retry is 2 minutes after the last
-        nRequestTime = max(nRequestTime + 2 * 60 * 1000000, nNow);
+        // Each retry is 2 minutes after the last -- and because nRequestTime
+        // is shared across peers, the same spacing is what strings requests
+        // for one inv across its announcers. For transactions that is pacing;
+        // for a new block it was the stall of issue #33: nine peers announce
+        // the block, the first one asked does not serve it, and the ninth
+        // candidate is not even asked until sixteen minutes in, with the pool
+        // mining on the stale tip the whole time.
+        //
+        // Blocks retry on the fetch timescale, but only while we are at the
+        // tip. That is where the stall costs something -- the pool keeps
+        // handing out jobs against a dead tip -- and it is also where the
+        // spacing is nearly free: one block is in flight, and its body has
+        // almost always arrived before the second announcer's turn comes
+        // round, so AlreadyHave drops the duplicate before a getdata is sent.
+        //
+        // While catching up neither half holds. A peer working through a
+        // 500-deep queue can be minutes away from block N without anything
+        // being wrong, so at 15s the second announcer is asked at +15s, the
+        // third at +30s, and the bulk download arrives once per announcer --
+        // up to eight times over. Nothing caps that: the eight-entry cap is on
+        // mapBlockFetch, which is a different mechanism from this spacing.
+        // Two minutes is the right number there, and it is what this line did
+        // before. Failover at the tip does not depend on this being 15s in any
+        // case: mapBlockFetch re-issues a stalled getdata to the next peer on
+        // its own timer, which is what actually fixes #33.
+        bool fFast = (inv.type == MSG_BLOCK && IsBlockFetchNearTip());
+        int64 nRetryUsecs = (fFast ? BLOCK_FETCH_RETRY_SECS : 2 * 60) * 1000000LL;
+        nRequestTime = max(nRequestTime + nRetryUsecs, nNow);
         mapAskFor.insert(make_pair(nRequestTime, inv));
     }