merge: !8 fix/tunnel-retry into deploy/all-2026-08-24

5 files changed, 182 insertions, 21 deletions

diff --git a/src/btfrv.cpp b/src/btfrv.cpp
index 8ab735c..7d705ef 100644
--- a/src/btfrv.cpp
+++ b/src/btfrv.cpp
@@ -377,17 +377,29 @@ bool RvServiceWaitPaired(RvSocket s, int nTimeoutSecs)
 }
 
 RvSocket RvClientConnect(const char* relay_host, unsigned short port,
-                         const unsigned char target_pubkey[32])
+                         const unsigned char target_pubkey[32],
+                         RvClientFail* pFail)
 {
+    if (pFail) *pFail = RVC_FAIL_DIAL;
     if (!RvInit()) return RV_INVALID;
     SOCKET s = ConnectTo(relay_host, port);
     if (s == INVALID_SOCKET) return RV_INVALID;
+    if (pFail) *pFail = RVC_FAIL_PAIR;
     unsigned char hdr[33];
     hdr[0] = (unsigned char)'C';
     memcpy(hdr + 1, target_pubkey, 32);
     if (!WriteN(s, hdr, 33)) { CLOSESOCK(s); return RV_INVALID; }
     unsigned char paired = 0;
-    if (!ReadN(s, &paired, 1) || paired != 0x01) { CLOSESOCK(s); return RV_INVALID; }
+    if (!ReadN(s, &paired, 1)) { CLOSESOCK(s); return RV_INVALID; }
+    if (paired != 0x01)
+    {
+        // The relay answered and declined: HandleIncoming writes 0x00 and
+        // closes when no service is parked under this pubkey right now.
+        if (pFail) *pFail = RVC_FAIL_REFUSED;
+        CLOSESOCK(s);
+        return RV_INVALID;
+    }
+    if (pFail) *pFail = RVC_FAIL_NONE;
     return (RvSocket)s;
 }
 
diff --git a/src/btfrv.h b/src/btfrv.h
index 9cd7476..1e85cb0 100644
--- a/src/btfrv.h
+++ b/src/btfrv.h
@@ -57,10 +57,25 @@ RvSocket RvServiceRegister(const char* relay_host, unsigned short port,
 bool RvServiceWaitPaired(RvSocket s, int nTimeoutSecs = 0);
 
 // ---- client side (the node dialing a .btf address) ----
+// Where a failed RvClientConnect stopped. RV_INVALID alone collapses three
+// different situations -- relay unreachable, relay reachable but nobody
+// registered, connection died mid-pairing -- and they point at different
+// components. The caller can only report the right one if this side says
+// which it saw.
+enum RvClientFail
+{
+    RVC_FAIL_NONE = 0,
+    RVC_FAIL_DIAL,    // TCP connect to the relay itself failed
+    RVC_FAIL_REFUSED, // relay answered 0x00: no service registered under this pubkey
+    RVC_FAIL_PAIR,    // relay connection died before pairing completed
+};
+
 // Connect through the relay to the service registered under `target_pubkey`.
-// Returns the paired socket, or RV_INVALID on error.
+// Returns the paired socket, or RV_INVALID on error -- with `pFail`, if
+// given, naming the stage that failed.
 RvSocket RvClientConnect(const char* relay_host, unsigned short port,
-                         const unsigned char target_pubkey[32]);
+                         const unsigned char target_pubkey[32],
+                         RvClientFail* pFail = 0);
 
 // Blocking read/write of exactly n bytes over a rendezvous socket. Return false
 // on error/EOF.
diff --git a/src/btftunnel.cpp b/src/btftunnel.cpp
index 4acca1a..4058f6b 100644
--- a/src/btftunnel.cpp
+++ b/src/btftunnel.cpp
@@ -231,18 +231,47 @@ static btf_socket_t StartTunnel(RvSocket rv, const unsigned char shared[32])
     return appEnd;
 }
 
+const char* TunnelFailText(TunnelFail f)
+{
+    switch (f)
+    {
+    case TUNNEL_FAIL_NONE:      return "no failure";
+    case TUNNEL_FAIL_INIT:      return "local socket/crypto init failed";
+    case TUNNEL_FAIL_DIAL:      return "meeting node unreachable (tcp dial failed)";
+    case TUNNEL_FAIL_REFUSED:   return "relay reachable, but no service registered there for this address";
+    case TUNNEL_FAIL_PAIR:      return "relay connection died before pairing completed";
+    case TUNNEL_FAIL_HANDSHAKE: return "end-to-end handshake failed";
+    case TUNNEL_FAIL_LOCAL:     return "local tunnel setup failed";
+    }
+    return "unknown failure";
+}
+
 btf_socket_t BtfClientTunnel(const char* meeting_host, unsigned short port,
                              const unsigned char target_pubkey[32],
-                             const unsigned char service_enc_pub[32])
+                             const unsigned char service_enc_pub[32],
+                             TunnelFail* pFail)
 {
+    if (pFail) *pFail = TUNNEL_FAIL_INIT;
     if (!TunnelInit()) return INVALID_SOCKET;
-    RvSocket rv = RvClientConnect(meeting_host, port, target_pubkey);
-    if (rv == RV_INVALID) return INVALID_SOCKET;
+    RvClientFail rvFail = RVC_FAIL_NONE;
+    RvSocket rv = RvClientConnect(meeting_host, port, target_pubkey, &rvFail);
+    if (rv == RV_INVALID)
+    {
+        if (pFail)
+            *pFail = rvFail == RVC_FAIL_DIAL    ? TUNNEL_FAIL_DIAL
+                   : rvFail == RVC_FAIL_REFUSED ? TUNNEL_FAIL_REFUSED
+                                                : TUNNEL_FAIL_PAIR;
+        return INVALID_SOCKET;
+    }
 
     unsigned char eph_pub[32], shared[32];
+    if (pFail) *pFail = TUNNEL_FAIL_HANDSHAKE;
     if (!ChanClientHandshake(service_enc_pub, eph_pub, shared)) { RvClose(rv); return INVALID_SOCKET; }
     if (!RvWriteN(rv, eph_pub, 32)) { RvClose(rv); return INVALID_SOCKET; }
-    return StartTunnel(rv, shared);
+    if (pFail) *pFail = TUNNEL_FAIL_LOCAL;
+    btf_socket_t app = StartTunnel(rv, shared);
+    if (app != INVALID_SOCKET && pFail) *pFail = TUNNEL_FAIL_NONE;
+    return app;
 }
 
 btf_socket_t BtfServiceWrap(RvSocket rv, const unsigned char my_enc_sk[32])
diff --git a/src/btftunnel.h b/src/btftunnel.h
index 24a7509..3cc87cd 100644
--- a/src/btftunnel.h
+++ b/src/btftunnel.h
@@ -36,12 +36,33 @@ namespace btf
 // Initialize (sockets + libsodium). Idempotent.
 bool TunnelInit();
 
+// Which stage of BtfClientTunnel failed. INVALID_SOCKET alone says only "no
+// tunnel"; a caller that then guesses at causes in its log line sends whoever
+// reads it to the wrong component (issue #17: "relay unreachable or pool not
+// listening" blamed two healthy parts). The stage is what this side actually
+// knows -- report that.
+enum TunnelFail
+{
+    TUNNEL_FAIL_NONE = 0,
+    TUNNEL_FAIL_INIT,      // local socket/crypto init failed
+    TUNNEL_FAIL_DIAL,      // TCP connect to the meeting node failed
+    TUNNEL_FAIL_REFUSED,   // relay reachable, but no service registered under the pubkey
+    TUNNEL_FAIL_PAIR,      // relay connection died before pairing completed
+    TUNNEL_FAIL_HANDSHAKE, // end-to-end channel handshake failed
+    TUNNEL_FAIL_LOCAL,     // local pump plumbing (socket pair / threads) failed
+};
+
+// Short description of a TunnelFail for log lines.
+const char* TunnelFailText(TunnelFail f);
+
 // Client: dial the service registered under `target_pubkey` at the meeting node
 // (host:port), box to its static x25519 `service_enc_pub`, and return a local
-// plaintext SOCKET tunneled end-to-end to the service. INVALID_SOCKET on error.
+// plaintext SOCKET tunneled end-to-end to the service. INVALID_SOCKET on error
+// -- with `pFail`, if given, naming the stage that failed.
 btf_socket_t BtfClientTunnel(const char* meeting_host, unsigned short port,
                              const unsigned char target_pubkey[32],
-                             const unsigned char service_enc_pub[32]);
+                             const unsigned char service_enc_pub[32],
+                             TunnelFail* pFail = 0);
 
 // Service: wrap an already-paired rendezvous socket (RvServiceRegister followed
 // by RvServiceWaitPaired -- the pairing byte MUST already be consumed) with
diff --git a/src/main.cpp b/src/main.cpp
index 681673c..6b8e0de 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -3970,6 +3970,14 @@ static void ParticipantHashWorker(ParticipantJobShared* sh, int idx, int nWorker
     RandomXDestroyMinerVM(vm);
 }
 
+// Run one participant session: resolve the pool, open the tunnel, subscribe,
+// hash, submit shares. Returns true only if the pool actually handed out work
+// -- at least one well-formed mining.notify arrived. Everything else is false,
+// including a tunnel that opened and was dropped before any job.
+//
+// The caller's reconnect backoff keys off this return value, so any path added
+// here that leaves the session without work MUST return false. "The session
+// ended tidily" is not the question being asked.
 static bool PoolParticipantMiner()
 {
     // ------------------------------------------------------------------
@@ -4047,6 +4055,33 @@ static bool PoolParticipantMiner()
     }
     LogPrint("worker", "[worker] pool resolved to meeting node %s\n", meetingHostPort.c_str());
 
+    // The descriptor names ONE meeting node, but the pool parks a
+    // registration at every relay it knows (one accept thread per relay,
+    // rpc.cpp:AcceptOneFn), so any seed relay can pair us. Insisting on the
+    // advertised entry forever turns one bad path into a miner that idles
+    // indefinitely while a working path exists (issue #17); after a few
+    // consecutive tunnel failures, rotate the seed relays into the attempts
+    // too. Only ONE participant session runs (StartMinerThreads), so these
+    // statics are single-threaded.
+    static const int TUNNEL_FAILOVER_AFTER = 3;
+    static int nTunnelFails = 0;      // consecutive tunnel failures, this pool
+    static size_t iFailoverRelay = 0; // rotation cursor over vBtfMeetingRelays
+    static std::string strFailPool;   // pool the counters belong to
+    if (strFailPool != strParticipantPool) {
+        strFailPool = strParticipantPool;
+        nTunnelFails = 0;
+        iFailoverRelay = 0;
+    }
+    if (nTunnelFails >= TUNNEL_FAILOVER_AFTER && !vBtfMeetingRelays.empty()) {
+        std::string alt = vBtfMeetingRelays[iFailoverRelay++ % vBtfMeetingRelays.size()];
+        if (alt != meetingHostPort) {
+            LogPrint("worker", "[worker] advertised meeting node %s failed %d"
+                     " times in a row -- trying seed relay %s instead\n",
+                     meetingHostPort.c_str(), nTunnelFails, alt.c_str());
+            meetingHostPort = alt;
+        }
+    }
+
     size_t colon = meetingHostPort.rfind(':');
     if (colon == std::string::npos) {
         LogPrint("worker", "[worker] bad meeting node format '%s'\n", meetingHostPort.c_str());
@@ -4066,15 +4101,18 @@ static bool PoolParticipantMiner()
     // Step 2b: open encrypted tunnel to pool
     // ------------------------------------------------------------------
     SetParticipantMiningStatus("opening tunnel");
+    btf::TunnelFail tunnelFail = btf::TUNNEL_FAIL_NONE;
     s = btf::BtfClientTunnel(host.c_str(), (unsigned short)port,
-                             target_pubkey, service_enc_pub);
+                             target_pubkey, service_enc_pub, &tunnelFail);
     if (s == INVALID_SOCKET) {
-        LogPrint("worker", "[worker] tunnel connect FAILED via %s"
-                 " (relay unreachable or pool not listening)\n",
-                 meetingHostPort.c_str());
-        SetParticipantMiningStatus("tunnel connect failed");
+        nTunnelFails++;
+        LogPrint("worker", "[worker] tunnel connect FAILED via %s: %s\n",
+                 meetingHostPort.c_str(), btf::TunnelFailText(tunnelFail));
+        SetParticipantMiningStatus(std::string("tunnel connect failed: ")
+                                   + btf::TunnelFailText(tunnelFail));
         return false;
     }
+    nTunnelFails = 0;
     LogPrint("worker", "[worker] tunnel open via %s\n", meetingHostPort.c_str());
     SetParticipantMiningStatus("tunnel open");
     }
@@ -4253,6 +4291,7 @@ connected:
     int  nBestHeightResolved = -1;      // nBestHeight when we last resolved
     double currentDifficulty = 1.0; // updated by mining.set_difficulty from pool
     bool haveJob         = false;
+    bool gotAnyJob       = false; // any job at all this session -- the return value
     bool gotSubscribeAck = false;
     bool gotAuthorizeAck = false;
     int64 authWaitStart  = 0;
@@ -4369,8 +4408,9 @@ connected:
                                     jobId = newJobId;
                                     memcpy(header, hdrBytes.data(), 80);
                                     memcpy(&target, tgtBytes.data(), 32);
-                                    haveJob = true;
-                                    nNonce  = 1;
+                                    haveJob   = true;
+                                    gotAnyJob = true;
+                                    nNonce    = 1;
                                     {
                                         std::lock_guard<std::mutex> lk(jobShared.mtx);
                                         memcpy(jobShared.header, header, 80);
@@ -4562,7 +4602,11 @@ connected:
              (unsigned long long)hashesSinceSample,
              (unsigned long long)sessionSharesSent,
              (unsigned long long)sessionSharesAccepted);
-    return true;
+    // Not "we got here", which is true of a tunnel the pool accepted and then
+    // dropped without ever sending a job -- exactly the case the caller's
+    // backoff exists for. Reaching the end of the session is not evidence the
+    // session was worth anything; a job is.
+    return gotAnyJob;
 }
 
 // ---------------------------------------------------------------------------
@@ -4759,11 +4803,51 @@ bool BitcoinMiner(int nThreadId)
     // Participant mode: connect to external pool instead of mining solo/operator
     if (nMineMode == MINE_PARTICIPANT)
     {
+        // An attempt that never got work backs off (10s doubling to 120s)
+        // instead of hammering the same path every flat 10s, and a streak
+        // lasting minutes is said at printf level: the identical once-per-10s
+        // LogPrint line is how a worker idled all night unnoticed (issue #17).
+        //
+        // Only a session the pool actually sent a job to resets either. The
+        // weaker test -- "the session returned" -- was gameable: a pool that
+        // completes the tunnel and drops without ever sending work leaves by
+        // the same tidy cleanup path a healthy one does, which would pin the
+        // retry at 10s forever and defeat the backoff in exactly the scenario
+        // it exists for.
+        int   nRetrySecs    = 10;
+        int64 nFailingSince = 0;
         while (fGenerateBitcoins && nMineMode == MINE_PARTICIPANT && !fShutdown) {
-            PoolParticipantMiner();
+            bool fGotWork = PoolParticipantMiner();
             if (fGenerateBitcoins && nMineMode == MINE_PARTICIPANT && !fShutdown) {
-                LogPrint("worker", "PoolParticipantMiner: reconnecting in 10s\n");
-                for (int i = 0; i < 10 && !fShutdown; i++) Sleep(1000);
+                if (fGotWork) {
+                    nRetrySecs    = 10;
+                    nFailingSince = 0;
+                } else {
+                    if (nFailingSince == 0)
+                        nFailingSince = GetTime();
+                    int64 nMins = (GetTime() - nFailingSince) / 60;
+                    if (nMins >= 1)
+                        printf("[worker] connection to pool %s has been failing"
+                               " for %lld minutes\n",
+                               strParticipantPool.c_str(), (long long)nMins);
+                }
+                LogPrint("worker", "PoolParticipantMiner: reconnecting in %ds\n",
+                         nRetrySecs);
+                // Poll everything the enclosing while polls, not just
+                // fShutdown. Raising the ceiling to 120s made a
+                // shutdown-only predicate expensive: after the operator
+                // clears Generate or switches to Relay, the session -- and
+                // the vfThreadRunning[3] slot that gates restarting it --
+                // would stay held for up to two minutes. Measured 119s
+                // before, 1s after.
+                for (int i = 0; i < nRetrySecs && !fShutdown && fGenerateBitcoins
+                                && nMineMode == MINE_PARTICIPANT; i++)
+                    Sleep(1000);
+                if (!fGotWork && nRetrySecs < 120) {
+                    nRetrySecs *= 2;
+                    if (nRetrySecs > 120)
+                        nRetrySecs = 120;
+                }
             }
         }
         return true;