Merge pull request 'wallet: verify SQLite wallet exports' (#50) from wallet-sqlite-verify into main

5 files changed, 165 insertions, 0 deletions

diff --git a/docs/storage.md b/docs/storage.md
index 2d748fa..0a441bc 100644
--- a/docs/storage.md
+++ b/docs/storage.md
@@ -111,6 +111,8 @@ overwrite an existing export and still does not change the runtime wallet
 backend. `CWalletDBSQLite::ScanRecords()` can stream those exported records back
 through the same visitor shape, so tests can compare BDB and SQLite record
 streams without introducing a SQLite runtime wallet loader yet.
+`-walletsqliteverify=FILE` compares the current `wallet.dat` stream with a
+SQLite export and prints only record counts and mismatch counts.
 
 ## `blkindex.dat`
 
diff --git a/src/main_gui.cpp b/src/main_gui.cpp
index 6e89270..caacf13 100644
--- a/src/main_gui.cpp
+++ b/src/main_gui.cpp
@@ -183,6 +183,8 @@ static void PrintUsage()
     printf("                              wallet.dat storage state)\n");
     printf("  /walletsqliteexport=FILE   (copy raw wallet.dat records into an\n");
     printf("                              experimental SQLite store, then exit)\n");
+    printf("  /walletsqliteverify=FILE   (compare wallet.dat with a SQLite export,\n");
+    printf("                              printing counts only, then exit)\n");
     printf("  /rescan                    (walk the chain for coins this wallet owns\n");
     printf("                              but never recorded, then exit)\n");
     printf("\n");
@@ -540,6 +542,15 @@ int main(int argc, char* argv[])
         return nRet;
     }
 
+    string strWalletSQLiteVerify =
+        argval2(argc, argv, "/walletsqliteverify", "-walletsqliteverify");
+    if (!strWalletSQLiteVerify.empty())
+    {
+        int nRet = CmdWalletSQLiteVerify(strWalletSQLiteVerify);
+        DBFlush(true);
+        return nRet;
+    }
+
     // Berkeley DB reports failure by throwing, and CDB's constructor lets it
     // through. Nothing on this path caught anything, so an unreadable
     // blkindex.dat or wallet.dat -- one written by another platform's Berkeley
diff --git a/src/selftest.cpp b/src/selftest.cpp
index 408dd41..4ba90c3 100644
--- a/src/selftest.cpp
+++ b/src/selftest.cpp
@@ -1134,6 +1134,8 @@ static int RunWalletSQLiteSelfTest()
     return nFail == 0 ? 0 : 1;
 }
 
+namespace {
+
 class CWalletRecordMapVisitor : public CWalletRecordVisitor
 {
 public:
@@ -1158,6 +1160,8 @@ public:
     }
 };
 
+} // namespace
+
 static int RunWalletSQLiteMigrationSelfTest()
 {
     fflush(stdout);
@@ -1215,6 +1219,17 @@ static int RunWalletSQLiteMigrationSelfTest()
                        FileContainsText(strExportOut, "SQLite wallet export written"),
                        "wallet.dat can be exported to SQLite") ? 0 : 1;
 
+        string strVerifyOut = tmp + "/sqlite-verify.txt";
+        vector<string> vVerifyArgs;
+        vVerifyArgs.push_back("-datadir=" + strWalletDir);
+        vVerifyArgs.push_back("-nomanagedtor");
+        vVerifyArgs.push_back("-nogui");
+        vVerifyArgs.push_back("-walletsqliteverify=" + strSQLite);
+        int nVerifyRet = RunBitflashChild(strExe, vVerifyArgs, NULL, &strVerifyOut);
+        nFail += Check(nVerifyRet == 0 &&
+                       FileContainsText(strVerifyOut, "verification:              ok"),
+                       "SQLite wallet export verifies against wallet.dat") ? 0 : 1;
+
         string strOverwriteOut = tmp + "/sqlite-export-overwrite.txt";
         int nOverwriteRet =
             RunBitflashChild(strExe, vExportArgs, NULL, &strOverwriteOut);
@@ -1265,6 +1280,32 @@ static int RunWalletSQLiteMigrationSelfTest()
         nFail += Check(db.ScanRecords(sqliteVisitor, strError) &&
                        mapSQLite == mapBDB,
                        "SQLite wallet scanner streams every exported record byte-for-byte") ? 0 : 1;
+
+        db.Close();
+        CWalletDBSQLite dbCorrupt;
+        vector<unsigned char> vchBadValue;
+        vchBadValue.push_back(0xba);
+        vchBadValue.push_back(0xad);
+        nFail += Check(!mapBDB.empty() &&
+                       dbCorrupt.Open(strSQLite, strError) &&
+                       dbCorrupt.BeginTransaction(strError) &&
+                       dbCorrupt.WriteRecord(mapBDB.begin()->first,
+                                             vchBadValue,
+                                             strError) &&
+                       dbCorrupt.CommitTransaction(strError) &&
+                       dbCorrupt.Checkpoint(strError),
+                       "SQLite export can be deliberately corrupted for verifier test") ? 0 : 1;
+        dbCorrupt.Close();
+
+        string strVerifyBadOut = tmp + "/sqlite-verify-bad.txt";
+        int nVerifyBadRet =
+            RunBitflashChild(strExe, vVerifyArgs, NULL, &strVerifyBadOut);
+        nFail += Check(nVerifyBadRet == 2 &&
+                       FileContainsText(strVerifyBadOut,
+                                        "record value mismatches:   1") &&
+                       FileContainsText(strVerifyBadOut,
+                                        "verification:              failed"),
+                       "SQLite wallet verifier detects a mismatched record value") ? 0 : 1;
     }
     catch (const std::exception& e)
     {
diff --git a/src/walletcmd.cpp b/src/walletcmd.cpp
index 20e7e33..36f6783 100644
--- a/src/walletcmd.cpp
+++ b/src/walletcmd.cpp
@@ -796,6 +796,8 @@ static vector<unsigned char> DataStreamBytesLocal(const CDataStream& ss)
     return vector<unsigned char>(ss.begin(), ss.end());
 }
 
+namespace {
+
 class CWalletSQLiteExportVisitor : public CWalletRecordVisitor
 {
 public:
@@ -818,6 +820,32 @@ public:
     }
 };
 
+class CWalletRecordMapVisitor : public CWalletRecordVisitor
+{
+public:
+    std::map<vector<unsigned char>, vector<unsigned char> >& mapRecords;
+
+    explicit CWalletRecordMapVisitor(
+        std::map<vector<unsigned char>, vector<unsigned char> >& mapRecordsIn)
+        : mapRecords(mapRecordsIn) { }
+
+    bool VisitWalletRecord(const CDataStream& ssKey,
+                           const CDataStream& ssValue,
+                           string& strErrorRet)
+    {
+        vector<unsigned char> vchKey = DataStreamBytesLocal(ssKey);
+        if (mapRecords.count(vchKey))
+        {
+            strErrorRet = "duplicate serialized wallet key";
+            return false;
+        }
+        mapRecords[vchKey] = DataStreamBytesLocal(ssValue);
+        return true;
+    }
+};
+
+} // namespace
+
 int CmdWalletSQLiteExport(const std::string& strDest)
 {
     AttachTerminal();
@@ -887,6 +915,85 @@ int CmdWalletSQLiteExport(const std::string& strDest)
     return 0;
 }
 
+int CmdWalletSQLiteVerify(const std::string& strPath)
+{
+    AttachTerminal();
+
+    if (strPath.empty())
+    {
+        fprintf(stderr, "Missing SQLite wallet export path.\n");
+        return 1;
+    }
+    if (!FileExists(strPath.c_str()))
+    {
+        fprintf(stderr, "SQLite wallet export does not exist: %s\n",
+                strPath.c_str());
+        return 1;
+    }
+
+    std::map<vector<unsigned char>, vector<unsigned char> > mapBDB;
+    std::map<vector<unsigned char>, vector<unsigned char> > mapSQLite;
+    string strError;
+
+    CWalletRecordMapVisitor bdbVisitor(mapBDB);
+    if (!ScanWalletRecords(bdbVisitor, strError))
+    {
+        fprintf(stderr, "Cannot scan wallet.dat records: %s\n", strError.c_str());
+        return 1;
+    }
+
+    CWalletDBSQLite db;
+    if (!db.Open(strPath, strError))
+    {
+        fprintf(stderr, "Cannot open SQLite wallet export: %s\n", strError.c_str());
+        return 1;
+    }
+    CWalletRecordMapVisitor sqliteVisitor(mapSQLite);
+    if (!db.ScanRecords(sqliteVisitor, strError))
+    {
+        fprintf(stderr, "Cannot scan SQLite wallet export: %s\n", strError.c_str());
+        return 1;
+    }
+
+    int nMissing = 0;
+    int nMismatched = 0;
+    for (std::map<vector<unsigned char>, vector<unsigned char> >::const_iterator it =
+             mapBDB.begin(); it != mapBDB.end(); ++it)
+    {
+        std::map<vector<unsigned char>, vector<unsigned char> >::const_iterator sit =
+            mapSQLite.find(it->first);
+        if (sit == mapSQLite.end())
+            nMissing++;
+        else if (sit->second != it->second)
+            nMismatched++;
+    }
+
+    int nExtra = 0;
+    for (std::map<vector<unsigned char>, vector<unsigned char> >::const_iterator it =
+             mapSQLite.begin(); it != mapSQLite.end(); ++it)
+    {
+        if (!mapBDB.count(it->first))
+            nExtra++;
+    }
+
+    printf("SQLite wallet export verification\n");
+    printf("  wallet.dat records:        %u\n", (unsigned int)mapBDB.size());
+    printf("  SQLite records:            %u\n", (unsigned int)mapSQLite.size());
+    printf("  missing records:           %d\n", nMissing);
+    printf("  extra records:             %d\n", nExtra);
+    printf("  record value mismatches:   %d\n", nMismatched);
+    if (nMissing == 0 && nExtra == 0 && nMismatched == 0)
+    {
+        printf("  verification:              ok\n");
+        fflush(stdout);
+        return 0;
+    }
+
+    printf("  verification:              failed\n");
+    fflush(stdout);
+    return 2;
+}
+
 int CmdEncryptWallet(const std::string& strPassphrase)
 {
     AttachTerminal();
diff --git a/src/walletcmd.h b/src/walletcmd.h
index b1a29d2..784c74c 100644
--- a/src/walletcmd.h
+++ b/src/walletcmd.h
@@ -75,6 +75,10 @@ int CmdWalletStorageCheck();
 // not change which backend the runtime wallet opens.
 int CmdWalletSQLiteExport(const std::string& strDest);
 
+// Diagnostic/migration staging: compare a SQLite wallet export with the current
+// wallet.dat raw records. Prints counts only, never record keys or values.
+int CmdWalletSQLiteVerify(const std::string& strPath);
+
 // Rewrite wallet.dat so private keys and the HD seed are encrypted with the
 // given passphrase. Returns 0 on success and exits without starting the node.
 int CmdEncryptWallet(const std::string& strPassphrase);