wallet: export BDB records to SQLite

9 files changed, 281 insertions, 3 deletions

diff --git a/docs/storage.md b/docs/storage.md
index f75f9b9..bb414b5 100644
--- a/docs/storage.md
+++ b/docs/storage.md
@@ -104,6 +104,12 @@ runtime wallet still opens Berkeley DB. The purpose is to prove schema creation,
 raw record write/read, close/reopen behavior, and value preservation before any
 BDB-to-SQLite migration or runtime backend flag exists.
 
+`-walletsqliteexport=FILE` is the next staging tool. It streams every raw
+`wallet.dat` record through `ScanWalletRecords()` and writes those same key and
+value bytes into the SQLite table inside one transaction. It refuses to
+overwrite an existing export and still does not change the runtime wallet
+backend.
+
 ## `blkindex.dat`
 
 `blkindex.dat` is accessed through `CTxDB`.
diff --git a/src/Makefile b/src/Makefile
index cfa40f6..b83ff58 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -189,6 +189,7 @@ tests: test_bip32 bitflash-node
         ./bitflash-node -selftest=wallet-hd
         ./bitflash-node -selftest=wallet-format
         ./bitflash-node -selftest=wallet-sqlite
+        ./bitflash-node -selftest=wallet-sqlite-migration
         ./bitflash-node -selftest=wallet-crypto
         ./bitflash-node -selftest=wallet-encrypt
         ./bitflash-node -selftest=net-message
diff --git a/src/main_gui.cpp b/src/main_gui.cpp
index 965c818..6e89270 100644
--- a/src/main_gui.cpp
+++ b/src/main_gui.cpp
@@ -114,8 +114,8 @@ static void PrintUsage()
     printf("  /gen\n");
     printf("  /nogui or /daemon\n");
     printf("  /selftest=wallet-keypool, wallet-hd, wallet-format, wallet-storage-sanity,\n");
-    printf("            db-env-reopen, wallet-sqlite, wallet-crypto, wallet-encrypt,\n");
-    printf("            wallet-portability,\n");
+    printf("            db-env-reopen, wallet-sqlite, wallet-sqlite-migration,\n");
+    printf("            wallet-crypto, wallet-encrypt, wallet-portability,\n");
     printf("            net-message,\n");
     printf("            consensus-limits, pool-stratum,\n");
     printf("            parse-money, socks5-proxy, or managed-tor\n");
@@ -181,6 +181,8 @@ static void PrintUsage()
     printf("                              (write the same audit as deterministic JSON)\n");
     printf("  /walletstoragecheck        (fail closed on inconsistent or unsafe\n");
     printf("                              wallet.dat storage state)\n");
+    printf("  /walletsqliteexport=FILE   (copy raw wallet.dat records into an\n");
+    printf("                              experimental SQLite store, then exit)\n");
     printf("  /rescan                    (walk the chain for coins this wallet owns\n");
     printf("                              but never recorded, then exit)\n");
     printf("\n");
@@ -529,6 +531,15 @@ int main(int argc, char* argv[])
         return nRet;
     }
 
+    string strWalletSQLiteExport =
+        argval2(argc, argv, "/walletsqliteexport", "-walletsqliteexport");
+    if (!strWalletSQLiteExport.empty())
+    {
+        int nRet = CmdWalletSQLiteExport(strWalletSQLiteExport);
+        DBFlush(true);
+        return nRet;
+    }
+
     // Berkeley DB reports failure by throwing, and CDB's constructor lets it
     // through. Nothing on this path caught anything, so an unreadable
     // blkindex.dat or wallet.dat -- one written by another platform's Berkeley
diff --git a/src/makefile.mingw b/src/makefile.mingw
index 2ff4e11..14e6b32 100644
--- a/src/makefile.mingw
+++ b/src/makefile.mingw
@@ -109,6 +109,7 @@ tests: test_bip32.exe bitflash.exe
         ./bitflash.exe -selftest=wallet-hd
         ./bitflash.exe -selftest=wallet-format
         ./bitflash.exe -selftest=wallet-sqlite
+        ./bitflash.exe -selftest=wallet-sqlite-migration
         ./bitflash.exe -selftest=wallet-crypto
         ./bitflash.exe -selftest=wallet-encrypt
         ./bitflash.exe -selftest=net-message
diff --git a/src/selftest.cpp b/src/selftest.cpp
index f9c4a8e..7a54271 100644
--- a/src/selftest.cpp
+++ b/src/selftest.cpp
@@ -1134,6 +1134,150 @@ static int RunWalletSQLiteSelfTest()
     return nFail == 0 ? 0 : 1;
 }
 
+class CWalletRecordMapVisitor : public CWalletRecordVisitor
+{
+public:
+    std::map<vector<unsigned char>, vector<unsigned char> >& mapRecords;
+
+    explicit CWalletRecordMapVisitor(
+        std::map<vector<unsigned char>, vector<unsigned char> >& mapRecordsIn)
+        : mapRecords(mapRecordsIn) { }
+
+    bool VisitWalletRecord(const CDataStream& ssKey,
+                           const CDataStream& ssValue,
+                           string& strErrorRet)
+    {
+        vector<unsigned char> vchKey = DataStreamBytes(ssKey);
+        if (mapRecords.count(vchKey))
+        {
+            strErrorRet = "duplicate serialized wallet key";
+            return false;
+        }
+        mapRecords[vchKey] = DataStreamBytes(ssValue);
+        return true;
+    }
+};
+
+static int RunWalletSQLiteMigrationSelfTest()
+{
+    fflush(stdout);
+    printf("wallet-sqlite-migration self-test\n");
+
+    std::string tmp;
+    std::string cwd;
+    if (!MakeTempDir(tmp))
+    {
+        printf("  FAIL could not create a temporary data directory\n");
+        return 1;
+    }
+    if (!GetCurrentDir(cwd))
+    {
+        printf("  FAIL could not read current directory\n");
+        RemoveTree(tmp);
+        return 1;
+    }
+
+    int nFail = 0;
+    printf("  temp root: %s\n", tmp.c_str());
+
+    try
+    {
+#ifdef _WIN32
+        string strExe = cwd + "\\bitflash.exe";
+#else
+        string strExe = cwd + "/bitflash-node";
+#endif
+        string strWalletDir = tmp + "/wallet";
+        nFail += Check(MakeDirLocal(strWalletDir),
+                       "temporary wallet directory can be created") ? 0 : 1;
+
+        string strCreateOut = tmp + "/create.txt";
+        vector<string> vCreateArgs;
+        vCreateArgs.push_back("-datadir=" + strWalletDir);
+        vCreateArgs.push_back("-nomanagedtor");
+        vCreateArgs.push_back("-nogui");
+        vCreateArgs.push_back("-newphrase");
+        int nCreateRet = RunBitflashChild(strExe, vCreateArgs, NULL, &strCreateOut);
+        nFail += Check(nCreateRet == 0 &&
+                       FileContainsText(strCreateOut, "Write these twelve words down"),
+                       "a phrase wallet can be created for SQLite export") ? 0 : 1;
+
+        string strSQLite = tmp + "/wallet.sqlite";
+        string strExportOut = tmp + "/sqlite-export.txt";
+        vector<string> vExportArgs;
+        vExportArgs.push_back("-datadir=" + strWalletDir);
+        vExportArgs.push_back("-nomanagedtor");
+        vExportArgs.push_back("-nogui");
+        vExportArgs.push_back("-walletsqliteexport=" + strSQLite);
+        int nExportRet = RunBitflashChild(strExe, vExportArgs, NULL, &strExportOut);
+        nFail += Check(nExportRet == 0 &&
+                       FileExists(strSQLite.c_str()) &&
+                       FileContainsText(strExportOut, "SQLite wallet export written"),
+                       "wallet.dat can be exported to SQLite") ? 0 : 1;
+
+        string strOverwriteOut = tmp + "/sqlite-export-overwrite.txt";
+        int nOverwriteRet =
+            RunBitflashChild(strExe, vExportArgs, NULL, &strOverwriteOut);
+        nFail += Check(nOverwriteRet != 0 &&
+                       FileContainsText(strOverwriteOut,
+                                        "Refusing to overwrite existing SQLite wallet export"),
+                       "SQLite wallet export refuses to overwrite") ? 0 : 1;
+        remove((strSQLite + "-wal").c_str());
+        remove((strSQLite + "-shm").c_str());
+
+        std::map<vector<unsigned char>, vector<unsigned char> > mapBDB;
+        string strOldDataDir = strSetDataDir;
+        string strScanError;
+        strSetDataDir = strWalletDir;
+        CWalletRecordMapVisitor visitor(mapBDB);
+        bool fScanOk = ScanWalletRecords(visitor, strScanError);
+        DBFlush(true);
+        strSetDataDir = strOldDataDir;
+        nFail += Check(fScanOk && !mapBDB.empty(),
+                       "wallet.dat records can be scanned for comparison") ? 0 : 1;
+
+        CWalletDBSQLite db;
+        string strError;
+        nFail += Check(db.Open(strSQLite, strError),
+                       "exported SQLite wallet opens") ? 0 : 1;
+        int nSQLiteRecords = 0;
+        nFail += Check(db.CountRecords(nSQLiteRecords, strError) &&
+                       nSQLiteRecords == (int)mapBDB.size(),
+                       "SQLite export has the same record count as wallet.dat") ? 0 : 1;
+
+        bool fAllMatch = true;
+        for (std::map<vector<unsigned char>, vector<unsigned char> >::const_iterator it =
+                 mapBDB.begin(); it != mapBDB.end(); ++it)
+        {
+            vector<unsigned char> vchSQLiteValue;
+            if (!db.ReadRecord(it->first, vchSQLiteValue, strError) ||
+                vchSQLiteValue != it->second)
+            {
+                fAllMatch = false;
+                break;
+            }
+        }
+        nFail += Check(fAllMatch,
+                       "SQLite export preserves every wallet.dat key/value byte-for-byte") ? 0 : 1;
+    }
+    catch (const std::exception& e)
+    {
+        printf("  FAIL exception: %s\n", e.what());
+        nFail++;
+    }
+    catch (...)
+    {
+        printf("  FAIL unknown exception\n");
+        nFail++;
+    }
+
+    RemoveTree(tmp);
+    printf("%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
+           nFail, nFail == 1 ? "" : "s");
+    fflush(stdout);
+    return nFail == 0 ? 0 : 1;
+}
+
 static void ClearWalletRuntimeForTest()
 {
     CRITICAL_BLOCK(cs_mapKeys)
@@ -2395,6 +2539,8 @@ int RunSelfTest(const std::string& name)
         return RunDbEnvReopenSelfTest();
     if (name == "wallet-sqlite")
         return RunWalletSQLiteSelfTest();
+    if (name == "wallet-sqlite-migration")
+        return RunWalletSQLiteMigrationSelfTest();
     if (name == "wallet-crypto")
         return RunWalletCryptoSelfTest();
     if (name == "wallet-encrypt")
@@ -2415,6 +2561,6 @@ int RunSelfTest(const std::string& name)
         return RunManagedTorSelfTest();
 
     printf("Unknown self-test '%s'\n", name.c_str());
-    printf("Known self-tests: wallet-keypool, wallet-hd, wallet-format, wallet-storage-sanity, db-env-reopen, wallet-sqlite, wallet-crypto, wallet-encrypt, wallet-portability, net-message, consensus-limits, pool-stratum, parse-money, socks5-proxy, managed-tor\n");
+    printf("Known self-tests: wallet-keypool, wallet-hd, wallet-format, wallet-storage-sanity, db-env-reopen, wallet-sqlite, wallet-sqlite-migration, wallet-crypto, wallet-encrypt, wallet-portability, net-message, consensus-limits, pool-stratum, parse-money, socks5-proxy, managed-tor\n");
     return 1;
 }
diff --git a/src/wallet_sqlite.cpp b/src/wallet_sqlite.cpp
index 4104ac6..4369af6 100644
--- a/src/wallet_sqlite.cpp
+++ b/src/wallet_sqlite.cpp
@@ -97,6 +97,16 @@ bool CWalletDBSQLite::RollbackTransaction(string& strError)
     return Exec("ROLLBACK;", strError);
 }
 
+bool CWalletDBSQLite::Checkpoint(string& strError)
+{
+    if (!pdb)
+    {
+        strError = "SQLite wallet is not open";
+        return false;
+    }
+    return Exec("PRAGMA wal_checkpoint(TRUNCATE);", strError);
+}
+
 bool CWalletDBSQLite::Exec(const char* pszSql, string& strError)
 {
     char* pszErr = NULL;
diff --git a/src/wallet_sqlite.h b/src/wallet_sqlite.h
index 9812d3a..5ef862d 100644
--- a/src/wallet_sqlite.h
+++ b/src/wallet_sqlite.h
@@ -18,6 +18,7 @@ public:
     bool BeginTransaction(string& strError);
     bool CommitTransaction(string& strError);
     bool RollbackTransaction(string& strError);
+    bool Checkpoint(string& strError);
 
     bool WriteRecord(const vector<unsigned char>& vchKey,
                      const vector<unsigned char>& vchValue,
diff --git a/src/walletcmd.cpp b/src/walletcmd.cpp
index 6fb09a3..20e7e33 100644
--- a/src/walletcmd.cpp
+++ b/src/walletcmd.cpp
@@ -12,6 +12,7 @@
 #include "headers_core.h"
 #include "bip32.h"
 #include "walletcmd.h"
+#include "wallet_sqlite.h"
 
 #undef printf
 
@@ -790,6 +791,102 @@ int CmdWalletStorageAudit(const std::string& strJsonOut)
     return 0;
 }
 
+static vector<unsigned char> DataStreamBytesLocal(const CDataStream& ss)
+{
+    return vector<unsigned char>(ss.begin(), ss.end());
+}
+
+class CWalletSQLiteExportVisitor : public CWalletRecordVisitor
+{
+public:
+    CWalletDBSQLite& db;
+    int& nCopied;
+
+    CWalletSQLiteExportVisitor(CWalletDBSQLite& dbIn, int& nCopiedIn)
+        : db(dbIn), nCopied(nCopiedIn) { }
+
+    bool VisitWalletRecord(const CDataStream& ssKey,
+                           const CDataStream& ssValue,
+                           string& strErrorRet)
+    {
+        if (!db.WriteRecord(DataStreamBytesLocal(ssKey),
+                            DataStreamBytesLocal(ssValue),
+                            strErrorRet))
+            return false;
+        nCopied++;
+        return true;
+    }
+};
+
+int CmdWalletSQLiteExport(const std::string& strDest)
+{
+    AttachTerminal();
+
+    if (strDest.empty())
+    {
+        fprintf(stderr, "Missing SQLite wallet export path.\n");
+        return 1;
+    }
+    if (FileExists(strDest.c_str()))
+    {
+        fprintf(stderr, "Refusing to overwrite existing SQLite wallet export: %s\n",
+                strDest.c_str());
+        return 1;
+    }
+
+    string strError;
+    CWalletDBSQLite db;
+    if (!db.Open(strDest, strError))
+    {
+        fprintf(stderr, "Cannot open SQLite wallet export: %s\n", strError.c_str());
+        return 1;
+    }
+    if (!db.BeginTransaction(strError))
+    {
+        fprintf(stderr, "Cannot begin SQLite wallet export: %s\n", strError.c_str());
+        return 1;
+    }
+
+    int nCopied = 0;
+    CWalletSQLiteExportVisitor visitor(db, nCopied);
+    if (!ScanWalletRecords(visitor, strError))
+    {
+        string strRollbackError;
+        db.RollbackTransaction(strRollbackError);
+        fprintf(stderr, "Cannot export wallet.dat to SQLite: %s\n", strError.c_str());
+        return 1;
+    }
+    if (!db.CommitTransaction(strError))
+    {
+        fprintf(stderr, "Cannot commit SQLite wallet export: %s\n", strError.c_str());
+        return 1;
+    }
+    if (!db.Checkpoint(strError))
+    {
+        fprintf(stderr, "Cannot checkpoint SQLite wallet export: %s\n", strError.c_str());
+        return 1;
+    }
+
+    int nCount = 0;
+    if (!db.CountRecords(nCount, strError))
+    {
+        fprintf(stderr, "Cannot verify SQLite wallet export count: %s\n",
+                strError.c_str());
+        return 1;
+    }
+    if (nCount != nCopied)
+    {
+        fprintf(stderr, "SQLite wallet export count mismatch: copied %d, stored %d\n",
+                nCopied, nCount);
+        return 1;
+    }
+
+    printf("SQLite wallet export written to %s\n", strDest.c_str());
+    printf("  records copied:            %d\n", nCopied);
+    fflush(stdout);
+    return 0;
+}
+
 int CmdEncryptWallet(const std::string& strPassphrase)
 {
     AttachTerminal();
diff --git a/src/walletcmd.h b/src/walletcmd.h
index 6077669..b1a29d2 100644
--- a/src/walletcmd.h
+++ b/src/walletcmd.h
@@ -70,6 +70,11 @@ int CmdWalletStorageAudit(const std::string& strJsonOut = "");
 // or migration-unsafe mix of storage records.
 int CmdWalletStorageCheck();
 
+// Diagnostic/migration staging: copy every raw wallet.dat record into an
+// experimental SQLite key/value store, preserving serialized bytes. This does
+// not change which backend the runtime wallet opens.
+int CmdWalletSQLiteExport(const std::string& strDest);
+
 // Rewrite wallet.dat so private keys and the HD seed are encrypted with the
 // given passphrase. Returns 0 on success and exits without starting the node.
 int CmdEncryptWallet(const std::string& strPassphrase);