Merge pull request 'wallet: add experimental SQLite record store' (#47) from wallet-sqlite-skeleton into main

6 files changed, 414 insertions, 9 deletions

diff --git a/src/Makefile b/src/Makefile
index 19ad33a..cfa40f6 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -40,7 +40,7 @@ FUZZ_LDFLAGS = -fsanitize=fuzzer,address,undefined
 
 LIBS = -L$(SECP_LIB) -lsecp256k1 \
        -L$(RANDOMX_DIR)/build -lrandomx \
-       -lsodium -lssl -lcrypto -ldb_cxx -lpthread \
+       -lsodium -lssl -lcrypto -ldb_cxx -lsqlite3 -lpthread \
        -lglfw -lGL
 
 # Every header a .cpp here includes has to be listed, not just the interesting
@@ -53,7 +53,7 @@ LIBS = -L$(SECP_LIB) -lsecp256k1 \
 HEADERS = headers_core.h headers.h util.h main.h serialize.h uint256.h \
           key.h bignum.h script.h db.h base58.h net.h nostr.h \
           randomx_pow.h btfaddr.h btfchan.h btfrv.h btftunnel.h market.h \
-          selftest.h walletcmd.h compat.h sockcount.h proxy.h tor.h font_roboto.h
+          selftest.h walletcmd.h wallet_sqlite.h compat.h sockcount.h proxy.h tor.h font_roboto.h
 
 IMGUI_OBJS = obj/imgui.o obj/imgui_draw.o obj/imgui_tables.o \
              obj/imgui_widgets.o obj/imgui_impl_glfw.o obj/imgui_impl_opengl3.o
@@ -61,7 +61,8 @@ IMGUI_OBJS = obj/imgui.o obj/imgui_draw.o obj/imgui_tables.o \
 NODE_OBJS = obj/util.o obj/script.o obj/db.o obj/net.o obj/main.o \
             obj/market.o obj/sha.o obj/nostr.o obj/randomx_pow.o \
             obj/btfaddr.o obj/btfchan.o obj/btfrv.o obj/btftunnel.o obj/rpc.o \
-            obj/selftest.o obj/bip32.o obj/walletcmd.o obj/proxy.o obj/tor.o
+            obj/selftest.o obj/bip32.o obj/walletcmd.o obj/wallet_sqlite.o \
+            obj/proxy.o obj/tor.o
 
 GUI_OBJS = obj/gui.o obj/main_gui.o
 
@@ -116,7 +117,7 @@ bitflash: $(IMGUI_OBJS) $(NODE_OBJS) $(GUI_OBJS)
 
 NODE_LIBS = -L$(SECP_LIB) -lsecp256k1 \
             -L$(RANDOMX_DIR)/build -lrandomx \
-            -lsodium -lssl -lcrypto -ldb_cxx -lpthread
+            -lsodium -lssl -lcrypto -ldb_cxx -lsqlite3 -lpthread
 
 NODE_ONLY_OBJS = $(patsubst obj/%,obj-node/%,$(NODE_OBJS)) \
                  obj-node/main_gui.o obj-node/headless.o
@@ -187,6 +188,7 @@ tests: test_bip32 bitflash-node
         ./bitflash-node -selftest=wallet-keypool
         ./bitflash-node -selftest=wallet-hd
         ./bitflash-node -selftest=wallet-format
+        ./bitflash-node -selftest=wallet-sqlite
         ./bitflash-node -selftest=wallet-crypto
         ./bitflash-node -selftest=wallet-encrypt
         ./bitflash-node -selftest=net-message
diff --git a/src/main_gui.cpp b/src/main_gui.cpp
index 705b4ef..965c818 100644
--- a/src/main_gui.cpp
+++ b/src/main_gui.cpp
@@ -114,7 +114,8 @@ static void PrintUsage()
     printf("  /gen\n");
     printf("  /nogui or /daemon\n");
     printf("  /selftest=wallet-keypool, wallet-hd, wallet-format, wallet-storage-sanity,\n");
-    printf("            db-env-reopen, wallet-crypto, wallet-encrypt, wallet-portability,\n");
+    printf("            db-env-reopen, wallet-sqlite, wallet-crypto, wallet-encrypt,\n");
+    printf("            wallet-portability,\n");
     printf("            net-message,\n");
     printf("            consensus-limits, pool-stratum,\n");
     printf("            parse-money, socks5-proxy, or managed-tor\n");
diff --git a/src/makefile.mingw b/src/makefile.mingw
index 333be96..2ff4e11 100644
--- a/src/makefile.mingw
+++ b/src/makefile.mingw
@@ -24,7 +24,7 @@ CXXFLAGS = -std=gnu++14 -O2 $(WARNFLAGS) -fpermissive -pthread \
 
 LIBS = -L$(DEPS)/lib -lsecp256k1 \
        -L$(RANDOMX)/build -lrandomx \
-       -lsodium -lssl -lcrypto -ldb_cxx -lpthread \
+       -lsodium -lssl -lcrypto -ldb_cxx -lsqlite3 -lpthread \
        -LC:/msys64/ucrt64/lib \
        -lglfw3 -lopengl32 -lgdi32 \
        -lkernel32 -luser32 -lcomdlg32 -lwinspool -lwinmm \
@@ -37,7 +37,8 @@ IMGUI_OBJS = obj/imgui.o obj/imgui_draw.o obj/imgui_tables.o \
 NODE_OBJS = obj/util.o obj/script.o obj/db.o obj/net.o obj/main.o \
             obj/market.o obj/sha.o obj/nostr.o obj/randomx_pow.o \
             obj/btfaddr.o obj/btfchan.o obj/btfrv.o obj/btftunnel.o obj/rpc.o \
-            obj/selftest.o obj/bip32.o obj/walletcmd.o obj/proxy.o obj/tor.o
+            obj/selftest.o obj/bip32.o obj/walletcmd.o obj/wallet_sqlite.o \
+            obj/proxy.o obj/tor.o
 
 GUI_OBJS = obj/gui.o obj/main_gui.o obj/bitflash_res.o
 
@@ -51,7 +52,7 @@ GUI_OBJS = obj/gui.o obj/main_gui.o obj/bitflash_res.o
 HEADERS = headers_core.h headers.h util.h main.h serialize.h uint256.h \
           key.h bignum.h script.h db.h base58.h net.h nostr.h \
           randomx_pow.h btfaddr.h btfchan.h btfrv.h btftunnel.h market.h \
-          selftest.h walletcmd.h sockcount.h proxy.h tor.h compat.h font_roboto.h
+          selftest.h walletcmd.h wallet_sqlite.h sockcount.h proxy.h tor.h compat.h font_roboto.h
 
 all: bitflash.exe
 
@@ -107,6 +108,7 @@ tests: test_bip32.exe bitflash.exe
         ./bitflash.exe -selftest=wallet-keypool
         ./bitflash.exe -selftest=wallet-hd
         ./bitflash.exe -selftest=wallet-format
+        ./bitflash.exe -selftest=wallet-sqlite
         ./bitflash.exe -selftest=wallet-crypto
         ./bitflash.exe -selftest=wallet-encrypt
         ./bitflash.exe -selftest=net-message
diff --git a/src/selftest.cpp b/src/selftest.cpp
index 31c168f..f9c4a8e 100644
--- a/src/selftest.cpp
+++ b/src/selftest.cpp
@@ -13,6 +13,7 @@
 #include "sockcount.h"
 #include "tor.h"
 #include "walletcmd.h"
+#include "wallet_sqlite.h"
 
 extern int RunPoolStratumSelfTest();
 
@@ -62,6 +63,11 @@ static std::string HexStrLocal(const std::vector<unsigned char>& v)
     return out;
 }
 
+static vector<unsigned char> DataStreamBytes(const CDataStream& ss)
+{
+    return vector<unsigned char>(ss.begin(), ss.end());
+}
+
 static bool MakeTempDir(std::string& out)
 {
 #ifdef _WIN32
@@ -1030,6 +1036,104 @@ static int RunDbEnvReopenSelfTest()
     return nFail == 0 ? 0 : 1;
 }
 
+static int RunWalletSQLiteSelfTest()
+{
+    fflush(stdout);
+    printf("wallet-sqlite self-test\n");
+
+    std::string tmp;
+    if (!MakeTempDir(tmp))
+    {
+        printf("  FAIL could not create a temporary data directory\n");
+        return 1;
+    }
+
+    int nFail = 0;
+    string strPath = tmp + "/wallet.sqlite";
+    string strError;
+
+    try
+    {
+        CDataStream ssKeySetting(SER_DISK);
+        ssKeySetting << make_pair(string("setting"), string("sqlite-roundtrip"));
+        CDataStream ssValueSetting(SER_DISK);
+        ssValueSetting << (int64)424242;
+
+        vector<unsigned char> vchPubKey;
+        vchPubKey.push_back(0x04);
+        for (int i = 0; i < 64; i++)
+            vchPubKey.push_back((unsigned char)i);
+        CDataStream ssKeyDefault(SER_DISK);
+        ssKeyDefault << string("defaultkey");
+        CDataStream ssValueDefault(SER_DISK);
+        ssValueDefault << vchPubKey;
+
+        {
+            CWalletDBSQLite db;
+            nFail += Check(db.Open(strPath, strError),
+                           "SQLite wallet opens and creates schema") ? 0 : 1;
+            nFail += Check(db.BeginTransaction(strError),
+                           "SQLite wallet begins a write transaction") ? 0 : 1;
+            nFail += Check(db.WriteRecord(DataStreamBytes(ssKeySetting),
+                                          DataStreamBytes(ssValueSetting),
+                                          strError),
+                           "SQLite wallet writes a serialized setting record") ? 0 : 1;
+            nFail += Check(db.WriteRecord(DataStreamBytes(ssKeyDefault),
+                                          DataStreamBytes(ssValueDefault),
+                                          strError),
+                           "SQLite wallet writes a serialized defaultkey record") ? 0 : 1;
+            nFail += Check(db.CommitTransaction(strError),
+                           "SQLite wallet commits a write transaction") ? 0 : 1;
+            int nCount = 0;
+            nFail += Check(db.CountRecords(nCount, strError) && nCount == 2,
+                           "SQLite wallet counts raw records") ? 0 : 1;
+        }
+
+        {
+            CWalletDBSQLite db;
+            nFail += Check(db.Open(strPath, strError),
+                           "SQLite wallet reopens") ? 0 : 1;
+            vector<unsigned char> vchRead;
+            nFail += Check(db.ReadRecord(DataStreamBytes(ssKeySetting),
+                                         vchRead,
+                                         strError),
+                           "SQLite wallet reads a serialized setting record") ? 0 : 1;
+            int64 nReadValue = 0;
+            CDataStream ssRead(vchRead, SER_DISK);
+            ssRead >> nReadValue;
+            nFail += Check(!ssRead.fail() && nReadValue == 424242,
+                           "SQLite wallet preserves serialized values byte-for-byte") ? 0 : 1;
+
+            vchRead.clear();
+            nFail += Check(db.ReadRecord(DataStreamBytes(ssKeyDefault),
+                                         vchRead,
+                                         strError),
+                           "SQLite wallet reads a serialized defaultkey record") ? 0 : 1;
+            vector<unsigned char> vchReadPubKey;
+            CDataStream ssReadDefault(vchRead, SER_DISK);
+            ssReadDefault >> vchReadPubKey;
+            nFail += Check(!ssReadDefault.fail() && vchReadPubKey == vchPubKey,
+                           "SQLite wallet preserves public key blobs") ? 0 : 1;
+        }
+    }
+    catch (const std::exception& e)
+    {
+        printf("  FAIL exception: %s\n", e.what());
+        nFail++;
+    }
+    catch (...)
+    {
+        printf("  FAIL unknown exception\n");
+        nFail++;
+    }
+
+    RemoveTree(tmp);
+    printf("%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
+           nFail, nFail == 1 ? "" : "s");
+    fflush(stdout);
+    return nFail == 0 ? 0 : 1;
+}
+
 static void ClearWalletRuntimeForTest()
 {
     CRITICAL_BLOCK(cs_mapKeys)
@@ -2289,6 +2393,8 @@ int RunSelfTest(const std::string& name)
         return RunWalletStorageSanitySelfTest();
     if (name == "db-env-reopen")
         return RunDbEnvReopenSelfTest();
+    if (name == "wallet-sqlite")
+        return RunWalletSQLiteSelfTest();
     if (name == "wallet-crypto")
         return RunWalletCryptoSelfTest();
     if (name == "wallet-encrypt")
@@ -2309,6 +2415,6 @@ int RunSelfTest(const std::string& name)
         return RunManagedTorSelfTest();
 
     printf("Unknown self-test '%s'\n", name.c_str());
-    printf("Known self-tests: wallet-keypool, wallet-hd, wallet-format, wallet-storage-sanity, db-env-reopen, wallet-crypto, wallet-encrypt, wallet-portability, net-message, consensus-limits, pool-stratum, parse-money, socks5-proxy, managed-tor\n");
+    printf("Known self-tests: wallet-keypool, wallet-hd, wallet-format, wallet-storage-sanity, db-env-reopen, wallet-sqlite, wallet-crypto, wallet-encrypt, wallet-portability, net-message, consensus-limits, pool-stratum, parse-money, socks5-proxy, managed-tor\n");
     return 1;
 }
diff --git a/src/wallet_sqlite.cpp b/src/wallet_sqlite.cpp
new file mode 100644
index 0000000..4104ac6
--- /dev/null
+++ b/src/wallet_sqlite.cpp
@@ -0,0 +1,254 @@
+// Copyright (c) 2026 Bitflash developers
+// Distributed under the MIT/X11 software license.
+
+#include "headers.h"
+#include "wallet_sqlite.h"
+
+static void SetSQLiteError(sqlite3* pdb, const string& strContext,
+                           string& strError)
+{
+    if (pdb)
+        strError = strContext + ": " + sqlite3_errmsg(pdb);
+    else
+        strError = strContext;
+}
+
+CWalletDBSQLite::CWalletDBSQLite() : pdb(NULL)
+{
+}
+
+CWalletDBSQLite::~CWalletDBSQLite()
+{
+    Close();
+}
+
+bool CWalletDBSQLite::Open(const string& strPath, string& strError)
+{
+    Close();
+
+    int ret = sqlite3_open_v2(strPath.c_str(), &pdb,
+                              SQLITE_OPEN_READWRITE |
+                              SQLITE_OPEN_CREATE |
+                              SQLITE_OPEN_FULLMUTEX,
+                              NULL);
+    if (ret != SQLITE_OK)
+    {
+        SetSQLiteError(pdb, "cannot open SQLite wallet", strError);
+        Close();
+        return false;
+    }
+
+    sqlite3_busy_timeout(pdb, 5000);
+
+    if (!Exec("PRAGMA journal_mode=WAL;", strError))
+    {
+        Close();
+        return false;
+    }
+    if (!Exec("PRAGMA synchronous=FULL;", strError))
+    {
+        Close();
+        return false;
+    }
+    if (!InitSchema(strError))
+    {
+        Close();
+        return false;
+    }
+    return true;
+}
+
+void CWalletDBSQLite::Close()
+{
+    if (pdb)
+    {
+        sqlite3_close(pdb);
+        pdb = NULL;
+    }
+}
+
+bool CWalletDBSQLite::BeginTransaction(string& strError)
+{
+    if (!pdb)
+    {
+        strError = "SQLite wallet is not open";
+        return false;
+    }
+    return Exec("BEGIN IMMEDIATE;", strError);
+}
+
+bool CWalletDBSQLite::CommitTransaction(string& strError)
+{
+    if (!pdb)
+    {
+        strError = "SQLite wallet is not open";
+        return false;
+    }
+    return Exec("COMMIT;", strError);
+}
+
+bool CWalletDBSQLite::RollbackTransaction(string& strError)
+{
+    if (!pdb)
+    {
+        strError = "SQLite wallet is not open";
+        return false;
+    }
+    return Exec("ROLLBACK;", strError);
+}
+
+bool CWalletDBSQLite::Exec(const char* pszSql, string& strError)
+{
+    char* pszErr = NULL;
+    int ret = sqlite3_exec(pdb, pszSql, NULL, NULL, &pszErr);
+    if (ret != SQLITE_OK)
+    {
+        if (pszErr)
+        {
+            strError = pszErr;
+            sqlite3_free(pszErr);
+        }
+        else
+            SetSQLiteError(pdb, "SQLite exec failed", strError);
+        return false;
+    }
+    return true;
+}
+
+bool CWalletDBSQLite::InitSchema(string& strError)
+{
+    return Exec("CREATE TABLE IF NOT EXISTS wallet_records ("
+                "key BLOB PRIMARY KEY NOT NULL,"
+                "value BLOB NOT NULL"
+                ");",
+                strError) &&
+           Exec("PRAGMA user_version=1;", strError);
+}
+
+bool CWalletDBSQLite::WriteRecord(const vector<unsigned char>& vchKey,
+                                  const vector<unsigned char>& vchValue,
+                                  string& strError)
+{
+    if (!pdb)
+    {
+        strError = "SQLite wallet is not open";
+        return false;
+    }
+    if (vchKey.empty())
+    {
+        strError = "SQLite wallet record key is empty";
+        return false;
+    }
+
+    sqlite3_stmt* stmt = NULL;
+    const char* pszSql =
+        "INSERT OR REPLACE INTO wallet_records(key, value) VALUES(?, ?);";
+    int ret = sqlite3_prepare_v2(pdb, pszSql, -1, &stmt, NULL);
+    if (ret != SQLITE_OK)
+    {
+        SetSQLiteError(pdb, "cannot prepare SQLite wallet write", strError);
+        return false;
+    }
+
+    ret = sqlite3_bind_blob(stmt, 1, &vchKey[0], (int)vchKey.size(),
+                            SQLITE_TRANSIENT);
+    if (ret == SQLITE_OK)
+        ret = sqlite3_bind_blob(stmt, 2,
+                                vchValue.empty() ? NULL : &vchValue[0],
+                                (int)vchValue.size(),
+                                SQLITE_TRANSIENT);
+    if (ret == SQLITE_OK)
+        ret = sqlite3_step(stmt);
+
+    bool fOk = (ret == SQLITE_DONE);
+    if (!fOk)
+        SetSQLiteError(pdb, "cannot write SQLite wallet record", strError);
+    sqlite3_finalize(stmt);
+    return fOk;
+}
+
+bool CWalletDBSQLite::ReadRecord(const vector<unsigned char>& vchKey,
+                                 vector<unsigned char>& vchValueRet,
+                                 string& strError)
+{
+    vchValueRet.clear();
+    if (!pdb)
+    {
+        strError = "SQLite wallet is not open";
+        return false;
+    }
+    if (vchKey.empty())
+    {
+        strError = "SQLite wallet record key is empty";
+        return false;
+    }
+
+    sqlite3_stmt* stmt = NULL;
+    const char* pszSql = "SELECT value FROM wallet_records WHERE key = ?;";
+    int ret = sqlite3_prepare_v2(pdb, pszSql, -1, &stmt, NULL);
+    if (ret != SQLITE_OK)
+    {
+        SetSQLiteError(pdb, "cannot prepare SQLite wallet read", strError);
+        return false;
+    }
+
+    ret = sqlite3_bind_blob(stmt, 1, &vchKey[0], (int)vchKey.size(),
+                            SQLITE_TRANSIENT);
+    if (ret != SQLITE_OK)
+    {
+        SetSQLiteError(pdb, "cannot bind SQLite wallet key", strError);
+        sqlite3_finalize(stmt);
+        return false;
+    }
+
+    ret = sqlite3_step(stmt);
+    if (ret == SQLITE_ROW)
+    {
+        const unsigned char* pch =
+            (const unsigned char*)sqlite3_column_blob(stmt, 0);
+        int nBytes = sqlite3_column_bytes(stmt, 0);
+        if (nBytes > 0 && pch)
+            vchValueRet.assign(pch, pch + nBytes);
+        sqlite3_finalize(stmt);
+        return true;
+    }
+
+    if (ret == SQLITE_DONE)
+        strError = "SQLite wallet record not found";
+    else
+        SetSQLiteError(pdb, "cannot read SQLite wallet record", strError);
+    sqlite3_finalize(stmt);
+    return false;
+}
+
+bool CWalletDBSQLite::CountRecords(int& nRecordsRet, string& strError)
+{
+    nRecordsRet = 0;
+    if (!pdb)
+    {
+        strError = "SQLite wallet is not open";
+        return false;
+    }
+
+    sqlite3_stmt* stmt = NULL;
+    int ret = sqlite3_prepare_v2(pdb,
+                                 "SELECT COUNT(*) FROM wallet_records;",
+                                 -1, &stmt, NULL);
+    if (ret != SQLITE_OK)
+    {
+        SetSQLiteError(pdb, "cannot prepare SQLite wallet count", strError);
+        return false;
+    }
+
+    ret = sqlite3_step(stmt);
+    if (ret != SQLITE_ROW)
+    {
+        SetSQLiteError(pdb, "cannot count SQLite wallet records", strError);
+        sqlite3_finalize(stmt);
+        return false;
+    }
+
+    nRecordsRet = sqlite3_column_int(stmt, 0);
+    sqlite3_finalize(stmt);
+    return true;
+}
diff --git a/src/wallet_sqlite.h b/src/wallet_sqlite.h
new file mode 100644
index 0000000..9812d3a
--- /dev/null
+++ b/src/wallet_sqlite.h
@@ -0,0 +1,40 @@
+// Copyright (c) 2026 Bitflash developers
+// Distributed under the MIT/X11 software license.
+
+#ifndef BITFLASH_WALLET_SQLITE_H
+#define BITFLASH_WALLET_SQLITE_H
+
+#include <sqlite3.h>
+
+class CWalletDBSQLite
+{
+public:
+    CWalletDBSQLite();
+    ~CWalletDBSQLite();
+
+    bool Open(const string& strPath, string& strError);
+    void Close();
+
+    bool BeginTransaction(string& strError);
+    bool CommitTransaction(string& strError);
+    bool RollbackTransaction(string& strError);
+
+    bool WriteRecord(const vector<unsigned char>& vchKey,
+                     const vector<unsigned char>& vchValue,
+                     string& strError);
+    bool ReadRecord(const vector<unsigned char>& vchKey,
+                    vector<unsigned char>& vchValueRet,
+                    string& strError);
+    bool CountRecords(int& nRecordsRet, string& strError);
+
+private:
+    sqlite3* pdb;
+
+    bool Exec(const char* pszSql, string& strError);
+    bool InitSchema(string& strError);
+
+    CWalletDBSQLite(const CWalletDBSQLite&);
+    void operator=(const CWalletDBSQLite&);
+};
+
+#endif