From 99f7c1dfebb1a15dc6f77a338a8b0aaa3962564b Mon Sep 17 00:00:00 2001 From: Bitflash Date: Thu, 30 Jul 2026 11:27:49 -0300 Subject: wallet: make wallet.dat a file you can actually back up (#46) * wallet: make wallet.dat a file you can actually back up Issue #40: a wallet.dat copied out on its own will not open anywhere else. The file is intact -- same bytes, same keys -- and the node refuses it with "Db::put: Invalid argument", thrown from a path that aborted the process. The reporter believes they lost funds to this, having saved the one file anybody would think to save. The chain, from the bottom: _endthread() is pthread_exit() on POSIX, and glibc implements that by throwing abi::__forced_unwind through the stack. That is not a real exception; catching it without rethrowing is a fatal error by contract. CATCH_PRINT_EXCEPTION ended in a bare catch(...), so every Linux shutdown ended: UNKNOWN EXCEPTION bitflash-node in ThreadSocketHandler() FATAL: exception not rethrown An abort, mid-shutdown, so nothing after StopNode() ran. Which mattered because Berkeley DB stamps every page with a log sequence number belonging to the environment that wrote it. Carried elsewhere, those numbers point at logs that do not exist and BDB refuses to write. DBFlush(true) calls lsn_reset, which is precisely the cure -- and DBFlush has been declared since 2009 and called from nowhere. Even had it been called, the abort stood in front of it. So the wallet was never released from its directory, and the one file users copy was the one file that could not travel. Three fixes, each covering a different case: - Rethrow abi::__forced_unwind, and call DBFlush(true) on both shutdown paths. Shutdown completes, and wallet.dat is left portable by construction. - Adopt orphaned files on open: a freshly created database environment beside pre-existing .dat files means they came from elsewhere, so reset their sequence numbers. This is what recovers wallets already stranded by older builds -- including, if they still hold the file, the reporter's. - BackupWallet(), reachable as /backupwallet=FILE, which checkpoints, copies, and clears the copy's sequence numbers. Bitcoin 0.1.0 had a backup command; this tree lost it in the wxWidgets removal and has shipped with no way to take a backup since. DBFlush's own txn_checkpoint and lsn_reset are wrapped too -- they throw, and throwing on the way out is how this started. Verified on Linux and Windows: shutdown exits 0 rather than 134, DBFlush runs, and a lone wallet.dat opens in an empty directory with no exceptions and byte-identical key records (defaultkey, key, name, setting, version) before and after. Not addressed here, and the larger half of #40: keys are generated as they are needed rather than derived from a seed, so any backup is a snapshot and coins paid to addresses created after it are not in it. /backupwallet says so on every run. The real answer is deterministic derivation, which is a wallet format change and its own piece of work. * README: document what a wallet backup actually requires The PR description claimed this section existed and it did not. Covers the two things that cost money and are not obvious: a lone wallet.dat is refused elsewhere because Berkeley DB ties it to the database/ directory beside it, and every backup is a snapshot because keys are made on demand rather than derived from a seed. --- src/db.cpp | 134 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- src/db.h | 6 +++ src/main_gui.cpp | 32 +++++++++++++ src/util.h | 32 +++++++++++++ 4 files changed, 201 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/db.cpp b/src/db.cpp index d43e4ef..fc5f180 100644 --- a/src/db.cpp +++ b/src/db.cpp @@ -59,7 +59,12 @@ CDB::CDB(const char* pszFile, const char* pszMode, bool fTxn) : pdb(NULL) { string strAppDir = GetAppDir(); string strLogDir = strAppDir + "/database"; - _mkdir(strLogDir.c_str()); + + // _mkdir returns 0 only when it actually created the directory, so + // this is also the test for "this environment did not exist until + // now". See the recovery block after dbenv.open() for why we care. + bool fFreshEnv = (_mkdir(strLogDir.c_str()) == 0); + printf("dbenv.open strAppDir=%s\n", strAppDir.c_str()); dbenv.set_lg_dir(strLogDir.c_str()); @@ -81,6 +86,41 @@ CDB::CDB(const char* pszFile, const char* pszMode, bool fTxn) : pdb(NULL) if (ret > 0) throw runtime_error(strprintf("CDB() : error %d opening database environment\n", ret)); fDbEnvInit = true; + + // A Berkeley DB file records, in every page header, a log sequence + // number belonging to the environment that wrote it. Carried into a + // different environment those numbers refer to logs that do not + // exist, and BDB refuses to write: "Db::put: Invalid argument", + // thrown from a path that used to abort the process. + // + // That is what happens when someone backs up wallet.dat on its own. + // It is the obviously important file, it is the only one people + // copy, and the environment it is welded to lives in a database/ + // subdirectory nobody has reason to suspect. The file is intact -- + // same bytes, same keys -- and the wallet still will not open. + // Issue #40 reports exactly this, after the fact. + // + // lsn_reset() exists for precisely this migration, so do it for + // them. A fresh environment beside pre-existing .dat files means + // those files came from somewhere else; in the normal case the + // environment is already there and none of this runs. + if (fFreshEnv) + { + static const char* pszBdbFiles[] = { "wallet.dat", "blkindex.dat" }; + for (int i = 0; i < (int)(sizeof(pszBdbFiles)/sizeof(pszBdbFiles[0])); i++) + { + const char* pszName = pszBdbFiles[i]; + if (!FileExists((strAppDir + "/" + pszName).c_str())) + continue; + try + { + if (dbenv.lsn_reset(pszName, 0) == 0) + printf("CDB() : adopted %s from another environment " + "(log sequence numbers reset)\n", pszName); + } + catch (...) { } // best effort; the open below reports real trouble + } + } } strFile = pszFile; @@ -137,7 +177,14 @@ void DBFlush(bool fShutdown) printf("DBFlush(%s)\n", fShutdown ? "true" : "false"); CRITICAL_BLOCK(cs_db) { - dbenv.txn_checkpoint(0, 0, 0); + // This runs on the way out. Anything that throws here and is not caught + // takes the process down before the wallet has been put down cleanly, + // which is the difference between a portable wallet.dat and one welded + // to this directory -- the lsn_reset just below is what frees it. + try { dbenv.txn_checkpoint(0, 0, 0); } + catch (const std::exception& e) + { printf("DBFlush() : checkpoint failed: %s\n", e.what()); } + map::iterator mi = mapFileUseCount.begin(); while (mi != mapFileUseCount.end()) { @@ -145,7 +192,9 @@ void DBFlush(bool fShutdown) int nRefCount = (*mi).second; if (nRefCount == 0) { - dbenv.lsn_reset(strFile.c_str(), 0); + try { dbenv.lsn_reset(strFile.c_str(), 0); } + catch (const std::exception& e) + { printf("DBFlush() : lsn_reset(%s) failed: %s\n", strFile.c_str(), e.what()); } mapFileUseCount.erase(mi++); } else @@ -580,3 +629,82 @@ bool LoadWallet() return true; } + + +// Write a copy of wallet.dat that can be opened anywhere. +// +// A plain file copy is not a backup here. Berkeley DB stamps every page with a +// log sequence number tied to the environment that wrote it, so the copy only +// works beside the database/ directory it grew up with -- which is why backing +// up "just wallet.dat", the one file anybody would think to save, produces +// something that will not open. See the recovery block in CDB::CDB and #40. +// +// So: flush outstanding writes, copy, then clear the copy's log sequence +// numbers. What lands on disk is a wallet.dat that stands on its own. +// +// This is still a point-in-time snapshot. Keys are generated as they are +// needed, not derived from a seed, so coins paid to an address created after +// this file was written are not spendable from it. Backing up once is not +// enough, and that is a property of the wallet format rather than of this +// function. +bool BackupWallet(const string& strDest) +{ + if (strDest.empty()) + return error("BackupWallet() : no destination given\n"); + + string strSrc = GetAppDir() + "/wallet.dat"; + if (!FileExists(strSrc.c_str())) + return error("BackupWallet() : %s does not exist\n", strSrc.c_str()); + + CRITICAL_BLOCK(cs_db) + { + // Committing first, so the copy is not missing the newest records. + // Both calls can throw, and a failed backup must not take the node + // down with it. + try { dbenv.txn_checkpoint(0, 0, 0); } + catch (const std::exception& e) + { return error("BackupWallet() : checkpoint failed: %s\n", e.what()); } + + try + { + FILE* pfIn = fopen(strSrc.c_str(), "rb"); + if (!pfIn) + return error("BackupWallet() : cannot read %s\n", strSrc.c_str()); + FILE* pfOut = fopen(strDest.c_str(), "wb"); + if (!pfOut) + { + fclose(pfIn); + return error("BackupWallet() : cannot write %s\n", strDest.c_str()); + } + + char buf[65536]; + size_t n; + bool fOk = true; + while ((n = fread(buf, 1, sizeof(buf), pfIn)) > 0) + if (fwrite(buf, 1, n, pfOut) != n) { fOk = false; break; } + if (ferror(pfIn)) + fOk = false; + fclose(pfIn); + // Flushed and closed before lsn_reset touches it. + if (fclose(pfOut) != 0) + fOk = false; + + if (!fOk) + { + remove(strDest.c_str()); // half a wallet is worse than none + return error("BackupWallet() : copy to %s failed\n", strDest.c_str()); + } + + // Without this the copy is welded to this node's database/ dir. + int ret = dbenv.lsn_reset(strDest.c_str(), 0); + if (ret != 0) + printf("BackupWallet() : warning -- lsn_reset returned %d; the copy " + "may only open beside this node's database/ directory\n", ret); + } + catch (const std::exception& e) + { return error("BackupWallet() : %s\n", e.what()); } + } + + printf("BackupWallet() : wrote %s\n", strDest.c_str()); + return true; +} diff --git a/src/db.h b/src/db.h index 6dc3a4f..987786c 100644 --- a/src/db.h +++ b/src/db.h @@ -403,6 +403,12 @@ public: bool LoadWallet(); +// Writes a wallet.dat that opens on its own, anywhere -- see the definition in +// db.cpp for why a plain file copy does not. Still a point-in-time snapshot: +// keys are made as needed rather than derived from a seed, so coins paid to an +// address created after this call are not spendable from the file it writes. +bool BackupWallet(const string& strDest); + inline bool SetAddressBookName(const string& strAddress, const string& strName) { return CWalletDB().WriteName(strAddress, strName); diff --git a/src/main_gui.cpp b/src/main_gui.cpp index c64dca9..976cd42 100644 --- a/src/main_gui.cpp +++ b/src/main_gui.cpp @@ -67,6 +67,10 @@ static void PrintUsage() printf(" /port=N (P2P listen port, default 8433)\n"); printf(" /btfseed=ADDRESS:ENCHEX (extra bootstrap peer, repeatable)\n"); printf("\n"); + printf("Wallet:\n"); + printf(" /backupwallet=FILE (write a wallet.dat that opens on its own,\n"); + printf(" then exit; run it again after new addresses)\n"); + printf("\n"); printf("Each option also accepts '-' instead of '/'.\n"); } @@ -219,6 +223,28 @@ int main(int argc, char* argv[]) return 1; } printf("Height=%d\n", nBestHeight); + + // Before the node opens sockets or touches anything: the wallet is loaded, + // which is all a backup needs, and finishing here means the copy is taken + // from a quiet directory rather than from under a running node. + string strBackup = argval2(argc, argv, "/backupwallet", "-backupwallet"); + if (!strBackup.empty()) + { + bool fOk = BackupWallet(strBackup); + if (fOk) + { + printf("Backed up to %s\n", strBackup.c_str()); + printf("This file opens on its own -- it does not need the database/ " + "directory beside it.\n"); + printf("It is a snapshot: coins paid to addresses created after now are " + "not in it, so back up again whenever you receive to a new address.\n"); + } + else + fprintf(stderr, "Backup failed -- nothing was written.\n"); + DBFlush(true); + return fOk ? 0 : 1; + } + ReacceptWalletTransactions(); if (!StartNode(strErrors)) { fprintf(stderr,"StartNode: %s\n",strErrors.c_str()); return 1; } @@ -249,6 +275,11 @@ int main(int argc, char* argv[]) printf("Running headless. Ctrl-C to stop.\n"); while (!fShutdown) Sleep(500); StopNode(); + // Checkpoints, releases wallet.dat from this directory's Berkeley DB + // environment, and closes it. Declared since 2009 and never once + // called here, which is why a wallet.dat copied elsewhere would not + // open -- issue #40. Not optional. + DBFlush(true); return 0; } @@ -258,6 +289,7 @@ int main(int argc, char* argv[]) int ret = RunGUI(argc, argv); fShutdown = true; StopNode(); + DBFlush(true); return ret; #endif } diff --git a/src/util.h b/src/util.h index 797ed27..25ef118 100644 --- a/src/util.h +++ b/src/util.h @@ -3,6 +3,11 @@ // file license.txt or http://www.opensource.org/licenses/mit-license.php. +// For abi::__forced_unwind, which CATCH_PRINT_EXCEPTION below must let past. +#ifndef _WIN32 +#include +#endif + #if defined(_MSC_VER) || defined(__BORLANDC__) typedef __int64 int64; typedef unsigned __int64 uint64; @@ -419,7 +424,34 @@ inline void heapchk() } \ } +// The forced-unwind arm has to come first, and it has to rethrow. +// +// On POSIX, _endthread() is pthread_exit(), and glibc implements that by +// throwing abi::__forced_unwind through the stack. It is not a real exception +// and catching it without rethrowing is a fatal error by contract: glibc +// notices and calls abort(). +// +// The bare catch(...) below did exactly that. Every shutdown on Linux died with +// +// UNKNOWN EXCEPTION bitflash-node in ThreadSocketHandler() +// FATAL: exception not rethrown +// +// which looked like nothing worse than an ugly exit, and was not: the process +// aborted partway through shutting down, so nothing after StopNode() ran. See +// main_gui.cpp, where DBFlush(true) is now called -- it could never have +// worked while this abort stood in front of it. Windows is unaffected; there +// _endthread() is the CRT's and unwinds nothing. +#ifdef _WIN32 +#define CATCH_FORCED_UNWIND +#else +#define CATCH_FORCED_UNWIND \ + catch (abi::__forced_unwind&) { \ + throw; \ + } +#endif + #define CATCH_PRINT_EXCEPTION(pszFn) \ + CATCH_FORCED_UNWIND \ catch (std::exception& e) { \ PrintException(&e, (pszFn)); \ } catch (...) { \ -- cgit v1.2.3