upleb.uk

Public git repos — served from a NIP-34 GRASP relay at git.upleb.uk

summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.github/workflows/ci.yml72
-rw-r--r--.gitignore8
-rw-r--r--Makefile169
-rw-r--r--README.md255
-rw-r--r--docs/dependencies.md18
-rw-r--r--docs/fair-launch.md70
-rw-r--r--docs/mining-calculator.md108
-rw-r--r--docs/pool-directory.md61
-rw-r--r--docs/release-verification.md94
-rw-r--r--docs/rendezvous.md336
-rw-r--r--docs/site.md115
-rw-r--r--docs/social.pngbin208957 -> 162267 bytes
-rw-r--r--docs/storage.md159
-rw-r--r--docs/tor.md231
-rw-r--r--docs/utxo-commitment.md125
-rw-r--r--docs/utxo-proofs.md61
-rw-r--r--scripts/bitflash-mining-calculator.py206
-rw-r--r--scripts/bitflash_chain.py450
-rwxr-xr-xscripts/build-explorer.py376
-rw-r--r--scripts/build-pool-index.py174
-rwxr-xr-xscripts/check-money-log-lines.sh38
-rwxr-xr-x[-rw-r--r--]scripts/make-release-checksums.sh2
-rwxr-xr-xscripts/package-windows-tor.sh187
-rw-r--r--scripts/prove-utxo.py186
-rw-r--r--scripts/smoke-managed-tor-onion.ps1174
-rw-r--r--scripts/verify-fair-launch.py166
-rwxr-xr-x[-rw-r--r--]scripts/verify-release.sh108
-rw-r--r--scripts/verify-utxo-proof.py146
-rw-r--r--scripts/verify-utxo-set.py131
-rw-r--r--src/Makefile110
-rw-r--r--src/bignum.h4
-rw-r--r--src/btfaddr.cpp114
-rw-r--r--src/btfaddr.h8
-rw-r--r--src/btfrv.cpp74
-rw-r--r--src/btfrv.h19
-rw-r--r--src/btftunnel.cpp140
-rw-r--r--src/btftunnel.h25
-rw-r--r--src/compactblock.cpp175
-rw-r--r--src/compactblock.h113
-rw-r--r--src/db.cpp1276
-rw-r--r--src/db.h183
-rw-r--r--src/fuzz_net_message.cpp32
-rw-r--r--src/fuzz_script.cpp59
-rw-r--r--src/gui.cpp275
-rw-r--r--src/headers_core.h5
-rw-r--r--src/main.cpp1000
-rw-r--r--src/main.h91
-rw-r--r--src/main_gui.cpp596
-rw-r--r--src/makefile.mingw25
-rw-r--r--src/net.cpp394
-rw-r--r--src/net.h150
-rw-r--r--src/nostr.cpp256
-rw-r--r--src/nostr.h15
-rw-r--r--src/proxy.cpp422
-rw-r--r--src/proxy.h34
-rw-r--r--src/randomx_pow.cpp150
-rw-r--r--src/randomx_pow.h7
-rw-r--r--src/rc/bitcoin.icobin372526 -> 150897 bytes
-rw-r--r--src/readme.txt28
-rw-r--r--src/rpc.cpp4490
-rw-r--r--src/selftest.cpp2611
-rw-r--r--src/selftest.h1
-rw-r--r--src/sockcount.h2
-rw-r--r--src/test_btfaddr.cpp33
-rw-r--r--src/test_compactblock.cpp133
-rw-r--r--src/tor.cpp892
-rw-r--r--src/tor.h24
-rw-r--r--src/util.cpp176
-rw-r--r--src/util.h93
-rw-r--r--src/wallet_sqlite.cpp374
-rw-r--r--src/wallet_sqlite.h120
-rw-r--r--src/walletcmd.cpp1039
-rw-r--r--src/walletcmd.h43
-rw-r--r--tests/fixtures/chain-tools/explorer-blocks.json1
-rw-r--r--tests/fixtures/chain-tools/fair-launch-report.json1
-rw-r--r--tests/fixtures/chain-tools/utxo-proof-verification.json1
-rw-r--r--tests/fixtures/chain-tools/utxo-proof.json1
-rw-r--r--tests/fixtures/chain-tools/utxo-report.json1
-rw-r--r--tests/scripts/test_bitflash_tools.py286
79 files changed, 19313 insertions, 1015 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..e22b5a4
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,72 @@
1name: CI
2
3on:
4 push:
5 pull_request:
6
7permissions:
8 contents: read
9
10concurrency:
11 group: ci-${{ github.workflow }}-${{ github.ref }}
12 cancel-in-progress: true
13
14jobs:
15 linux-headless:
16 name: Linux headless selftests
17 runs-on: ubuntu-latest
18 steps:
19 - name: Check out repository
20 uses: actions/checkout@v5
21
22 - name: Install Linux dependencies
23 run: make deps-linux
24
25 - name: Run Linux selftests
26 run: make tests
27
28 - name: Run Python tool regressions
29 run: python3 tests/scripts/test_bitflash_tools.py
30
31 - name: Run P2P message fuzz smoke
32 run: make fuzz-net-message-smoke
33
34 - name: Run script fuzz smoke
35 run: make fuzz-script-smoke
36
37 windows-ucrt64:
38 name: Windows UCRT64 selftests
39 runs-on: windows-latest
40 defaults:
41 run:
42 shell: msys2 {0}
43 steps:
44 - name: Check out repository
45 uses: actions/checkout@v5
46
47 - name: Set up MSYS2
48 uses: msys2/setup-msys2@v2
49 with:
50 msystem: UCRT64
51 update: true
52 install: >-
53 git
54 make
55 mingw-w64-ucrt-x86_64-gcc
56 mingw-w64-ucrt-x86_64-glfw
57 mingw-w64-ucrt-x86_64-openssl
58 mingw-w64-ucrt-x86_64-db
59 mingw-w64-ucrt-x86_64-sqlite3
60 mingw-w64-ucrt-x86_64-libsodium
61 mingw-w64-ucrt-x86_64-nlohmann-json
62 mingw-w64-ucrt-x86_64-boost
63 mingw-w64-ucrt-x86_64-cmake
64 autoconf
65 automake
66 libtool
67
68 - name: Build Windows dependencies
69 run: make deps-windows
70
71 - name: Run Windows selftests
72 run: cd src && make -f makefile.mingw tests
diff --git a/.gitignore b/.gitignore
index 111f7ba..77b1660 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,9 @@ src/obj/
6*.dll 6*.dll
7bitflash-node 7bitflash-node
8src/bitflash-node 8src/bitflash-node
9src/obj-node/
10src/test_bip32
11src/test_compactblock
9 12
10# Release artefacts (produced by 'make windows' / 'make linux') 13# Release artefacts (produced by 'make windows' / 'make linux')
11Bitflash-*-windows/ 14Bitflash-*-windows/
@@ -18,11 +21,16 @@ SHA256SUMS.asc
18# Runtime data / logs 21# Runtime data / logs
19debug.log 22debug.log
20db.log 23db.log
24build-*.log
21*.dat 25*.dat
22wallet.dat 26wallet.dat
23nostr.key 27nostr.key
24btf_enc.key 28btf_enc.key
25 29
30# Python cache (upstream's release commit accidentally shipped one)
31__pycache__/
32*.pyc
33
26# Third-party build trees (rebuilt by the build scripts) 34# Third-party build trees (rebuilt by the build scripts)
27RandomX/ 35RandomX/
28deps/ 36deps/
diff --git a/Makefile b/Makefile
index abd75c5..342ec07 100644
--- a/Makefile
+++ b/Makefile
@@ -2,6 +2,7 @@
2# 2#
3# make linux build Bitflash-*-x86_64.AppImage 3# make linux build Bitflash-*-x86_64.AppImage
4# make windows build Bitflash-*-windows.zip (from MSYS2 UCRT64) 4# make windows build Bitflash-*-windows.zip (from MSYS2 UCRT64)
5# make windows-tor build Bitflash-*-windows-with-tor.zip
5# make tests build and run standalone unit tests 6# make tests build and run standalone unit tests
6# make checksums create SHA256SUMS for release assets 7# make checksums create SHA256SUMS for release assets
7# make verify-release TAG=v1.2.13 8# make verify-release TAG=v1.2.13
@@ -10,7 +11,10 @@
10ROOT := $(shell pwd) 11ROOT := $(shell pwd)
11NPROC := $(shell nproc 2>/dev/null || echo 2) 12NPROC := $(shell nproc 2>/dev/null || echo 2)
12SUDO := $(shell [ "$$(id -u)" = "0" ] && echo "" || echo "sudo") 13SUDO := $(shell [ "$$(id -u)" = "0" ] && echo "" || echo "sudo")
13VERSION := 1.2.15 14# Not "1.2.20". This tree is upstream v1.2.20 plus our own hardening commits and
15# a hand-merged payout ledger, so a binary built here must not be mistakable for
16# the upstream release of that number -- it names every release artifact.
17VERSION := 1.2.20+sec
14 18
15# ---- Pinned third-party sources ------------------------------------------- 19# ---- Pinned third-party sources -------------------------------------------
16# 20#
@@ -33,10 +37,23 @@ VERSION := 1.2.15
33# resolves the object by its own hash). Bump deliberately, and rebuild and 37# resolves the object by its own hash). Bump deliberately, and rebuild and
34# resync before shipping the result. 38# resync before shipping the result.
35# 39#
40# The check has to decide the exit status, so each recipe below joins its
41# commands with && the whole way to the closing echo. Ending that chain with a
42# `;` meant a failed `test` only skipped the build steps: the echo still ran,
43# the target still exited 0, and make announced a pinned dependency that
44# nothing had verified.
45#
36# RandomX pin is the commit this tree has been building and syncing against 46# RandomX pin is the commit this tree has been building and syncing against
37# (post-v2.0.1 master, 2026-05-24) -- chosen to change nothing today. 47# (post-v2.0.1 master, 2026-05-24) -- identical to upstream's pin.
48#
49# libsecp256k1 is pinned to upstream v1.2.18's commit rather than the one this
50# fork carried, so signature validation matches the binaries the rest of the
51# network runs. Consensus code must not differ from the network by accident.
52# There must be exactly ONE assignment of each of these variables in this file:
53# upstream defines them near the top too, and GNU Make takes the last `:=`, so
54# a second block silently decides the build.
38SECP256K1_REPO := https://github.com/bitcoin-core/secp256k1 55SECP256K1_REPO := https://github.com/bitcoin-core/secp256k1
39SECP256K1_COMMIT := 6e2c8bc4ecdc6e71dbe7a368f360d8d453ce435d 56SECP256K1_COMMIT := 7fecac74aed8e1fd9078380d67dd04663705c989
40RANDOMX_REPO := https://github.com/tevador/RandomX 57RANDOMX_REPO := https://github.com/tevador/RandomX
41RANDOMX_COMMIT := 1e9d4b2df63fa6edf46b06789486e23c0fcaa55a 58RANDOMX_COMMIT := 1e9d4b2df63fa6edf46b06789486e23c0fcaa55a
42 59
@@ -48,6 +65,32 @@ APPIMAGETOOL_SHA256 := ed4ce84f0d9caff66f50bcca6ff6f35aae54ce8135408b3fa33abfc3c
48 65
49# ---- Linux ---------------------------------------------------------------- 66# ---- Linux ----------------------------------------------------------------
50 67
68# RandomX used to build in $(HOME)/RandomX, and deps-randomx would rm -rf that
69# path whenever the library was missing -- deleting a tree the build did not
70# create and does not own (issue #27; a hand-managed pinned checkout there is
71# what this machine's miners build against). The default is now in-tree, like
72# secp256k1-build. A host that deliberately keeps RandomX elsewhere can set
73# RANDOMX_DIR=... explicitly; the recipe below will still never delete a
74# directory that lacks the marker file this build writes after a successful
75# build. Exported so src/Makefile's own RANDOMX_DIR ?= picks the same tree.
76#
77# Every use of it below is quoted, and the recipe rejects a value that cannot
78# name exactly one directory before it reaches anything destructive. Unquoted,
79# the marker guard could be stepped around: a value holding a glob that matches
80# two directories made `[ -d $(RANDOMX_DIR) ]` a two-argument test, which dies
81# with "unexpected operator" instead of answering, so the && short-circuited,
82# the guard was skipped, and the rm -rf below ran against every match. A value
83# holding a space or a `&&` broke the outer test the same way and reported
84# "already built". Whitespace and glob characters are refused outright rather
85# than merely quoted, because src/Makefile interpolates this same value into
86# -I$(RANDOMX_DIR)/src and -L$(RANDOMX_DIR)/build unquoted, where neither can
87# survive. A relative value is refused rather than silently made absolute: it
88# would mean two different directories to this Makefile and to src/Makefile,
89# whose cwd is src/, and a command-line assignment reaches the sub-make through
90# MAKEFLAGS, where it outranks anything exported from here.
91RANDOMX_DIR ?= $(ROOT)/randomx-build
92export RANDOMX_DIR
93
51linux: deps-linux 94linux: deps-linux
52 $(MAKE) -C src -f Makefile -j$(NPROC) 95 $(MAKE) -C src -f Makefile -j$(NPROC)
53 strip src/bitflash 96 strip src/bitflash
@@ -67,46 +110,79 @@ linux-node: deps-linux
67 110
68deps-linux: deps-apt deps-secp256k1 deps-randomx 111deps-linux: deps-apt deps-secp256k1 deps-randomx
69 112
113# libboost-dev, not libboost-system-dev: Boost.System has been header-only
114# since 1.69 and Ubuntu 26.04 (resolute) no longer ships the -system package.
115# This tree only includes two boost headers and links no boost library. The
116# apt-get update comes first: a fresh image's package lists are empty, and
117# install then fails on the first name it cannot find.
118#
119# A failed update is deliberately not fatal: one unreachable repository should
120# not block an install the lists already on disk can satisfy. It does say so,
121# though -- "unable to locate package" several seconds later is otherwise hard
122# to connect back to it. The install is the last command in the branch, so its
123# status is the target's, and that one does abort.
70deps-apt: 124deps-apt:
71 @if command -v apt-get >/dev/null 2>&1; then \ 125 @if command -v apt-get >/dev/null 2>&1; then \
126 $(SUDO) apt-get update || \
127 echo " warning: apt-get update failed, using the package lists on disk"; \
72 $(SUDO) apt-get install -y build-essential cmake git pkg-config autoconf \ 128 $(SUDO) apt-get install -y build-essential cmake git pkg-config autoconf \
73 libtool libssl-dev libdb5.3++-dev libsodium-dev nlohmann-json3-dev \ 129 libtool libssl-dev libdb5.3++-dev libsodium-dev nlohmann-json3-dev \
74 libboost-system-dev libglfw3-dev libgl-dev python3-pil; \ 130 libboost-dev libglfw3-dev libgl-dev libsqlite3-dev \
131 python3-pil clang; \
75 else \ 132 else \
76 echo "Not an apt system — install: g++ cmake git autoconf libtool"; \ 133 echo "Not an apt system — install: g++ cmake git autoconf libtool"; \
77 echo " libssl libdb++ libsodium nlohmann-json boost glfw3 opengl python3-pil"; \ 134 echo " libssl libdb++ libsodium nlohmann-json boost glfw3 opengl sqlite3 python3-pil"; \
78 fi 135 fi
79 136
80deps-secp256k1: 137deps-secp256k1:
81 @if [ ! -f /usr/local/lib/libsecp256k1.a ]; then \ 138 @if [ ! -f /usr/local/lib/libsecp256k1.a ]; then \
82 echo "==> building libsecp256k1"; \ 139 echo "==> building libsecp256k1"; \
83 rm -rf $(ROOT)/secp256k1-build; \ 140 rm -rf $(ROOT)/secp256k1-build && \
84 git init -q $(ROOT)/secp256k1-build; \ 141 git init -q $(ROOT)/secp256k1-build && \
85 cd $(ROOT)/secp256k1-build && \ 142 cd $(ROOT)/secp256k1-build && \
86 git remote add origin $(SECP256K1_REPO) && \ 143 git remote add origin $(SECP256K1_REPO) && \
87 git fetch -q --depth 1 origin $(SECP256K1_COMMIT) && \ 144 git fetch -q --depth 1 origin $(SECP256K1_COMMIT) && \
88 git checkout -q $(SECP256K1_COMMIT) && \ 145 git checkout -q --detach FETCH_HEAD && \
89 ./autogen.sh && \ 146 test "$$(git rev-parse HEAD)" = "$(SECP256K1_COMMIT)" && \
147 echo "==> libsecp256k1 pinned at $(SECP256K1_COMMIT)" && \
148 cd $(ROOT)/secp256k1-build && ./autogen.sh && \
90 ./configure --enable-module-schnorrsig --enable-module-extrakeys \ 149 ./configure --enable-module-schnorrsig --enable-module-extrakeys \
91 --disable-shared --with-pic --disable-benchmark --disable-tests && \ 150 --disable-shared --with-pic --disable-benchmark --disable-tests && \
92 make -j$(NPROC) && $(SUDO) make install && $(SUDO) ldconfig; \ 151 make -j$(NPROC) && $(SUDO) make install && $(SUDO) ldconfig && \
93 echo "==> libsecp256k1 done"; \ 152 echo "==> libsecp256k1 done"; \
94 else \ 153 else \
95 echo "==> libsecp256k1 already installed"; \ 154 echo "==> libsecp256k1 already installed"; \
96 fi 155 fi
97 156
98deps-randomx: 157deps-randomx:
99 @if [ ! -f $(HOME)/RandomX/build/librandomx.a ]; then \ 158 @ws="$$(printf ' \t')"; \
159 case "$(RANDOMX_DIR)" in \
160 "") echo "ERROR: RANDOMX_DIR is empty"; exit 1 ;; \
161 /) echo "ERROR: RANDOMX_DIR is /, refusing"; exit 1 ;; \
162 *[$$ws]*) echo "ERROR: RANDOMX_DIR contains whitespace: $(RANDOMX_DIR)"; exit 1 ;; \
163 *[*?[]*) echo "ERROR: RANDOMX_DIR contains a glob character: $(RANDOMX_DIR)"; exit 1 ;; \
164 /*) ;; \
165 *) echo "ERROR: RANDOMX_DIR must be an absolute path: $(RANDOMX_DIR)"; exit 1 ;; \
166 esac
167 @if [ ! -f "$(RANDOMX_DIR)/build/librandomx.a" ]; then \
168 if [ -d "$(RANDOMX_DIR)" ] && [ ! -f "$(RANDOMX_DIR)/.bitflash-built" ]; then \
169 echo "ERROR: $(RANDOMX_DIR) exists but was not created by this build"; \
170 echo " refusing to rm -rf it -- remove it yourself or set RANDOMX_DIR"; \
171 exit 1; \
172 fi; \
100 echo "==> building RandomX"; \ 173 echo "==> building RandomX"; \
101 rm -rf $(HOME)/RandomX; \ 174 rm -rf "$(RANDOMX_DIR)" && \
102 git init -q $(HOME)/RandomX; \ 175 git init -q "$(RANDOMX_DIR)" && \
103 cd $(HOME)/RandomX && \ 176 cd "$(RANDOMX_DIR)" && \
104 git remote add origin $(RANDOMX_REPO) && \ 177 git remote add origin $(RANDOMX_REPO) && \
105 git fetch -q --depth 1 origin $(RANDOMX_COMMIT) && \ 178 git fetch -q --depth 1 origin $(RANDOMX_COMMIT) && \
106 git checkout -q $(RANDOMX_COMMIT); \ 179 git checkout -q --detach FETCH_HEAD && \
107 mkdir -p $(HOME)/RandomX/build; \ 180 test "$$(git rev-parse HEAD)" = "$(RANDOMX_COMMIT)" && \
108 cd $(HOME)/RandomX/build && cmake .. -DCMAKE_BUILD_TYPE=Release && \ 181 echo "==> RandomX pinned at $(RANDOMX_COMMIT)" && \
109 make -j$(NPROC) randomx; \ 182 mkdir -p "$(RANDOMX_DIR)/build" && \
183 cd "$(RANDOMX_DIR)/build" && cmake .. -DCMAKE_BUILD_TYPE=Release && \
184 make -j$(NPROC) randomx && \
185 touch "$(RANDOMX_DIR)/.bitflash-built" && \
110 echo "==> RandomX done"; \ 186 echo "==> RandomX done"; \
111 else \ 187 else \
112 echo "==> RandomX already built"; \ 188 echo "==> RandomX already built"; \
@@ -157,13 +233,22 @@ img = ico.convert('RGBA'); \
157img.save(d + '/bitflash.png'); \ 233img.save(d + '/bitflash.png'); \
158import shutil; shutil.copy(d + '/bitflash.png', \ 234import shutil; shutil.copy(d + '/bitflash.png', \
159d + '/usr/share/icons/hicolor/256x256/apps/bitflash.png')" 235d + '/usr/share/icons/hicolor/256x256/apps/bitflash.png')"
160 @for lib in libsodium.so.23 libdb_cxx-5.3.so libglfw.so.3; do \ 236 @for lib in libsodium.so.23 libdb_cxx-5.3.so libsqlite3.so.0 libglfw.so.3; do \
161 path=$$(ldconfig -p | grep " $$lib " | awk '{print $$NF}' | head -1); \ 237 path=$$(ldconfig -p | grep " $$lib " | awk '{print $$NF}' | head -1); \
162 [ -z "$$path" ] && path=$$(find /usr/lib /lib -name "$$lib" 2>/dev/null | head -1); \ 238 [ -z "$$path" ] && path=$$(find /usr/lib /lib -name "$$lib" 2>/dev/null | head -1); \
163 [ -n "$$path" ] && cp -L "$$path" $(APPDIR)/usr/lib/ && echo " bundled $$lib" || echo " missing $$lib"; \ 239 [ -n "$$path" ] && cp -L "$$path" $(APPDIR)/usr/lib/ && echo " bundled $$lib" || echo " missing $$lib"; \
164 done 240 done
241# appimagetool's output used to be piped through grep ... || true, which
242# discarded its exit status: the target succeeded and announced "Built:"
243# even when no AppImage existed. Run the tool bare so its status decides,
244# then require the artefact to exist, be non-empty, and be newer than the
245# binary it was packaged from before saying so.
165 @ARCH=x86_64 $(APPIMAGETOOL) --no-appstream $(APPDIR) \ 246 @ARCH=x86_64 $(APPIMAGETOOL) --no-appstream $(APPDIR) \
166 $(ROOT)/Bitflash-$(VERSION)-x86_64.AppImage 2>&1 | grep -E "^Built|^Error" || true 247 $(ROOT)/Bitflash-$(VERSION)-x86_64.AppImage
248 @test -s $(ROOT)/Bitflash-$(VERSION)-x86_64.AppImage || \
249 { echo "ERROR: Bitflash-$(VERSION)-x86_64.AppImage missing or empty"; exit 1; }
250 @test $(ROOT)/Bitflash-$(VERSION)-x86_64.AppImage -nt src/bitflash || \
251 { echo "ERROR: Bitflash-$(VERSION)-x86_64.AppImage is not newer than src/bitflash"; exit 1; }
167 @echo "Built: Bitflash-$(VERSION)-x86_64.AppImage" 252 @echo "Built: Bitflash-$(VERSION)-x86_64.AppImage"
168 253
169# ---- Windows (MSYS2 UCRT64) ----------------------------------------------- 254# ---- Windows (MSYS2 UCRT64) -----------------------------------------------
@@ -175,29 +260,42 @@ deps-windows:
175 mingw-w64-ucrt-x86_64-glfw \ 260 mingw-w64-ucrt-x86_64-glfw \
176 mingw-w64-ucrt-x86_64-openssl \ 261 mingw-w64-ucrt-x86_64-openssl \
177 mingw-w64-ucrt-x86_64-db \ 262 mingw-w64-ucrt-x86_64-db \
263 mingw-w64-ucrt-x86_64-sqlite3 \
178 mingw-w64-ucrt-x86_64-libsodium \ 264 mingw-w64-ucrt-x86_64-libsodium \
179 mingw-w64-ucrt-x86_64-nlohmann-json \ 265 mingw-w64-ucrt-x86_64-nlohmann-json \
180 mingw-w64-ucrt-x86_64-boost \ 266 mingw-w64-ucrt-x86_64-boost \
181 mingw-w64-ucrt-x86_64-cmake \ 267 mingw-w64-ucrt-x86_64-cmake \
182 autoconf automake libtool 268 autoconf automake libtool
183 @if [ ! -f deps/lib/libsecp256k1.a ]; then \ 269 @if [ ! -f deps/lib/libsecp256k1.a ]; then \
184 rm -rf secp256k1-build; \ 270 rm -rf secp256k1-build && \
185 git clone --depth 1 https://github.com/bitcoin-core/secp256k1 secp256k1-build; \ 271 git init -q secp256k1-build && \
186 cd secp256k1-build && ./autogen.sh && \ 272 cd secp256k1-build && \
273 git remote add origin $(SECP256K1_REPO) && \
274 git fetch -q --depth 1 origin $(SECP256K1_COMMIT) && \
275 git checkout -q --detach FETCH_HEAD && \
276 test "$$(git rev-parse HEAD)" = "$(SECP256K1_COMMIT)" && \
277 echo "==> secp256k1 pinned at $(SECP256K1_COMMIT)" && \
278 cd $(ROOT)/secp256k1-build && ./autogen.sh && \
187 ./configure --prefix=$(ROOT)/deps \ 279 ./configure --prefix=$(ROOT)/deps \
188 --enable-module-schnorrsig --enable-module-extrakeys \ 280 --enable-module-schnorrsig --enable-module-extrakeys \
189 --disable-shared --with-pic --disable-benchmark --disable-tests && \ 281 --disable-shared --with-pic --disable-benchmark --disable-tests && \
190 make install; \ 282 make install && \
191 echo "==> secp256k1 done"; \ 283 echo "==> secp256k1 done"; \
192 else \ 284 else \
193 echo "==> secp256k1 already built"; \ 285 echo "==> secp256k1 already built"; \
194 fi 286 fi
195 @if [ ! -f RandomX/build/librandomx.a ]; then \ 287 @if [ ! -f RandomX/build/librandomx.a ]; then \
196 rm -rf RandomX; \ 288 rm -rf RandomX && \
197 git clone --depth 1 https://github.com/tevador/RandomX; \ 289 git init -q RandomX && \
198 mkdir -p RandomX/build; \ 290 cd RandomX && \
199 cd RandomX/build && cmake .. -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release && \ 291 git remote add origin $(RANDOMX_REPO) && \
200 make randomx; \ 292 git fetch -q --depth 1 origin $(RANDOMX_COMMIT) && \
293 git checkout -q --detach FETCH_HEAD && \
294 test "$$(git rev-parse HEAD)" = "$(RANDOMX_COMMIT)" && \
295 echo "==> RandomX pinned at $(RANDOMX_COMMIT)" && \
296 mkdir -p $(ROOT)/RandomX/build && \
297 cd $(ROOT)/RandomX/build && cmake .. -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Release && \
298 make randomx && \
201 echo "==> RandomX done"; \ 299 echo "==> RandomX done"; \
202 else \ 300 else \
203 echo "==> RandomX already built"; \ 301 echo "==> RandomX already built"; \
@@ -213,6 +311,9 @@ windows: deps-windows
213 @zip -r Bitflash-$(VERSION)-windows.zip Bitflash-$(VERSION)-windows/ 311 @zip -r Bitflash-$(VERSION)-windows.zip Bitflash-$(VERSION)-windows/
214 @echo "Built: Bitflash-$(VERSION)-windows.zip" 312 @echo "Built: Bitflash-$(VERSION)-windows.zip"
215 313
314windows-tor: windows
315 ./scripts/package-windows-tor.sh
316
216# ---- Clean ---------------------------------------------------------------- 317# ---- Clean ----------------------------------------------------------------
217 318
218clean: 319clean:
@@ -224,6 +325,12 @@ clean:
224tests: 325tests:
225 $(MAKE) -C src -f Makefile tests 326 $(MAKE) -C src -f Makefile tests
226 327
328fuzz-net-message-smoke: deps-linux
329 $(MAKE) -C src -f Makefile fuzz-net-message-smoke
330
331fuzz-script-smoke: deps-linux
332 $(MAKE) -C src -f Makefile fuzz-script-smoke
333
227checksums: 334checksums:
228 ./scripts/make-release-checksums.sh 335 ./scripts/make-release-checksums.sh
229 336
@@ -233,6 +340,6 @@ sign-checksums:
233verify-release: 340verify-release:
234 ./scripts/verify-release.sh $(if $(TAG),$(TAG),latest) 341 ./scripts/verify-release.sh $(if $(TAG),$(TAG),latest)
235 342
236.PHONY: linux windows clean appimage \ 343.PHONY: linux windows windows-tor clean appimage \
237 tests checksums sign-checksums verify-release \ 344 tests fuzz-net-message-smoke fuzz-script-smoke checksums sign-checksums verify-release \
238 deps-linux deps-windows deps-apt deps-secp256k1 deps-randomx 345 deps-linux deps-windows deps-apt deps-secp256k1 deps-randomx
diff --git a/README.md b/README.md
index 9a55a5b..4453529 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,20 @@
1## Bitflash `BTF` 1<div align="center">
2
3<img src="docs/social.png" width="820" alt="Bitflash"/>
4
5# Bitflash `BTF`
6
2CPU-only cryptocurrency. A revival of Bitcoin 0.1.0 with RandomX proof of work and anonymous `.btf` addressing over Nostr. 7CPU-only cryptocurrency. A revival of Bitcoin 0.1.0 with RandomX proof of work and anonymous `.btf` addressing over Nostr.
3 8
4## Bitflash community 9![PoW](https://img.shields.io/badge/PoW-RandomX%20(CPU)-2ea44f?style=for-the-badge)
5Discord: https://discord.gg/TJG6vnXS9 10![Privacy](https://img.shields.io/badge/addresses-.btf%20anonymous-2ea44f?style=for-the-badge)
11![Tor](https://img.shields.io/badge/tor-onion%20service-7d4698?style=for-the-badge)
12![Fair Launch](https://img.shields.io/badge/premine-none-2ea44f?style=for-the-badge)
13[![Discord](https://img.shields.io/badge/discord-join-5865F2?style=for-the-badge)](https://discord.gg/n7B2Eamn3)
14
15**[Download](https://releases.bitflash.network/)** · **[Discord](https://discord.gg/n7B2Eamn3)**
16
17</div>
6 18
7--- 19---
8 20
@@ -12,7 +24,7 @@ Bitflash keeps Satoshi's original consensus rules and replaces two things:
12 24
13**RandomX proof of work.** Memory-hard algorithm used by Monero. A laptop competes equally with a server. ASICs and GPUs have no advantage. 25**RandomX proof of work.** Memory-hard algorithm used by Monero. A laptop competes equally with a server. ASICs and GPUs have no advantage.
14 26
15**Anonymous addressing.** Every node has a `.btf` address derived from its public key, similar to a Tor `.onion`. Nodes reach each other through encrypted rendezvous tunnels, so no port forwarding is needed and a node behind CGNAT works normally. 27**Anonymous addressing.** Every node has a `.btf` address derived from its public key, similar to a Tor `.onion`. Nodes reach each other through encrypted rendezvous tunnels by default, and can advertise a signed direct `.onion` endpoint for Tor peers.
16 28
17No premine. No ICO. 50 BTF per block, halving on schedule, 21M cap, ~2 minute blocks. 29No premine. No ICO. 50 BTF per block, halving on schedule, 21M cap, ~2 minute blocks.
18 30
@@ -20,27 +32,128 @@ No premine. No ICO. 50 BTF per block, halving on schedule, 21M cap, ~2 minute bl
20 32
21Worth being precise, because the difference matters if you are relying on it. 33Worth being precise, because the difference matters if you are relying on it.
22 34
23**A peer you reach over `.btf` does not learn your IP.** All outbound connections go through a rendezvous tunnel; there is no IP-based peer dialling left in the node. The relay forwards encrypted bytes and cannot read or alter them. 35**A peer you reach over `.btf` does not learn your IP.** Outbound peer discovery uses rendezvous tunnels by default. If both sides use Tor and the peer advertises a signed `.onion` endpoint, the node connects directly to that hidden service instead. In both cases the peer does not see your clearnet IP.
24 36
25**The rendezvous relay does see your IP.** It has to — it is the thing your TCP connection terminates on. Relays are run by volunteers, so treat that as a party who knows you are on the network. 37**The rendezvous relay sees your IP unless you use Tor.** It has to — it is the thing your TCP connection terminates on. With `-tor`, the relay sees Tor instead. With a signed direct `.onion` peer endpoint, the relay is not on that connection path at all.
26 38
27**The node still listens on 8433.** Nothing dials by IP any more, but the listener is still there, so anyone who already knows your address and can reach that port may connect directly. If that matters to you, firewall it. 39**The node still listens on 8433.** Nothing dials by IP any more, but the listener is still there, so anyone who already knows your address and can reach that port may connect directly. If that matters to you, firewall it.
28 40
29This is unlinkability between peers, not anonymity against a network observer. It is not Tor. 41This is unlinkability between peers, not anonymity against a network observer.
42For Tor routing and direct onion peer endpoints, start the node with `-tor`;
43see [Tor mode](docs/tor.md).
44For a plain SOCKS5 proxy without Tor-specific defaults, use
45`-socks=HOST:PORT`.
30 46
31--- 47---
32 48
33## Quick start 49## Quick start
34 50
51Download the [latest release](https://releases.bitflash.network/) and run. No
52install, no configuration — it connects automatically and starts syncing.
53
35**Linux:** make the `.AppImage` executable and run it. 54**Linux:** make the `.AppImage` executable and run it.
36 55
37**Windows:** extract the `-windows.zip` and run `Bitflash.exe`. 56**Windows:** extract the `-windows-with-tor.zip` and run `Bitflash.exe`. Tor is
57bundled and a managed Tor instance starts automatically, so onion transport
58works with no separate install. Pass `-nomanagedtor` to turn it off.
59
60Every release ships a `SHA256SUMS` covering the assets. Verifying takes a second
61and is worth doing:
62
63```bash
64sha256sum -c SHA256SUMS
65```
66
67For newer signed releases, verify the checksum file itself first:
68
69```bash
70gpg --verify SHA256SUMS.asc SHA256SUMS
71sha256sum -c SHA256SUMS
72```
73
74The helper below downloads the release assets, verifies `SHA256SUMS.asc` when it
75is present, then checks the hashes:
76
77```bash
78scripts/verify-release.sh latest
79```
80
81See [release verification](docs/release-verification.md) for the full release
82audit flow and the maintainer signing step.
83
84Public mirrors should make the latest checksum files available both under the
85versioned release directory and at the release root:
86
87```text
88https://releases.bitflash.network/v1.2.20/SHA256SUMS
89https://releases.bitflash.network/SHA256SUMS # alias to latest
90```
91
92Bitflash also ships a deterministic UTXO-set commitment tool:
93
94```bash
95python3 scripts/verify-utxo-set.py --out utxo-report.json --json
96```
97
98It reconstructs the best chain from local block files, computes the current
99UTXO root and supply, and writes canonical JSON that independent auditors can
100compare or timestamp externally with OpenTimestamps. See
101[UTXO-set commitments](docs/utxo-commitment.md).
102
103The same root can prove one unspent output without sharing the full UTXO set:
104
105```bash
106python3 scripts/prove-utxo.py TXID:VOUT --out utxo-proof.json --json
107python3 scripts/verify-utxo-proof.py utxo-proof.json
108```
109
110See [UTXO inclusion proofs](docs/utxo-proofs.md).
111
112Bitflash also ships a deterministic fair-launch verifier:
113
114```bash
115python3 scripts/verify-fair-launch.py --max-blocks 1000 --out bitflash-fair-launch-report.json --json
116```
117
118The report reads the local chain directly, checks the genesis launch baseline,
119and produces canonical JSON whose hash can be compared by independent auditors
120or timestamped externally with OpenTimestamps. See
121[fair-launch verification](docs/fair-launch.md).
122
123For a static local block explorer, generate HTML and JSON from either a data
124directory or explicit block files:
125
126```bash
127python3 scripts/build-explorer.py --datadir ~/.bitflash ./explorer-out
128python3 scripts/build-explorer.py ~/.bitflash/blk0001.dat ~/.bitflash/blk0002.dat ./explorer-out
129```
130
131The generated explorer is static and self-contained: `index.html`,
132`style.css`, `explorer.js`, `blocks.json`, `block/*.json`, and `logo.png`. It is
133safe to host behind a strict policy such as `default-src 'none'; img-src 'self';
134style-src 'self'; script-src 'self'; connect-src 'self'; base-uri 'none';
135form-action 'none'`.
38 136
39**Keep your node current.** Consensus rules have changed since the first 137**Keep your node current.** Consensus rules have changed since the first
40releases — 1.0.0 fixed a bug that let anyone spend anyone's coins. A node on an older build will accept 138releases — 1.2.1 fixed a bug that let anyone spend anyone's coins, and 1.2.2
139added a per-block signature-operation cap. A node on an older build will accept
41blocks that current nodes reject, which puts it on a different chain without any 140blocks that current nodes reject, which puts it on a different chain without any
42warning. 141warning.
43 142
143Two more reasons, both measured rather than theorised:
144
145- **Before 1.2.13** a node closed a disconnected peer's socket twice and left the
146 closed handle in its `select()` set, where it made `select()` fail on every
147 iteration. The node then spent its socket loop in an error path instead of
148 reading its peers — the deaf-node behaviour reported since 1.2.7. Two nodes
149 side by side on one machine, same network: the one without the fix held 11
150 peers and logged 741 spurious disconnections; the one with it held 21 and
151 logged none. On 1.2.13 in production, two mining nodes hold 13 and 18 peers
152 with zero.
153- **Before 1.2.11** a long-running Windows node accumulated sockets it never
154 released — 1262 of them in 26 hours on one machine, each holding an ephemeral
155 port. Restarting returned them; upgrading stops them accumulating.
156
44Your wallet and chain data live in `%APPDATA%\Bitflash` (Windows) or 157Your wallet and chain data live in `%APPDATA%\Bitflash` (Windows) or
45`~/.bitflash` (Linux) and are shared by every version, so upgrading is just 158`~/.bitflash` (Linux) and are shared by every version, so upgrading is just
46replacing the binary. Never delete that directory to "fix" something without a 159replacing the binary. Never delete that directory to "fix" something without a
@@ -50,6 +163,19 @@ backup — it holds your keys.
50 163
51## Your wallet: the phrase and the file 164## Your wallet: the phrase and the file
52 165
166Since 1.2.12 a wallet can hold twelve words that rebuild it. Since 1.2.13 those
167words also cover the address the window shows you. **Both still matter** — the
168phrase and the file back up different things, and the difference is where people
169lose money.
170
171Since 1.2.20 the file is a single self-contained `wallet.sqlite` by default. A
172fresh data directory starts on it; an existing Berkeley DB `wallet.dat` keeps
173working and the desktop app converts it in one click on first run, never
174touching the original, which stays as a fallback. Choose a backend explicitly
175with `-walletbackend=sqlite` or `-walletbackend=bdb`, recorded in a
176`wallet-backend` marker in the data directory. See
177[wallet storage](docs/storage.md).
178
53The derivation is written down in [docs/derivation.md](docs/derivation.md), with 179The derivation is written down in [docs/derivation.md](docs/derivation.md), with
54test vectors and a script that reproduces them from scratch. It is there so the 180test vectors and a script that reproduces them from scratch. It is there so the
55twelve words keep working even if this software does not: paths, address format 181twelve words keep working even if this software does not: paths, address format
@@ -63,8 +189,11 @@ bitflash -newphrase # create it, show it once, exit
63bitflash -restorephrase="twelve words here" # rebuild a wallet from it 189bitflash -restorephrase="twelve words here" # rebuild a wallet from it
64``` 190```
65 191
66`-newphrase` installs a BIP32 seed, derives the wallet's key pool and default 192`-newphrase` installs a BIP39/BIP32 seed and derives the wallet as a BIP44 tree
67receiving address from it, and prints the words once. It refuses if a phrase 193along `m/44'/4346950'/0'/…` — coin type **4346950** is `0x425446`, the ASCII
194bytes `BTF`, registered in [SLIP-0044](https://github.com/satoshilabs/slips/blob/master/slip-0044.md).
195It builds the key pool and default receiving address from the seed, and prints
196the words once. It refuses if a phrase
68already exists: replacing one silently would strand every coin on addresses the 197already exists: replacing one silently would strand every coin on addresses the
69written-down words no longer describe. 198written-down words no longer describe.
70 199
@@ -84,8 +213,17 @@ from the words. The wallet names such an address `Your Address (created before
84the recovery phrase)` so you can tell them apart. This is why file backups still 213the recovery phrase)` so you can tell them apart. This is why file backups still
85matter. 214matter.
86 215
216> **If you created a phrase on 1.2.12, run `-restorephrase` with the same twelve
217> words.** That release installed the seed but left the visible address and the
218> key pool random, so the wallet went on handing out addresses the words cannot
219> reproduce while the window said a phrase existed. Restoring repairs it.
220
87### The file 221### The file
88 222
223Under the SQLite default the wallet is one self-contained `wallet.sqlite`, and
224`-backupwallet` writes a matching `.sqlite` copy that opens on its own — no
225sidecar directory. The Berkeley DB notes below apply to a legacy `wallet.dat`.
226
89**Copying `wallet.dat` on its own is not a backup.** Berkeley DB ties the file 227**Copying `wallet.dat` on its own is not a backup.** Berkeley DB ties the file
90to the environment in the `database/` subdirectory beside it, so a lone 228to the environment in the `database/` subdirectory beside it, so a lone
91`wallet.dat` will not open elsewhere — the keys are all still in there, and the 229`wallet.dat` will not open elsewhere — the keys are all still in there, and the
@@ -123,7 +261,7 @@ as a verdict to somebody who has just lost a wallet.
123 261
124### When wallet.dat itself will not open 262### When wallet.dat itself will not open
125 263
126Everything above still runs through Berkeley DB. If the database is the thing 264A legacy `wallet.dat` runs on Berkeley DB. If that database is the thing
127that is broken — a build that will not read it, a file truncated by a bad copy, 265that is broken — a build that will not read it, a file truncated by a bad copy,
128an environment beyond recovery — the keys are usually still fine, and you can 266an environment beyond recovery — the keys are usually still fine, and you can
129take them out as text: 267take them out as text:
@@ -157,15 +295,48 @@ Open **Options** from the menu bar. Under Mining Mode:
157 295
158Pool announcements are published on Nostr with live status fields, so external tools can query the latest pool state by `.btf` address. 296Pool announcements are published on Nostr with live status fields, so external tools can query the latest pool state by `.btf` address.
159 297
298Pool operators also write a local `pool_status.json` every ten seconds and a
299`pool_rounds.json` proof ledger whenever they find blocks or pay miners. By
300default both are created in the data directory; use `-poolstatusfile=PATH` and
301`-poolroundsfile=PATH` to write them somewhere a dashboard or web sync can read.
302The files are informational only: nodes still discover pools through Nostr and
303`.btf`, not through any web domain.
304See [public pool directory](docs/pool-directory.md) for the website/API shape.
305
160**Participant** — mine to someone else's pool. Enter or select the pool's `.btf` address and enable Start Mining. 306**Participant** — mine to someone else's pool. Enter or select the pool's `.btf` address and enable Start Mining.
161 307
162SRBMiner and XMRig connect to operator pools through the `.btf` path: 308The built-in participant miner connects to operator pools through the `.btf`
309path. Use it — **third-party RandomX miners such as XMRig and SRBMiner cannot
310mine Bitflash pools**, and no amount of configuration changes that. In RandomX
311mode they speak the Monero-style JSON-RPC protocol (`login`/`job`/`submit`) and
312assume a Monero-style job: the nonce at byte 39 of the blob, and a 32-byte seed
313hash given as 64 hex characters. Bitflash pools speak Bitcoin-style Stratum
314(`mining.subscribe`/`mining.notify`/`mining.submit`) over an 80-byte header
315whose nonce is at byte 76 — byte 39 falls inside the merkle root — and key
316RandomX from a fixed 36-byte string that is not a hash at all.
317
318`-stratumbridge` carries that same Bitcoin-style Stratum over the `.btf` path,
319so a miner on this machine can reach a pool it otherwise could not address. It
320is a byte relay, not a protocol translator, so it helps only a miner that
321already speaks the pool's dialect, and it does not make the miners above work:
322
323```bash
324./bitflash -nogui -stratumbridge=POOL_BTF_ADDRESS -stratumbridgeport=3333
325```
326
327Estimate rewards and electricity cost locally:
163 328
164```bash 329```bash
165SRBMiner-MULTI --algorithm randomx --pool POOL_BTF_ADDRESS --wallet YOUR_BTF_ADDRESS --password x 330python3 scripts/bitflash-mining-calculator.py \
166xmrig -a rx/0 -o POOL_BTF_ADDRESS -u YOUR_BTF_ADDRESS -p x 331 --hashrate 21 --hashrate-unit kh/s \
332 --network-hashrate 1.56 --network-hashrate-unit mh/s \
333 --watts 140 --kwh-cost 0.10 \
334 --price 0.001 --pool-fee 1
167``` 335```
168 336
337See [mining calculator](docs/mining-calculator.md). Price is manual until
338Bitflash has a reliable public market.
339
169--- 340---
170 341
171## Headless / server mode 342## Headless / server mode
@@ -174,7 +345,11 @@ xmrig -a rx/0 -o POOL_BTF_ADDRESS -u YOUR_BTF_ADDRESS -p x
174./bitflash -nogui # node only 345./bitflash -nogui # node only
175./bitflash -nogui -gen # node + solo mining 346./bitflash -nogui -gen # node + solo mining
176./bitflash -nogui -gen -operator # pool operator 347./bitflash -nogui -gen -operator # pool operator
348./bitflash -nogui -gen -operator \
349 -poolstatusfile=/var/www/pool_status.json \
350 -poolroundsfile=/var/www/pool_rounds.json
177./bitflash -nogui -gen -participant=POOL_BTF_ADDRESS # mine to pool 351./bitflash -nogui -gen -participant=POOL_BTF_ADDRESS # mine to pool
352./bitflash -nogui -stratumbridge=POOL_BTF_ADDRESS # Stratum over .btf (see Mining)
178``` 353```
179 354
180Every option takes `-` or `/`. **Under MSYS2 use the `-` form** — the shell 355Every option takes `-` or `/`. **Under MSYS2 use the `-` form** — the shell
@@ -190,7 +365,13 @@ Other options worth knowing:
190 365
191```bash 366```bash
192-datadir=PATH # wallet and chain data elsewhere 367-datadir=PATH # wallet and chain data elsewhere
193-port=N # P2P listen port, default 8433 368-testnet # isolated test network genesis, datadir, port 18433, and magic
369-port=N # P2P listen port, default 8433; testnet default 18433
370-socks=HOST:PORT # SOCKS5 for outbound Nostr, .btf relay, and onion peer dials
371-tor[=HOST:PORT] # Tor mode; default local Tor SOCKS5 proxy is 127.0.0.1:9050
372-managedtor[=PATH] # start Tor, create a hidden service, advertise its onion
373-nomanagedtor # disable the automatic bundled-Tor startup
374-onionservice=HOST.onion:PORT # advertise this node's Tor hidden service
194-debug # verbose log; without it debug.log is nearly silent 375-debug # verbose log; without it debug.log is nearly silent
195-help # full list 376-help # full list
196``` 377```
@@ -199,10 +380,32 @@ Other options worth knowing:
199needed — the data directory takes an exclusive lock, so a second node pointed at 380needed — the data directory takes an exclusive lock, so a second node pointed at
200the same one will refuse to start. 381the same one will refuse to start.
201 382
383`-socks=127.0.0.1:9050` routes outbound Nostr discovery, `.btf` rendezvous
384dials, and direct `.onion` peer dials through a local SOCKS5 proxy such as Tor.
385When it is set, the node skips
386plain HTTP external-IP probes instead of leaking a direct request outside the
387proxy. IPv6 proxy endpoints use brackets, for example `-socks=[::1]:9050`.
388
389`-tor` is a privacy shorthand for the usual local Tor SOCKS5 listener at
390`127.0.0.1:9050`. It routes outbound Nostr discovery, `.btf` rendezvous dials,
391and signed direct `.onion` peer dials through Tor, and keeps external-IP probes
392disabled. Use `-tor=HOST:PORT` when Tor listens somewhere else.
393
394The desktop app starts a managed Tor automatically when no other Tor option is
395set, so onion transport works out of the box; `-nomanagedtor` turns that off.
396For a headless node, ask for it explicitly:
397
398`-managedtor[=PATH]` starts a Tor process for this node, writes a local `torrc`,
399creates a v3 hidden service for port `8433`, routes outbound discovery through
400that Tor instance, and signs the generated `.onion:8433` endpoint into the
401node's `.btf` descriptor. Without `PATH`, Bitflash looks for `tor/tor.exe`,
402`tor.exe`, or `tor` beside the binary and then on `PATH`. The hidden-service key
403is stored under the Bitflash data directory in `managed-tor/onion-service/`.
404
202### When something looks wrong 405### When something looks wrong
203 406
204**Menu > Diagnostics**, and the same report in `debug.log` every ten minutes: 407**Menu > Diagnostics**, and the same report in `debug.log` every ten minutes:
205peers held against peers `select()` is actually watching, blocks received and how 408peers held against peers the socket loop is actually watching, blocks received and how
206many arrived without a parent, the proof-of-work mode with the live miner thread 409many arrived without a parent, the proof-of-work mode with the live miner thread
207count, sockets by the part of the program that opened them, and per peer how long 410count, sockets by the part of the program that opened them, and per peer how long
208since the last message each way. There is a copy button — if you open an issue, 411since the last message each way. There is a copy button — if you open an issue,
@@ -284,6 +487,19 @@ make windows
284``` 487```
285Installs deps via pacman, produces `Bitflash-*-windows.zip`. 488Installs deps via pacman, produces `Bitflash-*-windows.zip`.
286 489
490To build the Windows package that also carries the Tor Expert Bundle:
491
492```bash
493make windows-tor
494```
495
496That produces `Bitflash-*-windows-with-tor.zip`. The release script downloads
497the Tor Expert Bundle from the official Tor archive, verifies its pinned SHA256,
498optionally verifies the Tor Project GPG signature when `gpg` is available, and
499places `tor/tor.exe` beside `Bitflash.exe` so `-managedtor` works without a
500separate Tor install. For release builds, use `TOR_VERIFY_GPG=required make
501windows-tor` to require the extra signature check.
502
287--- 503---
288 504
289## At a glance 505## At a glance
@@ -303,7 +519,8 @@ Installs deps via pacman, produces `Bitflash-*-windows.zip`.
303| Addressing | `.btf` rendezvous — see the caveats above | 519| Addressing | `.btf` rendezvous — see the caveats above |
304| Premine | None | 520| Premine | None |
305| Pool server | Built-in — `.btf` rendezvous only | 521| Pool server | Built-in — `.btf` rendezvous only |
306| Wallet recovery | Twelve-word phrase (BIP39 + BIP32), plus file backup | 522| Wallet recovery | Twelve-word phrase (BIP39 + BIP32 + BIP44, coin type 4346950), plus file backup |
523| Wallet storage | SQLite `wallet.sqlite` by default, or Berkeley DB `wallet.dat` |
307 524
308The halving interval is the number most people get wrong coming from Bitcoin. 525The halving interval is the number most people get wrong coming from Bitcoin.
309Same 210,000 blocks, but at two minutes instead of ten, so it arrives in about 526Same 210,000 blocks, but at two minutes instead of ten, so it arrives in about
@@ -311,4 +528,6 @@ ten months rather than four years.
311 528
312--- 529---
313 530
531Young network. Keep your node current and don't put in more than you are willing to lose.
532
314MIT. Built on Bitcoin 0.1.0 (Satoshi Nakamoto, 2009). 533MIT. Built on Bitcoin 0.1.0 (Satoshi Nakamoto, 2009).
diff --git a/docs/dependencies.md b/docs/dependencies.md
new file mode 100644
index 0000000..9fd45b6
--- /dev/null
+++ b/docs/dependencies.md
@@ -0,0 +1,18 @@
1# Build Dependencies
2
3Bitflash links consensus-adjacent cryptographic and proof-of-work code from
4external repositories. Fresh builds must not follow upstream `master` silently:
5a changed upstream branch can break reproducible builds or change behavior in
6code that every node relies on.
7
8The top-level `Makefile` pins source dependencies by full commit hash:
9
10| dependency | repository | pinned commit |
11|---|---|---|
12| libsecp256k1 | https://github.com/bitcoin-core/secp256k1 | `7fecac74aed8e1fd9078380d67dd04663705c989` |
13| RandomX | https://github.com/tevador/RandomX | `1e9d4b2df63fa6edf46b06789486e23c0fcaa55a` |
14
15When updating either dependency, rebuild from a clean dependency directory and
16run the full selftest suite before opening the PR. The PR should explain why the
17dependency changed and whether the update can affect consensus, addresses,
18signatures, proof-of-work validation, or mining.
diff --git a/docs/fair-launch.md b/docs/fair-launch.md
new file mode 100644
index 0000000..9603910
--- /dev/null
+++ b/docs/fair-launch.md
@@ -0,0 +1,70 @@
1# Bitflash fair-launch verifier
2
3Bitflash ships a deterministic fair-launch audit script:
4
5```bash
6python3 scripts/verify-fair-launch.py --max-blocks 1000 --out bitflash-fair-launch-report.json --json
7```
8
9The verifier reads `blk*.dat` directly from the local Bitflash data directory.
10It does not use Berkeley DB, does not call the node binary, and does not need
11network access. It uses the same shared Python chain parser as the UTXO
12commitment and proof tools, so block serialization fixes land in one place.
13
14The report proves the launch baseline that can be checked from the chain:
15
16- the genesis block hash matches the source constant;
17- the genesis merkle root matches the source constant;
18- the genesis transaction is a single coinbase transaction;
19- the genesis output total is exactly `50.00000000 BTF`;
20- the scanned block range and mined coinbase total are reproducible.
21
22For machines that do not have a synced data directory yet, the source constants
23can be checked without disk blocks:
24
25```bash
26python3 scripts/verify-fair-launch.py --reconstruct-genesis
27```
28
29## Deterministic output
30
31Use `--json` or `--out` for canonical JSON. The JSON is written with sorted
32keys, fixed separators, ASCII escaping, and one trailing newline. It deliberately
33does not include local paths or wall-clock generation time, so two auditors
34scanning the same block range produce the same bytes.
35
36That makes the report shareable:
37
38```bash
39python3 scripts/verify-fair-launch.py --max-blocks 1000 --out report.json --json
40sha256sum report.json
41```
42
43If another auditor runs the same command against the same chain prefix, the
44`sha256sum` should match.
45
46## OpenTimestamps
47
48OpenTimestamps is optional and external. It anchors the report in Bitcoin
49without adding any bridge or dependency to Bitflash consensus.
50
51After generating a canonical JSON report:
52
53```bash
54ots stamp report.json
55ots verify report.json.ots
56```
57
58The `.ots` file proves that the exact report bytes existed no later than the
59Bitcoin block that confirms the timestamp.
60
61## What this does not claim
62
63This is not a retroactive genesis commitment. The Bitflash genesis block already
64exists, so adding a new on-chain commitment to genesis would create a different
65chain. This verifier instead audits the launch state that is already on-chain
66and makes the result reproducible and timestampable.
67
68It also does not prove real-world miner identity. It proves the genesis state,
69the absence of non-coinbase premine in genesis, and the deterministic coinbase
70totals over the scanned prefix.
diff --git a/docs/mining-calculator.md b/docs/mining-calculator.md
new file mode 100644
index 0000000..e6cb61b
--- /dev/null
+++ b/docs/mining-calculator.md
@@ -0,0 +1,108 @@
1# Bitflash mining calculator
2
3`scripts/bitflash-mining-calculator.py` is a local WhatToMine-style calculator
4for Bitflash.
5
6It does not fetch market prices, does not talk to exchanges, and does not
7change consensus. Price is a manual input until Bitflash has a reliable public
8market.
9
10## Examples
11
12Estimate from network hashrate:
13
14```bash
15python3 scripts/bitflash-mining-calculator.py \
16 --hashrate 21 --hashrate-unit kh/s \
17 --network-hashrate 1.56 --network-hashrate-unit mh/s \
18 --watts 140 --kwh-cost 0.10 \
19 --price 0.001 --pool-fee 1
20```
21
22Estimate from Bitflash difficulty:
23
24```bash
25python3 scripts/bitflash-mining-calculator.py \
26 --hashrate 21 --hashrate-unit kh/s \
27 --difficulty 45600 \
28 --watts 140 --kwh-cost 0.10
29```
30
31Machine-readable output:
32
33```bash
34python3 scripts/bitflash-mining-calculator.py \
35 --hashrate 21 --hashrate-unit kh/s \
36 --network-hashrate 1.56 --network-hashrate-unit mh/s \
37 --json
38```
39
40## Bitflash Difficulty Scale
41
42Bitflash does not use Bitcoin's SHA256 difficulty-1 constant for pool shares.
43The chain's easiest RandomX target is `bnProofOfWorkLimit = ~uint256(0) >> 12`,
44so difficulty 1 expects about `4096` hashes.
45
46The calculator uses:
47
48```text
49network_hashrate = difficulty * 4096 / block_time_seconds
50```
51
52At the default two-minute block target:
53
54```text
55network_hashrate = difficulty * 4096 / 120
56```
57
58If you already know network hashrate, pass `--network-hashrate` directly and
59the script derives the equivalent Bitflash difficulty for display.
60
61## Reward Estimate
62
63The expected daily reward is:
64
65```text
66blocks_per_day = 86400 / block_time_seconds
67miner_share = miner_hashrate / network_hashrate
68gross_btf_per_day = blocks_per_day * block_reward * miner_share
69net_btf_per_day = gross_btf_per_day * (1 - pool_fee_percent / 100)
70```
71
72Electricity is:
73
74```text
75electricity_per_day = watts / 1000 * 24 * kwh_cost
76```
77
78This is an expectation, not a promise. Solo mining variance can be very large,
79especially when the miner has a small share of the network.
80
81## Future Website Use
82
83The script is intentionally standalone so the same math can later back a
84website widget or status dashboard. Until a public price exists, a website
85calculator should keep price as a manual field and label fiat profit as
86speculative.
87
88## Website Fields
89
90A public calculator should ask for:
91
92- miner hashrate and unit;
93- power draw in watts;
94- electricity price per kWh;
95- pool fee percent;
96- optional BTF price;
97- either current Bitflash difficulty or current network hashrate.
98
99The output should separate:
100
101- estimated BTF per day;
102- estimated gross fiat revenue, if a manual price was provided;
103- electricity cost;
104- net profit/loss;
105- the difficulty/hashrate assumptions used.
106
107Never present the fiat number as a market quote unless it comes from a real
108public market. Before that exists, label it as a manual scenario.
diff --git a/docs/pool-directory.md b/docs/pool-directory.md
new file mode 100644
index 0000000..ff0640a
--- /dev/null
+++ b/docs/pool-directory.md
@@ -0,0 +1,61 @@
1# Public Pool Directory
2
3`pool.bitflash.network` should be a public directory, not a network dependency.
4Nodes discover pools through Nostr and `.btf`; the website only helps humans
5compare operators.
6
7Operators can publish the `pool_status.json` and `pool_rounds.json` files
8produced by:
9
10```bash
11bitflash -nogui -gen -operator \
12 -poolstatusfile=/var/www/pool_status.json \
13 -poolroundsfile=/var/www/pool_rounds.json
14```
15
16Build a static directory from one or more status files:
17
18```bash
19python3 scripts/build-pool-index.py --out-dir public/pool \
20 https://operator-one.example/pool_status.json \
21 https://operator-two.example/pool_status.json
22```
23
24The generated output contains:
25
26- `index.html`: human-readable ranking page.
27- `pools.json`: machine-readable API for `api.bitflash.network/pools.json`.
28
29The website is a convenience layer, not a consensus or discovery dependency.
30Nodes discover pools through Nostr descriptors and `.btf`; a missing website
31must not stop mining or pool operation.
32
33When a source is an HTTP(S) `pool_status.json`, the directory links to a
34`pool_rounds.json` proof file next to it. That file records the block height,
35block hash, coinbase txid, coinbase value, pool fee, total shares, per-address
36share counts, computed payouts, and payout txids once paid.
37
38Ranking order is intentionally simple:
39
401. online pools first;
412. higher reported hashrate;
423. more authorized miners;
434. lower fee;
445. name as a stable tie-breaker.
45
46An entry is considered online when the source says the pool is running and its
47status file is no more than 120 seconds old.
48
49## Recommended Hostnames
50
51Use:
52
53```text
54pool.bitflash.network human-readable pool directory
55api.bitflash.network/pools.json
56status.bitflash.network relay/node health, not pool ranking
57docs.bitflash.network operator setup guide
58```
59
60The GUI may use the API as a shortcut for humans, but it should keep the
61existing decentralized pool discovery path as the source of truth.
diff --git a/docs/release-verification.md b/docs/release-verification.md
index ecacd5f..b8eaf5e 100644
--- a/docs/release-verification.md
+++ b/docs/release-verification.md
@@ -3,60 +3,100 @@
3Bitflash releases publish binaries and a `SHA256SUMS` file. The checksum file 3Bitflash releases publish binaries and a `SHA256SUMS` file. The checksum file
4proves that the file you downloaded matches the file the release page names. 4proves that the file you downloaded matches the file the release page names.
5A detached signature, `SHA256SUMS.asc`, proves that the checksum file itself was 5A detached signature, `SHA256SUMS.asc`, proves that the checksum file itself was
6signed by a trusted release key. 6signed by the Bitflash release key.
7 7
8That gives users two separate checks: 8That gives users two separate checks:
9 9
101. `gpg --verify SHA256SUMS.asc SHA256SUMS` checks who signed the checksums. 101. `gpg --verify SHA256SUMS.asc SHA256SUMS` — who signed the checksums.
112. `sha256sum -c SHA256SUMS` checks the binaries against those checksums. 112. `sha256sum -c SHA256SUMS` — the binaries against those checksums.
12 12
13## User Check 13But the first check is only worth something if you require a **specific** key.
14A good signature by *some* key proves nothing; anyone can make one.
14 15
15From a shell with `curl` or `wget` and `sha256sum`: 16## The release signing key
17
18```text
19fingerprint 910A 2B4C CA87 9E81 FB4B 2AEA 1D4A 53D3 B78A A4B8
20uid Bitflash Releases (bitflash.network release signing)
21 <releases@bitflash.network>
22```
23
24The public key is published at
25`https://releases.bitflash.network/bitflash-release-key.asc` and embedded in
26`scripts/verify-release.sh`, so the tool's trust anchor lives in this
27repository rather than on the server it verifies.
28
29## User check
30
31From a shell with `curl` or `wget`, `sha256sum` and `gpg`:
16 32
17```bash 33```bash
18scripts/verify-release.sh latest 34scripts/verify-release.sh latest
19``` 35```
20 36
21For a specific release: 37The public release host should expose latest-file aliases for convenience:
22 38
23```bash 39```text
24scripts/verify-release.sh v1.2.13 40https://releases.bitflash.network/SHA256SUMS
41https://releases.bitflash.network/SHA256SUMS.asc
25``` 42```
26 43
27For release audits, require the signature: 44Those aliases must point at the current versioned directory. They do not replace
45signature verification; they only make the common manual path shorter.
46
47For a specific release:
28 48
29```bash 49```bash
30scripts/verify-release.sh v1.2.13 --require-signature 50scripts/verify-release.sh v1.2.15
31``` 51```
32 52
33Older releases may not have `SHA256SUMS.asc`. In that case the script warns and 53The script downloads the release, imports the pinned key into a throwaway
34still checks file integrity. New release audits should use `--require-signature`. 54keyring, and **refuses anything that is not a valid signature by the key above**
35 55before checking the hashes. Releases from before signing was introduced ship
36## Maintainer Flow 56only `SHA256SUMS`; without `--require-signature` those are checked for integrity
57and reported as unsigned.
37 58
38After building release assets in the repository root: 59By hand, if you would rather not run the script:
39 60
40```bash 61```bash
41scripts/make-release-checksums.sh --sign --local-user RELEASE_KEY_ID 62curl -O https://releases.bitflash.network/bitflash-release-key.asc
63gpg --import bitflash-release-key.asc
64gpg --verify SHA256SUMS.asc SHA256SUMS # must name the fingerprint above
65sha256sum -c SHA256SUMS
42``` 66```
43 67
44Upload all built assets plus: 68## Maintainer flow
45 69
46```text 70The signing key lives on a dedicated location, not in the build tree. After
47SHA256SUMS 71building the release assets in the repository root, write and sign the
48SHA256SUMS.asc 72checksums:
73
74```bash
75sha256sum Bitflash-*.zip Bitflash-*.AppImage bitflash-node-* > SHA256SUMS
76gpg --armor --detach-sign --output SHA256SUMS.asc SHA256SUMS
49``` 77```
50 78
51Keep the private signing key offline or on a dedicated release machine. Publish 79Publish, alongside the assets, `SHA256SUMS` and `SHA256SUMS.asc`, and update
52the public key fingerprint in the release notes and keep using the same key for 80`latest.txt` on the releases host to the new tag. Keep using the same key; if it
53future releases unless there is a clearly announced rotation. 81is ever rotated, announce the new fingerprint clearly and update the embedded
82key in `verify-release.sh`.
83
84For Windows releases with managed Tor built in, run `make windows-tor`. It
85creates `Bitflash-*-windows-with-tor.zip` by downloading the Tor Expert Bundle,
86checking the pinned SHA256 in `scripts/package-windows-tor.sh`, optionally
87verifying the Tor Project `.asc` signature, and placing `tor/tor.exe` next to
88`Bitflash.exe`. For release builds, run `TOR_VERIFY_GPG=required make
89windows-tor` so the signature check is mandatory. Include that zip in
90`SHA256SUMS` like any other release asset.
54 91
55## Why This Matters 92## Why this matters
56 93
57`SHA256SUMS` alone protects against a broken download, but not against someone 94`SHA256SUMS` alone protects against a broken download, but not against someone
58replacing both a binary and the checksum file. Signing `SHA256SUMS` means an 95replacing both a binary and the checksum file. Signing `SHA256SUMS`, and
59attacker must also have the release signing key to make the replacement verify. 96requiring a pinned key, means an attacker must also hold the release signing key
97to make the replacement verify. Serving from bitflash.network makes this the
98guarantee that matters: without a pinned key, "trust the site" is the only
99thing standing behind a download.
60 100
61This is not reproducible builds yet. It is the smaller, immediate step that 101This is not reproducible builds yet. It is the smaller, immediate step that
62makes every release asset auditable by users before they run it. 102makes every release asset auditable before anyone runs it.
diff --git a/docs/rendezvous.md b/docs/rendezvous.md
new file mode 100644
index 0000000..68b5df3
--- /dev/null
+++ b/docs/rendezvous.md
@@ -0,0 +1,336 @@
1# Bitflash `.btf` Rendezvous Protocol
2
3This is the specification of how Bitflash nodes reach each other, and how miners
4reach pools, without revealing an IP address and without any central directory.
5It documents the protocol as implemented in `src/btfaddr.cpp`, `src/btfchan.cpp`,
6`src/btfrv.cpp`, `src/btftunnel.cpp` and `src/nostr.cpp`.
7
8The design is ported from the Itzal hidden service design. A `.btf` address names
9a public key, a node proves ownership of that key by signing a service
10descriptor, other nodes find the descriptor over Nostr, and the two sides then
11meet through an untrusted relay over an end to end encrypted tunnel. The relay
12carries ciphertext only and never learns the plaintext.
13
14## 1. Overview
15
16A connection to a `.btf` service has three layers:
17
181. **Discovery (Nostr).** The service publishes a signed descriptor as a Nostr
19 event. A caller fetches it by the service public key and verifies the
20 signature. The descriptor says which rendezvous relay the service is waiting
21 at, which encryption key to use, and optionally which Tor hidden service can
22 be dialed directly.
232. **Rendezvous (TCP).** Caller and service both connect out to the same relay
24 and are paired by public key. Neither needs an inbound port or a public IP.
25 After pairing the relay is a blind byte pipe.
263. **Direct onion fallback bypass.** If the descriptor carries a valid signed
27 `.onion:port` endpoint and the caller is running with Tor mode, the caller
28 tries that endpoint before using rendezvous. This removes the rendezvous
29 relay from the data path for peers that choose to run hidden services.
304. **Encrypted channel (X25519 + XChaCha20-Poly1305).** Over the paired pipe the
31 caller and service run an ephemeral to static handshake and then exchange
32 authenticated frames. The relay sees only ciphertext.
33
34The caller ends up holding an ordinary local socket that behaves like a direct
35TCP connection to the peer. The node treats it exactly like any other peer
36socket.
37
38## 2. Identities and keys
39
40A node holds two independent keypairs.
41
42- **Signing / identity key**: a secp256k1 x-only key (BIP340 Schnorr). This is
43 the `.btf` identity and also the node's Nostr identity. Persisted in
44 `nostr.key`.
45- **Encryption key**: an X25519 (Curve25519) keypair. Persisted in
46 `btf_enc.key`. Its public half is published in the descriptor as `enc` and is
47 used for the tunnel key agreement.
48
49The identity key names the service and authenticates the descriptor. The
50encryption key secures the tunnel. They are separate so the long term identity
51never performs Diffie-Hellman and the encryption key can be rotated in a new
52signed descriptor.
53
54## 3. The `.btf` address
55
56An address is the identity public key plus a short checksum, Base32 encoded, with
57the suffix `.btf`.
58
59```
60label = Base32( pubkey[32] || checksum[2] )
61address = label + ".btf"
62```
63
64- `pubkey` is the 32 byte secp256k1 x-only public key.
65- `checksum` is the first two bytes of `SHA256(".btf-checksum-v1" || pubkey)`.
66- Base32 uses the lowercase alphabet `abcdefghijklmnopqrstuvwxyz234567`, no
67 padding. The 34 byte payload encodes to 55 characters.
68- Parsing is case insensitive, trims surrounding whitespace, and strips a
69 trailing dot. A wrong checksum is rejected.
70
71The address is self certifying: it *is* the public key, so no registry has to be
72trusted to map a name to a key.
73
74## 4. Discovery over Nostr
75
76Nodes publish and read three kinds of parameterized replaceable Nostr events.
77Parameterized replaceable means the newest event for a given author and `d` tag
78supersedes the older one, so a descriptor is always the latest state.
79
80| Purpose | kind | `d` tag |
81|---|---|---|
82| Service descriptor | 38501 | `btf-descriptor-2` |
83| Rendezvous relay announcement | 38502 | `btf-relay-2` |
84| Pool announcement | 38503 | `btf-pool-2` |
85
86The Nostr event is signed with the node's identity key, so the event author
87equals the `.btf` public key. That binds the descriptor to the address at the
88Nostr layer, and the descriptor is *also* signed inside its own content (see
89below), so a relay operator cannot forge or swap a descriptor for an address it
90does not control.
91
92**Resolving an address.** To reach `X.btf`, a caller parses the address to the 32
93byte key and sends a Nostr `REQ` filtered by `authors = [X]`, `kinds = [38501]`,
94`#d = ["btf-descriptor-2"]`, `limit = 1`. It verifies the returned descriptor and
95uses its `meeting_node` and `enc`.
96
97**Relay auto discovery.** A relay operator runs a rendezvous relay and publishes a
98kind 38502 event carrying `host:port`. Every node that reads it adds the relay to
99the set it can register at and dial through. No seed list edit or maintainer
100approval is needed. A small curated seed list is still shipped for bootstrap:
101
102```
10392.246.128.180:8434
10431.44.4.249:8434
10590.156.222.107:8434
106```
107
108The default rendezvous port is `8434`.
109
110## 5. The service descriptor
111
112The descriptor is a JSON object, signed with the identity key over a domain
113separated digest of its own fields.
114
115Legacy descriptor:
116
117```json
118{
119 "v": 2,
120 "pubkey": "<hex 32-byte x-only identity key>",
121 "enc": "<hex 32-byte X25519 public key>",
122 "meeting_node": "host:port",
123 "created": <unix seconds>,
124 "sig": "<hex 64-byte BIP340 Schnorr signature>"
125}
126```
127
128Onion-capable descriptor:
129
130```json
131{
132 "v": 3,
133 "pubkey": "<hex 32-byte x-only identity key>",
134 "enc": "<hex 32-byte X25519 public key>",
135 "meeting_node": "host:port",
136 "onion": "<56-char-v3-name>.onion:8433",
137 "created": <unix seconds>,
138 "sig": "<legacy v2 signature>",
139 "sig3": "<signature over the onion-capable digest>"
140}
141```
142
143The legacy signed digest is:
144
145```
146digest = SHA256( ".btf-descriptor-v2"
147 || pubkey[32]
148 || enc (raw bytes of the string)
149 || meeting_node (raw bytes of the string)
150 || created (8 bytes, big endian) )
151```
152
153`sig` is kept on v3 descriptors so old nodes can still resolve and dial through
154the rendezvous relay. The onion-capable digest is:
155
156```
157digest3 = SHA256( ".btf-descriptor-v3-onion"
158 || pubkey[32]
159 || enc (raw bytes of the string)
160 || meeting_node (raw bytes of the string)
161 || onion (raw bytes of the normalized string)
162 || created (8 bytes, big endian) )
163```
164
165`sig` and `sig3` are deterministic BIP340 Schnorr signatures (no auxiliary
166randomness) over their respective digests by the identity key.
167
168**Verification** (untrusted input, must never throw): parse the JSON, require all
169fields with the right types, confirm `pubkey` equals the address being resolved,
170recompute the digest, and verify the Schnorr signature. Any failure rejects the
171descriptor.
172
173If `onion` is present, readers accept it only when it is a normalized v3 onion
174endpoint and `sig3` verifies. A bad or tampered onion field is ignored; the v2
175descriptor can still be used for rendezvous fallback if `sig` verifies.
176
177`created` lets a reader prefer the freshest descriptor when more than one is
178seen. A node without a direct onion endpoint only publishes a descriptor once it
179has actually registered at a `meeting_node`, so it never advertises a relay it
180cannot be reached at. A node with a direct onion endpoint may publish before
181rendezvous registration using `meeting_node = "rendezvous-pending"`; Tor peers
182can still dial the signed onion endpoint, and rendezvous becomes fallback once
183registration succeeds.
184
185## 6. Rendezvous transport
186
187The relay is a small TCP server. Every connection opens with a fixed 33 byte
188header:
189
190```
191byte 0 role: 'S' (service) or 'C' (client)
192bytes 1..32 target public key (the .btf identity, 32 bytes)
193```
194
195**Service registration (`S`).** The service connects out to the relay and sends
196`'S' || own_pubkey`. The relay stores the connection in a table keyed by the
197public key and leaves it open, waiting. There is one slot per key: a new
198registration for the same key replaces and closes the old one. The service does
199not block waiting to be paired at registration time; it is considered registered
200the moment the relay reads the header, which is what lets it publish a descriptor
201pointing at this relay.
202
203**Client dial (`C`).** The caller connects out to the same relay and sends
204`'C' || target_pubkey`. The relay looks up the waiting service:
205
206- If present, it removes it from the table, writes a single `0x01` byte to both
207 sockets, and then forwards raw bytes in both directions until either side
208 closes. From this point the relay is a blind pipe.
209- If absent, it writes `0x00` to the caller and closes.
210
211Both sides connect *outbound* to the relay, so neither needs a public IP or an
212open inbound port. The relay never sees the identity of the caller (the caller
213sends no key of its own, only the target it wants).
214
215**Liveness.** A registration socket is idle by design and can outlive the network
216path that created it (a NAT drops the idle mapping, the relay restarts, a route
217changes) with no FIN and nothing readable. TCP keepalive was measured and does
218not reliably detect this. The service therefore imposes its own timeout on the
219wait for the pairing byte and re-registers on expiry. Expiring is not a failure;
220it costs one reconnect per interval and returns the loop to a known state.
221
222## 7. The encrypted channel
223
224Once paired through the relay, the caller and service run an end to end handshake
225and then exchange authenticated frames. The primitive is libsodium
226`crypto_box_curve25519xchacha20poly1305` (X25519 key agreement, XChaCha20-Poly1305
227AEAD), the same construction as Rust's `crypto_box` ChaChaBox.
228
229**Handshake (ephemeral to static).**
230
2311. The caller generates a fresh ephemeral X25519 keypair, computes
232 `shared = X25519(eph_sk, service_enc_pub)` reduced to a box key
233 (`crypto_box_beforenm`), and sends the 32 byte ephemeral public key over the
234 paired pipe.
2352. The service reads the ephemeral public key and computes the same
236 `shared = X25519(enc_sk, eph_pub)`.
237
238Because the caller uses a new ephemeral key per connection, each session has an
239independent key and forward secrecy against later compromise of the service
240encryption key.
241
242**Framing.** Every message is length prefixed:
243
244```
245frame = len[4] (big endian, length of the body that follows)
246 body = nonce[24] || ciphertext || mac[16]
247```
248
249The nonce is 24 random bytes per frame (XChaCha20), the MAC is 16 bytes
250(Poly1305). A body shorter than `nonce + mac` or larger than 1 MiB is rejected.
251Decryption that fails authentication drops the frame and tears the tunnel down.
252
253The tunnel is transparent: the application is handed a local socket pair end and
254reads and writes plaintext. Two pump threads move data between that socket and
255the relay socket, encrypting outbound and decrypting inbound.
256
257## 8. End to end connection flow
258
259To dial `X.btf`:
260
2611. Parse `X.btf` to the 32 byte identity key.
2622. Fetch and verify the descriptor over Nostr (kind 38501). Obtain
263 `meeting_node` and `enc`.
2643. Connect to `meeting_node`, send `'C' || X`, and wait for `0x01`.
2654. Generate an ephemeral X25519 key, derive the shared key against `enc`, send
266 the ephemeral public key.
2675. Exchange XChaCha20-Poly1305 frames. Hand the application a local socket that
268 speaks the ordinary peer protocol.
269
270A service, symmetrically: registers at a relay (`'S' || own_pubkey`), publishes a
271descriptor pointing at that relay, waits for `0x01`, reads the caller ephemeral
272public key, derives the shared key with its encryption secret, and serves the
273tunnel.
274
275## 9. Security properties
276
277- **Address unforgeability.** The address is the identity key and the descriptor
278 is Schnorr signed by it, and the Nostr event is signed by it. A relay or Nostr
279 operator cannot publish a working descriptor for an address it does not own.
280- **Relay confidentiality.** The relay forwards ciphertext and never holds the
281 channel key. It cannot read or modify traffic undetected; a tampered frame
282 fails the Poly1305 tag and the tunnel closes.
283- **Caller anonymity to the service and relay.** The caller presents no identity
284 key, only the target it wants to reach. It connects outbound, so its address is
285 not published anywhere.
286- **Forward secrecy.** The caller's per connection ephemeral key means a later
287 compromise of the service encryption key does not decrypt past sessions.
288- **IP privacy.** Neither side needs a public IP or an inbound port for
289 rendezvous. With a signed `.onion` endpoint, the reachable side exposes only a
290 Tor hidden service and the relay is not in the peer connection path.
291
292## 10. Threat model and limitations
293
294- **The relay is trusted for availability, not for confidentiality.** It can
295 refuse to pair, drop a connection, or go offline. This is why relays are
296 plural, auto discovered, and re-registration is periodic. It cannot read
297 traffic.
298- **Traffic analysis is not addressed.** A network observer, or the relay, can
299 see connection timing and byte volumes. There is no padding or mixing.
300- **Single slot registration causes churn.** One waiting slot per key per relay
301 means a race or a stale registration can briefly make a service unreachable at
302 that relay. Multiple relays and the wait timeout bound the damage, but this is
303 a known source of connection churn.
304- **Descriptor freshness is best effort.** `created` is a self reported
305 timestamp; readers prefer the newest descriptor they have seen but there is no
306 global clock.
307- **Discovery depends on public Nostr relays.** If the relays a node uses are all
308 unreachable, discovery stalls until one is reachable again. Relay TLS is
309 verified, which closes a man in the middle path on discovery, but availability
310 still rests on the relay set.
311- **Direct onion needs Tor.** With `-managedtor`, Bitflash starts a Tor process,
312 writes a local hidden-service config, and advertises the generated onion
313 endpoint. With `-onionservice`, an operator can still run Tor manually and
314 provide the endpoint. In both cases Tor owns the hidden-service private key and
315 uptime.
316
317## 11. Wire reference
318
319| Item | Value |
320|---|---|
321| `.btf` Base32 alphabet | `abcdefghijklmnopqrstuvwxyz234567`, no padding |
322| Address payload | `pubkey[32] || checksum[2]`, 55 Base32 chars |
323| Address checksum | `SHA256(".btf-checksum-v1" || pubkey)[0..2]` |
324| Descriptor domain sep | `.btf-descriptor-v2`; optional onion sep `.btf-descriptor-v3-onion` |
325| Descriptor version | `2`; `3` when a signed onion endpoint is present |
326| Descriptor signature | BIP340 Schnorr, deterministic (no aux rand), v3 keeps `sig` plus `sig3` |
327| Nostr descriptor event | kind `38501`, `d = btf-descriptor-2` |
328| Nostr relay event | kind `38502`, `d = btf-relay-2` |
329| Nostr pool event | kind `38503`, `d = btf-pool-2` |
330| Rendezvous port (default) | `8434` |
331| Rendezvous header | `role[1] ('S'/'C') || pubkey[32]`, 33 bytes |
332| Rendezvous pairing byte | `0x01` paired, `0x00` no service |
333| Channel primitive | `crypto_box_curve25519xchacha20poly1305` |
334| Nonce / MAC / key sizes | 24 / 16 / 32 bytes |
335| Frame | `len[4 big endian] || nonce[24] || ct || mac[16]` |
336| Max frame body | 1 MiB |
diff --git a/docs/site.md b/docs/site.md
new file mode 100644
index 0000000..9526a61
--- /dev/null
+++ b/docs/site.md
@@ -0,0 +1,115 @@
1# Public Site Contract
2
3The public Bitflash sites are generated from this repository. If a deployed
4page and the source disagree, the deployed page is stale.
5
6## Canonical Navigation
7
8Every public page should use the same top navigation:
9
10| label | target |
11|---|---|
12| home | `https://bitflash.network/` |
13| code | `https://git.bitflash.network/bitflash/bitflash` |
14| docs | `https://docs.bitflash.network/` |
15| downloads | `https://releases.bitflash.network/` |
16| explorer | `https://explorer.bitflash.network/` |
17| status | `https://status.bitflash.network/` |
18
19The `code` link intentionally points at the repository, not the Forgejo
20instance root. A visitor choosing "code" wants the source tree.
21
22## Documentation Pages To Publish
23
24The docs index should expose these source files:
25
26| page | source | why it matters |
27|---|---|---|
28| Manual | `README.md` | run, mine, recover, operate |
29| Key derivation and address format | `docs/derivation.md` | independent wallet recovery |
30| Release verification | `docs/release-verification.md` | pinned signing-key flow |
31| Dependencies | `docs/dependencies.md` | pinned secp256k1/RandomX supply chain |
32| Rendezvous and .btf addressing | `docs/rendezvous.md` | network design and caveats |
33| Tor mode | `docs/tor.md` | `-tor`, `-socks`, onion relay operation |
34| Fair launch verifier | `docs/fair-launch.md` | no-premine launch audit |
35| UTXO set commitment | `docs/utxo-commitment.md` | deterministic supply/root audit |
36| UTXO inclusion proofs | `docs/utxo-proofs.md` | proof for one unspent output |
37| Mining calculator | `docs/mining-calculator.md` | local WhatToMine-style estimates |
38| Public pool directory | `docs/pool-directory.md` | `pool_status.json`, `pools.json`, ranking |
39
40## Release Mirror Aliases
41
42Each release remains under its versioned directory:
43
44```text
45https://releases.bitflash.network/v1.2.19/SHA256SUMS
46https://releases.bitflash.network/v1.2.19/SHA256SUMS.asc
47```
48
49The release root should also alias the latest files:
50
51```text
52https://releases.bitflash.network/SHA256SUMS
53https://releases.bitflash.network/SHA256SUMS.asc
54https://releases.bitflash.network/Bitflash-latest-windows.zip
55https://releases.bitflash.network/Bitflash-latest-x86_64.AppImage
56https://releases.bitflash.network/bitflash-node-latest-x86_64
57```
58
59The aliases are convenience only. Verification must still require the pinned
60release key documented in `docs/release-verification.md`.
61
62## Static Explorer
63
64`scripts/build-explorer.py` writes a static explorer with no backend:
65
66```text
67index.html
68style.css
69explorer.js
70logo.png
71blocks.json
72block/<height>.json
73```
74
75Serve it with a strict content policy:
76
77```text
78default-src 'none';
79img-src 'self';
80style-src 'self';
81script-src 'self';
82connect-src 'self';
83base-uri 'none';
84form-action 'none';
85frame-ancestors 'none';
86```
87
88The viewer must never use `innerHTML` for chain data. A malicious transaction
89must be rendered as text, not interpreted as markup.
90
91## Cache Policy
92
93Recommended defaults:
94
95| path | cache |
96|---|---|
97| HTML pages | `public, max-age=0, must-revalidate` |
98| `status.json` | `no-store` |
99| `blocks.json`, `block/*.json` | `public, max-age=0, must-revalidate` |
100| release assets | `public, max-age=14400, must-revalidate` |
101| `SHA256SUMS`, signatures, keys | `public, max-age=0, must-revalidate` |
102
103## Security Headers
104
105Recommended for all static pages:
106
107```text
108Strict-Transport-Security: max-age=31536000; includeSubDomains
109X-Content-Type-Options: nosniff
110Referrer-Policy: no-referrer
111Permissions-Policy: interest-cohort=(), geolocation=(), camera=(), microphone=()
112```
113
114Forgejo has its own application headers and should not blindly inherit the
115static-site CSP.
diff --git a/docs/social.png b/docs/social.png
index 19e2a3b..9a08b02 100644
--- a/docs/social.png
+++ b/docs/social.png
Binary files differ
diff --git a/docs/storage.md b/docs/storage.md
new file mode 100644
index 0000000..c7de4e7
--- /dev/null
+++ b/docs/storage.md
@@ -0,0 +1,159 @@
1# Bitflash Storage Inventory
2
3This document records the Berkeley DB surface that still exists in Bitflash.
4It is a migration map, not a promise that all stores should move at once.
5
6The immediate priority is `wallet.dat`: it holds spend authority and recovery
7metadata. `blkindex.dat` can be rebuilt from block files, so it is important for
8startup correctness but less dangerous than wallet storage.
9
10## Berkeley DB Environment
11
12All `CDB` stores use one Berkeley DB environment rooted at:
13
14```text
15<datadir>/database/
16```
17
18The environment creates log files such as:
19
20```text
21database/log.0000000001
22```
23
24Those logs are part of the legacy BDB risk model. They are not user-facing
25wallet files, but they can contain historical records while the environment is
26live. Wallet encryption already rewrites `wallet.dat` and purges BDB environment
27logs after the encrypted file is installed.
28
29## Common Records
30
31Every `CDB` file can carry:
32
33| Key | Value | Purpose |
34|---|---|---|
35| `"version"` | `int` | Database format version written when a DB is opened in create mode. |
36
37Keys and values are serialized with the existing `CDataStream` format. A future
38SQLite migration must preserve byte-for-byte value semantics until a higher
39level schema migration is intentionally introduced.
40
41## `wallet.dat`
42
43`wallet.dat` is accessed through `CWalletDB`.
44Read-only tooling should go through `ScanWalletRecords()` instead of opening a
45Berkeley DB cursor directly. That keeps migration checks, audits, and future
46backends behind one record-stream boundary. Wallet rewrite code should also use
47that boundary when it needs to copy existing records into a newly written
48wallet file.
49
50| Key shape | Value | Secret? | Meaning |
51|---|---|---:|---|
52| `("name", address)` | `string` | No | Address book label. |
53| `("tx", txid)` | `CWalletTx` | No | Wallet transaction state, including spend flags and metadata. |
54| `("key", pubkey)` | `CPrivKey` | Yes | Plain private key record for unencrypted wallets or old backups. |
55| `"defaultkey"` | `vector<unsigned char>` | No | Default public key shown by the wallet. |
56| `"hdmaster"` | `vector<unsigned char>` | Yes | Plain BIP32 master private key material. |
57| `"hdchaincode"` | `vector<unsigned char>` | Yes | Plain BIP32 chain code. |
58| `"hdnext"` | `unsigned int` | No | Legacy deterministic derivation next index. |
59| `"hdschema"` | `int` | No | Deterministic derivation schema (`legacy` or `bip44`). |
60| `"hdcointype"` | `unsigned int` | No | BIP44 coin type. Bitflash is `4346950`. |
61| `"hdreceivenext"` | `unsigned int` | No | BIP44 external receive-chain next index. |
62| `"hdchangenext"` | `unsigned int` | No | BIP44 internal change-chain next index. |
63| `("mkey", id)` | `CWalletMasterKey` | Sensitive | Encrypted wallet master-key metadata. |
64| `("ckey", pubkey)` | `vector<unsigned char>` | Yes | Private key encrypted under the wallet master key. |
65| `"cryptedhdmaster"` | `vector<unsigned char>` | Yes | Encrypted BIP32 master private key material. |
66| `"cryptedhdchaincode"` | `vector<unsigned char>` | Yes | Encrypted BIP32 chain code. |
67| `"walletminversion"` | `int` | No | Minimum wallet format version required to open this wallet safely. |
68| `("pool", index)` | `vector<unsigned char>` | No | Public key reserved in the key pool. The private key lives in `key`/`ckey`. |
69| `("setting", name)` | typed setting value | Usually no | Wallet/UI settings such as mining mode, fees, and backup timestamps. |
70
71### Wallet Migration Rules
72
73A future wallet backend must keep these properties:
74
751. Unknown future wallet formats fail closed with a clear message.
762. Encrypted wallets must encrypt private keys and HD seed material.
773. A locked encrypted wallet may expose public keys and balances, but not
78 private keys or decrypted HD seed material.
794. `-backupwallet` must produce a wallet file that opens in a different datadir
80 without copying `database/`, and must not leave a half-written final backup
81 path if the copy or install step fails.
825. `-recoveryaudit` must distinguish:
83 - no recovery phrase installed;
84 - recovery phrase present;
85 - recovery phrase encrypted and locked, requiring unlock before coverage can
86 be audited.
87
88## SQLite Wallet Store
89
90Since v1.2.20 SQLite is the default wallet backend. It is a byte-preserving
91record store, not a new wallet format -- the same serialized `CDataStream` bytes
92`wallet.dat` uses:
93
94```sql
95CREATE TABLE wallet_records (
96 key BLOB PRIMARY KEY NOT NULL,
97 value BLOB NOT NULL
98);
99PRAGMA user_version=1;
100```
101
102A fresh data directory starts on `wallet.sqlite`. An existing `wallet.dat` keeps
103running on Berkeley DB until it is converted: the GUI offers a one-click
104conversion on first run, and `-walletbackend=sqlite` / `-walletbackend=bdb`
105(recorded in a plain-text `wallet-backend` marker file in the datadir) select a
106backend explicitly. Conversion never modifies `wallet.dat`; it stays as a
107fallback, and the change takes effect on restart.
108
109Every wallet operation runs on either backend: read, write, erase, key-pool and
110HD-seed writes (the two halves of the seed bracketed in one SQLite transaction),
111`-backupwallet` (checkpoint then copy the self-contained file), `-encryptwallet`
112(an atomic rebuild-and-swap in one transaction), and the `-walletstorageaudit` /
113`-walletstoragecheck` diagnostics, which read the active backend's file.
114
115The migration tooling remains available for scripted use. `-walletsqliteexport=FILE`
116streams every raw `wallet.dat` record through `ScanWalletRecords()` into a SQLite
117file in one transaction, refusing to overwrite. `-walletsqliteverify=FILE`
118compares the two streams and prints only counts. `-walletsqliteloadcheck=FILE`
119parses an export with the same record types the loader uses.
120`-walletsqliterestore=FILE` rebuilds a `wallet.dat` from an export in an empty
121data directory. Failed exports and restores remove their incomplete output.
122
123## `blkindex.dat`
124
125`blkindex.dat` is accessed through `CTxDB`.
126
127| Key shape | Value | Rebuildable? | Meaning |
128|---|---|---:|---|
129| `("tx", txid)` | `CTxIndex` | Yes | Transaction disk position and spent-output index state. |
130| `("owner", hash160, CDiskTxPos)` | `int height` | Yes | Legacy owner transaction index cursor range. |
131| `("blockindex", blockhash)` | `CDiskBlockIndex` | Yes | Block index entry. |
132| `"hashBestChain"` | `uint256` | Yes | Current best-chain tip hash. |
133
134`blkindex.dat` can be rebuilt from `blk*.dat` plus consensus validation. A
135future migration may therefore prefer a rebuild path over a byte-preserving
136conversion path.
137
138## Other Legacy Stores
139
140None. Upstream still documents `reviews.dat` (`CReviewDB`) and `market.dat`
141(`CMarketDB`) here; this fork deleted the 2009 marketplace outright, along
142with `CReviewDB`, `market.cpp` and `market.h`, so neither file is opened or
143created by this tree. A datadir carrying them from an older build is simply
144ignored.
145
146The old IP address database (`addr.dat`) has been removed. Peer discovery is
147handled by Nostr, `.btf`, rendezvous relays, and direct onion peers.
148
149## Migration Order (shipped in v1.2.20)
150
1511. Storage self-tests were added around Berkeley DB behavior first.
1522. The SQLite backend landed behind an explicit flag, then behind a
153 `wallet-backend` marker and a GUI conversion wizard.
1543. BDB-to-SQLite migration is a one-way copy that leaves `wallet.dat` as a backup.
1554. SQLite became the default for new wallets only after cross-version restore,
156 encryption, backup, and recovery-audit tests passed -- and a live coin-proof
157 of receive, spend, restart, and encrypt on real mainnet funds.
1585. Treat `blkindex.dat` separately; it is safer to rebuild than to migrate under
159 pressure.
diff --git a/docs/tor.md b/docs/tor.md
new file mode 100644
index 0000000..0016f8b
--- /dev/null
+++ b/docs/tor.md
@@ -0,0 +1,231 @@
1# Tor mode
2
3Bitflash can route outbound discovery traffic through a local Tor SOCKS5 proxy.
4This is transport plumbing only: it does not change consensus, wallet keys,
5mining, `.btf` addresses, or the encrypted rendezvous protocol.
6
7## Client mode
8
9Start Tor locally, then start Bitflash with:
10
11```bash
12bitflash -tor
13```
14
15`-tor` is shorthand for:
16
17```bash
18bitflash -socks=127.0.0.1:9050
19```
20
21If Tor listens somewhere else:
22
23```bash
24bitflash -tor=127.0.0.1:9150
25bitflash -tor=[::1]:9050
26```
27
28For a SOCKS5 proxy that is not Tor, use `-socks` directly:
29
30```bash
31bitflash -socks=127.0.0.1:9050
32bitflash -socks=[::1]:9050
33```
34
35If both `-tor` and `-socks` are present, Tor mode wins and the node prints a
36warning. That keeps a command line with `-tor` from silently becoming a custom
37proxy route.
38
39## Managed Tor mode
40
41`-managedtor` starts a Tor process for this node, writes a local `torrc`, creates
42a v3 hidden service for the normal Bitflash P2P listener, routes outbound
43discovery through that Tor instance, and signs the generated `.onion:8433`
44endpoint into the node's `.btf` descriptor.
45
46```bash
47bitflash -managedtor
48```
49
50Without a value, Bitflash looks for Tor in this order:
51
521. `tor/tor.exe` or `tor/tor` beside the Bitflash binary;
532. `tor.exe` or `tor` beside the Bitflash binary;
543. `tor.exe` or `tor` on `PATH`.
55
56Windows releases can also be built as `Bitflash-*-windows-with-tor.zip`. That
57package includes the official Tor Expert Bundle under `tor/`, so the first path
58above exists immediately after extraction and `bitflash -managedtor` needs no
59separate Tor install.
60
61The bundled-Tor package is created by:
62
63```bash
64make windows-tor
65```
66
67The packaging script verifies a pinned SHA256 for the downloaded Tor Expert
68Bundle before copying it into the release directory. When `gpg` is available it
69also verifies the matching `.asc` signature against the pinned Tor Browser
70Developers signing-key fingerprint. Release builds should use
71`TOR_VERIFY_GPG=required make windows-tor` to make that second check mandatory.
72The Tor binary is bundled; the runtime data directory, hidden-service key, and
73control cookie are still created under the user's Bitflash data directory.
74
75To point at a specific Tor executable:
76
77```bash
78bitflash -managedtor=/opt/tor/bin/tor
79bitflash -managedtor=C:\Tor Browser\Browser\TorBrowser\Tor\tor.exe
80```
81
82Managed Tor writes its state under the Bitflash data directory:
83
84```text
85managed-tor/
86 torrc
87 data/
88 onion-service/
89```
90
91The private key for the onion address lives in `managed-tor/onion-service/`.
92Back up that directory if you want the same onion address after moving the node.
93Delete it only if you intentionally want a fresh onion endpoint.
94
95`-managedtor` takes precedence over `-tor` and `-socks`, because it must choose
96the local SOCKS port it starts. Diagnostics report the current managed Tor
97state, including the SOCKS port and onion endpoint once Tor has generated it.
98
99When Tor mode is enabled:
100
101- outbound Nostr relay dials use Tor;
102- outbound `.btf` rendezvous relay dials use Tor;
103- outbound direct peer dials prefer signed `.onion` endpoints when a peer
104 advertises one;
105- destination names are sent to Tor as SOCKS5 domain-name CONNECT requests, so
106 the local DNS resolver does not see them;
107- direct `.onion` dials are refused unless a SOCKS5/Tor proxy is enabled;
108- plain HTTP external-IP probes are skipped.
109
110Tor mode affects outbound discovery and relay dials. It does not encrypt
111`wallet.dat`, hide mining rewards on-chain, or change the `.btf` identity. A
112relay still sees the TCP client that connected to it; with Tor mode that client
113is the Tor exit or onion circuit endpoint instead of the user's direct IP.
114
115## What to test
116
117Use the public status page only as a rough health check. To prove the local node
118is using Tor/SOCKS, test from the node machine:
119
120```bash
121bitflash -tor -debug
122```
123
124Then confirm in `debug.log` that Nostr and rendezvous dials report the proxy
125path. A DNS leak test should show no local resolver lookup for relay hostnames:
126the SOCKS5 request uses the domain-name form and asks the proxy to resolve it.
127
128## Onion rendezvous relay
129
130An operator can expose a Bitflash rendezvous relay as a Tor hidden service. The
131node does not create hidden services itself; Tor owns that configuration.
132
133Example `torrc`:
134
135```text
136HiddenServiceDir /var/lib/tor/bitflash-rendezvous/
137HiddenServicePort 8434 127.0.0.1:8434
138```
139
140After restarting Tor, read the generated onion name:
141
142```bash
143sudo cat /var/lib/tor/bitflash-rendezvous/hostname
144```
145
146Then run the relay on localhost/VPS as usual and announce the onion endpoint from
147nodes that use Tor:
148
149```bash
150bitflash -tor -rvrelay=exampleexampleexampleexampleexampleexampleexampleexample.onion:8434
151bitflash -tor -announcerelay=exampleexampleexampleexampleexampleexampleexampleexample.onion:8434
152```
153
154For a pool operator that wants the pool descriptor to point at an onion
155rendezvous relay, use both:
156
157```bash
158bitflash -operator -tor \
159 -rvrelay=exampleexampleexampleexampleexampleexampleexampleexample.onion:8434 \
160 -announcerelay=exampleexampleexampleexampleexampleexampleexampleexample.onion:8434
161```
162
163## Manual direct full-node hidden service
164
165Managed Tor is the preferred path for ordinary nodes. Operators can still expose
166the P2P listener through a separately managed Tor service:
167
168```text
169HiddenServiceDir /var/lib/tor/bitflash-node/
170HiddenServicePort 8433 127.0.0.1:8433
171```
172
173After restarting Tor, read the generated onion name:
174
175```bash
176sudo cat /var/lib/tor/bitflash-node/hostname
177```
178
179Then start Bitflash with Tor outbound enabled and advertise that hidden service:
180
181```bash
182bitflash -tor \
183 -onionservice=exampleexampleexampleexampleexampleexampleexampleexample.onion:8433
184```
185
186That onion endpoint is signed into this node's self-certifying `.btf`
187descriptor. New peers that resolve the descriptor and also run with `-tor` try
188the `.onion:8433` P2P connection first. If it fails, they fall back to the
189ordinary encrypted rendezvous path. Older nodes ignore the onion field and still
190use the rendezvous relay because the descriptor keeps the legacy signature too.
191
192If the node has `-onionservice` but has not registered at any rendezvous relay
193yet, it can still publish an onion-capable descriptor with
194`meeting_node = rendezvous-pending`. Tor-capable peers can dial the hidden
195service directly from that descriptor. Rendezvous becomes a fallback, not a
196precondition for being discoverable.
197
198This means rendezvous relays stop being the only way to reach a node that has
199published a hidden service. They remain useful bootstrap/fallback infrastructure,
200but an already-discovered onion-capable peer can be reached directly over Tor.
201
202Tor owns the private key under `HiddenServiceDir`; back it up if you want the
203onion name to stay stable.
204
205## Release smoke test
206
207After building a Windows binary with managed Tor support, run the two-node smoke
208test before cutting a release:
209
210```powershell
211powershell -ExecutionPolicy Bypass -File scripts\smoke-managed-tor-onion.ps1 `
212 -Bitflash .\src\bitflash.exe `
213 -Tor C:\path\to\tor.exe
214```
215
216The script creates two temporary datadirs, starts node A with `-managedtor`,
217waits for it to publish its `.btf` identity and generated `.onion` endpoint,
218then starts node B with its own `-managedtor`, `-oniononly`, and
219`-connectbtf=<node-a>`. It passes only when node B logs that it connected to
220node A `via direct onion`; rendezvous fallback cannot satisfy the test.
221
222By default the script waits 120 seconds after node A's descriptor self-resolves
223before it starts node B. Tor can create the local `hostname` before the hidden
224service descriptor is visible enough for client circuits, and early dials often
225fail with SOCKS5 reply `0x01`. Override this with `-OnionWarmupSec N` when
226testing a slow or already-warmed Tor environment.
227
228This test still uses Nostr for descriptor discovery, because that is how nodes
229learn each other's signed `.btf` descriptors. It does not require a Bitflash
230rendezvous relay to carry the P2P connection: once the descriptor includes the
231signed onion endpoint, the actual node connection goes directly through Tor.
diff --git a/docs/utxo-commitment.md b/docs/utxo-commitment.md
new file mode 100644
index 0000000..ab43deb
--- /dev/null
+++ b/docs/utxo-commitment.md
@@ -0,0 +1,125 @@
1# Bitflash UTXO-set commitment
2
3`scripts/verify-utxo-set.py` reconstructs a deterministic commitment to the
4current UTXO set from local `blk*.dat` files.
5
6This is intentionally not a consensus rule yet. It is an audit tool: independent
7operators can run it against the same chain prefix and compare the resulting
8root and canonical JSON hash.
9
10```bash
11python3 scripts/verify-utxo-set.py --max-blocks 1000 --out utxo-report.json --json
12```
13
14Use `--max-blocks 0` or omit the option to scan every available block file:
15
16```bash
17python3 scripts/verify-utxo-set.py --out utxo-report.json --json
18```
19
20## What It Verifies
21
22The verifier:
23
24- parses Bitflash block files directly with the shared Python chain parser;
25- streams `blk*.dat` records instead of loading full block files into memory;
26- reconstructs the best chain rooted at the Bitflash genesis block;
27- ignores side blocks that may also be present in `blk*.dat`;
28- verifies every scanned block merkle root;
29- applies spends and creates outputs in deterministic order;
30- reports current UTXO count, current supply, mined coinbase total, fees, and
31 historical duplicate-output overwrites;
32- computes a deterministic UTXO root.
33
34## Commitment Algorithm
35
36The UTXO map is keyed by:
37
38```text
39txid:vout
40```
41
42Keys are sorted lexicographically. Each leaf is:
43
44```text
45SHA256("BTFUTXO1|" || outpoint || "|" || value_satoshis || "|" ||
46 script_pub_key_hex || "|" || created_height || "|" || created_txid)
47```
48
49The tree is then reduced pairwise. If a level has an odd number of hashes, the
50last hash is duplicated. Each parent is:
51
52```text
53SHA256("BTFNODE1|" || left_hash || right_hash)
54```
55
56The final hash is the UTXO-set root.
57
58## Inclusion Proofs
59
60After a root has been computed, `scripts/prove-utxo.py` can create a compact
61inclusion proof for one unspent output:
62
63```bash
64python3 scripts/prove-utxo.py TXID:VOUT --out utxo-proof.json --json
65python3 scripts/verify-utxo-proof.py utxo-proof.json
66```
67
68The verifier does not need local block files. It recomputes the target leaf,
69walks the sibling path, and compares the result with the expected root. See
70[UTXO inclusion proofs](utxo-proofs.md).
71
72## Deterministic Output
73
74`--json` and `--out` write canonical JSON with sorted keys, fixed separators,
75ASCII escaping, and one trailing newline. It contains no local path and no
76generation timestamp, so two auditors scanning the same chain prefix should
77produce byte-identical reports.
78
79```bash
80python3 scripts/verify-utxo-set.py --out utxo-report.json --json
81sha256sum utxo-report.json
82```
83
84## OpenTimestamps
85
86The canonical JSON can be timestamped in Bitcoin without adding any dependency
87to Bitflash consensus:
88
89```bash
90ots stamp utxo-report.json
91ots verify utxo-report.json.ots
92```
93
94## Historical Duplicates
95
96Bitflash inherited early Bitcoin behavior around duplicate transaction IDs.
97The verifier reports duplicate-output overwrites as warnings and uses
98last-write-wins semantics for the UTXO map, matching the legacy index model.
99Those warnings are part of the report so auditors can see the historical debt
100instead of relying on a polished summary.
101
102For synthetic chains or future clean consensus ranges, auditors can make
103duplicates a hard failure:
104
105```bash
106python3 scripts/verify-utxo-set.py --strict-duplicates
107```
108
109The default remains compatible with the current historical chain because it has
110known legacy duplicate-output overwrites.
111
112## Shared Parser
113
114The audit scripts share `scripts/bitflash_chain.py` for block parsing,
115main-chain selection, UTXO application, canonical JSON, and genesis
116reconstruction. That keeps `verify-utxo-set.py`, `verify-fair-launch.py`,
117`prove-utxo.py`, and the explorer on the same serialization rules.
118
119## Future Consensus Path
120
121This tool is the low-risk first step toward consensus commitments. A future
122hard-fork design can commit this root, or a Utreexo-style accumulator root, in
123coinbase or block headers after a fixed activation height. Until then this
124script provides reproducible supply and state commitments without changing what
125nodes accept.
diff --git a/docs/utxo-proofs.md b/docs/utxo-proofs.md
new file mode 100644
index 0000000..4947c3c
--- /dev/null
+++ b/docs/utxo-proofs.md
@@ -0,0 +1,61 @@
1# Bitflash UTXO inclusion proofs
2
3`scripts/prove-utxo.py` creates a deterministic inclusion proof for one
4currently unspent output. `scripts/verify-utxo-proof.py` verifies that proof
5against the UTXO root produced by `scripts/verify-utxo-set.py`.
6
7This is intentionally outside consensus. It turns the audit root into something
8useful for a specific coin without changing what nodes accept.
9
10## Create A Proof
11
12Use an unspent output in `txid:vout` form:
13
14```bash
15python3 scripts/prove-utxo.py TXID:VOUT --out utxo-proof.json --json
16```
17
18The proof contains:
19
20- the target UTXO;
21- its sorted leaf index;
22- the leaf hash;
23- the Merkle sibling path;
24- the UTXO root and chain tip used to create it.
25
26It does not contain the full UTXO set.
27
28## Verify A Proof
29
30Verify against the root embedded in the proof:
31
32```bash
33python3 scripts/verify-utxo-proof.py utxo-proof.json
34```
35
36Or verify against a root copied from another machine, dashboard, block explorer,
37or future node export:
38
39```bash
40python3 scripts/verify-utxo-proof.py utxo-proof.json \
41 --root ROOT_FROM_REPORT_OR_EXPORT
42```
43
44The verifier does not need local `blk*.dat` files. It only needs the proof and
45the expected root.
46
47## Scope
48
49This first proof format supports inclusion only. Absence and range proofs are
50left for a later design because they require a more careful commitment scheme
51and are easier to get subtly wrong.
52
53The proof uses the exact same leaf and node domains as the UTXO commitment:
54
55```text
56BTFUTXO1
57BTFNODE1
58```
59
60That keeps the verifier language-neutral and gives future implementations a
61small, stable target.
diff --git a/scripts/bitflash-mining-calculator.py b/scripts/bitflash-mining-calculator.py
new file mode 100644
index 0000000..1fcc1a9
--- /dev/null
+++ b/scripts/bitflash-mining-calculator.py
@@ -0,0 +1,206 @@
1#!/usr/bin/env python3
2"""Estimate Bitflash mining rewards and electricity cost.
3
4This is a local WhatToMine-style calculator for Bitflash. It does not fetch
5prices, does not talk to the node, and does not affect consensus.
6"""
7
8import argparse
9import json
10import math
11import sys
12
13
14HASHES_AT_DIFF_1 = 4096.0
15DEFAULT_BLOCK_TIME = 120.0
16DEFAULT_BLOCK_REWARD = 50.0
17
18
19UNITS = {
20 "h": 1.0,
21 "h/s": 1.0,
22 "hs": 1.0,
23 "kh": 1_000.0,
24 "kh/s": 1_000.0,
25 "khs": 1_000.0,
26 "mh": 1_000_000.0,
27 "mh/s": 1_000_000.0,
28 "mhs": 1_000_000.0,
29 "gh": 1_000_000_000.0,
30 "gh/s": 1_000_000_000.0,
31 "ghs": 1_000_000_000.0,
32}
33
34
35def parse_hashrate(value, unit):
36 try:
37 n = float(value)
38 except ValueError:
39 raise SystemExit("invalid hashrate: %s" % value)
40 mul = UNITS.get(unit.lower())
41 if mul is None:
42 raise SystemExit("unknown hashrate unit: %s" % unit)
43 if n < 0:
44 raise SystemExit("hashrate must not be negative")
45 return n * mul
46
47
48def fmt_hashrate(hps):
49 units = [("GH/s", 1_000_000_000.0), ("MH/s", 1_000_000.0), ("kH/s", 1_000.0), ("H/s", 1.0)]
50 for name, div in units:
51 if hps >= div:
52 return "%.4g %s" % (hps / div, name)
53 return "%.4g H/s" % hps
54
55
56def fmt_money(x):
57 return "%.8f" % x
58
59
60def estimate(args):
61 miner_hps = parse_hashrate(args.hashrate, args.hashrate_unit)
62 if args.network_hashrate is not None:
63 network_hps = parse_hashrate(args.network_hashrate, args.network_hashrate_unit)
64 difficulty = network_hps * args.block_time / HASHES_AT_DIFF_1
65 network_source = "network_hashrate"
66 else:
67 if args.difficulty is None:
68 raise SystemExit("provide either --network-hashrate or --difficulty")
69 difficulty = float(args.difficulty)
70 if difficulty <= 0:
71 raise SystemExit("difficulty must be positive")
72 network_hps = difficulty * HASHES_AT_DIFF_1 / args.block_time
73 network_source = "difficulty"
74
75 if network_hps <= 0:
76 raise SystemExit("network hashrate must be positive")
77
78 block_reward = float(args.block_reward)
79 block_time = float(args.block_time)
80 pool_fee_percent = float(args.pool_fee)
81 price = float(args.price)
82 watts = float(args.watts)
83 kwh_cost = float(args.kwh_cost)
84 if block_reward < 0 or block_time <= 0 or pool_fee_percent < 0 or price < 0 or watts < 0 or kwh_cost < 0:
85 raise SystemExit("numeric inputs must be non-negative; block time must be positive")
86
87 share = miner_hps / network_hps
88 blocks_per_day = 86400.0 / block_time
89 gross_btf_day = blocks_per_day * block_reward * share
90 net_btf_day = gross_btf_day * max(0.0, 1.0 - pool_fee_percent / 100.0)
91 revenue_day = net_btf_day * price
92 cost_day = watts / 1000.0 * 24.0 * kwh_cost
93 profit_day = revenue_day - cost_day
94 expected_seconds_per_block = math.inf if miner_hps <= 0 else difficulty * HASHES_AT_DIFF_1 / miner_hps
95
96 periods = {
97 "hour": 1.0 / 24.0,
98 "day": 1.0,
99 "week": 7.0,
100 "month_30d": 30.0,
101 }
102 rows = {}
103 for name, days in periods.items():
104 rows[name] = {
105 "btf": net_btf_day * days,
106 "revenue": revenue_day * days,
107 "electricity_cost": cost_day * days,
108 "profit": profit_day * days,
109 }
110
111 return {
112 "schema": "bitflash-mining-calculator-1",
113 "network_source": network_source,
114 "inputs": {
115 "miner_hashrate_hps": miner_hps,
116 "miner_hashrate": fmt_hashrate(miner_hps),
117 "network_hashrate_hps": network_hps,
118 "network_hashrate": fmt_hashrate(network_hps),
119 "difficulty": difficulty,
120 "block_time_seconds": block_time,
121 "block_reward_btf": block_reward,
122 "pool_fee_percent": pool_fee_percent,
123 "price_per_btf": price,
124 "power_watts": watts,
125 "electricity_per_kwh": kwh_cost,
126 },
127 "estimates": {
128 "miner_network_share": share,
129 "blocks_per_day": blocks_per_day,
130 "expected_time_to_solo_block_seconds": expected_seconds_per_block,
131 "expected_time_to_solo_block_hours": expected_seconds_per_block / 3600.0,
132 "gross_btf_per_day": gross_btf_day,
133 "net_btf_per_day": net_btf_day,
134 "periods": rows,
135 },
136 }
137
138
139def print_text(result):
140 i = result["inputs"]
141 e = result["estimates"]
142 print("Bitflash mining calculator")
143 print("schema: %s" % result["schema"])
144 print()
145 print("Network")
146 print(" source: %s" % result["network_source"])
147 print(" hashrate: %s" % i["network_hashrate"])
148 print(" difficulty: %.8f" % i["difficulty"])
149 print(" block time: %.0fs" % i["block_time_seconds"])
150 print(" reward: %s BTF" % fmt_money(i["block_reward_btf"]))
151 print()
152 print("Miner")
153 print(" hashrate: %s" % i["miner_hashrate"])
154 print(" share: %.8f%%" % (e["miner_network_share"] * 100.0))
155 if math.isinf(e["expected_time_to_solo_block_seconds"]):
156 print(" solo block: never")
157 else:
158 print(" solo block: %.2f hours expected average" % e["expected_time_to_solo_block_hours"])
159 print()
160 print("Economics")
161 print(" price: $%.8f / BTF" % i["price_per_btf"])
162 print(" power: %.2f W" % i["power_watts"])
163 print(" kWh cost: $%.4f" % i["electricity_per_kwh"])
164 print(" pool fee: %.2f%%" % i["pool_fee_percent"])
165 print()
166 print("Estimated rewards")
167 print(" %-9s %16s %12s %12s %12s" % ("period", "BTF", "revenue", "power", "profit"))
168 for name in ["hour", "day", "week", "month_30d"]:
169 row = e["periods"][name]
170 print(" %-9s %16s $%11.4f $%11.4f $%11.4f" % (
171 name,
172 fmt_money(row["btf"]),
173 row["revenue"],
174 row["electricity_cost"],
175 row["profit"],
176 ))
177 print()
178 print("Note: this is an expectation, not a promise. Solo mining variance can be huge.")
179
180
181def main(argv):
182 ap = argparse.ArgumentParser(description=__doc__)
183 ap.add_argument("--hashrate", required=True, help="miner hashrate number")
184 ap.add_argument("--hashrate-unit", default="kh/s", choices=sorted(UNITS), help="miner hashrate unit")
185 ap.add_argument("--network-hashrate", help="network hashrate number")
186 ap.add_argument("--network-hashrate-unit", default="kh/s", choices=sorted(UNITS), help="network hashrate unit")
187 ap.add_argument("--difficulty", type=float, help="Bitflash difficulty; diff 1 expects 4096 hashes")
188 ap.add_argument("--block-time", type=float, default=DEFAULT_BLOCK_TIME, help="target block time in seconds")
189 ap.add_argument("--block-reward", type=float, default=DEFAULT_BLOCK_REWARD, help="block reward in BTF")
190 ap.add_argument("--pool-fee", type=float, default=0.0, help="pool fee percent")
191 ap.add_argument("--price", type=float, default=0.0, help="price per BTF in USD or your chosen fiat unit")
192 ap.add_argument("--watts", type=float, default=0.0, help="miner or whole-system power in watts")
193 ap.add_argument("--kwh-cost", type=float, default=0.0, help="electricity cost per kWh")
194 ap.add_argument("--json", action="store_true", help="print JSON")
195 args = ap.parse_args(argv)
196
197 result = estimate(args)
198 if args.json:
199 print(json.dumps(result, sort_keys=True, indent=2))
200 else:
201 print_text(result)
202 return 0
203
204
205if __name__ == "__main__":
206 raise SystemExit(main(sys.argv[1:]))
diff --git a/scripts/bitflash_chain.py b/scripts/bitflash_chain.py
new file mode 100644
index 0000000..4a3231c
--- /dev/null
+++ b/scripts/bitflash_chain.py
@@ -0,0 +1,450 @@
1#!/usr/bin/env python3
2"""Shared Bitflash block/chain parser for deterministic audit tools."""
3
4import datetime as _dt
5import hashlib
6import json
7import mmap
8import os
9import struct
10from pathlib import Path
11
12
13MAGIC = bytes([0xBF, 0x20, 0x5C, 0xFD])
14MAX_BLOCK_PAYLOAD = 32 * 1024 * 1024
15UTXO_LEAF_DOMAIN = "BTFUTXO1"
16UTXO_NODE_DOMAIN_TAG = "BTFNODE1"
17UTXO_NODE_DOMAIN = (UTXO_NODE_DOMAIN_TAG + "|").encode("ascii")
18COIN = 100_000_000
19GENESIS_HASH = "5bd7cb255d814e48cebcdfb72da4dc87b34bd774227f8ceb546c5640f4bdc169"
20GENESIS_MERKLE = "1a45b4482532abb29b10e234d3f13132230525a339ecea91658ffa675a5b1325"
21GENESIS_TIME = 1_753_315_200
22GENESIS_BITS = 0x1F0FFFFF
23GENESIS_NONCE = 3141
24GENESIS_TEXT = "Bitflash 24/Jul/2026 Fair launch: one CPU one vote, no premine"
25
26
27class ParseError(Exception):
28 pass
29
30
31class Reader:
32 def __init__(self, data):
33 self.data = data
34 self.pos = 0
35
36 def take(self, n):
37 if self.pos + n > len(self.data):
38 raise ParseError("unexpected end of data")
39 out = self.data[self.pos:self.pos + n]
40 self.pos += n
41 return out
42
43 def u8(self):
44 return self.take(1)[0]
45
46 def u32(self):
47 return struct.unpack("<I", self.take(4))[0]
48
49 def i32(self):
50 return struct.unpack("<i", self.take(4))[0]
51
52 def i64(self):
53 return struct.unpack("<q", self.take(8))[0]
54
55
56def sha256(data):
57 return hashlib.sha256(data).digest()
58
59
60def sha256d(data):
61 return hashlib.sha256(hashlib.sha256(data).digest()).digest()
62
63
64def hash_hex(serialized):
65 return sha256d(serialized)[::-1].hex()
66
67
68def compact_size(r):
69 first = r.u8()
70 if first < 253:
71 return first
72 if first == 253:
73 return struct.unpack("<H", r.take(2))[0]
74 if first == 254:
75 return r.u32()
76 return struct.unpack("<Q", r.take(8))[0]
77
78
79def read_varbytes(r):
80 return r.take(compact_size(r))
81
82
83def ser_compact_size(n):
84 if n < 253:
85 return bytes([n])
86 if n <= 0xFFFF:
87 return b"\xfd" + struct.pack("<H", n)
88 if n <= 0xFFFFFFFF:
89 return b"\xfe" + struct.pack("<I", n)
90 return b"\xff" + struct.pack("<Q", n)
91
92
93def ser_varbytes(b):
94 return ser_compact_size(len(b)) + b
95
96
97def ser_tx(tx):
98 out = bytearray()
99 out += struct.pack("<i", tx["version"])
100 out += ser_compact_size(len(tx["vin"]))
101 for txin in tx["vin"]:
102 out += bytes.fromhex(txin["prevout_hash"])[::-1]
103 out += struct.pack("<I", txin["prevout_n"])
104 out += ser_varbytes(bytes.fromhex(txin["script_sig_hex"]))
105 out += struct.pack("<I", txin["sequence"])
106 out += ser_compact_size(len(tx["vout"]))
107 for txout in tx["vout"]:
108 out += struct.pack("<q", txout["value_satoshis"])
109 out += ser_varbytes(bytes.fromhex(txout["script_pub_key_hex"]))
110 out += struct.pack("<I", tx["lock_time"])
111 return bytes(out)
112
113
114def parse_tx(r):
115 start = r.pos
116 tx = {"version": r.i32(), "vin": [], "vout": [], "lock_time": 0}
117 for _ in range(compact_size(r)):
118 tx["vin"].append({
119 "prevout_hash": r.take(32)[::-1].hex(),
120 "prevout_n": r.u32(),
121 "script_sig_hex": read_varbytes(r).hex(),
122 "sequence": r.u32(),
123 })
124 for _ in range(compact_size(r)):
125 value = r.i64()
126 tx["vout"].append({
127 "value_satoshis": value,
128 "value_btf": format_money(value),
129 "script_pub_key_hex": read_varbytes(r).hex(),
130 })
131 tx["lock_time"] = r.u32()
132 raw = r.data[start:r.pos]
133 tx["txid"] = hash_hex(raw)
134 return tx
135
136
137def merkle_root(txids):
138 if not txids:
139 return "0" * 64
140 layer = [bytes.fromhex(x)[::-1] for x in txids]
141 while len(layer) > 1:
142 nxt = []
143 for i in range(0, len(layer), 2):
144 left = layer[i]
145 right = layer[i + 1] if i + 1 < len(layer) else left
146 nxt.append(sha256d(left + right))
147 layer = nxt
148 return layer[0][::-1].hex()
149
150
151def parse_block(payload, file_name="", offset=0):
152 r = Reader(payload)
153 header = r.take(80)
154 version, = struct.unpack("<i", header[0:4])
155 prev = header[4:36][::-1].hex()
156 merkle = header[36:68][::-1].hex()
157 ntime, bits, nonce = struct.unpack("<III", header[68:80])
158 txs = [parse_tx(r) for _ in range(compact_size(r))]
159 if r.pos != len(payload):
160 raise ParseError("trailing bytes inside block")
161 txids = [tx["txid"] for tx in txs]
162 return {
163 "hash": hash_hex(header),
164 "version": version,
165 "previous_hash": prev,
166 "merkle_root": merkle,
167 "computed_merkle_root": merkle_root(txids),
168 "time": ntime,
169 "time_utc": utc(ntime),
170 "bits": "0x%08x" % bits,
171 "nonce": nonce,
172 "tx_count": len(txs),
173 "transactions": txs,
174 "source_file": file_name,
175 "source_offset": offset,
176 "size": len(payload),
177 }
178
179
180def iter_block_files(datadir):
181 for path in sorted(Path(datadir).glob("blk*.dat")):
182 yield path
183
184
185def iter_blocks_from_file(path):
186 path = Path(path)
187 with path.open("rb") as f:
188 f.seek(0, os.SEEK_END)
189 file_size = f.tell()
190 if file_size == 0:
191 return
192 f.seek(0)
193 mm = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
194 try:
195 pos = 0
196 while True:
197 offset = mm.find(MAGIC, pos)
198 if offset < 0 or offset + 8 > file_size:
199 break
200 size = struct.unpack("<I", mm[offset + 4:offset + 8])[0]
201 payload_offset = offset + 8
202 end = payload_offset + size
203 if size == 0 or size > MAX_BLOCK_PAYLOAD or end > file_size:
204 pos = offset + 1
205 continue
206 payload = mm[payload_offset:end]
207 try:
208 yield parse_block(payload, path.name, payload_offset)
209 except ParseError:
210 pos = offset + 1
211 continue
212 pos = end
213 finally:
214 mm.close()
215
216
217def read_blocks_from_files(paths, max_blocks=0):
218 blocks = []
219 for path in paths:
220 for block in iter_blocks_from_file(path):
221 blocks.append(block)
222 if max_blocks and len(blocks) >= max_blocks:
223 return blocks
224 return blocks
225
226
227def read_blocks(datadir, max_blocks=0):
228 return read_blocks_from_files(iter_block_files(datadir), max_blocks)
229
230
231def select_main_chain(blocks):
232 by_hash = {}
233 for block in blocks:
234 by_hash.setdefault(block["hash"], block)
235 if GENESIS_HASH not in by_hash:
236 raise ParseError("genesis block not found in block files")
237
238 children = {}
239 for block in by_hash.values():
240 children.setdefault(block["previous_hash"], []).append(block)
241 for v in children.values():
242 v.sort(key=lambda b: (b["hash"], b["source_file"], b["source_offset"]))
243
244 best = []
245 stack = [(by_hash[GENESIS_HASH], [by_hash[GENESIS_HASH]])]
246 while stack:
247 block, path = stack.pop()
248 if len(path) > len(best) or (len(path) == len(best) and block["hash"] < best[-1]["hash"]):
249 best = path
250 for child in children.get(block["hash"], []):
251 stack.append((child, path + [child]))
252 return best
253
254
255def is_coinbase(tx):
256 return (
257 len(tx["vin"]) == 1
258 and tx["vin"][0]["prevout_hash"] == "0" * 64
259 and tx["vin"][0]["prevout_n"] == 0xFFFFFFFF
260 )
261
262
263def utxo_leaf_hash(outpoint, utxo):
264 payload = "|".join([
265 UTXO_LEAF_DOMAIN,
266 outpoint,
267 str(utxo["value_satoshis"]),
268 utxo["script_pub_key_hex"],
269 str(utxo["created_height"]),
270 utxo["created_txid"],
271 ]).encode("ascii")
272 return sha256(payload)
273
274
275def utxo_root(utxos):
276 leaves = [utxo_leaf_hash(k, utxos[k]) for k in sorted(utxos)]
277 if not leaves:
278 return sha256(b"BTFUTXO1|empty").hex()
279 layer = leaves
280 while len(layer) > 1:
281 nxt = []
282 for i in range(0, len(layer), 2):
283 left = layer[i]
284 right = layer[i + 1] if i + 1 < len(layer) else left
285 nxt.append(sha256(UTXO_NODE_DOMAIN + left + right))
286 layer = nxt
287 return layer[0].hex()
288
289
290def apply_blocks(blocks, strict_duplicates=False):
291 utxos = {}
292 spent = 0
293 missing_spends = []
294 duplicate_outputs = []
295 total_coinbase = 0
296 total_fees = 0
297 normal_txs = 0
298
299 for height, block in enumerate(blocks):
300 if block["computed_merkle_root"] != block["merkle_root"]:
301 raise ParseError("bad merkle root at height %d" % height)
302 for tx in block["transactions"]:
303 coinbase = is_coinbase(tx)
304 input_value = 0
305 output_value = sum(out["value_satoshis"] for out in tx["vout"])
306 if coinbase:
307 total_coinbase += output_value
308 else:
309 normal_txs += 1
310 for txin in tx["vin"]:
311 prev = "%s:%d" % (txin["prevout_hash"], txin["prevout_n"])
312 prev_utxo = utxos.pop(prev, None)
313 if prev_utxo is None:
314 missing_spends.append({
315 "height": height,
316 "txid": tx["txid"],
317 "missing_outpoint": prev,
318 })
319 else:
320 spent += prev_utxo["value_satoshis"]
321 input_value += prev_utxo["value_satoshis"]
322 if input_value >= output_value:
323 total_fees += input_value - output_value
324 for n, out in enumerate(tx["vout"]):
325 key = "%s:%d" % (tx["txid"], n)
326 if key in utxos:
327 if strict_duplicates:
328 raise ParseError("duplicate output at height %d: %s" % (height, key))
329 duplicate_outputs.append({"height": height, "outpoint": key})
330 utxos[key] = {
331 "txid": tx["txid"],
332 "vout": n,
333 "value_satoshis": out["value_satoshis"],
334 "value_btf": out["value_btf"],
335 "script_pub_key_hex": out["script_pub_key_hex"],
336 "created_height": height,
337 "created_block_hash": block["hash"],
338 "created_txid": tx["txid"],
339 "coinbase": coinbase,
340 }
341
342 total_unspent = sum(u["value_satoshis"] for u in utxos.values())
343 return {
344 "utxos": utxos,
345 "stats": {
346 "utxo_count": len(utxos),
347 "spent_outputs": spent,
348 "total_unspent_satoshis": total_unspent,
349 "total_unspent_btf": format_money(total_unspent),
350 "total_coinbase_satoshis": total_coinbase,
351 "total_coinbase_btf": format_money(total_coinbase),
352 "coinbase_minus_unspent_satoshis": total_coinbase - total_unspent,
353 "coinbase_minus_unspent_btf": format_money(total_coinbase - total_unspent),
354 "total_fees_satoshis": total_fees,
355 "total_fees_btf": format_money(total_fees),
356 "normal_transactions": normal_txs,
357 "missing_spends": missing_spends,
358 "duplicate_outputs": duplicate_outputs,
359 },
360 }
361
362
363def genesis_transaction():
364 script_sig = (
365 bytes.fromhex("04ffff001d")
366 + b"\x01\x04"
367 + bytes([len(GENESIS_TEXT)])
368 + GENESIS_TEXT.encode("ascii")
369 )
370 pubkey_num = bytes.fromhex(
371 "5f1df16b2b704c8a578d0bbaf74d385cde12c11ee50455f3c438ef4c3fbcf649"
372 "b6de611feae06279a60939e028a8d65c10b73071a6f16719274855feb0fd8a6704"
373 )
374 script_pub_key = bytes([65]) + pubkey_num[::-1] + b"\xac"
375 tx = {
376 "version": 1,
377 "vin": [{
378 "prevout_hash": "0" * 64,
379 "prevout_n": 0xFFFFFFFF,
380 "script_sig_hex": script_sig.hex(),
381 "sequence": 0xFFFFFFFF,
382 }],
383 "vout": [{
384 "value_satoshis": 50 * COIN,
385 "value_btf": format_money(50 * COIN),
386 "script_pub_key_hex": script_pub_key.hex(),
387 }],
388 "lock_time": 0,
389 }
390 tx["txid"] = hash_hex(ser_tx(tx))
391 return tx
392
393
394def reconstructed_genesis():
395 tx = genesis_transaction()
396 header = (
397 struct.pack("<i", 1)
398 + bytes(32)
399 + bytes.fromhex(tx["txid"])[::-1]
400 + struct.pack("<III", GENESIS_TIME, GENESIS_BITS, GENESIS_NONCE)
401 )
402 return {
403 "hash": hash_hex(header),
404 "version": 1,
405 "previous_hash": "0" * 64,
406 "merkle_root": tx["txid"],
407 "computed_merkle_root": tx["txid"],
408 "time": GENESIS_TIME,
409 "time_utc": utc(GENESIS_TIME),
410 "bits": "0x%08x" % GENESIS_BITS,
411 "nonce": GENESIS_NONCE,
412 "tx_count": 1,
413 "transactions": [tx],
414 "source_file": "reconstructed-from-source-constants",
415 "source_offset": 0,
416 "size": 80 + len(ser_compact_size(1)) + len(ser_tx(tx)),
417 }
418
419
420def ser_block(block):
421 header = (
422 struct.pack("<i", block["version"])
423 + bytes.fromhex(block["previous_hash"])[::-1]
424 + bytes.fromhex(block["merkle_root"])[::-1]
425 + struct.pack("<III", block["time"], int(block["bits"], 16), block["nonce"])
426 )
427 txs = block["transactions"]
428 return header + ser_compact_size(len(txs)) + b"".join(ser_tx(tx) for tx in txs)
429
430
431def format_money(value):
432 sign = "-" if value < 0 else ""
433 value = abs(value)
434 return "%s%d.%08d" % (sign, value // COIN, value % COIN)
435
436
437def utc(ts):
438 return _dt.datetime.fromtimestamp(ts, _dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
439
440
441def default_datadir():
442 if os.name == "nt" and os.environ.get("APPDATA"):
443 return os.path.join(os.environ["APPDATA"], "Bitflash")
444 if os.environ.get("HOME"):
445 return os.path.join(os.environ["HOME"], ".bitflash")
446 return "."
447
448
449def canonical_bytes(report):
450 return (json.dumps(report, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + "\n").encode("ascii")
diff --git a/scripts/build-explorer.py b/scripts/build-explorer.py
new file mode 100755
index 0000000..dbb8e17
--- /dev/null
+++ b/scripts/build-explorer.py
@@ -0,0 +1,376 @@
1#!/usr/bin/env python3
2"""Build a static Bitflash block explorer from local blk*.dat files."""
3
4import argparse
5import hashlib
6import json
7import shutil
8import sys
9from pathlib import Path
10
11import bitflash_chain as chain
12
13
14ADDRESS_VERSION = 25
15B58 = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"
16
17
18def b58check(payload):
19 v = bytes([ADDRESS_VERSION]) + payload
20 v += chain.sha256d(v)[:4]
21 n = int.from_bytes(v, "big")
22 s = ""
23 while n > 0:
24 n, r = divmod(n, 58)
25 s = B58[r] + s
26 for byte in v:
27 if byte == 0:
28 s = "1" + s
29 else:
30 break
31 return s
32
33
34def hash160(b):
35 return hashlib.new("ripemd160", hashlib.sha256(b).digest()).digest()
36
37
38def spk_info(script_hex):
39 try:
40 spk = bytes.fromhex(script_hex)
41 except ValueError:
42 return {"type": "malformed", "hex": script_hex[:80]}
43 if (
44 len(spk) == 25 and spk[0] == 0x76 and spk[1] == 0xA9
45 and spk[2] == 0x14 and spk[23] == 0x88 and spk[24] == 0xAC
46 ):
47 return {"type": "p2pkh", "address": b58check(spk[3:23])}
48 if len(spk) >= 35 and spk[-1] == 0xAC:
49 if spk[0] == 0x41 and len(spk) == 67:
50 pk = spk[1:66]
51 return {"type": "p2pk", "pubkey": pk.hex(), "address": b58check(hash160(pk))}
52 if spk[0] == 0x21 and len(spk) == 35:
53 pk = spk[1:34]
54 return {"type": "p2pk", "pubkey": pk.hex(), "address": b58check(hash160(pk))}
55 return {"type": "other", "hex": script_hex[:80]}
56
57
58def explorer_tx(tx):
59 vin = []
60 for txin in tx["vin"]:
61 coinbase = (
62 txin["prevout_hash"] == "0" * 64
63 and txin["prevout_n"] == 0xFFFFFFFF
64 )
65 vin.append({
66 "coinbase": coinbase,
67 "prev": None if coinbase else txin["prevout_hash"],
68 "vout": None if coinbase else txin["prevout_n"],
69 })
70 vout = []
71 for out in tx["vout"]:
72 vout.append(dict({"value": out["value_satoshis"]}, **spk_info(out["script_pub_key_hex"])))
73 return {"txid": tx["txid"], "vin": vin, "vout": vout, "size": len(chain.ser_tx(tx))}
74
75
76def parse_args(argv):
77 ap = argparse.ArgumentParser(description=__doc__)
78 ap.add_argument("--datadir", help="Bitflash data directory containing blk*.dat")
79 ap.add_argument("--max-blocks", type=int, default=0,
80 help="number of blocks to scan; 0 means all block files")
81 ap.add_argument("paths", nargs="+",
82 help="with --datadir: <out-dir>; otherwise: <blk*.dat>... <out-dir>")
83 args = ap.parse_args(argv)
84 if args.datadir:
85 if len(args.paths) != 1:
86 raise SystemExit("usage with --datadir: build-explorer.py --datadir DATADIR <out-dir>")
87 args.block_files = None
88 args.out_dir = args.paths[0]
89 else:
90 if len(args.paths) < 2:
91 raise SystemExit("usage: build-explorer.py <blk*.dat>... <out-dir>")
92 args.block_files = args.paths[:-1]
93 args.out_dir = args.paths[-1]
94 return args
95
96
97def load_blocks(args):
98 if args.datadir:
99 return chain.read_blocks(args.datadir, args.max_blocks)
100 return chain.read_blocks_from_files(args.block_files, args.max_blocks)
101
102
103def build_explorer(raw_blocks, out_dir):
104 try:
105 blocks = chain.select_main_chain(raw_blocks)
106 except chain.ParseError as e:
107 raise SystemExit(str(e))
108 out = Path(out_dir)
109 (out / "block").mkdir(parents=True, exist_ok=True)
110
111 summaries = []
112 for height, block in enumerate(blocks):
113 txs = [explorer_tx(tx) for tx in block["transactions"]]
114 detail = {
115 "height": height,
116 "hash": block["hash"],
117 "prev": block["previous_hash"],
118 "merkleRoot": block["merkle_root"],
119 "version": block["version"],
120 "time": block["time"],
121 "bits": int(block["bits"], 16),
122 "nonce": block["nonce"],
123 "size": block["size"],
124 "txs": txs,
125 }
126 cb = txs[0]["vout"][0] if txs and txs[0]["vout"] else {}
127 summaries.append({
128 "height": height,
129 "hash": block["hash"],
130 "time": block["time"],
131 "nTx": len(txs),
132 "size": block["size"],
133 "cbValue": cb.get("value", 0),
134 "cbAddr": cb.get("address", ""),
135 })
136 (out / "block" / ("%d.json" % height)).write_text(
137 json.dumps(detail, sort_keys=True, separators=(",", ":")) + "\n",
138 encoding="ascii",
139 )
140
141 summaries.sort(key=lambda x: -x["height"])
142 (out / "blocks.json").write_text(
143 json.dumps({
144 "network": "bitflash",
145 "tipHeight": len(blocks) - 1,
146 "count": len(blocks),
147 "blocks": summaries,
148 }, sort_keys=True, separators=(",", ":")) + "\n",
149 encoding="ascii",
150 )
151 (out / "index.html").write_text(INDEX_HTML, encoding="utf-8")
152 (out / "style.css").write_text(STYLE_CSS, encoding="utf-8")
153 (out / "explorer.js").write_text(EXPLORER_JS, encoding="utf-8")
154 logo = Path(__file__).resolve().parents[1] / "docs" / "logo.png"
155 if logo.exists():
156 shutil.copyfile(logo, out / "logo.png")
157 return len(blocks), len(blocks) - 1
158
159
160def main(argv):
161 args = parse_args(argv)
162 raw_blocks = load_blocks(args)
163 if not raw_blocks:
164 raise SystemExit("no Bitflash blocks parsed")
165 count, tip = build_explorer(raw_blocks, args.out_dir)
166 print("explorer built: %d main-chain blocks, tip %d, into %s" % (count, tip, args.out_dir))
167 return 0
168
169
170INDEX_HTML = r"""<!doctype html>
171<html lang="en">
172<head>
173<meta charset="utf-8">
174<meta name="viewport" content="width=device-width, initial-scale=1">
175<title>Bitflash Block Explorer</title>
176<meta name="description" content="Read-only Bitflash block explorer generated from local blk*.dat files.">
177<link rel="icon" href="logo.png" type="image/png">
178<link rel="stylesheet" href="style.css">
179<script src="explorer.js" defer></script>
180</head>
181<body>
182<header class="topbar">
183 <a class="brand" href="https://bitflash.network/"><img src="logo.png" width="22" height="22" alt="">Bitflash</a>
184 <nav aria-label="Bitflash sites">
185 <a href="https://bitflash.network/">home</a>
186 <a href="https://git.bitflash.network/bitflash/bitflash">code</a>
187 <a href="https://docs.bitflash.network/">docs</a>
188 <a href="https://releases.bitflash.network/">downloads</a>
189 <a href="https://explorer.bitflash.network/" aria-current="page">explorer</a>
190 <a href="https://status.bitflash.network/">status</a>
191 </nav>
192 <span class="meta" id="tip"></span>
193</header>
194<div class="wrap">
195 <h1>Block explorer <button class="link" id="home" hidden>back to blocks</button></h1>
196 <p class="note">Read-only view of the main chain: blocks, their transactions, and the coinbase payout. There is no mempool and no per-address history here.</p>
197 <div id="listview">
198 <input id="q" placeholder="Search by height or block hash">
199 <div class="scroll"><table>
200 <thead><tr><th>Height</th><th>Time (UTC)</th><th>Txs</th><th>Size</th><th>Coinbase</th><th>To</th></tr></thead>
201 <tbody id="rows"></tbody>
202 </table></div>
203 <p class="mut" id="more"></p>
204 </div>
205 <div id="detail" hidden></div>
206</div>
207</body>
208</html>
209"""
210
211
212STYLE_CSS = r""":root{
213 color-scheme: light dark;
214 --bg:#fff; --bg-2:#f6f6f6; --fg:#111; --dim:#6b6b6b; --line:#d7d7d7; --line-hard:#111;
215 --accent:#0b7a34; --accent-2:#12a047; --bar:3rem;
216 --mono: ui-monospace,"Cascadia Mono","Segoe UI Mono",Consolas,"DejaVu Sans Mono","Liberation Mono","Courier New",monospace;
217}
218@media (prefers-color-scheme: dark){:root{
219 --bg:#000; --bg-2:#0b0b0b; --fg:#dcdcdc; --dim:#8a8a8a; --line:#242424; --line-hard:#4a4a4a;
220 --accent:#2ee06a; --accent-2:#6bff9f;
221}}
222*{box-sizing:border-box;}
223html,body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.55 var(--mono);-webkit-text-size-adjust:100%;}
224a{color:var(--accent);}
225.topbar{display:flex;align-items:center;gap:1rem;flex-wrap:wrap;min-height:var(--bar);
226 padding:.5rem clamp(.9rem,2.5vw,3rem);border-bottom:1px solid var(--line);position:sticky;top:0;background:var(--bg);z-index:5;}
227.brand{display:flex;align-items:center;gap:.6rem;text-decoration:none;color:var(--fg);font-weight:bold;}
228.brand img{width:22px;height:22px;display:block;}
229.topbar nav{display:flex;flex-wrap:wrap;gap:.15rem 1.4rem;font-size:.86rem;}
230.topbar nav a{text-decoration:none;color:var(--dim);}
231.topbar nav a::before{content:"/";color:var(--line-hard);margin-right:.3rem;}
232.topbar nav a:hover{color:var(--fg);text-decoration:underline;}
233.topbar nav a[aria-current="page"]{color:var(--accent);font-weight:bold;}
234.topbar nav a[aria-current="page"]::before{color:var(--accent);}
235.topbar .meta{margin-left:auto;font-size:.76rem;color:var(--dim);letter-spacing:.06em;}
236.wrap{padding:1.4rem clamp(.9rem,2.5vw,3rem) 4rem;}
237h1{font-size:1rem;margin:0 0 .3rem;letter-spacing:.02em;}
238h2{font-size:.95rem;margin:1.2rem 0 .4rem;}
239.mut{color:var(--dim);}
240.note{color:var(--dim);font-size:.82rem;max-width:60rem;margin:.2rem 0 1rem;}
241input{width:100%;max-width:28rem;padding:.5rem .7rem;border:1px solid var(--line);background:var(--bg-2);color:var(--fg);font:inherit;font-size:.86rem;}
242input:focus{outline:none;border-color:var(--accent);}
243.scroll{overflow-x:auto;}
244table{width:100%;border-collapse:collapse;margin-top:.8rem;font-size:.86rem;}
245th,td{text-align:left;padding:.5rem .6rem;border-bottom:1px solid var(--line);font-variant-numeric:tabular-nums;}
246th{font-size:.72rem;text-transform:uppercase;letter-spacing:.1em;color:var(--dim);font-weight:normal;}
247tr.blk{cursor:pointer;}
248tr.blk:hover td{background:var(--bg-2);}
249code{font-size:.8rem;word-break:break-all;color:var(--fg);}
250.card{border:1px solid var(--line);background:var(--bg-2);padding:.9rem 1rem;margin-top:.8rem;}
251.kv{display:grid;grid-template-columns:11rem 1fr;gap:.25rem .8rem;font-size:.82rem;}
252.kv div:nth-child(odd){color:var(--dim);}
253.tx{border-top:1px solid var(--line);padding:.7rem 0;}
254.io{display:grid;grid-template-columns:1fr 1fr;gap:1rem;}
255@media (max-width:640px){.io{grid-template-columns:1fr;}.kv{grid-template-columns:7rem 1fr;}.topbar .meta{margin-left:0;width:100%;}}
256.pill{display:inline-block;font-size:.68rem;padding:.05rem .5rem;border:1px solid var(--line);color:var(--dim);text-transform:uppercase;letter-spacing:.05em;}
257button.link{background:none;border:none;color:var(--accent);cursor:pointer;font:inherit;padding:0;text-decoration:underline;}
258"""
259
260
261EXPLORER_JS = r"""
262var SUM = [], SHOWN = 200;
263function btf(sat){ return (sat/1e8).toFixed(8).replace(/0+$/,'').replace(/\.$/,'')+" BTF"; }
264function ts(t){ return new Date(t*1000).toISOString().replace('T',' ').replace('.000Z',' UTC'); }
265function short(h){ return h ? h.slice(0,10)+"..."+h.slice(-6) : "-"; }
266function E(tag, cls, text){
267 var e = document.createElement(tag);
268 if(cls) e.className = cls;
269 if(text !== undefined) e.textContent = text;
270 return e;
271}
272function code(text){ return E('code', '', text); }
273function clear(e){ while(e.firstChild) e.removeChild(e.firstChild); }
274function append(parent){
275 for(var i=1;i<arguments.length;i++) parent.appendChild(arguments[i]);
276 return parent;
277}
278function row(cells, cls){
279 var tr = E('tr', cls || '');
280 cells.forEach(function(c){ append(tr, E('td', '', c)); });
281 return tr;
282}
283function renderList(filter){
284 var q=(filter||"").trim().toLowerCase(), rows=document.getElementById('rows');
285 clear(rows);
286 var list = q ? SUM.filter(function(b){ return String(b.height)===q || b.hash.indexOf(q)===0; }) : SUM.slice(0,SHOWN);
287 if(!list.length){
288 var empty = E('tr');
289 var td = E('td', 'mut', 'no match');
290 td.colSpan = 6;
291 append(empty, td); append(rows, empty);
292 }
293 list.forEach(function(b){
294 var tr = row([String(b.height), ts(b.time), String(b.nTx), b.size+' B', btf(b.cbValue), ''], 'blk');
295 tr.dataset.h = b.height;
296 clear(tr.children[5]);
297 append(tr.children[5], code(b.cbAddr || '-'));
298 tr.onclick = function(){ openBlock(tr.dataset.h); };
299 append(rows, tr);
300 });
301 document.getElementById('more').textContent = (!q && SUM.length>SHOWN) ? ("showing latest "+SHOWN+" of "+SUM.length+" blocks; search to find any") : "";
302}
303function kv(parent, key, value, isCode){
304 append(parent, E('div', '', key));
305 append(parent, isCode ? append(E('div'), code(value)) : E('div', '', value));
306}
307function ioSide(title, items, render){
308 var side = E('div');
309 append(side, E('div', 'mut', title));
310 items.forEach(function(item){ append(side, render(item)); });
311 return side;
312}
313function openBlock(h){
314 fetch('block/'+h+'.json').then(function(r){return r.json();}).then(function(b){
315 var detail = document.getElementById('detail');
316 clear(detail);
317 append(detail, E('h2', '', 'Block '+b.height));
318 var card = E('div', 'card'), grid = E('div', 'kv');
319 kv(grid, 'Height', String(b.height));
320 kv(grid, 'Hash', b.hash, true);
321 var prev = E('div');
322 if(b.height > 0){
323 var btn = E('button', 'link');
324 append(btn, code(short(b.prev)));
325 btn.onclick = function(){ openBlock(b.height - 1); };
326 append(prev, btn);
327 } else {
328 append(prev, code(short(b.prev)));
329 }
330 append(grid, E('div', '', 'Previous'), prev);
331 kv(grid, 'Merkle root', b.merkleRoot, true);
332 kv(grid, 'Time (UTC)', ts(b.time));
333 kv(grid, 'Bits / Nonce', b.bits.toString(16)+' / '+b.nonce);
334 kv(grid, 'Size / Txs', b.size+' B / '+b.txs.length);
335 append(card, grid); append(detail, card);
336 b.txs.forEach(function(t, i){
337 var tx = E('div', 'tx');
338 var head = E('div');
339 append(head, code(t.txid));
340 if(i === 0) append(head, E('span', 'pill', 'coinbase'));
341 var io = E('div', 'io');
342 append(io, ioSide('Inputs', t.vin, function(v){
343 var d = E('div');
344 if(v.coinbase) append(d, E('span', 'pill', 'coinbase'));
345 else append(d, code(short(v.prev)+':'+v.vout));
346 return d;
347 }));
348 append(io, ioSide('Outputs', t.vout, function(o){
349 var d = E('div');
350 var to = o.address ? o.address : (o.pubkey ? 'pubkey '+o.pubkey.slice(0,16)+'...' : o.type);
351 append(d, document.createTextNode(btf(o.value)+' -> '), code(to));
352 return d;
353 }));
354 append(tx, head, io); append(detail, tx);
355 });
356 document.getElementById('listview').hidden = true;
357 detail.hidden = false;
358 document.getElementById('home').hidden = false;
359 window.scrollTo(0,0);
360 });
361}
362document.getElementById('home').onclick=function(){
363 document.getElementById('detail').hidden = true;
364 document.getElementById('listview').hidden = false;
365 this.hidden = true;
366};
367document.getElementById('q').oninput=function(){ renderList(this.value); };
368fetch('blocks.json').then(function(r){return r.json();}).then(function(d){
369 SUM=d.blocks; document.getElementById('tip').textContent='tip height '+d.tipHeight+' - '+d.count+' blocks';
370 renderList('');
371});
372"""
373
374
375if __name__ == "__main__":
376 raise SystemExit(main(sys.argv[1:]))
diff --git a/scripts/build-pool-index.py b/scripts/build-pool-index.py
new file mode 100644
index 0000000..d3eb386
--- /dev/null
+++ b/scripts/build-pool-index.py
@@ -0,0 +1,174 @@
1#!/usr/bin/env python3
2#
3# Build a static public pool directory from one or more Bitflash pool_status.json
4# files. Inputs may be local paths or http(s) URLs.
5
6import argparse
7import html
8import json
9import os
10import time
11import urllib.parse
12import urllib.request
13
14
15def load_json(source):
16 if source.startswith("http://") or source.startswith("https://"):
17 with urllib.request.urlopen(source, timeout=10) as r:
18 return json.loads(r.read().decode("utf-8"))
19 with open(source, "r", encoding="utf-8-sig") as f:
20 return json.load(f)
21
22
23def normalize_pool(source, obj, now):
24 operator = obj.get("operator", {})
25 pool = obj.get("pool", {})
26 node = obj.get("node", {})
27 rounds_file = operator.get("roundsFile") or "pool_rounds.json"
28 rounds_url = ""
29 if source.startswith("http://") or source.startswith("https://"):
30 rounds_url = urllib.parse.urljoin(source, rounds_file)
31 updated = int(obj.get("updatedAt") or 0)
32 age = max(0, now - updated) if updated else None
33 online = bool(node.get("poolRunning")) and age is not None and age <= 120
34 return {
35 "source": source,
36 "name": operator.get("name") or "Bitflash Pool",
37 "btfAddress": operator.get("btfAddress") or "",
38 "feePercent": float(operator.get("feePercent") or 0.0),
39 "dashboardUrl": operator.get("dashboardUrl") or "",
40 "height": int(node.get("height") or 0),
41 "online": online,
42 "ageSeconds": age,
43 "authorizedMiners": int(pool.get("authorizedMiners") or 0),
44 "blocksFoundSession": int(pool.get("blocksFoundSession") or 0),
45 "roundShares": int(pool.get("roundShares") or 0),
46 "hashRate": float(pool.get("hashRate") or 0.0),
47 "roundsUrl": rounds_url,
48 "recentRounds": obj.get("recentRounds", []),
49 }
50
51
52def rank_key(pool):
53 return (
54 1 if pool["online"] else 0,
55 pool["hashRate"],
56 pool["authorizedMiners"],
57 -pool["feePercent"],
58 pool["name"].lower(),
59 )
60
61
62def render_html(pools, generated_at):
63 rows = []
64 for p in pools:
65 dash = p["dashboardUrl"]
66 rounds = p.get("roundsUrl", "")
67 # Only linkify http(s). The dashboard URL comes from an operator's
68 # self-published status file, so a "javascript:" or "data:" scheme
69 # would become a clickable script-injection on this public page.
70 dash_lower = dash.lower()
71 if dash_lower.startswith("http://") or dash_lower.startswith("https://"):
72 dash_html = '<a href="{0}" rel="noopener noreferrer">dashboard</a>'.format(
73 html.escape(dash, quote=True))
74 else:
75 dash_html = "-"
76 rounds_lower = rounds.lower()
77 if rounds_lower.startswith("http://") or rounds_lower.startswith("https://"):
78 rounds_html = '<a href="{0}" rel="noopener noreferrer">proof</a>'.format(
79 html.escape(rounds, quote=True))
80 else:
81 rounds_html = "-"
82 rows.append(
83 "<tr>"
84 "<td>{status}</td>"
85 "<td><strong>{name}</strong><br><code>{addr}</code></td>"
86 "<td>{fee:.2f}%</td>"
87 "<td>{miners}</td>"
88 "<td>{hashrate:.2f}</td>"
89 "<td>{blocks}</td>"
90 "<td>{height}</td>"
91 "<td>{age}</td>"
92 "<td>{dash}</td>"
93 "<td>{rounds}</td>"
94 "</tr>".format(
95 status="online" if p["online"] else "stale",
96 name=html.escape(p["name"]),
97 addr=html.escape(p["btfAddress"]),
98 fee=p["feePercent"],
99 miners=p["authorizedMiners"],
100 hashrate=p["hashRate"],
101 blocks=p["blocksFoundSession"],
102 height=p["height"],
103 age="{}s".format(p["ageSeconds"]) if p["ageSeconds"] is not None else "-",
104 dash=dash_html,
105 rounds=rounds_html,
106 )
107 )
108
109 return """<!doctype html>
110<html lang="en">
111<head>
112 <meta charset="utf-8">
113 <meta name="viewport" content="width=device-width, initial-scale=1">
114 <title>Bitflash Pools</title>
115 <style>
116 body {{ font: 15px/1.45 system-ui, sans-serif; margin: 32px; color: #111; }}
117 table {{ border-collapse: collapse; width: 100%; }}
118 th, td {{ border-bottom: 1px solid #ddd; padding: 10px; text-align: left; vertical-align: top; }}
119 th {{ font-size: 12px; text-transform: uppercase; letter-spacing: .08em; }}
120 code {{ font-size: 12px; word-break: break-all; }}
121 .note {{ color: #555; max-width: 780px; }}
122 </style>
123</head>
124<body>
125 <h1>Bitflash Pools</h1>
126 <p class="note">This directory is informational. Bitflash nodes discover pools through Nostr and .btf; this page only helps humans compare public operators.</p>
127 <table>
128 <thead>
129 <tr><th>Status</th><th>Pool</th><th>Fee</th><th>Miners</th><th>H/s</th><th>Blocks</th><th>Height</th><th>Age</th><th>Link</th><th>Proof</th></tr>
130 </thead>
131 <tbody>
132 {rows}
133 </tbody>
134 </table>
135 <p class="note">Generated at {generated_at}. JSON: <a href="pools.json">pools.json</a>.</p>
136</body>
137</html>
138""".format(rows="\n ".join(rows), generated_at=generated_at)
139
140
141def main():
142 ap = argparse.ArgumentParser()
143 ap.add_argument("--out-dir", default="pool-site")
144 ap.add_argument("sources", nargs="+", help="pool_status.json paths or URLs")
145 args = ap.parse_args()
146
147 now = int(time.time())
148 pools = []
149 errors = []
150 for source in args.sources:
151 try:
152 pools.append(normalize_pool(source, load_json(source), now))
153 except Exception as e:
154 errors.append({"source": source, "error": str(e)})
155
156 pools.sort(key=rank_key, reverse=True)
157 os.makedirs(args.out_dir, exist_ok=True)
158
159 generated_at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(now))
160 with open(os.path.join(args.out_dir, "pools.json"), "w", encoding="utf-8") as f:
161 json.dump({
162 "schema": "bitflash-pool-directory-1",
163 "generatedAt": generated_at,
164 "pools": pools,
165 "errors": errors,
166 }, f, indent=2, sort_keys=True)
167 f.write("\n")
168
169 with open(os.path.join(args.out_dir, "index.html"), "w", encoding="utf-8") as f:
170 f.write(render_html(pools, generated_at))
171
172
173if __name__ == "__main__":
174 main()
diff --git a/scripts/check-money-log-lines.sh b/scripts/check-money-log-lines.sh
new file mode 100755
index 0000000..c9f348b
--- /dev/null
+++ b/scripts/check-money-log-lines.sh
@@ -0,0 +1,38 @@
1#!/usr/bin/env bash
2# Money-moving payout lines must print every time they fire: they are the
3# audit trail for obligations created, moved and destroyed, and LogPrint's
4# burst collapse can stand one line in for any number of events inside one
5# 3-second window (issue #3: 100 voids totalling 4,950 BTF read as one
6# 49.49 BTF line). Every line carrying one of the phrases below must use
7# LogPrintAlways, which keeps the category gate but skips the collapse.
8#
9# Adding a payout line that reports money moving? Use LogPrintAlways and
10# add its phrase here. A phrase found in a plain LogPrint call fails this
11# check; anything else passes, so a non-money line is never held up by it.
12set -euo pipefail
13cd "$(dirname "$0")/.."
14
15phrases='was orphaned -- voiding|orphaned while queued|\[payout\] paid %s to %s|FAILED to pay|writing off|SKIPPED bad address'
16
17# Call sites span several lines; flatten each call to a single line first so
18# the phrase and the macro it belongs to are matched together. Only a line
19# whose first token is the macro starts a call, so comments mentioning
20# LogPrint are never mistaken for one.
21bad=$(awk '
22 /^[ \t]*LogPrint(Always)?[(]/ {
23 if (collecting) print buf
24 buf = $0; collecting = 1; next
25 }
26 collecting {
27 buf = buf $0
28 if (/;[ \t]*$/) { print buf; collecting = 0 }
29 }
30 END { if (collecting) print buf }
31' src/rpc.cpp | grep -E '^[[:space:]]*LogPrint[(]' | grep -E "$phrases" || true)
32
33if [ -n "$bad" ]; then
34 echo "error: money-moving payout line(s) on collapsing LogPrint:" >&2
35 echo "$bad" >&2
36 exit 1
37fi
38echo "ok: every money-moving payout line in src/rpc.cpp uses LogPrintAlways"
diff --git a/scripts/make-release-checksums.sh b/scripts/make-release-checksums.sh
index 5b29e2d..9c4f45a 100644..100755
--- a/scripts/make-release-checksums.sh
+++ b/scripts/make-release-checksums.sh
@@ -10,6 +10,7 @@ Usage: scripts/make-release-checksums.sh [--sign] [--local-user KEYID]
10 10
11Creates SHA256SUMS for release assets in the current directory: 11Creates SHA256SUMS for release assets in the current directory:
12 Bitflash-*-windows.zip 12 Bitflash-*-windows.zip
13 Bitflash-*-windows-with-tor.zip
13 Bitflash-*-x86_64.AppImage 14 Bitflash-*-x86_64.AppImage
14 bitflash-node-*-x86_64 15 bitflash-node-*-x86_64
15 16
@@ -44,6 +45,7 @@ fi
44assets=() 45assets=()
45for pattern in \ 46for pattern in \
46 "Bitflash-*-windows.zip" \ 47 "Bitflash-*-windows.zip" \
48 "Bitflash-*-windows-with-tor.zip" \
47 "Bitflash-*-x86_64.AppImage" \ 49 "Bitflash-*-x86_64.AppImage" \
48 "bitflash-node-*-x86_64" 50 "bitflash-node-*-x86_64"
49do 51do
diff --git a/scripts/package-windows-tor.sh b/scripts/package-windows-tor.sh
new file mode 100755
index 0000000..c8cca3b
--- /dev/null
+++ b/scripts/package-windows-tor.sh
@@ -0,0 +1,187 @@
1#!/usr/bin/env bash
2set -euo pipefail
3
4ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
5VERSION="${VERSION:-$(sed -n 's/^VERSION[[:space:]]*:= *//p' "$ROOT/Makefile" | head -1)}"
6
7TOR_VERSION="${TOR_VERSION:-15.0.19}"
8TOR_ARCHIVE_NAME="tor-expert-bundle-windows-x86_64-${TOR_VERSION}.tar.gz"
9TOR_URL="${TOR_URL:-https://archive.torproject.org/tor-package-archive/torbrowser/${TOR_VERSION}/${TOR_ARCHIVE_NAME}}"
10TOR_SHA256="${TOR_SHA256:-6AC067402C7B4A3DC37887ED3754B3914B67FDC220C966190683E9CCF91ABF0F}"
11TOR_ASC_URL="${TOR_ASC_URL:-${TOR_URL}.asc}"
12TOR_SIGNING_KEY_FINGERPRINT="${TOR_SIGNING_KEY_FINGERPRINT:-EF6E286DDA85EA2A4BA7DE684E2C6E8793298290}"
13TOR_VERIFY_GPG="${TOR_VERIFY_GPG:-auto}"
14
15BUNDLE_DIR="${BUNDLE_DIR:-$ROOT/Bitflash-${VERSION}-windows}"
16OUT_ZIP="${OUT_ZIP:-$ROOT/Bitflash-${VERSION}-windows-with-tor.zip}"
17CACHE_DIR="${CACHE_DIR:-$ROOT/.cache}"
18ARCHIVE="${TOR_ARCHIVE:-$CACHE_DIR/$TOR_ARCHIVE_NAME}"
19ASC_FILE="${TOR_ASC:-$ARCHIVE.asc}"
20
21usage() {
22 cat <<EOF
23Usage: scripts/package-windows-tor.sh
24
25Creates Bitflash-${VERSION}-windows-with-tor.zip from an existing
26Bitflash-${VERSION}-windows/ release directory.
27
28Environment overrides:
29 VERSION Bitflash release version, default read from Makefile
30 TOR_VERSION Tor Expert Bundle version, default ${TOR_VERSION}
31 TOR_URL Download URL, default official Tor archive URL
32 TOR_ASC_URL Signature URL, default TOR_URL.asc
33 TOR_SHA256 Expected archive SHA256
34 TOR_ARCHIVE Use an existing archive instead of downloading
35 TOR_ASC Use an existing detached signature
36 TOR_VERIFY_GPG auto, required, or off; default auto
37 BUNDLE_DIR Existing unpacked Windows release directory
38 OUT_ZIP Output zip path
39EOF
40}
41
42if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
43 usage
44 exit 0
45fi
46
47need() {
48 command -v "$1" >/dev/null 2>&1 || {
49 echo "missing required command: $1" >&2
50 exit 1
51 }
52}
53
54need sha256sum
55need tar
56need zip
57
58download() {
59 local url="$1"
60 local out="$2"
61 if command -v curl >/dev/null 2>&1; then
62 curl -L --fail --output "$out" "$url"
63 elif command -v wget >/dev/null 2>&1; then
64 wget -O "$out" "$url"
65 else
66 echo "missing required command: curl or wget" >&2
67 exit 1
68 fi
69}
70
71if [ ! -d "$BUNDLE_DIR" ]; then
72 echo "missing Windows release directory: $BUNDLE_DIR" >&2
73 echo "run 'make windows' first, or set BUNDLE_DIR" >&2
74 exit 1
75fi
76if [ ! -f "$BUNDLE_DIR/Bitflash.exe" ]; then
77 echo "missing Bitflash.exe in $BUNDLE_DIR" >&2
78 exit 1
79fi
80BUNDLE_PARENT="$(cd "$(dirname "$BUNDLE_DIR")" && pwd)"
81BUNDLE_NAME="$(basename "$BUNDLE_DIR")"
82OUT_PARENT="$(mkdir -p "$(dirname "$OUT_ZIP")" && cd "$(dirname "$OUT_ZIP")" && pwd)"
83OUT_NAME="$(basename "$OUT_ZIP")"
84OUT_ZIP_ABS="$OUT_PARENT/$OUT_NAME"
85
86mkdir -p "$CACHE_DIR"
87if [ ! -f "$ARCHIVE" ]; then
88 download "$TOR_URL" "$ARCHIVE"
89fi
90
91got="$(sha256sum "$ARCHIVE" | awk '{print tolower($1)}')"
92want="$(printf '%s' "$TOR_SHA256" | tr 'A-F' 'a-f')"
93if [ "$got" != "$want" ]; then
94 echo "Tor archive SHA256 mismatch" >&2
95 echo " expected: $want" >&2
96 echo " got: $got" >&2
97 exit 1
98fi
99
100tmp="$(mktemp -d "${TMPDIR:-/tmp}/bitflash-tor-package.XXXXXX")"
101trap 'rm -rf "$tmp"' EXIT
102
103case "$TOR_VERIFY_GPG" in
104 auto|required|off) ;;
105 *) echo "TOR_VERIFY_GPG must be auto, required, or off" >&2; exit 2 ;;
106esac
107
108if [ "$TOR_VERIFY_GPG" != "off" ]; then
109 if command -v gpg >/dev/null 2>&1; then
110 if [ ! -f "$ASC_FILE" ]; then
111 download "$TOR_ASC_URL" "$ASC_FILE"
112 fi
113 gnupg_home="$tmp/gnupg"
114 mkdir -p "$gnupg_home"
115 chmod 0700 "$gnupg_home"
116 gpg --homedir "$gnupg_home" --batch --auto-key-locate nodefault,wkd,keyserver \
117 --locate-keys torbrowser@torproject.org >/dev/null 2>&1 || {
118 if [ "$TOR_VERIFY_GPG" = "required" ]; then
119 echo "could not import Tor Browser Developers signing key" >&2
120 exit 1
121 fi
122 echo "WARNING: skipped Tor GPG signature verification; signing key import failed" >&2
123 }
124 if gpg --homedir "$gnupg_home" --batch --with-colons --fingerprint "$TOR_SIGNING_KEY_FINGERPRINT" \
125 2>/dev/null | grep -q "^fpr:::::::::${TOR_SIGNING_KEY_FINGERPRINT}:"; then
126 gpg --homedir "$gnupg_home" --batch --verify "$ASC_FILE" "$ARCHIVE" >/dev/null 2>&1 || {
127 echo "Tor archive GPG signature verification failed" >&2
128 exit 1
129 }
130 echo "Tor archive GPG signature verified against ${TOR_SIGNING_KEY_FINGERPRINT}"
131 elif [ "$TOR_VERIFY_GPG" = "required" ]; then
132 echo "Tor signing key fingerprint mismatch" >&2
133 exit 1
134 else
135 echo "WARNING: skipped Tor GPG signature verification; signing key fingerprint was not available" >&2
136 fi
137 elif [ "$TOR_VERIFY_GPG" = "required" ]; then
138 echo "missing required command for TOR_VERIFY_GPG=required: gpg" >&2
139 exit 1
140 else
141 echo "WARNING: gpg not found; relying on pinned SHA256 only" >&2
142 fi
143fi
144
145mkdir -p "$tmp/extract"
146tar -xzf "$ARCHIVE" -C "$tmp/extract"
147
148if [ ! -f "$tmp/extract/tor/tor.exe" ]; then
149 echo "Tor archive did not contain tor/tor.exe" >&2
150 exit 1
151fi
152
153rm -rf "$BUNDLE_DIR/tor"
154mkdir -p "$BUNDLE_DIR/tor"
155cp -R "$tmp/extract/tor/." "$BUNDLE_DIR/tor/"
156if [ -d "$tmp/extract/docs" ]; then
157 mkdir -p "$BUNDLE_DIR/tor/docs"
158 cp -R "$tmp/extract/docs/." "$BUNDLE_DIR/tor/docs/"
159fi
160if [ -d "$tmp/extract/data" ]; then
161 mkdir -p "$BUNDLE_DIR/tor/data-defaults"
162 cp -R "$tmp/extract/data/." "$BUNDLE_DIR/tor/data-defaults/"
163fi
164
165cat > "$BUNDLE_DIR/TOR-EXPERT-BUNDLE.txt" <<EOF
166Bitflash managed Tor bundle
167
168This Windows package includes the Tor Expert Bundle so -managedtor can start
169Tor without requiring a separate Tor install.
170
171Tor Expert Bundle: ${TOR_VERSION}
172Archive URL: ${TOR_URL}
173Archive SHA256: ${TOR_SHA256}
174
175Bitflash uses tor/tor.exe from this package. Runtime state, hidden-service keys,
176and control authentication cookies are written under the user's Bitflash data
177directory, not inside this release directory.
178EOF
179
180rm -f "$OUT_ZIP_ABS"
181(
182 cd "$BUNDLE_PARENT"
183 zip -qr "$OUT_ZIP_ABS" "$BUNDLE_NAME"
184)
185
186echo "Bundled Tor Expert Bundle ${TOR_VERSION}"
187echo "Built: $OUT_ZIP_ABS"
diff --git a/scripts/prove-utxo.py b/scripts/prove-utxo.py
new file mode 100644
index 0000000..105e677
--- /dev/null
+++ b/scripts/prove-utxo.py
@@ -0,0 +1,186 @@
1#!/usr/bin/env python3
2"""Create a Bitflash UTXO inclusion proof from local blk*.dat files.
3
4The proof is deliberately outside consensus. It proves that one txid:vout is
5included in the deterministic UTXO root produced by verify-utxo-set.py.
6"""
7
8import argparse
9import hashlib
10import json
11import sys
12from pathlib import Path
13
14import bitflash_chain as chain
15
16SCHEMA = "bitflash-utxo-inclusion-proof-1"
17
18
19def canonical_bytes(obj):
20 return (json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + "\n").encode("ascii")
21
22
23def split_outpoint(outpoint):
24 parts = outpoint.split(":")
25 if len(parts) != 2 or len(parts[0]) != 64:
26 raise SystemExit("outpoint must be txid:vout")
27 try:
28 vout = int(parts[1], 10)
29 except ValueError:
30 raise SystemExit("vout must be a decimal integer")
31 if vout < 0:
32 raise SystemExit("vout must not be negative")
33 return parts[0].lower(), vout
34
35
36def build_merkle_proof(keys, leaf_hashes, index):
37 current_index = index
38 layer = list(leaf_hashes)
39 proof = []
40 level = 0
41 while len(layer) > 1:
42 if current_index % 2 == 0:
43 sibling_index = current_index + 1
44 duplicated = sibling_index >= len(layer)
45 sibling = layer[current_index] if duplicated else layer[sibling_index]
46 side = "right"
47 else:
48 sibling_index = current_index - 1
49 duplicated = False
50 sibling = layer[sibling_index]
51 side = "left"
52 proof.append({
53 "level": level,
54 "side": side,
55 "hash": sibling.hex(),
56 "duplicated": duplicated,
57 })
58
59 next_layer = []
60 for i in range(0, len(layer), 2):
61 left = layer[i]
62 right = layer[i + 1] if i + 1 < len(layer) else left
63 next_layer.append(hashlib.sha256(chain.UTXO_NODE_DOMAIN + left + right).digest())
64 layer = next_layer
65 current_index //= 2
66 level += 1
67 return proof, layer[0].hex()
68
69
70def build_proof(args):
71 txid, vout = split_outpoint(args.outpoint)
72 outpoint = "%s:%d" % (txid, vout)
73
74 raw_blocks = chain.read_blocks(args.datadir, args.max_blocks)
75 if not raw_blocks:
76 raise SystemExit("no Bitflash blocks found in %s" % args.datadir)
77 try:
78 blocks = chain.select_main_chain(raw_blocks)
79 state = chain.apply_blocks(blocks, strict_duplicates=args.strict_duplicates)
80 except chain.ParseError as e:
81 raise SystemExit(str(e))
82 utxos = state["utxos"]
83 stats = state["stats"]
84 if outpoint not in utxos:
85 raise SystemExit("outpoint is not unspent in the scanned chain: %s" % outpoint)
86
87 keys = sorted(utxos)
88 leaf_hashes = [chain.utxo_leaf_hash(k, utxos[k]) for k in keys]
89 index = keys.index(outpoint)
90 proof_path, root = build_merkle_proof(keys, leaf_hashes, index)
91 direct_root = chain.utxo_root(utxos)
92 if root != direct_root:
93 raise SystemExit("internal proof root mismatch")
94
95 tip = blocks[-1]
96 utxo = utxos[outpoint]
97 report = {
98 "schema": SCHEMA,
99 "deterministic": True,
100 "chain": {
101 "name": "Bitflash",
102 "ticker": "BTF",
103 "message_start_hex": chain.MAGIC.hex(),
104 "coin": chain.COIN,
105 },
106 "commitment": {
107 "algorithm": "sorted-outpoint-merkle-sha256",
108 "leaf_domain": chain.UTXO_LEAF_DOMAIN,
109 "node_domain": chain.UTXO_NODE_DOMAIN_TAG,
110 "root": root,
111 "leaf_count": len(keys),
112 },
113 "target": {
114 "outpoint": outpoint,
115 "txid": txid,
116 "vout": vout,
117 "leaf_index": index,
118 "leaf_hash": leaf_hashes[index].hex(),
119 "utxo": utxo,
120 },
121 "proof": proof_path,
122 "scanned": {
123 "raw_block_count": len(raw_blocks),
124 "side_blocks_ignored": len(raw_blocks) - len(blocks),
125 "main_chain_block_count": len(blocks),
126 "first_height": 0,
127 "last_height": len(blocks) - 1,
128 "tip_hash": tip["hash"],
129 "tip_time": tip["time"],
130 "tip_time_utc": tip["time_utc"],
131 },
132 "warnings": {
133 "legacy_duplicate_output_overwrites": len(stats["duplicate_outputs"]),
134 },
135 "verdict": "UTXO_INCLUSION_PROOF_CREATED",
136 }
137 return report
138
139
140def print_text(proof):
141 data_hash = hashlib.sha256(canonical_bytes(proof)).hexdigest()
142 target = proof["target"]
143 utxo = target["utxo"]
144 print("Bitflash UTXO inclusion proof")
145 print("schema: %s" % proof["schema"])
146 print("canonical proof sha256: %s" % data_hash)
147 print()
148 print("Target")
149 print(" outpoint: %s" % target["outpoint"])
150 print(" value: %s BTF" % utxo["value_btf"])
151 print(" height: %d" % utxo["created_height"])
152 print(" index: %d of %d" % (target["leaf_index"], proof["commitment"]["leaf_count"]))
153 print()
154 print("Commitment")
155 print(" root: %s" % proof["commitment"]["root"])
156 print(" path: %d sibling hashes" % len(proof["proof"]))
157 print(" tip: %s" % proof["scanned"]["tip_hash"])
158 print()
159 print("Verdict: %s" % proof["verdict"])
160
161
162def main(argv):
163 ap = argparse.ArgumentParser(description=__doc__)
164 ap.add_argument("outpoint", help="unspent output to prove, as txid:vout")
165 ap.add_argument("--datadir", default=chain.default_datadir(), help="Bitflash data directory")
166 ap.add_argument("--max-blocks", type=int, default=0,
167 help="number of blocks to scan; 0 means all block files")
168 ap.add_argument("--strict-duplicates", action="store_true",
169 help="fail instead of using legacy last-write-wins duplicate output handling")
170 ap.add_argument("--json", action="store_true", help="print canonical JSON")
171 ap.add_argument("--out", help="write canonical JSON proof to this path")
172 args = ap.parse_args(argv)
173
174 proof = build_proof(args)
175 data = canonical_bytes(proof)
176 if args.out:
177 Path(args.out).write_bytes(data)
178 if args.json:
179 sys.stdout.buffer.write(data)
180 else:
181 print_text(proof)
182 return 0
183
184
185if __name__ == "__main__":
186 raise SystemExit(main(sys.argv[1:]))
diff --git a/scripts/smoke-managed-tor-onion.ps1 b/scripts/smoke-managed-tor-onion.ps1
new file mode 100644
index 0000000..70658c7
--- /dev/null
+++ b/scripts/smoke-managed-tor-onion.ps1
@@ -0,0 +1,174 @@
1param(
2 [string]$Bitflash,
3 [string]$Tor,
4 [string]$WorkDir,
5 [int]$TimeoutSec = 600,
6 [int]$OnionWarmupSec = 120
7)
8
9$ErrorActionPreference = "Stop"
10
11function Resolve-RepoRoot {
12 $scriptDir = Split-Path -Parent $PSCommandPath
13 return (Resolve-Path (Join-Path $scriptDir "..")).Path
14}
15
16function Get-FreeTcpPort {
17 $listener = [Net.Sockets.TcpListener]::new([Net.IPAddress]::Loopback, 0)
18 $listener.Start()
19 try {
20 return ([Net.IPEndPoint]$listener.LocalEndpoint).Port
21 }
22 finally {
23 $listener.Stop()
24 }
25}
26
27function Wait-RegexInFile {
28 param(
29 [string]$Path,
30 [string]$Pattern,
31 [datetime]$Deadline,
32 [string]$Label
33 )
34
35 while ((Get-Date) -lt $Deadline) {
36 if (Test-Path $Path) {
37 $text = Get-Content -Raw -ErrorAction SilentlyContinue $Path
38 if ($null -ne $text) {
39 $m = [regex]::Match($text, $Pattern)
40 if ($m.Success) {
41 return $m
42 }
43 }
44 }
45 Start-Sleep -Milliseconds 500
46 }
47 throw "timed out waiting for $Label"
48}
49
50function Stop-TestProcesses {
51 param(
52 [System.Diagnostics.Process[]]$Processes,
53 [string[]]$DataDirKeys
54 )
55
56 foreach ($p in $Processes) {
57 try {
58 $p.Refresh()
59 if (!$p.HasExited) {
60 Stop-Process -Id $p.Id -Force
61 }
62 }
63 catch {}
64 }
65
66 foreach ($key in $DataDirKeys) {
67 if ([string]::IsNullOrWhiteSpace($key)) {
68 continue
69 }
70 try {
71 Get-CimInstance Win32_Process -Filter "name = 'tor.exe'" |
72 Where-Object { $_.CommandLine -like "*$key*" } |
73 ForEach-Object { Stop-Process -Id $_.ProcessId -Force }
74 }
75 catch {}
76 }
77}
78
79$repo = Resolve-RepoRoot
80if ([string]::IsNullOrWhiteSpace($Bitflash)) {
81 $Bitflash = Join-Path $repo "src\bitflash.exe"
82}
83if ([string]::IsNullOrWhiteSpace($Tor)) {
84 $candidates = @(
85 (Join-Path $repo "src\tor\tor.exe"),
86 (Join-Path $repo "Bitflash-$(Select-String -Path (Join-Path $repo 'Makefile') -Pattern '^VERSION[ ]*:=' | ForEach-Object { ($_.Line -split ':=')[1].Trim() })-windows\tor\tor.exe")
87 )
88 foreach ($candidate in $candidates) {
89 if (Test-Path $candidate) {
90 $Tor = $candidate
91 break
92 }
93 }
94}
95if ([string]::IsNullOrWhiteSpace($WorkDir)) {
96 $WorkDir = Join-Path $env:TEMP ("bitflash-managed-tor-two-node-" + (Get-Date -Format "yyyyMMddHHmmss"))
97}
98
99if (!(Test-Path $Bitflash)) {
100 throw "Bitflash binary not found: $Bitflash"
101}
102if (!(Test-Path $Tor)) {
103 throw "Tor binary not found. Pass -Tor or build the Windows-with-Tor package."
104}
105
106New-Item -ItemType Directory -Force -Path $WorkDir | Out-Null
107$dataA = Join-Path $WorkDir "node-a"
108$dataB = Join-Path $WorkDir "node-b"
109New-Item -ItemType Directory -Force -Path $dataA | Out-Null
110New-Item -ItemType Directory -Force -Path $dataB | Out-Null
111
112$portA = Get-FreeTcpPort
113$portB = Get-FreeTcpPort
114$deadline = (Get-Date).AddSeconds($TimeoutSec)
115$oldPath = $env:PATH
116$env:PATH = "C:\msys64\ucrt64\bin;C:\msys64\usr\bin;" + $env:PATH
117
118$procA = $null
119$procB = $null
120$dataKeys = @((Split-Path $WorkDir -Leaf))
121
122try {
123 $argsA = @(
124 "-datadir=$dataA",
125 "-port=$portA",
126 "-managedtor=$Tor",
127 "-nogui",
128 "-debug",
129 "-checkblocks=0"
130 )
131 $procA = Start-Process -FilePath $Bitflash -ArgumentList $argsA -WorkingDirectory (Split-Path $Bitflash -Parent) -PassThru -WindowStyle Hidden
132 $logA = Join-Path $dataA "debug.log"
133
134 $btfMatch = Wait-RegexInFile -Path $logA -Pattern "Nostr: node \.btf address = ([a-z2-7]+\.btf)" -Deadline $deadline -Label "node A .btf address"
135 $btfA = $btfMatch.Groups[1].Value
136
137 $onionMatch = Wait-RegexInFile -Path $logA -Pattern "Managed Tor hidden service ready: ([a-z2-7]{56}\.onion:\d+)" -Deadline $deadline -Label "node A managed onion"
138 $onionA = $onionMatch.Groups[1].Value
139
140 Wait-RegexInFile -Path $logA -Pattern "Nostr: \.btf self-resolve OK via .*" -Deadline $deadline -Label "node A descriptor self-resolve" | Out-Null
141 if ($OnionWarmupSec -gt 0) {
142 Write-Host "Waiting $OnionWarmupSec second(s) for Tor hidden-service descriptor propagation"
143 Start-Sleep -Seconds $OnionWarmupSec
144 }
145
146 $argsB = @(
147 "-datadir=$dataB",
148 "-port=$portB",
149 "-managedtor=$Tor",
150 "-oniononly",
151 "-connectbtf=$btfA",
152 "-nogui",
153 "-debug",
154 "-checkblocks=0"
155 )
156 $procB = Start-Process -FilePath $Bitflash -ArgumentList $argsB -WorkingDirectory (Split-Path $Bitflash -Parent) -PassThru -WindowStyle Hidden
157 $logB = Join-Path $dataB "debug.log"
158
159 Wait-RegexInFile -Path $logB -Pattern ("connected " + [regex]::Escape($btfA) + " via direct onion ") -Deadline $deadline -Label "node B direct onion connection to node A" | Out-Null
160
161 Write-Host "Managed Tor two-node onion smoke test passed"
162 Write-Host " node A .btf: $btfA"
163 Write-Host " node A onion: $onionA"
164 Write-Host " node A port: $portA"
165 Write-Host " node B port: $portB"
166 Write-Host " work dir: $WorkDir"
167}
168finally {
169 $env:PATH = $oldPath
170 $procs = @()
171 if ($null -ne $procA) { $procs += $procA }
172 if ($null -ne $procB) { $procs += $procB }
173 Stop-TestProcesses -Processes $procs -DataDirKeys $dataKeys
174}
diff --git a/scripts/verify-fair-launch.py b/scripts/verify-fair-launch.py
new file mode 100644
index 0000000..c33c8be
--- /dev/null
+++ b/scripts/verify-fair-launch.py
@@ -0,0 +1,166 @@
1#!/usr/bin/env python3
2"""Produce a deterministic Bitflash fair-launch audit report."""
3
4import argparse
5import hashlib
6import sys
7from pathlib import Path
8
9import bitflash_chain as chain
10
11
12def build_report(blocks, source_mode):
13 genesis = blocks[0]
14 launch_outputs = genesis["transactions"][0]["vout"] if genesis["transactions"] else []
15 coinbase_outputs = []
16 total_mined = 0
17 non_coinbase_txs = 0
18
19 for height, block in enumerate(blocks):
20 if not block["transactions"]:
21 continue
22 cb = block["transactions"][0]
23 if chain.is_coinbase(cb):
24 amount = sum(out["value_satoshis"] for out in cb["vout"])
25 total_mined += amount
26 coinbase_outputs.append({
27 "height": height,
28 "block_hash": block["hash"],
29 "coinbase_txid": cb["txid"],
30 "value_satoshis": amount,
31 "value_btf": chain.format_money(amount),
32 "outputs": cb["vout"],
33 })
34 non_coinbase_txs += sum(1 for tx in block["transactions"] if not chain.is_coinbase(tx))
35
36 checks = {
37 "genesis_hash_matches_source": genesis["hash"] == chain.GENESIS_HASH,
38 "genesis_merkle_matches_source": genesis["merkle_root"] == chain.GENESIS_MERKLE,
39 "computed_merkle_matches_header": genesis["computed_merkle_root"] == genesis["merkle_root"],
40 "genesis_has_single_transaction": genesis["tx_count"] == 1,
41 "genesis_transaction_is_coinbase": bool(genesis["transactions"] and chain.is_coinbase(genesis["transactions"][0])),
42 "genesis_supply_is_50_btf": sum(out["value_satoshis"] for out in launch_outputs) == 50 * chain.COIN,
43 }
44
45 report = {
46 "schema": "bitflash-fair-launch-report-1",
47 "deterministic": True,
48 "source_mode": source_mode,
49 "chain": {
50 "name": "Bitflash",
51 "ticker": "BTF",
52 "message_start_hex": chain.MAGIC.hex(),
53 "coin": chain.COIN,
54 },
55 "source_constants": {
56 "genesis_hash": chain.GENESIS_HASH,
57 "genesis_merkle_root": chain.GENESIS_MERKLE,
58 "genesis_time": chain.GENESIS_TIME,
59 "genesis_time_utc": chain.utc(chain.GENESIS_TIME),
60 "genesis_bits": "0x%08x" % chain.GENESIS_BITS,
61 "genesis_nonce": chain.GENESIS_NONCE,
62 "genesis_message": chain.GENESIS_TEXT,
63 },
64 "scanned": {
65 "block_count": len(blocks),
66 "first_height": 0,
67 "last_height": len(blocks) - 1,
68 "non_coinbase_transactions": non_coinbase_txs,
69 "total_coinbase_satoshis": total_mined,
70 "total_coinbase_btf": chain.format_money(total_mined),
71 },
72 "genesis": {
73 "hash": genesis["hash"],
74 "merkle_root": genesis["merkle_root"],
75 "computed_merkle_root": genesis["computed_merkle_root"],
76 "time": genesis["time"],
77 "time_utc": genesis["time_utc"],
78 "bits": genesis["bits"],
79 "nonce": genesis["nonce"],
80 "tx_count": genesis["tx_count"],
81 "coinbase_txid": genesis["transactions"][0]["txid"] if genesis["transactions"] else "",
82 "outputs": launch_outputs,
83 },
84 "coinbase_outputs": coinbase_outputs,
85 "checks": checks,
86 "verdict": "FAIR_LAUNCH_BASELINE_VERIFIED" if all(checks.values()) else "CHECKS_FAILED",
87 "external_timestamping": {
88 "status": "not_embedded",
89 "recommended": "OpenTimestamps",
90 "instructions": [
91 "Save the canonical JSON report.",
92 "Run: ots stamp bitflash-fair-launch-report.json",
93 "Later verify with: ots verify bitflash-fair-launch-report.json.ots",
94 ],
95 },
96 }
97 report["attestation_payload_sha256"] = hashlib.sha256(chain.canonical_bytes(report)).hexdigest()
98 return report
99
100
101def print_text(report):
102 checks = report["checks"]
103 print("Bitflash fair-launch verifier")
104 print("schema: %s" % report["schema"])
105 print("source: %s" % report["source_mode"])
106 print("attestation payload sha256: %s" % report["attestation_payload_sha256"])
107 print()
108 print("Genesis")
109 print(" hash: %s" % report["genesis"]["hash"])
110 print(" merkle: %s" % report["genesis"]["merkle_root"])
111 print(" time: %s" % report["genesis"]["time_utc"])
112 print(" txs: %d" % report["genesis"]["tx_count"])
113 print(" supply: %s BTF" % chain.format_money(sum(o["value_satoshis"] for o in report["genesis"]["outputs"])))
114 print()
115 print("Scanned")
116 print(" blocks: %d" % report["scanned"]["block_count"])
117 print(" height: %d..%d" % (report["scanned"]["first_height"], report["scanned"]["last_height"]))
118 print(" minted: %s BTF" % report["scanned"]["total_coinbase_btf"])
119 print(" normal transactions in range: %d" % report["scanned"]["non_coinbase_transactions"])
120 print()
121 print("Checks")
122 for key in sorted(checks):
123 print(" %s %s" % ("OK " if checks[key] else "FAIL", key))
124 print()
125 print("Verdict: %s" % report["verdict"])
126 print()
127 print("OpenTimestamps")
128 print(" canonical JSON can be stamped later with: ots stamp <report.json>")
129
130
131def main(argv):
132 ap = argparse.ArgumentParser(description=__doc__)
133 ap.add_argument("--datadir", default=chain.default_datadir(), help="Bitflash data directory")
134 ap.add_argument("--max-blocks", type=int, default=100, help="number of blocks to scan from blk*.dat")
135 ap.add_argument("--json", action="store_true", help="print canonical JSON")
136 ap.add_argument("--out", help="write canonical JSON to this path")
137 ap.add_argument("--reconstruct-genesis", action="store_true",
138 help="do not read disk; reconstruct only the genesis block from source constants")
139 args = ap.parse_args(argv)
140
141 if args.reconstruct_genesis:
142 blocks = [chain.reconstructed_genesis()]
143 source_mode = "reconstructed-genesis"
144 else:
145 blocks = chain.read_blocks(args.datadir, args.max_blocks)
146 if not blocks:
147 raise SystemExit("no Bitflash blocks found in %s; use --reconstruct-genesis for a source-only check" % args.datadir)
148 source_mode = "local-block-files"
149
150 try:
151 report = build_report(blocks, source_mode)
152 except chain.ParseError as e:
153 raise SystemExit(str(e))
154 data = chain.canonical_bytes(report)
155
156 if args.out:
157 Path(args.out).write_bytes(data)
158 if args.json:
159 sys.stdout.buffer.write(data)
160 else:
161 print_text(report)
162 return 0 if report["verdict"] == "FAIR_LAUNCH_BASELINE_VERIFIED" else 1
163
164
165if __name__ == "__main__":
166 raise SystemExit(main(sys.argv[1:]))
diff --git a/scripts/verify-release.sh b/scripts/verify-release.sh
index c651e78..4fd4558 100644..100755
--- a/scripts/verify-release.sh
+++ b/scripts/verify-release.sh
@@ -1,23 +1,50 @@
1#!/usr/bin/env bash 1#!/usr/bin/env bash
2set -euo pipefail 2set -euo pipefail
3 3
4repo="bitflash-coin/bitflash" 4# Verifies a Bitflash release: the checksum file's signature against a pinned
5tag="${1:-latest}" 5# key, then the assets against the checksums.
6#
7# The signing key is embedded below, not fetched. That is the whole point of a
8# pinned key: the trust anchor ships with this script (in git), so it does not
9# depend on the same server that serves the release. A server that is
10# compromised can swap the binaries and the checksums, but it cannot forge a
11# signature by this key.
12
13host="https://releases.bitflash.network"
6require_signature=0 14require_signature=0
7 15
16# The Bitflash release signing key. Verification requires this exact key --
17# gpg's own "good signature" is not enough, because a good signature by *some*
18# key proves nothing. See issue #138.
19RELEASE_KEY_FP="910A2B4CCA879E81FB4B2AEA1D4A53D3B78AA4B8"
20read -r -d '' RELEASE_KEY <<'KEY' || true
21-----BEGIN PGP PUBLIC KEY BLOCK-----
22
23mDMEanSOmRYJKwYBBAHaRw8BAQdAXj24i2BYUcAmOpoXwjE4ZTynHsifCZ+niwQK
24HqO0nMa0UEJpdGZsYXNoIFJlbGVhc2VzIChiaXRmbGFzaC5uZXR3b3JrIHJlbGVh
25c2Ugc2lnbmluZykgPHJlbGVhc2VzQGJpdGZsYXNoLm5ldHdvcms+iJAEExYKADgW
26IQSRCitMyoeegftLKuodSlPTt4qkuAUCanSOmQIbAwULCQgHAgYVCgkICwIEFgID
27AQIeAQIXgAAKCRAdSlPTt4qkuIi9AP96X3OcstiCTTgKOr16Vhds3LyuZ19q3Apq
284KAcNuOUMAD+NqgJ1CqkLGPXfZLCtYBCRHXNYWxMMEk+ViLcRtdLMg4=
29=mxic
30-----END PGP PUBLIC KEY BLOCK-----
31KEY
32
8usage() { 33usage() {
9 cat <<'EOF' 34 cat <<'EOF'
10Usage: scripts/verify-release.sh [tag|latest] [--require-signature] 35Usage: scripts/verify-release.sh [tag|latest] [--require-signature]
11 36
12Downloads the release checksum file and every asset named in it, then verifies: 37Downloads the release checksum file and every asset named in it, then verifies:
13 1. SHA256SUMS.asc, if present, against SHA256SUMS with gpg. 38 1. SHA256SUMS.asc against SHA256SUMS, requiring the pinned Bitflash release
39 key (fingerprint 910A 2B4C CA87 9E81 FB4B 2AEA 1D4A 53D3 B78A A4B8).
14 2. SHA256SUMS against the downloaded assets. 40 2. SHA256SUMS against the downloaded assets.
15 41
16Use --require-signature for release audits. Without it, older releases that 42Releases from before signing was introduced ship only SHA256SUMS; without
17ship only SHA256SUMS are checked for integrity and reported as unsigned. 43--require-signature those are checked for integrity and reported as unsigned.
18EOF 44EOF
19} 45}
20 46
47tag="${1:-latest}"
21for arg in "${@:2}"; do 48for arg in "${@:2}"; do
22 case "$arg" in 49 case "$arg" in
23 --require-signature) require_signature=1 ;; 50 --require-signature) require_signature=1 ;;
@@ -25,19 +52,9 @@ for arg in "${@:2}"; do
25 *) echo "unknown option: $arg" >&2; usage >&2; exit 2 ;; 52 *) echo "unknown option: $arg" >&2; usage >&2; exit 2 ;;
26 esac 53 esac
27done 54done
55if [ "$tag" = "-h" ] || [ "$tag" = "--help" ]; then usage; exit 0; fi
28 56
29if [ "$tag" = "-h" ] || [ "$tag" = "--help" ]; then 57need() { command -v "$1" >/dev/null 2>&1 || { echo "missing required command: $1" >&2; exit 1; }; }
30 usage
31 exit 0
32fi
33
34need() {
35 if ! command -v "$1" >/dev/null 2>&1; then
36 echo "missing required command: $1" >&2
37 exit 1
38 fi
39}
40
41need sed 58need sed
42need awk 59need awk
43 60
@@ -46,39 +63,25 @@ if command -v sha256sum >/dev/null 2>&1; then
46elif command -v shasum >/dev/null 2>&1; then 63elif command -v shasum >/dev/null 2>&1; then
47 sha_check=(shasum -a 256 -c SHA256SUMS) 64 sha_check=(shasum -a 256 -c SHA256SUMS)
48else 65else
49 echo "missing required command: sha256sum or shasum" >&2 66 echo "missing required command: sha256sum or shasum" >&2; exit 1
50 exit 1
51fi 67fi
52 68
53download() { 69download() {
54 local url="$1" 70 if command -v curl >/dev/null 2>&1; then curl -fsSL "$1" -o "$2"
55 local out="$2" 71 elif command -v wget >/dev/null 2>&1; then wget -q "$1" -O "$2"
56 if command -v curl >/dev/null 2>&1; then 72 else echo "missing required command: curl or wget" >&2; exit 1; fi
57 curl -fsSL "$url" -o "$out"
58 elif command -v wget >/dev/null 2>&1; then
59 wget -q "$url" -O "$out"
60 else
61 echo "missing required command: curl or wget" >&2
62 exit 1
63 fi
64} 73}
65 74
66if [ "$tag" = "latest" ]; then 75if [ "$tag" = "latest" ]; then
67 tag="$(download "https://api.github.com/repos/$repo/releases/latest" - | 76 tag="$(download "$host/latest.txt" - 2>/dev/null | tr -d '[:space:]' || true)"
68 sed -n 's/.*"tag_name"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | 77 [ -n "$tag" ] || { echo "could not resolve the latest release tag from $host/latest.txt" >&2; exit 1; }
69 head -1)"
70 if [ -z "$tag" ]; then
71 echo "could not resolve latest release tag" >&2
72 exit 1
73 fi
74fi 78fi
75 79
76workdir="$(mktemp -d "${TMPDIR:-/tmp}/bitflash-release-verify.XXXXXX")" 80workdir="$(mktemp -d "${TMPDIR:-/tmp}/bitflash-release-verify.XXXXXX")"
77cleanup() { rm -rf "$workdir"; } 81cleanup() { rm -rf "$workdir"; }
78trap cleanup EXIT 82trap cleanup EXIT
79 83
80base_url="https://github.com/$repo/releases/download/$tag" 84base_url="$host/$tag"
81
82echo "Verifying Bitflash $tag" 85echo "Verifying Bitflash $tag"
83echo "Working directory: $workdir" 86echo "Working directory: $workdir"
84cd "$workdir" 87cd "$workdir"
@@ -86,27 +89,32 @@ cd "$workdir"
86download "$base_url/SHA256SUMS" SHA256SUMS 89download "$base_url/SHA256SUMS" SHA256SUMS
87 90
88if download "$base_url/SHA256SUMS.asc" SHA256SUMS.asc 2>/dev/null; then 91if download "$base_url/SHA256SUMS.asc" SHA256SUMS.asc 2>/dev/null; then
89 if ! command -v gpg >/dev/null 2>&1; then 92 need gpg
90 echo "SHA256SUMS.asc exists, but gpg is not installed" >&2 93 # Verify in a throwaway keyring holding ONLY the pinned key, and require that
94 # the signature validates against it with the pinned fingerprint. This is the
95 # difference between "signed" and "signed by Bitflash".
96 keyring="$workdir/keyring"
97 mkdir -p "$keyring"; chmod 700 "$keyring"
98 printf '%s\n' "$RELEASE_KEY" | GNUPGHOME="$keyring" gpg --batch --quiet --import 2>/dev/null
99 status="$(GNUPGHOME="$keyring" gpg --batch --status-fd=1 --verify SHA256SUMS.asc SHA256SUMS 2>/dev/null || true)"
100 if printf '%s' "$status" | grep -q "VALIDSIG.*$RELEASE_KEY_FP"; then
101 echo "Signature OK -- signed by the pinned Bitflash release key"
102 else
103 echo "SIGNATURE REJECTED: not a valid signature by the pinned release key" >&2
104 echo " expected key: $RELEASE_KEY_FP" >&2
91 exit 1 105 exit 1
92 fi 106 fi
93 gpg --verify SHA256SUMS.asc SHA256SUMS
94 echo "Signature OK"
95else 107else
96 if [ "$require_signature" -eq 1 ]; then 108 if [ "$require_signature" -eq 1 ]; then
97 echo "release does not publish SHA256SUMS.asc" >&2 109 echo "release does not publish SHA256SUMS.asc" >&2; exit 1
98 exit 1
99 fi 110 fi
100 echo "WARNING: no SHA256SUMS.asc found; checking hashes only" 111 echo "WARNING: no SHA256SUMS.asc found; checking hashes only (unsigned release)"
101fi 112fi
102 113
103awk '{print $2}' SHA256SUMS | while IFS= read -r file; do 114awk '{print $2}' SHA256SUMS | while IFS= read -r file; do
104 file="${file#\*}" 115 file="${file#\*}"; file="${file#./}"
105 file="${file#./}"
106 [ -z "$file" ] && continue 116 [ -z "$file" ] && continue
107 case "$file" in 117 case "$file" in */*|..*) echo "refusing unexpected checksum path: $file" >&2; exit 1 ;; esac
108 */*|..*) echo "refusing unexpected checksum path: $file" >&2; exit 1 ;;
109 esac
110 echo "Downloading $file" 118 echo "Downloading $file"
111 download "$base_url/$file" "$file" 119 download "$base_url/$file" "$file"
112done 120done
diff --git a/scripts/verify-utxo-proof.py b/scripts/verify-utxo-proof.py
new file mode 100644
index 0000000..50d9356
--- /dev/null
+++ b/scripts/verify-utxo-proof.py
@@ -0,0 +1,146 @@
1#!/usr/bin/env python3
2"""Verify a Bitflash UTXO inclusion proof against a UTXO root."""
3
4import argparse
5import hashlib
6import json
7import sys
8from pathlib import Path
9
10import bitflash_chain as chain
11
12SCHEMA = "bitflash-utxo-inclusion-proof-1"
13
14
15def canonical_bytes(obj):
16 return (json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + "\n").encode("ascii")
17
18
19def load_proof(path):
20 try:
21 return json.loads(Path(path).read_text(encoding="ascii"))
22 except Exception as e:
23 raise SystemExit("could not read proof: %s" % e)
24
25
26def verify(proof, expected_root=None):
27 failures = []
28 if proof.get("schema") != SCHEMA:
29 failures.append("unexpected schema")
30 commitment = proof.get("commitment", {})
31 if commitment.get("algorithm") != "sorted-outpoint-merkle-sha256":
32 failures.append("unexpected commitment algorithm")
33 if commitment.get("leaf_domain") != chain.UTXO_LEAF_DOMAIN:
34 failures.append("unexpected leaf domain")
35 if commitment.get("node_domain") != chain.UTXO_NODE_DOMAIN_TAG:
36 failures.append("unexpected node domain")
37
38 target = proof.get("target", {})
39 outpoint = target.get("outpoint", "")
40 utxo = target.get("utxo", {})
41 if outpoint != "%s:%s" % (target.get("txid"), target.get("vout")):
42 failures.append("target outpoint does not match txid:vout")
43
44 try:
45 current = chain.utxo_leaf_hash(outpoint, utxo)
46 leaf_ok = target.get("leaf_hash") == current.hex()
47 except Exception as e:
48 failures.append("could not hash target leaf: %s" % e)
49 current = b"\x00" * 32
50 leaf_ok = False
51
52 if not leaf_ok:
53 failures.append("target leaf hash mismatch")
54
55 for i, step in enumerate(proof.get("proof", [])):
56 try:
57 sibling = bytes.fromhex(step["hash"])
58 except Exception:
59 failures.append("bad sibling hash at proof level %d" % i)
60 sibling = b"\x00" * 32
61 if len(sibling) != 32:
62 failures.append("bad sibling hash length at proof level %d" % i)
63 side = step.get("side")
64 if side == "right":
65 current = chain.sha256(chain.UTXO_NODE_DOMAIN + current + sibling)
66 elif side == "left":
67 current = chain.sha256(chain.UTXO_NODE_DOMAIN + sibling + current)
68 else:
69 failures.append("bad sibling side at proof level %d" % i)
70
71 computed_root = current.hex()
72 root = expected_root or commitment.get("root")
73 if computed_root != root:
74 failures.append("computed root does not match expected root")
75
76 return {
77 "schema": "bitflash-utxo-proof-verification-1",
78 "deterministic": True,
79 "target": {
80 "outpoint": outpoint,
81 "value_satoshis": utxo.get("value_satoshis"),
82 "value_btf": utxo.get("value_btf"),
83 "created_height": utxo.get("created_height"),
84 },
85 "commitment": {
86 "expected_root": root,
87 "computed_root": computed_root,
88 "leaf_hash": target.get("leaf_hash"),
89 "path_length": len(proof.get("proof", [])),
90 },
91 "checks": {
92 "schema_is_supported": proof.get("schema") == SCHEMA,
93 "leaf_hash_matches_target": leaf_ok,
94 "root_matches": computed_root == root,
95 },
96 "failures": failures,
97 "verdict": "UTXO_PROOF_VERIFIED" if not failures else "UTXO_PROOF_FAILED",
98 }
99
100
101def print_text(result):
102 data_hash = hashlib.sha256(canonical_bytes(result)).hexdigest()
103 print("Bitflash UTXO proof verifier")
104 print("schema: %s" % result["schema"])
105 print("canonical verification sha256: %s" % data_hash)
106 print()
107 print("Target")
108 print(" outpoint: %s" % result["target"]["outpoint"])
109 print(" value: %s BTF" % result["target"]["value_btf"])
110 print(" height: %s" % result["target"]["created_height"])
111 print()
112 print("Commitment")
113 print(" expected: %s" % result["commitment"]["expected_root"])
114 print(" computed: %s" % result["commitment"]["computed_root"])
115 print(" path: %d sibling hashes" % result["commitment"]["path_length"])
116 print()
117 print("Checks")
118 for key in sorted(result["checks"]):
119 print(" %s %s" % ("OK " if result["checks"][key] else "FAIL", key))
120 if result["failures"]:
121 print()
122 print("Failures")
123 for failure in result["failures"]:
124 print(" - %s" % failure)
125 print()
126 print("Verdict: %s" % result["verdict"])
127
128
129def main(argv):
130 ap = argparse.ArgumentParser(description=__doc__)
131 ap.add_argument("proof", help="canonical JSON proof from prove-utxo.py")
132 ap.add_argument("--root", help="expected UTXO root; defaults to root embedded in proof")
133 ap.add_argument("--json", action="store_true", help="print canonical JSON")
134 args = ap.parse_args(argv)
135
136 result = verify(load_proof(args.proof), args.root)
137 data = canonical_bytes(result)
138 if args.json:
139 sys.stdout.buffer.write(data)
140 else:
141 print_text(result)
142 return 0 if result["verdict"] == "UTXO_PROOF_VERIFIED" else 1
143
144
145if __name__ == "__main__":
146 raise SystemExit(main(sys.argv[1:]))
diff --git a/scripts/verify-utxo-set.py b/scripts/verify-utxo-set.py
new file mode 100644
index 0000000..b5bf3d5
--- /dev/null
+++ b/scripts/verify-utxo-set.py
@@ -0,0 +1,131 @@
1#!/usr/bin/env python3
2"""Build a deterministic Bitflash UTXO-set commitment from local blk*.dat."""
3
4import argparse
5import hashlib
6import sys
7from pathlib import Path
8
9import bitflash_chain as chain
10
11
12def build_report(raw_blocks, include_utxos=False, strict_duplicates=False):
13 blocks = chain.select_main_chain(raw_blocks)
14 state = chain.apply_blocks(blocks, strict_duplicates=strict_duplicates)
15 utxos = state["utxos"]
16 stats = state["stats"]
17 tip = blocks[-1]
18 root = chain.utxo_root(utxos)
19 checks = {
20 "genesis_hash_matches_source": blocks[0]["hash"] == chain.GENESIS_HASH,
21 "genesis_merkle_matches_source": blocks[0]["merkle_root"] == chain.GENESIS_MERKLE,
22 "all_scanned_merkle_roots_match": all(b["computed_merkle_root"] == b["merkle_root"] for b in blocks),
23 "no_missing_spends": len(stats["missing_spends"]) == 0,
24 }
25 warnings = {
26 "legacy_duplicate_output_overwrites": len(stats["duplicate_outputs"]),
27 }
28 report = {
29 "schema": "bitflash-utxo-set-commitment-1",
30 "deterministic": True,
31 "chain": {
32 "name": "Bitflash",
33 "ticker": "BTF",
34 "message_start_hex": chain.MAGIC.hex(),
35 "coin": chain.COIN,
36 },
37 "commitment": {
38 "algorithm": "sorted-outpoint-merkle-sha256",
39 "leaf_domain": "BTFUTXO1",
40 "node_domain": "BTFNODE1",
41 "root": root,
42 },
43 "scanned": {
44 "raw_block_count": len(raw_blocks),
45 "side_blocks_ignored": len(raw_blocks) - len(blocks),
46 "main_chain_block_count": len(blocks),
47 "first_height": 0,
48 "last_height": len(blocks) - 1,
49 "tip_hash": tip["hash"],
50 "tip_time": tip["time"],
51 "tip_time_utc": tip["time_utc"],
52 },
53 "stats": stats,
54 "checks": checks,
55 "warnings": warnings,
56 "verdict": "UTXO_COMMITMENT_VERIFIED" if all(checks.values()) else "CHECKS_FAILED",
57 }
58 if include_utxos:
59 report["utxos"] = {k: utxos[k] for k in sorted(utxos)}
60 return report
61
62
63def print_text(report):
64 data_hash = hashlib.sha256(chain.canonical_bytes(report)).hexdigest()
65 print("Bitflash UTXO-set commitment verifier")
66 print("schema: %s" % report["schema"])
67 print("canonical report sha256: %s" % data_hash)
68 print()
69 print("Commitment")
70 print(" root: %s" % report["commitment"]["root"])
71 print(" algo: %s" % report["commitment"]["algorithm"])
72 print()
73 print("Scanned")
74 print(" main chain blocks: %d" % report["scanned"]["main_chain_block_count"])
75 print(" raw disk blocks: %d" % report["scanned"]["raw_block_count"])
76 print(" side ignored: %d" % report["scanned"]["side_blocks_ignored"])
77 print(" height: %d..%d" % (report["scanned"]["first_height"], report["scanned"]["last_height"]))
78 print(" tip: %s" % report["scanned"]["tip_hash"])
79 print()
80 print("Supply")
81 print(" utxos: %d" % report["stats"]["utxo_count"])
82 print(" supply: %s BTF" % report["stats"]["total_unspent_btf"])
83 print(" mined: %s BTF" % report["stats"]["total_coinbase_btf"])
84 print(" delta: %s BTF" % report["stats"]["coinbase_minus_unspent_btf"])
85 print(" fees: %s BTF" % report["stats"]["total_fees_btf"])
86 if report["warnings"]["legacy_duplicate_output_overwrites"]:
87 print(" legacy duplicate overwrites: %d" % report["warnings"]["legacy_duplicate_output_overwrites"])
88 print()
89 print("Checks")
90 for key in sorted(report["checks"]):
91 print(" %s %s" % ("OK " if report["checks"][key] else "FAIL", key))
92 print()
93 print("Verdict: %s" % report["verdict"])
94 print()
95 print("OpenTimestamps")
96 print(" canonical JSON can be stamped later with: ots stamp <report.json>")
97
98
99def main(argv):
100 ap = argparse.ArgumentParser(description=__doc__)
101 ap.add_argument("--datadir", default=chain.default_datadir(), help="Bitflash data directory")
102 ap.add_argument("--max-blocks", type=int, default=0,
103 help="number of blocks to scan; 0 means all block files")
104 ap.add_argument("--include-utxos", action="store_true",
105 help="include the full sorted UTXO map in JSON output")
106 ap.add_argument("--strict-duplicates", action="store_true",
107 help="fail instead of using legacy last-write-wins duplicate output handling")
108 ap.add_argument("--json", action="store_true", help="print canonical JSON")
109 ap.add_argument("--out", help="write canonical JSON to this path")
110 args = ap.parse_args(argv)
111
112 blocks = chain.read_blocks(args.datadir, args.max_blocks)
113 if not blocks:
114 raise SystemExit("no Bitflash blocks found in %s" % args.datadir)
115 try:
116 report = build_report(blocks, include_utxos=args.include_utxos,
117 strict_duplicates=args.strict_duplicates)
118 except chain.ParseError as e:
119 raise SystemExit(str(e))
120 data = chain.canonical_bytes(report)
121 if args.out:
122 Path(args.out).write_bytes(data)
123 if args.json:
124 sys.stdout.buffer.write(data)
125 else:
126 print_text(report)
127 return 0 if report["verdict"] == "UTXO_COMMITMENT_VERIFIED" else 1
128
129
130if __name__ == "__main__":
131 raise SystemExit(main(sys.argv[1:]))
diff --git a/src/Makefile b/src/Makefile
index 171afe0..c997ae3 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -2,10 +2,13 @@
2# Called by: make linux (root Makefile) 2# Called by: make linux (root Makefile)
3 3
4CXX ?= g++ 4CXX ?= g++
5FUZZ_CXX ?= clang++
5 6
6SECP_INC ?= /usr/local/include 7SECP_INC ?= /usr/local/include
7SECP_LIB ?= /usr/local/lib 8SECP_LIB ?= /usr/local/lib
8RANDOMX_DIR ?= $(HOME)/RandomX 9# The root Makefile exports RANDOMX_DIR (default: <repo>/randomx-build, built
10# by deps-randomx). This fallback only applies when src/ is built standalone.
11RANDOMX_DIR ?= $(abspath $(CURDIR)/..)/randomx-build
9 12
10IMGUI_DIR = imgui 13IMGUI_DIR = imgui
11 14
@@ -29,14 +32,17 @@ WARNFLAGS = -Wformat -Werror=format \
29 -Wno-narrowing -Wno-unused-result 32 -Wno-narrowing -Wno-unused-result
30 33
31CXXFLAGS = -std=gnu++14 -O2 $(WARNFLAGS) -fpermissive -pthread \ 34CXXFLAGS = -std=gnu++14 -O2 $(WARNFLAGS) -fpermissive -pthread \
32 -DSECP256K1_STATIC \ 35 -DSECP256K1_STATIC -DBITFLASH_NODE_BUILD \
33 -I$(SECP_INC) -I$(RANDOMX_DIR)/src \ 36 -I$(SECP_INC) -I$(RANDOMX_DIR)/src \
34 -I/usr/include/nlohmann \ 37 -I/usr/include/nlohmann \
35 -I$(IMGUI_DIR) 38 -I$(IMGUI_DIR)
39FUZZ_CXXFLAGS = $(filter-out -O2,$(CXXFLAGS)) -O1 -g -fno-omit-frame-pointer \
40 -fsanitize=fuzzer-no-link,address,undefined
41FUZZ_LDFLAGS = -fsanitize=fuzzer,address,undefined
36 42
37LIBS = -L$(SECP_LIB) -lsecp256k1 \ 43LIBS = -L$(SECP_LIB) -lsecp256k1 \
38 -L$(RANDOMX_DIR)/build -lrandomx \ 44 -L$(RANDOMX_DIR)/build -lrandomx \
39 -lsodium -lssl -lcrypto -ldb_cxx -lpthread \ 45 -lsodium -lssl -lcrypto -ldb_cxx -lsqlite3 -lpthread \
40 -lglfw -lGL 46 -lglfw -lGL
41 47
42# Every header a .cpp here includes has to be listed, not just the interesting 48# Every header a .cpp here includes has to be listed, not just the interesting
@@ -49,7 +55,8 @@ LIBS = -L$(SECP_LIB) -lsecp256k1 \
49HEADERS = headers_core.h headers.h util.h main.h serialize.h uint256.h \ 55HEADERS = headers_core.h headers.h util.h main.h serialize.h uint256.h \
50 key.h bignum.h script.h db.h base58.h net.h nostr.h \ 56 key.h bignum.h script.h db.h base58.h net.h nostr.h \
51 randomx_pow.h btfaddr.h btfchan.h btfrv.h btftunnel.h \ 57 randomx_pow.h btfaddr.h btfchan.h btfrv.h btftunnel.h \
52 selftest.h walletcmd.h compat.h sockcount.h font_roboto.h 58 selftest.h walletcmd.h wallet_sqlite.h compat.h sockcount.h proxy.h tor.h font_roboto.h \
59 compactblock.h
53 60
54IMGUI_OBJS = obj/imgui.o obj/imgui_draw.o obj/imgui_tables.o \ 61IMGUI_OBJS = obj/imgui.o obj/imgui_draw.o obj/imgui_tables.o \
55 obj/imgui_widgets.o obj/imgui_impl_glfw.o obj/imgui_impl_opengl3.o 62 obj/imgui_widgets.o obj/imgui_impl_glfw.o obj/imgui_impl_opengl3.o
@@ -57,7 +64,8 @@ IMGUI_OBJS = obj/imgui.o obj/imgui_draw.o obj/imgui_tables.o \
57NODE_OBJS = obj/util.o obj/script.o obj/db.o obj/net.o obj/main.o \ 64NODE_OBJS = obj/util.o obj/script.o obj/db.o obj/net.o obj/main.o \
58 obj/sha.o obj/nostr.o obj/randomx_pow.o \ 65 obj/sha.o obj/nostr.o obj/randomx_pow.o \
59 obj/btfaddr.o obj/btfchan.o obj/btfrv.o obj/btftunnel.o obj/rpc.o \ 66 obj/btfaddr.o obj/btfchan.o obj/btfrv.o obj/btftunnel.o obj/rpc.o \
60 obj/selftest.o obj/bip32.o obj/walletcmd.o 67 obj/selftest.o obj/bip32.o obj/walletcmd.o obj/wallet_sqlite.o \
68 obj/proxy.o obj/tor.o obj/compactblock.o
61 69
62GUI_OBJS = obj/gui.o obj/main_gui.o 70GUI_OBJS = obj/gui.o obj/main_gui.o
63 71
@@ -96,7 +104,7 @@ obj/imgui_impl_opengl3.o: $(IMGUI_DIR)/imgui_impl_opengl3.cpp
96 $(CXX) -c $(CXXFLAGS) -o $@ $< 104 $(CXX) -c $(CXXFLAGS) -o $@ $<
97 105
98bitflash: $(IMGUI_OBJS) $(NODE_OBJS) $(GUI_OBJS) 106bitflash: $(IMGUI_OBJS) $(NODE_OBJS) $(GUI_OBJS)
99 $(CXX) $(CXXFLAGS) -o $@ $^ $(LIBS) 107 $(CXX) $(CXXFLAGS) $(LDHARDFLAGS) -o $@ $^ $(LIBS)
100 @echo "Built: bitflash" 108 @echo "Built: bitflash"
101 109
102# ---- headless node --------------------------------------------------------- 110# ---- headless node ---------------------------------------------------------
@@ -110,13 +118,31 @@ bitflash: $(IMGUI_OBJS) $(NODE_OBJS) $(GUI_OBJS)
110# Objects go to obj-node/ because main_gui.cpp is compiled twice, with and 118# Objects go to obj-node/ because main_gui.cpp is compiled twice, with and
111# without -DBITFLASH_NO_GUI, and the two must not overwrite each other. 119# without -DBITFLASH_NO_GUI, and the two must not overwrite each other.
112 120
121# RandomX links a hand-written assembly JIT stub, jit_compiler_x86_static.S,
122# which carries no .note.GNU-stack marker. Older binutils -- Debian 12's, which
123# is what builds against the oldest glibc we ship to -- reads a missing marker
124# as "this object wants an executable stack" and marks the whole binary RWE,
125# switching NX off for the process. Newer toolchains default the other way, so
126# identical source produced GNU_STACK RW on Ubuntu 26.04 and RWE on bookworm,
127# announced by nothing louder than a linker warning in the middle of the build.
128#
129# Assert it here rather than inherit whichever default the build host happens
130# to carry. The JIT maps its own W^X pages and never executes from the stack,
131# so nothing needs the permission this takes away. Check with:
132# readelf -lW bitflash-node | grep -A1 GNU_STACK # expect RW, not RWE
133LDHARDFLAGS = -Wl,-z,noexecstack
134
113NODE_LIBS = -L$(SECP_LIB) -lsecp256k1 \ 135NODE_LIBS = -L$(SECP_LIB) -lsecp256k1 \
114 -L$(RANDOMX_DIR)/build -lrandomx \ 136 -L$(RANDOMX_DIR)/build -lrandomx \
115 -lsodium -lssl -lcrypto -ldb_cxx -lpthread 137 -lsodium -lssl -lcrypto -ldb_cxx -lsqlite3 -lpthread
116 138
117NODE_ONLY_OBJS = $(patsubst obj/%,obj-node/%,$(NODE_OBJS)) \ 139NODE_ONLY_OBJS = $(patsubst obj/%,obj-node/%,$(NODE_OBJS)) \
118 obj-node/main_gui.o obj-node/headless.o 140 obj-node/main_gui.o obj-node/headless.o
119 141
142FUZZ_BASE_OBJS = $(patsubst obj/%,obj-fuzz/%,$(NODE_OBJS))
143FUZZ_NET_MESSAGE_OBJS = $(FUZZ_BASE_OBJS) obj-fuzz/fuzz_net_message.o
144FUZZ_SCRIPT_OBJS = $(FUZZ_BASE_OBJS) obj-fuzz/fuzz_script.o
145
120obj-node/%.o: %.cpp $(HEADERS) 146obj-node/%.o: %.cpp $(HEADERS)
121 @mkdir -p obj-node 147 @mkdir -p obj-node
122 $(CXX) -c $(CXXFLAGS) -DBITFLASH_NO_GUI -o $@ $< 148 $(CXX) -c $(CXXFLAGS) -DBITFLASH_NO_GUI -o $@ $<
@@ -126,21 +152,85 @@ obj-node/sha.o: sha.cpp sha.h
126 $(CXX) -c $(CXXFLAGS) -DBITFLASH_NO_GUI -O3 -o $@ $< 152 $(CXX) -c $(CXXFLAGS) -DBITFLASH_NO_GUI -O3 -o $@ $<
127 153
128bitflash-node: $(NODE_ONLY_OBJS) 154bitflash-node: $(NODE_ONLY_OBJS)
129 $(CXX) $(CXXFLAGS) -o $@ $^ $(NODE_LIBS) 155 $(CXX) $(CXXFLAGS) $(LDHARDFLAGS) -o $@ $^ $(NODE_LIBS)
130 @echo "Built: bitflash-node (headless, no libGL needed)" 156 @echo "Built: bitflash-node (headless, no libGL needed)"
131 157
158obj-fuzz/%.o: %.cpp $(HEADERS)
159 @mkdir -p obj-fuzz
160 $(FUZZ_CXX) -c $(FUZZ_CXXFLAGS) -DBITFLASH_FUZZ -o $@ $<
161
162obj-fuzz/sha.o: sha.cpp sha.h
163 @mkdir -p obj-fuzz
164 $(FUZZ_CXX) -c $(FUZZ_CXXFLAGS) -DBITFLASH_FUZZ -O1 -o $@ $<
165
166obj-fuzz/fuzz_net_message.o: fuzz_net_message.cpp $(HEADERS)
167 @mkdir -p obj-fuzz
168 $(FUZZ_CXX) -c $(FUZZ_CXXFLAGS) -DBITFLASH_FUZZ -o $@ $<
169
170obj-fuzz/fuzz_script.o: fuzz_script.cpp $(HEADERS)
171 @mkdir -p obj-fuzz
172 $(FUZZ_CXX) -c $(FUZZ_CXXFLAGS) -DBITFLASH_FUZZ -o $@ $<
173
174fuzz-net-message: $(FUZZ_NET_MESSAGE_OBJS)
175 $(FUZZ_CXX) $(filter-out -fsanitize=fuzzer-no-link%,$(FUZZ_CXXFLAGS)) \
176 $(FUZZ_LDFLAGS) -o $@ $^ $(NODE_LIBS)
177 @echo "Built: fuzz-net-message"
178
179fuzz-script: $(FUZZ_SCRIPT_OBJS)
180 $(FUZZ_CXX) $(filter-out -fsanitize=fuzzer-no-link%,$(FUZZ_CXXFLAGS)) \
181 $(FUZZ_LDFLAGS) -o $@ $^ $(NODE_LIBS)
182 @echo "Built: fuzz-script"
183
184fuzz-net-message-smoke: fuzz-net-message
185 @mkdir -p ../work/fuzz-corpus/net-message
186 @printf '\277\040\134\375ping\000\000\000\000\000\000\000\000\000\000\000\000' > ../work/fuzz-corpus/net-message/ping-header
187 @printf '\277\040\134\375version\000\000\000\000\000\000\000\000\000' > ../work/fuzz-corpus/net-message/version-header
188 ./fuzz-net-message -runs=128 ../work/fuzz-corpus/net-message
189
190fuzz-script-smoke: fuzz-script
191 @mkdir -p ../work/fuzz-corpus/script
192 @printf '\000\121' > ../work/fuzz-corpus/script/op-true
193 @printf '\001\166\251\024\001\002\003\004\005\006\007\010\011\012\013\014\015\016\017\020\021\022\023\024\210\254' > ../work/fuzz-corpus/script/p2pkh-shape
194 @printf '\000\115\377\377' > ../work/fuzz-corpus/script/truncated-pushdata2
195 ./fuzz-script -runs=128 ../work/fuzz-corpus/script
196
132clean: 197clean:
133 rm -rf obj obj-node bitflash bitflash-node test_bip32 198 rm -rf obj obj-node obj-fuzz bitflash bitflash-node fuzz-net-message fuzz-script test_bip32 test_compactblock
134 199
135test_bip32: test_bip32.cpp bip32.cpp bip32.h bip39_english.h 200test_bip32: test_bip32.cpp bip32.cpp bip32.h bip39_english.h
136 $(CXX) $(CXXFLAGS) -o $@ test_bip32.cpp bip32.cpp -lssl -lcrypto 201 $(CXX) $(CXXFLAGS) -o $@ test_bip32.cpp bip32.cpp -lssl -lcrypto
137 202
138tests: test_bip32 bitflash-node 203# Upstream shipped test_compactblock.cpp with no build rule at all, so it was
204# never compiled there. It links the standalone way test_bip32 does: it supplies
205# its own mapTransactions and GetAppDir so main.cpp stays out of the link, and
206# only needs sha.cpp/util.cpp for Hash() and the logging helpers.
207#
208# TEST_LIBS is overridable for the same reason NODE_LIBS is: the pinned dep tree
209# ships libsodium as a static archive only, so a build against it passes
210# TEST_LIBS="... -l:libsodium.a ...".
211TEST_LIBS ?= -lsodium -lssl -lcrypto -lpthread
212
213test_compactblock: test_compactblock.cpp compactblock.cpp compactblock.h $(HEADERS)
214 $(CXX) $(CXXFLAGS) -o $@ test_compactblock.cpp compactblock.cpp sha.cpp util.cpp \
215 $(TEST_LIBS)
216
217tests: test_bip32 test_compactblock bitflash-node
139 ./test_bip32 218 ./test_bip32
219 ./test_compactblock
140 ./bitflash-node -selftest=wallet-keypool 220 ./bitflash-node -selftest=wallet-keypool
141 ./bitflash-node -selftest=wallet-hd 221 ./bitflash-node -selftest=wallet-hd
142 ./bitflash-node -selftest=wallet-format 222 ./bitflash-node -selftest=wallet-format
223 ./bitflash-node -selftest=wallet-sqlite
224 ./bitflash-node -selftest=wallet-sqlite-migration
143 ./bitflash-node -selftest=wallet-crypto 225 ./bitflash-node -selftest=wallet-crypto
144 ./bitflash-node -selftest=wallet-encrypt 226 ./bitflash-node -selftest=wallet-encrypt
145 ./bitflash-node -selftest=net-message 227 ./bitflash-node -selftest=net-message
228 ./bitflash-node -selftest=network-params
146 ./bitflash-node -selftest=consensus-limits 229 ./bitflash-node -selftest=consensus-limits
230 ./bitflash-node -selftest=pool-stratum
231 ./bitflash-node -selftest=payout-ledger
232 ./bitflash-node -selftest=parse-money
233 ./bitflash-node -selftest=socks5-proxy
234 ./bitflash-node -selftest=managed-tor
235
236.PHONY: tests fuzz-net-message-smoke fuzz-script-smoke
diff --git a/src/bignum.h b/src/bignum.h
index 9da104b..6a295b2 100644
--- a/src/bignum.h
+++ b/src/bignum.h
@@ -155,7 +155,9 @@ public:
155 for (int i = 0; i < 8; i++) 155 for (int i = 0; i < 8; i++)
156 { 156 {
157 unsigned char c = (n >> 56) & 0xff; 157 unsigned char c = (n >> 56) & 0xff;
158 n <<= 8; 158 n = (int64)((uint64)n << 8); // shift in unsigned space; <<= on a
159 // signed int64 with the top byte set
160 // is overflow UB (fuzzer/UBSan finding)
159 if (fLeadingZeroes) 161 if (fLeadingZeroes)
160 { 162 {
161 if (c == 0) 163 if (c == 0)
diff --git a/src/btfaddr.cpp b/src/btfaddr.cpp
index 6edcc91..502220e 100644
--- a/src/btfaddr.cpp
+++ b/src/btfaddr.cpp
@@ -19,6 +19,8 @@
19#include <cstring> 19#include <cstring>
20#include <algorithm> 20#include <algorithm>
21#include <cctype> 21#include <cctype>
22#include <cerrno>
23#include <cstdlib>
22 24
23using json = nlohmann::json; 25using json = nlohmann::json;
24 26
@@ -166,10 +168,47 @@ static bool HexDecode(const std::string& s, unsigned char* out, size_t n)
166 return true; 168 return true;
167} 169}
168 170
171bool NormalizeOnionEndpoint(const std::string& in, std::string& out)
172{
173 out.clear();
174 std::string s = ToLowerTrim(in);
175 size_t colon = s.rfind(':');
176 if (colon == std::string::npos || colon == 0 || colon + 1 >= s.size())
177 return false;
178
179 std::string host = s.substr(0, colon);
180 std::string port = s.substr(colon + 1);
181 while (!host.empty() && host.back() == '.') host.pop_back();
182 static const char* suffix = ".onion";
183 const size_t suffixLen = strlen(suffix);
184 if (host.size() != 56 + suffixLen)
185 return false;
186 if (host.compare(host.size() - suffixLen, suffixLen, suffix) != 0)
187 return false;
188 for (size_t i = 0; i < 56; i++)
189 {
190 char c = host[i];
191 if (!((c >= 'a' && c <= 'z') || (c >= '2' && c <= '7')))
192 return false;
193 }
194 for (size_t i = 0; i < port.size(); i++)
195 if (!isdigit((unsigned char)port[i]))
196 return false;
197
198 errno = 0;
199 char* end = NULL;
200 long nPort = strtol(port.c_str(), &end, 10);
201 if (errno == ERANGE || end == NULL || *end != '\0' || nPort <= 0 || nPort > 65535)
202 return false;
203
204 out = host + ":" + port;
205 return true;
206}
207
169// Digest signed over the descriptor fields (domain-separated). 208// Digest signed over the descriptor fields (domain-separated).
170static void DescriptorDigest(const unsigned char pubkey[32], const std::string& enc, 209static void DescriptorDigestV2(const unsigned char pubkey[32], const std::string& enc,
171 const std::string& meeting_node, uint64_t created, 210 const std::string& meeting_node, uint64_t created,
172 unsigned char out[32]) 211 unsigned char out[32])
173{ 212{
174 static const char* DS = ".btf-descriptor-v2"; 213 static const char* DS = ".btf-descriptor-v2";
175 SHA256_CTX ctx; 214 SHA256_CTX ctx;
@@ -184,9 +223,27 @@ static void DescriptorDigest(const unsigned char pubkey[32], const std::string&
184 SHA256_Final(out, &ctx); 223 SHA256_Final(out, &ctx);
185} 224}
186 225
226static void DescriptorDigestV3(const unsigned char pubkey[32], const std::string& enc,
227 const std::string& meeting_node, const std::string& onion,
228 uint64_t created, unsigned char out[32])
229{
230 static const char* DS = ".btf-descriptor-v3-onion";
231 SHA256_CTX ctx;
232 SHA256_Init(&ctx);
233 SHA256_Update(&ctx, (const unsigned char*)DS, strlen(DS));
234 SHA256_Update(&ctx, pubkey, 32);
235 SHA256_Update(&ctx, (const unsigned char*)enc.data(), enc.size());
236 SHA256_Update(&ctx, (const unsigned char*)meeting_node.data(), meeting_node.size());
237 SHA256_Update(&ctx, (const unsigned char*)onion.data(), onion.size());
238 unsigned char be[8];
239 for (int i = 0; i < 8; i++) be[i] = (unsigned char)((created >> (8 * (7 - i))) & 0xff);
240 SHA256_Update(&ctx, be, 8);
241 SHA256_Final(out, &ctx);
242}
243
187std::string SignDescriptor(void* ctxv, const unsigned char seckey[32], 244std::string SignDescriptor(void* ctxv, const unsigned char seckey[32],
188 const std::string& enc, const std::string& meeting_node, 245 const std::string& enc, const std::string& meeting_node,
189 uint64_t created) 246 uint64_t created, const std::string& onion)
190{ 247{
191 secp256k1_context* ctx = (secp256k1_context*)ctxv; 248 secp256k1_context* ctx = (secp256k1_context*)ctxv;
192 secp256k1_keypair kp; 249 secp256k1_keypair kp;
@@ -197,17 +254,28 @@ std::string SignDescriptor(void* ctxv, const unsigned char seckey[32],
197 secp256k1_xonly_pubkey_serialize(ctx, pubkey, &xpub); 254 secp256k1_xonly_pubkey_serialize(ctx, pubkey, &xpub);
198 255
199 unsigned char digest[32]; 256 unsigned char digest[32];
200 DescriptorDigest(pubkey, enc, meeting_node, created, digest); 257 DescriptorDigestV2(pubkey, enc, meeting_node, created, digest);
201 258
202 unsigned char sig[64]; 259 unsigned char sig[64];
203 // Deterministic signature (NULL aux_rand), matching Itzal's no-aux-rand signing. 260 // Deterministic signature (NULL aux_rand), matching Itzal's no-aux-rand signing.
204 if (!secp256k1_schnorrsig_sign32(ctx, sig, digest, &kp, NULL)) return std::string(); 261 if (!secp256k1_schnorrsig_sign32(ctx, sig, digest, &kp, NULL)) return std::string();
205 262
206 json v = json::object(); 263 json v = json::object();
207 v["v"] = 2; 264 v["v"] = onion.empty() ? 2 : 3;
208 v["pubkey"] = HexEncode(pubkey, 32); 265 v["pubkey"] = HexEncode(pubkey, 32);
209 v["enc"] = enc; 266 v["enc"] = enc;
210 v["meeting_node"] = meeting_node; 267 v["meeting_node"] = meeting_node;
268 if (!onion.empty())
269 {
270 std::string onionNorm;
271 if (!NormalizeOnionEndpoint(onion, onionNorm)) return std::string();
272 unsigned char digest3[32];
273 DescriptorDigestV3(pubkey, enc, meeting_node, onionNorm, created, digest3);
274 unsigned char sig3[64];
275 if (!secp256k1_schnorrsig_sign32(ctx, sig3, digest3, &kp, NULL)) return std::string();
276 v["onion"] = onionNorm;
277 v["sig3"] = HexEncode(sig3, 64);
278 }
211 v["created"] = created; 279 v["created"] = created;
212 v["sig"] = HexEncode(sig, 64); 280 v["sig"] = HexEncode(sig, 64);
213 return v.dump(); 281 return v.dump();
@@ -242,15 +310,47 @@ bool VerifyDescriptor(void* ctxv, const std::string& jsonStr,
242 if (!HexDecode(v["sig"].get<std::string>(), sig, 64)) return false; 310 if (!HexDecode(v["sig"].get<std::string>(), sig, 64)) return false;
243 311
244 unsigned char digest[32]; 312 unsigned char digest[32];
245 DescriptorDigest(expect_pubkey, enc, meeting_node, created, digest); 313 DescriptorDigestV2(expect_pubkey, enc, meeting_node, created, digest);
246 314
247 secp256k1_xonly_pubkey xpub; 315 secp256k1_xonly_pubkey xpub;
248 if (!secp256k1_xonly_pubkey_parse(ctx, &xpub, expect_pubkey)) return false; 316 if (!secp256k1_xonly_pubkey_parse(ctx, &xpub, expect_pubkey)) return false;
249 if (!secp256k1_schnorrsig_verify(ctx, sig, digest, 32, &xpub)) return false; 317 if (!secp256k1_schnorrsig_verify(ctx, sig, digest, 32, &xpub)) return false;
250 318
319 std::string onion;
320 if (v.contains("onion"))
321 {
322 // A descriptor that carries an onion endpoint must prove it. The
323 // endpoint is covered by sig3 and only sig3, so any failure here --
324 // missing sig3, bad hex, unnormalised endpoint, bad signature --
325 // means the descriptor was tampered with, and a tampered
326 // descriptor is rejected outright. Upstream dropped just the onion
327 // and accepted the rest, which let any relay convert an onion
328 // descriptor into a rendezvous-only one by deleting a field.
329 //
330 // Residual, by design: removing BOTH "onion" and "sig3" still
331 // verifies, because the result is indistinguishable from a peer
332 // that never had an onion endpoint. Closing that requires binding
333 // the onion into the primary sig (the V2 digest), which every
334 // upstream 1.2.18/1.2.19 verifier would reject -- an upstream
335 // protocol change, not something this fork can do unilaterally.
336 if (!v["onion"].is_string()) return false;
337 if (!v.contains("sig3") || !v["sig3"].is_string()) return false;
338 std::string onionNorm;
339 std::string onionRaw = v["onion"].get<std::string>();
340 if (!NormalizeOnionEndpoint(onionRaw, onionNorm) || onionNorm != onionRaw)
341 return false;
342 unsigned char sig3[64];
343 unsigned char digest3[32];
344 DescriptorDigestV3(expect_pubkey, enc, meeting_node, onionNorm, created, digest3);
345 if (!HexDecode(v["sig3"].get<std::string>(), sig3, 64)) return false;
346 if (!secp256k1_schnorrsig_verify(ctx, sig3, digest3, 32, &xpub)) return false;
347 onion = onionNorm;
348 }
349
251 memcpy(out.pubkey, expect_pubkey, 32); 350 memcpy(out.pubkey, expect_pubkey, 32);
252 out.enc = enc; 351 out.enc = enc;
253 out.meeting_node = meeting_node; 352 out.meeting_node = meeting_node;
353 out.onion = onion;
254 out.created = created; 354 out.created = created;
255 return true; 355 return true;
256 } 356 }
diff --git a/src/btfaddr.h b/src/btfaddr.h
index 1addf8f..55a3ebd 100644
--- a/src/btfaddr.h
+++ b/src/btfaddr.h
@@ -52,15 +52,21 @@ struct Descriptor
52 unsigned char pubkey[32]; 52 unsigned char pubkey[32];
53 std::string enc; // service x25519 public key (hex) for the E2E channel 53 std::string enc; // service x25519 public key (hex) for the E2E channel
54 std::string meeting_node; // rendezvous node the service is registered at 54 std::string meeting_node; // rendezvous node the service is registered at
55 std::string onion; // optional direct Tor hidden-service endpoint (host.onion:port)
55 uint64_t created; 56 uint64_t created;
56}; 57};
57 58
59// Normalize and validate a direct Tor hidden-service endpoint. Only v3 onion
60// names are accepted: 56 base32 chars plus ".onion", followed by ":PORT".
61bool NormalizeOnionEndpoint(const std::string& in, std::string& out);
62
58// Sign a descriptor. `ctx` is a secp256k1_context* (passed as void* to keep this 63// Sign a descriptor. `ctx` is a secp256k1_context* (passed as void* to keep this
59// header free of secp headers); `seckey` is the 32-byte service secret key. 64// header free of secp headers); `seckey` is the 32-byte service secret key.
60// Returns the descriptor JSON as a string, or "" on failure. 65// Returns the descriptor JSON as a string, or "" on failure.
61std::string SignDescriptor(void* ctx, const unsigned char seckey[32], 66std::string SignDescriptor(void* ctx, const unsigned char seckey[32],
62 const std::string& enc, const std::string& meeting_node, 67 const std::string& enc, const std::string& meeting_node,
63 uint64_t created); 68 uint64_t created,
69 const std::string& onion = std::string());
64 70
65// Verify a descriptor is validly signed by `expect_pubkey` (the key the resolved 71// Verify a descriptor is validly signed by `expect_pubkey` (the key the resolved
66// `.btf` address encodes). Returns true and fills `out` on success. 72// `.btf` address encodes). Returns true and fills `out` on success.
diff --git a/src/btfrv.cpp b/src/btfrv.cpp
index 905717f..7d705ef 100644
--- a/src/btfrv.cpp
+++ b/src/btfrv.cpp
@@ -5,6 +5,9 @@
5// Windows/Winsock implementation (Layer 6 will #ifdef this for Linux/BSD sockets). 5// Windows/Winsock implementation (Layer 6 will #ifdef this for Linux/BSD sockets).
6 6
7#include "btfrv.h" 7#include "btfrv.h"
8#ifdef BITFLASH_NODE_BUILD
9#include "proxy.h"
10#endif
8 11
9// Portable sockets: Winsock on Windows, BSD sockets on Linux/POSIX (for the 12// Portable sockets: Winsock on Windows, BSD sockets on Linux/POSIX (for the
10// headless relay `bitflashd` that runs on the VPS). 13// headless relay `bitflashd` that runs on the VPS).
@@ -37,6 +40,8 @@ typedef int SOCKET;
37#include <thread> 40#include <thread>
38#include <mutex> 41#include <mutex>
39#include <atomic> 42#include <atomic>
43#include <chrono>
44#include <system_error>
40 45
41namespace btf 46namespace btf
42{ 47{
@@ -92,6 +97,11 @@ static bool WriteN(SOCKET s, const void* buf, int n)
92 97
93static SOCKET ConnectTo(const char* host, unsigned short port) 98static SOCKET ConnectTo(const char* host, unsigned short port)
94{ 99{
100#ifdef BITFLASH_NODE_BUILD
101 SOCKET s = BtfConnectSocket(host, port, SOCK_RV_DIAL, 10);
102 if (s == INVALID_SOCKET)
103 return INVALID_SOCKET;
104#else
95 struct addrinfo hints, *res = NULL; 105 struct addrinfo hints, *res = NULL;
96 memset(&hints, 0, sizeof(hints)); 106 memset(&hints, 0, sizeof(hints));
97 hints.ai_family = AF_UNSPEC; // IPv4 or IPv6 (some relays are v6-only) 107 hints.ai_family = AF_UNSPEC; // IPv4 or IPv6 (some relays are v6-only)
@@ -130,6 +140,7 @@ static SOCKET ConnectTo(const char* host, unsigned short port)
130 } 140 }
131 freeaddrinfo(res); 141 freeaddrinfo(res);
132 if (s == INVALID_SOCKET) return INVALID_SOCKET; 142 if (s == INVALID_SOCKET) return INVALID_SOCKET;
143#endif
133 144
134 // Clear the timeout -- the socket is now a live tunnel and must not 145 // Clear the timeout -- the socket is now a live tunnel and must not
135 // time out when idle. recv() will block until data arrives or peer closes. 146 // time out when idle. recv() will block until data arrives or peer closes.
@@ -200,8 +211,28 @@ static void HandleIncoming(SOCKET s)
200 bool a = WriteN(svc, &ok, 1); 211 bool a = WriteN(svc, &ok, 1);
201 bool b = WriteN(s, &ok, 1); 212 bool b = WriteN(s, &ok, 1);
202 if (!a || !b) { CLOSESOCK(svc); CLOSESOCK(s); return; } 213 if (!a || !b) { CLOSESOCK(svc); CLOSESOCK(s); return; }
203 std::thread(Forward, s, svc).detach(); 214 // One thread pumps s -> svc; this thread pumps svc -> s. Spawning a
204 std::thread(Forward, svc, s).detach(); 215 // thread per direction left no clean way to fail the second spawn:
216 // std::thread throws std::system_error when pthread_create is refused
217 // (NPROC limit, container pids cgroup, memory pressure), the first
218 // pump was already detached, and closing its sockets from here would
219 // race it -- so nothing caught the throw and it reached
220 // std::terminate, killing the relay over one pair it could not serve.
221 // Running one direction on this thread leaves exactly one spawn to
222 // guard, and a failure here has nothing running yet to race.
223 try
224 {
225 std::thread(Forward, s, svc).detach();
226 }
227 catch (const std::system_error& e)
228 {
229 fprintf(stderr, "rendezvous relay: cannot spawn forward thread (%s), pair dropped\n",
230 e.what());
231 CLOSESOCK(svc);
232 CLOSESOCK(s);
233 return;
234 }
235 Forward(svc, s);
205 } 236 }
206 else 237 else
207 { 238 {
@@ -227,6 +258,7 @@ bool RvRelayRun(unsigned short port)
227 if (bind(g_listen, (struct sockaddr*)&addr, sizeof(addr)) != 0) { CLOSESOCK(g_listen); return false; } 258 if (bind(g_listen, (struct sockaddr*)&addr, sizeof(addr)) != 0) { CLOSESOCK(g_listen); return false; }
228 if (listen(g_listen, 16) != 0) { CLOSESOCK(g_listen); return false; } 259 if (listen(g_listen, 16) != 0) { CLOSESOCK(g_listen); return false; }
229 260
261 int nSpawnFails = 0;
230 while (!g_stop) 262 while (!g_stop)
231 { 263 {
232 struct sockaddr_in cli; 264 struct sockaddr_in cli;
@@ -237,7 +269,27 @@ bool RvRelayRun(unsigned short port)
237 if (g_stop) break; 269 if (g_stop) break;
238 continue; 270 continue;
239 } 271 }
240 std::thread(HandleIncoming, s).detach(); 272 try
273 {
274 std::thread(HandleIncoming, s).detach();
275 nSpawnFails = 0;
276 }
277 catch (const std::system_error& e)
278 {
279 // A refused pthread_create is backpressure, not a reason to die:
280 // the condition behind it (NPROC, pids cgroup, memory) does not
281 // clear in milliseconds, and the connection count feeding it is
282 // attacker-influenced. Drop this one peer, stop accepting for a
283 // moment -- longer each time, capped -- and keep serving the
284 // connections already established.
285 fprintf(stderr, "rendezvous relay: cannot spawn connection thread (%s), connection dropped\n",
286 e.what());
287 CLOSESOCK(s);
288 nSpawnFails++;
289 int nWaitMs = nSpawnFails < 5 ? nSpawnFails * 1000 : 5000;
290 for (int i = 0; i < nWaitMs && !g_stop; i += 100)
291 std::this_thread::sleep_for(std::chrono::milliseconds(100));
292 }
241 } 293 }
242 CLOSESOCK(g_listen); 294 CLOSESOCK(g_listen);
243 g_listen = INVALID_SOCKET; 295 g_listen = INVALID_SOCKET;
@@ -325,17 +377,29 @@ bool RvServiceWaitPaired(RvSocket s, int nTimeoutSecs)
325} 377}
326 378
327RvSocket RvClientConnect(const char* relay_host, unsigned short port, 379RvSocket RvClientConnect(const char* relay_host, unsigned short port,
328 const unsigned char target_pubkey[32]) 380 const unsigned char target_pubkey[32],
381 RvClientFail* pFail)
329{ 382{
383 if (pFail) *pFail = RVC_FAIL_DIAL;
330 if (!RvInit()) return RV_INVALID; 384 if (!RvInit()) return RV_INVALID;
331 SOCKET s = ConnectTo(relay_host, port); 385 SOCKET s = ConnectTo(relay_host, port);
332 if (s == INVALID_SOCKET) return RV_INVALID; 386 if (s == INVALID_SOCKET) return RV_INVALID;
387 if (pFail) *pFail = RVC_FAIL_PAIR;
333 unsigned char hdr[33]; 388 unsigned char hdr[33];
334 hdr[0] = (unsigned char)'C'; 389 hdr[0] = (unsigned char)'C';
335 memcpy(hdr + 1, target_pubkey, 32); 390 memcpy(hdr + 1, target_pubkey, 32);
336 if (!WriteN(s, hdr, 33)) { CLOSESOCK(s); return RV_INVALID; } 391 if (!WriteN(s, hdr, 33)) { CLOSESOCK(s); return RV_INVALID; }
337 unsigned char paired = 0; 392 unsigned char paired = 0;
338 if (!ReadN(s, &paired, 1) || paired != 0x01) { CLOSESOCK(s); return RV_INVALID; } 393 if (!ReadN(s, &paired, 1)) { CLOSESOCK(s); return RV_INVALID; }
394 if (paired != 0x01)
395 {
396 // The relay answered and declined: HandleIncoming writes 0x00 and
397 // closes when no service is parked under this pubkey right now.
398 if (pFail) *pFail = RVC_FAIL_REFUSED;
399 CLOSESOCK(s);
400 return RV_INVALID;
401 }
402 if (pFail) *pFail = RVC_FAIL_NONE;
339 return (RvSocket)s; 403 return (RvSocket)s;
340} 404}
341 405
diff --git a/src/btfrv.h b/src/btfrv.h
index 9cd7476..1e85cb0 100644
--- a/src/btfrv.h
+++ b/src/btfrv.h
@@ -57,10 +57,25 @@ RvSocket RvServiceRegister(const char* relay_host, unsigned short port,
57bool RvServiceWaitPaired(RvSocket s, int nTimeoutSecs = 0); 57bool RvServiceWaitPaired(RvSocket s, int nTimeoutSecs = 0);
58 58
59// ---- client side (the node dialing a .btf address) ---- 59// ---- client side (the node dialing a .btf address) ----
60// Where a failed RvClientConnect stopped. RV_INVALID alone collapses three
61// different situations -- relay unreachable, relay reachable but nobody
62// registered, connection died mid-pairing -- and they point at different
63// components. The caller can only report the right one if this side says
64// which it saw.
65enum RvClientFail
66{
67 RVC_FAIL_NONE = 0,
68 RVC_FAIL_DIAL, // TCP connect to the relay itself failed
69 RVC_FAIL_REFUSED, // relay answered 0x00: no service registered under this pubkey
70 RVC_FAIL_PAIR, // relay connection died before pairing completed
71};
72
60// Connect through the relay to the service registered under `target_pubkey`. 73// Connect through the relay to the service registered under `target_pubkey`.
61// Returns the paired socket, or RV_INVALID on error. 74// Returns the paired socket, or RV_INVALID on error -- with `pFail`, if
75// given, naming the stage that failed.
62RvSocket RvClientConnect(const char* relay_host, unsigned short port, 76RvSocket RvClientConnect(const char* relay_host, unsigned short port,
63 const unsigned char target_pubkey[32]); 77 const unsigned char target_pubkey[32],
78 RvClientFail* pFail = 0);
64 79
65// Blocking read/write of exactly n bytes over a rendezvous socket. Return false 80// Blocking read/write of exactly n bytes over a rendezvous socket. Return false
66// on error/EOF. 81// on error/EOF.
diff --git a/src/btftunnel.cpp b/src/btftunnel.cpp
index 63d178b..4058f6b 100644
--- a/src/btftunnel.cpp
+++ b/src/btftunnel.cpp
@@ -25,8 +25,12 @@ typedef int SOCKET;
25#endif 25#endif
26#include "sockcount.h" 26#include "sockcount.h"
27#include <cstring> 27#include <cstring>
28#include <cstdio>
29#include <memory>
30#include <new>
28#include <vector> 31#include <vector>
29#include <thread> 32#include <thread>
33#include <system_error>
30 34
31namespace btf 35namespace btf
32{ 36{
@@ -119,35 +123,155 @@ static void PumpIn(SOCKET pumpEnd, RvSocket rv, std::vector<unsigned char> key)
119 shutdown((SOCKET)rv, SD_BOTH); 123 shutdown((SOCKET)rv, SD_BOTH);
120} 124}
121 125
126// The pump end and the rendezvous socket, owned jointly by whoever is still
127// using them. Closed once, by the last reference to go.
128//
129// Joint ownership rather than "the inbound pump owns the cleanup", because
130// either pump can reach its exit first and BOTH touch BOTH descriptors on the
131// way out -- PumpOut ends in shutdown(rv), shutdown(pumpEnd) and PumpIn ends in
132// the mirror image. A single owner therefore has no exit at which the other is
133// guaranteed to be finished. Holding a reference for the whole body of each
134// pump gives it one: the close cannot happen until the last pump has stopped
135// touching the descriptors, so no shutdown() can land on a number that has
136// already been closed and handed back out by the kernel to an unrelated
137// connection.
138struct TunnelEnds
139{
140 SOCKET pumpEnd;
141 RvSocket rv;
142 TunnelEnds(SOCKET p, RvSocket r) : pumpEnd(p), rv(r) {}
143 ~TunnelEnds() { BtfCloseSocket(pumpEnd); RvClose(rv); }
144private:
145 TunnelEnds(const TunnelEnds&);
146 TunnelEnds& operator=(const TunnelEnds&);
147};
148
122// Start the two pumps and hand back the app-facing socket. When both pumps 149// Start the two pumps and hand back the app-facing socket. When both pumps
123// finish, close the pump end and the rendezvous socket. 150// finish, close the pump end and the rendezvous socket.
151//
152// That sentence used to be a comment describing something the code did not do.
153// The inbound pump closed both descriptors as soon as it exited, without
154// waiting for the outbound pump, which then called shutdown() on both at its
155// own exit. Whether that shutdown hit a live descriptor or a closed one came
156// down to which pump got there first, and the rendezvous socket dying before
157// the app socket -- a relay restart, the far side hanging up -- is precisely
158// the order that puts the inbound pump ahead. Measured over 25 such teardowns
159// against the real code: 29 shutdowns landed on an already-closed descriptor.
160//
161// appEnd is not part of this. It is handed to the caller on success and closed
162// here on every failure path; no pump ever touches it.
163//
164// Every tunnel is two threads, and a tunnel is one peer connection (a .btf
165// dial or accept, a pool miner, a participant's pool session) -- so the
166// thread count scales with peer count, which is attacker-influenced. When
167// pthread_create is refused std::thread throws std::system_error; uncaught it
168// reached std::terminate and took the whole process down over one connection
169// it could not serve. Both spawns are guarded, and the failure paths return
170// INVALID_SOCKET, which every caller already handles by dropping just that
171// connection.
124static btf_socket_t StartTunnel(RvSocket rv, const unsigned char shared[32]) 172static btf_socket_t StartTunnel(RvSocket rv, const unsigned char shared[32])
125{ 173{
126 btf_socket_t appEnd, pumpEnd; 174 btf_socket_t appEnd, pumpEnd;
127 if (!MakeLocalPair(appEnd, pumpEnd)) { RvClose(rv); return INVALID_SOCKET; } 175 if (!MakeLocalPair(appEnd, pumpEnd)) { RvClose(rv); return INVALID_SOCKET; }
128 176
129 std::vector<unsigned char> key(shared, shared + 32); 177 std::vector<unsigned char> key(shared, shared + 32);
130 std::thread(PumpOut, pumpEnd, rv, key).detach(); 178
131 std::thread([pumpEnd, rv, key]() { 179 // make_shared, not shared_ptr(new ...): it allocates before it constructs,
132 PumpIn(pumpEnd, rv, key); 180 // so it either comes back owning both descriptors or throws having taken
181 // ownership of neither. The two-step form can run the deleter and then
182 // throw, which would close both here and again below.
183 std::shared_ptr<TunnelEnds> ends;
184 try
185 {
186 ends = std::make_shared<TunnelEnds>(pumpEnd, rv);
187 }
188 catch (const std::bad_alloc&)
189 {
190 fprintf(stderr, "btf tunnel: out of memory starting tunnel, tunnel dropped\n");
133 BtfCloseSocket(pumpEnd); 191 BtfCloseSocket(pumpEnd);
192 BtfCloseSocket(appEnd);
134 RvClose(rv); 193 RvClose(rv);
135 }).detach(); 194 return INVALID_SOCKET;
195 }
196
197 try
198 {
199 std::thread([ends, key]() { PumpIn(ends->pumpEnd, ends->rv, key); }).detach();
200 }
201 catch (const std::system_error& e)
202 {
203 // Nothing is running yet: `ends` is the only reference, and returning
204 // drops it, which is what closes pumpEnd and rv.
205 fprintf(stderr, "btf tunnel: cannot spawn inbound pump (%s), tunnel dropped\n",
206 e.what());
207 BtfCloseSocket(appEnd);
208 return INVALID_SOCKET;
209 }
210
211 try
212 {
213 std::thread([ends, key]() { PumpOut(ends->pumpEnd, ends->rv, key); }).detach();
214 }
215 catch (const std::system_error& e)
216 {
217 // The inbound pump may be parked in RvReadN(rv) or may already have
218 // exited -- when the spawn is refused because the process is loaded,
219 // the rendezvous socket is often dead for the same reason, and then
220 // PumpIn returns on its first length read. Either way this thread
221 // still holds a reference, so both descriptors are still open: the
222 // shutdown wakes the pump if it is parked and is harmless if it is
223 // not. The close belongs to whichever reference goes last.
224 fprintf(stderr, "btf tunnel: cannot spawn outbound pump (%s), tunnel dropped\n",
225 e.what());
226 shutdown((SOCKET)ends->rv, SD_BOTH);
227 shutdown(ends->pumpEnd, SD_BOTH);
228 BtfCloseSocket(appEnd);
229 return INVALID_SOCKET;
230 }
136 return appEnd; 231 return appEnd;
137} 232}
138 233
234const char* TunnelFailText(TunnelFail f)
235{
236 switch (f)
237 {
238 case TUNNEL_FAIL_NONE: return "no failure";
239 case TUNNEL_FAIL_INIT: return "local socket/crypto init failed";
240 case TUNNEL_FAIL_DIAL: return "meeting node unreachable (tcp dial failed)";
241 case TUNNEL_FAIL_REFUSED: return "relay reachable, but no service registered there for this address";
242 case TUNNEL_FAIL_PAIR: return "relay connection died before pairing completed";
243 case TUNNEL_FAIL_HANDSHAKE: return "end-to-end handshake failed";
244 case TUNNEL_FAIL_LOCAL: return "local tunnel setup failed";
245 }
246 return "unknown failure";
247}
248
139btf_socket_t BtfClientTunnel(const char* meeting_host, unsigned short port, 249btf_socket_t BtfClientTunnel(const char* meeting_host, unsigned short port,
140 const unsigned char target_pubkey[32], 250 const unsigned char target_pubkey[32],
141 const unsigned char service_enc_pub[32]) 251 const unsigned char service_enc_pub[32],
252 TunnelFail* pFail)
142{ 253{
254 if (pFail) *pFail = TUNNEL_FAIL_INIT;
143 if (!TunnelInit()) return INVALID_SOCKET; 255 if (!TunnelInit()) return INVALID_SOCKET;
144 RvSocket rv = RvClientConnect(meeting_host, port, target_pubkey); 256 RvClientFail rvFail = RVC_FAIL_NONE;
145 if (rv == RV_INVALID) return INVALID_SOCKET; 257 RvSocket rv = RvClientConnect(meeting_host, port, target_pubkey, &rvFail);
258 if (rv == RV_INVALID)
259 {
260 if (pFail)
261 *pFail = rvFail == RVC_FAIL_DIAL ? TUNNEL_FAIL_DIAL
262 : rvFail == RVC_FAIL_REFUSED ? TUNNEL_FAIL_REFUSED
263 : TUNNEL_FAIL_PAIR;
264 return INVALID_SOCKET;
265 }
146 266
147 unsigned char eph_pub[32], shared[32]; 267 unsigned char eph_pub[32], shared[32];
268 if (pFail) *pFail = TUNNEL_FAIL_HANDSHAKE;
148 if (!ChanClientHandshake(service_enc_pub, eph_pub, shared)) { RvClose(rv); return INVALID_SOCKET; } 269 if (!ChanClientHandshake(service_enc_pub, eph_pub, shared)) { RvClose(rv); return INVALID_SOCKET; }
149 if (!RvWriteN(rv, eph_pub, 32)) { RvClose(rv); return INVALID_SOCKET; } 270 if (!RvWriteN(rv, eph_pub, 32)) { RvClose(rv); return INVALID_SOCKET; }
150 return StartTunnel(rv, shared); 271 if (pFail) *pFail = TUNNEL_FAIL_LOCAL;
272 btf_socket_t app = StartTunnel(rv, shared);
273 if (app != INVALID_SOCKET && pFail) *pFail = TUNNEL_FAIL_NONE;
274 return app;
151} 275}
152 276
153btf_socket_t BtfServiceWrap(RvSocket rv, const unsigned char my_enc_sk[32]) 277btf_socket_t BtfServiceWrap(RvSocket rv, const unsigned char my_enc_sk[32])
diff --git a/src/btftunnel.h b/src/btftunnel.h
index 24a7509..3cc87cd 100644
--- a/src/btftunnel.h
+++ b/src/btftunnel.h
@@ -36,12 +36,33 @@ namespace btf
36// Initialize (sockets + libsodium). Idempotent. 36// Initialize (sockets + libsodium). Idempotent.
37bool TunnelInit(); 37bool TunnelInit();
38 38
39// Which stage of BtfClientTunnel failed. INVALID_SOCKET alone says only "no
40// tunnel"; a caller that then guesses at causes in its log line sends whoever
41// reads it to the wrong component (issue #17: "relay unreachable or pool not
42// listening" blamed two healthy parts). The stage is what this side actually
43// knows -- report that.
44enum TunnelFail
45{
46 TUNNEL_FAIL_NONE = 0,
47 TUNNEL_FAIL_INIT, // local socket/crypto init failed
48 TUNNEL_FAIL_DIAL, // TCP connect to the meeting node failed
49 TUNNEL_FAIL_REFUSED, // relay reachable, but no service registered under the pubkey
50 TUNNEL_FAIL_PAIR, // relay connection died before pairing completed
51 TUNNEL_FAIL_HANDSHAKE, // end-to-end channel handshake failed
52 TUNNEL_FAIL_LOCAL, // local pump plumbing (socket pair / threads) failed
53};
54
55// Short description of a TunnelFail for log lines.
56const char* TunnelFailText(TunnelFail f);
57
39// Client: dial the service registered under `target_pubkey` at the meeting node 58// Client: dial the service registered under `target_pubkey` at the meeting node
40// (host:port), box to its static x25519 `service_enc_pub`, and return a local 59// (host:port), box to its static x25519 `service_enc_pub`, and return a local
41// plaintext SOCKET tunneled end-to-end to the service. INVALID_SOCKET on error. 60// plaintext SOCKET tunneled end-to-end to the service. INVALID_SOCKET on error
61// -- with `pFail`, if given, naming the stage that failed.
42btf_socket_t BtfClientTunnel(const char* meeting_host, unsigned short port, 62btf_socket_t BtfClientTunnel(const char* meeting_host, unsigned short port,
43 const unsigned char target_pubkey[32], 63 const unsigned char target_pubkey[32],
44 const unsigned char service_enc_pub[32]); 64 const unsigned char service_enc_pub[32],
65 TunnelFail* pFail = 0);
45 66
46// Service: wrap an already-paired rendezvous socket (RvServiceRegister followed 67// Service: wrap an already-paired rendezvous socket (RvServiceRegister followed
47// by RvServiceWaitPaired -- the pairing byte MUST already be consumed) with 68// by RvServiceWaitPaired -- the pairing byte MUST already be consumed) with
diff --git a/src/compactblock.cpp b/src/compactblock.cpp
new file mode 100644
index 0000000..1ca6118
--- /dev/null
+++ b/src/compactblock.cpp
@@ -0,0 +1,175 @@
1// Copyright (c) 2026 Bitflash developers
2// Distributed under the MIT/X11 software license.
3//
4// See compactblock.h.
5
6#include "headers.h"
7#include "compactblock.h"
8#include <sodium.h>
9#include <map>
10
11uint256 CCmpctBlock::HeaderHash() const
12{
13 // Same technique CBlock::GetHash() uses: the header fields are laid out
14 // contiguously and in the same order here as in CBlock, so hashing the
15 // raw member range gives the identical block-identity hash.
16 return Hash(BEGIN(nVersion), END(nNonce));
17}
18
19uint64 ComputeShortTxId(const uint256& hashPrevBlock, uint64 shortIdNonce, const uint256& txid)
20{
21 // Derive a 16-byte SipHash key from (hashPrevBlock, shortIdNonce), so
22 // short IDs are specific to this block: they can't be precomputed
23 // before the block (and its freshly-chosen nonce) exist, and the same
24 // transaction gets a different short ID in every block it appears in
25 // (relevant across reorgs/re-announcements).
26 unsigned char keyMaterial[32 + sizeof(uint64)];
27 memcpy(keyMaterial, BEGIN(hashPrevBlock), 32);
28 memcpy(keyMaterial + 32, &shortIdNonce, sizeof(uint64));
29 uint256 keyHash = Hash((const char*)keyMaterial, (const char*)keyMaterial + sizeof(keyMaterial));
30
31 unsigned char out[crypto_shorthash_BYTES]; // 8 bytes
32 crypto_shorthash(out, (const unsigned char*)BEGIN(txid), 32, (const unsigned char*)BEGIN(keyHash));
33
34 uint64 id = 0;
35 memcpy(&id, out, sizeof(uint64));
36 return id & 0xFFFFFFFFFFFFULL; // low 48 bits -- matches the wire field width
37}
38
39bool BuildCompactBlock(const CBlock& block, CCmpctBlock& out)
40{
41 if (block.vtx.empty())
42 return false; // every real block has at least the coinbase
43
44 out.nVersion = block.nVersion;
45 out.hashPrevBlock = block.hashPrevBlock;
46 out.hashMerkleRoot = block.hashMerkleRoot;
47 out.nTime = block.nTime;
48 out.nBits = block.nBits;
49 out.nNonce = block.nNonce;
50
51 // Upstream ignored the RAND_bytes return value here. On failure nonceBytes
52 // is left uninitialised and an indeterminate shortIdNonce goes out on the
53 // wire -- which is precisely the value this file relies on being
54 // unpredictable to stop short IDs being precomputed or ground against.
55 // Fail closed instead, matching every other RAND_bytes call site in this
56 // tree (bip32.cpp, db.cpp): the caller falls back to full-block relay,
57 // which is always correct.
58 unsigned char nonceBytes[sizeof(uint64)];
59 if (RAND_bytes(nonceBytes, sizeof(nonceBytes)) != 1)
60 return false;
61 memcpy(&out.shortIdNonce, nonceBytes, sizeof(uint64));
62
63 out.vPrefilledIndex.clear();
64 out.vPrefilledTx.clear();
65 out.vShortTxId.clear();
66
67 // Only the coinbase is prefilled (index 0) -- see file comment in
68 // compactblock.h for why that's enough for this chain's scale.
69 out.vPrefilledIndex.push_back(0);
70 out.vPrefilledTx.push_back(block.vtx[0]);
71
72 for (size_t i = 1; i < block.vtx.size(); i++)
73 out.vShortTxId.push_back(ComputeShortTxId(out.hashPrevBlock, out.shortIdNonce, block.vtx[i].GetHash()));
74
75 return true;
76}
77
78bool TryReconstructBlock(const CCmpctBlock& cmpct, CBlock& outBlock, std::vector<unsigned int>& vMissingIndex)
79{
80 vMissingIndex.clear();
81
82 size_t nTotal = cmpct.vPrefilledIndex.size() + cmpct.vShortTxId.size();
83 if (cmpct.vPrefilledIndex.size() != cmpct.vPrefilledTx.size() || nTotal == 0)
84 return false; // malformed
85
86 // nTotal comes straight off the wire once this is wired into the P2P
87 // layer, and serialize.h only bounds a vector length by MAX_SIZE
88 // (0x02000000), so an unbounded assign() below would let one 60-odd-byte
89 // message ask this node for ~33M default-constructed CTransactions. The
90 // smallest a transaction can serialise to is 10 bytes (nVersion, empty
91 // vin, empty vout, nLockTime), so no block within MAX_BLOCK_SIZE can hold
92 // more than MAX_BLOCK_SIZE/10 of them; anything above that is not a block
93 // we could ever accept. Rejecting here just falls back to full-block
94 // relay, which is always correct.
95 if (nTotal > MAX_BLOCK_SIZE / 10)
96 return false; // implausible tx count -- refuse to size a vector from it
97
98 outBlock.SetNull();
99 outBlock.nVersion = cmpct.nVersion;
100 outBlock.hashPrevBlock = cmpct.hashPrevBlock;
101 outBlock.hashMerkleRoot = cmpct.hashMerkleRoot;
102 outBlock.nTime = cmpct.nTime;
103 outBlock.nBits = cmpct.nBits;
104 outBlock.nNonce = cmpct.nNonce;
105 outBlock.vtx.assign(nTotal, CTransaction());
106
107 std::vector<bool> vFilled(nTotal, false);
108 for (size_t i = 0; i < cmpct.vPrefilledIndex.size(); i++)
109 {
110 unsigned int idx = cmpct.vPrefilledIndex[i];
111 if (idx >= nTotal)
112 return false; // malformed
113 outBlock.vtx[idx] = cmpct.vPrefilledTx[i];
114 vFilled[idx] = true;
115 }
116
117 // Index every mempool tx by its short ID under this block's keying, so
118 // resolving each block position is a map lookup rather than a linear
119 // scan of the mempool per short ID.
120 std::map<uint64, const CTransaction*> mapShortIdToTx;
121 for (std::map<uint256, CTransaction>::const_iterator mi = mapTransactions.begin();
122 mi != mapTransactions.end(); ++mi)
123 {
124 uint64 sid = ComputeShortTxId(cmpct.hashPrevBlock, cmpct.shortIdNonce, mi->first);
125 // A short-ID collision between two distinct mempool transactions is
126 // possible (48 bits, birthday bound ~2^24 mempool entries) though
127 // very unlikely at this chain's scale; on collision, prefer keeping
128 // neither mapped rather than risk resolving to the wrong tx --
129 // that position just falls through to getblocktxn like a genuine
130 // miss, which is always correct, only occasionally not optimal.
131 if (mapShortIdToTx.count(sid))
132 mapShortIdToTx[sid] = NULL;
133 else
134 mapShortIdToTx[sid] = &mi->second;
135 }
136
137 size_t shortIdPos = 0;
138 for (size_t idx = 0; idx < nTotal; idx++)
139 {
140 if (vFilled[idx])
141 continue;
142 if (shortIdPos >= cmpct.vShortTxId.size())
143 return false; // malformed: more unfilled positions than short IDs
144 uint64 sid = cmpct.vShortTxId[shortIdPos++];
145
146 std::map<uint64, const CTransaction*>::const_iterator found = mapShortIdToTx.find(sid);
147 if (found != mapShortIdToTx.end() && found->second != NULL)
148 {
149 outBlock.vtx[idx] = *found->second;
150 vFilled[idx] = true;
151 }
152 else
153 {
154 vMissingIndex.push_back(idx);
155 }
156 }
157
158 return vMissingIndex.empty();
159}
160
161bool FillMissingAndFinish(CBlock& outBlock, const std::vector<unsigned int>& vMissingIndex,
162 const std::vector<CTransaction>& vTxn)
163{
164 if (vMissingIndex.size() != vTxn.size())
165 return false; // peer didn't answer what was asked
166
167 for (size_t i = 0; i < vMissingIndex.size(); i++)
168 {
169 unsigned int idx = vMissingIndex[i];
170 if (idx >= outBlock.vtx.size())
171 return false; // malformed
172 outBlock.vtx[idx] = vTxn[i];
173 }
174 return true;
175}
diff --git a/src/compactblock.h b/src/compactblock.h
new file mode 100644
index 0000000..745ee3b
--- /dev/null
+++ b/src/compactblock.h
@@ -0,0 +1,113 @@
1// Copyright (c) 2026 Bitflash developers
2// Distributed under the MIT/X11 software license.
3//
4// Compact block relay (BIP152-style). Instead of sending a peer's full set
5// of transactions for a block they're about to receive -- almost all of
6// which they already have in their mempool from normal tx relay -- send a
7// short (48-bit, keyed-hash) ID per transaction instead. A peer that already
8// has every referenced transaction reconstructs the full block locally from
9// its mempool without any of the transaction bytes crossing the wire at
10// all; only genuinely missing transactions (typically just the coinbase)
11// need a follow-up round trip.
12//
13// Compatibility: this is purely additive. A peer that never sends
14// "sendcmpct" is never sent a compact block -- it keeps getting full
15// "block" messages exactly as before (see the getdata handler in
16// main.cpp). Old, unupgraded nodes on the live network are completely
17// unaffected; ProcessMessage's unknown-command handling means they simply
18// never see the new message types. No coordinated network upgrade needed.
19//
20// Deliberately scoped down from full BIP152: no high-bandwidth (unsolicited
21// push) mode, no differential-index encoding for prefilled transactions,
22// only the current chain tip is ever offered in compact form (an old block
23// is unlikely to still overlap much with any peer's current mempool, so the
24// bandwidth win doesn't apply -- full "block" is sent for those, same as
25// today). Only the coinbase is prefilled; every other transaction position
26// carries a short ID.
27
28#ifndef BITFLASH_COMPACTBLOCK_H
29#define BITFLASH_COMPACTBLOCK_H
30
31// Requires CBlock/CTransaction already visible -- #include "headers.h" (or
32// main.h) before this file. Deliberately doesn't include main.h itself:
33// main.h has no include guard (matches this codebase's existing headers, it
34// relies on being pulled in exactly once via headers.h), so a second
35// #include "main.h" here would break every translation unit that already
36// brought it in via headers.h first.
37#include <vector>
38
39// The compact representation of a block: header fields, a per-block nonce
40// (keys the short-ID hash so IDs can't be precomputed/ground across blocks),
41// the coinbase in full, and a 48-bit short ID for every other transaction
42// in block order.
43class CCmpctBlock
44{
45public:
46 // header
47 int nVersion;
48 uint256 hashPrevBlock;
49 uint256 hashMerkleRoot;
50 unsigned int nTime;
51 unsigned int nBits;
52 unsigned int nNonce;
53
54 // Keys the short-ID hash for this block specifically -- chosen fresh
55 // per block (not reused from nNonce, which is public PoW data an
56 // attacker already knows and could otherwise grind against).
57 uint64 shortIdNonce;
58
59 // Sent in full. In this implementation: always exactly the coinbase,
60 // at index 0.
61 std::vector<unsigned int> vPrefilledIndex;
62 std::vector<CTransaction> vPrefilledTx;
63
64 // One short ID per NON-prefilled transaction, in ascending block-index
65 // order (i.e. the Nth entry here belongs to the Nth block position that
66 // isn't in vPrefilledIndex).
67 std::vector<uint64> vShortTxId;
68
69 IMPLEMENT_SERIALIZE
70 (
71 READWRITE(this->nVersion);
72 nVersion = this->nVersion;
73 READWRITE(hashPrevBlock);
74 READWRITE(hashMerkleRoot);
75 READWRITE(nTime);
76 READWRITE(nBits);
77 READWRITE(nNonce);
78 READWRITE(shortIdNonce);
79 READWRITE(vPrefilledIndex);
80 READWRITE(vPrefilledTx);
81 READWRITE(vShortTxId);
82 )
83
84 uint256 HeaderHash() const;
85};
86
87// Compute the 48-bit short ID for a transaction under a given block's
88// short-ID nonce. Keyed (via the block's hashPrevBlock + shortIdNonce) so
89// IDs from one block can't be precomputed before that block exists, and
90// differ block to block even for the same transaction.
91uint64 ComputeShortTxId(const uint256& hashPrevBlock, uint64 shortIdNonce, const uint256& txid);
92
93// Build the compact form of `block` (which must have vtx[0] as the
94// coinbase, as every block on this chain does).
95bool BuildCompactBlock(const CBlock& block, CCmpctBlock& out);
96
97// Attempt to reconstruct a full block from a received compact block using
98// the local mempool. On full success, returns true and fills `outBlock`.
99// On partial success (some transactions not found locally), returns false
100// and fills `vMissingIndex` with the block-position indices that still need
101// to be requested via "getblocktxn" -- outBlock is left partially built
102// (coinbase and any resolved positions filled; caller must not use it until
103// FillMissingAndFinish completes it).
104bool TryReconstructBlock(const CCmpctBlock& cmpct, CBlock& outBlock, std::vector<unsigned int>& vMissingIndex);
105
106// Complete a partially-reconstructed block (from a prior TryReconstructBlock
107// that returned false) using the transactions received in a "blocktxn"
108// response. vTxn must be in the same order as the indices this node
109// requested via "getblocktxn" for this block.
110bool FillMissingAndFinish(CBlock& outBlock, const std::vector<unsigned int>& vMissingIndex,
111 const std::vector<CTransaction>& vTxn);
112
113#endif
diff --git a/src/db.cpp b/src/db.cpp
index 6f9461b..bf1958d 100644
--- a/src/db.cpp
+++ b/src/db.cpp
@@ -3,6 +3,7 @@
3// file license.txt or http://www.opensource.org/licenses/mit-license.php. 3// file license.txt or http://www.opensource.org/licenses/mit-license.php.
4 4
5#include "headers.h" 5#include "headers.h"
6#include "wallet_sqlite.h"
6 7
7 8
8 9
@@ -52,8 +53,34 @@ static bool fDbEnvInit = false;
52// Why LoadWallet() refused, in words meant for the person who has to act on it. 53// Why LoadWallet() refused, in words meant for the person who has to act on it.
53// Empty unless the reason is one we can name. 54// Empty unless the reason is one we can name.
54string strWalletLoadError; 55string strWalletLoadError;
55DbEnv dbenv(0u); 56static DbEnv* pdbenv = NULL;
57static FILE* pdbErrFile = NULL;
56static map<string, int> mapFileUseCount; 58static map<string, int> mapFileUseCount;
59static CWalletDBSQLite* pWalletSQLiteRuntime = NULL;
60
61DbEnv& GetDbEnv()
62{
63 if (!pdbenv)
64 pdbenv = new DbEnv(0u);
65 return *pdbenv;
66}
67
68static void CloseDbEnv()
69{
70 if (pdbenv)
71 {
72 try { pdbenv->close(0); }
73 catch (...) { }
74 delete pdbenv;
75 pdbenv = NULL;
76 }
77 if (pdbErrFile)
78 {
79 fclose(pdbErrFile);
80 pdbErrFile = NULL;
81 }
82 fDbEnvInit = false;
83}
57 84
58class CDBInit 85class CDBInit
59{ 86{
@@ -64,15 +91,189 @@ public:
64 ~CDBInit() 91 ~CDBInit()
65 { 92 {
66 if (fDbEnvInit) 93 if (fDbEnvInit)
67 { 94 CloseDbEnv();
68 try { dbenv.close(0); }
69 catch (...) { }
70 fDbEnvInit = false;
71 }
72 } 95 }
73} 96}
74instance_of_cdbinit; 97instance_of_cdbinit;
75 98
99bool WalletSQLiteRuntimeActive()
100{
101 return pWalletSQLiteRuntime != NULL;
102}
103
104bool WalletSQLiteRuntimeOpen(const string& strPath, string& strErrorRet, bool fMustExist)
105{
106 strErrorRet.clear();
107 // Callers that load an existing wallet want the clear "does not exist"
108 // message rather than silently creating a blank one; the create path passes
109 // fMustExist=false so Open() (SQLITE_OPEN_CREATE) can make a fresh file.
110 if (fMustExist && !FileExists(strPath.c_str()))
111 {
112 strErrorRet = strprintf("SQLite wallet does not exist: %s",
113 strPath.c_str());
114 return false;
115 }
116
117 WalletSQLiteRuntimeClose();
118 std::auto_ptr<CWalletDBSQLite> db(new CWalletDBSQLite());
119 if (!db->Open(strPath, strErrorRet))
120 return false;
121
122 pWalletSQLiteRuntime = db.release();
123 return true;
124}
125
126void WalletSQLiteRuntimeClose()
127{
128 if (pWalletSQLiteRuntime)
129 {
130 // Fold the write-ahead log into the main file on the way out. WAL +
131 // synchronous=FULL already makes every committed write durable across a
132 // kill, so this is housekeeping, not the thing that protects the data:
133 // if it fails, or the node is killed before it runs, the wallet is
134 // still whole. Best effort, never throws.
135 string strCheckpointError;
136 pWalletSQLiteRuntime->Checkpoint(strCheckpointError);
137 delete pWalletSQLiteRuntime;
138 pWalletSQLiteRuntime = NULL;
139 }
140}
141
142bool WalletSQLiteRuntimeReadRecord(const vector<unsigned char>& vchKey,
143 vector<unsigned char>& vchValueRet)
144{
145 if (!pWalletSQLiteRuntime)
146 return false;
147 string strError;
148 return pWalletSQLiteRuntime->ReadRecord(vchKey, vchValueRet, strError);
149}
150
151bool WalletSQLiteRuntimeWriteRecord(const vector<unsigned char>& vchKey,
152 const vector<unsigned char>& vchValue,
153 bool fOverwrite)
154{
155 if (!pWalletSQLiteRuntime)
156 return false;
157 string strError;
158 return pWalletSQLiteRuntime->WriteRecord(vchKey, vchValue, strError,
159 fOverwrite);
160}
161
162bool WalletSQLiteRuntimeEraseRecord(const vector<unsigned char>& vchKey)
163{
164 if (!pWalletSQLiteRuntime)
165 return false;
166 string strError;
167 return pWalletSQLiteRuntime->EraseRecord(vchKey, strError);
168}
169
170// Transaction bracket for the SQLite runtime. When the SQLite backend is not
171// active these are no-ops that succeed, so a caller can wrap a group of related
172// writes and Berkeley DB keeps its existing per-write behaviour unchanged while
173// SQLite gets all-or-nothing.
174bool WalletSQLiteRuntimeBeginTxn()
175{
176 if (!pWalletSQLiteRuntime)
177 return true;
178 string strError;
179 if (pWalletSQLiteRuntime->BeginTransaction(strError))
180 return true;
181 printf("WalletSQLiteRuntimeBeginTxn() : %s\n", strError.c_str());
182 return false;
183}
184
185bool WalletSQLiteRuntimeCommitTxn()
186{
187 if (!pWalletSQLiteRuntime)
188 return true;
189 string strError;
190 if (pWalletSQLiteRuntime->CommitTransaction(strError))
191 return true;
192 printf("WalletSQLiteRuntimeCommitTxn() : %s\n", strError.c_str());
193 return false;
194}
195
196void WalletSQLiteRuntimeRollbackTxn()
197{
198 if (!pWalletSQLiteRuntime)
199 return;
200 string strError;
201 pWalletSQLiteRuntime->RollbackTransaction(strError);
202}
203
204// Open the SQLite wallet read-only as the active runtime. Used by the one-shot
205// storage diagnostics, which run before the normal wallet load and must be able
206// to inspect a wallet that the loader would refuse -- so this opens the file for
207// scanning without loading or validating its records.
208bool WalletSQLiteRuntimeOpenReadOnly(const string& strPath, string& strErrorRet)
209{
210 strErrorRet.clear();
211 if (!FileExists(strPath.c_str()))
212 {
213 strErrorRet = strprintf("SQLite wallet does not exist: %s", strPath.c_str());
214 return false;
215 }
216
217 WalletSQLiteRuntimeClose();
218 std::auto_ptr<CWalletDBSQLite> db(new CWalletDBSQLite());
219 if (!db->OpenReadOnly(strPath, strErrorRet))
220 return false;
221
222 pWalletSQLiteRuntime = db.release();
223 return true;
224}
225
226bool WalletSQLiteRuntimeCheckpoint(string& strErrorRet)
227{
228 if (!pWalletSQLiteRuntime)
229 {
230 strErrorRet = "SQLite wallet backend is not active";
231 return false;
232 }
233 return pWalletSQLiteRuntime->Checkpoint(strErrorRet);
234}
235
236// The persistent record of which backend this datadir uses, so the choice
237// survives a restart without a command-line flag. A plain text file holding
238// "sqlite" or "bdb" -- visible and hand-editable on purpose, no hidden state.
239static string WalletBackendMarkerPath()
240{
241 return GetAppDir() + "/wallet-backend";
242}
243
244string ReadWalletBackendMarker()
245{
246 FILE* pf = fopen(WalletBackendMarkerPath().c_str(), "rb");
247 if (!pf)
248 return "";
249 char buf[16] = {0};
250 size_t n = fread(buf, 1, sizeof(buf) - 1, pf);
251 fclose(pf);
252 string s(buf, n);
253 while (!s.empty() && isspace((unsigned char)s[s.size() - 1]))
254 s.erase(s.size() - 1);
255 while (!s.empty() && isspace((unsigned char)s[0]))
256 s.erase(0, 1);
257 if (s == "sqlite" || s == "bdb")
258 return s;
259 return "";
260}
261
262bool WriteWalletBackendMarker(const string& strBackend)
263{
264 if (strBackend != "sqlite" && strBackend != "bdb")
265 return false;
266 FILE* pf = fopen(WalletBackendMarkerPath().c_str(), "wb");
267 if (!pf)
268 return false;
269 bool fOk = fwrite(strBackend.c_str(), 1, strBackend.size(), pf) == strBackend.size();
270 if (fputc('\n', pf) == EOF)
271 fOk = false;
272 if (fclose(pf) != 0)
273 fOk = false;
274 return fOk;
275}
276
76 277
77CDB::CDB(const char* pszFile, const char* pszMode, bool fTxn) : pdb(NULL) 278CDB::CDB(const char* pszFile, const char* pszMode, bool fTxn) : pdb(NULL)
78{ 279{
@@ -104,24 +305,30 @@ CDB::CDB(const char* pszFile, const char* pszMode, bool fTxn) : pdb(NULL)
104 305
105 printf("dbenv.open strAppDir=%s\n", strAppDir.c_str()); 306 printf("dbenv.open strAppDir=%s\n", strAppDir.c_str());
106 307
107 dbenv.set_lg_dir(strLogDir.c_str()); 308 DbEnv& env = GetDbEnv();
108 dbenv.set_lg_max(10000000); 309 env.set_lg_dir(strLogDir.c_str());
109 dbenv.set_lk_max_locks(10000); 310 env.set_lg_max(10000000);
110 dbenv.set_lk_max_objects(10000); 311 env.set_lk_max_locks(10000);
111 dbenv.set_errfile(fopen("db.log", "a")); /// debug 312 env.set_lk_max_objects(10000);
112 ///dbenv.log_set_config(DB_LOG_AUTO_REMOVE, 1); /// causes corruption 313 pdbErrFile = fopen("db.log", "a"); /// debug
113 ret = dbenv.open(strAppDir.c_str(), 314 if (pdbErrFile)
114 DB_CREATE | 315 env.set_errfile(pdbErrFile);
115 DB_INIT_LOCK | 316 ///env.log_set_config(DB_LOG_AUTO_REMOVE, 1); /// causes corruption
116 DB_INIT_LOG | 317 ret = env.open(strAppDir.c_str(),
117 DB_INIT_MPOOL | 318 DB_CREATE |
118 DB_INIT_TXN | 319 DB_INIT_LOCK |
119 DB_THREAD | 320 DB_INIT_LOG |
120 DB_PRIVATE | 321 DB_INIT_MPOOL |
121 DB_RECOVER, 322 DB_INIT_TXN |
122 0); 323 DB_THREAD |
324 DB_PRIVATE |
325 DB_RECOVER,
326 0);
123 if (ret > 0) 327 if (ret > 0)
328 {
329 CloseDbEnv();
124 throw runtime_error(strprintf("CDB() : error %d opening database environment\n", ret)); 330 throw runtime_error(strprintf("CDB() : error %d opening database environment\n", ret));
331 }
125 fDbEnvInit = true; 332 fDbEnvInit = true;
126 333
127 // A Berkeley DB file records, in every page header, a log sequence 334 // A Berkeley DB file records, in every page header, a log sequence
@@ -151,7 +358,7 @@ CDB::CDB(const char* pszFile, const char* pszMode, bool fTxn) : pdb(NULL)
151 continue; 358 continue;
152 try 359 try
153 { 360 {
154 if (dbenv.lsn_reset(pszName, 0) == 0) 361 if (env.lsn_reset(pszName, 0) == 0)
155 printf("CDB() : adopted %s from another environment " 362 printf("CDB() : adopted %s from another environment "
156 "(log sequence numbers reset)\n", pszName); 363 "(log sequence numbers reset)\n", pszName);
157 } 364 }
@@ -164,7 +371,7 @@ CDB::CDB(const char* pszFile, const char* pszMode, bool fTxn) : pdb(NULL)
164 ++mapFileUseCount[strFile]; 371 ++mapFileUseCount[strFile];
165 } 372 }
166 373
167 pdb = new Db(&dbenv, 0); 374 pdb = new Db(&GetDbEnv(), 0);
168 375
169 ret = pdb->open(NULL, // Txn pointer 376 ret = pdb->open(NULL, // Txn pointer
170 pszFile, // Filename 377 pszFile, // Filename
@@ -199,7 +406,8 @@ void CDB::Close()
199 try { pdb->close(0); } catch (...) { } 406 try { pdb->close(0); } catch (...) { }
200 delete pdb; 407 delete pdb;
201 pdb = NULL; 408 pdb = NULL;
202 try { dbenv.txn_checkpoint(0, 0, 0); } catch (...) { } 409 if (fDbEnvInit)
410 try { GetDbEnv().txn_checkpoint(0, 0, 0); } catch (...) { }
203 411
204 CRITICAL_BLOCK(cs_db) 412 CRITICAL_BLOCK(cs_db)
205 --mapFileUseCount[strFile]; 413 --mapFileUseCount[strFile];
@@ -279,7 +487,8 @@ void DBFlush(bool fShutdown)
279 // takes the process down before the wallet has been put down cleanly, 487 // takes the process down before the wallet has been put down cleanly,
280 // which is the difference between a portable wallet.dat and one welded 488 // which is the difference between a portable wallet.dat and one welded
281 // to this directory -- the lsn_reset just below is what frees it. 489 // to this directory -- the lsn_reset just below is what frees it.
282 try { dbenv.txn_checkpoint(0, 0, 0); } 490 DbEnv& env = GetDbEnv();
491 try { env.txn_checkpoint(0, 0, 0); }
283 catch (const std::exception& e) 492 catch (const std::exception& e)
284 { printf("DBFlush() : checkpoint failed: %s\n", e.what()); } 493 { printf("DBFlush() : checkpoint failed: %s\n", e.what()); }
285 494
@@ -290,7 +499,7 @@ void DBFlush(bool fShutdown)
290 int nRefCount = (*mi).second; 499 int nRefCount = (*mi).second;
291 if (nRefCount == 0) 500 if (nRefCount == 0)
292 { 501 {
293 try { dbenv.lsn_reset(strFile.c_str(), 0); } 502 try { env.lsn_reset(strFile.c_str(), 0); }
294 catch (const std::exception& e) 503 catch (const std::exception& e)
295 { printf("DBFlush() : lsn_reset(%s) failed: %s\n", strFile.c_str(), e.what()); } 504 { printf("DBFlush() : lsn_reset(%s) failed: %s\n", strFile.c_str(), e.what()); }
296 mapFileUseCount.erase(mi++); 505 mapFileUseCount.erase(mi++);
@@ -310,9 +519,8 @@ void DBFlush(bool fShutdown)
310 // Found while making wallet encryption clean up after itself: 519 // Found while making wallet encryption clean up after itself:
311 // calling DBFlush(true) a second time crashed here every time. 520 // calling DBFlush(true) a second time crashed here every time.
312 if (mapFileUseCount.empty()) 521 if (mapFileUseCount.empty())
313 try { dbenv.log_archive(NULL, DB_ARCH_REMOVE); } catch (...) { } 522 try { env.log_archive(NULL, DB_ARCH_REMOVE); } catch (...) { }
314 try { dbenv.close(0); } catch (...) { } 523 CloseDbEnv();
315 fDbEnvInit = false;
316 } 524 }
317 } 525 }
318} 526}
@@ -614,6 +822,294 @@ bool CTxDB::LoadBlockIndex()
614// CWalletDB 822// CWalletDB
615// 823//
616 824
825CWalletDB::CWalletDB(const char* pszMode, bool fTxn) :
826 CDB(WalletSQLiteRuntimeActive() ? NULL : "wallet.dat", pszMode, fTxn)
827{
828}
829
830class CSQLiteWalletRuntimeLoadVisitor : public CWalletRecordVisitor
831{
832public:
833 bool fHaveStoredMineMode;
834 bool fHaveStoredHDCoinType;
835 string strLastType;
836
837 CSQLiteWalletRuntimeLoadVisitor() :
838 fHaveStoredMineMode(false),
839 fHaveStoredHDCoinType(false)
840 {
841 }
842
843 bool VisitWalletRecord(const CDataStream& ssKeyIn,
844 const CDataStream& ssValueIn,
845 string& strErrorRet)
846 {
847 CDataStream ssKey = ssKeyIn;
848 CDataStream ssValue = ssValueIn;
849 string strType;
850 ssKey >> strType;
851 strLastType = strType;
852
853 if (strType == "name")
854 {
855 string strAddress;
856 ssKey >> strAddress;
857 ssValue >> mapAddressBook[strAddress];
858 }
859 else if (strType == "tx")
860 {
861 uint256 hash;
862 ssKey >> hash;
863 CWalletTx& wtx = mapWallet[hash];
864 ssValue >> wtx;
865
866 if (wtx.GetHash() != hash)
867 printf("Error in SQLite wallet export, hash mismatch\n");
868 }
869 else if (strType == "key")
870 {
871 vector<unsigned char> vchPubKey;
872 ssKey >> vchPubKey;
873 CPrivKey vchPrivKey;
874 ssValue >> vchPrivKey;
875
876 mapKeys[vchPubKey] = vchPrivKey;
877 mapPubKeys[Hash160(vchPubKey)] = vchPubKey;
878 }
879 else if (strType == "mkey")
880 {
881 unsigned int nID = 0;
882 ssKey >> nID;
883 CWalletMasterKey kMasterKey;
884 ssValue >> kMasterKey;
885 mapMasterKeys[nID] = kMasterKey;
886 nWalletMasterKeyMaxID = max(nWalletMasterKeyMaxID, nID);
887 fWalletEncrypted = true;
888 fWalletLocked = true;
889 }
890 else if (strType == "ckey")
891 {
892 vector<unsigned char> vchPubKey;
893 ssKey >> vchPubKey;
894 vector<unsigned char> vchCryptedSecret;
895 ssValue >> vchCryptedSecret;
896 if (!AddCryptedKey(vchPubKey, vchCryptedSecret))
897 {
898 strErrorRet = "SQLite wallet export has an unreadable encrypted key record";
899 return false;
900 }
901 }
902 else if (strType == "defaultkey")
903 {
904 ssValue >> vchDefaultKey;
905 }
906 else if (strType == "hdmaster")
907 {
908 ssValue >> vchHDMaster;
909 }
910 else if (strType == "hdchaincode")
911 {
912 ssValue >> vchHDChainCode;
913 }
914 else if (strType == "hdnext")
915 {
916 ssValue >> nHDNext;
917 }
918 else if (strType == "hdschema")
919 {
920 ssValue >> nHDKeySchema;
921 }
922 else if (strType == "hdcointype")
923 {
924 ssValue >> nHDCoinType;
925 fHaveStoredHDCoinType = true;
926 }
927 else if (strType == "hdreceivenext")
928 {
929 ssValue >> nHDReceiveNext;
930 }
931 else if (strType == "hdchangenext")
932 {
933 ssValue >> nHDChangeNext;
934 }
935 else if (strType == "cryptedhdmaster")
936 {
937 ssValue >> vchCryptedHDMaster;
938 fWalletEncrypted = true;
939 fWalletLocked = true;
940 }
941 else if (strType == "cryptedhdchaincode")
942 {
943 ssValue >> vchCryptedHDChainCode;
944 fWalletEncrypted = true;
945 fWalletLocked = true;
946 }
947 else if (strType == "walletminversion")
948 {
949 int nMinVersion = 0;
950 ssValue >> nMinVersion;
951 if (nMinVersion > WALLET_FORMAT_SUPPORTED)
952 {
953 strWalletLoadError = strprintf(
954 "This SQLite wallet export was written by a newer version of "
955 "Bitflash (wallet format %d; this build understands %d). "
956 "Upgrade Bitflash before opening it.",
957 nMinVersion, WALLET_FORMAT_SUPPORTED);
958 strErrorRet = strWalletLoadError;
959 return false;
960 }
961 }
962 else if (strType == "pool")
963 {
964 int64 nIndex;
965 ssKey >> nIndex;
966 vector<unsigned char> vchPubKey;
967 ssValue >> vchPubKey;
968 mapKeyPool[nIndex] = vchPubKey;
969 }
970 else if (strType == "setting")
971 {
972 string strKey;
973 ssKey >> strKey;
974 if (strKey == "nTransactionFee") ssValue >> nTransactionFee;
975 if (strKey == "addrIncoming") ssValue >> addrIncoming;
976
977 if (!fMineModeFromCommandLine)
978 {
979 if (strKey == "nMineMode") { ssValue >> nMineMode; fHaveStoredMineMode = true; }
980 if (strKey == "fGenerateBitcoins") ssValue >> fGenerateBitcoins;
981 if (strKey == "strParticipantPool") ssValue >> strParticipantPool;
982 }
983 }
984
985 if (ssKey.fail())
986 {
987 strErrorRet = strprintf("malformed SQLite wallet record key while handling '%s'",
988 strLastType.c_str());
989 return false;
990 }
991 if (ssValue.fail())
992 {
993 strErrorRet = strprintf("malformed SQLite wallet record value while handling '%s'",
994 strLastType.c_str());
995 return false;
996 }
997 return true;
998 }
999
1000 vector<unsigned char> vchDefaultKey;
1001};
1002
1003static bool FinishLoadedWallet(const vector<unsigned char>& vchDefaultKey,
1004 bool fMayCreateDefaultKey)
1005{
1006 if (!vchDefaultKey.empty() && mapPubKeys.count(Hash160(vchDefaultKey)))
1007 {
1008 keyUser.SetPubKey(vchDefaultKey);
1009 if (mapKeys.count(vchDefaultKey))
1010 keyUser.SetPrivKey(mapKeys[vchDefaultKey]);
1011 }
1012 else if (IsWalletEncrypted())
1013 {
1014 printf("LoadWallet: encrypted wallet has no usable default public key. "
1015 "The file was left untouched.\n");
1016 return false;
1017 }
1018 else
1019 {
1020 if (!fMayCreateDefaultKey)
1021 {
1022 strWalletLoadError =
1023 "SQLite wallet export has no usable default public key";
1024 printf("LoadWallet: %s\n", strWalletLoadError.c_str());
1025 return false;
1026 }
1027
1028 keyUser.MakeNewKey();
1029 if (!AddKey(keyUser))
1030 return false;
1031 if (!SetAddressBookName(PubKeyToAddress(keyUser.GetPubKey()), "Your Address"))
1032 return false;
1033 CWalletDB().WriteDefaultKey(keyUser.GetPubKey());
1034 }
1035
1036 return true;
1037}
1038
1039static bool ReconcileLoadedWalletMetadata(bool fHaveStoredMineMode,
1040 bool fHaveStoredHDCoinType)
1041{
1042 // An encrypted wallet has a seed; it just has not been decrypted yet.
1043 //
1044 // The reset in the else branch was written when "no seed in memory after
1045 // loading" could only mean "this wallet has no seed". With encryption that
1046 // is no longer true, and taking the else branch here wiped the schema and
1047 // the derivation counters that had just been read correctly out of the
1048 // file. Unlocking later restores the seed but nothing restores those, so a
1049 // BIP44 wallet came back deriving legacy addresses -- different addresses
1050 // from the ones its own recovery phrase produces, while the audit reported
1051 // the phrase as covering nothing.
1052 if (HaveHDSeed() || !vchCryptedHDMaster.empty())
1053 {
1054 if (nHDKeySchema == HD_SCHEMA_NONE)
1055 {
1056 nHDKeySchema = HD_SCHEMA_LEGACY;
1057 printf("LoadWallet: deterministic seed has no schema metadata; "
1058 "treating it as legacy m/index'\n");
1059 }
1060 else if (nHDKeySchema != HD_SCHEMA_LEGACY &&
1061 nHDKeySchema != HD_SCHEMA_BIP44)
1062 {
1063 printf("LoadWallet: deterministic wallet schema %d (%s) is not "
1064 "supported by this build\n",
1065 nHDKeySchema, HDKeySchemaName(nHDKeySchema).c_str());
1066 return false;
1067 }
1068 if (!fHaveStoredHDCoinType)
1069 {
1070 nHDCoinType = HD_BIP44_COIN_TYPE_BITFLASH;
1071 printf("LoadWallet: deterministic seed has no BIP44 coin type metadata; "
1072 "using Bitflash coin type %u\n", nHDCoinType);
1073 }
1074 }
1075 else
1076 {
1077 nHDKeySchema = HD_SCHEMA_NONE;
1078 nHDNext = 0;
1079 nHDReceiveNext = 0;
1080 nHDChangeNext = 0;
1081 nHDCoinType = HD_BIP44_COIN_TYPE_BITFLASH;
1082 }
1083
1084 // fGenerateBitcoins and nMineMode only mean anything together, and a
1085 // wallet.dat can easily hold one without the other: Bitcoin 0.1.0 already
1086 // wrote fGenerateBitcoins when mining was toggled, years before this fork
1087 // had a mode at all. Restoring that lone flag put a node into the one state
1088 // the miner cannot act on -- generate on, mode relay -- so every miner
1089 // thread started and returned at the relay guard, and the machine sat there
1090 // looking like it was mining while hashing nothing. That happened to a
1091 // 32-core node on the first 1.2.11 start.
1092 //
1093 // So the flag is only believed when the mode it belongs to was stored with
1094 // it, and the pair is made coherent either way.
1095 if (!fMineModeFromCommandLine)
1096 {
1097 if (!fHaveStoredMineMode)
1098 {
1099 if (fGenerateBitcoins)
1100 printf("LoadWallet: ignoring a stored fGenerateBitcoins with no mining mode "
1101 "beside it -- it predates this setting and cannot be read on its own\n");
1102 fGenerateBitcoins = 0;
1103 }
1104 else
1105 {
1106 fGenerateBitcoins = (nMineMode != MINE_RELAY) ? 1 : 0;
1107 }
1108 }
1109
1110 return true;
1111}
1112
617bool CWalletDB::LoadWallet(vector<unsigned char>& vchDefaultKeyRet) 1113bool CWalletDB::LoadWallet(vector<unsigned char>& vchDefaultKeyRet)
618{ 1114{
619 // Whether wallet.dat actually carried a mining mode, as opposed to just the 1115 // Whether wallet.dat actually carried a mining mode, as opposed to just the
@@ -872,9 +1368,9 @@ bool CWalletDB::LoadWallet(vector<unsigned char>& vchDefaultKeyRet)
872 } 1368 }
873 if (!fHaveStoredHDCoinType) 1369 if (!fHaveStoredHDCoinType)
874 { 1370 {
875 nHDCoinType = HD_BIP44_COIN_TYPE_BITFLASH_PROVISIONAL; 1371 nHDCoinType = HD_BIP44_COIN_TYPE_BITFLASH;
876 printf("LoadWallet: deterministic seed has no BIP44 coin type metadata; " 1372 printf("LoadWallet: deterministic seed has no BIP44 coin type metadata; "
877 "using provisional Bitflash coin type %u\n", nHDCoinType); 1373 "using Bitflash coin type %u\n", nHDCoinType);
878 } 1374 }
879 } 1375 }
880 else 1376 else
@@ -883,7 +1379,7 @@ bool CWalletDB::LoadWallet(vector<unsigned char>& vchDefaultKeyRet)
883 nHDNext = 0; 1379 nHDNext = 0;
884 nHDReceiveNext = 0; 1380 nHDReceiveNext = 0;
885 nHDChangeNext = 0; 1381 nHDChangeNext = 0;
886 nHDCoinType = HD_BIP44_COIN_TYPE_BITFLASH_PROVISIONAL; 1382 nHDCoinType = HD_BIP44_COIN_TYPE_BITFLASH;
887 } 1383 }
888 1384
889 // fGenerateBitcoins and nMineMode only mean anything together, and a 1385 // fGenerateBitcoins and nMineMode only mean anything together, and a
@@ -924,34 +1420,269 @@ bool CWalletDB::LoadWallet(vector<unsigned char>& vchDefaultKeyRet)
924 return true; 1420 return true;
925} 1421}
926 1422
1423bool ScanWalletRecords(CWalletRecordVisitor& visitor, string& strErrorRet)
1424{
1425 // Under the SQLite backend there is no Berkeley DB cursor to walk; scan the
1426 // active SQLite store instead. This is the single choke point every "read
1427 // every wallet record" caller goes through (storage audit and check among
1428 // them), so routing it here makes those honour -walletbackend=sqlite.
1429 if (WalletSQLiteRuntimeActive())
1430 {
1431 strErrorRet.clear();
1432 return pWalletSQLiteRuntime->ScanRecords(visitor, strErrorRet);
1433 }
1434
1435 class CWalletScanDB : public CWalletDB
1436 {
1437 public:
1438 CWalletScanDB() : CWalletDB("r") { }
1439 using CDB::GetCursor;
1440 using CDB::ReadAtCursor;
1441 };
1442
1443 strErrorRet.clear();
1444 CWalletScanDB dbWallet;
1445 Dbc* pcursor = dbWallet.GetCursor();
1446 if (!pcursor)
1447 {
1448 strErrorRet = "could not open a wallet cursor";
1449 return false;
1450 }
1451 CAutoCursor cursorGuard(pcursor);
1452
1453 loop
1454 {
1455 CDataStream ssKey(SER_DISK);
1456 CDataStream ssValue(SER_DISK);
1457 int ret = dbWallet.ReadAtCursor(pcursor, ssKey, ssValue);
1458 if (ret == DB_NOTFOUND)
1459 break;
1460 if (ret != 0)
1461 {
1462 strErrorRet = strprintf("could not read wallet record: Berkeley DB error %d", ret);
1463 return false;
1464 }
1465 if (!visitor.VisitWalletRecord(ssKey, ssValue, strErrorRet))
1466 return false;
1467 }
1468
1469 return true;
1470}
1471
927bool LoadWallet() 1472bool LoadWallet()
928{ 1473{
929 vector<unsigned char> vchDefaultKey; 1474 vector<unsigned char> vchDefaultKey;
930 if (!CWalletDB("cr").LoadWallet(vchDefaultKey)) 1475 if (!CWalletDB("cr").LoadWallet(vchDefaultKey))
931 return false; 1476 return false;
932 1477
933 if (!vchDefaultKey.empty() && mapPubKeys.count(Hash160(vchDefaultKey))) 1478 return FinishLoadedWallet(vchDefaultKey, true);
1479}
1480
1481bool LoadWalletFromSQLite(const string& strPath)
1482{
1483 strWalletLoadError.clear();
1484
1485 if (!FileExists(strPath.c_str()))
934 { 1486 {
935 // Set keyUser 1487 strWalletLoadError = strprintf("SQLite wallet export does not exist: %s",
936 keyUser.SetPubKey(vchDefaultKey); 1488 strPath.c_str());
937 if (mapKeys.count(vchDefaultKey)) 1489 printf("LoadWallet: %s\n", strWalletLoadError.c_str());
938 keyUser.SetPrivKey(mapKeys[vchDefaultKey]); 1490 return false;
939 } 1491 }
940 else if (IsWalletEncrypted()) 1492
1493 CWalletDBSQLite sqlite;
1494 string strError;
1495 if (!sqlite.OpenReadOnly(strPath, strError))
941 { 1496 {
942 printf("LoadWallet: encrypted wallet has no usable default public key. " 1497 strWalletLoadError = strError;
943 "The file was left untouched.\n"); 1498 printf("LoadWallet: cannot open SQLite wallet export: %s\n",
1499 strError.c_str());
944 return false; 1500 return false;
945 } 1501 }
946 else 1502
1503 CSQLiteWalletRuntimeLoadVisitor visitor;
1504 try
947 { 1505 {
948 // Create new keyUser and set as default key 1506 CRITICAL_BLOCK(cs_mapKeys)
949 keyUser.MakeNewKey(); 1507 CRITICAL_BLOCK(cs_mapWallet)
950 if (!AddKey(keyUser)) 1508 {
951 return false; 1509 if (!sqlite.ScanRecords(visitor, strError))
952 if (!SetAddressBookName(PubKeyToAddress(keyUser.GetPubKey()), "Your Address")) 1510 {
953 return false; 1511 if (strWalletLoadError.empty())
954 CWalletDB().WriteDefaultKey(keyUser.GetPubKey()); 1512 strWalletLoadError = strError;
1513 printf("LoadWallet: SQLite wallet export could not be read. "
1514 "Failed while handling a '%s' record: %s\n",
1515 visitor.strLastType.c_str(), strError.c_str());
1516 return false;
1517 }
1518 }
1519 }
1520 catch (const std::exception& e)
1521 {
1522 strWalletLoadError = strprintf(
1523 "SQLite wallet export could not be read while handling a '%s' record: %s",
1524 visitor.strLastType.c_str(), e.what());
1525 printf("LoadWallet: %s\n", strWalletLoadError.c_str());
1526 return false;
1527 }
1528 catch (...)
1529 {
1530 strWalletLoadError = strprintf(
1531 "SQLite wallet export could not be read while handling a '%s' record",
1532 visitor.strLastType.c_str());
1533 printf("LoadWallet: %s\n", strWalletLoadError.c_str());
1534 return false;
1535 }
1536
1537 if (!ReconcileLoadedWalletMetadata(visitor.fHaveStoredMineMode,
1538 visitor.fHaveStoredHDCoinType))
1539 {
1540 if (strWalletLoadError.empty())
1541 strWalletLoadError = "SQLite wallet metadata is not supported by this build";
1542 return false;
1543 }
1544
1545 return FinishLoadedWallet(visitor.vchDefaultKey, false);
1546}
1547
1548// Open a SQLite wallet read-write and keep it open as the active runtime
1549// backend, then load its records into memory the same way LoadWalletFromSQLite
1550// does. Unlike the read-only staging path, this leaves pWalletSQLiteRuntime set
1551// so every later CWalletDB read/write/erase routes to this file for the life of
1552// the process. The Berkeley DB wallet.dat is never opened.
1553bool LoadWalletFromSQLiteRuntime(const string& strPath)
1554{
1555 strWalletLoadError.clear();
1556
1557 if (!FileExists(strPath.c_str()))
1558 {
1559 strWalletLoadError = strprintf("SQLite wallet does not exist: %s",
1560 strPath.c_str());
1561 printf("LoadWallet: %s\n", strWalletLoadError.c_str());
1562 return false;
1563 }
1564
1565 string strError;
1566 if (!WalletSQLiteRuntimeOpen(strPath, strError))
1567 {
1568 strWalletLoadError = strError;
1569 printf("LoadWallet: cannot open SQLite wallet backend: %s\n",
1570 strError.c_str());
1571 return false;
1572 }
1573
1574 CSQLiteWalletRuntimeLoadVisitor visitor;
1575 try
1576 {
1577 CRITICAL_BLOCK(cs_mapKeys)
1578 CRITICAL_BLOCK(cs_mapWallet)
1579 {
1580 if (!pWalletSQLiteRuntime->ScanRecords(visitor, strError))
1581 {
1582 WalletSQLiteRuntimeClose();
1583 if (strWalletLoadError.empty())
1584 strWalletLoadError = strError;
1585 printf("LoadWallet: SQLite wallet backend could not be read. "
1586 "Failed while handling a '%s' record: %s\n",
1587 visitor.strLastType.c_str(), strError.c_str());
1588 return false;
1589 }
1590 }
1591 }
1592 catch (const std::exception& e)
1593 {
1594 WalletSQLiteRuntimeClose();
1595 strWalletLoadError = strprintf(
1596 "SQLite wallet backend could not be read while handling a '%s' record: %s",
1597 visitor.strLastType.c_str(), e.what());
1598 printf("LoadWallet: %s\n", strWalletLoadError.c_str());
1599 return false;
1600 }
1601 catch (...)
1602 {
1603 WalletSQLiteRuntimeClose();
1604 strWalletLoadError = strprintf(
1605 "SQLite wallet backend could not be read while handling a '%s' record",
1606 visitor.strLastType.c_str());
1607 printf("LoadWallet: %s\n", strWalletLoadError.c_str());
1608 return false;
1609 }
1610
1611 // Refuse an empty or non-wallet SQLite file up front, with a message that
1612 // names the likely cause. FinishLoadedWallet below already rejects a
1613 // missing default key (it is the fMayCreateDefaultKey=false path), but its
1614 // wording -- "no usable default public key" -- reads as corruption when the
1615 // real story is usually "this file was never an exported wallet". A blank
1616 // wallet.sqlite left by a typo or a half-run export should say so.
1617 if (visitor.vchDefaultKey.empty())
1618 {
1619 WalletSQLiteRuntimeClose();
1620 strWalletLoadError =
1621 "SQLite wallet has no default key record; it may be empty or not a "
1622 "wallet export. Create it with -walletsqliteexport.";
1623 printf("LoadWallet: %s\n", strWalletLoadError.c_str());
1624 return false;
1625 }
1626
1627 if (!ReconcileLoadedWalletMetadata(visitor.fHaveStoredMineMode,
1628 visitor.fHaveStoredHDCoinType))
1629 {
1630 WalletSQLiteRuntimeClose();
1631 if (strWalletLoadError.empty())
1632 strWalletLoadError = "SQLite wallet metadata is not supported by this build";
1633 return false;
1634 }
1635
1636 // Never fabricate a default key: an export always carries one, and a file
1637 // that does not is refused with the wallet left open to no writes rather
1638 // than quietly minted into a new identity.
1639 if (!FinishLoadedWallet(visitor.vchDefaultKey, false))
1640 {
1641 WalletSQLiteRuntimeClose();
1642 return false;
1643 }
1644
1645 return true;
1646}
1647
1648// Create a brand-new wallet directly in the SQLite backend. For a fresh install
1649// where SQLite is the default there is no wallet.dat to export from, so open a
1650// new wallet.sqlite (Open() creates the file and its schema) and let
1651// FinishLoadedWallet mint the first default key straight into it -- the same
1652// call the Berkeley DB create path uses, so a new wallet is identical either
1653// way. Refuses to touch an existing file, and cleans up on failure so a broken
1654// half-initialised wallet.sqlite is never left for the next start to load.
1655bool CreateNewSQLiteWallet(const string& strPath)
1656{
1657 strWalletLoadError.clear();
1658
1659 if (FileExists(strPath.c_str()))
1660 {
1661 strWalletLoadError = strprintf("SQLite wallet already exists: %s",
1662 strPath.c_str());
1663 printf("CreateNewSQLiteWallet: %s\n", strWalletLoadError.c_str());
1664 return false;
1665 }
1666
1667 string strError;
1668 if (!WalletSQLiteRuntimeOpen(strPath, strError, false))
1669 {
1670 strWalletLoadError = strError;
1671 printf("CreateNewSQLiteWallet: cannot create SQLite wallet: %s\n",
1672 strError.c_str());
1673 return false;
1674 }
1675
1676 vector<unsigned char> vchNoDefaultKey;
1677 if (!FinishLoadedWallet(vchNoDefaultKey, true))
1678 {
1679 WalletSQLiteRuntimeClose();
1680 remove(strPath.c_str());
1681 remove((strPath + "-wal").c_str());
1682 remove((strPath + "-shm").c_str());
1683 if (strWalletLoadError.empty())
1684 strWalletLoadError = "could not initialise the new SQLite wallet";
1685 return false;
955 } 1686 }
956 1687
957 return true; 1688 return true;
@@ -974,21 +1705,99 @@ bool LoadWallet()
974// this file was written are not spendable from it. Backing up once is not 1705// this file was written are not spendable from it. Backing up once is not
975// enough, and that is a property of the wallet format rather than of this 1706// enough, and that is a property of the wallet format rather than of this
976// function. 1707// function.
1708// Back up the active SQLite wallet. The store is checkpointed so the WAL is
1709// folded into the main file, then the single self-contained wallet.sqlite is
1710// copied through a temp file and renamed into place (same temp/old/rename dance
1711// as the Berkeley DB path, for Windows' non-overwriting rename). No lsn_reset:
1712// a checkpointed SQLite file needs nothing beside it to open.
1713static bool BackupWalletSQLite(const string& strDest)
1714{
1715 string strSrc = GetAppDir() + "/wallet.sqlite";
1716 if (!FileExists(strSrc.c_str()))
1717 return error("BackupWallet() : %s does not exist\n", strSrc.c_str());
1718
1719 string strCheckpointError;
1720 if (!WalletSQLiteRuntimeCheckpoint(strCheckpointError))
1721 return error("BackupWallet() : checkpoint failed: %s\n", strCheckpointError.c_str());
1722
1723 string strTemp = strprintf("%s.tmp.%lld", strDest.c_str(), GetTime());
1724 for (int i = 1; FileExists(strTemp.c_str()); i++)
1725 strTemp = strprintf("%s.tmp.%lld.%d", strDest.c_str(), GetTime(), i);
1726 string strOld = strprintf("%s.old.%lld", strDest.c_str(), GetTime());
1727 for (int i = 1; FileExists(strOld.c_str()); i++)
1728 strOld = strprintf("%s.old.%lld.%d", strDest.c_str(), GetTime(), i);
1729
1730 FILE* pfIn = fopen(strSrc.c_str(), "rb");
1731 if (!pfIn)
1732 return error("BackupWallet() : cannot read %s\n", strSrc.c_str());
1733 FILE* pfOut = fopen(strTemp.c_str(), "wb");
1734 if (!pfOut)
1735 {
1736 fclose(pfIn);
1737 return error("BackupWallet() : cannot write %s\n", strTemp.c_str());
1738 }
1739
1740 char buf[65536];
1741 size_t n;
1742 bool fOk = true;
1743 while ((n = fread(buf, 1, sizeof(buf), pfIn)) > 0)
1744 if (fwrite(buf, 1, n, pfOut) != n) { fOk = false; break; }
1745 if (ferror(pfIn))
1746 fOk = false;
1747 fclose(pfIn);
1748 if (fclose(pfOut) != 0)
1749 fOk = false;
1750 if (!fOk)
1751 {
1752 remove(strTemp.c_str()); // half a wallet is worse than none
1753 return error("BackupWallet() : copy to %s failed\n", strTemp.c_str());
1754 }
1755
1756 bool fHadOldDest = FileExists(strDest.c_str());
1757 if (fHadOldDest && rename(strDest.c_str(), strOld.c_str()) != 0)
1758 {
1759 remove(strTemp.c_str());
1760 return error("BackupWallet() : could not move old %s aside\n", strDest.c_str());
1761 }
1762 if (rename(strTemp.c_str(), strDest.c_str()) != 0)
1763 {
1764 remove(strTemp.c_str());
1765 if (fHadOldDest)
1766 rename(strOld.c_str(), strDest.c_str());
1767 return error("BackupWallet() : could not install %s\n", strDest.c_str());
1768 }
1769 if (fHadOldDest)
1770 remove(strOld.c_str());
1771 return true;
1772}
1773
977bool BackupWallet(const string& strDest) 1774bool BackupWallet(const string& strDest)
978{ 1775{
979 if (strDest.empty()) 1776 if (strDest.empty())
980 return error("BackupWallet() : no destination given\n"); 1777 return error("BackupWallet() : no destination given\n");
981 1778
1779 // Under the SQLite backend the live wallet is wallet.sqlite, not wallet.dat.
1780 if (WalletSQLiteRuntimeActive())
1781 return BackupWalletSQLite(strDest);
1782
982 string strSrc = GetAppDir() + "/wallet.dat"; 1783 string strSrc = GetAppDir() + "/wallet.dat";
983 if (!FileExists(strSrc.c_str())) 1784 if (!FileExists(strSrc.c_str()))
984 return error("BackupWallet() : %s does not exist\n", strSrc.c_str()); 1785 return error("BackupWallet() : %s does not exist\n", strSrc.c_str());
985 1786
1787 string strTemp = strprintf("%s.tmp.%lld", strDest.c_str(), GetTime());
1788 for (int i = 1; FileExists(strTemp.c_str()); i++)
1789 strTemp = strprintf("%s.tmp.%lld.%d", strDest.c_str(), GetTime(), i);
1790 string strOld = strprintf("%s.old.%lld", strDest.c_str(), GetTime());
1791 for (int i = 1; FileExists(strOld.c_str()); i++)
1792 strOld = strprintf("%s.old.%lld.%d", strDest.c_str(), GetTime(), i);
1793
986 CRITICAL_BLOCK(cs_db) 1794 CRITICAL_BLOCK(cs_db)
987 { 1795 {
988 // Committing first, so the copy is not missing the newest records. 1796 // Committing first, so the copy is not missing the newest records.
989 // Both calls can throw, and a failed backup must not take the node 1797 // Both calls can throw, and a failed backup must not take the node
990 // down with it. 1798 // down with it.
991 try { dbenv.txn_checkpoint(0, 0, 0); } 1799 DbEnv& env = GetDbEnv();
1800 try { env.txn_checkpoint(0, 0, 0); }
992 catch (const std::exception& e) 1801 catch (const std::exception& e)
993 { return error("BackupWallet() : checkpoint failed: %s\n", e.what()); } 1802 { return error("BackupWallet() : checkpoint failed: %s\n", e.what()); }
994 1803
@@ -997,11 +1806,11 @@ bool BackupWallet(const string& strDest)
997 FILE* pfIn = fopen(strSrc.c_str(), "rb"); 1806 FILE* pfIn = fopen(strSrc.c_str(), "rb");
998 if (!pfIn) 1807 if (!pfIn)
999 return error("BackupWallet() : cannot read %s\n", strSrc.c_str()); 1808 return error("BackupWallet() : cannot read %s\n", strSrc.c_str());
1000 FILE* pfOut = fopen(strDest.c_str(), "wb"); 1809 FILE* pfOut = fopen(strTemp.c_str(), "wb");
1001 if (!pfOut) 1810 if (!pfOut)
1002 { 1811 {
1003 fclose(pfIn); 1812 fclose(pfIn);
1004 return error("BackupWallet() : cannot write %s\n", strDest.c_str()); 1813 return error("BackupWallet() : cannot write %s\n", strTemp.c_str());
1005 } 1814 }
1006 1815
1007 char buf[65536]; 1816 char buf[65536];
@@ -1018,18 +1827,39 @@ bool BackupWallet(const string& strDest)
1018 1827
1019 if (!fOk) 1828 if (!fOk)
1020 { 1829 {
1021 remove(strDest.c_str()); // half a wallet is worse than none 1830 remove(strTemp.c_str()); // half a wallet is worse than none
1022 return error("BackupWallet() : copy to %s failed\n", strDest.c_str()); 1831 return error("BackupWallet() : copy to %s failed\n", strTemp.c_str());
1023 } 1832 }
1024 1833
1025 // Without this the copy is welded to this node's database/ dir. 1834 // Without this the copy is welded to this node's database/ dir.
1026 int ret = dbenv.lsn_reset(strDest.c_str(), 0); 1835 int ret = env.lsn_reset(strTemp.c_str(), 0);
1027 if (ret != 0) 1836 if (ret != 0)
1028 printf("BackupWallet() : warning -- lsn_reset returned %d; the copy " 1837 printf("BackupWallet() : warning -- lsn_reset returned %d; the copy "
1029 "may only open beside this node's database/ directory\n", ret); 1838 "may only open beside this node's database/ directory\n", ret);
1839
1840 bool fHadOldDest = FileExists(strDest.c_str());
1841 if (fHadOldDest && rename(strDest.c_str(), strOld.c_str()) != 0)
1842 {
1843 remove(strTemp.c_str());
1844 return error("BackupWallet() : could not move old %s aside\n", strDest.c_str());
1845 }
1846 if (rename(strTemp.c_str(), strDest.c_str()) != 0)
1847 {
1848 remove(strTemp.c_str());
1849 if (fHadOldDest)
1850 rename(strOld.c_str(), strDest.c_str());
1851 return error("BackupWallet() : could not install %s\n", strDest.c_str());
1852 }
1853 if (fHadOldDest)
1854 remove(strOld.c_str());
1030 } 1855 }
1031 catch (const std::exception& e) 1856 catch (const std::exception& e)
1032 { return error("BackupWallet() : %s\n", e.what()); } 1857 {
1858 remove(strTemp.c_str());
1859 if (FileExists(strOld.c_str()) && !FileExists(strDest.c_str()))
1860 rename(strOld.c_str(), strDest.c_str());
1861 return error("BackupWallet() : %s\n", e.what());
1862 }
1033 } 1863 }
1034 1864
1035 // Recorded here rather than by whoever called, so that a backup taken with 1865 // Recorded here rather than by whoever called, so that a backup taken with
@@ -1307,73 +2137,71 @@ public:
1307 { 2137 {
1308 return Write(make_pair(string("pool"), nIndex), vchPubKey); 2138 return Write(make_pair(string("pool"), nIndex), vchPubKey);
1309 } 2139 }
2140};
1310 2141
1311 bool CopyPublicRecordsTo(CWalletRewriteDB& dbTo, int& nCopiedRet, string& strErrorRet) 2142static bool IsWalletEncryptionPrivateRecord(const string& strType)
2143{
2144 return strType == "key" ||
2145 strType == "pool" ||
2146 strType == "hdmaster" ||
2147 strType == "hdchaincode" ||
2148 strType == "mkey" ||
2149 strType == "ckey" ||
2150 strType == "cryptedhdmaster" ||
2151 strType == "cryptedhdchaincode" ||
2152 strType == "walletminversion";
2153}
2154
2155class CWalletPublicRecordCopyVisitor : public CWalletRecordVisitor
2156{
2157public:
2158 CWalletRewriteDB& dbTo;
2159 int& nCopied;
2160
2161 CWalletPublicRecordCopyVisitor(CWalletRewriteDB& dbToIn, int& nCopiedIn)
2162 : dbTo(dbToIn), nCopied(nCopiedIn) { }
2163
2164 bool VisitWalletRecord(const CDataStream& ssKey,
2165 const CDataStream& ssValue,
2166 string& strErrorRet)
1312 { 2167 {
1313 nCopiedRet = 0; 2168 string strType;
1314 Dbc* pcursor = GetCursor(); 2169 try
1315 if (!pcursor)
1316 { 2170 {
1317 strErrorRet = "could not open a wallet cursor"; 2171 CDataStream ssType = ssKey;
1318 return false; 2172 ssType >> strType;
1319 } 2173 }
1320 2174 catch (...)
1321 unsigned int fFlags = DB_NEXT;
1322 loop
1323 { 2175 {
1324 CDataStream ssKey; 2176 strErrorRet = "wallet.dat has a record whose key type cannot be read";
1325 CDataStream ssValue; 2177 return false;
1326 int ret = ReadAtCursor(pcursor, ssKey, ssValue, fFlags); 2178 }
1327 if (ret == DB_NOTFOUND)
1328 break;
1329 if (ret != 0)
1330 {
1331 pcursor->close();
1332 strErrorRet = strprintf("could not read wallet record: Berkeley DB error %d", ret);
1333 return false;
1334 }
1335
1336 string strType;
1337 try
1338 {
1339 CDataStream ssType = ssKey;
1340 ssType >> strType;
1341 }
1342 catch (...)
1343 {
1344 pcursor->close();
1345 strErrorRet = "wallet.dat has a record whose key type cannot be read";
1346 return false;
1347 }
1348 2179
1349 if (strType == "key" || 2180 if (IsWalletEncryptionPrivateRecord(strType))
1350 strType == "pool" || 2181 return true;
1351 strType == "hdmaster" ||
1352 strType == "hdchaincode" ||
1353 strType == "mkey" ||
1354 strType == "ckey" ||
1355 strType == "cryptedhdmaster" ||
1356 strType == "cryptedhdchaincode" ||
1357 strType == "walletminversion")
1358 continue;
1359 2182
1360 if (!dbTo.WriteRaw(ssKey, ssValue)) 2183 if (!dbTo.WriteRaw(ssKey, ssValue))
1361 { 2184 {
1362 pcursor->close(); 2185 strErrorRet = strprintf("could not copy wallet record '%s'", strType.c_str());
1363 strErrorRet = strprintf("could not copy wallet record '%s'", strType.c_str()); 2186 return false;
1364 return false;
1365 }
1366 nCopiedRet++;
1367 } 2187 }
1368 2188 nCopied++;
1369 pcursor->close();
1370 return true; 2189 return true;
1371 } 2190 }
1372}; 2191};
1373 2192
1374static string WalletEncryptBackupPath() 2193static bool CopyPublicWalletRecordsTo(CWalletRewriteDB& dbTo,
2194 int& nCopiedRet,
2195 string& strErrorRet)
1375{ 2196{
1376 string strBase = GetAppDir() + "/wallet.dat.before-encrypt."; 2197 nCopiedRet = 0;
2198 CWalletPublicRecordCopyVisitor visitor(dbTo, nCopiedRet);
2199 return ScanWalletRecords(visitor, strErrorRet);
2200}
2201
2202static string WalletEncryptBackupPathFor(const char* pszBaseName)
2203{
2204 string strBase = GetAppDir() + "/" + pszBaseName + ".before-encrypt.";
1377 string strPath = strBase + i64tostr(GetTime()) + ".bak"; 2205 string strPath = strBase + i64tostr(GetTime()) + ".bak";
1378 for (int i = 1; FileExists(strPath.c_str()); i++) 2206 for (int i = 1; FileExists(strPath.c_str()); i++)
1379 strPath = strBase + i64tostr(GetTime()) + "." + itostr(i) + ".bak"; 2207 strPath = strBase + i64tostr(GetTime()) + "." + itostr(i) + ".bak";
@@ -1396,6 +2224,199 @@ static bool EncryptPrivateKeyForWallet(const CKeyingMaterial& vchMasterKey,
1396 return true; 2224 return true;
1397} 2225}
1398 2226
2227static void SecureClearBytes(vector<unsigned char>& vch)
2228{
2229 if (!vch.empty())
2230 OPENSSL_cleanse(&vch[0], vch.size());
2231 vch.clear();
2232}
2233
2234// The SQLite counterpart of CWalletPublicRecordCopyVisitor: same skip list,
2235// same byte-for-byte copy, but the destination is a CWalletDBSQLite record
2236// store rather than a Berkeley DB file.
2237class CWalletPublicRecordCopyToSQLiteVisitor : public CWalletRecordVisitor
2238{
2239public:
2240 CWalletDBSQLite& dbTo;
2241 int& nCopied;
2242
2243 CWalletPublicRecordCopyToSQLiteVisitor(CWalletDBSQLite& dbToIn, int& nCopiedIn)
2244 : dbTo(dbToIn), nCopied(nCopiedIn) { }
2245
2246 bool VisitWalletRecord(const CDataStream& ssKey,
2247 const CDataStream& ssValue,
2248 string& strErrorRet)
2249 {
2250 string strType;
2251 try
2252 {
2253 CDataStream ssType = ssKey;
2254 ssType >> strType;
2255 }
2256 catch (...)
2257 {
2258 strErrorRet = "SQLite wallet has a record whose key type cannot be read";
2259 return false;
2260 }
2261
2262 if (IsWalletEncryptionPrivateRecord(strType))
2263 return true;
2264
2265 vector<unsigned char> vchKey(ssKey.begin(), ssKey.end());
2266 vector<unsigned char> vchValue(ssValue.begin(), ssValue.end());
2267 bool fOk = dbTo.WriteRecord(vchKey, vchValue, strErrorRet, true);
2268 if (!vchKey.empty())
2269 memset(&vchKey[0], 0, vchKey.size());
2270 if (!vchValue.empty())
2271 memset(&vchValue[0], 0, vchValue.size());
2272 if (!fOk)
2273 {
2274 strErrorRet = strprintf("could not copy wallet record '%s': %s",
2275 strType.c_str(), strErrorRet.c_str());
2276 return false;
2277 }
2278 nCopied++;
2279 return true;
2280 }
2281};
2282
2283// Persist an encrypted wallet under the SQLite backend. Everything the caller
2284// computed -- the master key, the re-encrypted private keys, the fresh crypted
2285// key pool, the encrypted HD seed -- lands in a brand new wallet.sqlite built
2286// in one transaction, which is then swapped in for the plaintext original.
2287//
2288// A fresh file rather than an in-place rewrite for two reasons: it keeps the
2289// whole change atomic (a crash before the rename leaves the old wallet whole),
2290// and it guarantees no plaintext key survives in a freed page the way an
2291// in-place DELETE would leave behind until a VACUUM.
2292static bool EncryptWalletPersistSQLite(
2293 const vector<pair<vector<unsigned char>, vector<unsigned char> > >& vCryptedKeys,
2294 const vector<pair<vector<unsigned char>, vector<unsigned char> > >& vNewPoolKeys,
2295 const vector<pair<int64, vector<unsigned char> > >& vNewPool,
2296 const CWalletMasterKey& kMasterKey,
2297 const vector<unsigned char>& vchCryptedHDMasterNew,
2298 const vector<unsigned char>& vchCryptedHDChainCodeNew,
2299 unsigned int nHDNextNew,
2300 int& nCopiedRet,
2301 string& strBackupRet,
2302 string& strErrorRet)
2303{
2304 nCopiedRet = 0;
2305 if (!pWalletSQLiteRuntime)
2306 {
2307 strErrorRet = "SQLite wallet backend is not active";
2308 return false;
2309 }
2310
2311 string strWalletPath = GetAppDir() + "/wallet.sqlite";
2312 string strTempPath = GetAppDir() + "/" +
2313 strprintf("wallet.sqlite.encrypting.%lld", GetTime());
2314 remove(strTempPath.c_str());
2315 remove((strTempPath + "-wal").c_str());
2316 remove((strTempPath + "-shm").c_str());
2317
2318 {
2319 CWalletDBSQLite dbTmp;
2320 if (!dbTmp.Open(strTempPath, strErrorRet))
2321 {
2322 remove(strTempPath.c_str());
2323 return false;
2324 }
2325 if (!dbTmp.BeginTransaction(strErrorRet))
2326 {
2327 dbTmp.Close();
2328 remove(strTempPath.c_str());
2329 return false;
2330 }
2331
2332 // The public records come from the live SQLite store -- under this
2333 // backend ScanWalletRecords() has no Berkeley DB behind it to read.
2334 CWalletPublicRecordCopyToSQLiteVisitor visitor(dbTmp, nCopiedRet);
2335 bool fOk = pWalletSQLiteRuntime->ScanRecords(visitor, strErrorRet);
2336
2337 // Encrypted metadata, serialized byte-for-byte the way the Berkeley DB
2338 // path (CWalletRewriteDB) writes it, so a wallet encrypted under either
2339 // backend is the same set of records. ckey uses INSERT-only to match
2340 // WriteCryptedKey; the rest overwrite.
2341 fOk = fOk &&
2342 dbTmp.WriteTypedRecord(string("walletminversion"),
2343 (int)WALLET_FORMAT_ENCRYPTED, strErrorRet, true) &&
2344 dbTmp.WriteTypedRecord(make_pair(string("mkey"),
2345 (unsigned int)(nWalletMasterKeyMaxID + 1)),
2346 kMasterKey, strErrorRet, true);
2347 for (size_t i = 0; fOk && i < vCryptedKeys.size(); i++)
2348 fOk = dbTmp.WriteTypedRecord(make_pair(string("ckey"), vCryptedKeys[i].first),
2349 vCryptedKeys[i].second, strErrorRet, false);
2350 for (size_t i = 0; fOk && i < vNewPoolKeys.size(); i++)
2351 fOk = dbTmp.WriteTypedRecord(make_pair(string("ckey"), vNewPoolKeys[i].first),
2352 vNewPoolKeys[i].second, strErrorRet, false);
2353 if (fOk && !vchCryptedHDMasterNew.empty())
2354 fOk = dbTmp.WriteTypedRecord(string("cryptedhdmaster"),
2355 vchCryptedHDMasterNew, strErrorRet, true) &&
2356 dbTmp.WriteTypedRecord(string("cryptedhdchaincode"),
2357 vchCryptedHDChainCodeNew, strErrorRet, true);
2358 if (fOk)
2359 fOk = dbTmp.WriteTypedRecord(string("hdnext"), nHDNextNew, strErrorRet, true);
2360 for (size_t i = 0; fOk && i < vNewPool.size(); i++)
2361 fOk = dbTmp.WriteTypedRecord(make_pair(string("pool"), vNewPool[i].first),
2362 vNewPool[i].second, strErrorRet, true);
2363
2364 if (!fOk)
2365 {
2366 string strIgnore;
2367 dbTmp.RollbackTransaction(strIgnore);
2368 dbTmp.Close();
2369 remove(strTempPath.c_str());
2370 remove((strTempPath + "-wal").c_str());
2371 remove((strTempPath + "-shm").c_str());
2372 return false;
2373 }
2374
2375 if (!dbTmp.CommitTransaction(strErrorRet))
2376 {
2377 dbTmp.Close();
2378 remove(strTempPath.c_str());
2379 remove((strTempPath + "-wal").c_str());
2380 remove((strTempPath + "-shm").c_str());
2381 return false;
2382 }
2383 string strCheckpointError;
2384 dbTmp.Checkpoint(strCheckpointError);
2385 dbTmp.Close();
2386 }
2387
2388 // The rebuilt wallet is committed and self-contained. Close the live handle
2389 // so the file can be swapped: Windows will not rename a file that still has
2390 // an open handle, and the node exits right after this.
2391 WalletSQLiteRuntimeClose();
2392
2393 string strBackupPath = WalletEncryptBackupPathFor("wallet.sqlite");
2394 if (rename(strWalletPath.c_str(), strBackupPath.c_str()) != 0)
2395 {
2396 strErrorRet = strprintf("could not move the original SQLite wallet to %s",
2397 strBackupPath.c_str());
2398 remove(strTempPath.c_str());
2399 return false;
2400 }
2401 // The moved-aside wallet owns any leftover write-ahead log or shared-memory
2402 // sidecar; drop stale ones so they cannot shadow the new wallet.
2403 remove((strWalletPath + "-wal").c_str());
2404 remove((strWalletPath + "-shm").c_str());
2405
2406 if (rename(strTempPath.c_str(), strWalletPath.c_str()) != 0)
2407 {
2408 rename(strBackupPath.c_str(), strWalletPath.c_str());
2409 strErrorRet = "could not install the encrypted SQLite wallet; the original was restored";
2410 remove(strTempPath.c_str());
2411 return false;
2412 }
2413 remove((strTempPath + "-wal").c_str());
2414 remove((strTempPath + "-shm").c_str());
2415
2416 strBackupRet = strBackupPath;
2417 return true;
2418}
2419
1399bool EncryptWallet(const string& strPassphrase, string& strBackupRet, string& strErrorRet) 2420bool EncryptWallet(const string& strPassphrase, string& strBackupRet, string& strErrorRet)
1400{ 2421{
1401 strBackupRet.clear(); 2422 strBackupRet.clear();
@@ -1415,6 +2436,7 @@ bool EncryptWallet(const string& strPassphrase, string& strBackupRet, string& st
1415 map<vector<unsigned char>, CPrivKey> mapPlainKeys; 2436 map<vector<unsigned char>, CPrivKey> mapPlainKeys;
1416 vector<unsigned char> vchPlainHDMaster; 2437 vector<unsigned char> vchPlainHDMaster;
1417 vector<unsigned char> vchPlainHDChainCode; 2438 vector<unsigned char> vchPlainHDChainCode;
2439 bool fHadHDSeed = false;
1418 unsigned int nHDNextSnapshot = 0; 2440 unsigned int nHDNextSnapshot = 0;
1419 { 2441 {
1420 CRITICAL_BLOCK(cs_mapKeys) 2442 CRITICAL_BLOCK(cs_mapKeys)
@@ -1425,6 +2447,7 @@ bool EncryptWallet(const string& strPassphrase, string& strBackupRet, string& st
1425 { 2447 {
1426 vchPlainHDMaster = vchHDMaster; 2448 vchPlainHDMaster = vchHDMaster;
1427 vchPlainHDChainCode = vchHDChainCode; 2449 vchPlainHDChainCode = vchHDChainCode;
2450 fHadHDSeed = !vchPlainHDMaster.empty() || !vchPlainHDChainCode.empty();
1428 nHDNextSnapshot = nHDNext; 2451 nHDNextSnapshot = nHDNext;
1429 } 2452 }
1430 } 2453 }
@@ -1475,11 +2498,13 @@ bool EncryptWallet(const string& strPassphrase, string& strBackupRet, string& st
1475 2498
1476 vector<unsigned char> vchCryptedHDMasterNew; 2499 vector<unsigned char> vchCryptedHDMasterNew;
1477 vector<unsigned char> vchCryptedHDChainCodeNew; 2500 vector<unsigned char> vchCryptedHDChainCodeNew;
1478 if (!vchPlainHDMaster.empty() || !vchPlainHDChainCode.empty()) 2501 if (fHadHDSeed)
1479 { 2502 {
1480 if (vchPlainHDMaster.size() != 32 || vchPlainHDChainCode.size() != 32) 2503 if (vchPlainHDMaster.size() != 32 || vchPlainHDChainCode.size() != 32)
1481 { 2504 {
1482 strErrorRet = "wallet has an incomplete HD seed"; 2505 strErrorRet = "wallet has an incomplete HD seed";
2506 SecureClearBytes(vchPlainHDMaster);
2507 SecureClearBytes(vchPlainHDChainCode);
1483 return false; 2508 return false;
1484 } 2509 }
1485 if (!EncryptSecret(vchMasterKey, vchPlainHDMaster, 2510 if (!EncryptSecret(vchMasterKey, vchPlainHDMaster,
@@ -1488,8 +2513,12 @@ bool EncryptWallet(const string& strPassphrase, string& strBackupRet, string& st
1488 WalletSecretIV("hdchaincode"), vchCryptedHDChainCodeNew)) 2513 WalletSecretIV("hdchaincode"), vchCryptedHDChainCodeNew))
1489 { 2514 {
1490 strErrorRet = "could not encrypt the HD seed"; 2515 strErrorRet = "could not encrypt the HD seed";
2516 SecureClearBytes(vchPlainHDMaster);
2517 SecureClearBytes(vchPlainHDChainCode);
1491 return false; 2518 return false;
1492 } 2519 }
2520 SecureClearBytes(vchPlainHDMaster);
2521 SecureClearBytes(vchPlainHDChainCode);
1493 } 2522 }
1494 2523
1495 vector<pair<int64, vector<unsigned char> > > vNewPool; 2524 vector<pair<int64, vector<unsigned char> > > vNewPool;
@@ -1498,7 +2527,7 @@ bool EncryptWallet(const string& strPassphrase, string& strBackupRet, string& st
1498 for (int i = 0; i < KEYPOOL_SIZE; i++) 2527 for (int i = 0; i < KEYPOOL_SIZE; i++)
1499 { 2528 {
1500 CKey key; 2529 CKey key;
1501 if (!vchPlainHDMaster.empty()) 2530 if (fHadHDSeed)
1502 { 2531 {
1503 string strDeriveError; 2532 string strDeriveError;
1504 if (!DeriveHDKey(nHDNextNew, key, strDeriveError)) 2533 if (!DeriveHDKey(nHDNextNew, key, strDeriveError))
@@ -1526,15 +2555,32 @@ bool EncryptWallet(const string& strPassphrase, string& strBackupRet, string& st
1526 vNewPool.push_back(make_pair((int64)i + 1, vchPubKey)); 2555 vNewPool.push_back(make_pair((int64)i + 1, vchPubKey));
1527 } 2556 }
1528 2557
2558 // Every ciphertext is computed; nothing has touched the store yet. Under
2559 // the SQLite backend, persist by rebuilding wallet.sqlite in one atomic
2560 // transaction and swapping it in. The Berkeley DB path below is unchanged.
2561 if (WalletSQLiteRuntimeActive())
2562 {
2563 int nCopiedSQLite = 0;
2564 if (!EncryptWalletPersistSQLite(vCryptedKeys, vNewPoolKeys, vNewPool,
2565 kMasterKey, vchCryptedHDMasterNew,
2566 vchCryptedHDChainCodeNew, nHDNextNew,
2567 nCopiedSQLite, strBackupRet, strErrorRet))
2568 return false;
2569 printf("EncryptWallet() : copied %d public record(s), encrypted %u key(s), "
2570 "rebuilt %d key-pool entry(s) [SQLite]\n",
2571 nCopiedSQLite, (unsigned int)(vCryptedKeys.size() + vNewPoolKeys.size()),
2572 (int)vNewPool.size());
2573 return true;
2574 }
2575
1529 string strTempFile = strprintf("wallet.encrypting.%lld.dat", GetTime()); 2576 string strTempFile = strprintf("wallet.encrypting.%lld.dat", GetTime());
1530 string strTempPath = GetAppDir() + "/" + strTempFile; 2577 string strTempPath = GetAppDir() + "/" + strTempFile;
1531 remove(strTempPath.c_str()); 2578 remove(strTempPath.c_str());
1532 2579
1533 int nCopied = 0; 2580 int nCopied = 0;
1534 { 2581 {
1535 CWalletRewriteDB dbSource("wallet.dat", "r");
1536 CWalletRewriteDB dbTarget(strTempFile.c_str(), "c+"); 2582 CWalletRewriteDB dbTarget(strTempFile.c_str(), "c+");
1537 if (!dbSource.CopyPublicRecordsTo(dbTarget, nCopied, strErrorRet)) 2583 if (!CopyPublicWalletRecordsTo(dbTarget, nCopied, strErrorRet))
1538 { 2584 {
1539 remove(strTempPath.c_str()); 2585 remove(strTempPath.c_str());
1540 return false; 2586 return false;
@@ -1594,7 +2640,7 @@ bool EncryptWallet(const string& strPassphrase, string& strBackupRet, string& st
1594 DBFlush(true); 2640 DBFlush(true);
1595 2641
1596 string strWalletPath = GetAppDir() + "/wallet.dat"; 2642 string strWalletPath = GetAppDir() + "/wallet.dat";
1597 string strBackupPath = WalletEncryptBackupPath(); 2643 string strBackupPath = WalletEncryptBackupPathFor("wallet.dat");
1598 if (rename(strWalletPath.c_str(), strBackupPath.c_str()) != 0) 2644 if (rename(strWalletPath.c_str(), strBackupPath.c_str()) != 0)
1599 { 2645 {
1600 strErrorRet = strprintf("could not move the original wallet to %s", strBackupPath.c_str()); 2646 strErrorRet = strprintf("could not move the original wallet to %s", strBackupPath.c_str());
diff --git a/src/db.h b/src/db.h
index 37a252f..859e3fb 100644
--- a/src/db.h
+++ b/src/db.h
@@ -46,7 +46,7 @@ public:
46}; 46};
47 47
48 48
49extern DbEnv dbenv; 49extern DbEnv& GetDbEnv();
50extern void DBFlush(bool fShutdown); 50extern void DBFlush(bool fShutdown);
51// Close the environment and delete its logs. Only the encryption command calls 51// Close the environment and delete its logs. Only the encryption command calls
52// this: the logs hold the plaintext records the old wallet wrote. 52// this: the logs hold the plaintext records the old wallet wrote.
@@ -235,7 +235,7 @@ public:
235 if (!pdb) 235 if (!pdb)
236 return false; 236 return false;
237 DbTxn* ptxn = NULL; 237 DbTxn* ptxn = NULL;
238 int ret = dbenv.txn_begin(GetTxn(), &ptxn, 0); 238 int ret = GetDbEnv().txn_begin(GetTxn(), &ptxn, 0);
239 if (!ptxn || ret != 0) 239 if (!ptxn || ret != 0)
240 return false; 240 return false;
241 vTxn.push_back(ptxn); 241 vTxn.push_back(ptxn);
@@ -333,6 +333,24 @@ private:
333// Legacy IP-based peer address database (CAddrDB / addr.dat) removed -- 333// Legacy IP-based peer address database (CAddrDB / addr.dat) removed --
334// peer discovery is entirely Nostr/.btf-based now. 334// peer discovery is entirely Nostr/.btf-based now.
335 335
336bool WalletSQLiteRuntimeActive();
337bool WalletSQLiteRuntimeOpen(const string& strPath, string& strErrorRet, bool fMustExist=true);
338void WalletSQLiteRuntimeClose();
339bool WalletSQLiteRuntimeReadRecord(const vector<unsigned char>& vchKey,
340 vector<unsigned char>& vchValueRet);
341bool WalletSQLiteRuntimeWriteRecord(const vector<unsigned char>& vchKey,
342 const vector<unsigned char>& vchValue,
343 bool fOverwrite);
344bool WalletSQLiteRuntimeEraseRecord(const vector<unsigned char>& vchKey);
345bool WalletSQLiteRuntimeBeginTxn();
346bool WalletSQLiteRuntimeCommitTxn();
347void WalletSQLiteRuntimeRollbackTxn();
348bool WalletSQLiteRuntimeOpenReadOnly(const string& strPath, string& strErrorRet);
349bool WalletSQLiteRuntimeCheckpoint(string& strErrorRet);
350string ReadWalletBackendMarker();
351bool WriteWalletBackendMarker(const string& strBackend);
352bool CreateNewSQLiteWallet(const string& strPath);
353
336 354
337 355
338 356
@@ -340,64 +358,141 @@ private:
340class CWalletDB : public CDB 358class CWalletDB : public CDB
341{ 359{
342public: 360public:
343 CWalletDB(const char* pszMode="r+", bool fTxn=false) : CDB("wallet.dat", pszMode, fTxn) { } 361 CWalletDB(const char* pszMode="r+", bool fTxn=false);
344private: 362private:
345 CWalletDB(const CWalletDB&); 363 CWalletDB(const CWalletDB&);
346 void operator=(const CWalletDB&); 364 void operator=(const CWalletDB&);
365
366 template<typename K, typename T>
367 bool ReadWalletRecord(const K& key, T& value)
368 {
369 if (!WalletSQLiteRuntimeActive())
370 return Read(key, value);
371
372 CDataStream ssKey(SER_DISK);
373 ssKey.reserve(1000);
374 ssKey << key;
375 vector<unsigned char> vchKey(ssKey.begin(), ssKey.end());
376 vector<unsigned char> vchValue;
377
378 bool fOk = WalletSQLiteRuntimeReadRecord(vchKey, vchValue);
379 if (!vchKey.empty())
380 memset(&vchKey[0], 0, vchKey.size());
381 if (!ssKey.empty())
382 memset(&ssKey[0], 0, ssKey.size());
383 if (!fOk)
384 return false;
385
386 CDataStream ssValue(vchValue, SER_DISK);
387 ssValue >> value;
388 bool fReadOk = !ssValue.fail();
389 if (!vchValue.empty())
390 memset(&vchValue[0], 0, vchValue.size());
391 if (!ssValue.empty())
392 memset(&ssValue[0], 0, ssValue.size());
393 return fReadOk;
394 }
395
396 template<typename K, typename T>
397 bool WriteWalletRecord(const K& key, const T& value, bool fOverwrite=true)
398 {
399 if (!WalletSQLiteRuntimeActive())
400 return Write(key, value, fOverwrite);
401
402 CDataStream ssKey(SER_DISK);
403 ssKey.reserve(1000);
404 ssKey << key;
405 CDataStream ssValue(SER_DISK);
406 ssValue.reserve(10000);
407 ssValue << value;
408
409 vector<unsigned char> vchKey(ssKey.begin(), ssKey.end());
410 vector<unsigned char> vchValue(ssValue.begin(), ssValue.end());
411 bool fOk = WalletSQLiteRuntimeWriteRecord(vchKey, vchValue, fOverwrite);
412
413 if (!vchKey.empty())
414 memset(&vchKey[0], 0, vchKey.size());
415 if (!vchValue.empty())
416 memset(&vchValue[0], 0, vchValue.size());
417 if (!ssKey.empty())
418 memset(&ssKey[0], 0, ssKey.size());
419 if (!ssValue.empty())
420 memset(&ssValue[0], 0, ssValue.size());
421 return fOk;
422 }
423
424 template<typename K>
425 bool EraseWalletRecord(const K& key)
426 {
427 if (!WalletSQLiteRuntimeActive())
428 return Erase(key);
429
430 CDataStream ssKey(SER_DISK);
431 ssKey.reserve(1000);
432 ssKey << key;
433 vector<unsigned char> vchKey(ssKey.begin(), ssKey.end());
434 bool fOk = WalletSQLiteRuntimeEraseRecord(vchKey);
435
436 if (!vchKey.empty())
437 memset(&vchKey[0], 0, vchKey.size());
438 if (!ssKey.empty())
439 memset(&ssKey[0], 0, ssKey.size());
440 return fOk;
441 }
347public: 442public:
348 bool ReadName(const string& strAddress, string& strName) 443 bool ReadName(const string& strAddress, string& strName)
349 { 444 {
350 strName = ""; 445 strName = "";
351 return Read(make_pair(string("name"), strAddress), strName); 446 return ReadWalletRecord(make_pair(string("name"), strAddress), strName);
352 } 447 }
353 448
354 bool WriteName(const string& strAddress, const string& strName) 449 bool WriteName(const string& strAddress, const string& strName)
355 { 450 {
356 mapAddressBook[strAddress] = strName; 451 mapAddressBook[strAddress] = strName;
357 return Write(make_pair(string("name"), strAddress), strName); 452 return WriteWalletRecord(make_pair(string("name"), strAddress), strName);
358 } 453 }
359 454
360 bool EraseName(const string& strAddress) 455 bool EraseName(const string& strAddress)
361 { 456 {
362 mapAddressBook.erase(strAddress); 457 mapAddressBook.erase(strAddress);
363 return Erase(make_pair(string("name"), strAddress)); 458 return EraseWalletRecord(make_pair(string("name"), strAddress));
364 } 459 }
365 460
366 bool ReadTx(uint256 hash, CWalletTx& wtx) 461 bool ReadTx(uint256 hash, CWalletTx& wtx)
367 { 462 {
368 return Read(make_pair(string("tx"), hash), wtx); 463 return ReadWalletRecord(make_pair(string("tx"), hash), wtx);
369 } 464 }
370 465
371 bool WriteTx(uint256 hash, const CWalletTx& wtx) 466 bool WriteTx(uint256 hash, const CWalletTx& wtx)
372 { 467 {
373 return Write(make_pair(string("tx"), hash), wtx); 468 return WriteWalletRecord(make_pair(string("tx"), hash), wtx);
374 } 469 }
375 470
376 bool EraseTx(uint256 hash) 471 bool EraseTx(uint256 hash)
377 { 472 {
378 return Erase(make_pair(string("tx"), hash)); 473 return EraseWalletRecord(make_pair(string("tx"), hash));
379 } 474 }
380 475
381 bool ReadKey(const vector<unsigned char>& vchPubKey, CPrivKey& vchPrivKey) 476 bool ReadKey(const vector<unsigned char>& vchPubKey, CPrivKey& vchPrivKey)
382 { 477 {
383 vchPrivKey.clear(); 478 vchPrivKey.clear();
384 return Read(make_pair(string("key"), vchPubKey), vchPrivKey); 479 return ReadWalletRecord(make_pair(string("key"), vchPubKey), vchPrivKey);
385 } 480 }
386 481
387 bool WriteKey(const vector<unsigned char>& vchPubKey, const CPrivKey& vchPrivKey) 482 bool WriteKey(const vector<unsigned char>& vchPubKey, const CPrivKey& vchPrivKey)
388 { 483 {
389 return Write(make_pair(string("key"), vchPubKey), vchPrivKey, false); 484 return WriteWalletRecord(make_pair(string("key"), vchPubKey), vchPrivKey, false);
390 } 485 }
391 486
392 bool ReadDefaultKey(vector<unsigned char>& vchPubKey) 487 bool ReadDefaultKey(vector<unsigned char>& vchPubKey)
393 { 488 {
394 vchPubKey.clear(); 489 vchPubKey.clear();
395 return Read(string("defaultkey"), vchPubKey); 490 return ReadWalletRecord(string("defaultkey"), vchPubKey);
396 } 491 }
397 492
398 bool WriteDefaultKey(const vector<unsigned char>& vchPubKey) 493 bool WriteDefaultKey(const vector<unsigned char>& vchPubKey)
399 { 494 {
400 return Write(string("defaultkey"), vchPubKey); 495 return WriteWalletRecord(string("defaultkey"), vchPubKey);
401 } 496 }
402 497
403 // --- Deterministic (BIP32) seed --------------------------------------- 498 // --- Deterministic (BIP32) seed ---------------------------------------
@@ -415,62 +510,75 @@ public:
415 { 510 {
416 vchMasterRet.clear(); 511 vchMasterRet.clear();
417 vchChainCodeRet.clear(); 512 vchChainCodeRet.clear();
418 return Read(string("hdmaster"), vchMasterRet) 513 return ReadWalletRecord(string("hdmaster"), vchMasterRet)
419 && Read(string("hdchaincode"), vchChainCodeRet); 514 && ReadWalletRecord(string("hdchaincode"), vchChainCodeRet);
420 } 515 }
421 516
422 bool WriteHDMaster(const vector<unsigned char>& vchMaster, const vector<unsigned char>& vchChainCode) 517 bool WriteHDMaster(const vector<unsigned char>& vchMaster, const vector<unsigned char>& vchChainCode)
423 { 518 {
424 return Write(string("hdmaster"), vchMaster) 519 // hdmaster and hdchaincode are halves of one seed: a wallet with one but
425 && Write(string("hdchaincode"), vchChainCode); 520 // not the other is an incomplete HD wallet (-walletstoragecheck flags
521 // exactly that). Under the SQLite runtime write them in one transaction
522 // so a failure or crash between the two cannot leave that half-state.
523 // Berkeley DB keeps its existing per-write behaviour -- the bracket is a
524 // no-op there.
525 if (!WalletSQLiteRuntimeBeginTxn())
526 return false;
527 if (!WriteWalletRecord(string("hdmaster"), vchMaster) ||
528 !WriteWalletRecord(string("hdchaincode"), vchChainCode))
529 {
530 WalletSQLiteRuntimeRollbackTxn();
531 return false;
532 }
533 return WalletSQLiteRuntimeCommitTxn();
426 } 534 }
427 535
428 bool WriteWalletMinVersion(int nVersion) 536 bool WriteWalletMinVersion(int nVersion)
429 { 537 {
430 return Write(string("walletminversion"), nVersion); 538 return WriteWalletRecord(string("walletminversion"), nVersion);
431 } 539 }
432 540
433 bool WriteMasterKey(unsigned int nID, const CWalletMasterKey& kMasterKey) 541 bool WriteMasterKey(unsigned int nID, const CWalletMasterKey& kMasterKey)
434 { 542 {
435 return Write(make_pair(string("mkey"), nID), kMasterKey, true); 543 return WriteWalletRecord(make_pair(string("mkey"), nID), kMasterKey, true);
436 } 544 }
437 545
438 bool WriteCryptedKey(const vector<unsigned char>& vchPubKey, 546 bool WriteCryptedKey(const vector<unsigned char>& vchPubKey,
439 const vector<unsigned char>& vchCryptedSecret) 547 const vector<unsigned char>& vchCryptedSecret)
440 { 548 {
441 return Write(make_pair(string("ckey"), vchPubKey), vchCryptedSecret, false); 549 return WriteWalletRecord(make_pair(string("ckey"), vchPubKey), vchCryptedSecret, false);
442 } 550 }
443 551
444 bool WriteCryptedHDMaster(const vector<unsigned char>& vchCryptedMaster, 552 bool WriteCryptedHDMaster(const vector<unsigned char>& vchCryptedMaster,
445 const vector<unsigned char>& vchCryptedChainCode) 553 const vector<unsigned char>& vchCryptedChainCode)
446 { 554 {
447 return Write(string("cryptedhdmaster"), vchCryptedMaster) 555 return WriteWalletRecord(string("cryptedhdmaster"), vchCryptedMaster)
448 && Write(string("cryptedhdchaincode"), vchCryptedChainCode); 556 && WriteWalletRecord(string("cryptedhdchaincode"), vchCryptedChainCode);
449 } 557 }
450 558
451 bool WriteHDNext(unsigned int nNext) 559 bool WriteHDNext(unsigned int nNext)
452 { 560 {
453 return Write(string("hdnext"), nNext); 561 return WriteWalletRecord(string("hdnext"), nNext);
454 } 562 }
455 563
456 bool WriteHDSchema(int nSchema) 564 bool WriteHDSchema(int nSchema)
457 { 565 {
458 return Write(string("hdschema"), nSchema); 566 return WriteWalletRecord(string("hdschema"), nSchema);
459 } 567 }
460 568
461 bool WriteHDCoinType(unsigned int nCoinType) 569 bool WriteHDCoinType(unsigned int nCoinType)
462 { 570 {
463 return Write(string("hdcointype"), nCoinType); 571 return WriteWalletRecord(string("hdcointype"), nCoinType);
464 } 572 }
465 573
466 bool WriteHDReceiveNext(unsigned int nNext) 574 bool WriteHDReceiveNext(unsigned int nNext)
467 { 575 {
468 return Write(string("hdreceivenext"), nNext); 576 return WriteWalletRecord(string("hdreceivenext"), nNext);
469 } 577 }
470 578
471 bool WriteHDChangeNext(unsigned int nNext) 579 bool WriteHDChangeNext(unsigned int nNext)
472 { 580 {
473 return Write(string("hdchangenext"), nNext); 581 return WriteWalletRecord(string("hdchangenext"), nNext);
474 } 582 }
475 583
476 // Keys generated ahead of time and not yet handed out. The private key of 584 // Keys generated ahead of time and not yet handed out. The private key of
@@ -478,30 +586,43 @@ public:
478 // record only says that this one is still unspoken for. 586 // record only says that this one is still unspoken for.
479 bool WritePool(int64 nIndex, const vector<unsigned char>& vchPubKey) 587 bool WritePool(int64 nIndex, const vector<unsigned char>& vchPubKey)
480 { 588 {
481 return Write(make_pair(string("pool"), nIndex), vchPubKey); 589 return WriteWalletRecord(make_pair(string("pool"), nIndex), vchPubKey);
482 } 590 }
483 591
484 bool ErasePool(int64 nIndex) 592 bool ErasePool(int64 nIndex)
485 { 593 {
486 return Erase(make_pair(string("pool"), nIndex)); 594 return EraseWalletRecord(make_pair(string("pool"), nIndex));
487 } 595 }
488 596
489 template<typename T> 597 template<typename T>
490 bool ReadSetting(const string& strKey, T& value) 598 bool ReadSetting(const string& strKey, T& value)
491 { 599 {
492 return Read(make_pair(string("setting"), strKey), value); 600 return ReadWalletRecord(make_pair(string("setting"), strKey), value);
493 } 601 }
494 602
495 template<typename T> 603 template<typename T>
496 bool WriteSetting(const string& strKey, const T& value) 604 bool WriteSetting(const string& strKey, const T& value)
497 { 605 {
498 return Write(make_pair(string("setting"), strKey), value); 606 return WriteWalletRecord(make_pair(string("setting"), strKey), value);
499 } 607 }
500 608
501 bool LoadWallet(vector<unsigned char>& vchDefaultKeyRet); 609 bool LoadWallet(vector<unsigned char>& vchDefaultKeyRet);
502}; 610};
503 611
612class CWalletRecordVisitor
613{
614public:
615 virtual ~CWalletRecordVisitor() { }
616 virtual bool VisitWalletRecord(const CDataStream& ssKey,
617 const CDataStream& ssValue,
618 string& strErrorRet) = 0;
619};
620
621bool ScanWalletRecords(CWalletRecordVisitor& visitor, string& strErrorRet);
622
504bool LoadWallet(); 623bool LoadWallet();
624bool LoadWalletFromSQLite(const string& strPath);
625bool LoadWalletFromSQLiteRuntime(const string& strPath);
505 626
506// Writes a wallet.dat that opens on its own, anywhere -- see the definition in 627// Writes a wallet.dat that opens on its own, anywhere -- see the definition in
507// db.cpp for why a plain file copy does not. Still a point-in-time snapshot: 628// db.cpp for why a plain file copy does not. Still a point-in-time snapshot:
diff --git a/src/fuzz_net_message.cpp b/src/fuzz_net_message.cpp
new file mode 100644
index 0000000..1104d90
--- /dev/null
+++ b/src/fuzz_net_message.cpp
@@ -0,0 +1,32 @@
1// Copyright (c) 2026 Bitflash developers
2// Distributed under the MIT/X11 software license.
3//
4// libFuzzer harness for the P2P message envelope parser. It feeds arbitrary
5// receive-buffer bytes into ProcessMessages() through a dummy CNode, without
6// opening sockets or starting node threads.
7
8#include "headers_core.h"
9
10#include <stdint.h>
11#include <stddef.h>
12
13// Stand-ins normally supplied by the GUI/headless entry objects.
14map<string,string> mapAddressBook;
15bool gPoolServerRunning = false;
16void MainFrameRepaint() {}
17
18extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
19{
20 // Keep this first target focused on parser behavior, not multi-megabyte
21 // allocation pressure. The protocol size guards have their own selftest.
22 if (size > 4096)
23 return 0;
24
25 CNode node(INVALID_SOCKET, CAddress("127.0.0.1"));
26 node.nVersion = 0; // exercise pre-version gating plus the version parser
27 if (size > 0)
28 node.vRecv.write((const char*)data, (int)size);
29
30 ProcessMessages(&node);
31 return 0;
32}
diff --git a/src/fuzz_script.cpp b/src/fuzz_script.cpp
new file mode 100644
index 0000000..28c84f6
--- /dev/null
+++ b/src/fuzz_script.cpp
@@ -0,0 +1,59 @@
1// Copyright (c) 2026 Bitflash developers
2// Distributed under the MIT/X11 software license.
3//
4// libFuzzer harness for raw script parsing and evaluation. It keeps the
5// target deliberately small: arbitrary bytes become one CScript, then the
6// parser, standard-script extractors and EvalScript all see that same input.
7
8#include "headers_core.h"
9
10#include <stdint.h>
11#include <stddef.h>
12
13// Stand-ins normally supplied by the GUI/headless entry objects.
14map<string,string> mapAddressBook;
15bool gPoolServerRunning = false;
16void MainFrameRepaint() {}
17
18extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size)
19{
20 if (size < 1 || size > 4096)
21 return 0;
22
23 try
24 {
25 int nHashType = data[0];
26 const unsigned char* pbegin = data + 1;
27 const unsigned char* pend = data + size;
28 CScript script(pbegin, pend);
29
30 CScript::const_iterator pc = script.begin();
31 opcodetype opcode;
32 vector<unsigned char> vchPush;
33 while (pc < script.end())
34 {
35 if (!script.GetOp(pc, opcode, vchPush))
36 break;
37 }
38
39 script.GetSigOpCount();
40 script.ToString();
41
42 vector<unsigned char> vchPubKey;
43 ExtractPubKey(script, false, vchPubKey);
44 uint160 hash160;
45 ExtractHash160(script, hash160);
46
47 CTransaction tx;
48 tx.vin.push_back(CTxIn(COutPoint(), CScript()));
49 tx.vout.push_back(CTxOut(0, CScript()));
50
51 vector<vector<unsigned char> > stack;
52 EvalScript(script, tx, 0, nHashType, &stack);
53 }
54 catch (...)
55 {
56 }
57
58 return 0;
59}
diff --git a/src/gui.cpp b/src/gui.cpp
index 8f1d204..ff6059a 100644
--- a/src/gui.cpp
+++ b/src/gui.cpp
@@ -11,10 +11,16 @@
11#include <ctime> 11#include <ctime>
12#include <cstdio> 12#include <cstdio>
13#include <sstream> 13#include <sstream>
14#ifdef _WIN32
15#include <shellapi.h> // ShellExecuteA, to open the website in the browser
16#else
17#include <unistd.h>
18#endif
14#include <iomanip> 19#include <iomanip>
15#include <atomic> 20#include <atomic>
16#include <mutex> 21#include <mutex>
17#include <thread> 22#include <thread>
23#include <system_error>
18#include "bip32.h" 24#include "bip32.h"
19#include "walletcmd.h" 25#include "walletcmd.h"
20#include "font_roboto.h" 26#include "font_roboto.h"
@@ -172,7 +178,11 @@ static bool PathIsInsideDataDir(const std::string& strPath)
172 178
173static void SetDefaultBackupPath() 179static void SetDefaultBackupPath()
174{ 180{
175 std::string path = DefaultBackupDir() + "/wallet-backup-" + BackupTimeSuffix() + ".dat"; 181 // Match the extension to the live backend: BackupWallet copies wallet.sqlite
182 // under the SQLite backend and wallet.dat under Berkeley DB, so a ".dat"
183 // default name would put SQLite contents in a misleadingly-named file.
184 const char* pszExt = WalletSQLiteRuntimeActive() ? ".sqlite" : ".dat";
185 std::string path = DefaultBackupDir() + "/wallet-backup-" + BackupTimeSuffix() + pszExt;
176 strncpy(g_backupPath, path.c_str(), sizeof(g_backupPath)-1); 186 strncpy(g_backupPath, path.c_str(), sizeof(g_backupPath)-1);
177 g_backupPath[sizeof(g_backupPath)-1] = '\0'; 187 g_backupPath[sizeof(g_backupPath)-1] = '\0';
178} 188}
@@ -910,8 +920,16 @@ static void DrawOptionsDialog()
910 920
911 if (nMineMode == MINE_OPERATOR) { 921 if (nMineMode == MINE_OPERATOR) {
912 if (!gPoolServerRunning) { 922 if (!gPoolServerRunning) {
913 gPoolServerRunning = true; 923 // Claim the server is up only once it is. The flag used to
914 _beginthread(ThreadRPCServer, 0, NULL); 924 // be set first and the result never looked at, so a refused
925 // spawn left an operator with a pool that reports itself
926 // running, no RPC server for miners to reach, and nothing
927 // anywhere saying why. The flag also latches: with it set,
928 // this branch is never retried.
929 if (_beginthread(ThreadRPCServer, 0, NULL) == (uintptr_t)-1)
930 printf("Error: _beginthread(ThreadRPCServer) failed -- pool server not started\n");
931 else
932 gPoolServerRunning = true;
915 } else if (!gPoolRunning) { 933 } else if (!gPoolRunning) {
916 gPoolRunning = true; 934 gPoolRunning = true;
917 } 935 }
@@ -1124,11 +1142,28 @@ static void DrawRestorePhraseDialog()
1124 } 1142 }
1125 g_restoreDerived.store(0); 1143 g_restoreDerived.store(0);
1126 g_restoreRecovered.store(0); 1144 g_restoreRecovered.store(0);
1145 // Set before the spawn, not after: RestoreThread clears this
1146 // flag itself when it finishes, and a short restore can beat
1147 // us back here and have its `false` overwritten, leaving the
1148 // dialog disabled forever.
1127 g_restoreRunning.store(true); 1149 g_restoreRunning.store(true);
1128 // On its own thread: a restore walks the whole chain once per 1150 // On its own thread: a restore walks the whole chain once per
1129 // batch and would otherwise freeze the window for as long as it 1151 // batch and would otherwise freeze the window for as long as it
1130 // takes. 1152 // takes. A refused spawn is one restore that does not happen,
1131 std::thread(RestoreThread, std::string(g_restoreInput)).detach(); 1153 // not a reason to take the window down with an uncaught
1154 // std::system_error -- and the flag has to come back off, or
1155 // the dialog sits greyed out reporting a search nobody is
1156 // running.
1157 try
1158 {
1159 std::thread(RestoreThread, std::string(g_restoreInput)).detach();
1160 }
1161 catch (const std::system_error& e)
1162 {
1163 g_restoreRunning.store(false);
1164 std::lock_guard<std::mutex> lock(g_restoreResultMutex);
1165 g_restoreResult = std::string("Could not start the search: ") + e.what();
1166 }
1132 } 1167 }
1133 } 1168 }
1134 if (fBusy) 1169 if (fBusy)
@@ -1250,15 +1285,15 @@ static void DrawWalletSafetyDialog()
1250 if (g_recoveryAudit.fHaveSeed) 1285 if (g_recoveryAudit.fHaveSeed)
1251 { 1286 {
1252 ImGui::Text("Derivation schema: %s", HDKeySchemaName(g_recoveryAudit.nSchema).c_str()); 1287 ImGui::Text("Derivation schema: %s", HDKeySchemaName(g_recoveryAudit.nSchema).c_str());
1253 ImGui::Text("BIP44 coin type: %u (provisional BITFLASH)", g_recoveryAudit.nCoinType); 1288 ImGui::Text("BIP44 coin type: %u (SLIP-0044 BITFLASH)", g_recoveryAudit.nCoinType);
1254 } 1289 }
1255 ImGui::Text("Phrase-backed spendable balance: %s BTF", 1290 ImGui::Text("Phrase-backed spendable balance: %s BTF",
1256 FmtMoney(g_recoveryAudit.nRecoverableCredit).c_str()); 1291 FmtMoney(g_recoveryAudit.nRecoverableCredit).c_str());
1257 ImGui::Text("Wallet.dat-only spendable balance: %s BTF", 1292 ImGui::Text("Not phrase-backed spendable balance: %s BTF",
1258 FmtMoney(g_recoveryAudit.nLegacyCredit).c_str()); 1293 FmtMoney(g_recoveryAudit.nLegacyCredit).c_str());
1259 ImGui::Text("Phrase-backed immature mining rewards: %s BTF", 1294 ImGui::Text("Phrase-backed immature mining rewards: %s BTF",
1260 FmtMoney(g_recoveryAudit.nRecoverableImmatureCredit).c_str()); 1295 FmtMoney(g_recoveryAudit.nRecoverableImmatureCredit).c_str());
1261 ImGui::Text("Wallet.dat-only immature mining rewards: %s BTF", 1296 ImGui::Text("Not phrase-backed immature mining rewards: %s BTF",
1262 FmtMoney(g_recoveryAudit.nLegacyImmatureCredit).c_str()); 1297 FmtMoney(g_recoveryAudit.nLegacyImmatureCredit).c_str());
1263 if (!g_recoveryAudit.fDeriveComplete) 1298 if (!g_recoveryAudit.fDeriveComplete)
1264 { 1299 {
@@ -1272,8 +1307,8 @@ static void DrawWalletSafetyDialog()
1272 ImGui::PushStyleColor(ImGuiCol_Text, ImVec4(1.0f, 0.75f, 0.25f, 1.0f)); 1307 ImGui::PushStyleColor(ImGuiCol_Text, ImVec4(1.0f, 0.75f, 0.25f, 1.0f));
1273 ImGui::TextWrapped( 1308 ImGui::TextWrapped(
1274 "Some coins are on keys the phrase does not reproduce. " 1309 "Some coins are on keys the phrase does not reproduce. "
1275 "Keep wallet.dat backups until that balance has been moved to a " 1310 "Keep a file backup of this wallet until that balance has been "
1276 "phrase-backed address."); 1311 "moved to a phrase-backed address.");
1277 ImGui::PopStyleColor(); 1312 ImGui::PopStyleColor();
1278 } 1313 }
1279 else if (HaveHDSeed() && 1314 else if (HaveHDSeed() &&
@@ -1369,16 +1404,38 @@ static void DrawDiagnosticsDialog()
1369 ImGui::End(); 1404 ImGui::End();
1370} 1405}
1371 1406
1407static void OpenURL(const char* pszUrl)
1408{
1409#ifdef _WIN32
1410 ShellExecuteA(NULL, "open", pszUrl, NULL, NULL, SW_SHOWNORMAL);
1411#else
1412 std::string cmd = std::string("xdg-open '") + pszUrl + "' >/dev/null 2>&1 &";
1413 int r = system(cmd.c_str());
1414 (void)r;
1415#endif
1416}
1417
1372static void DrawAboutDialog() 1418static void DrawAboutDialog()
1373{ 1419{
1374 if (!g_showAbout) return; 1420 if (!g_showAbout) return;
1375 ImGui::SetNextWindowSize(ImVec2(380.0f, 170.0f), ImGuiCond_Always); 1421 ImGui::SetNextWindowSize(ImVec2(380.0f, 205.0f), ImGuiCond_Always);
1376 ImGui::SetNextWindowPos(ImGui::GetMainViewport()->GetCenter(), 1422 ImGui::SetNextWindowPos(ImGui::GetMainViewport()->GetCenter(),
1377 ImGuiCond_Always, ImVec2(0.5f, 0.5f)); 1423 ImGuiCond_Always, ImVec2(0.5f, 0.5f));
1378 if (ImGui::Begin("About Bitflash", &g_showAbout, 1424 if (ImGui::Begin("About Bitflash", &g_showAbout,
1379 ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoCollapse)) 1425 ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoCollapse))
1380 { 1426 {
1381 ImGui::TextColored(ImVec4(1.0f, 0.6f, 0.0f, 1.0f), "Bitflash BTF v1.1.0"); 1427 ImGui::TextColored(ImVec4(1.0f, 0.6f, 0.0f, 1.0f),
1428 "Bitflash BTF v" BITFLASH_VERSION_STRING);
1429 ImGui::Spacing();
1430 ImGui::PushStyleColor(ImGuiCol_Text, ImVec4(0.40f, 0.70f, 1.0f, 1.0f));
1431 ImGui::TextUnformatted("https://bitflash.network");
1432 ImGui::PopStyleColor();
1433 if (ImGui::IsItemHovered())
1434 {
1435 ImGui::SetMouseCursor(ImGuiMouseCursor_Hand);
1436 if (ImGui::IsMouseClicked(0))
1437 OpenURL("https://bitflash.network");
1438 }
1382 ImGui::Spacing(); 1439 ImGui::Spacing();
1383 ImGui::TextWrapped( 1440 ImGui::TextWrapped(
1384 "CPU-only cryptocurrency. RandomX proof of work. " 1441 "CPU-only cryptocurrency. RandomX proof of work. "
@@ -1391,6 +1448,184 @@ static void DrawAboutDialog()
1391 ImGui::End(); 1448 ImGui::End();
1392} 1449}
1393 1450
1451// First-run offer to move a Berkeley DB wallet to the SQLite backend. Only the
1452// button actions touch anything: "Convert" exports wallet.dat to wallet.sqlite
1453// and records the backend marker (wallet.dat is left untouched, so a restart
1454// picks up SQLite while the old file stays as a fallback); "Keep Berkeley DB"
1455// just records that choice so the offer does not return; "Decide later" leaves
1456// no marker and asks again next start.
1457static bool g_showBackendWizard = false;
1458static std::string g_wizardStatus;
1459static bool g_wizardConverted = false;
1460static int g_restartArgc = 0;
1461static char** g_restartArgv = NULL;
1462
1463#ifdef _WIN32
1464// Quote one argument so CommandLineToArgvW parses it back to exactly this
1465// string: only backslashes that immediately precede a double quote (including
1466// the closing one) are doubled, so ordinary Windows paths are not mangled.
1467static std::string QuoteArgWin(const std::string& a)
1468{
1469 if (!a.empty() && a.find_first_of(" \t\"") == std::string::npos)
1470 return a;
1471 std::string r = "\"";
1472 size_t bs = 0;
1473 for (size_t i = 0; i < a.size(); i++)
1474 {
1475 if (a[i] == '\\') { bs++; continue; }
1476 if (a[i] == '"') { r.append(bs * 2 + 1, '\\'); r += '"'; bs = 0; continue; }
1477 r.append(bs, '\\'); bs = 0; r += a[i];
1478 }
1479 r.append(bs * 2, '\\');
1480 r += "\"";
1481 return r;
1482}
1483#endif
1484
1485// Relaunch Bitflash with the same arguments and datadir, then ask this instance
1486// to exit -- but only once the relaunch has actually started, so a failure never
1487// leaves the user with no node running. The new instance waits (-restartwait)
1488// before it opens the network port and the block index, so this one is gone
1489// first and they never fight over the same files.
1490static void RestartApplication()
1491{
1492 bool fLaunched = false;
1493#ifdef _WIN32
1494 char exePath[MAX_PATH];
1495 if (GetModuleFileNameA(NULL, exePath, MAX_PATH) == 0)
1496 return;
1497 std::string strCmd = QuoteArgWin(exePath);
1498 bool fHasDataDir = false;
1499 for (int i = 1; i < g_restartArgc; i++)
1500 {
1501 std::string a = g_restartArgv[i] ? g_restartArgv[i] : "";
1502 if (a.find("-datadir=") == 0 || a.find("/datadir=") == 0)
1503 fHasDataDir = true;
1504 strCmd += " " + QuoteArgWin(a);
1505 }
1506 if (!fHasDataDir)
1507 strCmd += " " + QuoteArgWin("-datadir=" + GetAppDir());
1508 strCmd += " -restartwait";
1509
1510 std::vector<char> cmdBuf(strCmd.begin(), strCmd.end());
1511 cmdBuf.push_back('\0');
1512 STARTUPINFOA si; memset(&si, 0, sizeof(si)); si.cb = sizeof(si);
1513 PROCESS_INFORMATION pi; memset(&pi, 0, sizeof(pi));
1514 if (CreateProcessA(NULL, &cmdBuf[0], NULL, NULL, FALSE, 0, NULL, NULL, &si, &pi))
1515 {
1516 CloseHandle(pi.hProcess);
1517 CloseHandle(pi.hThread);
1518 fLaunched = true;
1519 }
1520#else
1521 pid_t pid = fork();
1522 if (pid == 0)
1523 {
1524 // Child: wait for the parent to release the port and files, then re-exec.
1525 // /proc/self/exe is the reliable absolute path; argv[0] may be relative
1526 // or a bare name that execv() cannot resolve.
1527 sleep(2);
1528 char exePath[4096];
1529 ssize_t len = readlink("/proc/self/exe", exePath, sizeof(exePath) - 1);
1530 if (len > 0)
1531 {
1532 exePath[len] = '\0';
1533 execv(exePath, g_restartArgv);
1534 }
1535 if (g_restartArgv)
1536 execv(g_restartArgv[0], g_restartArgv);
1537 _exit(127);
1538 }
1539 fLaunched = (pid > 0);
1540#endif
1541
1542 if (fLaunched)
1543 {
1544 fShutdown = true;
1545 if (glfwGetCurrentContext())
1546 glfwSetWindowShouldClose(glfwGetCurrentContext(), true);
1547 }
1548 else
1549 g_wizardStatus = "Could not relaunch automatically -- please restart "
1550 "Bitflash yourself to switch to the SQLite wallet.";
1551}
1552
1553static void DrawBackendWizardDialog()
1554{
1555 if (!g_showBackendWizard) return;
1556 ImGui::SetNextWindowSize(ImVec2(560.0f, 0.0f), ImGuiCond_Always);
1557 ImGui::SetNextWindowPos(ImGui::GetMainViewport()->GetCenter(),
1558 ImGuiCond_Always, ImVec2(0.5f, 0.5f));
1559 if (ImGui::Begin("Wallet Storage", &g_showBackendWizard,
1560 ImGuiWindowFlags_NoResize | ImGuiWindowFlags_NoCollapse))
1561 {
1562 if (g_wizardConverted)
1563 {
1564 ImGui::TextColored(ImVec4(0.35f, 1.0f, 0.45f, 1.0f), "Converted to SQLite.");
1565 ImGui::TextWrapped("%s", g_wizardStatus.c_str());
1566 ImGui::Spacing();
1567 ImGui::TextWrapped(
1568 "The switch takes effect when Bitflash restarts. Until then it is "
1569 "still running on the old Berkeley DB wallet. Your wallet.dat is "
1570 "kept untouched as a fallback.");
1571 ImGui::Spacing();
1572 if (ImGui::Button("Restart Now", ImVec2(140.0f, 0.0f)))
1573 RestartApplication();
1574 ImGui::SameLine();
1575 if (ImGui::Button("Later", ImVec2(120.0f, 0.0f)))
1576 g_showBackendWizard = false;
1577 ImGui::Spacing();
1578 ImGui::TextDisabled(
1579 "If you keep mining before restarting, new coins go to the old "
1580 "wallet until you switch.");
1581 }
1582 else
1583 {
1584 ImGui::TextWrapped(
1585 "Bitflash can store your wallet in a new SQLite format: a single "
1586 "file that is easier to back up and does not depend on the "
1587 "database/ directory beside it.");
1588 ImGui::Spacing();
1589 ImGui::TextWrapped(
1590 "Your current Berkeley DB wallet (wallet.dat) is kept exactly as it "
1591 "is -- nothing is deleted, and you can go back to it at any time. "
1592 "The change takes effect after you restart.");
1593 ImGui::Spacing();
1594 if (ImGui::Button("Convert to SQLite", ImVec2(180.0f, 0.0f)))
1595 {
1596 int nCopied = 0;
1597 std::string strError;
1598 if (DoConvertWalletToSQLite(nCopied, strError))
1599 {
1600 g_wizardStatus =
1601 strprintf("%d record(s) copied to wallet.sqlite.", nCopied);
1602 g_wizardConverted = true;
1603 }
1604 else
1605 g_wizardStatus = "Conversion failed: " + strError;
1606 }
1607 ImGui::SameLine();
1608 if (ImGui::Button("Keep Berkeley DB", ImVec2(160.0f, 0.0f)))
1609 {
1610 WriteWalletBackendMarker("bdb");
1611 g_showBackendWizard = false;
1612 }
1613 ImGui::SameLine();
1614 if (ImGui::Button("Decide later", ImVec2(120.0f, 0.0f)))
1615 g_showBackendWizard = false;
1616
1617 if (!g_wizardStatus.empty())
1618 {
1619 ImGui::Spacing();
1620 ImGui::PushStyleColor(ImGuiCol_Text, ImVec4(1.0f, 0.45f, 0.35f, 1.0f));
1621 ImGui::TextWrapped("%s", g_wizardStatus.c_str());
1622 ImGui::PopStyleColor();
1623 }
1624 }
1625 }
1626 ImGui::End();
1627}
1628
1394// --------------------------------------------------------------------------- 1629// ---------------------------------------------------------------------------
1395// Entry point 1630// Entry point
1396// --------------------------------------------------------------------------- 1631// ---------------------------------------------------------------------------
@@ -1457,6 +1692,21 @@ int RunGUI(int argc, char* argv[])
1457 g_mineRadio = nMineMode; 1692 g_mineRadio = nMineMode;
1458 strncpy(g_participantPool, strParticipantPool.c_str(), sizeof(g_participantPool)-1); 1693 strncpy(g_participantPool, strParticipantPool.c_str(), sizeof(g_participantPool)-1);
1459 1694
1695 // Kept for the wizard's "Restart Now": relaunch with the same command line.
1696 g_restartArgc = argc;
1697 g_restartArgv = argv;
1698
1699 // First run on a Berkeley DB wallet with no recorded choice yet: offer to
1700 // convert to SQLite. Never when already on SQLite, once wallet.sqlite exists,
1701 // or once a choice has been recorded in the backend marker.
1702 if (!WalletSQLiteRuntimeActive() &&
1703 FileExists((GetAppDir() + "/wallet.dat").c_str()) &&
1704 !FileExists((GetAppDir() + "/wallet.sqlite").c_str()) &&
1705 ReadWalletBackendMarker().empty())
1706 {
1707 g_showBackendWizard = true;
1708 }
1709
1460 int frame = 0; 1710 int frame = 0;
1461 while (!glfwWindowShouldClose(window) && !fShutdown) 1711 while (!glfwWindowShouldClose(window) && !fShutdown)
1462 { 1712 {
@@ -1480,6 +1730,7 @@ int RunGUI(int argc, char* argv[])
1480 DrawRestorePhraseDialog(); 1730 DrawRestorePhraseDialog();
1481 DrawDiagnosticsDialog(); 1731 DrawDiagnosticsDialog();
1482 DrawAboutDialog(); 1732 DrawAboutDialog();
1733 DrawBackendWizardDialog();
1483 DrawStatusBar(); 1734 DrawStatusBar();
1484 1735
1485 ImGui::Render(); 1736 ImGui::Render();
diff --git a/src/headers_core.h b/src/headers_core.h
index 6a60b52..2824126 100644
--- a/src/headers_core.h
+++ b/src/headers_core.h
@@ -1,6 +1,11 @@
1// Core node headers -- no GUI framework dependency. 1// Core node headers -- no GUI framework dependency.
2// Included by gui.cpp and all non-GUI source files. 2// Included by gui.cpp and all non-GUI source files.
3 3
4// The one place the human-readable release version lives -- bump it at release.
5// It is what the About dialog shows. Protocol and transaction versions elsewhere
6// (nVersion fields) are unrelated and must not be changed for a release.
7#define BITFLASH_VERSION_STRING "1.2.20"
8
4#ifdef _MSC_VER 9#ifdef _MSC_VER
5#pragma warning(disable:4786) 10#pragma warning(disable:4786)
6#endif 11#endif
diff --git a/src/main.cpp b/src/main.cpp
index 77aba89..0fa6b89 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -14,6 +14,7 @@
14#include <atomic> 14#include <atomic>
15#include <mutex> 15#include <mutex>
16#include <thread> 16#include <thread>
17#include <system_error>
17#include "bip32.h" 18#include "bip32.h"
18#include "btfaddr.h" 19#include "btfaddr.h"
19#include "btftunnel.h" 20#include "btftunnel.h"
@@ -41,12 +42,30 @@ unsigned int nTransactionsUpdated = 0;
41map<COutPoint, CInPoint> mapNextTx; 42map<COutPoint, CInPoint> mapNextTx;
42 43
43map<uint256, CBlockIndex*> mapBlockIndex; 44map<uint256, CBlockIndex*> mapBlockIndex;
44// Bitflash genesis -- mined under RandomX PoW on 2026-07-22 45// Bitflash mainnet genesis -- mined under RandomX PoW for the 2026 stable
45// (nTime 1753142400, nBits 0x1f0fffff, RandomX key "Bitflash/RandomX/v1/...") 46// relaunch. Testnet gets its own resettable genesis below so block indexes,
46// Bitflash 2026 stable relaunch genesis (2-min blocks, RandomX). Mined values. 47// headers, and wallet scans cannot be accidentally mixed across networks.
47unsigned int GENESIS_NONCE = 3141; 48static const unsigned int MAINNET_GENESIS_TIME = 1753315200; // 2026-07-24 00:00:00 UTC
48uint256 hashGenesisBlock("0x5bd7cb255d814e48cebcdfb72da4dc87b34bd774227f8ceb546c5640f4bdc169"); 49static const unsigned int MAINNET_GENESIS_NONCE = 3141;
49uint256 hashGenesisMerkleRoot("0x1a45b4482532abb29b10e234d3f13132230525a339ecea91658ffa675a5b1325"); 50static const char* MAINNET_GENESIS_TIMESTAMP =
51 "Bitflash 24/Jul/2026 Fair launch: one CPU one vote, no premine";
52static const char* MAINNET_GENESIS_HASH =
53 "0x5bd7cb255d814e48cebcdfb72da4dc87b34bd774227f8ceb546c5640f4bdc169";
54static const char* MAINNET_GENESIS_MERKLE =
55 "0x1a45b4482532abb29b10e234d3f13132230525a339ecea91658ffa675a5b1325";
56
57static const unsigned int TESTNET_GENESIS_TIME = 1787184000; // 2026-08-20 00:00:00 UTC
58static const unsigned int TESTNET_GENESIS_NONCE = 3199;
59static const char* TESTNET_GENESIS_TIMESTAMP =
60 "Bitflash testnet 20/Aug/2026: resettable public test chain";
61static const char* TESTNET_GENESIS_HASH =
62 "0xa9d11c6d697bcb7aea0653bc088b12dcf9f44e732df205162cf085fd9d9963eb";
63static const char* TESTNET_GENESIS_MERKLE =
64 "0x510b293105e3ff85b8d4c3920e357f85d1a86bf199eb4e7d11f04f380030ec5c";
65
66unsigned int GENESIS_NONCE = MAINNET_GENESIS_NONCE;
67uint256 hashGenesisBlock(MAINNET_GENESIS_HASH);
68uint256 hashGenesisMerkleRoot(MAINNET_GENESIS_MERKLE);
50CBlockIndex* pindexGenesisBlock = NULL; 69CBlockIndex* pindexGenesisBlock = NULL;
51int nBestHeight = -1; 70int nBestHeight = -1;
52uint256 hashBestChain = 0; 71uint256 hashBestChain = 0;
@@ -64,6 +83,137 @@ CBlockIndex* pindexBest = NULL;
64map<uint256, CBlock*> mapOrphanBlocks; 83map<uint256, CBlock*> mapOrphanBlocks;
65multimap<uint256, CBlock*> mapOrphanBlocksByPrev; 84multimap<uint256, CBlock*> mapOrphanBlocksByPrev;
66 85
86// Block bodies we have sent a getdata for and not yet received, so the
87// failover pass in SendMessages can tell "requested and being served" from
88// "requested and gone quiet" and re-issue the request to a different peer
89// (#33). Keyed by block hash; setAsked is the peers already tried since the
90// round last restarted, so the retry rotates instead of hammering the
91// announcer that is not serving. Guarded by cs_main, which both the message
92// handlers and SendMessages already hold.
93//
94// Capped small on purpose. At the tip there is one block in flight; during
95// initial download there are legitimately hundreds outstanding to one peer,
96// and rotating those would duplicate the whole download to every other peer.
97//
98// The cap is a shared resource and nothing on the path from inv to insertion
99// can tell a real block hash from 32 random bytes -- AlreadyHave only asks
100// whether we already hold the block, and there is no header to check work
101// against. So the table does not refuse a new hash when it is full; it makes
102// room. See MakeRoomInBlockFetch for the rule, which is what stops one peer
103// filling all eight slots and switching failover off for everybody.
104//
105// The initialisers matter: the insertion path uses operator[] and then tests
106// nTimeFirstAsked == 0 to detect a fresh entry. That worked on
107// value-initialisation alone, but only for as long as nobody gave this struct
108// a constructor.
109struct CBlockFetch
110{
111 int64 nTimeFirstAsked = 0; // when the first getdata for this block went out
112 int64 nTimeAsked = 0; // when the latest getdata went out
113 CAddress addrAnnounced; // peer whose announcement put this hash here
114 set<CAddress> setAsked; // peers tried since the round last restarted
115};
116map<uint256, CBlockFetch> mapBlockFetch;
117static const int BLOCK_FETCH_TRACK_MAX = 8;
118// A block nobody has served for this long is not a fetch problem: the hash may
119// have been announced and never existed. Drop the entry rather than getdata
120// forever; a re-announcement recreates it.
121static const int BLOCK_FETCH_GIVEUP_SECS = 10 * 60;
122
123// Failover firing at all is worth saying out loud: it means peers are
124// announcing blocks they then do not serve, which is the pathology of #33 and
125// the thing nobody could otherwise see. Naming the peer on every re-request is
126// a different matter. A peer can drive that line as often as it likes by
127// announcing hashes it will not serve, and what it prints is a peer address,
128// which is the one thing the .btf transport otherwise takes trouble not to
129// spread around. So the per-peer detail sits behind the net category and the
130// unconditional line is a count over a window no peer can shorten: the first
131// failover after a quiet spell prints at once, because the change from "blocks
132// arrive" to "blocks are announced and not served" is the event worth seeing,
133// and everything after it is summarised once a minute.
134static const int BLOCK_FETCH_REPORT_SECS = 60;
135static int64 nBlockFetchWindowStart = 0;
136static int nBlockFetchFailovers = 0;
137
138// How stale our own best block may be before we stop calling this the tip.
139// Blocks target two minutes, so this is thirty intervals: far longer than any
140// stall worth failing over from -- the one in #33 ran sixteen minutes -- and
141// far shorter than catching up, which starts hours or days back.
142static const int BLOCK_FETCH_TIP_AGE_SECS = 60 * 60;
143
144// Whether this node is tracking the tip rather than still catching up.
145//
146// Measured off the age of our own best block, not off any height a peer told
147// us. nStartingHeight is optional in the version message and every node
148// released so far sends the short form, so it is -1 for essentially every
149// peer; and where it is filled in later it is filled in from the height of a
150// block that peer served us, which during catch-up tracks our own height and
151// would report "at the tip" for the whole download. Block timestamps are the
152// one clock here that peers cannot move: CheckBlock refuses anything more than
153// two hours ahead and AcceptBlock refuses anything at or before the previous
154// median, so the only way to make this read "at the tip" falsely is to mine,
155// and all that buys is the 15s spacing this code had unconditionally anyway.
156bool IsBlockFetchNearTip()
157{
158 return pindexBest && (int64)pindexBest->nTime > GetTime() - BLOCK_FETCH_TIP_AGE_SECS;
159}
160
161// Drop one entry so a hash announced by addrNew can be tracked.
162//
163// Refusing the insertion instead was a remote off-switch for the whole
164// feature. An inv carrying eight hashes that never existed filled every slot
165// -- nothing between inv and here can tell those from real blocks -- and no
166// genuinely new block was tracked again until the give-up timer expired ten
167// minutes later, re-armable for one message per ten minutes from a single
168// connection.
169//
170// So evict instead, by occupancy: a peer may only take a slot from a peer
171// holding strictly more slots than itself, and otherwise recycles one of its
172// own. That is what keeps eviction from becoming the same attack in a new
173// shape. A peer that has filled the table is by definition the heaviest
174// holder, so its own entries are the only ones it can displace, while a peer
175// announcing its first block holds nothing and therefore always evicts from
176// the heaviest -- a genuine announcement is never refused a slot, whatever the
177// attacker holds or however many connections it holds it from. The one case
178// that costs a real fetch is eight distinct peers holding one entry each,
179// where a ninth takes the oldest; that is a busy healthy node, not an attack,
180// and losing the oldest of eight beats refusing the ninth.
181static void MakeRoomInBlockFetch(const CAddress& addrNew)
182{
183 if (mapBlockFetch.empty())
184 return;
185
186 map<CAddress, int> mapHeld;
187 for (map<uint256, CBlockFetch>::iterator mi = mapBlockFetch.begin(); mi != mapBlockFetch.end(); ++mi)
188 mapHeld[(*mi).second.addrAnnounced]++;
189
190 int nMine = mapHeld.count(addrNew) ? mapHeld[addrNew] : 0;
191
192 int nMax = 0;
193 CAddress addrVictim;
194 for (map<CAddress, int>::iterator mi = mapHeld.begin(); mi != mapHeld.end(); ++mi)
195 if ((*mi).second > nMax)
196 {
197 nMax = (*mi).second;
198 addrVictim = (*mi).first;
199 }
200
201 // Nobody is holding more than this peer, so it recycles its own oldest
202 // rather than taking a slot off an equal.
203 if (nMax <= nMine)
204 addrVictim = addrNew;
205
206 map<uint256, CBlockFetch>::iterator miVictim = mapBlockFetch.end();
207 for (map<uint256, CBlockFetch>::iterator mi = mapBlockFetch.begin(); mi != mapBlockFetch.end(); ++mi)
208 if ((*mi).second.addrAnnounced == addrVictim &&
209 (miVictim == mapBlockFetch.end() ||
210 (*mi).second.nTimeFirstAsked < (*miVictim).second.nTimeFirstAsked))
211 miVictim = mi;
212
213 if (miVictim != mapBlockFetch.end())
214 mapBlockFetch.erase(miVictim);
215}
216
67map<uint256, CDataStream*> mapOrphanTransactions; 217map<uint256, CDataStream*> mapOrphanTransactions;
68multimap<uint256, CDataStream*> mapOrphanTransactionsByPrev; 218multimap<uint256, CDataStream*> mapOrphanTransactionsByPrev;
69 219
@@ -109,7 +259,11 @@ string strPayoutFile; // participant: /payoutfile with one addres
109int nHashThreads = 0; // participant: /hashthreads override (0 = cores-1) 259int nHashThreads = 0; // participant: /hashthreads override (0 = cores-1)
110string strPoolName = "Bitflash Pool"; 260string strPoolName = "Bitflash Pool";
111string strPoolDashboardUrl; 261string strPoolDashboardUrl;
262string strPoolStatusFile;
263string strPoolRoundsFile;
112double dPoolFeePercent = 0.0; 264double dPoolFeePercent = 0.0;
265bool fStratumBridge = false;
266int nStratumBridgePort = 3333;
113 267
114static std::atomic<uint64> gParticipantSharesSent{0}; 268static std::atomic<uint64> gParticipantSharesSent{0};
115static std::atomic<uint64> gParticipantSharesAccepted{0}; 269static std::atomic<uint64> gParticipantSharesAccepted{0};
@@ -118,6 +272,63 @@ static std::atomic<uint64> gParticipantHashRateX1000{0};
118static std::mutex gParticipantStatusMutex; 272static std::mutex gParticipantStatusMutex;
119static std::string gParticipantStatus; 273static std::string gParticipantStatus;
120 274
275void SelectGenesisParams(bool fTestNetIn)
276{
277 static bool fGenesisParamsSelected = false;
278 static bool fGenesisParamsTestNet = false;
279 if (fGenesisParamsSelected && fGenesisParamsTestNet == fTestNetIn)
280 return;
281
282 fGenesisParamsSelected = true;
283 fGenesisParamsTestNet = fTestNetIn;
284
285 if (fTestNetIn)
286 {
287 GENESIS_NONCE = TESTNET_GENESIS_NONCE;
288 hashGenesisBlock = uint256(TESTNET_GENESIS_HASH);
289 hashGenesisMerkleRoot = uint256(TESTNET_GENESIS_MERKLE);
290 }
291 else
292 {
293 GENESIS_NONCE = MAINNET_GENESIS_NONCE;
294 hashGenesisBlock = uint256(MAINNET_GENESIS_HASH);
295 hashGenesisMerkleRoot = uint256(MAINNET_GENESIS_MERKLE);
296 }
297}
298
299void SelectChainParams(bool fTestNetIn)
300{
301 SelectNetworkParams(fTestNetIn);
302 SelectGenesisParams(fTestNetIn);
303}
304
305static CBlock BuildGenesisBlock()
306{
307 const char* pszTimestamp = IsTestNet() ?
308 TESTNET_GENESIS_TIMESTAMP : MAINNET_GENESIS_TIMESTAMP;
309
310 CTransaction txNew;
311 txNew.vin.resize(1);
312 txNew.vout.resize(1);
313 txNew.vin[0].scriptSig = CScript() << 486604799 << CBigNum(4) <<
314 vector<unsigned char>((unsigned char*)pszTimestamp,
315 (unsigned char*)pszTimestamp + strlen(pszTimestamp));
316 txNew.vout[0].nValue = 50 * COIN;
317 txNew.vout[0].scriptPubKey = CScript() <<
318 CBigNum("0x5F1DF16B2B704C8A578D0BBAF74D385CDE12C11EE50455F3C438EF4C3FBCF649B6DE611FEAE06279A60939E028A8D65C10B73071A6F16719274855FEB0FD8A6704") <<
319 OP_CHECKSIG;
320
321 CBlock block;
322 block.vtx.push_back(txNew);
323 block.hashPrevBlock = 0;
324 block.hashMerkleRoot = block.BuildMerkleTree();
325 block.nVersion = 1;
326 block.nTime = IsTestNet() ? TESTNET_GENESIS_TIME : MAINNET_GENESIS_TIME;
327 block.nBits = 0x1f0fffff; // = bnProofOfWorkLimit (Bitflash RandomX floor)
328 block.nNonce = GENESIS_NONCE;
329 return block;
330}
331
121void SetParticipantMiningStatus(const std::string& status) 332void SetParticipantMiningStatus(const std::string& status)
122{ 333{
123 // Updates the GUI status string only. Logging is done at each call site 334 // Updates the GUI status string only. Logging is done at each call site
@@ -190,6 +401,9 @@ int ParticipantHashThreadCount()
190 return nWorkers - 1; 401 return nWorkers - 1;
191} 402}
192 403
404static void SecureClearBytes(vector<unsigned char>& vch);
405static void SecureClearKeyingMaterial(CKeyingMaterial& vch);
406
193bool AddKey(const CKey& key) 407bool AddKey(const CKey& key)
194{ 408{
195 if (IsWalletEncrypted()) 409 if (IsWalletEncrypted())
@@ -199,9 +413,13 @@ bool AddKey(const CKey& key)
199 vector<unsigned char> vchCryptedSecret; 413 vector<unsigned char> vchCryptedSecret;
200 if (IsWalletLocked()) 414 if (IsWalletLocked())
201 return error("AddKey() : wallet is locked\n"); 415 return error("AddKey() : wallet is locked\n");
202 if (!EncryptSecret(vWalletMasterKey, vector<unsigned char>(vchPrivKey.begin(), vchPrivKey.end()), 416 vector<unsigned char> vchPlainSecret(vchPrivKey.begin(), vchPrivKey.end());
203 WalletKeyIV(key.GetPubKey()), vchCryptedSecret)) 417 if (!EncryptSecret(vWalletMasterKey, vchPlainSecret, WalletKeyIV(key.GetPubKey()), vchCryptedSecret))
418 {
419 SecureClearBytes(vchPlainSecret);
204 return error("AddKey() : encrypting key failed\n"); 420 return error("AddKey() : encrypting key failed\n");
421 }
422 SecureClearBytes(vchPlainSecret);
205 if (!AddCryptedKey(key.GetPubKey(), vchCryptedSecret)) 423 if (!AddCryptedKey(key.GetPubKey(), vchCryptedSecret))
206 return false; 424 return false;
207 return CWalletDB().WriteCryptedKey(key.GetPubKey(), vchCryptedSecret); 425 return CWalletDB().WriteCryptedKey(key.GetPubKey(), vchCryptedSecret);
@@ -225,17 +443,31 @@ bool IsWalletLocked()
225 return fWalletEncrypted && fWalletLocked; 443 return fWalletEncrypted && fWalletLocked;
226} 444}
227 445
446static void SecureClearBytes(vector<unsigned char>& vch)
447{
448 if (!vch.empty())
449 OPENSSL_cleanse(&vch[0], vch.size());
450 vch.clear();
451}
452
453static void SecureClearKeyingMaterial(CKeyingMaterial& vch)
454{
455 if (!vch.empty())
456 OPENSSL_cleanse(&vch[0], vch.size());
457 vch.clear();
458}
459
228void LockWallet() 460void LockWallet()
229{ 461{
230 if (!fWalletEncrypted) 462 if (!fWalletEncrypted)
231 return; 463 return;
232 CRITICAL_BLOCK(cs_mapKeys) 464 CRITICAL_BLOCK(cs_mapKeys)
233 { 465 {
234 vWalletMasterKey.clear(); 466 SecureClearKeyingMaterial(vWalletMasterKey);
235 if (!vchCryptedHDMaster.empty()) 467 if (!vchCryptedHDMaster.empty())
236 vchHDMaster.clear(); 468 SecureClearBytes(vchHDMaster);
237 if (!vchCryptedHDChainCode.empty()) 469 if (!vchCryptedHDChainCode.empty())
238 vchHDChainCode.clear(); 470 SecureClearBytes(vchHDChainCode);
239 fWalletLocked = true; 471 fWalletLocked = true;
240 } 472 }
241} 473}
@@ -260,14 +492,14 @@ bool DeriveWalletPassphraseKey(const string& strPassphrase,
260 chKey, chIV); 492 chKey, chIV);
261 if (n != 32) 493 if (n != 32)
262 { 494 {
263 memset(chKey, 0, sizeof(chKey)); 495 OPENSSL_cleanse(chKey, sizeof(chKey));
264 memset(chIV, 0, sizeof(chIV)); 496 OPENSSL_cleanse(chIV, sizeof(chIV));
265 return false; 497 return false;
266 } 498 }
267 vchKeyRet.assign(chKey, chKey + sizeof(chKey)); 499 vchKeyRet.assign(chKey, chKey + sizeof(chKey));
268 vchIVRet.assign(chIV, chIV + sizeof(chIV)); 500 vchIVRet.assign(chIV, chIV + sizeof(chIV));
269 memset(chKey, 0, sizeof(chKey)); 501 OPENSSL_cleanse(chKey, sizeof(chKey));
270 memset(chIV, 0, sizeof(chIV)); 502 OPENSSL_cleanse(chIV, sizeof(chIV));
271 return true; 503 return true;
272} 504}
273 505
@@ -328,6 +560,8 @@ bool DecryptSecret(const CKeyingMaterial& vchKey,
328 vchPlaintextRet.swap(vchOut); 560 vchPlaintextRet.swap(vchOut);
329 fOk = true; 561 fOk = true;
330 } 562 }
563 if (!fOk)
564 SecureClearBytes(vchOut);
331 EVP_CIPHER_CTX_free(ctx); 565 EVP_CIPHER_CTX_free(ctx);
332 return fOk; 566 return fOk;
333} 567}
@@ -403,6 +637,7 @@ bool GetWalletPrivKey(const vector<unsigned char>& vchPubKey,
403 } 637 }
404 638
405 CPrivKey vchPrivKey(vchPlaintext.begin(), vchPlaintext.end()); 639 CPrivKey vchPrivKey(vchPlaintext.begin(), vchPlaintext.end());
640 SecureClearBytes(vchPlaintext);
406 CKey key; 641 CKey key;
407 if (!key.SetPrivKey(vchPrivKey) || key.GetPubKey() != vchPubKey) 642 if (!key.SetPrivKey(vchPrivKey) || key.GetPubKey() != vchPubKey)
408 { 643 {
@@ -437,11 +672,20 @@ bool UnlockWallet(const string& strPassphrase, string& strErrorRet)
437 vector<unsigned char> vchMasterPlain; 672 vector<unsigned char> vchMasterPlain;
438 if (!DecryptSecret(vchPassKey, kMasterKey.vchCryptedKey, 673 if (!DecryptSecret(vchPassKey, kMasterKey.vchCryptedKey,
439 vchPassIV, vchMasterPlain)) 674 vchPassIV, vchMasterPlain))
675 {
676 SecureClearBytes(vchPassIV);
440 continue; 677 continue;
678 }
441 if (vchMasterPlain.size() != 32) 679 if (vchMasterPlain.size() != 32)
680 {
681 SecureClearBytes(vchMasterPlain);
682 SecureClearBytes(vchPassIV);
442 continue; 683 continue;
684 }
443 685
444 CKeyingMaterial vchCandidate(vchMasterPlain.begin(), vchMasterPlain.end()); 686 CKeyingMaterial vchCandidate(vchMasterPlain.begin(), vchMasterPlain.end());
687 SecureClearBytes(vchMasterPlain);
688 SecureClearBytes(vchPassIV);
445 bool fValid = mapCryptedKeys.empty(); 689 bool fValid = mapCryptedKeys.empty();
446 for (map<vector<unsigned char>, vector<unsigned char> >::iterator mci = mapCryptedKeys.begin(); 690 for (map<vector<unsigned char>, vector<unsigned char> >::iterator mci = mapCryptedKeys.begin();
447 mci != mapCryptedKeys.end(); ++mci) 691 mci != mapCryptedKeys.end(); ++mci)
@@ -450,6 +694,7 @@ bool UnlockWallet(const string& strPassphrase, string& strErrorRet)
450 if (!DecryptSecret(vchCandidate, (*mci).second, WalletKeyIV((*mci).first), vchPlaintext)) 694 if (!DecryptSecret(vchCandidate, (*mci).second, WalletKeyIV((*mci).first), vchPlaintext))
451 break; 695 break;
452 CPrivKey vchPrivKey(vchPlaintext.begin(), vchPlaintext.end()); 696 CPrivKey vchPrivKey(vchPlaintext.begin(), vchPlaintext.end());
697 SecureClearBytes(vchPlaintext);
453 CKey key; 698 CKey key;
454 if (key.SetPrivKey(vchPrivKey) && key.GetPubKey() == (*mci).first) 699 if (key.SetPrivKey(vchPrivKey) && key.GetPubKey() == (*mci).first)
455 fValid = true; 700 fValid = true;
@@ -465,10 +710,12 @@ bool UnlockWallet(const string& strPassphrase, string& strErrorRet)
465 WalletSecretIV("hdmaster"), vchPlainHDMaster) || 710 WalletSecretIV("hdmaster"), vchPlainHDMaster) ||
466 vchPlainHDMaster.size() != 32) 711 vchPlainHDMaster.size() != 32)
467 { 712 {
713 SecureClearBytes(vchPlainHDMaster);
468 strErrorRet = "could not decrypt HD master key"; 714 strErrorRet = "could not decrypt HD master key";
469 return false; 715 return false;
470 } 716 }
471 vchHDMaster = vchPlainHDMaster; 717 vchHDMaster = vchPlainHDMaster;
718 SecureClearBytes(vchPlainHDMaster);
472 } 719 }
473 if (!vchCryptedHDChainCode.empty()) 720 if (!vchCryptedHDChainCode.empty())
474 { 721 {
@@ -477,10 +724,12 @@ bool UnlockWallet(const string& strPassphrase, string& strErrorRet)
477 WalletSecretIV("hdchaincode"), vchPlainHDChainCode) || 724 WalletSecretIV("hdchaincode"), vchPlainHDChainCode) ||
478 vchPlainHDChainCode.size() != 32) 725 vchPlainHDChainCode.size() != 32)
479 { 726 {
727 SecureClearBytes(vchPlainHDChainCode);
480 strErrorRet = "could not decrypt HD chain code"; 728 strErrorRet = "could not decrypt HD chain code";
481 return false; 729 return false;
482 } 730 }
483 vchHDChainCode = vchPlainHDChainCode; 731 vchHDChainCode = vchPlainHDChainCode;
732 SecureClearBytes(vchPlainHDChainCode);
484 } 733 }
485 fWalletLocked = false; 734 fWalletLocked = false;
486 return true; 735 return true;
@@ -625,7 +874,7 @@ unsigned int nHDNext = 0;
625int nHDKeySchema = HD_SCHEMA_NONE; 874int nHDKeySchema = HD_SCHEMA_NONE;
626unsigned int nHDReceiveNext = 0; 875unsigned int nHDReceiveNext = 0;
627unsigned int nHDChangeNext = 0; 876unsigned int nHDChangeNext = 0;
628unsigned int nHDCoinType = HD_BIP44_COIN_TYPE_BITFLASH_PROVISIONAL; 877unsigned int nHDCoinType = HD_BIP44_COIN_TYPE_BITFLASH;
629 878
630string HDKeySchemaName(int nSchema) 879string HDKeySchemaName(int nSchema)
631{ 880{
@@ -781,7 +1030,7 @@ bool SetHDSeedFromMnemonic(const string& strMnemonic, string& strErrorRet)
781 1030
782 if (!CWalletDB().WriteHDMaster(master.privateKey, master.chainCode) || 1031 if (!CWalletDB().WriteHDMaster(master.privateKey, master.chainCode) ||
783 !CWalletDB().WriteHDSchema(HD_SCHEMA_BIP44) || 1032 !CWalletDB().WriteHDSchema(HD_SCHEMA_BIP44) ||
784 !CWalletDB().WriteHDCoinType(HD_BIP44_COIN_TYPE_BITFLASH_PROVISIONAL) || 1033 !CWalletDB().WriteHDCoinType(HD_BIP44_COIN_TYPE_BITFLASH) ||
785 !CWalletDB().WriteHDNext(0) || 1034 !CWalletDB().WriteHDNext(0) ||
786 !CWalletDB().WriteHDReceiveNext(1) || 1035 !CWalletDB().WriteHDReceiveNext(1) ||
787 !CWalletDB().WriteHDChangeNext(0)) 1036 !CWalletDB().WriteHDChangeNext(0))
@@ -794,7 +1043,7 @@ bool SetHDSeedFromMnemonic(const string& strMnemonic, string& strErrorRet)
794 nHDKeySchema = HD_SCHEMA_BIP44; 1043 nHDKeySchema = HD_SCHEMA_BIP44;
795 nHDReceiveNext = 1; 1044 nHDReceiveNext = 1;
796 nHDChangeNext = 0; 1045 nHDChangeNext = 0;
797 nHDCoinType = HD_BIP44_COIN_TYPE_BITFLASH_PROVISIONAL; 1046 nHDCoinType = HD_BIP44_COIN_TYPE_BITFLASH;
798 1047
799 // Index 0 is spoken for before it is derived, so a failure below cannot 1048 // Index 0 is spoken for before it is derived, so a failure below cannot
800 // leave the pool free to hand it out as an ordinary key. The cost of 1049 // leave the pool free to hand it out as an ordinary key. The cost of
@@ -1670,7 +1919,7 @@ void CWalletTx::RelayWalletTransaction(CTxDB& txdb)
1670 uint256 hash = GetHash(); 1919 uint256 hash = GetHash();
1671 if (!txdb.ContainsTx(hash)) 1920 if (!txdb.ContainsTx(hash))
1672 { 1921 {
1673 if (LogAcceptsCategory("net")) printf("Relaying wtx %s\n", hash.ToString().substr(0,6).c_str()); 1922 if (LogAcceptsCategory("relay")) printf("Relaying wtx %s\n", hash.ToString().substr(0,6).c_str());
1674 RelayMessage(CInv(MSG_TX, hash), (CTransaction)*this); 1923 RelayMessage(CInv(MSG_TX, hash), (CTransaction)*this);
1675 } 1924 }
1676 } 1925 }
@@ -2480,11 +2729,21 @@ string GetAppDir()
2480 { 2729 {
2481 return "."; 2730 return ".";
2482 } 2731 }
2483 static bool fMkdirDone; 2732 static string strCreatedBaseDir;
2484 if (!fMkdirDone) 2733 if (strCreatedBaseDir != strDir)
2485 { 2734 {
2486 fMkdirDone = true;
2487 _mkdir(strDir.c_str()); 2735 _mkdir(strDir.c_str());
2736 strCreatedBaseDir = strDir;
2737 }
2738 if (IsTestNet())
2739 {
2740 strDir += "/testnet";
2741 static string strCreatedTestnetDir;
2742 if (strCreatedTestnetDir != strDir)
2743 {
2744 _mkdir(strDir.c_str());
2745 strCreatedTestnetDir = strDir;
2746 }
2488 } 2747 }
2489 return strDir; 2748 return strDir;
2490} 2749}
@@ -2636,22 +2895,36 @@ bool LoadBlockIndex(bool fAllowNew)
2636 // CTxOut(nValue=50.00000000, scriptPubKey=0x5F1DF16B2B704C8A578D0B) 2895 // CTxOut(nValue=50.00000000, scriptPubKey=0x5F1DF16B2B704C8A578D0B)
2637 // vMerkleTree: 4a5e1e 2896 // vMerkleTree: 4a5e1e
2638 2897
2639 // Bitflash genesis block -- own header (fair launch, 2026 stable relaunch) 2898 // Bitflash genesis block -- own header (fair launch)
2640 char* pszTimestamp = "Bitflash 24/Jul/2026 Fair launch: one CPU one vote, no premine"; 2899 //
2641 CTransaction txNew; 2900 // Upstream moved the construction into BuildGenesisBlock() so mainnet
2642 txNew.vin.resize(1); 2901 // and the isolated testnet can share it. The mainnet inputs it uses are
2643 txNew.vout.resize(1); 2902 // byte-identical to what this block built inline before the merge --
2644 txNew.vin[0].scriptSig = CScript() << 486604799 << CBigNum(4) << vector<unsigned char>((unsigned char*)pszTimestamp, (unsigned char*)pszTimestamp + strlen(pszTimestamp)); 2903 // timestamp string, nTime, nNonce, scriptPubKey, nBits and nVersion all
2645 txNew.vout[0].nValue = 50 * COIN; 2904 // verified equal against MAINNET_GENESIS_* -- so the mainnet genesis
2646 txNew.vout[0].scriptPubKey = CScript() << CBigNum("0x5F1DF16B2B704C8A578D0BBAF74D385CDE12C11EE50455F3C438EF4C3FBCF649B6DE611FEAE06279A60939E028A8D65C10B73071A6F16719274855FEB0FD8A6704") << OP_CHECKSIG; 2905 // hash is unchanged. The warning below is why that was checked rather
2647 CBlock block; 2906 // than assumed, and it still applies to those constants:
2648 block.vtx.push_back(txNew); 2907 //
2649 block.hashPrevBlock = 0; 2908 // Do not change MAINNET_GENESIS_TIMESTAMP, the scriptPubKey,
2650 block.hashMerkleRoot = block.BuildMerkleTree(); 2909 // MAINNET_GENESIS_TIME or MAINNET_GENESIS_NONCE. They are the inputs to
2651 block.nVersion = 1; 2910 // the genesis hash, and the genesis hash is consensus. Changing one on
2652 block.nTime = 1753315200; // 2026-07-24 00:00:00 UTC (stable relaunch) 2911 // its own trips the assert below, which is loud and reverts cleanly;
2653 block.nBits = 0x1f0fffff; // = bnProofOfWorkLimit (Bitflash RandomX floor) 2912 // changing one and regenerating GENESIS_NONCE and hashGenesisBlock to
2654 block.nNonce = GENESIS_NONCE; 2913 // match -- which is exactly what the re-mining branch below prints out
2914 // and invites -- gives a node that starts without complaint, builds its
2915 // own chain from height 0, and rejects every real peer as being on a
2916 // different chain. That one fails silently, and the damage scales with
2917 // how many people run the "fixed" build.
2918 //
2919 // MAINNET_GENESIS_TIME is 1753315200, which is 2025-07-24T00:00:00Z,
2920 // while the headline in MAINNET_GENESIS_TIMESTAMP reads 2026 -- as does
2921 // the comment upstream put on that constant, which is wrong about the
2922 // year. This is a known discrepancy and not a typo in the value: it is
2923 // only consensus-critical as an input to the genesis hash, the block
2924 // hashes to hashGenesisBlock with it, and every node on the network
2925 // agrees. The mismatch is in the launched chain and cannot be corrected
2926 // -- see the warning above before deciding otherwise.
2927 CBlock block = BuildGenesisBlock();
2655 2928
2656 // If the nonce has not been found yet, mine the genesis once (RandomX) 2929 // If the nonce has not been found yet, mine the genesis once (RandomX)
2657 // and print the values to be hardcoded in the constants above. 2930 // and print the values to be hardcoded in the constants above.
@@ -2676,6 +2949,11 @@ bool LoadBlockIndex(bool fAllowNew)
2676 printf("powhash = %s\n", block.GetPoWHash().ToString().c_str()); 2949 printf("powhash = %s\n", block.GetPoWHash().ToString().c_str());
2677 printf("hash = %s\n", block.GetHash().ToString().c_str()); 2950 printf("hash = %s\n", block.GetHash().ToString().c_str());
2678 printf("merkle = %s\n", block.hashMerkleRoot.ToString().c_str()); 2951 printf("merkle = %s\n", block.hashMerkleRoot.ToString().c_str());
2952 printf("----------------------------------------\n");
2953 printf("These constants bring up a NEW chain. Do not paste them\n");
2954 printf("into a build for this network to make an edited nTime or\n");
2955 printf("pszTimestamp fit -- a node whose genesis differs starts\n");
2956 printf("cleanly on a chain of its own and rejects every real peer.\n");
2679 printf("========================================\n"); 2957 printf("========================================\n");
2680 } 2958 }
2681 2959
@@ -3221,15 +3499,61 @@ bool ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv)
3221 vRecv >> *pblock; 3499 vRecv >> *pblock;
3222 3500
3223 //// debug print 3501 //// debug print
3224 if (LogAcceptsCategory("net")) { printf("received block:\n"); pblock->print(); } 3502 if (LogAcceptsCategory("relay")) { printf("received block:\n"); pblock->print(); }
3225 3503
3226 CInv inv(MSG_BLOCK, pblock->GetHash()); 3504 CInv inv(MSG_BLOCK, pblock->GetHash());
3227 pfrom->AddInventoryKnown(inv); 3505 pfrom->AddInventoryKnown(inv);
3228 3506
3507 // GetHash() covers the 80-byte header and nothing else, so the hash
3508 // coming out right does not make this the block we asked for. Every
3509 // peer that saw X announced holds X's header, and can answer our
3510 // getdata with that header bolted onto a transaction list of its own
3511 // invention -- the reply still hashes to X. hashMerkleRoot is the only
3512 // thing binding the header to the body, so check it here, before the
3513 // mapBlockFetch entry is touched and before CheckBlock spends a
3514 // RandomX hash on a body already known to be forged.
3515 //
3516 // There is no honest way to arrive here. Peer traffic rides the .btf
3517 // channel, which is XChaCha20-Poly1305, so a body that arrives at all
3518 // arrived intact: a mismatch is the peer's own construction, not line
3519 // noise. Drop the peer and leave the fetch outstanding so the failover
3520 // sweep asks somebody else for the real X, rather than recording X as
3521 // delivered on the strength of a header anyone could copy.
3522 if (pblock->hashMerkleRoot != pblock->BuildMerkleTree())
3523 {
3524 if (LogAcceptsCategory("net"))
3525 printf("peer %s served a forged body for %s\n",
3526 pfrom->addr.ToString().c_str(), inv.ToString().c_str());
3527 // One line per connection: ProcessMessages keeps draining the
3528 // receive buffer after fDisconnect is set, so a peer that packs
3529 // many forged bodies into one buffer must not get one log line
3530 // for each of them.
3531 if (!pfrom->fDisconnect)
3532 printf("block fetch: a peer answered the request for %s with a body that is not "
3533 "that block -- dropping it\n", inv.hash.ToString().substr(0,14).c_str());
3534 pfrom->fDisconnect = true;
3535 return error("ProcessMessage() : forged block body for %s",
3536 inv.hash.ToString().substr(0,14).c_str());
3537 }
3538
3229 if (ProcessBlock(pfrom, pblock.release())) 3539 if (ProcessBlock(pfrom, pblock.release()))
3230 { 3540 {
3231 mapAlreadyAskedFor.erase(inv); 3541 mapAlreadyAskedFor.erase(inv);
3232 3542
3543 // The body arrived and validated, so the fetch really is over:
3544 // the block is now in mapBlockIndex, or parked in mapOrphanBlocks,
3545 // and either way AlreadyHave answers for it from here on.
3546 //
3547 // This has to stay inside the success branch, next to the erase
3548 // above. When validation rejects a body nothing is stored,
3549 // AlreadyHave stays false, and the mapAskFor entry was already
3550 // consumed when the getdata went out -- so erasing the tracking
3551 // here would leave the block requested from nobody at all until
3552 // some peer happened to re-announce it. That is exactly the stall
3553 // of #33, and erasing before ProcessBlock made it reachable on
3554 // demand.
3555 mapBlockFetch.erase(inv.hash);
3556
3233 // A peer that just handed us a block is demonstrably at least at 3557 // A peer that just handed us a block is demonstrably at least at
3234 // that block's height. Without this, nStartingHeight is a snapshot 3558 // that block's height. Without this, nStartingHeight is a snapshot
3235 // taken once during the version handshake and never touched again, 3559 // taken once during the version handshake and never touched again,
@@ -3354,12 +3678,119 @@ bool SendMessages(CNode* pto)
3354 const CInv& inv = (*pto->mapAskFor.begin()).second; 3678 const CInv& inv = (*pto->mapAskFor.begin()).second;
3355 if (LogAcceptsCategory("net")) printf("sending getdata: %s\n", inv.ToString().c_str()); 3679 if (LogAcceptsCategory("net")) printf("sending getdata: %s\n", inv.ToString().c_str());
3356 if (!AlreadyHave(txdb, inv)) 3680 if (!AlreadyHave(txdb, inv))
3681 {
3357 vAskFor.push_back(inv); 3682 vAskFor.push_back(inv);
3683 // Blocks get their delivery tracked so the failover pass
3684 // below can chase the ones that go quiet.
3685 if (inv.type == MSG_BLOCK)
3686 {
3687 // Full, and this is a hash we are not already tracking:
3688 // make room rather than drop it on the floor, or one peer
3689 // announcing eight hashes turns failover off for everyone.
3690 if (!mapBlockFetch.count(inv.hash) &&
3691 mapBlockFetch.size() >= BLOCK_FETCH_TRACK_MAX)
3692 MakeRoomInBlockFetch(pto->addr);
3693
3694 if (mapBlockFetch.count(inv.hash) ||
3695 mapBlockFetch.size() < BLOCK_FETCH_TRACK_MAX)
3696 {
3697 CBlockFetch& fetch = mapBlockFetch[inv.hash];
3698 if (fetch.nTimeFirstAsked == 0)
3699 {
3700 fetch.nTimeFirstAsked = GetTime();
3701 // Who is answerable for this hash existing. Set
3702 // once, on creation: the failover sweep asks other
3703 // peers later, and none of them chose to announce
3704 // it, so none of them should be charged for it.
3705 fetch.addrAnnounced = pto->addr;
3706 }
3707 fetch.nTimeAsked = GetTime();
3708 fetch.setAsked.insert(pto->addr);
3709 }
3710 }
3711 }
3358 pto->mapAskFor.erase(pto->mapAskFor.begin()); 3712 pto->mapAskFor.erase(pto->mapAskFor.begin());
3359 } 3713 }
3360 if (!vAskFor.empty()) 3714 if (!vAskFor.empty())
3361 pto->PushMessage("getdata", vAskFor); 3715 pto->PushMessage("getdata", vAskFor);
3362 3716
3717
3718 //
3719 // Message: getdata (block fetch failover)
3720 //
3721 // A requested block that has not arrived in BLOCK_FETCH_RETRY_SECS is
3722 // not still in transit -- a body is at most 1 MB and somebody
3723 // demonstrably has it -- so ask somebody else instead of waiting on
3724 // whoever was asked first (#33). Each peer the message handler visits
3725 // takes at most one turn per block per round; once every connected
3726 // peer has had its turn and a further interval passes with nobody
3727 // new to ask, the round restarts. A peer that declared itself a
3728 // client cannot serve block bodies and is no use as a target.
3729 if (!pto->fClient)
3730 {
3731 vector<CInv> vFailover;
3732 int64 nNowSecs = GetTime();
3733 for (map<uint256, CBlockFetch>::iterator mi = mapBlockFetch.begin(); mi != mapBlockFetch.end();)
3734 {
3735 CBlockFetch& fetch = (*mi).second;
3736 CInv inv(MSG_BLOCK, (*mi).first);
3737 if (AlreadyHave(txdb, inv))
3738 {
3739 // Arrived by some other road -- our own miner, an orphan
3740 // parked in mapOrphanBlocks. Nothing left to chase.
3741 mapBlockFetch.erase(mi++);
3742 continue;
3743 }
3744 if (nNowSecs - fetch.nTimeFirstAsked > BLOCK_FETCH_GIVEUP_SECS)
3745 {
3746 printf("block fetch: giving up on %s -- no peer served it in %d s\n",
3747 inv.ToString().c_str(), (int)(nNowSecs - fetch.nTimeFirstAsked));
3748 mapBlockFetch.erase(mi++);
3749 continue;
3750 }
3751 if (nNowSecs - fetch.nTimeAsked >= BLOCK_FETCH_RETRY_SECS)
3752 {
3753 // The handler loop visits every peer within ~100ms, so a
3754 // request stale by two full intervals means nobody outside
3755 // setAsked was there to ask. Start the round over.
3756 if (nNowSecs - fetch.nTimeAsked >= 2 * BLOCK_FETCH_RETRY_SECS)
3757 fetch.setAsked.clear();
3758 if (!fetch.setAsked.count(pto->addr))
3759 {
3760 // Which block, how long, and which peer we are trying
3761 // next. Behind the category because it names a peer
3762 // and a peer decides how often it is written.
3763 if (LogAcceptsCategory("net"))
3764 printf("block fetch failover: %s unanswered for %d s, re-requesting from %s\n",
3765 inv.ToString().c_str(), (int)(nNowSecs - fetch.nTimeAsked),
3766 pto->addr.ToString().c_str());
3767
3768 // The unconditional half: no addresses, and at most
3769 // one line per BLOCK_FETCH_REPORT_SECS however hard a
3770 // peer tries. See the counters for the reasoning.
3771 nBlockFetchFailovers++;
3772 if (nNowSecs - nBlockFetchWindowStart >= BLOCK_FETCH_REPORT_SECS)
3773 {
3774 printf("block fetch: %d block request%s went unanswered and %s been "
3775 "re-asked of other peers (%d tracked)\n",
3776 nBlockFetchFailovers, nBlockFetchFailovers == 1 ? "" : "s",
3777 nBlockFetchFailovers == 1 ? "has" : "have",
3778 (int)mapBlockFetch.size());
3779 nBlockFetchWindowStart = nNowSecs;
3780 nBlockFetchFailovers = 0;
3781 }
3782
3783 vFailover.push_back(inv);
3784 fetch.nTimeAsked = nNowSecs;
3785 fetch.setAsked.insert(pto->addr);
3786 }
3787 }
3788 ++mi;
3789 }
3790 if (!vFailover.empty())
3791 pto->PushMessage("getdata", vFailover);
3792 }
3793
3363 } 3794 }
3364 return true; 3795 return true;
3365} 3796}
@@ -3620,6 +4051,14 @@ static void ParticipantHashWorker(ParticipantJobShared* sh, int idx, int nWorker
3620 RandomXDestroyMinerVM(vm); 4051 RandomXDestroyMinerVM(vm);
3621} 4052}
3622 4053
4054// Run one participant session: resolve the pool, open the tunnel, subscribe,
4055// hash, submit shares. Returns true only if the pool actually handed out work
4056// -- at least one well-formed mining.notify arrived. Everything else is false,
4057// including a tunnel that opened and was dropped before any job.
4058//
4059// The caller's reconnect backoff keys off this return value, so any path added
4060// here that leaves the session without work MUST return false. "The session
4061// ended tidily" is not the question being asked.
3623static bool PoolParticipantMiner() 4062static bool PoolParticipantMiner()
3624{ 4063{
3625 // ------------------------------------------------------------------ 4064 // ------------------------------------------------------------------
@@ -3697,6 +4136,33 @@ static bool PoolParticipantMiner()
3697 } 4136 }
3698 LogPrint("worker", "[worker] pool resolved to meeting node %s\n", meetingHostPort.c_str()); 4137 LogPrint("worker", "[worker] pool resolved to meeting node %s\n", meetingHostPort.c_str());
3699 4138
4139 // The descriptor names ONE meeting node, but the pool parks a
4140 // registration at every relay it knows (one accept thread per relay,
4141 // rpc.cpp:AcceptOneFn), so any seed relay can pair us. Insisting on the
4142 // advertised entry forever turns one bad path into a miner that idles
4143 // indefinitely while a working path exists (issue #17); after a few
4144 // consecutive tunnel failures, rotate the seed relays into the attempts
4145 // too. Only ONE participant session runs (StartMinerThreads), so these
4146 // statics are single-threaded.
4147 static const int TUNNEL_FAILOVER_AFTER = 3;
4148 static int nTunnelFails = 0; // consecutive tunnel failures, this pool
4149 static size_t iFailoverRelay = 0; // rotation cursor over vBtfMeetingRelays
4150 static std::string strFailPool; // pool the counters belong to
4151 if (strFailPool != strParticipantPool) {
4152 strFailPool = strParticipantPool;
4153 nTunnelFails = 0;
4154 iFailoverRelay = 0;
4155 }
4156 if (nTunnelFails >= TUNNEL_FAILOVER_AFTER && !vBtfMeetingRelays.empty()) {
4157 std::string alt = vBtfMeetingRelays[iFailoverRelay++ % vBtfMeetingRelays.size()];
4158 if (alt != meetingHostPort) {
4159 LogPrint("worker", "[worker] advertised meeting node %s failed %d"
4160 " times in a row -- trying seed relay %s instead\n",
4161 meetingHostPort.c_str(), nTunnelFails, alt.c_str());
4162 meetingHostPort = alt;
4163 }
4164 }
4165
3700 size_t colon = meetingHostPort.rfind(':'); 4166 size_t colon = meetingHostPort.rfind(':');
3701 if (colon == std::string::npos) { 4167 if (colon == std::string::npos) {
3702 LogPrint("worker", "[worker] bad meeting node format '%s'\n", meetingHostPort.c_str()); 4168 LogPrint("worker", "[worker] bad meeting node format '%s'\n", meetingHostPort.c_str());
@@ -3716,15 +4182,18 @@ static bool PoolParticipantMiner()
3716 // Step 2b: open encrypted tunnel to pool 4182 // Step 2b: open encrypted tunnel to pool
3717 // ------------------------------------------------------------------ 4183 // ------------------------------------------------------------------
3718 SetParticipantMiningStatus("opening tunnel"); 4184 SetParticipantMiningStatus("opening tunnel");
4185 btf::TunnelFail tunnelFail = btf::TUNNEL_FAIL_NONE;
3719 s = btf::BtfClientTunnel(host.c_str(), (unsigned short)port, 4186 s = btf::BtfClientTunnel(host.c_str(), (unsigned short)port,
3720 target_pubkey, service_enc_pub); 4187 target_pubkey, service_enc_pub, &tunnelFail);
3721 if (s == INVALID_SOCKET) { 4188 if (s == INVALID_SOCKET) {
3722 LogPrint("worker", "[worker] tunnel connect FAILED via %s" 4189 nTunnelFails++;
3723 " (relay unreachable or pool not listening)\n", 4190 LogPrint("worker", "[worker] tunnel connect FAILED via %s: %s\n",
3724 meetingHostPort.c_str()); 4191 meetingHostPort.c_str(), btf::TunnelFailText(tunnelFail));
3725 SetParticipantMiningStatus("tunnel connect failed"); 4192 SetParticipantMiningStatus(std::string("tunnel connect failed: ")
4193 + btf::TunnelFailText(tunnelFail));
3726 return false; 4194 return false;
3727 } 4195 }
4196 nTunnelFails = 0;
3728 LogPrint("worker", "[worker] tunnel open via %s\n", meetingHostPort.c_str()); 4197 LogPrint("worker", "[worker] tunnel open via %s\n", meetingHostPort.c_str());
3729 SetParticipantMiningStatus("tunnel open"); 4198 SetParticipantMiningStatus("tunnel open");
3730 } 4199 }
@@ -3857,9 +4326,32 @@ connected:
3857 std::vector<ParticipantShareHit> shareHits; 4326 std::vector<ParticipantShareHit> shareHits;
3858 std::mutex shareHitsMtx; 4327 std::mutex shareHitsMtx;
3859 std::vector<std::thread> hashThreads; 4328 std::vector<std::thread> hashThreads;
3860 for (int i = 0; i < nWorkers; i++) 4329 try
3861 hashThreads.emplace_back(ParticipantHashWorker, &jobShared, i, nWorkers, 4330 {
3862 &shareHits, &shareHitsMtx); 4331 for (int i = 0; i < nWorkers; i++)
4332 hashThreads.emplace_back(ParticipantHashWorker, &jobShared, i, nWorkers,
4333 &shareHits, &shareHitsMtx);
4334 }
4335 catch (const std::system_error& e)
4336 {
4337 // A refused pthread_create used to reach std::terminate here and
4338 // kill the worker mid-session -- observed twice on an otherwise
4339 // healthy participant. Stop the workers that did start the same way
4340 // the disconnect path below does, drop this pool connection, and let
4341 // the caller's reconnect loop try again once the pressure clears.
4342 LogPrint("worker", "[worker] cannot spawn hash threads (%s)"
4343 " -- dropping pool connection\n", e.what());
4344 SetParticipantMiningStatus("hash thread spawn failed");
4345 {
4346 std::lock_guard<std::mutex> lk(jobShared.mtx);
4347 jobShared.stop = true;
4348 jobShared.haveJob = false;
4349 }
4350 for (std::thread& t : hashThreads)
4351 t.join();
4352 closesocket(s);
4353 return false;
4354 }
3863 LogPrint("worker", "[worker] spawned %d hash threads on shared RandomX dataset\n", 4355 LogPrint("worker", "[worker] spawned %d hash threads on shared RandomX dataset\n",
3864 nWorkers); 4356 nWorkers);
3865 const uint64 sessionStartHashes = gParticipantHashes.load(); 4357 const uint64 sessionStartHashes = gParticipantHashes.load();
@@ -3870,8 +4362,17 @@ connected:
3870 std::string jobId; 4362 std::string jobId;
3871 unsigned char header[80] = {}; 4363 unsigned char header[80] = {};
3872 uint256 target; 4364 uint256 target;
4365 // Cached answer to "which block is this job built on", for the
4366 // mining.notify handler below -- see the comment there. Session-local, so
4367 // a reconnect re-resolves rather than trusting an answer obtained before
4368 // a possible change of pool.
4369 uint256 hashPrevResolved; // prevhash the cached answer describes
4370 bool fPrevResolved = false; // whether that answer exists yet
4371 int nPrevHeight = -1; // its height, or -1 == unknown to us
4372 int nBestHeightResolved = -1; // nBestHeight when we last resolved
3873 double currentDifficulty = 1.0; // updated by mining.set_difficulty from pool 4373 double currentDifficulty = 1.0; // updated by mining.set_difficulty from pool
3874 bool haveJob = false; 4374 bool haveJob = false;
4375 bool gotAnyJob = false; // any job at all this session -- the return value
3875 bool gotSubscribeAck = false; 4376 bool gotSubscribeAck = false;
3876 bool gotAuthorizeAck = false; 4377 bool gotAuthorizeAck = false;
3877 int64 authWaitStart = 0; 4378 int64 authWaitStart = 0;
@@ -3988,8 +4489,9 @@ connected:
3988 jobId = newJobId; 4489 jobId = newJobId;
3989 memcpy(header, hdrBytes.data(), 80); 4490 memcpy(header, hdrBytes.data(), 80);
3990 memcpy(&target, tgtBytes.data(), 32); 4491 memcpy(&target, tgtBytes.data(), 32);
3991 haveJob = true; 4492 haveJob = true;
3992 nNonce = 1; 4493 gotAnyJob = true;
4494 nNonce = 1;
3993 { 4495 {
3994 std::lock_guard<std::mutex> lk(jobShared.mtx); 4496 std::lock_guard<std::mutex> lk(jobShared.mtx);
3995 memcpy(jobShared.header, header, 80); 4497 memcpy(jobShared.header, header, 80);
@@ -3998,8 +4500,64 @@ connected:
3998 jobShared.jobSeq++; 4500 jobShared.jobSeq++;
3999 jobShared.haveJob = true; 4501 jobShared.haveJob = true;
4000 } 4502 }
4001 LogPrint("worker", "[pool->worker] new job %s received\n", 4503 // Log the chain position the pool is
4002 jobId.c_str()); 4504 // building on, so a stale or private
4505 // fork is auditable from worker logs.
4506 // The prevhash comes from the header we
4507 // actually hash (== p[2] of the notify);
4508 // a brief cs_main lookup is safe here --
4509 // this thread holds no other lock.
4510 //
4511 // Resolving it is the expensive half, and
4512 // this is the socket-reading thread: block
4513 // validation can hold cs_main for a long
4514 // time, and every moment spent waiting on
4515 // it here is a moment set_difficulty and
4516 // share responses are not being handled.
4517 // Most notifies in a round are a new job
4518 // on the SAME tip, so the answer is cached
4519 // and re-resolved only when it could come
4520 // out differently: a prevhash we have not
4521 // resolved, or one we could NOT resolve
4522 // last time while the chain has since
4523 // moved (unknown -> known). A block's
4524 // height is fixed once mapBlockIndex knows
4525 // it, so a cached "known" never goes
4526 // stale. nBestHeight is read unlocked on
4527 // purpose -- it only decides whether to
4528 // repeat a lookup, never what gets
4529 // printed, so a stale read costs at most
4530 // one deferred or one extra lookup.
4531 // The line still prints in full for every
4532 // job, cached or not.
4533 uint256 hashPrev;
4534 memcpy(&hashPrev, header + 4, 32);
4535 if (LogAcceptsCategory("worker"))
4536 {
4537 int nBestHeightNow = nBestHeight;
4538 if (!fPrevResolved
4539 || hashPrev != hashPrevResolved
4540 || (nPrevHeight < 0 && nBestHeightNow != nBestHeightResolved))
4541 {
4542 nPrevHeight = -1;
4543 CRITICAL_BLOCK(cs_main)
4544 {
4545 map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(hashPrev);
4546 if (mi != mapBlockIndex.end())
4547 nPrevHeight = (*mi).second->nHeight;
4548 }
4549 hashPrevResolved = hashPrev;
4550 nBestHeightResolved = nBestHeightNow;
4551 fPrevResolved = true;
4552 }
4553 if (nPrevHeight >= 0)
4554 LogPrint("worker", "[pool->worker] new job %s received, prevhash %s (known, job height %d)\n",
4555 jobId.c_str(), hashPrev.GetHex().c_str(),
4556 nPrevHeight + 1);
4557 else
4558 LogPrint("worker", "[pool->worker] new job %s received, prevhash %s (unknown to our chain)\n",
4559 jobId.c_str(), hashPrev.GetHex().c_str());
4560 }
4003 SetParticipantMiningStatus("hashing job " + jobId); 4561 SetParticipantMiningStatus("hashing job " + jobId);
4004 } 4562 }
4005 } 4563 }
@@ -4125,9 +4683,197 @@ connected:
4125 (unsigned long long)hashesSinceSample, 4683 (unsigned long long)hashesSinceSample,
4126 (unsigned long long)sessionSharesSent, 4684 (unsigned long long)sessionSharesSent,
4127 (unsigned long long)sessionSharesAccepted); 4685 (unsigned long long)sessionSharesAccepted);
4686 // Not "we got here", which is true of a tunnel the pool accepted and then
4687 // dropped without ever sending a job -- exactly the case the caller's
4688 // backoff exists for. Reaching the end of the session is not evidence the
4689 // session was worth anything; a job is.
4690 return gotAnyJob;
4691}
4692
4693// ---------------------------------------------------------------------------
4694// Local Stratum bridge -- lets external miners speak plain Stratum to
4695// 127.0.0.1 while Bitflash opens the .btf tunnel to the selected pool.
4696// ---------------------------------------------------------------------------
4697
4698struct StratumBridgeClientCtx
4699{
4700 SOCKET miner;
4701 std::string pool;
4702};
4703
4704static bool BridgeSendAll(SOCKET s, const char* p, int n)
4705{
4706 int off = 0;
4707 while (off < n)
4708 {
4709 int r = send(s, p + off, n - off, 0);
4710 if (r <= 0)
4711 return false;
4712 off += r;
4713 }
4128 return true; 4714 return true;
4129} 4715}
4130 4716
4717static void BridgePump(SOCKET a, SOCKET b)
4718{
4719 char buf[8192];
4720 while (!fShutdown)
4721 {
4722 fd_set rd;
4723 FD_ZERO(&rd);
4724 FD_SET(a, &rd);
4725 FD_SET(b, &rd);
4726 SOCKET maxfd = a > b ? a : b;
4727 struct timeval tv = {1, 0};
4728 int r = select((int)maxfd + 1, &rd, NULL, NULL, &tv);
4729 if (r <= 0)
4730 continue;
4731
4732 if (FD_ISSET(a, &rd))
4733 {
4734 int n = recv(a, buf, sizeof(buf), 0);
4735 if (n <= 0 || !BridgeSendAll(b, buf, n))
4736 break;
4737 }
4738 if (FD_ISSET(b, &rd))
4739 {
4740 int n = recv(b, buf, sizeof(buf), 0);
4741 if (n <= 0 || !BridgeSendAll(a, buf, n))
4742 break;
4743 }
4744 }
4745}
4746
4747static void ThreadStratumBridgeClient(void* arg)
4748{
4749 StratumBridgeClientCtx* ctx = (StratumBridgeClientCtx*)arg;
4750 SOCKET miner = ctx->miner;
4751 std::string pool = ctx->pool;
4752 delete ctx;
4753
4754 unsigned char target_pubkey[32];
4755 if (!btf::ParseAddress(pool, target_pubkey))
4756 {
4757 LogPrint("worker", "[bridge] invalid pool .btf address: %s\n", pool.c_str());
4758 BtfCloseSocket(miner);
4759 return;
4760 }
4761
4762 std::string meetingHostPort;
4763 unsigned char service_enc_pub[32];
4764 if (!BtfResolve(pool, meetingHostPort, service_enc_pub))
4765 {
4766 LogPrint("worker", "[bridge] failed to resolve pool %s\n", pool.c_str());
4767 BtfCloseSocket(miner);
4768 return;
4769 }
4770
4771 size_t colon = meetingHostPort.rfind(':');
4772 if (colon == std::string::npos)
4773 {
4774 LogPrint("worker", "[bridge] bad pool endpoint '%s'\n", meetingHostPort.c_str());
4775 BtfCloseSocket(miner);
4776 return;
4777 }
4778 std::string host = meetingHostPort.substr(0, colon);
4779 int port = atoi(meetingHostPort.substr(colon + 1).c_str());
4780 if (port <= 0 || port > 65535)
4781 {
4782 LogPrint("worker", "[bridge] bad pool port in '%s'\n", meetingHostPort.c_str());
4783 BtfCloseSocket(miner);
4784 return;
4785 }
4786
4787 SOCKET poolSock = btf::BtfClientTunnel(host.c_str(), (unsigned short)port,
4788 target_pubkey, service_enc_pub);
4789 if (poolSock == INVALID_SOCKET)
4790 {
4791 LogPrint("worker", "[bridge] tunnel connect failed for pool %s via %s\n",
4792 pool.c_str(), meetingHostPort.c_str());
4793 BtfCloseSocket(miner);
4794 return;
4795 }
4796
4797 LogPrint("worker", "[bridge] miner connected to pool %s via %s\n",
4798 pool.c_str(), meetingHostPort.c_str());
4799 BridgePump(miner, poolSock);
4800 BtfCloseSocket(poolSock);
4801 BtfCloseSocket(miner);
4802 LogPrint("worker", "[bridge] miner disconnected from pool %s\n", pool.c_str());
4803}
4804
4805void ThreadStratumBridge(void*)
4806{
4807 if (strParticipantPool.empty())
4808 {
4809 printf("Stratum bridge: no pool selected. Use /stratumbridge=POOL_BTF_ADDRESS\n");
4810 return;
4811 }
4812
4813 SOCKET listener = BtfSocketTag(socket(AF_INET, SOCK_STREAM, IPPROTO_TCP),
4814 SOCK_STRATUM_BRIDGE);
4815 if (listener == INVALID_SOCKET)
4816 {
4817 printf("Stratum bridge: could not create listen socket\n");
4818 return;
4819 }
4820
4821 int yes = 1;
4822 setsockopt(listener, SOL_SOCKET, SO_REUSEADDR, (char*)&yes, sizeof(yes));
4823
4824 sockaddr_in addr;
4825 memset(&addr, 0, sizeof(addr));
4826 addr.sin_family = AF_INET;
4827 addr.sin_addr.s_addr = htonl(0x7f000001);
4828 addr.sin_port = htons((unsigned short)nStratumBridgePort);
4829
4830 if (bind(listener, (sockaddr*)&addr, sizeof(addr)) != 0 ||
4831 listen(listener, 16) != 0)
4832 {
4833 printf("Stratum bridge: could not listen on 127.0.0.1:%d\n",
4834 nStratumBridgePort);
4835 BtfCloseSocket(listener);
4836 return;
4837 }
4838
4839 printf("Stratum bridge: listening on 127.0.0.1:%d for pool %s\n",
4840 nStratumBridgePort, strParticipantPool.c_str());
4841
4842 while (!fShutdown)
4843 {
4844 fd_set rd;
4845 FD_ZERO(&rd);
4846 FD_SET(listener, &rd);
4847 struct timeval tv = {1, 0};
4848 if (select((int)listener + 1, &rd, NULL, NULL, &tv) <= 0)
4849 continue;
4850
4851 sockaddr_in cli;
4852#ifdef _WIN32
4853 int len = sizeof(cli);
4854#else
4855 socklen_t len = sizeof(cli);
4856#endif
4857 SOCKET miner = BtfSocketTag(accept(listener, (sockaddr*)&cli, &len),
4858 SOCK_STRATUM_BRIDGE);
4859 if (miner == INVALID_SOCKET)
4860 continue;
4861
4862 StratumBridgeClientCtx* ctx = new StratumBridgeClientCtx();
4863 ctx->miner = miner;
4864 ctx->pool = strParticipantPool;
4865 if (_beginthread(ThreadStratumBridgeClient, 0, ctx) == (uintptr_t)-1)
4866 {
4867 delete ctx;
4868 BtfCloseSocket(miner);
4869 LogPrint("worker", "[bridge] failed to spawn client thread\n");
4870 }
4871 }
4872
4873 BtfCloseSocket(listener);
4874 printf("Stratum bridge: stopped\n");
4875}
4876
4131bool BitcoinMiner(int nThreadId) 4877bool BitcoinMiner(int nThreadId)
4132{ 4878{
4133 // Relay mode: never mines, just relays/syncs. (Shouldn't normally get here 4879 // Relay mode: never mines, just relays/syncs. (Shouldn't normally get here
@@ -4138,11 +4884,51 @@ bool BitcoinMiner(int nThreadId)
4138 // Participant mode: connect to external pool instead of mining solo/operator 4884 // Participant mode: connect to external pool instead of mining solo/operator
4139 if (nMineMode == MINE_PARTICIPANT) 4885 if (nMineMode == MINE_PARTICIPANT)
4140 { 4886 {
4887 // An attempt that never got work backs off (10s doubling to 120s)
4888 // instead of hammering the same path every flat 10s, and a streak
4889 // lasting minutes is said at printf level: the identical once-per-10s
4890 // LogPrint line is how a worker idled all night unnoticed (issue #17).
4891 //
4892 // Only a session the pool actually sent a job to resets either. The
4893 // weaker test -- "the session returned" -- was gameable: a pool that
4894 // completes the tunnel and drops without ever sending work leaves by
4895 // the same tidy cleanup path a healthy one does, which would pin the
4896 // retry at 10s forever and defeat the backoff in exactly the scenario
4897 // it exists for.
4898 int nRetrySecs = 10;
4899 int64 nFailingSince = 0;
4141 while (fGenerateBitcoins && nMineMode == MINE_PARTICIPANT && !fShutdown) { 4900 while (fGenerateBitcoins && nMineMode == MINE_PARTICIPANT && !fShutdown) {
4142 PoolParticipantMiner(); 4901 bool fGotWork = PoolParticipantMiner();
4143 if (fGenerateBitcoins && nMineMode == MINE_PARTICIPANT && !fShutdown) { 4902 if (fGenerateBitcoins && nMineMode == MINE_PARTICIPANT && !fShutdown) {
4144 LogPrint("worker", "PoolParticipantMiner: reconnecting in 10s\n"); 4903 if (fGotWork) {
4145 for (int i = 0; i < 10 && !fShutdown; i++) Sleep(1000); 4904 nRetrySecs = 10;
4905 nFailingSince = 0;
4906 } else {
4907 if (nFailingSince == 0)
4908 nFailingSince = GetTime();
4909 int64 nMins = (GetTime() - nFailingSince) / 60;
4910 if (nMins >= 1)
4911 printf("[worker] connection to pool %s has been failing"
4912 " for %lld minutes\n",
4913 strParticipantPool.c_str(), (long long)nMins);
4914 }
4915 LogPrint("worker", "PoolParticipantMiner: reconnecting in %ds\n",
4916 nRetrySecs);
4917 // Poll everything the enclosing while polls, not just
4918 // fShutdown. Raising the ceiling to 120s made a
4919 // shutdown-only predicate expensive: after the operator
4920 // clears Generate or switches to Relay, the session -- and
4921 // the vfThreadRunning[3] slot that gates restarting it --
4922 // would stay held for up to two minutes. Measured 119s
4923 // before, 1s after.
4924 for (int i = 0; i < nRetrySecs && !fShutdown && fGenerateBitcoins
4925 && nMineMode == MINE_PARTICIPANT; i++)
4926 Sleep(1000);
4927 if (!fGotWork && nRetrySecs < 120) {
4928 nRetrySecs *= 2;
4929 if (nRetrySecs > 120)
4930 nRetrySecs = 120;
4931 }
4146 } 4932 }
4147 } 4933 }
4148 return true; 4934 return true;
@@ -4244,7 +5030,6 @@ bool BitcoinMiner(int nThreadId)
4244 map<uint256, CTxIndex> mapTestPool; 5030 map<uint256, CTxIndex> mapTestPool;
4245 vector<char> vfAlreadyAdded(mapTransactions.size()); 5031 vector<char> vfAlreadyAdded(mapTransactions.size());
4246 bool fFoundSomething = true; 5032 bool fFoundSomething = true;
4247 unsigned int nBlockSize = 0;
4248 // Assembly must stop below the consensus cap that CheckBlock() 5033 // Assembly must stop below the consensus cap that CheckBlock()
4249 // enforces, or we mine a block our own node then rejects. 5034 // enforces, or we mine a block our own node then rejects.
4250 // 5035 //
@@ -4255,9 +5040,30 @@ bool BitcoinMiner(int nThreadId)
4255 // meant solving a block and throwing it away -- with this pool at 5040 // meant solving a block and throwing it away -- with this pool at
4256 // ~90% of hashrate, that is the network stalling, not just us. 5041 // ~90% of hashrate, that is the network stalling, not just us.
4257 // 5042 //
4258 // Half the cap, keeping the original margin: the test runs before 5043 // The budget test then sat only on this outer while, and the
4259 // a transaction is added, so the loop can overshoot by one tx. 5044 // comment claimed that bounded the overshoot to a single
4260 while (fFoundSomething && nBlockSize < MAX_BLOCK_SIZE/2) 5045 // transaction. It did not. The inner for walks all of
5046 // mapTransactions with no test of its own, so one pass could append
5047 // the entire mempool: the overshoot was bounded by mempool size,
5048 // which is to say not bounded at all. Ask per transaction, before
5049 // appending, and the overshoot is zero rather than small.
5050 //
5051 // Seeded with what the block already weighs -- header, tx count,
5052 // coinbase -- because CheckBlock() weighs the whole serialized
5053 // block and not just the transactions we chose to add to it.
5054 unsigned int nBlockSize = ::GetSerializeSize(*pblock, SER_DISK);
5055 // Sigops are a budget too, and CheckBlock() enforces one that no
5056 // assembler here counted against. Bytes do not bound validation
5057 // cost: every OP_CHECKSIG is an elliptic-curve verification, so a
5058 // template well inside the size limit can still be refused for
5059 // demanding more work than a block is allowed to. Counted with
5060 // CTransaction::GetSigOpCount, the function CheckBlock() itself
5061 // sums, so the two cannot disagree about what a transaction costs.
5062 //
5063 // Seeded from the coinbase because CheckBlock() sums over all of
5064 // vtx, coinbase included.
5065 unsigned int nBlockSigOps = pblock->vtx[0].GetSigOpCount();
5066 while (fFoundSomething && nBlockSize < MAX_BLOCK_SIZE_GEN)
4261 { 5067 {
4262 fFoundSomething = false; 5068 fFoundSomething = false;
4263 unsigned int n = 0; 5069 unsigned int n = 0;
@@ -4269,6 +5075,29 @@ bool BitcoinMiner(int nThreadId)
4269 if (tx.IsCoinBase() || !tx.IsFinal()) 5075 if (tx.IsCoinBase() || !tx.IsFinal())
4270 continue; 5076 continue;
4271 5077
5078 // Skip, not break: a transaction too big for the room left
5079 // says nothing about the next one, which may still fit.
5080 //
5081 // This is also the only ceiling on a single transaction's
5082 // size anywhere on the path into a block. CheckTransaction
5083 // does not cap it and neither does AcceptTransaction, so
5084 // the mempool will hold one of any size the wire allows;
5085 // without this test a lone 1.5 MB transaction offering a
5086 // fat fee is selected, and every template built from that
5087 // mempool is one CheckBlock() throws away. The fee is never
5088 // paid, because the block never confirms.
5089 //
5090 // Deliberately before ConnectInputs and not after:
5091 // ConnectInputs adds this transaction's fee to nFees by
5092 // reference on success. Passing it and then declining the
5093 // transaction would credit the coinbase for a fee no
5094 // transaction in the block pays, and the block would be
5095 // rejected for a wholly different reason.
5096 unsigned int nTxSize = ::GetSerializeSize(tx, SER_NETWORK);
5097 unsigned int nTxSigOps = tx.GetSigOpCount();
5098 if (!BlockTemplateFits(nBlockSize, nBlockSigOps, nTxSize, nTxSigOps))
5099 continue;
5100
4272 // Transaction fee requirements, mainly only needed for flood control 5101 // Transaction fee requirements, mainly only needed for flood control
4273 // Under 10K (about 80 inputs) is free for first 100 transactions 5102 // Under 10K (about 80 inputs) is free for first 100 transactions
4274 // Base rate is 0.01 per KB 5103 // Base rate is 0.01 per KB
@@ -4280,7 +5109,8 @@ bool BitcoinMiner(int nThreadId)
4280 swap(mapTestPool, mapTestPoolTmp); 5109 swap(mapTestPool, mapTestPoolTmp);
4281 5110
4282 pblock->vtx.push_back(tx); 5111 pblock->vtx.push_back(tx);
4283 nBlockSize += ::GetSerializeSize(tx, SER_NETWORK); 5112 nBlockSize += nTxSize;
5113 nBlockSigOps += nTxSigOps;
4284 vfAlreadyAdded[n] = true; 5114 vfAlreadyAdded[n] = true;
4285 fFoundSomething = true; 5115 fFoundSomething = true;
4286 } 5116 }
@@ -4329,7 +5159,7 @@ bool BitcoinMiner(int nThreadId)
4329 //// debug print 5159 //// debug print
4330 if (LogAcceptsCategory("net")) printf("BitcoinMiner:\n"); 5160 if (LogAcceptsCategory("net")) printf("BitcoinMiner:\n");
4331 if (LogAcceptsCategory("net")) printf("RandomX proof-of-work found \n powhash: %s \ntarget: %s\n", hash.GetHex().c_str(), hashTarget.GetHex().c_str()); 5161 if (LogAcceptsCategory("net")) printf("RandomX proof-of-work found \n powhash: %s \ntarget: %s\n", hash.GetHex().c_str(), hashTarget.GetHex().c_str());
4332 if (LogAcceptsCategory("net")) pblock->print(); 5162 if (LogAcceptsCategory("relay")) pblock->print();
4333 5163
4334 SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_NORMAL); 5164 SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_NORMAL);
4335 CRITICAL_BLOCK(cs_main) 5165 CRITICAL_BLOCK(cs_main)
@@ -4435,6 +5265,9 @@ WalletRecoveryAudit GetWalletRecoveryAudit()
4435 CRITICAL_BLOCK(cs_keyPool) 5265 CRITICAL_BLOCK(cs_keyPool)
4436 { 5266 {
4437 audit.fHaveSeed = HaveHDSeed(); 5267 audit.fHaveSeed = HaveHDSeed();
5268 audit.fSeedEncryptedLocked = !audit.fHaveSeed &&
5269 (!vchCryptedHDMaster.empty() ||
5270 !vchCryptedHDChainCode.empty());
4438 audit.nSchema = nHDKeySchema; 5271 audit.nSchema = nHDKeySchema;
4439 audit.nDerivedKnown = nHDNext; 5272 audit.nDerivedKnown = nHDNext;
4440 audit.nReceiveNext = nHDReceiveNext; 5273 audit.nReceiveNext = nHDReceiveNext;
@@ -4681,9 +5514,17 @@ bool SelectCoins(int64 nTargetValue, set<CWalletTx*>& setCoinsRet)
4681 5514
4682 5515
4683 5516
4684bool CreateTransaction(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew, int64& nFeeRequiredRet) 5517// presultRet, when given, says WHY this returned false: the caller cannot work
5518// it out afterwards. "Coin selection came up short" and "the wallet could not
5519// build the transaction" are indistinguishable from the outside -- both are
5520// just `false` -- and the difference decides whether a pool defers a payment
5521// or counts it against the payment's retry budget. See WalletSendResult.
5522bool CreateTransaction(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew, int64& nFeeRequiredRet,
5523 WalletSendResult* presultRet)
4685{ 5524{
4686 nFeeRequiredRet = 0; 5525 nFeeRequiredRet = 0;
5526 if (presultRet)
5527 *presultRet = WALLET_SEND_FAILED;
4687 CRITICAL_BLOCK(cs_main) 5528 CRITICAL_BLOCK(cs_main)
4688 { 5529 {
4689 // txdb must be opened before the mapWallet lock 5530 // txdb must be opened before the mapWallet lock
@@ -4714,10 +5555,21 @@ bool CreateTransaction(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew, in
4714 // fixed this in 107d9e288. 5555 // fixed this in 107d9e288.
4715 int64 nTotalValue = nValue + nFee; 5556 int64 nTotalValue = nValue + nFee;
4716 5557
4717 // Choose coins to use 5558 // Choose coins to use.
5559 //
5560 // Failing here is a funding shortfall and nothing else: there
5561 // is not enough spendable coin for nValue plus the fee this
5562 // pass is working with. On the second and later passes that
5563 // fee is GetMinFee() of the transaction the first pass built,
5564 // so this is also how a wallet that can afford the amount but
5565 // not the 11 * CENT charged at 10,000 bytes reports itself.
4718 set<CWalletTx*> setCoins; 5566 set<CWalletTx*> setCoins;
4719 if (!SelectCoins(nTotalValue, setCoins)) 5567 if (!SelectCoins(nTotalValue, setCoins))
5568 {
5569 if (presultRet)
5570 *presultRet = WALLET_SEND_INSUFFICIENT_FUNDS;
4720 return false; 5571 return false;
5572 }
4721 int64 nValueIn = 0; 5573 int64 nValueIn = 0;
4722 foreach(CWalletTx* pcoin, setCoins) 5574 foreach(CWalletTx* pcoin, setCoins)
4723 nValueIn += pcoin->GetCredit(); 5575 nValueIn += pcoin->GetCredit();
@@ -4787,6 +5639,8 @@ bool CreateTransaction(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew, in
4787 } 5639 }
4788 } 5640 }
4789 } 5641 }
5642 if (presultRet)
5643 *presultRet = WALLET_SEND_OK;
4790 return true; 5644 return true;
4791} 5645}
4792 5646
@@ -4849,13 +5703,24 @@ bool CommitTransactionSpent(const CWalletTx& wtxNew)
4849 5703
4850 5704
4851 5705
4852bool SendMoney(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew) 5706// outcomeRet reports what happened rather than only whether it worked. A
5707// caller that has to decide what a failure MEANS -- the pool payout thread --
5708// cannot reconstruct it from a balance reading afterwards, because the fee
5709// that was actually required depends on the size of a transaction that was
5710// never built. See WalletSendResult.
5711bool SendMoney(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew,
5712 SendMoneyOutcome& outcomeRet)
4853{ 5713{
5714 outcomeRet.result = WALLET_SEND_FAILED;
5715 outcomeRet.nFeeRequired = 0;
4854 CRITICAL_BLOCK(cs_main) 5716 CRITICAL_BLOCK(cs_main)
4855 { 5717 {
4856 int64 nFeeRequired; 5718 int64 nFeeRequired;
4857 if (!CreateTransaction(scriptPubKey, nValue, wtxNew, nFeeRequired)) 5719 WalletSendResult createResult = WALLET_SEND_FAILED;
5720 if (!CreateTransaction(scriptPubKey, nValue, wtxNew, nFeeRequired, &createResult))
4858 { 5721 {
5722 outcomeRet.result = createResult;
5723 outcomeRet.nFeeRequired = nFeeRequired;
4859 string strError; 5724 string strError;
4860 if (nValue + nFeeRequired > GetBalance()) 5725 if (nValue + nFeeRequired > GetBalance())
4861 // Not an oversized transaction -- this branch is simply "the 5726 // Not an oversized transaction -- this branch is simply "the
@@ -4893,6 +5758,13 @@ bool SendMoney(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew)
4893 } 5758 }
4894 wtxNew.RelayWalletTransaction(); 5759 wtxNew.RelayWalletTransaction();
4895 } 5760 }
5761 outcomeRet.result = WALLET_SEND_OK;
4896 MainFrameRepaint(); 5762 MainFrameRepaint();
4897 return true; 5763 return true;
4898} 5764}
5765
5766bool SendMoney(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew)
5767{
5768 SendMoneyOutcome ignored;
5769 return SendMoney(scriptPubKey, nValue, wtxNew, ignored);
5770}
diff --git a/src/main.h b/src/main.h
index 27f89ac..26b3b69 100644
--- a/src/main.h
+++ b/src/main.h
@@ -33,6 +33,35 @@ static const unsigned int MAX_BLOCK_SIGOPS = 20000;
33// limit (32 MB), not a block rule. Bitcoin 0.1.0 predated the later 1 MB block 33// limit (32 MB), not a block rule. Bitcoin 0.1.0 predated the later 1 MB block
34// cap, so this tree inherited no real block-size consensus limit. 34// cap, so this tree inherited no real block-size consensus limit.
35static const unsigned int MAX_BLOCK_SIZE = 1000000; 35static const unsigned int MAX_BLOCK_SIZE = 1000000;
36// How much of that cap a template we build may fill. Half, which is what both
37// assemblers already aimed at -- this only gives the number a name and one
38// place to live. They had already drifted on how it was enforced, which is the
39// bug this exists to stop repeating.
40//
41// NOT a consensus rule. CheckBlock() decides what the network accepts; this
42// decides what we are willing to produce, and it has to stay under that.
43static const unsigned int MAX_BLOCK_SIZE_GEN = MAX_BLOCK_SIZE / 2;
44
45// Room for one more transaction in a template that currently weighs
46// nBlockSize bytes and spends nBlockSigOps signature operations?
47//
48// Asked BEFORE the transaction is appended, never after. A template that has
49// already overshot is one CheckBlock() rejects, and asking afterwards only
50// measures the waste. Both assemblers call this, so neither can drift into a
51// different idea of what fits.
52//
53// Size gets a margin (half the cap); sigops do not. There is no cheap way to
54// spend the sigop budget usefully and no reason to reserve headroom in it, so
55// assembly holds itself to the same number CheckBlock() enforces -- and to the
56// same comparison. CheckBlock() fails on `> MAX_BLOCK_SIGOPS`, so a block
57// sitting exactly on the limit is valid and this has to accept it too.
58// Assembly and validation disagreeing by one is how the size bug started.
59inline bool BlockTemplateFits(unsigned int nBlockSize, unsigned int nBlockSigOps,
60 unsigned int nTxSize, unsigned int nTxSigOps)
61{
62 return (nBlockSize + nTxSize <= MAX_BLOCK_SIZE_GEN &&
63 nBlockSigOps + nTxSigOps <= MAX_BLOCK_SIGOPS);
64}
36static const int64 COIN = 100000000; 65static const int64 COIN = 100000000;
37// Total Bitflash emission: identical to Bitcoin (21 million). 66// Total Bitflash emission: identical to Bitcoin (21 million).
38// MAX_MONEY guards against the value overflow bug (CVE-2010-5139), which in 67// MAX_MONEY guards against the value overflow bug (CVE-2010-5139), which in
@@ -65,6 +94,8 @@ extern map<uint256, CBlockIndex*> mapBlockIndex;
65extern uint256 hashGenesisBlock; 94extern uint256 hashGenesisBlock;
66extern uint256 hashGenesisMerkleRoot; 95extern uint256 hashGenesisMerkleRoot;
67extern unsigned int GENESIS_NONCE; 96extern unsigned int GENESIS_NONCE;
97void SelectGenesisParams(bool fTestNetIn);
98void SelectChainParams(bool fTestNetIn);
68extern CBlockIndex* pindexGenesisBlock; 99extern CBlockIndex* pindexGenesisBlock;
69extern int nBestHeight; 100extern int nBestHeight;
70extern uint256 hashBestChain; 101extern uint256 hashBestChain;
@@ -97,7 +128,11 @@ extern bool fPoolMineBlind; // /poolmineblind: mine on even when deaf to
97extern int nHashThreads; // participant: /hashthreads override (0 = cores-1) 128extern int nHashThreads; // participant: /hashthreads override (0 = cores-1)
98extern string strPoolName; // operator: announced pool name 129extern string strPoolName; // operator: announced pool name
99extern string strPoolDashboardUrl; // operator: optional dashboard URL 130extern string strPoolDashboardUrl; // operator: optional dashboard URL
131extern string strPoolStatusFile; // operator: optional JSON status output path
132extern string strPoolRoundsFile; // operator: optional public round proof path
100extern double dPoolFeePercent; // operator: announced fee percent 133extern double dPoolFeePercent; // operator: announced fee percent
134extern bool fStratumBridge; // participant helper: local Stratum bridge
135extern int nStratumBridgePort; // local bridge listen port, default 3333
101extern bool fSoloMineTest; // /solomine: mine without requiring a peer (local test) 136extern bool fSoloMineTest; // /solomine: mine without requiring a peer (local test)
102// Threads to hash with; 0 = decide from the hardware. Set by /genproclimit. 137// Threads to hash with; 0 = decide from the hardware. Set by /genproclimit.
103extern int nMinerThreads; 138extern int nMinerThreads;
@@ -226,9 +261,9 @@ static const unsigned int HD_BIP44_PURPOSE = 44;
226static const unsigned int HD_BIP44_ACCOUNT = 0; 261static const unsigned int HD_BIP44_ACCOUNT = 0;
227static const unsigned int HD_BIP44_CHAIN_RECEIVE = 0; 262static const unsigned int HD_BIP44_CHAIN_RECEIVE = 0;
228static const unsigned int HD_BIP44_CHAIN_CHANGE = 1; 263static const unsigned int HD_BIP44_CHAIN_CHANGE = 1;
229// Provisional until Bitflash has an official SLIP-0044 assignment. 264// Official SLIP-0044 assignment (registered 2026-08-07):
230// Proposed registry row: 4346950 | BITFLASH | Bitflash. 265// 4346950 | BITFLASH | Bitflash (4346950 == 0x425446 == "BTF")
231static const unsigned int HD_BIP44_COIN_TYPE_BITFLASH_PROVISIONAL = 4346950; 266static const unsigned int HD_BIP44_COIN_TYPE_BITFLASH = 4346950;
232extern int nHDKeySchema; 267extern int nHDKeySchema;
233extern unsigned int nHDReceiveNext; // BIP44 external chain 268extern unsigned int nHDReceiveNext; // BIP44 external chain
234extern unsigned int nHDChangeNext; // BIP44 internal chain 269extern unsigned int nHDChangeNext; // BIP44 internal chain
@@ -285,6 +320,7 @@ void LimitOrphanBlocks(unsigned int nMaxOrphans);
285// distinguishable, which the dedup filter would otherwise collapse into one. 320// distinguishable, which the dedup filter would otherwise collapse into one.
286bool BitcoinMiner(int nThreadId = 1); 321bool BitcoinMiner(int nThreadId = 1);
287void ThreadRPCServer(void* parg); // rpc.cpp -- .btf pool server 322void ThreadRPCServer(void* parg); // rpc.cpp -- .btf pool server
323void ThreadStratumBridge(void* parg); // main.cpp -- local Stratum -> .btf bridge
288void GetParticipantMiningStats(uint64& sharesSent, uint64& sharesAccepted, double& hashRate); 324void GetParticipantMiningStats(uint64& sharesSent, uint64& sharesAccepted, double& hashRate);
289void SetParticipantMiningStatus(const std::string& status); 325void SetParticipantMiningStatus(const std::string& status);
290std::string GetParticipantMiningStatus(); 326std::string GetParticipantMiningStatus();
@@ -301,6 +337,7 @@ int64 GetBalance();
301struct WalletRecoveryAudit 337struct WalletRecoveryAudit
302{ 338{
303 bool fHaveSeed; 339 bool fHaveSeed;
340 bool fSeedEncryptedLocked;
304 bool fDeriveComplete; 341 bool fDeriveComplete;
305 int nSchema; 342 int nSchema;
306 unsigned int nDerivedKnown; 343 unsigned int nDerivedKnown;
@@ -320,12 +357,13 @@ struct WalletRecoveryAudit
320 WalletRecoveryAudit() 357 WalletRecoveryAudit()
321 { 358 {
322 fHaveSeed = false; 359 fHaveSeed = false;
360 fSeedEncryptedLocked = false;
323 fDeriveComplete = true; 361 fDeriveComplete = true;
324 nSchema = HD_SCHEMA_NONE; 362 nSchema = HD_SCHEMA_NONE;
325 nDerivedKnown = 0; 363 nDerivedKnown = 0;
326 nReceiveNext = 0; 364 nReceiveNext = 0;
327 nChangeNext = 0; 365 nChangeNext = 0;
328 nCoinType = HD_BIP44_COIN_TYPE_BITFLASH_PROVISIONAL; 366 nCoinType = HD_BIP44_COIN_TYPE_BITFLASH;
329 nRecoverableTx = 0; 367 nRecoverableTx = 0;
330 nLegacyTx = 0; 368 nLegacyTx = 0;
331 nRecoverableImmatureTx = 0; 369 nRecoverableImmatureTx = 0;
@@ -337,9 +375,52 @@ struct WalletRecoveryAudit
337 } 375 }
338}; 376};
339WalletRecoveryAudit GetWalletRecoveryAudit(); 377WalletRecoveryAudit GetWalletRecoveryAudit();
340bool CreateTransaction(CScript scriptPubKey, int64 nValue, CWalletTx& txNew, int64& nFeeRequiredRet); 378
379// Why a send did not go through.
380//
381// The pool's payout thread has to decide, on every failed SendMoney, whether
382// the failure was the wallet's condition (wait for it to change) or a fault
383// (something is wrong with this payment). It used to guess by re-reading
384// GetBalance() afterwards and allowing a fixed CENT of headroom for the fee.
385// That cannot work: the fee here is a step function of the SERIALISED SIZE of
386// a transaction that no longer exists, and the step is from 0 to 11 * CENT at
387// 10,000 bytes -- so a one-CENT allowance is either unnecessary or an order of
388// magnitude short, never right. A pool wallet close to broke selects many
389// small inputs BECAUSE it is close to broke, which is what pushes the
390// transaction over 10 KB, so the guess is wrong exactly when the money is
391// tight.
392//
393// The wallet already knows the answer; it just did not say. SelectCoins()
394// failing is "there is not enough spendable coin to cover the amount plus
395// whatever fee this size of transaction requires", and it is the only reason
396// CreateTransaction() can fail that is about funding rather than about the
397// wallet being broken.
398enum WalletSendResult
399{
400 WALLET_SEND_OK = 0,
401 WALLET_SEND_INSUFFICIENT_FUNDS, // coin selection could not cover amount+fee
402 WALLET_SEND_FAILED // anything else: could not build, sign or accept
403};
404
405// What the wallet did with a send request. nFeeRequired is the fee the
406// transaction turned out to need, which for a failed send is what coin
407// selection was asked for on top of the amount and could not find -- 0 if it
408// never got that far.
409struct SendMoneyOutcome
410{
411 WalletSendResult result = WALLET_SEND_OK;
412 int64 nFeeRequired = 0;
413};
414
415bool CreateTransaction(CScript scriptPubKey, int64 nValue, CWalletTx& txNew, int64& nFeeRequiredRet,
416 WalletSendResult* presultRet=NULL);
341bool CommitTransactionSpent(const CWalletTx& wtxNew); 417bool CommitTransactionSpent(const CWalletTx& wtxNew);
418// The three-argument form is kept, and kept as a distinct declaration rather
419// than a defaulted argument, because gui.cpp declares SendMoney itself instead
420// of including this header.
342bool SendMoney(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew); 421bool SendMoney(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew);
422bool SendMoney(CScript scriptPubKey, int64 nValue, CWalletTx& wtxNew,
423 SendMoneyOutcome& outcomeRet);
343 424
344 425
345 426
diff --git a/src/main_gui.cpp b/src/main_gui.cpp
index bee4b95..2e4eb91 100644
--- a/src/main_gui.cpp
+++ b/src/main_gui.cpp
@@ -1,7 +1,10 @@
1// Bitflash entry point -- starts node threads then runs GUI (or headless). 1// Bitflash entry point -- starts node threads then runs GUI (or headless).
2 2
3#include "headers_core.h" 3#include "headers_core.h"
4#include "nostr.h"
5#include "proxy.h"
4#include "selftest.h" 6#include "selftest.h"
7#include "tor.h"
5#include "walletcmd.h" 8#include "walletcmd.h"
6#include <thread> // hardware_concurrency, to sanity-check /genproclimit 9#include <thread> // hardware_concurrency, to sanity-check /genproclimit
7#ifndef _WIN32 10#ifndef _WIN32
@@ -108,9 +111,16 @@ static void PrintUsage()
108 printf(" /help, -help, --help, /?\n"); 111 printf(" /help, -help, --help, /?\n");
109 printf(" /datadir=PATH\n"); 112 printf(" /datadir=PATH\n");
110 printf(" /debug\n"); 113 printf(" /debug\n");
114 printf(" /debugrelay (/debug plus per-message relay chatter and block dumps)\n");
115 printf(" /maxdebugfilesize=N (roll debug.log at N MB, default 1024; 0 = never)\n");
111 printf(" /gen\n"); 116 printf(" /gen\n");
112 printf(" /nogui or /daemon\n"); 117 printf(" /nogui or /daemon\n");
113 printf(" /selftest=wallet-keypool, wallet-hd, wallet-format, wallet-crypto, or wallet-encrypt\n"); 118 printf(" /selftest=wallet-keypool, wallet-hd, wallet-format, wallet-storage-sanity,\n");
119 printf(" db-env-reopen, wallet-sqlite, wallet-sqlite-migration,\n");
120 printf(" wallet-crypto, wallet-encrypt, wallet-portability,\n");
121 printf(" net-message,\n");
122 printf(" consensus-limits, pool-stratum,\n");
123 printf(" parse-money, network-params, socks5-proxy, or managed-tor\n");
114 printf("\n"); 124 printf("\n");
115 printf("Mining mode:\n"); 125 printf("Mining mode:\n");
116 printf(" /operator\n"); 126 printf(" /operator\n");
@@ -118,22 +128,36 @@ static void PrintUsage()
118 printf(" /payoutaddr=BTF_ADDRESS (participant: pay shares to this address, no wallet needed)\n"); 128 printf(" /payoutaddr=BTF_ADDRESS (participant: pay shares to this address, no wallet needed)\n");
119 printf(" /payoutfile=FILE (participant: rotate payout addresses from file)\n"); 129 printf(" /payoutfile=FILE (participant: rotate payout addresses from file)\n");
120 printf(" /hashthreads=N (participant: mining threads, default cores-1)\n"); 130 printf(" /hashthreads=N (participant: mining threads, default cores-1)\n");
131 printf(" /stratumbridge=POOL_BTF_ADDRESS\n");
132 printf(" /stratumbridgeport=N (default 3333; listen on 127.0.0.1)\n");
121 printf(" /solomine\n"); 133 printf(" /solomine\n");
122 printf(" /genproclimit=N (mining threads; 0 or absent = every core but one)\n"); 134 printf(" /genproclimit=N (mining threads; 0 or absent = every core but one)\n");
135 printf(" /nolargepages (do not ask for 2 MB pages for the RandomX\n");
136 printf(" cache, dataset and scratchpads)\n");
123 printf(" /checkblocks=N (blocks re-verified at startup, default 288, 0 = all)\n"); 137 printf(" /checkblocks=N (blocks re-verified at startup, default 288, 0 = all)\n");
124 printf("\n"); 138 printf("\n");
125 printf("Pool operator announcement:\n"); 139 printf("Pool operator announcement:\n");
126 printf(" /poolname=NAME\n"); 140 printf(" /poolname=NAME\n");
127 printf(" /poolfee=PCT\n"); 141 printf(" /poolfee=PCT\n");
128 printf(" /pooldashboard=URL (alias: /pooldash=URL)\n"); 142 printf(" /pooldashboard=URL (alias: /pooldash=URL)\n");
143 printf(" /poolstatusfile=PATH (write pool status JSON for dashboards)\n");
144 printf(" /poolroundsfile=PATH (write public pool round proofs JSON)\n");
129 printf("\n"); 145 printf("\n");
130 printf(".btf and rendezvous:\n"); 146 printf(".btf and rendezvous:\n");
131 printf(" /connectbtf=PEER_BTF_ADDRESS\n"); 147 printf(" /connectbtf=PEER_BTF_ADDRESS\n");
132 printf(" /rvrelay=HOST:PORT\n"); 148 printf(" /rvrelay=HOST:PORT\n");
133 printf(" /announcerelay=HOST:PORT\n"); 149 printf(" /announcerelay=HOST:PORT\n");
150 printf(" /onionservice=HOST.onion:PORT (advertise this node's Tor hidden service)\n");
134 printf("\n"); 151 printf("\n");
135 printf("Network:\n"); 152 printf("Network:\n");
136 printf(" /port=N (P2P listen port, default 8433)\n"); 153 printf(" /testnet (isolated test network genesis, datadir, port, and magic)\n");
154 printf(" /port=N (P2P listen port, default 8433; testnet 18433)\n");
155 printf(" /socks=HOST:PORT (SOCKS5 proxy for Nostr, .btf relay, and onion dials)\n");
156 printf(" /tor[=HOST:PORT] (Tor mode; default SOCKS5 proxy is 127.0.0.1:9050)\n");
157 printf(" /managedtor[=PATH] (start Tor, create a hidden service, advertise its onion)\n");
158 printf(" /automanagedtor (auto-start a bundled Tor if one sits next to the executable)\n");
159 printf(" /nomanagedtor (disable automatic bundled Tor startup)\n");
160 printf(" /oniononly (do not fall back to rendezvous when a .btf onion dial fails)\n");
137 printf(" /btfseed=ADDRESS:ENCHEX (extra bootstrap peer, repeatable)\n"); 161 printf(" /btfseed=ADDRESS:ENCHEX (extra bootstrap peer, repeatable)\n");
138 printf("\n"); 162 printf("\n");
139 printf("Wallet:\n"); 163 printf("Wallet:\n");
@@ -160,6 +184,26 @@ static void PrintUsage()
160 printf(" /recoveryaudit (show how much spendable balance is\n"); 184 printf(" /recoveryaudit (show how much spendable balance is\n");
161 printf(" covered by the recovery phrase;\n"); 185 printf(" covered by the recovery phrase;\n");
162 printf(" exits 2 when wallet.dat is still needed)\n"); 186 printf(" exits 2 when wallet.dat is still needed)\n");
187 printf(" /walletstorageaudit (count wallet.dat record types without\n");
188 printf(" printing wallet values, then exit)\n");
189 printf(" /walletstorageauditjson=FILE\n");
190 printf(" (write the same audit as deterministic JSON)\n");
191 printf(" /walletstoragecheck (fail closed on inconsistent or unsafe\n");
192 printf(" wallet.dat storage state)\n");
193 printf(" /walletsqliteexport=FILE (copy raw wallet.dat records into an\n");
194 printf(" SQLite store, then exit)\n");
195 printf(" /walletsqliteverify=FILE (compare wallet.dat with a SQLite export,\n");
196 printf(" printing counts only, then exit)\n");
197 printf(" /walletsqliterestore=FILE (rebuild wallet.dat from a SQLite export in\n");
198 printf(" an empty data directory, then exit)\n");
199 printf(" /walletsqliteloadcheck=FILE\n");
200 printf(" (parse a SQLite export like the wallet loader)\n");
201 printf(" /walletsqlite=FILE (stage-load a SQLite wallet export, then exit)\n");
202 printf(" /walletbackend=sqlite (opt-in: run the node on <datadir>/wallet.sqlite\n");
203 printf(" instead of wallet.dat; export one first)\n");
204 printf(" /createwallet (headless only: permit creating a new wallet in a\n");
205 printf(" datadir that has none -- otherwise refused, since\n");
206 printf(" a missing wallet is usually a mount/backup fault)\n");
163 printf(" /rescan (walk the chain for coins this wallet owns\n"); 207 printf(" /rescan (walk the chain for coins this wallet owns\n");
164 printf(" but never recorded, then exit)\n"); 208 printf(" but never recorded, then exit)\n");
165 printf("\n"); 209 printf("\n");
@@ -173,9 +217,60 @@ static void ParseStartupArguments(int argc, char* argv[])
173 if (arg(argc,argv,"/datadir") || arg(argc,argv,"-datadir")) 217 if (arg(argc,argv,"/datadir") || arg(argc,argv,"-datadir"))
174 strSetDataDir = argval2(argc, argv, "/datadir", "-datadir"); 218 strSetDataDir = argval2(argc, argv, "/datadir", "-datadir");
175 219
220 bool fUseTestNet = arg(argc, argv, "/testnet") || arg(argc, argv, "-testnet");
221 SelectChainParams(fUseTestNet);
222
176 if (arg(argc,argv,"/debug") || arg(argc,argv,"-debug")) 223 if (arg(argc,argv,"/debug") || arg(argc,argv,"-debug"))
177 fDebug = true; 224 fDebug = true;
178 225
226 // /debugrelay -- everything /debug shows, plus the per-message firehose
227 // ("relay" and "nostrdiscovery" categories: sending: tx, Relaying wtx,
228 // raw CBlock dumps, per-peer discovery lines). That traffic was ~90% of
229 // a 13-hour production debug.log (issue #35), so plain /debug leaves it
230 // off; the message text itself is unchanged for anyone grepping it.
231 if (arg(argc,argv,"/debugrelay") || arg(argc,argv,"-debugrelay"))
232 {
233 fDebug = true;
234 fDebugRelay = true;
235 }
236
237 // /maxdebugfilesize=N -- roll debug.log to debug.log.old once it passes
238 // N MB, checked on each write. Default is 1024 (see util.cpp); 0 turns
239 // the check off and leaves rotation entirely to logrotate.
240 string strMaxDebugSize = argval2(argc, argv, "/maxdebugfilesize", "-maxdebugfilesize");
241 if (!strMaxDebugSize.empty())
242 {
243 // Parsed with atoi64() before, which is strtoll() with the endptr
244 // thrown away: it returns 0 for anything it cannot read, so
245 // /maxdebugfilesize=1G and =1024MB both came out as 0 -- "never
246 // roll", identical to asking for that on purpose, with nothing said
247 // to the operator. The old code also multiplied without a range
248 // check, so a big enough N wrapped to a negative cap, which the roll
249 // reads as "off" too. Every rejection is now reported and leaves the
250 // default in place; the value is a plain count of MB, no suffix.
251 const char* psz = strMaxDebugSize.c_str();
252 char* pszEnd = NULL;
253 errno = 0;
254 int64 nMB = strtoll(psz, &pszEnd, 10);
255 // INT64_MAX, without depending on which limits header this tree pulls in.
256 const int64 nMaxMB = (int64)((~(uint64)0) >> 1) / (1024 * 1024);
257 const char* pszWhy = NULL;
258 if (pszEnd == psz)
259 pszWhy = "not a number";
260 else if (*pszEnd != '\0')
261 pszWhy = "trailing characters -- the value is a count of MB, with no unit suffix";
262 else if (errno == ERANGE || nMB < 0)
263 pszWhy = "not a size in MB";
264 else if (nMB > nMaxMB)
265 pszWhy = "larger than a file size can be";
266 if (pszWhy)
267 fprintf(stderr, "Ignoring /maxdebugfilesize=%s: %s (keeping %lld MB)\n",
268 strMaxDebugSize.c_str(), pszWhy,
269 (long long)(nMaxDebugFileSize / 1024 / 1024));
270 else
271 nMaxDebugFileSize = nMB * 1024 * 1024;
272 }
273
179 if (arg(argc,argv,"/gen") || arg(argc,argv,"-gen")) 274 if (arg(argc,argv,"/gen") || arg(argc,argv,"-gen"))
180 { 275 {
181 fGenerateBitcoins = 1; 276 fGenerateBitcoins = 1;
@@ -195,6 +290,11 @@ static void ParseStartupArguments(int argc, char* argv[])
195 if (!strCheckBlocks.empty()) 290 if (!strCheckBlocks.empty())
196 nCheckBlocksOnLoad = atoi(strCheckBlocks.c_str()); 291 nCheckBlocksOnLoad = atoi(strCheckBlocks.c_str());
197 292
293 // Must be read before LoadBlockIndex(), which is what first calls
294 // RandomXInit() and fixes how the cache is allocated.
295 if (arg(argc,argv,"/nolargepages") || arg(argc,argv,"-nolargepages"))
296 fRandomXLargePages = false;
297
198 // /genproclimit=N -- threads to hash with. 0 or absent means automatic, 298 // /genproclimit=N -- threads to hash with. 0 or absent means automatic,
199 // which is every core but one. Named after Bitcoin's own option so it reads 299 // which is every core but one. Named after Bitcoin's own option so it reads
200 // familiarly to anyone who has run one of these before. 300 // familiarly to anyone who has run one of these before.
@@ -240,6 +340,22 @@ static void ParseStartupArguments(int argc, char* argv[])
240 if (!hashThreads.empty()) 340 if (!hashThreads.empty())
241 nHashThreads = atoi(hashThreads.c_str()); 341 nHashThreads = atoi(hashThreads.c_str());
242 342
343 if (arg(argc,argv,"/stratumbridge") || arg(argc,argv,"-stratumbridge"))
344 {
345 fStratumBridge = true;
346 string bridgePool = argval2(argc, argv, "/stratumbridge", "-stratumbridge");
347 if (!bridgePool.empty())
348 strParticipantPool = bridgePool;
349 }
350
351 string bridgePort = argval2(argc, argv, "/stratumbridgeport", "-stratumbridgeport");
352 if (!bridgePort.empty())
353 {
354 int nPort = atoi(bridgePort.c_str());
355 if (nPort > 0 && nPort <= 65535)
356 nStratumBridgePort = nPort;
357 }
358
243 string poolName = argval2(argc, argv, "/poolname", "-poolname"); 359 string poolName = argval2(argc, argv, "/poolname", "-poolname");
244 if (!poolName.empty()) 360 if (!poolName.empty())
245 strPoolName = poolName; 361 strPoolName = poolName;
@@ -260,9 +376,18 @@ static void ParseStartupArguments(int argc, char* argv[])
260 if (!poolFee.empty()) 376 if (!poolFee.empty())
261 dPoolFeePercent = atof(poolFee.c_str()); 377 dPoolFeePercent = atof(poolFee.c_str());
262 378
379 string poolStatusFile = argval2(argc, argv, "/poolstatusfile", "-poolstatusfile");
380 if (!poolStatusFile.empty())
381 strPoolStatusFile = poolStatusFile;
382
383 string poolRoundsFile = argval2(argc, argv, "/poolroundsfile", "-poolroundsfile");
384 if (!poolRoundsFile.empty())
385 strPoolRoundsFile = poolRoundsFile;
386
263 string btfConnect = argval2(argc, argv, "/connectbtf", "-connectbtf"); 387 string btfConnect = argval2(argc, argv, "/connectbtf", "-connectbtf");
264 if (!btfConnect.empty()) 388 if (!btfConnect.empty())
265 strBtfConnect = btfConnect; 389 strBtfConnect = btfConnect;
390 fBtfOnionOnly = arg(argc, argv, "/oniononly") || arg(argc, argv, "-oniononly");
266 391
267 string rvRelay = argval2(argc, argv, "/rvrelay", "-rvrelay"); 392 string rvRelay = argval2(argc, argv, "/rvrelay", "-rvrelay");
268 if (!rvRelay.empty()) 393 if (!rvRelay.empty())
@@ -283,10 +408,102 @@ static void ParseStartupArguments(int argc, char* argv[])
283 } 408 }
284 } 409 }
285 410
411 // net.cpp has described nListenPort as "tunable via /port" since it was
412 // written, but nothing ever read the option, so the port was fixed at 8433
413 // and a second node could not start on a machine already running one. Parse
414 // it before Tor setup so a managed hidden service maps the real listener.
415 string strPort = argval2(argc, argv, "/port", "-port");
416 if (!strPort.empty())
417 {
418 int nPort = atoi(strPort.c_str());
419 if (nPort <= 0 || nPort > 65535)
420 fprintf(stderr, "Ignoring /port=%s: not a port number\n", strPort.c_str());
421 else
422 {
423 nListenPort = htons((unsigned short)nPort);
424 // Or we would announce a port we never bound.
425 addrLocalHost.port = nListenPort;
426 }
427 }
428
286 string announceRelay = argval2(argc, argv, "/announcerelay", "-announcerelay"); 429 string announceRelay = argval2(argc, argv, "/announcerelay", "-announcerelay");
287 if (!announceRelay.empty()) 430 if (!announceRelay.empty())
288 strBtfAnnounceRelay = announceRelay; 431 strBtfAnnounceRelay = announceRelay;
289 432
433 string onionService = argval2(argc, argv, "/onionservice", "-onionservice");
434 if (!onionService.empty())
435 {
436 string err;
437 if (!BtfSetLocalOnionEndpoint(onionService, err))
438 fprintf(stderr, "Ignoring /onionservice=%s: %s\n", onionService.c_str(), err.c_str());
439 else
440 fprintf(stderr, "Direct onion peer endpoint advertised: %s\n",
441 BtfLocalOnionEndpoint().c_str());
442 }
443
444 bool fManagedTor = arg(argc, argv, "/managedtor") || arg(argc, argv, "-managedtor");
445 bool fNoManagedTor = arg(argc, argv, "/nomanagedtor") || arg(argc, argv, "-nomanagedtor");
446 bool fTorMode = arg(argc, argv, "/tor") || arg(argc, argv, "-tor");
447 string socksProxy = argval2(argc, argv, "/socks", "-socks");
448 string bundledTorPath;
449 // Managed Tor is opt-in on this fork. Upstream 1.2.19 auto-starts a
450 // bundled tor whenever one sits next to the executable, which silently
451 // moves Nostr/.btf traffic onto Tor, creates a hidden service, and starts
452 // publishing descriptors with meeting_node="rendezvous-pending" on nodes
453 // that never asked for it -- against our pinned-relay posture. The
454 // upstream behavior remains available under the explicit /automanagedtor.
455 bool fAutoManagedTor = !fManagedTor && !fNoManagedTor && !fTorMode &&
456 socksProxy.empty() &&
457 (arg(argc, argv, "/automanagedtor") || arg(argc, argv, "-automanagedtor")) &&
458 BtfBundledTorPath(bundledTorPath);
459 if (fManagedTor || fAutoManagedTor)
460 {
461 if (fManagedTor && (fTorMode || !socksProxy.empty()))
462 fprintf(stderr, "Warning: /managedtor takes precedence over /tor and /socks\n");
463
464 string torPath = fManagedTor ? argval2(argc, argv, "/managedtor", "-managedtor") : bundledTorPath;
465 string err;
466 if (!BtfStartManagedTor(torPath, err))
467 {
468 if (fAutoManagedTor)
469 fprintf(stderr, "Bundled Tor not enabled automatically: %s\n", err.c_str());
470 else
471 fprintf(stderr, "Ignoring /managedtor=%s: %s\n", torPath.c_str(), err.c_str());
472 }
473 else
474 fprintf(stderr, "Managed Tor enabled%s: %s\n",
475 fAutoManagedTor ? " automatically" : "",
476 BtfManagedTorStatus().c_str());
477 }
478 else if (fTorMode)
479 {
480 if (!socksProxy.empty())
481 fprintf(stderr, "Warning: /tor takes precedence over /socks=%s\n",
482 socksProxy.c_str());
483
484 string torProxy = argval2(argc, argv, "/tor", "-tor");
485 string err;
486 if (!BtfEnableTorProxy(torProxy, err))
487 fprintf(stderr, "Ignoring /tor=%s: %s\n", torProxy.c_str(), err.c_str());
488 else
489 fprintf(stderr, "Tor mode enabled through SOCKS5 proxy %s\n",
490 BtfSocks5ProxyName().c_str());
491 }
492 else
493 {
494 if (!socksProxy.empty())
495 {
496 string err;
497 if (!BtfSetSocks5Proxy(socksProxy, err))
498 fprintf(stderr, "Ignoring /socks=%s: %s\n", socksProxy.c_str(), err.c_str());
499 else
500 fprintf(stderr, "SOCKS5 proxy enabled for outbound discovery/relay dials: %s\n",
501 BtfSocks5ProxyName().c_str());
502 }
503 }
504 if (fBtfOnionOnly && !BtfSocks5ProxyEnabled())
505 fprintf(stderr, "Warning: /oniononly without /tor, /managedtor, or /socks cannot dial .onion peers\n");
506
290 // /btfseed=ADDRESS:ENCHEX -- extra bootstrap peers, repeatable. Useful for 507 // /btfseed=ADDRESS:ENCHEX -- extra bootstrap peers, repeatable. Useful for
291 // testing the seed path and for private networks that ship no compiled list. 508 // testing the seed path and for private networks that ship no compiled list.
292 for (int i = 1; i < argc; i++) 509 for (int i = 1; i < argc; i++)
@@ -306,22 +523,6 @@ static void ParseStartupArguments(int argc, char* argv[])
306 vBtfExtraSeeds.push_back(make_pair(val.substr(0, colon), val.substr(colon + 1))); 523 vBtfExtraSeeds.push_back(make_pair(val.substr(0, colon), val.substr(colon + 1)));
307 } 524 }
308 525
309 // net.cpp has described nListenPort as "tunable via /port" since it was
310 // written, but nothing ever read the option, so the port was fixed at 8433
311 // and a second node could not start on a machine already running one.
312 string strPort = argval2(argc, argv, "/port", "-port");
313 if (!strPort.empty())
314 {
315 int nPort = atoi(strPort.c_str());
316 if (nPort <= 0 || nPort > 65535)
317 fprintf(stderr, "Ignoring /port=%s: not a port number\n", strPort.c_str());
318 else
319 {
320 nListenPort = htons((unsigned short)nPort);
321 // Or we would announce a port we never bound.
322 addrLocalHost.port = nListenPort;
323 }
324 }
325} 526}
326 527
327// Report a startup failure somewhere the user will actually see it. 528// Report a startup failure somewhere the user will actually see it.
@@ -368,6 +569,36 @@ static void FatalStartupError(bool fHeadless, const string& strWhat, const strin
368#endif 569#endif
369} 570}
370 571
572#ifdef _WIN32
573// Windows equivalent of the SIGINT/SIGTERM handler the headless loop installs on
574// POSIX. Without it a headless node on Windows had no way to reach the clean
575// StopNode()+DBFlush() path: Ctrl-C, closing the console window, a service stop
576// and OS shutdown all killed it mid-write, and a wallet.dat left unflushed by
577// Berkeley DB will not reopen. For CLOSE/LOGOFF/SHUTDOWN the OS terminates us
578// shortly after the handler returns, so we set fShutdown and then block until the
579// main loop signals the flush is done -- returning any sooner throws the flush
580// away, which is the whole bug. (A `taskkill /F` is still an uncatchable hard
581// kill; nothing can help there, and nothing should pretend to.)
582static volatile bool g_fHeadlessShutdownDone = false;
583static BOOL WINAPI HeadlessConsoleCtrlHandler(DWORD dwCtrlType)
584{
585 switch (dwCtrlType)
586 {
587 case CTRL_C_EVENT:
588 case CTRL_BREAK_EVENT:
589 case CTRL_CLOSE_EVENT:
590 case CTRL_LOGOFF_EVENT:
591 case CTRL_SHUTDOWN_EVENT:
592 fShutdown = true;
593 while (!g_fHeadlessShutdownDone)
594 Sleep(50);
595 return TRUE;
596 default:
597 return FALSE;
598 }
599}
600#endif
601
371int main(int argc, char* argv[]) 602int main(int argc, char* argv[])
372{ 603{
373 if (arg(argc,argv,"/help") || arg(argc,argv,"-help") || 604 if (arg(argc,argv,"/help") || arg(argc,argv,"-help") ||
@@ -384,10 +615,119 @@ int main(int argc, char* argv[])
384 615
385 ParseStartupArguments(argc, argv); 616 ParseStartupArguments(argc, argv);
386 617
618#ifdef _WIN32
619 // Set by the GUI "Restart Now" button: give the previous instance a moment
620 // to release the network port and the block-index files before this one
621 // opens them, so a relaunch does not race the process it is replacing.
622 if (arg(argc, argv, "/restartwait") || arg(argc, argv, "-restartwait"))
623 Sleep(2500);
624#endif
625
626 // Hidden self-test helper. It must run before LoadWallet(): several
627 // storage-sanity scenarios deliberately make wallet.dat unsafe to load.
628 string strSelfTestMutateWallet =
629 argval2(argc, argv, "/selftestmutatewallet", "-selftestmutatewallet");
630 if (!strSelfTestMutateWallet.empty())
631 return RunSelfTestMutateWallet(strSelfTestMutateWallet);
632
387 string strSelfTest = argval2(argc, argv, "/selftest", "-selftest"); 633 string strSelfTest = argval2(argc, argv, "/selftest", "-selftest");
388 if (!strSelfTest.empty()) 634 if (!strSelfTest.empty())
389 return RunSelfTest(strSelfTest); 635 return RunSelfTest(strSelfTest);
390 636
637 // Storage diagnostics must run before LoadWallet(). Their job is to
638 // explain wallet.dat states that the normal wallet loader may refuse.
639 string strWalletStorageAuditJson =
640 argval2(argc, argv, "/walletstorageauditjson", "-walletstorageauditjson");
641 bool fDiagWantsAudit = arg(argc,argv,"/walletstorageaudit") ||
642 arg(argc,argv,"-walletstorageaudit") ||
643 !strWalletStorageAuditJson.empty();
644 bool fDiagWantsCheck = arg(argc,argv,"/walletstoragecheck") ||
645 arg(argc,argv,"-walletstoragecheck");
646
647 // These diagnostics normally read wallet.dat. Under -walletbackend=sqlite
648 // the live wallet is wallet.sqlite, so open it read-only as the active
649 // store and let the same scan walk the right file. This runs before the
650 // normal backend resolution further down, so validate the value here too.
651 string strDiagBackend = argval2(argc, argv, "/walletbackend", "-walletbackend");
652 if ((fDiagWantsAudit || fDiagWantsCheck) && !strDiagBackend.empty() &&
653 strDiagBackend != "sqlite" && strDiagBackend != "bdb")
654 {
655 AttachTerminal();
656 fprintf(stderr, "Unknown wallet backend '%s'. Use -walletbackend=sqlite "
657 "or -walletbackend=bdb.\n", strDiagBackend.c_str());
658 return 1;
659 }
660 bool fDiagSQLite = (strDiagBackend == "sqlite") &&
661 (fDiagWantsAudit || fDiagWantsCheck);
662 if (fDiagSQLite)
663 {
664 AttachTerminal();
665 string strSQLitePath = GetAppDir() + "/wallet.sqlite";
666 string strOpenErr;
667 if (!WalletSQLiteRuntimeOpenReadOnly(strSQLitePath, strOpenErr))
668 {
669 fprintf(stderr, "Cannot open the SQLite wallet for diagnostics: %s\n",
670 strOpenErr.c_str());
671 return 1;
672 }
673 }
674
675 if (fDiagWantsAudit)
676 {
677 int nRet = CmdWalletStorageAudit(strWalletStorageAuditJson);
678 if (fDiagSQLite)
679 WalletSQLiteRuntimeClose();
680 else
681 DBFlush(true);
682 return nRet;
683 }
684
685 if (fDiagWantsCheck)
686 {
687 int nRet = CmdWalletStorageCheck();
688 if (fDiagSQLite)
689 WalletSQLiteRuntimeClose();
690 else
691 DBFlush(true);
692 return nRet;
693 }
694
695 string strWalletSQLiteExport =
696 argval2(argc, argv, "/walletsqliteexport", "-walletsqliteexport");
697 if (!strWalletSQLiteExport.empty())
698 {
699 int nRet = CmdWalletSQLiteExport(strWalletSQLiteExport);
700 DBFlush(true);
701 return nRet;
702 }
703
704 string strWalletSQLiteVerify =
705 argval2(argc, argv, "/walletsqliteverify", "-walletsqliteverify");
706 if (!strWalletSQLiteVerify.empty())
707 {
708 int nRet = CmdWalletSQLiteVerify(strWalletSQLiteVerify);
709 DBFlush(true);
710 return nRet;
711 }
712
713 string strWalletSQLiteRestore =
714 argval2(argc, argv, "/walletsqliterestore", "-walletsqliterestore");
715 if (!strWalletSQLiteRestore.empty())
716 {
717 int nRet = CmdWalletSQLiteRestore(strWalletSQLiteRestore);
718 DBFlush(true);
719 return nRet;
720 }
721
722 string strWalletSQLiteLoadCheck =
723 argval2(argc, argv, "/walletsqliteloadcheck", "-walletsqliteloadcheck");
724 if (!strWalletSQLiteLoadCheck.empty())
725 {
726 int nRet = CmdWalletSQLiteLoadCheck(strWalletSQLiteLoadCheck);
727 DBFlush(true);
728 return nRet;
729 }
730
391 // Berkeley DB reports failure by throwing, and CDB's constructor lets it 731 // Berkeley DB reports failure by throwing, and CDB's constructor lets it
392 // through. Nothing on this path caught anything, so an unreadable 732 // through. Nothing on this path caught anything, so an unreadable
393 // blkindex.dat or wallet.dat -- one written by another platform's Berkeley 733 // blkindex.dat or wallet.dat -- one written by another platform's Berkeley
@@ -432,18 +772,197 @@ int main(int argc, char* argv[])
432 printf("Loading wallet...\n"); 772 printf("Loading wallet...\n");
433 try 773 try
434 { 774 {
435 if (!LoadWallet()) 775 string strWalletBackend =
776 argval2(argc, argv, "/walletbackend", "-walletbackend");
777 string strWalletSQLite =
778 argval2(argc, argv, "/walletsqlite", "-walletsqlite");
779
780 // A misspelled backend must not fall through to Berkeley DB silently --
781 // someone who typed -walletbackend=sqlit meant to run on SQLite, and
782 // opening wallet.dat instead of telling them is the wrong surprise.
783 if (!strWalletBackend.empty() &&
784 strWalletBackend != "sqlite" && strWalletBackend != "bdb")
436 { 785 {
437 // LoadWallet() explains itself into strWalletLoadError when it 786 AttachTerminal();
438 // knows why -- an unsupported wallet format, for one, which is the 787 FatalStartupError(fHeadlessStartup,
439 // case this whole path exists to make visible. 788 strprintf("Unknown wallet backend '%s'.", strWalletBackend.c_str()),
440 FatalStartupError(fHeadlessStartup, "Cannot open wallet.dat.", 789 "Use -walletbackend=sqlite for the SQLite backend, or "
441 strWalletLoadError); 790 "-walletbackend=bdb for the Berkeley DB wallet.dat.");
442 return 1; 791 return 1;
443 } 792 }
444 793
794 if (strWalletBackend == "sqlite")
795 {
796 // Explicit opt-in via the flag. The node runs on
797 // <datadir>/wallet.sqlite and never opens wallet.dat, but the user
798 // has to have exported one first -- there is no silent migration,
799 // and wallet.dat is left
800 // exactly where it is so a plain restart goes back to it.
801 AttachTerminal();
802 string strSQLitePath = GetAppDir() + "/wallet.sqlite";
803 if (!FileExists(strSQLitePath.c_str()))
804 {
805 string strGuide = strprintf(
806 "No SQLite wallet was found at\n"
807 " %s\n\n"
808 "The SQLite wallet backend is opt-in and needs an exported\n"
809 "wallet first. Your Berkeley DB wallet.dat is not touched by\n"
810 "any of this.\n\n"
811 " Step 1 Export your existing wallet to SQLite:\n"
812 " bitflash -walletsqliteexport=\"%s\"\n"
813 " Step 2 Start again with the SQLite backend:\n"
814 " bitflash -walletbackend=sqlite\n\n"
815 "To go back to Berkeley DB at any time, just start without\n"
816 "-walletbackend=sqlite. wallet.dat is still your wallet.",
817 strSQLitePath.c_str(), strSQLitePath.c_str());
818 FatalStartupError(fHeadlessStartup,
819 "The SQLite wallet backend needs an exported wallet first.",
820 strGuide);
821 return 1;
822 }
823
824 if (!LoadWalletFromSQLiteRuntime(strSQLitePath))
825 {
826 FatalStartupError(fHeadlessStartup,
827 "Cannot open the SQLite wallet backend.",
828 strWalletLoadError);
829 return 1;
830 }
831
832 fprintf(stderr,
833 "Wallet backend: SQLite\n"
834 " %s\n"
835 "Your Berkeley DB wallet.dat is left untouched. Restart without\n"
836 "-walletbackend=sqlite to go back to it.\n",
837 strSQLitePath.c_str());
838 fflush(stderr);
839 // Fall through and run the node normally: every CWalletDB read,
840 // write, and erase now routes to this SQLite file.
841 }
842 else if (!strWalletSQLite.empty())
843 {
844 AttachTerminal();
845 if (!LoadWalletFromSQLite(strWalletSQLite))
846 {
847 FatalStartupError(fHeadlessStartup,
848 "Cannot open SQLite wallet export.",
849 strWalletLoadError);
850 return 1;
851 }
852
853 fprintf(stderr, "SQLite wallet loaded: read-only staging backend\n");
854 fprintf(stderr, "SQLite runtime writes are not enabled yet; exiting before "
855 "network, mining, GUI, or wallet mutation starts.\n");
856 fflush(stderr);
857 DBFlush(true);
858 return 0;
859 }
860 else if (strWalletBackend == "bdb")
861 {
862 // Explicit -walletbackend=bdb overrides the marker and the default:
863 // force Berkeley DB (LoadWallet creates wallet.dat if none exists).
864 if (!LoadWallet())
865 {
866 FatalStartupError(fHeadlessStartup, "Cannot open wallet.dat.",
867 strWalletLoadError);
868 return 1;
869 }
870 }
871 else
872 {
873 // No explicit -walletbackend on the command line. Resolve it from
874 // the persistent marker, then defaults: an existing wallet.sqlite
875 // stays on SQLite, an existing wallet.dat stays on Berkeley DB (the
876 // GUI offers to convert), and a fresh install starts on SQLite --
877 // the new default. wallet.dat is never touched by any of this.
878 string strSQLitePath = GetAppDir() + "/wallet.sqlite";
879 bool fSQLiteExists = FileExists(strSQLitePath.c_str());
880 bool fWalletDatExists = FileExists((GetAppDir() + "/wallet.dat").c_str());
881 string strMarker = ReadWalletBackendMarker();
882
883 bool fUseSQLite = false;
884 bool fCreateSQLite = false;
885 if (strMarker == "sqlite")
886 {
887 if (fSQLiteExists) fUseSQLite = true;
888 else if (fWalletDatExists) fUseSQLite = false; // marker stale; real wallet is BDB
889 else { fUseSQLite = true; fCreateSQLite = true; }
890 }
891 else if (strMarker == "bdb")
892 {
893 fUseSQLite = false;
894 }
895 else // no marker recorded yet
896 {
897 if (fSQLiteExists && !fWalletDatExists) fUseSQLite = true;
898 else if (fWalletDatExists) fUseSQLite = false;
899 else { fUseSQLite = true; fCreateSQLite = true; }
900 }
901
902 // A headless node whose datadir holds no wallet of either kind is
903 // far more often an operations fault -- datadir not mounted, or
904 // wallet.dat caught mid-rename by a backup -- than a first run.
905 // Creating one here is not the harmless default it looks like:
906 // success also writes the backend marker, and from then on
907 // restoring the real wallet.dat no longer takes effect, because the
908 // marker keeps the node on the empty wallet it just made. Under
909 // 1.2.19 the same fault created a wallet.dat that a restore
910 // overwrote, so operators could recover by putting the file back.
911 // Refuse instead, and name the flag that makes it deliberate.
912 if (fCreateSQLite && fHeadlessStartup &&
913 !arg(argc, argv, "/createwallet") && !arg(argc, argv, "-createwallet"))
914 {
915 AttachTerminal();
916 fprintf(stderr,
917 "No wallet found in %s\n"
918 "Refusing to create one. On a headless node an absent wallet is\n"
919 "usually a mount or backup fault, and creating one would record a\n"
920 "backend marker that makes restoring the real wallet a no-op.\n"
921 "Pass -createwallet to start a new one deliberately, or\n"
922 "-walletbackend=bdb to run on a Berkeley DB wallet.dat.\n",
923 GetAppDir().c_str());
924 fflush(stderr);
925 return 1;
926 }
927
928 if (fUseSQLite)
929 {
930 AttachTerminal();
931 bool fOk = fCreateSQLite
932 ? CreateNewSQLiteWallet(strSQLitePath)
933 : LoadWalletFromSQLiteRuntime(strSQLitePath);
934 if (!fOk)
935 {
936 FatalStartupError(fHeadlessStartup,
937 fCreateSQLite ? "Cannot create the SQLite wallet."
938 : "Cannot open the SQLite wallet.",
939 strWalletLoadError);
940 return 1;
941 }
942 // Record the choice so it is stable from here on.
943 if (ReadWalletBackendMarker() != "sqlite")
944 WriteWalletBackendMarker("sqlite");
945 fprintf(stderr, "Wallet backend: SQLite\n %s\n%s",
946 strSQLitePath.c_str(),
947 fCreateSQLite ? "A new wallet was created here.\n" : "");
948 fflush(stderr);
949 // Fall through and run the node; CWalletDB routes to this file.
950 }
951 else if (!LoadWallet())
952 {
953 // LoadWallet() explains itself into strWalletLoadError when it
954 // knows why -- an unsupported wallet format, for one, which is
955 // the case this whole path exists to make visible.
956 FatalStartupError(fHeadlessStartup, "Cannot open wallet.dat.",
957 strWalletLoadError);
958 return 1;
959 }
960 }
961
445 // After the block index, so there is a chain to compare the wallet 962 // After the block index, so there is a chain to compare the wallet
446 // against, and before anything reports a balance. 963 // against, and before anything reports a balance. Works on either
964 // backend: RescanSpentFlags writes corrected flags through CWalletDB,
965 // which routes to SQLite when that backend is active.
447 RescanSpentFlags(); 966 RescanSpentFlags();
448 } 967 }
449 catch (const std::exception& e) 968 catch (const std::exception& e)
@@ -668,7 +1187,12 @@ int main(int argc, char* argv[])
668 return 0; 1187 return 0;
669 } 1188 }
670 1189
671 if (!StartNode(strErrors)) { fprintf(stderr,"StartNode: %s\n",strErrors.c_str()); return 1; } 1190 if (!StartNode(strErrors))
1191 {
1192 fprintf(stderr,"StartNode: %s\n",strErrors.c_str());
1193 BtfStopManagedTor();
1194 return 1;
1195 }
672 1196
673 if (nMineMode == MINE_OPERATOR) { 1197 if (nMineMode == MINE_OPERATOR) {
674 gPoolServerRunning = true; 1198 gPoolServerRunning = true;
@@ -676,6 +1200,10 @@ int main(int argc, char* argv[])
676 if (_beginthread(ThreadRPCServer, 0, NULL) == (uintptr_t)-1) 1200 if (_beginthread(ThreadRPCServer, 0, NULL) == (uintptr_t)-1)
677 printf("Error: _beginthread(ThreadRPCServer) failed\n"); 1201 printf("Error: _beginthread(ThreadRPCServer) failed\n");
678 } 1202 }
1203 if (fStratumBridge) {
1204 if (_beginthread(ThreadStratumBridge, 0, NULL) == (uintptr_t)-1)
1205 printf("Error: _beginthread(ThreadStratumBridge) failed\n");
1206 }
679 if (fGenerateBitcoins) 1207 if (fGenerateBitcoins)
680 StartMinerThreads(); 1208 StartMinerThreads();
681 1209
@@ -688,18 +1216,30 @@ int main(int argc, char* argv[])
688#endif 1216#endif
689 1217
690 if (fHeadless) { 1218 if (fHeadless) {
691#ifndef _WIN32 1219#ifdef _WIN32
1220 SetConsoleCtrlHandler(HeadlessConsoleCtrlHandler, TRUE);
1221#else
692 auto sig=[](int){fShutdown=true;}; 1222 auto sig=[](int){fShutdown=true;};
693 signal(SIGINT,sig); signal(SIGTERM,sig); 1223 signal(SIGINT,sig); signal(SIGTERM,sig);
694#endif 1224#endif
695 printf("Running headless. Ctrl-C to stop.\n"); 1225 printf("Running headless. Ctrl-C to stop.\n");
696 while (!fShutdown) Sleep(500); 1226 while (!fShutdown) Sleep(500);
697 StopNode(); 1227 StopNode();
1228 BtfStopManagedTor();
698 // Checkpoints, releases wallet.dat from this directory's Berkeley DB 1229 // Checkpoints, releases wallet.dat from this directory's Berkeley DB
699 // environment, and closes it. Declared since 2009 and never once 1230 // environment, and closes it. Declared since 2009 and never once
700 // called here, which is why a wallet.dat copied elsewhere would not 1231 // called here, which is why a wallet.dat copied elsewhere would not
701 // open -- issue #40. Not optional. 1232 // open -- issue #40. Not optional.
702 DBFlush(true); 1233 DBFlush(true);
1234 // Checkpoint and close the SQLite wallet backend if one is active. A
1235 // no-op on the Berkeley DB path. Safe to kill before this runs: WAL +
1236 // synchronous=FULL already made every committed write durable.
1237 WalletSQLiteRuntimeClose();
1238#ifdef _WIN32
1239 // wallet.dat is safely flushed and closed now; let a pending console
1240 // control handler (CLOSE/LOGOFF/SHUTDOWN) return so the OS can finish.
1241 g_fHeadlessShutdownDone = true;
1242#endif
703 return 0; 1243 return 0;
704 } 1244 }
705 1245
@@ -709,7 +1249,9 @@ int main(int argc, char* argv[])
709 int ret = RunGUI(argc, argv); 1249 int ret = RunGUI(argc, argv);
710 fShutdown = true; 1250 fShutdown = true;
711 StopNode(); 1251 StopNode();
1252 BtfStopManagedTor();
712 DBFlush(true); 1253 DBFlush(true);
1254 WalletSQLiteRuntimeClose();
713 return ret; 1255 return ret;
714#endif 1256#endif
715} 1257}
diff --git a/src/makefile.mingw b/src/makefile.mingw
index 72f8bbf..362794a 100644
--- a/src/makefile.mingw
+++ b/src/makefile.mingw
@@ -14,14 +14,9 @@ WARNFLAGS = -Wformat -Werror=format \
14 -Wno-deprecated-declarations -Wno-write-strings \ 14 -Wno-deprecated-declarations -Wno-write-strings \
15 -Wno-narrowing -Wno-unused-result 15 -Wno-narrowing -Wno-unused-result
16 16
17# FD_SETSIZE has to be defined before winsock2.h and it has to be the same
18# number in every translation unit -- fd_set carries an array of that length,
19# so disagreeing about it means disagreeing about sizeof(fd_set). That is why
20# it lives here as a -D and not as a #define in a header. Windows defaults to
21# 64, which capped this node at 63 watchable peers no matter how many it had.
22CXXFLAGS = -std=gnu++14 -O2 $(WARNFLAGS) -fpermissive -pthread \ 17CXXFLAGS = -std=gnu++14 -O2 $(WARNFLAGS) -fpermissive -pthread \
23 -DWIN32 -DNOPCH -DFD_SETSIZE=1024 \ 18 -DWIN32 -DNOPCH \
24 -DSECP256K1_STATIC \ 19 -DSECP256K1_STATIC -DBITFLASH_NODE_BUILD \
25 -I$(DEPS)/include -I$(RANDOMX)/src \ 20 -I$(DEPS)/include -I$(RANDOMX)/src \
26 -I$(DEPS)/include/nlohmann \ 21 -I$(DEPS)/include/nlohmann \
27 -IC:/msys64/ucrt64/include \ 22 -IC:/msys64/ucrt64/include \
@@ -29,7 +24,7 @@ CXXFLAGS = -std=gnu++14 -O2 $(WARNFLAGS) -fpermissive -pthread \
29 24
30LIBS = -L$(DEPS)/lib -lsecp256k1 \ 25LIBS = -L$(DEPS)/lib -lsecp256k1 \
31 -L$(RANDOMX)/build -lrandomx \ 26 -L$(RANDOMX)/build -lrandomx \
32 -lsodium -lssl -lcrypto -ldb_cxx -lpthread \ 27 -lsodium -lssl -lcrypto -ldb_cxx -lsqlite3 -lpthread \
33 -LC:/msys64/ucrt64/lib \ 28 -LC:/msys64/ucrt64/lib \
34 -lglfw3 -lopengl32 -lgdi32 \ 29 -lglfw3 -lopengl32 -lgdi32 \
35 -lkernel32 -luser32 -lcomdlg32 -lwinspool -lwinmm \ 30 -lkernel32 -luser32 -lcomdlg32 -lwinspool -lwinmm \
@@ -42,7 +37,8 @@ IMGUI_OBJS = obj/imgui.o obj/imgui_draw.o obj/imgui_tables.o \
42NODE_OBJS = obj/util.o obj/script.o obj/db.o obj/net.o obj/main.o \ 37NODE_OBJS = obj/util.o obj/script.o obj/db.o obj/net.o obj/main.o \
43 obj/sha.o obj/nostr.o obj/randomx_pow.o \ 38 obj/sha.o obj/nostr.o obj/randomx_pow.o \
44 obj/btfaddr.o obj/btfchan.o obj/btfrv.o obj/btftunnel.o obj/rpc.o \ 39 obj/btfaddr.o obj/btfchan.o obj/btfrv.o obj/btftunnel.o obj/rpc.o \
45 obj/selftest.o obj/bip32.o obj/walletcmd.o 40 obj/selftest.o obj/bip32.o obj/walletcmd.o obj/wallet_sqlite.o \
41 obj/proxy.o obj/tor.o
46 42
47GUI_OBJS = obj/gui.o obj/main_gui.o obj/bitflash_res.o 43GUI_OBJS = obj/gui.o obj/main_gui.o obj/bitflash_res.o
48 44
@@ -56,7 +52,7 @@ GUI_OBJS = obj/gui.o obj/main_gui.o obj/bitflash_res.o
56HEADERS = headers_core.h headers.h util.h main.h serialize.h uint256.h \ 52HEADERS = headers_core.h headers.h util.h main.h serialize.h uint256.h \
57 key.h bignum.h script.h db.h base58.h net.h nostr.h \ 53 key.h bignum.h script.h db.h base58.h net.h nostr.h \
58 randomx_pow.h btfaddr.h btfchan.h btfrv.h btftunnel.h \ 54 randomx_pow.h btfaddr.h btfchan.h btfrv.h btftunnel.h \
59 selftest.h walletcmd.h sockcount.h compat.h font_roboto.h 55 selftest.h walletcmd.h wallet_sqlite.h sockcount.h proxy.h tor.h compat.h font_roboto.h
60 56
61all: bitflash.exe 57all: bitflash.exe
62 58
@@ -112,7 +108,16 @@ tests: test_bip32.exe bitflash.exe
112 ./bitflash.exe -selftest=wallet-keypool 108 ./bitflash.exe -selftest=wallet-keypool
113 ./bitflash.exe -selftest=wallet-hd 109 ./bitflash.exe -selftest=wallet-hd
114 ./bitflash.exe -selftest=wallet-format 110 ./bitflash.exe -selftest=wallet-format
111 ./bitflash.exe -selftest=wallet-sqlite
112 ./bitflash.exe -selftest=wallet-sqlite-migration
115 ./bitflash.exe -selftest=wallet-crypto 113 ./bitflash.exe -selftest=wallet-crypto
116 ./bitflash.exe -selftest=wallet-encrypt 114 ./bitflash.exe -selftest=wallet-encrypt
115 ./bitflash.exe -selftest=wallet-sqlite-encrypt
116 ./bitflash.exe -selftest=wallet-backend-default
117 ./bitflash.exe -selftest=wallet-convert
117 ./bitflash.exe -selftest=net-message 118 ./bitflash.exe -selftest=net-message
119 ./bitflash.exe -selftest=network-params
118 ./bitflash.exe -selftest=consensus-limits 120 ./bitflash.exe -selftest=consensus-limits
121 ./bitflash.exe -selftest=pool-stratum
122 ./bitflash.exe -selftest=socks5-proxy
123 ./bitflash.exe -selftest=managed-tor
diff --git a/src/net.cpp b/src/net.cpp
index 34c608c..0576402 100644
--- a/src/net.cpp
+++ b/src/net.cpp
@@ -13,15 +13,35 @@
13#include <thread> // hardware_concurrency, for the miner thread count 13#include <thread> // hardware_concurrency, for the miner thread count
14#ifdef _WIN32 14#ifdef _WIN32
15#include <winsock2.h> 15#include <winsock2.h>
16#else
17#include <poll.h>
16#endif 18#endif
17#include <openssl/rand.h> 19#include <openssl/rand.h>
20#include <cerrno>
21#include <cstdlib>
18#include "btfaddr.h" 22#include "btfaddr.h"
19#include "btftunnel.h" 23#include "btftunnel.h"
24#include "proxy.h"
25#include "tor.h"
20 26
21void ThreadReconnectCachedBtfPeers(void* parg); 27void ThreadReconnectCachedBtfPeers(void* parg);
22void ThreadMessageHandler2(void* parg); 28void ThreadMessageHandler2(void* parg);
23void ThreadSocketHandler2(void* parg); 29void ThreadSocketHandler2(void* parg);
24 30
31#ifdef _WIN32
32typedef WSAPOLLFD BtfPollFd;
33static int BtfPoll(BtfPollFd* pfd, unsigned int nfd, int nTimeoutMs)
34{
35 return WSAPoll(pfd, nfd, nTimeoutMs);
36}
37#else
38typedef struct pollfd BtfPollFd;
39static int BtfPoll(BtfPollFd* pfd, unsigned int nfd, int nTimeoutMs)
40{
41 return poll(pfd, (nfds_t)nfd, nTimeoutMs);
42}
43#endif
44
25 45
26 46
27 47
@@ -30,10 +50,14 @@ void ThreadSocketHandler2(void* parg);
30// 50//
31// Global state variables 51// Global state variables
32// 52//
53static const char pchMainnetMessageStart[4] = { 0xbf, 0x20, 0x5c, 0xfd };
54static const char pchTestnetMessageStart[4] = { (char)0xce, (char)0xe2, (char)0xc0, (char)0xff };
55static bool fTestNet = false;
56char pchMessageStart[4] = { 0xbf, 0x20, 0x5c, 0xfd };
33bool fClient = false; 57bool fClient = false;
34uint64 nLocalServices = (fClient ? 0 : NODE_NETWORK); 58uint64 nLocalServices = (fClient ? 0 : NODE_NETWORK);
35CAddress addrLocalHost(0, DEFAULT_PORT, nLocalServices); 59CAddress addrLocalHost(0, GetDefaultPort(), nLocalServices);
36unsigned short nListenPort = DEFAULT_PORT; // local P2P port (tunable via /port) 60unsigned short nListenPort = GetDefaultPort(); // local P2P port (tunable via /port)
37CNode nodeLocalHost(INVALID_SOCKET, CAddress("127.0.0.1", nLocalServices)); 61CNode nodeLocalHost(INVALID_SOCKET, CAddress("127.0.0.1", nLocalServices));
38CNode* pnodeLocalHost = &nodeLocalHost; 62CNode* pnodeLocalHost = &nodeLocalHost;
39bool fShutdown = false; 63bool fShutdown = false;
@@ -45,6 +69,8 @@ deque<pair<int64, CInv> > vRelayExpiration;
45CCriticalSection cs_mapRelay; 69CCriticalSection cs_mapRelay;
46map<CInv, int64> mapAlreadyAskedFor; 70map<CInv, int64> mapAlreadyAskedFor;
47string strBtfConnect; // .btf peer to keep connected to (from /connectbtf) 71string strBtfConnect; // .btf peer to keep connected to (from /connectbtf)
72bool fBtfOnionOnly = false; // require direct onion transport for .btf peers
73static const int BTF_ONION_CONNECT_TIMEOUT_SECS = 120;
48 74
49int nPeersWatched = 0; 75int nPeersWatched = 0;
50int64 nBlocksReceived = 0; 76int64 nBlocksReceived = 0;
@@ -68,6 +94,29 @@ static int64 nBtfHandshakeSilent = 0;
68static string strBtfLastDialFail; 94static string strBtfLastDialFail;
69static string strBtfLastHandshakeTimeout; 95static string strBtfLastHandshakeTimeout;
70 96
97bool IsTestNet()
98{
99 return fTestNet;
100}
101
102unsigned short GetDefaultPort()
103{
104 return htons(fTestNet ? TESTNET_PORT : MAINNET_PORT);
105}
106
107void SelectNetworkParams(bool fTestNetIn)
108{
109 if (fTestNet == fTestNetIn)
110 return;
111
112 fTestNet = fTestNetIn;
113 memcpy(pchMessageStart,
114 fTestNet ? pchTestnetMessageStart : pchMainnetMessageStart,
115 sizeof(pchMessageStart));
116 nListenPort = GetDefaultPort();
117 addrLocalHost.port = nListenPort;
118}
119
71void BtfChurnNoteResolveAttempt() 120void BtfChurnNoteResolveAttempt()
72{ 121{
73 CRITICAL_BLOCK(cs_btfChurn) 122 CRITICAL_BLOCK(cs_btfChurn)
@@ -153,7 +202,7 @@ static const char* pszSockSite[SOCK_SITES] = {
153 "external-ip probe", "listen socket", "inbound accept", 202 "external-ip probe", "listen socket", "inbound accept",
154 "rendezvous dial", "rendezvous listen", "rendezvous accept", 203 "rendezvous dial", "rendezvous listen", "rendezvous accept",
155 "loopback pair listener", "loopback pair app end", "loopback pair pump end", 204 "loopback pair listener", "loopback pair app end", "loopback pair pump end",
156 "nostr relay" 205 "nostr relay", "direct onion peer", "stratum bridge"
157}; 206};
158 207
159static string SockAccountingText() 208static string SockAccountingText()
@@ -246,20 +295,27 @@ string GetDiagnosticsText()
246 int64 nNow = GetTime(); 295 int64 nNow = GetTime();
247 string str; 296 string str;
248 297
249 int nInbound = 0, nHeld = 0; 298 int nInbound = 0, nHeld = 0, nDirectOnion = 0, nRendezvous = 0;
250 vector<CNode*> vCopy; 299 vector<CNode*> vCopy;
251 CRITICAL_BLOCK(cs_vNodes) 300 CRITICAL_BLOCK(cs_vNodes)
252 { 301 {
253 vCopy = vNodes; 302 vCopy = vNodes;
254 nHeld = (int)vNodes.size(); 303 nHeld = (int)vNodes.size();
255 foreach(CNode* pnode, vNodes) 304 foreach(CNode* pnode, vNodes)
305 {
256 if (pnode->fInbound) 306 if (pnode->fInbound)
257 nInbound++; 307 nInbound++;
308 if (pnode->strBtfMeeting.find(".onion:") != string::npos)
309 nDirectOnion++;
310 else if (!pnode->strBtfMeeting.empty())
311 nRendezvous++;
312 }
258 } 313 }
259 314
260 int nMedian = GetPeerMedianHeight(); 315 int nMedian = GetPeerMedianHeight();
261 316
262 str += "Bitflash node diagnostics\n"; 317 str += "Bitflash node diagnostics\n";
318 str += strprintf(" network %s\n", IsTestNet() ? "testnet" : "mainnet");
263 str += strprintf(" uptime %s\n", 319 str += strprintf(" uptime %s\n",
264 FormatAge(nNodeStartTime ? nNow - nNodeStartTime : -1).c_str()); 320 FormatAge(nNodeStartTime ? nNow - nNodeStartTime : -1).c_str());
265 if (nMedian < 0) 321 if (nMedian < 0)
@@ -274,9 +330,26 @@ string GetDiagnosticsText()
274 330
275 // The number that would have made the deafness obvious. Anything held but 331 // The number that would have made the deafness obvious. Anything held but
276 // not watched is a socket this node will never read again. 332 // not watched is a socket this node will never read again.
277 str += strprintf(" peers watched %d of %d that select() can hold%s\n", 333 str += strprintf(" peers watched %d of %d held by poll%s\n",
278 nPeersWatched, (int)FD_SETSIZE - 1, 334 nPeersWatched, nHeld,
279 nHeld > nPeersWatched ? " <-- NOT ALL PEERS ARE BEING READ" : ""); 335 nHeld > nPeersWatched ? " <-- NOT ALL PEERS ARE BEING READ" : "");
336 str += strprintf(" managed Tor %s\n", BtfManagedTorStatus().c_str());
337
338 string strTorMode = "disabled";
339 if (BtfManagedTorEnabled())
340 strTorMode = "managed";
341 else if (BtfTorProxyEnabled())
342 strTorMode = "external";
343 else if (BtfSocks5ProxyEnabled())
344 strTorMode = "socks5";
345 string strOnion = BtfLocalOnionEndpoint();
346 str += strprintf(" Tor mode %s%s\n",
347 strTorMode.c_str(),
348 fBtfOnionOnly ? " (onion-only)" : "");
349 str += strprintf(" onion endpoint %s\n",
350 strOnion.empty() ? "none" : strOnion.c_str());
351 str += strprintf(" .btf peers %d direct onion, %d rendezvous\n",
352 nDirectOnion, nRendezvous);
280 353
281 if (nBlocksReceived > 0) 354 if (nBlocksReceived > 0)
282 str += strprintf(" blocks received %lld (%lld arrived without a parent, %.1f%%)\n", 355 str += strprintf(" blocks received %lld (%lld arrived without a parent, %.1f%%)\n",
@@ -293,6 +366,7 @@ string GetDiagnosticsText()
293 nMining, 366 nMining,
294 (RandomXFastReady() ? 2080 : 256) + 2 * nMining).c_str() 367 (RandomXFastReady() ? 2080 : 256) + 2 * nMining).c_str()
295 : ", not mining"); 368 : ", not mining");
369 str += strprintf(" large pages %s\n", RandomXLargePagesStatus());
296 370
297 str += SockAccountingText(); 371 str += SockAccountingText();
298 372
@@ -361,6 +435,9 @@ string GetDiagnosticsText()
361 435
362bool GetMyExternalIP(unsigned int& ipRet) 436bool GetMyExternalIP(unsigned int& ipRet)
363{ 437{
438 if (BtfSocks5ProxyEnabled())
439 return error("GetMyExternalIP() skipped while SOCKS5 proxy is enabled\n");
440
364 // Try several plain-text IP echo services in order. 441 // Try several plain-text IP echo services in order.
365 // Each returns just the IPv4 address as the first line of the HTTP body. 442 // Each returns just the IPv4 address as the first line of the HTTP body.
366 struct { const char* host; const char* path; } services[] = { 443 struct { const char* host; const char* path; } services[] = {
@@ -524,6 +601,7 @@ struct CachedBtfPeer
524{ 601{
525 string btfAddr; 602 string btfAddr;
526 string meeting; 603 string meeting;
604 string onion;
527 string encHex; 605 string encHex;
528 int64 lastSeen; 606 int64 lastSeen;
529 // The peer's own signed descriptor, as it announced itself. Kept verbatim 607 // The peer's own signed descriptor, as it announced itself. Kept verbatim
@@ -582,6 +660,8 @@ void LoadCachedBtfPeers(vector<CachedBtfPeer>& out)
582 CachedBtfPeer p; 660 CachedBtfPeer p;
583 p.btfAddr = o["btf"].get<string>(); 661 p.btfAddr = o["btf"].get<string>();
584 p.meeting = o["meeting"].get<string>(); 662 p.meeting = o["meeting"].get<string>();
663 if (o.contains("onion") && o["onion"].is_string())
664 btf::NormalizeOnionEndpoint(o["onion"].get<string>(), p.onion);
585 p.encHex = o["enc"].get<string>(); 665 p.encHex = o["enc"].get<string>();
586 p.lastSeen = o.value("seen", (int64)0); 666 p.lastSeen = o.value("seen", (int64)0);
587 if (o.contains("desc") && o["desc"].is_string()) 667 if (o.contains("desc") && o["desc"].is_string())
@@ -605,7 +685,8 @@ static CCriticalSection cs_btfPeerCache;
605// learned about by exchange must not cost us the ability to pass it on. 685// learned about by exchange must not cost us the ability to pass it on.
606static void RememberBtfPeer(const string& strBtfAddr, const string& strMeeting, 686static void RememberBtfPeer(const string& strBtfAddr, const string& strMeeting,
607 const unsigned char enc_pub[32], 687 const unsigned char enc_pub[32],
608 const string& strDesc = string()) 688 const string& strDesc = string(),
689 const string& strOnion = string())
609{ 690{
610 CRITICAL_BLOCK(cs_btfPeerCache) 691 CRITICAL_BLOCK(cs_btfPeerCache)
611 { 692 {
@@ -613,11 +694,15 @@ static void RememberBtfPeer(const string& strBtfAddr, const string& strMeeting,
613 LoadCachedBtfPeers(peers); 694 LoadCachedBtfPeers(peers);
614 695
615 string strKeepDesc = strDesc; 696 string strKeepDesc = strDesc;
697 string strKeepOnion;
698 btf::NormalizeOnionEndpoint(strOnion, strKeepOnion);
616 for (size_t i = 0; i < peers.size(); i++) 699 for (size_t i = 0; i < peers.size(); i++)
617 if (peers[i].btfAddr == strBtfAddr) 700 if (peers[i].btfAddr == strBtfAddr)
618 { 701 {
619 if (strKeepDesc.empty()) 702 if (strKeepDesc.empty())
620 strKeepDesc = peers[i].desc; 703 strKeepDesc = peers[i].desc;
704 if (strKeepOnion.empty())
705 strKeepOnion = peers[i].onion;
621 peers.erase(peers.begin() + i); 706 peers.erase(peers.begin() + i);
622 break; 707 break;
623 } 708 }
@@ -625,6 +710,7 @@ static void RememberBtfPeer(const string& strBtfAddr, const string& strMeeting,
625 CachedBtfPeer p; 710 CachedBtfPeer p;
626 p.btfAddr = strBtfAddr; 711 p.btfAddr = strBtfAddr;
627 p.meeting = strMeeting; 712 p.meeting = strMeeting;
713 p.onion = strKeepOnion;
628 p.encHex = BytesToHex(enc_pub, 32); 714 p.encHex = BytesToHex(enc_pub, 32);
629 p.lastSeen = GetTime(); 715 p.lastSeen = GetTime();
630 p.desc = strKeepDesc; 716 p.desc = strKeepDesc;
@@ -639,6 +725,8 @@ static void RememberBtfPeer(const string& strBtfAddr, const string& strMeeting,
639 nlohmann::json o; 725 nlohmann::json o;
640 o["btf"] = q.btfAddr; 726 o["btf"] = q.btfAddr;
641 o["meeting"] = q.meeting; 727 o["meeting"] = q.meeting;
728 if (!q.onion.empty())
729 o["onion"] = q.onion;
642 o["enc"] = q.encHex; 730 o["enc"] = q.encHex;
643 o["seen"] = q.lastSeen; 731 o["seen"] = q.lastSeen;
644 if (!q.desc.empty()) 732 if (!q.desc.empty())
@@ -727,7 +815,7 @@ static int TryBtfSeeds()
727 foreach(const string& strRelay, relays) 815 foreach(const string& strRelay, relays)
728 { 816 {
729 if (fShutdown) return nConnected; 817 if (fShutdown) return nConnected;
730 if (ConnectNodeBtfResolved(s.first, strRelay, enc)) 818 if (ConnectNodeBtfResolved(s.first, strRelay, "", enc))
731 { 819 {
732 LogPrint("net", "btfseed: reached %s via %s\n", 820 LogPrint("net", "btfseed: reached %s via %s\n",
733 s.first.c_str(), strRelay.c_str()); 821 s.first.c_str(), strRelay.c_str());
@@ -765,7 +853,7 @@ void ThreadReconnectCachedBtfPeers(void* parg)
765 if (fShutdown) return; 853 if (fShutdown) return;
766 unsigned char enc[32]; 854 unsigned char enc[32];
767 if (!HexToBytes(p.encHex, enc, 32)) continue; 855 if (!HexToBytes(p.encHex, enc, 32)) continue;
768 if (ConnectNodeBtfResolved(p.btfAddr, p.meeting, enc)) 856 if (ConnectNodeBtfResolved(p.btfAddr, p.meeting, p.onion, enc))
769 { 857 {
770 nConnected++; 858 nConnected++;
771 if (nConnected >= 8) break; // enough to bootstrap; the rest can wait 859 if (nConnected >= 8) break; // enough to bootstrap; the rest can wait
@@ -856,14 +944,37 @@ int BtfPexAccept(const vector<string>& vDesc)
856 if (!HexToBytes(d.enc, enc, 32)) 944 if (!HexToBytes(d.enc, enc, 32))
857 continue; 945 continue;
858 946
859 RememberBtfPeer(strAddr, d.meeting_node, enc, strDesc); 947 RememberBtfPeer(strAddr, d.meeting_node, enc, strDesc, d.onion);
860 nKept++; 948 nKept++;
861 } 949 }
862 return nKept; 950 return nKept;
863} 951}
864 952
953static bool SplitHostPort(const string& strHostPort, string& strHost, unsigned short& nPort)
954{
955 size_t colon = strHostPort.rfind(':');
956 if (colon == string::npos || colon == 0 || colon + 1 >= strHostPort.size())
957 return false;
958 string port = strHostPort.substr(colon + 1);
959 for (size_t i = 0; i < port.size(); i++)
960 if (!isdigit((unsigned char)port[i]))
961 return false;
962
963 errno = 0;
964 char* end = NULL;
965 long n = strtol(port.c_str(), &end, 10);
966 if (errno == ERANGE || end == NULL || *end != '\0' || n <= 0 || n > 65535)
967 return false;
968
969 strHost = strHostPort.substr(0, colon);
970 nPort = (unsigned short)n;
971 return true;
972}
973
865static CNode* ConnectNodeBtfTail(const string& strBtfAddr, const unsigned char pk[32], 974static CNode* ConnectNodeBtfTail(const string& strBtfAddr, const unsigned char pk[32],
866 const string& strMeeting, const unsigned char enc_pub[32]) 975 const string& strMeeting, const string& strOnion,
976 const unsigned char enc_pub[32],
977 const string& strDesc = string())
867{ 978{
868 CAddress addr = BtfMarkerAddr(pk); 979 CAddress addr = BtfMarkerAddr(pk);
869 CNode* pnode = FindNode(addr.ip); 980 CNode* pnode = FindNode(addr.ip);
@@ -873,16 +984,54 @@ static CNode* ConnectNodeBtfTail(const string& strBtfAddr, const unsigned char p
873 return pnode; 984 return pnode;
874 } 985 }
875 986
876 BtfChurnNoteDialAttempt(strBtfAddr, strMeeting); 987 string strOnionNorm;
877 size_t colon = strMeeting.rfind(':'); 988 if (BtfSocks5ProxyEnabled() && btf::NormalizeOnionEndpoint(strOnion, strOnionNorm))
878 if (colon == string::npos)
879 { 989 {
880 BtfChurnNoteDialResult(strBtfAddr, strMeeting, false); 990 string strOnionHost;
991 unsigned short nOnionPort = 0;
992 if (SplitHostPort(strOnionNorm, strOnionHost, nOnionPort))
993 {
994 BtfChurnNoteDialAttempt(strBtfAddr, strOnionNorm);
995 SOCKET hOnionSocket = BtfConnectSocket(strOnionHost, nOnionPort, SOCK_ONION_PEER,
996 BTF_ONION_CONNECT_TIMEOUT_SECS);
997 if (hOnionSocket != INVALID_SOCKET)
998 {
999 if (fDebug)
1000 LogPrint("net", "connected %s via direct onion %s\n",
1001 strBtfAddr.c_str(), strOnionNorm.c_str());
1002 BtfChurnNoteDialResult(strBtfAddr, strOnionNorm, true);
1003 RememberBtfPeer(strBtfAddr, strMeeting, enc_pub, strDesc, strOnionNorm);
1004
1005 pnode = new CNode(hOnionSocket, addr, false);
1006 pnode->strBtfAddr = strBtfAddr;
1007 pnode->strBtfMeeting = string("onion ") + strOnionNorm;
1008 pnode->AddRef();
1009 CRITICAL_BLOCK(cs_vNodes)
1010 vNodes.push_back(pnode);
1011 return pnode;
1012 }
1013 if (fDebug)
1014 LogPrint("net", "ConnectNodeBtf: direct onion to %s at %s failed%s\n",
1015 strBtfAddr.c_str(), strOnionNorm.c_str(),
1016 fBtfOnionOnly ? "" : ", falling back to rendezvous");
1017 BtfChurnNoteDialResult(strBtfAddr, strOnionNorm, false);
1018 if (fBtfOnionOnly)
1019 return NULL;
1020 }
1021 }
1022
1023 if (fBtfOnionOnly)
1024 {
1025 if (fDebug)
1026 LogPrint("net", "ConnectNodeBtf: onion-only mode refused rendezvous fallback for %s\n",
1027 strBtfAddr.c_str());
881 return NULL; 1028 return NULL;
882 } 1029 }
883 string strHost = strMeeting.substr(0, colon); 1030
884 int nPort = atoi(strMeeting.substr(colon + 1).c_str()); 1031 BtfChurnNoteDialAttempt(strBtfAddr, strMeeting);
885 if (nPort <= 0 || nPort > 65535) 1032 string strHost;
1033 unsigned short nPort = 0;
1034 if (!SplitHostPort(strMeeting, strHost, nPort))
886 { 1035 {
887 BtfChurnNoteDialResult(strBtfAddr, strMeeting, false); 1036 BtfChurnNoteDialResult(strBtfAddr, strMeeting, false);
888 return NULL; 1037 return NULL;
@@ -902,7 +1051,7 @@ static CNode* ConnectNodeBtfTail(const string& strBtfAddr, const unsigned char p
902 BtfChurnNoteDialResult(strBtfAddr, strMeeting, true); 1051 BtfChurnNoteDialResult(strBtfAddr, strMeeting, true);
903 1052
904 // This one answered -- worth trying first next time we start. 1053 // This one answered -- worth trying first next time we start.
905 RememberBtfPeer(strBtfAddr, strMeeting, enc_pub); 1054 RememberBtfPeer(strBtfAddr, strMeeting, enc_pub, strDesc, strOnionNorm);
906 1055
907 // Add node 1056 // Add node
908 pnode = new CNode(hSocket, addr, false); 1057 pnode = new CNode(hSocket, addr, false);
@@ -932,9 +1081,10 @@ CNode* ConnectNodeBtf(const string& strBtfAddr)
932 LogPrint("net", "trying %s\n", strBtfAddr.c_str()); 1081 LogPrint("net", "trying %s\n", strBtfAddr.c_str());
933 1082
934 string strMeeting; 1083 string strMeeting;
1084 string strOnion;
935 unsigned char enc_pub[32]; 1085 unsigned char enc_pub[32];
936 BtfChurnNoteResolveAttempt(); 1086 BtfChurnNoteResolveAttempt();
937 if (!BtfResolve(strBtfAddr, strMeeting, enc_pub)) 1087 if (!BtfResolve(strBtfAddr, strMeeting, enc_pub, &strOnion))
938 { 1088 {
939 BtfChurnNoteResolveResult(false); 1089 BtfChurnNoteResolveResult(false);
940 if (fDebug) 1090 if (fDebug)
@@ -942,7 +1092,7 @@ CNode* ConnectNodeBtf(const string& strBtfAddr)
942 return NULL; 1092 return NULL;
943 } 1093 }
944 BtfChurnNoteResolveResult(true); 1094 BtfChurnNoteResolveResult(true);
945 return ConnectNodeBtfTail(strBtfAddr, pk, strMeeting, enc_pub); 1095 return ConnectNodeBtfTail(strBtfAddr, pk, strMeeting, strOnion, enc_pub);
946} 1096}
947 1097
948// Same as ConnectNodeBtf, but skips the Nostr resolve step because the caller 1098// Same as ConnectNodeBtf, but skips the Nostr resolve step because the caller
@@ -951,7 +1101,8 @@ CNode* ConnectNodeBtf(const string& strBtfAddr)
951// address). Makes zero relay connections -- only talks to the peer's 1101// address). Makes zero relay connections -- only talks to the peer's
952// rendezvous meeting node. 1102// rendezvous meeting node.
953CNode* ConnectNodeBtfResolved(const string& strBtfAddr, const string& strMeeting, 1103CNode* ConnectNodeBtfResolved(const string& strBtfAddr, const string& strMeeting,
954 const unsigned char enc_pub[32]) 1104 const string& strOnion, const unsigned char enc_pub[32],
1105 const string& strDesc)
955{ 1106{
956 unsigned char pk[32]; 1107 unsigned char pk[32];
957 if (!btf::ParseAddress(strBtfAddr, pk)) 1108 if (!btf::ParseAddress(strBtfAddr, pk))
@@ -961,7 +1112,7 @@ CNode* ConnectNodeBtfResolved(const string& strBtfAddr, const string& strMeeting
961 } 1112 }
962 if (BtfLocalAddress() == strBtfAddr) 1113 if (BtfLocalAddress() == strBtfAddr)
963 return NULL; // ourselves 1114 return NULL; // ourselves
964 return ConnectNodeBtfTail(strBtfAddr, pk, strMeeting, enc_pub); 1115 return ConnectNodeBtfTail(strBtfAddr, pk, strMeeting, strOnion, enc_pub, strDesc);
965} 1116}
966 1117
967// Anonymous inbound listener (this node's `.btf` hidden service). Registers at 1118// Anonymous inbound listener (this node's `.btf` hidden service). Registers at
@@ -1127,7 +1278,7 @@ void CNode::Disconnect()
1127 1278
1128 // Invalidate before anything else can look at it. This used to close the 1279 // Invalidate before anything else can look at it. This used to close the
1129 // socket and leave the handle number sitting in the object, so ~CNode 1280 // socket and leave the handle number sitting in the object, so ~CNode
1130 // closed it a second time and the select loop kept arming it in the 1281 // closed it a second time and the socket loop kept arming it in the
1131 // meantime. See the note on ~CNode in net.h. 1282 // meantime. See the note on ~CNode in net.h.
1132 BtfCloseSocket(hSocket); 1283 BtfCloseSocket(hSocket);
1133 hSocket = INVALID_SOCKET; 1284 hSocket = INVALID_SOCKET;
@@ -1276,52 +1427,49 @@ void ThreadSocketHandler2(void* parg)
1276 // 1427 //
1277 // Find which sockets have data to receive 1428 // Find which sockets have data to receive
1278 // 1429 //
1279 struct timeval timeout; 1430 static const int nPollTimeoutMs = 50; // frequency to poll pnode->vSend
1280 timeout.tv_sec = 0; 1431
1281 timeout.tv_usec = 50000; // frequency to poll pnode->vSend 1432 struct NodePollResult
1282 1433 {
1283 fd_set fdsetRecv; 1434 CNode* pnode;
1284 fd_set fdsetSend; 1435 SOCKET hSocket;
1285 FD_ZERO(&fdsetRecv); 1436 bool fRecv;
1286 FD_ZERO(&fdsetSend); 1437 bool fSend;
1287 SOCKET hSocketMax = 0; 1438 bool fInvalid;
1288 FD_SET(hListenSocket, &fdsetRecv); 1439 NodePollResult(CNode* pnodeIn, SOCKET hSocketIn)
1289 hSocketMax = max(hSocketMax, hListenSocket); 1440 : pnode(pnodeIn), hSocket(hSocketIn), fRecv(false), fSend(false), fInvalid(false) {}
1290 // FD_SET drops silently once the set is full, so count what goes in 1441 };
1291 // and say so. A peer that never makes it into the set is never read, 1442
1292 // and that is indistinguishable from a peer with nothing to say -- 1443 vector<BtfPollFd> vPoll;
1293 // the failure this whole path used to hide. 1444 vector<NodePollResult> vPollNodes;
1294 unsigned int nWatched = 1; // the listen socket 1445
1446 BtfPollFd listenPoll;
1447 memset(&listenPoll, 0, sizeof(listenPoll));
1448 listenPoll.fd = hListenSocket;
1449 listenPoll.events = POLLIN;
1450 vPoll.push_back(listenPoll);
1451
1295 CRITICAL_BLOCK(cs_vNodes) 1452 CRITICAL_BLOCK(cs_vNodes)
1296 { 1453 {
1297 foreach(CNode* pnode, vNodes) 1454 foreach(CNode* pnode, vNodes)
1298 { 1455 {
1299 if (nWatched >= FD_SETSIZE)
1300 break;
1301 // A disconnected node stays in vNodes until its references go. 1456 // A disconnected node stays in vNodes until its references go.
1302 // Arming its closed handle asks select() to watch a number the 1457 // Do not watch a closed handle number: the OS may already have
1303 // operating system may already have given to somebody else. 1458 // handed it to a different connection.
1304 if (pnode->hSocket == INVALID_SOCKET) 1459 if (pnode->hSocket == INVALID_SOCKET)
1305 continue; 1460 continue;
1306 FD_SET(pnode->hSocket, &fdsetRecv); 1461
1307 nWatched++; 1462 BtfPollFd pfd;
1308 hSocketMax = max(hSocketMax, pnode->hSocket); 1463 memset(&pfd, 0, sizeof(pfd));
1464 pfd.fd = pnode->hSocket;
1465 pfd.events = POLLIN;
1309 TRY_CRITICAL_BLOCK(pnode->cs_vSend) 1466 TRY_CRITICAL_BLOCK(pnode->cs_vSend)
1310 if (!pnode->vSend.empty()) 1467 if (!pnode->vSend.empty())
1311 FD_SET(pnode->hSocket, &fdsetSend); 1468 pfd.events |= POLLOUT;
1469 vPoll.push_back(pfd);
1470 vPollNodes.push_back(NodePollResult(pnode, pnode->hSocket));
1312 } 1471 }
1313 if (vNodes.size() + 1 > (size_t)FD_SETSIZE) 1472 nPeersWatched = (int)vPollNodes.size();
1314 {
1315 static int64 nLastWarned = 0;
1316 if (GetTime() - nLastWarned > 60)
1317 {
1318 nLastWarned = GetTime();
1319 printf("WARNING: %d peers but select() can only watch %d -- %d are not being read\n",
1320 (int)vNodes.size(), (int)FD_SETSIZE - 1,
1321 (int)vNodes.size() - (int)FD_SETSIZE + 1);
1322 }
1323 }
1324 nPeersWatched = (int)nWatched - 1; // less the listen socket
1325 } 1473 }
1326 1474
1327 // Say the state of the node out loud now and then, so a log pulled off 1475 // Say the state of the node out loud now and then, so a log pulled off
@@ -1342,32 +1490,17 @@ void ThreadSocketHandler2(void* parg)
1342 } 1490 }
1343 1491
1344 vfThreadRunning[0] = false; 1492 vfThreadRunning[0] = false;
1345 int nSelect = select(hSocketMax + 1, &fdsetRecv, &fdsetSend, NULL, &timeout); 1493 int nPoll = BtfPoll(&vPoll[0], (unsigned int)vPoll.size(), nPollTimeoutMs);
1346 vfThreadRunning[0] = true; 1494 vfThreadRunning[0] = true;
1347 CheckForShutdown(0); 1495 CheckForShutdown(0);
1348 if (nSelect == SOCKET_ERROR) 1496 if (nPoll == SOCKET_ERROR)
1349 { 1497 {
1350 // This used to answer a failed select() by marking every
1351 // descriptor from 0 to hSocketMax ready, which is code written for
1352 // POSIX, where a descriptor is a small integer. A Windows SOCKET is
1353 // a kernel handle in the thousands, so the loop iterated over
1354 // numbers that are not sockets and told the rest of the function
1355 // that every peer had data. One bad handle in the set makes
1356 // select() fail on every pass, so a single closed socket put the
1357 // whole thread into that state permanently.
1358 //
1359 // Find the bad handle instead and let it be dropped.
1360 int nErr = WSAGetLastError(); 1498 int nErr = WSAGetLastError();
1361 LogPrint("net", "select failed: %d\n", nErr); 1499 LogPrint("net", "poll failed: %d\n", nErr);
1362 FD_ZERO(&fdsetRecv);
1363 FD_ZERO(&fdsetSend);
1364 CRITICAL_BLOCK(cs_vNodes) 1500 CRITICAL_BLOCK(cs_vNodes)
1365 { 1501 {
1366 foreach(CNode* pnode, vNodes) 1502 foreach(CNode* pnode, vNodes)
1367 { 1503 {
1368 // Already disconnected: its handle is gone on purpose, and
1369 // reporting it here would name the wrong node for a select()
1370 // failure it did not cause.
1371 if (pnode->hSocket == INVALID_SOCKET) 1504 if (pnode->hSocket == INVALID_SOCKET)
1372 continue; 1505 continue;
1373 int nType = 0; 1506 int nType = 0;
@@ -1376,24 +1509,27 @@ void ThreadSocketHandler2(void* parg)
1376#else 1509#else
1377 socklen_t nTypeLen = sizeof(nType); 1510 socklen_t nTypeLen = sizeof(nType);
1378#endif 1511#endif
1379 if (pnode->hSocket != INVALID_SOCKET && 1512 if (getsockopt(pnode->hSocket, SOL_SOCKET, SO_TYPE, (char*)&nType, &nTypeLen) != 0)
1380 getsockopt(pnode->hSocket, SOL_SOCKET, SO_TYPE, (char*)&nType, &nTypeLen) != 0)
1381 { 1513 {
1382 printf("dropping node %s: its socket is no longer valid\n", 1514 printf("dropping node %s: its socket is no longer valid\n",
1383 pnode->addr.ToString().c_str()); 1515 pnode->addr.ToString().c_str());
1384 pnode->fDisconnect = true; 1516 pnode->fDisconnect = true;
1385 1517
1386 // Stop referring to it, or it goes back into the next 1518 // Stop referring to it, or it goes into the next poll
1387 // select() set and fails it again. 1519 // set and fails it again.
1388 // 1520 //
1389 // Marking fDisconnect is not enough: a node is only 1521 // Marking fDisconnect is not enough: a node is only
1390 // removed from vNodes once it has drained or once 1522 // removed from vNodes once it has drained or once
1391 // DISCONNECT_DRAIN_SECS has passed, so one dead 1523 // DISCONNECT_DRAIN_SECS has passed, so one dead
1392 // descriptor kept failing select() for a full minute -- 1524 // descriptor kept failing the wait for a full minute --
1393 // and a failed select() zeroes both sets above, which 1525 // and a failed wait skips the service loop below
1394 // means NO peer is read on that pass. Measured on a 1526 // entirely, which means NO peer is read on that pass.
1395 // wedged pool: 1,027 consecutive passes over 61 1527 // Measured on a wedged pool under the old select()
1396 // seconds, one bad socket, the entire node deaf. 1528 // implementation: 1,027 consecutive passes over 61
1529 // seconds, one bad socket, the entire node deaf. poll()
1530 // reports a bad descriptor as POLLNVAL on that entry
1531 // rather than failing the whole call, so this path is
1532 // now the backstop rather than the common case.
1397 // 1533 //
1398 // Not closed, only forgotten. getsockopt just told us 1534 // Not closed, only forgotten. getsockopt just told us
1399 // this descriptor is not ours any more; closing it now 1535 // this descriptor is not ours any more; closing it now
@@ -1402,7 +1538,17 @@ void ThreadSocketHandler2(void* parg)
1402 } 1538 }
1403 } 1539 }
1404 } 1540 }
1405 Sleep(timeout.tv_usec/1000); 1541 Sleep(nPollTimeoutMs);
1542 }
1543 else
1544 {
1545 for (size_t i = 0; i < vPollNodes.size(); i++)
1546 {
1547 short revents = vPoll[i + 1].revents; // poll[0] is the listener
1548 vPollNodes[i].fRecv = (revents & (POLLIN | POLLERR | POLLHUP)) != 0;
1549 vPollNodes[i].fSend = (revents & POLLOUT) != 0;
1550 vPollNodes[i].fInvalid = (revents & POLLNVAL) != 0;
1551 }
1406 } 1552 }
1407 RandAddSeed(); 1553 RandAddSeed();
1408 1554
@@ -1418,7 +1564,7 @@ void ThreadSocketHandler2(void* parg)
1418 // 1564 //
1419 // Accept new connections 1565 // Accept new connections
1420 // 1566 //
1421 if (FD_ISSET(hListenSocket, &fdsetRecv)) 1567 if (nPoll != SOCKET_ERROR && (vPoll[0].revents & POLLIN))
1422 { 1568 {
1423 struct sockaddr_in sockaddr; 1569 struct sockaddr_in sockaddr;
1424#ifdef _WIN32 1570#ifdef _WIN32
@@ -1437,18 +1583,47 @@ void ThreadSocketHandler2(void* parg)
1437 { 1583 {
1438 // Refuse rather than accept a connection this node cannot 1584 // Refuse rather than accept a connection this node cannot
1439 // watch. Without this the listen socket kept taking peers -- 1585 // watch. Without this the listen socket kept taking peers --
1440 // it is always in the select set -- while everything past the 1586 // it is always watched -- while everything past the
1441 // limit sat open and unread, which fed on itself: the deafer 1587 // limit sat open and unread, which fed on itself: the deafer
1442 // the node got, the more connections it collected. 1588 // the node got, the more connections it collected.
1443 unsigned int nNodes = 0; 1589 unsigned int nNodes = 0;
1444 CRITICAL_BLOCK(cs_vNodes) 1590 CRITICAL_BLOCK(cs_vNodes)
1445 nNodes = (unsigned int)vNodes.size(); 1591 nNodes = (unsigned int)vNodes.size();
1592 // Also count how many connections this one address already
1593 // holds. A single IP that keeps opening sockets -- a broken
1594 // node or a deliberate connection-exhaustion flood -- must not
1595 // be able to take the whole table, which is what starved the
1596 // relays.
1597 //
1598 // Loopback is exempt. Onion peers reach a managed-Tor node
1599 // through its local hidden-service listener, so every one of
1600 // them accepts as 127.0.0.1; capping loopback would cap all
1601 // inbound onion peers as a single group, which is the opposite
1602 // of where the network is headed. The global MAX_CONNECTIONS
1603 // still bounds the total, and the direct-IP flood this defends
1604 // against never arrives over loopback. (Per-onion-identity
1605 // limiting, once a peer's identity is known past accept(), is
1606 // the right tool for onion and is left for later.)
1607 bool fLoopback = (addr.GetByte(3) == 127);
1608 unsigned int nFromThisIP = 0;
1609 if (!fLoopback)
1610 CRITICAL_BLOCK(cs_vNodes)
1611 for (vector<CNode*>::iterator it = vNodes.begin(); it != vNodes.end(); ++it)
1612 if ((*it)->addr.ip == addr.ip)
1613 nFromThisIP++;
1614
1446 if (nNodes >= MAX_CONNECTIONS) 1615 if (nNodes >= MAX_CONNECTIONS)
1447 { 1616 {
1448 LogPrint("net", "refusing connection from %s, already at %u\n", 1617 LogPrint("net", "refusing connection from %s, already at %u\n",
1449 addr.ToString().c_str(), nNodes); 1618 addr.ToString().c_str(), nNodes);
1450 BtfCloseSocket(hSocket); 1619 BtfCloseSocket(hSocket);
1451 } 1620 }
1621 else if (!fLoopback && nFromThisIP >= MAX_CONNECTIONS_PER_IP)
1622 {
1623 LogPrint("net", "refusing connection from %s, already %u from that address\n",
1624 addr.ToString().c_str(), nFromThisIP);
1625 BtfCloseSocket(hSocket);
1626 }
1452 else 1627 else
1453 { 1628 {
1454 LogPrint("net", "accepted connection from %s\n", addr.ToString().c_str()); 1629 LogPrint("net", "accepted connection from %s\n", addr.ToString().c_str());
@@ -1464,22 +1639,28 @@ void ThreadSocketHandler2(void* parg)
1464 // 1639 //
1465 // Service each socket 1640 // Service each socket
1466 // 1641 //
1467 vector<CNode*> vNodesCopy; 1642 for (size_t i = 0; i < vPollNodes.size(); i++)
1468 CRITICAL_BLOCK(cs_vNodes)
1469 vNodesCopy = vNodes;
1470 foreach(CNode* pnode, vNodesCopy)
1471 { 1643 {
1472 CheckForShutdown(0); 1644 CheckForShutdown(0);
1473 SOCKET hSocket = pnode->hSocket; 1645 NodePollResult& poll = vPollNodes[i];
1474 // Forgotten above because it was already invalid. FD_ISSET on it 1646 CNode* pnode = poll.pnode;
1475 // is meaningless and recv()/send() would be operating on -1. 1647 SOCKET hSocket = poll.hSocket;
1476 if (hSocket == INVALID_SOCKET) 1648
1649 if (hSocket == INVALID_SOCKET || pnode->hSocket != hSocket)
1477 continue; 1650 continue;
1478 1651
1652 if (poll.fInvalid)
1653 {
1654 LogPrint("net", "poll reported invalid socket for %s\n",
1655 pnode->addr.ToString().c_str());
1656 pnode->fDisconnect = true;
1657 continue;
1658 }
1659
1479 // 1660 //
1480 // Receive 1661 // Receive
1481 // 1662 //
1482 if (FD_ISSET(hSocket, &fdsetRecv)) 1663 if (poll.fRecv)
1483 { 1664 {
1484 TRY_CRITICAL_BLOCK(pnode->cs_vRecv) 1665 TRY_CRITICAL_BLOCK(pnode->cs_vRecv)
1485 { 1666 {
@@ -1519,7 +1700,7 @@ void ThreadSocketHandler2(void* parg)
1519 // 1700 //
1520 // Send 1701 // Send
1521 // 1702 //
1522 if (FD_ISSET(hSocket, &fdsetSend)) 1703 if (poll.fSend)
1523 { 1704 {
1524 TRY_CRITICAL_BLOCK(pnode->cs_vSend) 1705 TRY_CRITICAL_BLOCK(pnode->cs_vSend)
1525 { 1706 {
@@ -1915,6 +2096,14 @@ bool StartNode(string& strError)
1915 // IP address, and port for the socket that is being bound 2096 // IP address, and port for the socket that is being bound
1916 int nRetryLimit = 15; 2097 int nRetryLimit = 15;
1917 struct sockaddr_in sockaddr = addrLocalHost.GetSockAddr(); 2098 struct sockaddr_in sockaddr = addrLocalHost.GetSockAddr();
2099 // Managed Tor forwards hidden-service traffic to 127.0.0.1:<port>, which a
2100 // LAN-only bind cannot receive -- so a node that advertises an onion
2101 // endpoint must listen on all local interfaces. Upstream 1.2.19 bound
2102 // INADDR_ANY unconditionally, which also widened the P2P listener of every
2103 // non-Tor node to VPN tunnels, docker bridges and secondary NICs. Keep the
2104 // wildcard bind conditional on actually having an onion endpoint.
2105 if (!BtfLocalOnionEndpoint().empty())
2106 sockaddr.sin_addr.s_addr = INADDR_ANY;
1918 if (bind(hListenSocket, (struct sockaddr*)&sockaddr, sizeof(sockaddr)) == SOCKET_ERROR) 2107 if (bind(hListenSocket, (struct sockaddr*)&sockaddr, sizeof(sockaddr)) == SOCKET_ERROR)
1919 { 2108 {
1920 int nErr = WSAGetLastError(); 2109 int nErr = WSAGetLastError();
@@ -1959,7 +2148,7 @@ bool StartNode(string& strError)
1959 if (addrIncoming.ip) 2148 if (addrIncoming.ip)
1960 addrLocalHost.ip = addrIncoming.ip; 2149 addrLocalHost.ip = addrIncoming.ip;
1961 2150
1962 _beginthread([](void*) { 2151 if (_beginthread([](void*) {
1963 unsigned int ip = addrLocalHost.ip; 2152 unsigned int ip = addrLocalHost.ip;
1964 if (GetMyExternalIP(ip)) { 2153 if (GetMyExternalIP(ip)) {
1965 addrLocalHost.ip = ip; 2154 addrLocalHost.ip = ip;
@@ -1967,7 +2156,8 @@ bool StartNode(string& strError)
1967 CWalletDB().WriteSetting("addrIncoming", addrIncoming); 2156 CWalletDB().WriteSetting("addrIncoming", addrIncoming);
1968 printf("External IP updated: %s\n", addrLocalHost.ToStringIP().c_str()); 2157 printf("External IP updated: %s\n", addrLocalHost.ToStringIP().c_str());
1969 } 2158 }
1970 }, 0, NULL); 2159 }, 0, NULL) == (uintptr_t)-1)
2160 printf("Error: _beginthread(GetMyExternalIP) failed\n");
1971 2161
1972 // Peers that answered last time, dialled straight away. The relays are the 2162 // Peers that answered last time, dialled straight away. The relays are the
1973 // only way to find anyone otherwise, and reaching them plus walking their 2163 // only way to find anyone otherwise, and reaching them plus walking their
diff --git a/src/net.h b/src/net.h
index 8b0245a..0988542 100644
--- a/src/net.h
+++ b/src/net.h
@@ -12,7 +12,13 @@ class CBlockIndex;
12 12
13 13
14 14
15static const unsigned short DEFAULT_PORT = htons(8433); 15// Held in host byte order, unlike the DEFAULT_PORT constant these replace,
16// which was htons(8433). Nothing is lost: net.cpp's GetDefaultPort() applies
17// htons() itself and is what every caller now uses, so the value that reaches
18// a sockaddr is byte-for-byte what it was. Mainnet stays on 8433.
19static const unsigned short MAINNET_PORT = 8433;
20static const unsigned short TESTNET_PORT = 18433;
21static const unsigned int PUBLISH_HOPS = 5;
16enum 22enum
17{ 23{
18 NODE_NETWORK = (1 << 0), 24 NODE_NETWORK = (1 << 0),
@@ -23,13 +29,24 @@ enum
23 29
24 30
25 31
32// Defined in main.cpp. True while this node is tracking the tip rather than
33// still catching up, measured off the age of our own best block -- see there
34// for why that and not a peer-reported height. Declared here because net.h is
35// parsed before main.h and CNode::AskFor needs it.
36bool IsBlockFetchNearTip();
37
26bool GetMyExternalIP(unsigned int& ipRet); 38bool GetMyExternalIP(unsigned int& ipRet);
39bool IsTestNet();
40unsigned short GetDefaultPort();
41void SelectNetworkParams(bool fTestNetIn);
27CNode* ConnectNodeBtf(const string& strBtfAddr); 42CNode* ConnectNodeBtf(const string& strBtfAddr);
28CNode* ConnectNodeBtfResolved(const string& strBtfAddr, const string& strMeeting, 43CNode* ConnectNodeBtfResolved(const string& strBtfAddr, const string& strMeeting,
29 const unsigned char enc_pub[32]); 44 const string& strOnion, const unsigned char enc_pub[32],
45 const string& strDesc=string());
30void ThreadBtfAccept(void* parg); 46void ThreadBtfAccept(void* parg);
31void ThreadBtfConnect(void* parg); 47void ThreadBtfConnect(void* parg);
32extern string strBtfConnect; 48extern string strBtfConnect;
49extern bool fBtfOnionOnly;
33// Bootstrap seeds added at runtime with /btfseed=ADDRESS:ENCHEX (repeatable). 50// Bootstrap seeds added at runtime with /btfseed=ADDRESS:ENCHEX (repeatable).
34// The compiled-in list lives in net.cpp; these are appended to it. 51// The compiled-in list lives in net.cpp; these are appended to it.
35extern std::vector<std::pair<std::string, std::string> > vBtfExtraSeeds; 52extern std::vector<std::pair<std::string, std::string> > vBtfExtraSeeds;
@@ -87,30 +104,52 @@ static const int BTF_RECV_TIMEOUT_SECS = 30 * 60;
87static const int BTF_SEND_STALL_SECS = 10 * 60; 104static const int BTF_SEND_STALL_SECS = 10 * 60;
88static const int BTF_HANDSHAKE_GRACE_SECS = 60; 105static const int BTF_HANDSHAKE_GRACE_SECS = 60;
89 106
107// How long a requested block gets before it is asked for again, in seconds.
108// Used twice: CNode::AskFor spaces the queued per-announcer block retries this
109// far apart instead of the two minutes transactions get, and the failover
110// sweep in main.cpp re-issues a getdata to a different peer once a sent
111// request has gone unanswered this long. A block body is at most 1 MB and
112// somebody demonstrably has it -- they announced it -- so on this timescale
113// "not here yet" means "not coming from that peer", not "still in transit".
114static const int BLOCK_FETCH_RETRY_SECS = 15;
115
90// A serialized object cannot be larger than MAX_SIZE, so no valid network 116// A serialized object cannot be larger than MAX_SIZE, so no valid network
91// message needs the old 256 MB header allowance. Keep the wire cap aligned with 117// message needs the old 256 MB header allowance. Keep the wire cap aligned with
92// what the deserializer can accept and bound how long one peer may sit on a 118// what the deserializer can accept. How long a peer may sit on a partial
93// partial payload after a complete header has arrived. 119// payload after a complete header has arrived is BTF_MSG_WAIT_MAX_SECS, below.
94static const unsigned int MAX_PROTOCOL_MESSAGE_SIZE = MAX_SIZE; 120static const unsigned int MAX_PROTOCOL_MESSAGE_SIZE = MAX_SIZE;
95static const int BTF_INCOMPLETE_MESSAGE_TIMEOUT_SECS = 2 * 60;
96 121
97// Ceiling on simultaneous connections. 122// Ceiling on simultaneous connections.
98// 123//
99// ThreadSocketHandler watches every peer through one select(), and select() 124// ThreadSocketHandler watches every peer in one socket loop. This used to be a
100// cannot watch more descriptors than FD_SETSIZE. On Windows FD_SET simply 125// select() loop, which made FD_SETSIZE the real limit and silently starved
101// stops adding once the set is full -- no error, no log -- so every socket 126// sockets past that bitmap. It is now poll()/WSAPoll(), so the ceiling below is
102// past the limit stays open and is never read again. Nothing capped vNodes, 127// a deliberate resource bound rather than a descriptor-set accident.
103// so a node accepted connections it had no way to service, and because new
104// entries go on the end of vNodes it was always the newest connection that
105// starved, including the ones dialed to fix connectivity.
106// 128//
107// Measured on two nodes before this ceiling existed: 277 and 90 CNode objects 129// Measured on two nodes before this ceiling existed: 277 and 90 CNode objects
108// against 93 and 90 live sockets, and a status bar reporting hundreds of 130// against 93 and 90 live sockets, and a status bar reporting hundreds of
109// peers for a node that was really talking to thirty. 131// peers for a node that was really talking to thirty.
110// 132//
111// 125 is Bitcoin's number and sits far below FD_SETSIZE on both platforms. 133// 125 is Bitcoin's number and remains a conservative public-node default.
112static const unsigned int MAX_CONNECTIONS = 125; 134static const unsigned int MAX_CONNECTIONS = 125;
113 135
136// Cap inbound connections from any single address. Without it one source can
137// hold every slot up to MAX_CONNECTIONS -- exactly what happened to the
138// rendezvous relays, where a single IP parked hundreds of connections and
139// starved real peers. A legitimate peer, even a relay talking to a relay,
140// needs only a handful; 32 leaves generous room while denying any one address
141// more than a quarter of the table. The firewall connlimit on the relays does
142// the same thing a layer lower; this makes every build carry the defense.
143//
144// Loopback (127.0.0.0/8) is exempt in the accept path. A managed-Tor node
145// receives every inbound onion peer over its local hidden-service listener, so
146// they all appear as 127.0.0.1 -- capping loopback would cap onion peers as a
147// group, which is exactly wrong for a network moving toward onion-only. The
148// global MAX_CONNECTIONS still bounds the total; a per-onion-identity limit,
149// applied once identity is known after accept(), is the right future tool for
150// onion floods.
151static const unsigned int MAX_CONNECTIONS_PER_IP = 32;
152
114// A node marked for disconnect is normally held until its buffers drain, so 153// A node marked for disconnect is normally held until its buffers drain, so
115// a last message still goes out. But vSend cannot drain through a socket 154// a last message still goes out. But vSend cannot drain through a socket
116// whose far end is gone, and that made such a node immortal: it stayed in 155// whose far end is gone, and that made such a node immortal: it stayed in
@@ -176,7 +215,7 @@ static const int BTF_BEHIND_WARN_INTERVAL_SECS = 5 * 60;
176// 215//
177// These counters exist so it can. Cheap to keep, and they name the failures 216// These counters exist so it can. Cheap to keep, and they name the failures
178// that actually happened rather than the ones that sound impressive. 217// that actually happened rather than the ones that sound impressive.
179extern int nPeersWatched; // peers in the last select() set 218extern int nPeersWatched; // peers in the last poll() set
180extern int64 nBlocksReceived; // blocks handed to ProcessBlock 219extern int64 nBlocksReceived; // blocks handed to ProcessBlock
181extern int64 nBlocksWithoutParent; // ...of those, how many arrived orphaned 220extern int64 nBlocksWithoutParent; // ...of those, how many arrived orphaned
182 221
@@ -236,15 +275,17 @@ void CheckForShutdown(int n);
236// The message start string is designed to be unlikely to occur in normal data. 275// The message start string is designed to be unlikely to occur in normal data.
237// The characters are rarely used upper ascii, not valid as UTF-8, and produce 276// The characters are rarely used upper ascii, not valid as UTF-8, and produce
238// a large 4-byte int at any alignment. 277// a large 4-byte int at any alignment.
239// Magic bytes unique to the Bitflash network (0xbf = "BF"). Second byte bumped 278// Magic bytes unique to the selected Bitflash network. Mainnet uses 0xbf =
240// to 0x20 for the 2026 stable relaunch so the old test chain can't interfere. 279// "BF"; testnet swaps the whole sequence so a test node cannot handshake with
241static const char pchMessageStart[4] = { 0xbf, 0x20, 0x5c, 0xfd }; 280// mainnet before consensus-level testnet activation exists.
281extern char pchMessageStart[4];
242 282
243class CMessageHeader 283class CMessageHeader
244{ 284{
245public: 285public:
246 enum { COMMAND_SIZE=12 }; 286 enum { COMMAND_SIZE=12 };
247 char pchMessageStart[sizeof(::pchMessageStart)]; 287 enum { MESSAGE_START_SIZE=4 };
288 char pchMessageStart[MESSAGE_START_SIZE];
248 char pchCommand[COMMAND_SIZE]; 289 char pchCommand[COMMAND_SIZE];
249 unsigned int nMessageSize; 290 unsigned int nMessageSize;
250 291
@@ -335,7 +376,7 @@ public:
335 nServices = 0; 376 nServices = 0;
336 memcpy(pchReserved, pchIPv4, sizeof(pchReserved)); 377 memcpy(pchReserved, pchIPv4, sizeof(pchReserved));
337 ip = 0; 378 ip = 0;
338 port = DEFAULT_PORT; 379 port = GetDefaultPort();
339 nTime = GetAdjustedTime(); 380 nTime = GetAdjustedTime();
340 nLastFailed = 0; 381 nLastFailed = 0;
341 } 382 }
@@ -365,7 +406,7 @@ public:
365 nServices = nServicesIn; 406 nServices = nServicesIn;
366 memcpy(pchReserved, pchIPv4, sizeof(pchReserved)); 407 memcpy(pchReserved, pchIPv4, sizeof(pchReserved));
367 ip = 0; 408 ip = 0;
368 port = DEFAULT_PORT; 409 port = GetDefaultPort();
369 nTime = GetAdjustedTime(); 410 nTime = GetAdjustedTime();
370 nLastFailed = 0; 411 nLastFailed = 0;
371 412
@@ -656,9 +697,6 @@ public:
656 int64 nLastSend; 697 int64 nLastSend;
657 int64 nLastRecv; 698 int64 nLastRecv;
658 int64 nLastSendEmpty; 699 int64 nLastSendEmpty;
659 int64 nIncompleteMessageStart;
660 unsigned int nIncompleteMessageSize;
661 string strIncompleteMessageCommand;
662 string strBtfAddr; 700 string strBtfAddr;
663 string strBtfMeeting; 701 string strBtfMeeting;
664 702
@@ -699,11 +737,13 @@ public:
699 nLastSend = 0; 737 nLastSend = 0;
700 nLastRecv = 0; 738 nLastRecv = 0;
701 nLastSendEmpty = GetTime(); 739 nLastSendEmpty = GetTime();
702 nIncompleteMessageStart = 0;
703 nIncompleteMessageSize = 0;
704 strIncompleteMessageCommand.clear();
705 strBtfAddr.clear(); 740 strBtfAddr.clear();
706 strBtfMeeting.clear(); 741 strBtfMeeting.clear();
742 // Not waiting on anything yet. Left indeterminate, the first partial
743 // message this peer sends is measured against whatever was in this
744 // memory, and ProcessMessages drops it as unreadable on first contact.
745 nMsgWaitSince = 0;
746 nMsgWaitSize = 0;
707 nStartingHeight = -1; 747 nStartingHeight = -1;
708 748
709 // Push a version message 749 // Push a version message
@@ -798,13 +838,51 @@ public:
798 static int64 nLastTime; 838 static int64 nLastTime;
799 nLastTime = nNow = max(nNow, ++nLastTime); 839 nLastTime = nNow = max(nNow, ++nLastTime);
800 840
801 // Each retry is 2 minutes after the last 841 // Each retry is 2 minutes after the last -- and because nRequestTime
802 nRequestTime = max(nRequestTime + 2 * 60 * 1000000, nNow); 842 // is shared across peers, the same spacing is what strings requests
843 // for one inv across its announcers. For transactions that is pacing;
844 // for a new block it was the stall of issue #33: nine peers announce
845 // the block, the first one asked does not serve it, and the ninth
846 // candidate is not even asked until sixteen minutes in, with the pool
847 // mining on the stale tip the whole time.
848 //
849 // Blocks retry on the fetch timescale, but only while we are at the
850 // tip. That is where the stall costs something -- the pool keeps
851 // handing out jobs against a dead tip -- and it is also where the
852 // spacing is nearly free: one block is in flight, and its body has
853 // almost always arrived before the second announcer's turn comes
854 // round, so AlreadyHave drops the duplicate before a getdata is sent.
855 //
856 // While catching up neither half holds. A peer working through a
857 // 500-deep queue can be minutes away from block N without anything
858 // being wrong, so at 15s the second announcer is asked at +15s, the
859 // third at +30s, and the bulk download arrives once per announcer --
860 // up to eight times over. Nothing caps that: the eight-entry cap is on
861 // mapBlockFetch, which is a different mechanism from this spacing.
862 // Two minutes is the right number there, and it is what this line did
863 // before. Failover at the tip does not depend on this being 15s in any
864 // case: mapBlockFetch re-issues a stalled getdata to the next peer on
865 // its own timer, which is what actually fixes #33.
866 bool fFast = (inv.type == MSG_BLOCK && IsBlockFetchNearTip());
867 int64 nRetryUsecs = (fFast ? BLOCK_FETCH_RETRY_SECS : 2 * 60) * 1000000LL;
868 nRequestTime = max(nRequestTime + nRetryUsecs, nNow);
803 mapAskFor.insert(make_pair(nRequestTime, inv)); 869 mapAskFor.insert(make_pair(nRequestTime, inv));
804 } 870 }
805 871
806 872
807 873
874 // "sending: tx (N bytes)" alone was ~71% of a 13-hour production
875 // debug.log (issue #35), so tx sends are demoted to the /debugrelay-gated
876 // "relay" category; every other command's send trace stays on "net".
877 // The line is emitted in pieces across BeginMessage/EndMessage/
878 // AbortMessage, so all three must agree on the category or orphaned
879 // "(N bytes)" fragments would leak into a plain -debug log.
880 // Valid only mid-message (nPushPos != -1).
881 const char* SendLogCategory() const
882 {
883 return strcmp(GetMessageCommand(), "tx") == 0 ? "relay" : "net";
884 }
885
808 void BeginMessage(const char* pszCommand) 886 void BeginMessage(const char* pszCommand)
809 { 887 {
810 EnterCriticalSection(&cs_vSend); 888 EnterCriticalSection(&cs_vSend);
@@ -812,17 +890,19 @@ public:
812 AbortMessage(); 890 AbortMessage();
813 nPushPos = vSend.size(); 891 nPushPos = vSend.size();
814 vSend << CMessageHeader(pszCommand, 0); 892 vSend << CMessageHeader(pszCommand, 0);
815 if (LogAcceptsCategory("net")) printf("sending: %-12s ", pszCommand); 893 if (LogAcceptsCategory(SendLogCategory())) printf("sending: %-12s ", pszCommand);
816 } 894 }
817 895
818 void AbortMessage() 896 void AbortMessage()
819 { 897 {
820 if (nPushPos == -1) 898 if (nPushPos == -1)
821 return; 899 return;
900 // Capture before nPushPos is cleared; GetMessageCommand() needs it.
901 const char* pszCategory = SendLogCategory();
822 vSend.resize(nPushPos); 902 vSend.resize(nPushPos);
823 nPushPos = -1; 903 nPushPos = -1;
824 LeaveCriticalSection(&cs_vSend); 904 LeaveCriticalSection(&cs_vSend);
825 if (LogAcceptsCategory("net")) printf("(aborted)\n"); 905 if (LogAcceptsCategory(pszCategory)) printf("(aborted)\n");
826 } 906 }
827 907
828 void EndMessage() 908 void EndMessage()
@@ -830,7 +910,11 @@ public:
830 extern int nDropMessagesTest; 910 extern int nDropMessagesTest;
831 if (nDropMessagesTest > 0 && GetRand(nDropMessagesTest) == 0) 911 if (nDropMessagesTest > 0 && GetRand(nDropMessagesTest) == 0)
832 { 912 {
833 if (LogAcceptsCategory("net")) printf("dropmessages DROPPING SEND MESSAGE\n"); 913 // Same category as the "sending: " this line interrupts, so a tx
914 // does not announce its own drop under "net" having been gated
915 // off under "relay". nPushPos can be -1 here, and then
916 // GetMessageCommand() returns "" and this is "net" as before.
917 if (LogAcceptsCategory(SendLogCategory())) printf("dropmessages DROPPING SEND MESSAGE\n");
834 AbortMessage(); 918 AbortMessage();
835 return; 919 return;
836 } 920 }
@@ -842,10 +926,10 @@ public:
842 unsigned int nSize = vSend.size() - nPushPos - sizeof(CMessageHeader); 926 unsigned int nSize = vSend.size() - nPushPos - sizeof(CMessageHeader);
843 memcpy((char*)&vSend[nPushPos] + offsetof(CMessageHeader, nMessageSize), &nSize, sizeof(nSize)); 927 memcpy((char*)&vSend[nPushPos] + offsetof(CMessageHeader, nMessageSize), &nSize, sizeof(nSize));
844 928
845 if (LogAcceptsCategory("net")) printf("(%d bytes) ", nSize); 929 if (LogAcceptsCategory(SendLogCategory())) printf("(%d bytes) ", nSize);
846 //for (int i = nPushPos+sizeof(CMessageHeader); i < min(vSend.size(), nPushPos+sizeof(CMessageHeader)+20U); i++) 930 //for (int i = nPushPos+sizeof(CMessageHeader); i < min(vSend.size(), nPushPos+sizeof(CMessageHeader)+20U); i++)
847 // printf("%02x ", vSend[i] & 0xff); 931 // printf("%02x ", vSend[i] & 0xff);
848 if (LogAcceptsCategory("net")) printf("\n"); 932 if (LogAcceptsCategory(SendLogCategory())) printf("\n");
849 933
850 nPushPos = -1; 934 nPushPos = -1;
851 LeaveCriticalSection(&cs_vSend); 935 LeaveCriticalSection(&cs_vSend);
diff --git a/src/nostr.cpp b/src/nostr.cpp
index 3f43302..f44dc08 100644
--- a/src/nostr.cpp
+++ b/src/nostr.cpp
@@ -7,6 +7,10 @@
7#include <openssl/ssl.h> 7#include <openssl/ssl.h>
8#include <openssl/err.h> 8#include <openssl/err.h>
9#include <openssl/x509.h> 9#include <openssl/x509.h>
10#include <openssl/x509v3.h>
11#ifdef _WIN32
12#include <wincrypt.h>
13#endif
10// We link libsecp256k1 statically (without this the header assumes DLL import). 14// We link libsecp256k1 statically (without this the header assumes DLL import).
11// Both makefiles also pass -DSECP256K1_STATIC; the guard keeps that from 15// Both makefiles also pass -DSECP256K1_STATIC; the guard keeps that from
12// warning while leaving this file correct if built without the flag. 16// warning while leaving this file correct if built without the flag.
@@ -26,6 +30,7 @@
26 30
27#include "btfaddr.h" 31#include "btfaddr.h"
28#include "btfchan.h" 32#include "btfchan.h"
33#include "proxy.h"
29 34
30using json = nlohmann::json; 35using json = nlohmann::json;
31 36
@@ -127,6 +132,8 @@ vector<string> vBtfMeetingRelays = {
127bool fBtfRelayPinned = false; 132bool fBtfRelayPinned = false;
128string strBtfActiveRelay; 133string strBtfActiveRelay;
129static CCriticalSection cs_activeRelay; 134static CCriticalSection cs_activeRelay;
135static string strBtfOnionEndpoint;
136static CCriticalSection cs_onionEndpoint;
130 137
131// Only the rendezvous we actually registered at (BtfSetActiveRelay, called from 138// Only the rendezvous we actually registered at (BtfSetActiveRelay, called from
132// ThreadBtfAccept once registration succeeds). Empty until then, on purpose: it 139// ThreadBtfAccept once registration succeeds). Empty until then, on purpose: it
@@ -144,10 +151,43 @@ string BtfActiveRelay()
144 151
145void BtfSetActiveRelay(const string& relay) 152void BtfSetActiveRelay(const string& relay)
146{ 153{
154 // With /rvrelay set, the descriptor must never name anything but the pin.
155 // No caller can reach here with another relay today -- both registration
156 // paths draw their list from BtfAllRelays(), which returns the pin alone --
157 // but the descriptor is the one thing a miner obeys, so the invariant is
158 // enforced at the sink instead of being retrusted to every future caller.
159 if (fBtfRelayPinned &&
160 find(vBtfMeetingRelays.begin(), vBtfMeetingRelays.end(), relay) == vBtfMeetingRelays.end())
161 {
162 LogPrint("pool", "refusing to advertise meeting relay %s: /rvrelay pins %s\n",
163 relay.c_str(),
164 vBtfMeetingRelays.empty() ? "an empty list" : vBtfMeetingRelays[0].c_str());
165 return;
166 }
147 CRITICAL_BLOCK(cs_activeRelay) 167 CRITICAL_BLOCK(cs_activeRelay)
148 strBtfActiveRelay = relay; 168 strBtfActiveRelay = relay;
149} 169}
150 170
171bool BtfSetLocalOnionEndpoint(const string& endpoint, string& errOut)
172{
173 string norm;
174 if (!btf::NormalizeOnionEndpoint(endpoint, norm))
175 {
176 errOut = "expected v3 onion endpoint HOST.onion:PORT";
177 return false;
178 }
179 CRITICAL_BLOCK(cs_onionEndpoint)
180 strBtfOnionEndpoint = norm;
181 return true;
182}
183
184string BtfLocalOnionEndpoint()
185{
186 CRITICAL_BLOCK(cs_onionEndpoint)
187 return strBtfOnionEndpoint;
188 return "";
189}
190
151// A relay this node advertises on Nostr for others to discover (/announcerelay). 191// A relay this node advertises on Nostr for others to discover (/announcerelay).
152string strBtfAnnounceRelay; 192string strBtfAnnounceRelay;
153// Relays learned from other operators' Nostr announcements. 193// Relays learned from other operators' Nostr announcements.
@@ -196,6 +236,18 @@ vector<string> BtfAllRelays()
196 return all; 236 return all;
197} 237}
198 238
239// Relays learned from Nostr, whether or not a pin is in force. Reporting only:
240// it exists so the pool can name what /rvrelay is excluding. Anything choosing
241// a relay to register or dial at must keep using BtfAllRelays().
242vector<string> BtfDiscoveredRelays()
243{
244 vector<string> out;
245 CRITICAL_BLOCK(cs_discoveredRelays)
246 foreach(const string& r, g_discoveredRelays)
247 out.push_back(r);
248 return out;
249}
250
199 251
200// 252//
201// Utilities 253// Utilities
@@ -387,10 +439,16 @@ std::string BtfLocalDescriptor()
387 string meeting; 439 string meeting;
388 CRITICAL_BLOCK(cs_activeRelay) 440 CRITICAL_BLOCK(cs_activeRelay)
389 meeting = strBtfActiveRelay; 441 meeting = strBtfActiveRelay;
442 string onion = BtfLocalOnionEndpoint();
390 if (meeting.empty()) 443 if (meeting.empty())
391 return ""; 444 {
445 if (onion.empty())
446 return "";
447 meeting = "rendezvous-pending";
448 }
392 return btf::SignDescriptor(g_nostrKey.ctx, g_nostrKey.seckey, 449 return btf::SignDescriptor(g_nostrKey.ctx, g_nostrKey.seckey,
393 g_nostrKey.EncPubHex(), meeting, (uint64_t)GetTime()); 450 g_nostrKey.EncPubHex(), meeting, (uint64_t)GetTime(),
451 onion);
394} 452}
395 453
396// The shared secp256k1 context, so net.cpp can verify descriptors that arrive 454// The shared secp256k1 context, so net.cpp can verify descriptors that arrive
@@ -468,6 +526,41 @@ static bool BuildSignedEvent(CNostrKey& key, int kind, const json& tags,
468static const uint64 MAX_WS_FRAME = 1 << 20; // 1 MiB per frame 526static const uint64 MAX_WS_FRAME = 1 << 20; // 1 MiB per frame
469static const uint64 MAX_WS_MESSAGE = 4 << 20; // 4 MiB reassembled 527static const uint64 MAX_WS_MESSAGE = 4 << 20; // 4 MiB reassembled
470 528
529static int BtfConnectedNodeCount()
530{
531 int nNodes = 0;
532 CRITICAL_BLOCK(cs_vNodes)
533 {
534 nNodes = (int)vNodes.size();
535 }
536 return nNodes;
537}
538
539#ifdef _WIN32
540static bool BtfLoadWindowsRootCertificates(SSL_CTX* sslctx)
541{
542 HCERTSTORE hStore = CertOpenSystemStoreA(0, "ROOT");
543 if (!hStore)
544 return false;
545
546 bool fLoaded = false;
547 X509_STORE* x509store = SSL_CTX_get_cert_store(sslctx);
548 PCCERT_CONTEXT pCert = NULL;
549 while ((pCert = CertEnumCertificatesInStore(hStore, pCert)) != NULL)
550 {
551 const unsigned char* p = pCert->pbCertEncoded;
552 X509* x509 = d2i_X509(NULL, &p, pCert->cbCertEncoded);
553 if (!x509)
554 continue;
555 if (X509_STORE_add_cert(x509store, x509) == 1)
556 fLoaded = true;
557 X509_free(x509);
558 }
559 CertCloseStore(hStore, 0);
560 return fLoaded;
561}
562#endif
563
471class CWebSocket 564class CWebSocket
472{ 565{
473public: 566public:
@@ -523,53 +616,8 @@ public:
523 size_t colon = host.find(':'); 616 size_t colon = host.find(':');
524 if (colon != string::npos) { port = atoi(host.substr(colon + 1).c_str()); host = host.substr(0, colon); } 617 if (colon != string::npos) { port = atoi(host.substr(colon + 1).c_str()); host = host.substr(0, colon); }
525 618
526 // Resolve and connect (TCP). AF_UNSPEC so we get both A and AAAA 619 hSocket = BtfConnectSocket(host, (unsigned short)port, SOCK_NOSTR, timeoutSec);
527 // records back and try them in the order the resolver prefers, 620 if (hSocket == INVALID_SOCKET)
528 // rather than hard-failing whenever the IPv4 path to a relay is
529 // filtered/rate-limited (increasingly common for CDN-fronted
530 // relays) while a working IPv6 route exists right there in the
531 // same getaddrinfo() result.
532 struct addrinfo hints, *res = NULL, *rp = NULL;
533 memset(&hints, 0, sizeof(hints));
534 hints.ai_family = AF_UNSPEC;
535 hints.ai_socktype = SOCK_STREAM;
536 char portstr[16]; sprintf(portstr, "%d", port);
537 if (getaddrinfo(host.c_str(), portstr, &hints, &res) != 0 || !res)
538 return error("Nostr: getaddrinfo %s failed", host.c_str());
539
540 bool fConnected = false;
541 for (rp = res; rp != NULL; rp = rp->ai_next)
542 {
543 hSocket = BtfSocketTag(socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol), SOCK_NOSTR);
544 if (hSocket == INVALID_SOCKET)
545 continue;
546
547 // Socket receive/send timeout: Windows takes a DWORD of
548 // milliseconds; POSIX takes a struct timeval.
549#ifdef _WIN32
550 DWORD tv = timeoutSec * 1000;
551 setsockopt(hSocket, SOL_SOCKET, SO_RCVTIMEO, (char*)&tv, sizeof(tv));
552 setsockopt(hSocket, SOL_SOCKET, SO_SNDTIMEO, (char*)&tv, sizeof(tv));
553#else
554 struct timeval tv;
555 tv.tv_sec = timeoutSec;
556 tv.tv_usec = 0;
557 setsockopt(hSocket, SOL_SOCKET, SO_RCVTIMEO, (char*)&tv, sizeof(tv));
558 setsockopt(hSocket, SOL_SOCKET, SO_SNDTIMEO, (char*)&tv, sizeof(tv));
559#endif
560
561 if (connect(hSocket, rp->ai_addr, (int)rp->ai_addrlen) == 0)
562 {
563 fConnected = true;
564 break;
565 }
566
567 BtfCloseSocket(hSocket);
568 hSocket = INVALID_SOCKET;
569 }
570 freeaddrinfo(res);
571
572 if (!fConnected)
573 return error("Nostr: connect %s:%d failed", host.c_str(), port); 621 return error("Nostr: connect %s:%d failed", host.c_str(), port);
574 622
575 // TLS 623 // TLS
@@ -585,22 +633,35 @@ public:
585 // so a MITM cannot forge them, but it could drop the rendezvous 633 // so a MITM cannot forge them, but it could drop the rendezvous
586 // registration (leaving the pool unreachable to miners) or answer 634 // registration (leaving the pool unreachable to miners) or answer
587 // subscriptions with whatever it liked. 635 // subscriptions with whatever it liked.
588 if (!SSL_CTX_set_default_verify_paths(sslctx))
589 return error("Nostr: no system CA store available, refusing %s", host.c_str());
590 SSL_CTX_set_verify(sslctx, SSL_VERIFY_PEER, NULL); 636 SSL_CTX_set_verify(sslctx, SSL_VERIFY_PEER, NULL);
637
638 // OpenSSL's default verify paths are usually empty on Windows, so
639 // the system ROOT store is loaded too -- without it verification
640 // fails closed there and every relay becomes unreachable.
641 bool fTrustLoaded = (SSL_CTX_set_default_verify_paths(sslctx) == 1);
642#ifdef _WIN32
643 fTrustLoaded = BtfLoadWindowsRootCertificates(sslctx) || fTrustLoaded;
644#endif
645 if (!fTrustLoaded)
646 return error("Nostr: failed to load TLS trust store");
647
648 // Floor the protocol version. Verification says who the peer is,
649 // not how weak a cipher suite we will agree to; without this a
650 // relay offering nothing better still gets TLS 1.0.
591 SSL_CTX_set_min_proto_version(sslctx, TLS1_2_VERSION); 651 SSL_CTX_set_min_proto_version(sslctx, TLS1_2_VERSION);
592 652
593 ssl = SSL_new(sslctx); 653 ssl = SSL_new(sslctx);
594 if (!ssl) return false; 654 if (!ssl) return false;
595 SSL_set_fd(ssl, (int)hSocket); 655 SSL_set_fd(ssl, (int)hSocket);
596 SSL_set_tlsext_host_name(ssl, host.c_str()); // SNI 656 if (SSL_set_tlsext_host_name(ssl, host.c_str()) != 1) // SNI
657 return error("Nostr: failed to set TLS SNI for %s", host.c_str());
597 658
598 // Verifying the chain is not enough on its own: without this the 659 // Verifying the chain is not enough on its own: without this the
599 // relay could present any certificate a trusted CA ever issued, 660 // relay could present any certificate a trusted CA ever issued,
600 // for any name. SSL_set1_host binds the check to the host we asked 661 // for any name. SSL_set1_host binds the check to the host we asked
601 // for. 662 // for.
602 if (!SSL_set1_host(ssl, host.c_str())) 663 if (SSL_set1_host(ssl, host.c_str()) != 1)
603 return error("Nostr: cannot pin verification to %s", host.c_str()); 664 return error("Nostr: failed to set TLS hostname for %s", host.c_str());
604 665
605 if (SSL_connect(ssl) != 1) 666 if (SSL_connect(ssl) != 1)
606 { 667 {
@@ -846,17 +907,21 @@ static void PublishDescriptor(CWebSocket& ws, CNostrKey& key)
846 // meeting_node is the rendezvous relay this node's hidden service 907 // meeting_node is the rendezvous relay this node's hidden service
847 // (ThreadBtfAccept in net.cpp) is registered at -- where clients dial us. 908 // (ThreadBtfAccept in net.cpp) is registered at -- where clients dial us.
848 string meeting_node = BtfActiveRelay(); 909 string meeting_node = BtfActiveRelay();
910 string onion = BtfLocalOnionEndpoint();
849 if (meeting_node.empty()) 911 if (meeting_node.empty())
850 { 912 {
851 // Not registered anywhere yet. Publishing now would advertise a meeting 913 if (onion.empty())
852 // node we can't be reached at, and peers would waste dials on it. The 914 {
853 // ThreadNostrSeed loop republishes on every pass, so skipping costs 915 // Not registered anywhere yet, and no direct onion endpoint exists.
854 // nothing but the delay until ThreadBtfAccept registers. 916 // Publishing now would advertise a meeting node we can't be reached at.
855 LogPrint("nostr", "Nostr: skipping descriptor publish, no rendezvous registered yet\n"); 917 LogPrint("nostr", "Nostr: skipping descriptor publish, no rendezvous registered yet\n");
856 return; 918 return;
919 }
920 meeting_node = "rendezvous-pending";
857 } 921 }
858 string desc = btf::SignDescriptor(key.ctx, key.seckey, key.EncPubHex(), 922 string desc = btf::SignDescriptor(key.ctx, key.seckey, key.EncPubHex(),
859 meeting_node, (uint64_t)GetTime()); 923 meeting_node, (uint64_t)GetTime(),
924 onion);
860 if (desc.empty()) 925 if (desc.empty())
861 return; 926 return;
862 json tags = json::array({ json::array({ "d", BTF_DESC_DTAG }) }); 927 json tags = json::array({ json::array({ "d", BTF_DESC_DTAG }) });
@@ -1062,7 +1127,8 @@ static void HandlePoolAnnouncement(const json& ev)
1062// this socket for reuse, and only a completed exchange leaves it in a state 1127// this socket for reuse, and only a completed exchange leaves it in a state
1063// the next caller can trust. 1128// the next caller can trust.
1064static bool ResolveDescriptor(CWebSocket& ws, void* ctx, const string& btfAddr, 1129static bool ResolveDescriptor(CWebSocket& ws, void* ctx, const string& btfAddr,
1065 btf::Descriptor& out, bool& fExchangeOk) 1130 btf::Descriptor& out, bool& fExchangeOk,
1131 string* rawOut = NULL)
1066{ 1132{
1067 fExchangeOk = false; 1133 fExchangeOk = false;
1068 unsigned char pk[32]; 1134 unsigned char pk[32];
@@ -1071,6 +1137,7 @@ static bool ResolveDescriptor(CWebSocket& ws, void* ctx, const string& btfAddr,
1071 json filter = json::object(); 1137 json filter = json::object();
1072 filter["authors"] = json::array({ HexEncode(pk, 32) }); 1138 filter["authors"] = json::array({ HexEncode(pk, 32) });
1073 filter["kinds"] = json::array({ BTF_DESC_KIND }); 1139 filter["kinds"] = json::array({ BTF_DESC_KIND });
1140 filter["#d"] = json::array({ BTF_DESC_DTAG });
1074 filter["limit"] = 1; 1141 filter["limit"] = 1;
1075 json req = json::array({ "REQ", "btf-resolve", filter }); 1142 json req = json::array({ "REQ", "btf-resolve", filter });
1076 if (!ws.SendText(req.dump())) 1143 if (!ws.SendText(req.dump()))
@@ -1097,7 +1164,11 @@ static bool ResolveDescriptor(CWebSocket& ws, void* ctx, const string& btfAddr,
1097 const json& ev = j[2]; 1164 const json& ev = j[2];
1098 if (!fFound && ev.contains("content") && 1165 if (!fFound && ev.contains("content") &&
1099 btf::VerifyDescriptor(ctx, ev["content"].get<string>(), pk, out)) 1166 btf::VerifyDescriptor(ctx, ev["content"].get<string>(), pk, out))
1167 {
1100 fFound = true; 1168 fFound = true;
1169 if (rawOut)
1170 *rawOut = ev["content"].get<string>();
1171 }
1101 // Deliberately keep reading to EOSE rather than returning here. 1172 // Deliberately keep reading to EOSE rather than returning here.
1102 // Returning early leaves the rest of the relay's answer queued on a 1173 // Returning early leaves the rest of the relay's answer queued on a
1103 // socket that goes straight back into the pool, and the next caller 1174 // socket that goes straight back into the pool, and the next caller
@@ -1117,8 +1188,10 @@ static bool ResolveDescriptor(CWebSocket& ws, void* ctx, const string& btfAddr,
1117// public discovery relays for the owner's self-certified descriptor. Used by 1188// public discovery relays for the owner's self-certified descriptor. Used by
1118// ConnectNodeBtf in net.cpp. 1189// ConnectNodeBtf in net.cpp.
1119bool BtfResolve(const std::string& btfAddr, std::string& meetingHostPort, 1190bool BtfResolve(const std::string& btfAddr, std::string& meetingHostPort,
1120 unsigned char enc_pub[32]) 1191 unsigned char enc_pub[32], std::string* onionHostPort)
1121{ 1192{
1193 if (onionHostPort)
1194 onionHostPort->clear();
1122 if (!EnsureNostrKey()) 1195 if (!EnsureNostrKey())
1123 return false; 1196 return false;
1124 for (int i = 0; i < nNostrRelays && !fShutdown; i++) 1197 for (int i = 0; i < nNostrRelays && !fShutdown; i++)
@@ -1146,10 +1219,15 @@ bool BtfResolve(const std::string& btfAddr, std::string& meetingHostPort,
1146 } 1219 }
1147 lease.fOk = fExchangeOk; 1220 lease.fOk = fExchangeOk;
1148 if (d.meeting_node.empty() || d.meeting_node == "rendezvous-pending") 1221 if (d.meeting_node.empty() || d.meeting_node == "rendezvous-pending")
1149 continue; // descriptor predates the owner's relay config 1222 {
1223 if (!onionHostPort || d.onion.empty())
1224 continue; // caller needs rendezvous, or descriptor has no direct path
1225 }
1150 if (!HexDecode(d.enc, enc_pub, 32)) 1226 if (!HexDecode(d.enc, enc_pub, 32))
1151 continue; 1227 continue;
1152 meetingHostPort = d.meeting_node; 1228 meetingHostPort = d.meeting_node;
1229 if (onionHostPort)
1230 *onionHostPort = d.onion;
1153 return true; 1231 return true;
1154 } 1232 }
1155 CATCH_PRINT_EXCEPTION("BtfResolve") 1233 CATCH_PRINT_EXCEPTION("BtfResolve")
@@ -1207,6 +1285,7 @@ bool BtfResolveMany(const std::vector<std::string>& btfAddrs,
1207 json filter = json::object(); 1285 json filter = json::object();
1208 filter["authors"] = authors; 1286 filter["authors"] = authors;
1209 filter["kinds"] = json::array({ BTF_DESC_KIND }); 1287 filter["kinds"] = json::array({ BTF_DESC_KIND });
1288 filter["#d"] = json::array({ BTF_DESC_DTAG });
1210 filter["limit"] = (int)authors.size(); 1289 filter["limit"] = (int)authors.size();
1211 json req = json::array({ "REQ", "btf-resolve-many", filter }); 1290 json req = json::array({ "REQ", "btf-resolve-many", filter });
1212 if (!ws.SendText(req.dump())) 1291 if (!ws.SendText(req.dump()))
@@ -1237,12 +1316,15 @@ bool BtfResolveMany(const std::vector<std::string>& btfAddrs,
1237 btf::Descriptor d; 1316 btf::Descriptor d;
1238 if (!btf::VerifyDescriptor(g_nostrKey.ctx, ev["content"].get<string>(), pk, d)) 1317 if (!btf::VerifyDescriptor(g_nostrKey.ctx, ev["content"].get<string>(), pk, d))
1239 continue; 1318 continue;
1240 if (d.meeting_node.empty() || d.meeting_node == "rendezvous-pending") 1319 if ((d.meeting_node.empty() || d.meeting_node == "rendezvous-pending") &&
1320 d.onion.empty())
1241 continue; 1321 continue;
1242 BtfResolvedPeer rp; 1322 BtfResolvedPeer rp;
1243 if (!HexDecode(d.enc, rp.enc_pub, 32)) 1323 if (!HexDecode(d.enc, rp.enc_pub, 32))
1244 continue; 1324 continue;
1245 rp.meetingHostPort = d.meeting_node; 1325 rp.meetingHostPort = d.meeting_node;
1326 rp.onionHostPort = d.onion;
1327 rp.descriptor = ev["content"].get<string>();
1246 out[ai->second] = rp; 1328 out[ai->second] = rp;
1247 pending.erase(ai->second); 1329 pending.erase(ai->second);
1248 } 1330 }
@@ -1383,7 +1465,7 @@ static bool SeedFromRelay(CNostrKey& key, const string& relay)
1383 else if (createdAt > it->second) it->second = createdAt; // refresh 1465 else if (createdAt > it->second) it->second = createdAt; // refresh
1384 } 1466 }
1385 if (fNew && fDebug) 1467 if (fNew && fDebug)
1386 LogPrint("nostr", "Nostr: discovered .btf peer %s\n", peerAddr.c_str()); 1468 LogPrint("nostrdiscovery", "Nostr: discovered .btf peer %s\n", peerAddr.c_str());
1387 if (++nPeers > 500) break; 1469 if (++nPeers > 500) break;
1388 } 1470 }
1389 else if (type == "EOSE") 1471 else if (type == "EOSE")
@@ -1617,8 +1699,9 @@ static void ConnectDiscoveredBtfPeers()
1617 // plus a random sample, not one or the other. 1699 // plus a random sample, not one or the other.
1618 int64 now = GetTime(); 1700 int64 now = GetTime();
1619 int nBudget = (int)BTF_DIALS_PER_PASS; 1701 int nBudget = (int)BTF_DIALS_PER_PASS;
1702 int nNodeCount = BtfConnectedNodeCount();
1620 { 1703 {
1621 int nRoom = (int)BTF_TARGET_CONN - (int)vNodes.size(); 1704 int nRoom = (int)BTF_TARGET_CONN - nNodeCount;
1622 if (nRoom < nBudget) nBudget = nRoom; 1705 if (nRoom < nBudget) nBudget = nRoom;
1623 } 1706 }
1624 if (nBudget < 0) nBudget = 0; 1707 if (nBudget < 0) nBudget = 0;
@@ -1632,7 +1715,6 @@ static void ConnectDiscoveredBtfPeers()
1632 // Reliability slice: walk the tiered (good -> untried -> bad) order. 1715 // Reliability slice: walk the tiered (good -> untried -> bad) order.
1633 for (const string& addr : peers) 1716 for (const string& addr : peers)
1634 { 1717 {
1635 if ((int)vNodes.size() >= (int)BTF_TARGET_CONN) break;
1636 if ((int)candidates.size() >= nSortedSlots) break; 1718 if ((int)candidates.size() >= nSortedSlots) break;
1637 map<string, int64>::iterator bi = g_btfPeerBackoff.find(addr); 1719 map<string, int64>::iterator bi = g_btfPeerBackoff.find(addr);
1638 if (bi != g_btfPeerBackoff.end() && now < bi->second) 1720 if (bi != g_btfPeerBackoff.end() && now < bi->second)
@@ -1657,7 +1739,7 @@ static void ConnectDiscoveredBtfPeers()
1657 int nWant = min(nRandomSlots, (int)pool.size()); 1739 int nWant = min(nRandomSlots, (int)pool.size());
1658 for (int i = 0; i < nWant; i++) 1740 for (int i = 0; i < nWant; i++)
1659 { 1741 {
1660 if ((int)vNodes.size() + (int)candidates.size() >= (int)BTF_TARGET_CONN) break; 1742 if (nNodeCount + (int)candidates.size() >= (int)BTF_TARGET_CONN) break;
1661 if ((int)candidates.size() >= nBudget) break; 1743 if ((int)candidates.size() >= nBudget) break;
1662 size_t j = i + (size_t)GetRand((uint64)(pool.size() - i)); 1744 size_t j = i + (size_t)GetRand((uint64)(pool.size() - i));
1663 std::swap(pool[i], pool[j]); 1745 std::swap(pool[i], pool[j]);
@@ -1712,20 +1794,20 @@ static void ConnectDiscoveredBtfPeers()
1712 // not the shared Nostr relays, so there's no thundering-herd effect here. 1794 // not the shared Nostr relays, so there's no thundering-herd effect here.
1713 // Each thread writes its result back via shared state guarded by cs_btfPeerState. 1795 // Each thread writes its result back via shared state guarded by cs_btfPeerState.
1714 struct DialCtx { string addr; BtfResolvedPeer rp; }; 1796 struct DialCtx { string addr; BtfResolvedPeer rp; };
1715 vector<DialCtx*> ctxs;
1716 for (const string& addr : toDial) 1797 for (const string& addr : toDial)
1717 { 1798 {
1718 DialCtx* ctx = new DialCtx{addr, resolved[addr]}; 1799 DialCtx* ctx = new DialCtx{addr, resolved[addr]};
1719 ctxs.push_back(ctx); 1800 uintptr_t tid = _beginthread([](void* arg) {
1720 _beginthread([](void* arg) {
1721 DialCtx* ctx = (DialCtx*)arg; 1801 DialCtx* ctx = (DialCtx*)arg;
1722 string addr = ctx->addr; 1802 string addr = ctx->addr;
1723 string meeting = ctx->rp.meetingHostPort; 1803 string meeting = ctx->rp.meetingHostPort;
1804 string onion = ctx->rp.onionHostPort;
1805 string desc = ctx->rp.descriptor;
1724 unsigned char enc_pub[32]; 1806 unsigned char enc_pub[32];
1725 memcpy(enc_pub, ctx->rp.enc_pub, 32); 1807 memcpy(enc_pub, ctx->rp.enc_pub, 32);
1726 delete ctx; 1808 delete ctx;
1727 1809
1728 CNode* pnode = ConnectNodeBtfResolved(addr, meeting, enc_pub); 1810 CNode* pnode = ConnectNodeBtfResolved(addr, meeting, onion, enc_pub, desc);
1729 CRITICAL_BLOCK(cs_btfPeerState) 1811 CRITICAL_BLOCK(cs_btfPeerState)
1730 { 1812 {
1731 if (pnode) 1813 if (pnode)
@@ -1750,6 +1832,21 @@ static void ConnectDiscoveredBtfPeers()
1750 } 1832 }
1751 } 1833 }
1752 }, 0, ctx); 1834 }, 0, ctx);
1835 if (tid == (uintptr_t)-1)
1836 {
1837 LogPrint("nostr", "Nostr: failed to create .btf dial thread for %s\n", addr.c_str());
1838 CRITICAL_BLOCK(cs_btfPeerState)
1839 {
1840 int n = ++g_btfPeerFails[addr];
1841 int64 cap = (n >= BTF_CHRONIC_FAIL_THRESHOLD) ? BTF_BACKOFF_CAP_SECS_DEAD
1842 : BTF_BACKOFF_CAP_SECS;
1843 int64 secs = 30;
1844 for (int k = 1; k < n && secs < cap; k++) secs *= 2;
1845 if (secs > cap) secs = cap;
1846 g_btfPeerBackoff[addr] = GetTime() + secs;
1847 }
1848 delete ctx;
1849 }
1753 } 1850 }
1754 1851
1755 // Wait long enough for all parallel dials to finish (each has a 10s socket 1852 // Wait long enough for all parallel dials to finish (each has a 10s socket
@@ -1772,11 +1869,12 @@ void ThreadNostrSeed(void* parg)
1772 LogPrint("nostr", "Nostr: node pubkey = %s\n", key.PubKeyHex().c_str()); 1869 LogPrint("nostr", "Nostr: node pubkey = %s\n", key.PubKeyHex().c_str());
1773 LogPrint("nostr", "Nostr: node .btf address = %s\n", key.BtfAddress().c_str()); 1870 LogPrint("nostr", "Nostr: node .btf address = %s\n", key.BtfAddress().c_str());
1774 1871
1775 // The self-test below publishes a real descriptor, and PublishDescriptor now 1872 // The self-test below publishes a real descriptor. Rendezvous-only nodes
1776 // refuses to publish before ThreadBtfAccept has registered a rendezvous. Give 1873 // need ThreadBtfAccept to register first; Tor nodes can publish as soon as
1777 // registration a bounded head start so the test measures the publish/resolve 1874 // their signed onion endpoint exists, with rendezvous marked pending.
1778 // chain instead of losing the race against it. 1875 for (int i = 0; i < 60 && !fShutdown &&
1779 for (int i = 0; i < 60 && !fShutdown && BtfActiveRelay().empty(); i++) 1876 BtfActiveRelay().empty() &&
1877 BtfLocalOnionEndpoint().empty(); i++)
1780 Sleep(1000); 1878 Sleep(1000);
1781 1879
1782 // One-time self-test of the rendezvous: publish our descriptor to a relay, 1880 // One-time self-test of the rendezvous: publish our descriptor to a relay,
@@ -1852,7 +1950,7 @@ void ThreadNostrSeed(void* parg)
1852 1950
1853 // Re-discover periodically. Sleep less aggressively when 1951 // Re-discover periodically. Sleep less aggressively when
1854 // we have no connections yet so we find peers faster on startup. 1952 // we have no connections yet so we find peers faster on startup.
1855 int nSleepSecs = vNodes.empty() ? 20 : 60; 1953 int nSleepSecs = (BtfConnectedNodeCount() == 0) ? 20 : 60;
1856 for (int i = 0; i < nSleepSecs && !fShutdown; i++) 1954 for (int i = 0; i < nSleepSecs && !fShutdown; i++)
1857 Sleep(1000); 1955 Sleep(1000);
1858 } 1956 }
diff --git a/src/nostr.h b/src/nostr.h
index 827b2f3..0dd919d 100644
--- a/src/nostr.h
+++ b/src/nostr.h
@@ -40,6 +40,13 @@ std::string BtfActiveRelay();
40// Record the relay this node just registered its service at (thread-safe). 40// Record the relay this node just registered its service at (thread-safe).
41void BtfSetActiveRelay(const std::string& relay); 41void BtfSetActiveRelay(const std::string& relay);
42 42
43// Optional Tor hidden-service endpoint for this node's normal P2P listener.
44// Set by /onionservice=HOST.onion:PORT. The endpoint is signed into this node's
45// .btf descriptor, so peers with -tor can dial it directly before falling back
46// to rendezvous relays.
47bool BtfSetLocalOnionEndpoint(const std::string& endpoint, std::string& errOut);
48std::string BtfLocalOnionEndpoint();
49
43// If set (via /announcerelay=host:port), this node announces that relay on 50// If set (via /announcerelay=host:port), this node announces that relay on
44// Nostr so other nodes discover it automatically -- no manual seed-list edit. 51// Nostr so other nodes discover it automatically -- no manual seed-list edit.
45extern std::string strBtfAnnounceRelay; 52extern std::string strBtfAnnounceRelay;
@@ -53,6 +60,9 @@ extern bool fBtfRelayPinned;
53bool LooksLikeRelay(const std::string& r); 60bool LooksLikeRelay(const std::string& r);
54// Every relay to try: curated seeds + relays discovered from Nostr announcements. 61// Every relay to try: curated seeds + relays discovered from Nostr announcements.
55std::vector<std::string> BtfAllRelays(); 62std::vector<std::string> BtfAllRelays();
63// Relays discovered from Nostr, ignoring the pin. For reporting what a pin
64// excludes -- never for choosing a relay to register or dial at.
65std::vector<std::string> BtfDiscoveredRelays();
56 66
57struct BtfPoolAnnouncement 67struct BtfPoolAnnouncement
58{ 68{
@@ -102,12 +112,15 @@ void* BtfSecpContext();
102// self-certified descriptor and return its meeting node ("host:port") and 112// self-certified descriptor and return its meeting node ("host:port") and
103// x25519 public key for the end-to-end channel. 113// x25519 public key for the end-to-end channel.
104bool BtfResolve(const std::string& btfAddr, std::string& meetingHostPort, 114bool BtfResolve(const std::string& btfAddr, std::string& meetingHostPort,
105 unsigned char enc_pub[32]); 115 unsigned char enc_pub[32],
116 std::string* onionHostPort = NULL);
106 117
107// Result of resolving one .btf address via BtfResolveMany. 118// Result of resolving one .btf address via BtfResolveMany.
108struct BtfResolvedPeer 119struct BtfResolvedPeer
109{ 120{
110 std::string meetingHostPort; 121 std::string meetingHostPort;
122 std::string onionHostPort;
123 std::string descriptor;
111 unsigned char enc_pub[32]; 124 unsigned char enc_pub[32];
112}; 125};
113 126
diff --git a/src/proxy.cpp b/src/proxy.cpp
new file mode 100644
index 0000000..eed054c
--- /dev/null
+++ b/src/proxy.cpp
@@ -0,0 +1,422 @@
1// Copyright (c) 2026 Bitflash developers
2// Distributed under the MIT/X11 software license.
3
4#include "headers_core.h"
5#include "proxy.h"
6#include "sockcount.h"
7
8#include <errno.h>
9
10static CCriticalSection cs_socks5;
11static bool g_fSocks5Proxy = false;
12static bool g_fTorProxy = false;
13static std::string g_socks5Host;
14static unsigned short g_socks5Port = 0;
15static const char* DEFAULT_TOR_SOCKS5_PROXY = "127.0.0.1:9050";
16static const int TOR_CONNECT_TIMEOUT_SECS = 120;
17
18static void SetSocketTimeout(SOCKET s, int nTimeoutSecs)
19{
20 if (nTimeoutSecs <= 0)
21 return;
22#ifdef _WIN32
23 DWORD tv = (DWORD)nTimeoutSecs * 1000;
24 setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tv, sizeof(tv));
25 setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, (const char*)&tv, sizeof(tv));
26#else
27 struct timeval tv;
28 tv.tv_sec = nTimeoutSecs;
29 tv.tv_usec = 0;
30 setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
31 setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
32#endif
33}
34
35static bool ReadN(SOCKET s, void* buf, int n)
36{
37 char* p = (char*)buf;
38 int off = 0;
39 while (off < n)
40 {
41 int r = recv(s, p + off, n - off, 0);
42 if (r <= 0)
43 return false;
44 off += r;
45 }
46 return true;
47}
48
49static bool WriteN(SOCKET s, const void* buf, int n)
50{
51 const char* p = (const char*)buf;
52 int off = 0;
53 while (off < n)
54 {
55 int r = send(s, p + off, n - off, 0);
56 if (r <= 0)
57 return false;
58 off += r;
59 }
60 return true;
61}
62
63static bool ValidProxyHostChar(char c)
64{
65 return isalnum((unsigned char)c) || c == '.' || c == '-' || c == '_';
66}
67
68static bool ValidProxyIpv6HostChar(char c)
69{
70 return isxdigit((unsigned char)c) || c == ':' || c == '.';
71}
72
73static std::string LowerProxyHost(std::string host)
74{
75 if (!host.empty() && host[host.size() - 1] == '.')
76 host.erase(host.size() - 1);
77 for (size_t i = 0; i < host.size(); i++)
78 host[i] = (char)tolower((unsigned char)host[i]);
79 return host;
80}
81
82bool BtfIsTorOnionHost(const std::string& host)
83{
84 std::string lower = LowerProxyHost(host);
85 static const char* suffix = ".onion";
86 size_t suffixLen = strlen(suffix);
87 return lower.size() > suffixLen &&
88 lower.compare(lower.size() - suffixLen, suffixLen, suffix) == 0;
89}
90
91static bool ParsePort(const std::string& port, unsigned short& portOut,
92 std::string& errOut)
93{
94 if (port.empty())
95 {
96 errOut = "port is empty";
97 return false;
98 }
99 for (size_t i = 0; i < port.size(); i++)
100 {
101 if (!isdigit((unsigned char)port[i]))
102 {
103 errOut = "port is not numeric";
104 return false;
105 }
106 }
107
108 errno = 0;
109 char* end = NULL;
110 long nPort = strtol(port.c_str(), &end, 10);
111 if (errno == ERANGE || end == NULL || *end != '\0' || nPort <= 0 || nPort > 65535)
112 {
113 errOut = "port is out of range";
114 return false;
115 }
116 portOut = (unsigned short)nPort;
117 return true;
118}
119
120bool BtfParseSocks5Proxy(const std::string& spec, std::string& hostOut,
121 unsigned short& portOut, std::string& errOut)
122{
123 hostOut.clear();
124 portOut = 0;
125 errOut.clear();
126
127 std::string host;
128 std::string port;
129 bool fBracketedIpv6 = false;
130 if (!spec.empty() && spec[0] == '[')
131 {
132 size_t close = spec.find(']');
133 if (close == std::string::npos || close == 1 || close + 1 >= spec.size() || spec[close + 1] != ':')
134 {
135 errOut = "expected [IPv6]:PORT";
136 return false;
137 }
138 host = spec.substr(1, close - 1);
139 port = spec.substr(close + 2);
140 fBracketedIpv6 = true;
141 }
142 else
143 {
144 size_t colon = spec.rfind(':');
145 if (colon == std::string::npos || colon == 0 || colon + 1 >= spec.size())
146 {
147 errOut = "expected HOST:PORT";
148 return false;
149 }
150 host = spec.substr(0, colon);
151 port = spec.substr(colon + 1);
152 if (host.find(':') != std::string::npos)
153 {
154 errOut = "IPv6 proxy hosts must use [ADDR]:PORT";
155 return false;
156 }
157 }
158
159 if (host.size() > 255)
160 {
161 errOut = "host is too long";
162 return false;
163 }
164 if (fBracketedIpv6)
165 {
166 bool fHasColon = false;
167 for (size_t i = 0; i < host.size(); i++)
168 {
169 fHasColon = fHasColon || host[i] == ':';
170 if (!ValidProxyIpv6HostChar(host[i]))
171 {
172 errOut = "IPv6 proxy host contains unsupported characters";
173 return false;
174 }
175 }
176 if (!fHasColon)
177 {
178 errOut = "bracketed proxy host is not IPv6";
179 return false;
180 }
181 }
182 else
183 {
184 for (size_t i = 0; i < host.size(); i++)
185 {
186 if (!ValidProxyHostChar(host[i]))
187 {
188 errOut = "host contains unsupported characters";
189 return false;
190 }
191 }
192 }
193 if (!ParsePort(port, portOut, errOut))
194 return false;
195
196 hostOut = host;
197 return true;
198}
199
200static bool SetSocks5Proxy(const std::string& spec, bool fTorMode,
201 std::string& errOut)
202{
203 std::string host;
204 unsigned short port = 0;
205 if (!BtfParseSocks5Proxy(spec, host, port, errOut))
206 return false;
207 CRITICAL_BLOCK(cs_socks5)
208 {
209 g_socks5Host = host;
210 g_socks5Port = port;
211 g_fSocks5Proxy = true;
212 g_fTorProxy = fTorMode;
213 }
214 return true;
215}
216
217bool BtfSetSocks5Proxy(const std::string& spec, std::string& errOut)
218{
219 return SetSocks5Proxy(spec, false, errOut);
220}
221
222bool BtfEnableTorProxy(const std::string& spec, std::string& errOut)
223{
224 return SetSocks5Proxy(spec.empty() ? DEFAULT_TOR_SOCKS5_PROXY : spec, true, errOut);
225}
226
227void BtfClearSocks5Proxy()
228{
229 CRITICAL_BLOCK(cs_socks5)
230 {
231 g_fSocks5Proxy = false;
232 g_fTorProxy = false;
233 g_socks5Host.clear();
234 g_socks5Port = 0;
235 }
236}
237
238bool BtfSocks5ProxyEnabled()
239{
240 CRITICAL_BLOCK(cs_socks5)
241 return g_fSocks5Proxy;
242 return false;
243}
244
245bool BtfTorProxyEnabled()
246{
247 CRITICAL_BLOCK(cs_socks5)
248 return g_fSocks5Proxy && g_fTorProxy;
249 return false;
250}
251
252std::string BtfSocks5ProxyName()
253{
254 CRITICAL_BLOCK(cs_socks5)
255 {
256 if (!g_fSocks5Proxy)
257 return "";
258 if (g_socks5Host.find(':') != std::string::npos)
259 return strprintf("[%s]:%u", g_socks5Host.c_str(), (unsigned)g_socks5Port);
260 return strprintf("%s:%u", g_socks5Host.c_str(), (unsigned)g_socks5Port);
261 }
262 return "";
263}
264
265static SOCKET ConnectDirect(const std::string& host, unsigned short port,
266 int nSockSite, int nTimeoutSecs)
267{
268 struct addrinfo hints, *res = NULL, *rp = NULL;
269 memset(&hints, 0, sizeof(hints));
270 hints.ai_family = AF_UNSPEC;
271 hints.ai_socktype = SOCK_STREAM;
272 char portstr[16];
273 sprintf(portstr, "%u", (unsigned)port);
274 if (getaddrinfo(host.c_str(), portstr, &hints, &res) != 0 || !res)
275 return INVALID_SOCKET;
276
277 SOCKET s = INVALID_SOCKET;
278 for (rp = res; rp != NULL; rp = rp->ai_next)
279 {
280 s = BtfSocketTag(socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol), nSockSite);
281 if (s == INVALID_SOCKET)
282 continue;
283 SetSocketTimeout(s, nTimeoutSecs);
284 if (connect(s, rp->ai_addr, (int)rp->ai_addrlen) == 0)
285 break;
286 BtfCloseSocket(s);
287 s = INVALID_SOCKET;
288 }
289 freeaddrinfo(res);
290 return s;
291}
292
293static bool Socks5Connect(SOCKET s, const std::string& destHost,
294 unsigned short destPort)
295{
296 if (destHost.empty() || destHost.size() > 255)
297 return false;
298
299 unsigned char hello[3] = { 0x05, 0x01, 0x00 }; // SOCKS5, one method, no auth
300 if (!WriteN(s, hello, sizeof(hello)))
301 {
302 error("SOCKS5 hello write failed for %s:%u\n",
303 destHost.c_str(), (unsigned)destPort);
304 return false;
305 }
306 unsigned char choice[2] = { 0, 0 };
307 if (!ReadN(s, choice, sizeof(choice)))
308 {
309 error("SOCKS5 method selection read failed for %s:%u\n",
310 destHost.c_str(), (unsigned)destPort);
311 return false;
312 }
313 if (choice[0] != 0x05 || choice[1] != 0x00)
314 {
315 error("SOCKS5 proxy rejected no-auth method for %s:%u (version=0x%02x method=0x%02x)\n",
316 destHost.c_str(), (unsigned)destPort,
317 (unsigned)choice[0], (unsigned)choice[1]);
318 return false;
319 }
320
321 std::vector<unsigned char> req;
322 req.reserve(7 + destHost.size());
323 req.push_back(0x05); // version
324 req.push_back(0x01); // CONNECT
325 req.push_back(0x00); // reserved
326 req.push_back(0x03); // domain name; proxy resolves, avoiding local DNS leak
327 req.push_back((unsigned char)destHost.size());
328 req.insert(req.end(), destHost.begin(), destHost.end());
329 req.push_back((unsigned char)(destPort >> 8));
330 req.push_back((unsigned char)(destPort & 0xff));
331 if (!WriteN(s, &req[0], (int)req.size()))
332 {
333 error("SOCKS5 CONNECT request write failed for %s:%u\n",
334 destHost.c_str(), (unsigned)destPort);
335 return false;
336 }
337
338 unsigned char rep[4] = {0,0,0,0};
339 if (!ReadN(s, rep, sizeof(rep)))
340 {
341 error("SOCKS5 CONNECT reply read failed for %s:%u\n",
342 destHost.c_str(), (unsigned)destPort);
343 return false;
344 }
345 if (rep[0] != 0x05)
346 {
347 error("SOCKS5 CONNECT reply has invalid version 0x%02x for %s:%u\n",
348 (unsigned)rep[0], destHost.c_str(), (unsigned)destPort);
349 return false;
350 }
351 if (rep[1] != 0x00)
352 {
353 error("SOCKS5 CONNECT to %s:%u failed with reply 0x%02x\n",
354 destHost.c_str(), (unsigned)destPort, (unsigned)rep[1]);
355 return false;
356 }
357 int nAddr = 0;
358 if (rep[3] == 0x01) nAddr = 4;
359 else if (rep[3] == 0x04) nAddr = 16;
360 else if (rep[3] == 0x03)
361 {
362 unsigned char len = 0;
363 if (!ReadN(s, &len, 1))
364 return false;
365 nAddr = len;
366 }
367 else
368 {
369 error("SOCKS5 CONNECT reply has invalid address type 0x%02x for %s:%u\n",
370 (unsigned)rep[3], destHost.c_str(), (unsigned)destPort);
371 return false;
372 }
373
374 std::vector<unsigned char> discard(nAddr + 2);
375 if (!ReadN(s, &discard[0], (int)discard.size()))
376 {
377 error("SOCKS5 CONNECT bind address read failed for %s:%u\n",
378 destHost.c_str(), (unsigned)destPort);
379 return false;
380 }
381 return true;
382}
383
384SOCKET BtfConnectSocket(const std::string& destHost, unsigned short destPort,
385 int nSockSite, int nTimeoutSecs)
386{
387 std::string proxyHost;
388 unsigned short proxyPort = 0;
389 bool fProxy = false;
390 bool fTorProxy = false;
391 CRITICAL_BLOCK(cs_socks5)
392 {
393 fProxy = g_fSocks5Proxy;
394 fTorProxy = g_fTorProxy;
395 proxyHost = g_socks5Host;
396 proxyPort = g_socks5Port;
397 }
398
399 if (!fProxy)
400 {
401 if (BtfIsTorOnionHost(destHost))
402 {
403 error("Refusing direct .onion connection to %s:%u; use /tor or /socks\n",
404 destHost.c_str(), (unsigned)destPort);
405 return INVALID_SOCKET;
406 }
407 return ConnectDirect(destHost, destPort, nSockSite, nTimeoutSecs);
408 }
409
410 if (fTorProxy && nTimeoutSecs > 0 && nTimeoutSecs < TOR_CONNECT_TIMEOUT_SECS)
411 nTimeoutSecs = TOR_CONNECT_TIMEOUT_SECS;
412
413 SOCKET s = ConnectDirect(proxyHost, proxyPort, nSockSite, nTimeoutSecs);
414 if (s == INVALID_SOCKET)
415 return INVALID_SOCKET;
416 if (!Socks5Connect(s, destHost, destPort))
417 {
418 BtfCloseSocket(s);
419 return INVALID_SOCKET;
420 }
421 return s;
422}
diff --git a/src/proxy.h b/src/proxy.h
new file mode 100644
index 0000000..6429cf2
--- /dev/null
+++ b/src/proxy.h
@@ -0,0 +1,34 @@
1// Copyright (c) 2026 Bitflash developers
2// Distributed under the MIT/X11 software license.
3//
4// Optional SOCKS5 outbound proxy support. This is transport plumbing only:
5// no consensus, wallet, mining, or .btf identity data depends on it.
6
7#ifndef BITFLASH_PROXY_H
8#define BITFLASH_PROXY_H
9
10#ifdef _WIN32
11#ifndef WIN32_LEAN_AND_MEAN
12#define WIN32_LEAN_AND_MEAN 1
13#endif
14#include <winsock2.h>
15#else
16#include "compat.h"
17#endif
18
19#include <string>
20
21bool BtfParseSocks5Proxy(const std::string& spec, std::string& hostOut,
22 unsigned short& portOut, std::string& errOut);
23bool BtfSetSocks5Proxy(const std::string& spec, std::string& errOut);
24bool BtfEnableTorProxy(const std::string& spec, std::string& errOut);
25void BtfClearSocks5Proxy();
26bool BtfSocks5ProxyEnabled();
27bool BtfTorProxyEnabled();
28bool BtfIsTorOnionHost(const std::string& host);
29std::string BtfSocks5ProxyName();
30
31SOCKET BtfConnectSocket(const std::string& destHost, unsigned short destPort,
32 int nSockSite, int nTimeoutSecs);
33
34#endif
diff --git a/src/randomx_pow.cpp b/src/randomx_pow.cpp
index d617ba2..a509543 100644
--- a/src/randomx_pow.cpp
+++ b/src/randomx_pow.cpp
@@ -8,6 +8,7 @@
8#include <thread> 8#include <thread>
9#include <vector> 9#include <vector>
10#include <mutex> 10#include <mutex>
11#include <system_error>
11 12
12// Fixed key (seed) that determines the RandomX cache/dataset. Keeping it 13// Fixed key (seed) that determines the RandomX cache/dataset. Keeping it
13// constant avoids per-epoch "reseed" logic; for the CPU+RAM fairness goal 14// constant avoids per-epoch "reseed" logic; for the CPU+RAM fairness goal
@@ -19,40 +20,117 @@ static randomx_cache* g_cache = NULL;
19static randomx_dataset* g_dataset = NULL; 20static randomx_dataset* g_dataset = NULL;
20static randomx_vm* g_vmVerify = NULL; 21static randomx_vm* g_vmVerify = NULL;
21static CCriticalSection g_csVerify; 22static CCriticalSection g_csVerify;
23static std::mutex g_csInit;
22static bool g_fInit = false; 24static bool g_fInit = false;
23static bool g_fFast = false; 25static bool g_fFast = false;
24 26
27// RandomX reads a 256 MB cache, a 2 GB dataset and a 2 MB scratchpad per VM
28// at random, so most reads miss the TLB on 4 KB pages. 2 MB pages recover
29// roughly a tenth of the hash rate.
30//
31// The pool must be reserved first (vm.nr_hugepages), and RandomX returns NULL
32// instead of falling back, so each allocation below retries without the flag.
33bool fRandomXLargePages = true; // cleared by -nolargepages
34static bool g_fLargeCache = false;
35static bool g_fLargeDataset = false;
36static bool g_fLargeAny = false;
37
38// How to reserve the large-page pool, for the hint printed when a large-page
39// allocation falls back to normal pages. The mechanism is OS-specific.
40static const char* LargePagesHint()
41{
42#ifdef _WIN32
43 return "enable the \"Lock pages in memory\" privilege and run elevated";
44#else
45 return "reserve them with: sysctl vm.nr_hugepages=1280";
46#endif
47}
48
49static randomx_flags WithLargePages(randomx_flags f)
50{
51 return (randomx_flags)(f | RANDOMX_FLAG_LARGE_PAGES);
52}
53
54// Ask for a cache in large pages, then plain pages. Sets fLargeOut to say
55// which one answered.
56static randomx_cache* AllocCache(randomx_flags flags, bool& fLargeOut)
57{
58 fLargeOut = false;
59 if (fRandomXLargePages)
60 {
61 randomx_cache* c = randomx_alloc_cache(WithLargePages(flags));
62 if (c)
63 {
64 fLargeOut = true;
65 return c;
66 }
67 }
68 return randomx_alloc_cache(flags);
69}
70
25 71
26bool RandomXInit() 72bool RandomXInit()
27{ 73{
28 if (g_fInit) 74 if (g_fInit)
29 return true; 75 return true;
30 76
77 std::lock_guard<std::mutex> lock(g_csInit);
78 if (g_fInit)
79 return true;
80
31 // Detect the best flags for this CPU (JIT, hardware AES, Argon2). 81 // Detect the best flags for this CPU (JIT, hardware AES, Argon2).
32 g_flags = randomx_get_flags(); 82 g_flags = randomx_get_flags();
33 83
34 g_cache = randomx_alloc_cache(g_flags); 84 g_cache = AllocCache(g_flags, g_fLargeCache);
35 if (!g_cache) 85 if (!g_cache)
36 { 86 {
37 // Try without JIT as a fallback 87 // Try without JIT as a fallback
38 g_flags = RANDOMX_FLAG_DEFAULT; 88 g_flags = RANDOMX_FLAG_DEFAULT;
39 g_cache = randomx_alloc_cache(g_flags); 89 g_cache = AllocCache(g_flags, g_fLargeCache);
40 if (!g_cache) 90 if (!g_cache)
41 return error("RandomX: failed to allocate cache (256 MB)"); 91 return error("RandomX: failed to allocate cache (256 MB)");
42 } 92 }
43 randomx_init_cache(g_cache, RANDOMX_KEY, strlen(RANDOMX_KEY)); 93 randomx_init_cache(g_cache, RANDOMX_KEY, strlen(RANDOMX_KEY));
44 94
45 // Verification VM in light mode (cache only) 95 // Verification VM in light mode (cache only). The scratchpad is only 2 MB
46 g_vmVerify = randomx_create_vm(g_flags, g_cache, NULL); 96 // and this VM is created once, but it hashes every block the node ever
97 // validates, so it gets the same treatment.
98 bool fLargeVerify = false;
99 if (fRandomXLargePages)
100 {
101 g_vmVerify = randomx_create_vm(WithLargePages(g_flags), g_cache, NULL);
102 fLargeVerify = (g_vmVerify != NULL);
103 }
104 if (!g_vmVerify)
105 g_vmVerify = randomx_create_vm(g_flags, g_cache, NULL);
47 if (!g_vmVerify) 106 if (!g_vmVerify)
48 return error("RandomX: failed to create verification VM"); 107 return error("RandomX: failed to create verification VM");
49 108
109 g_fLargeAny = g_fLargeCache || fLargeVerify;
50 g_fInit = true; 110 g_fInit = true;
51 printf("RandomX: initialized (cache 256 MB, flags=%d)\n", (int)g_flags); 111 printf("RandomX: initialized (cache 256 MB, flags=%d, large pages: %s)\n",
112 (int)g_flags, RandomXLargePagesStatus());
113 if (fRandomXLargePages && !g_fLargeCache)
114 printf("RandomX: large pages unavailable for the cache; %s "
115 "for about 10%% more hash rate\n", LargePagesHint());
52 return true; 116 return true;
53} 117}
54 118
55 119
120const char* RandomXLargePagesStatus()
121{
122 if (!fRandomXLargePages)
123 return "off (-nolargepages)";
124 if (g_fLargeCache && g_fLargeDataset)
125 return "cache + dataset";
126 if (g_fLargeCache)
127 return "cache only";
128 if (g_fLargeDataset)
129 return "dataset only";
130 return g_fLargeAny ? "scratchpad only" : "unavailable";
131}
132
133
56// One dataset, once, no matter how many miners ask for it. 134// One dataset, once, no matter how many miners ask for it.
57// 135//
58// Every miner thread calls this at startup. The g_fFast guard below is only 136// Every miner thread calls this at startup. The g_fFast guard below is only
@@ -82,8 +160,18 @@ bool RandomXInitDataset(int nThreads)
82 if (g_fFast) 160 if (g_fFast)
83 return true; 161 return true;
84 162
85 randomx_flags fastFlags = (randomx_flags)(g_flags | RANDOMX_FLAG_FULL_MEM); 163 // Only RANDOMX_FLAG_LARGE_PAGES means anything to randomx_alloc_dataset;
86 g_dataset = randomx_alloc_dataset(fastFlags); 164 // the rest of g_flags belongs to the VMs that read it.
165 if (fRandomXLargePages)
166 {
167 g_dataset = randomx_alloc_dataset(WithLargePages(RANDOMX_FLAG_DEFAULT));
168 g_fLargeDataset = (g_dataset != NULL);
169 if (!g_dataset)
170 printf("RandomX: large pages unavailable for the 2 GB dataset; %s "
171 "for about 10%% more hash rate\n", LargePagesHint());
172 }
173 if (!g_dataset)
174 g_dataset = randomx_alloc_dataset(RANDOMX_FLAG_DEFAULT);
87 if (!g_dataset) 175 if (!g_dataset)
88 { 176 {
89 printf("RandomX: not enough memory for the 2 GB dataset, staying in light mode\n"); 177 printf("RandomX: not enough memory for the 2 GB dataset, staying in light mode\n");
@@ -96,13 +184,28 @@ bool RandomXInitDataset(int nThreads)
96 184
97 std::vector<std::thread> workers; 185 std::vector<std::thread> workers;
98 unsigned long per = total / nThreads; 186 unsigned long per = total / nThreads;
99 for (int i = 0; i < nThreads; i++) 187 try
188 {
189 for (int i = 0; i < nThreads; i++)
190 {
191 unsigned long start = i * per;
192 unsigned long count = (i == nThreads - 1) ? (total - start) : per;
193 workers.emplace_back([start, count]() {
194 randomx_init_dataset(g_dataset, g_cache, start, count);
195 });
196 }
197 }
198 catch (const std::system_error& e)
100 { 199 {
101 unsigned long start = i * per; 200 // Same refusal as the allocation failure above: light mode is slow
102 unsigned long count = (i == nThreads - 1) ? (total - start) : per; 201 // but alive, std::terminate is neither. Join the workers that did
103 workers.emplace_back([start, count]() { 202 // start before releasing the dataset out from under them.
104 randomx_init_dataset(g_dataset, g_cache, start, count); 203 for (auto& w : workers) w.join();
105 }); 204 randomx_release_dataset(g_dataset);
205 g_dataset = NULL;
206 printf("RandomX: cannot spawn dataset init threads (%s), staying in light mode\n",
207 e.what());
208 return false;
106 } 209 }
107 for (auto& w : workers) w.join(); 210 for (auto& w : workers) w.join();
108 211
@@ -137,13 +240,26 @@ void* RandomXCreateMinerVM()
137{ 240{
138 if (!g_fInit && !RandomXInit()) 241 if (!g_fInit && !RandomXInit())
139 return NULL; 242 return NULL;
243
244 randomx_flags flags = g_flags;
245 randomx_cache* cache = g_cache;
246 randomx_dataset* dataset = NULL;
140 if (g_fFast && g_dataset) 247 if (g_fFast && g_dataset)
141 { 248 {
142 randomx_flags fastFlags = (randomx_flags)(g_flags | RANDOMX_FLAG_FULL_MEM); 249 flags = (randomx_flags)(g_flags | RANDOMX_FLAG_FULL_MEM);
143 return randomx_create_vm(fastFlags, NULL, g_dataset); 250 cache = NULL;
251 dataset = g_dataset;
252 }
253
254 // Each thread gets its own 2 MB scratchpad, so the large-page pool can run
255 // dry partway through a fleet of miners. Threads that miss out still run.
256 if (fRandomXLargePages)
257 {
258 randomx_vm* vm = randomx_create_vm(WithLargePages(flags), cache, dataset);
259 if (vm)
260 return vm;
144 } 261 }
145 // Light mode: own VM sharing the cache 262 return randomx_create_vm(flags, cache, dataset);
146 return randomx_create_vm(g_flags, g_cache, NULL);
147} 263}
148 264
149 265
diff --git a/src/randomx_pow.h b/src/randomx_pow.h
index 0027571..355b708 100644
--- a/src/randomx_pow.h
+++ b/src/randomx_pow.h
@@ -41,4 +41,11 @@ uint256 RandomXHashWithVM(void* vm, const void* pHeader, size_t nSize);
41// True when the 2 GB dataset is ready (fast mode active). 41// True when the 2 GB dataset is ready (fast mode active).
42bool RandomXFastReady(); 42bool RandomXFastReady();
43 43
44// Ask for large (2 MB) pages for the cache, dataset and scratchpads. Set false
45// by -nolargepages. Must be set before the first RandomXInit().
46extern bool fRandomXLargePages;
47
48// Which allocations actually got large pages, for the status display.
49const char* RandomXLargePagesStatus();
50
44#endif 51#endif
diff --git a/src/rc/bitcoin.ico b/src/rc/bitcoin.ico
index b768077..33323de 100644
--- a/src/rc/bitcoin.ico
+++ b/src/rc/bitcoin.ico
Binary files differ
diff --git a/src/readme.txt b/src/readme.txt
index 6af1694..455884c 100644
--- a/src/readme.txt
+++ b/src/readme.txt
@@ -16,7 +16,7 @@ This product includes software developed by the OpenSSL Project.
16Differences from the original Bitcoin 16Differences from the original Bitcoin
17------------------------------------- 17-------------------------------------
18- Own genesis block (22/Jul/2026 headline, independent chain) 18- Own genesis block (22/Jul/2026 headline, independent chain)
19- Own magic bytes (0xbf 0x1a 0x5c 0xfd) and P2P port 8433 19- Own magic bytes (0xbf 0x20 0x5c 0xfd) and P2P port 8433
20- Addresses start with "B" (ADDRESSVERSION = 25) 20- Addresses start with "B" (ADDRESSVERSION = 25)
21- Consensus fixes applied: value overflow (CVE-2010-5139), duplicate inputs, 21- Consensus fixes applied: value overflow (CVE-2010-5139), duplicate inputs,
22 money range checks (MoneyRange) 22 money range checks (MoneyRange)
@@ -41,21 +41,45 @@ The resulting executable is src/bitflash.exe.
41 41
42Command-line options (in addition to the original ones) 42Command-line options (in addition to the original ones)
43------------------------------------------------------- 43-------------------------------------------------------
44 /port=N listen on P2P port N (default 8433) 44 /testnet use isolated test network genesis, datadir, port 18433, and message magic
45 /port=N listen on P2P port N (default 8433; testnet default 18433)
46 /socks=HOST:PORT
47 route outbound Nostr, .btf rendezvous, and direct onion peer
48 dials through a SOCKS5 proxy, e.g. /socks=127.0.0.1:9050 for local Tor
49 or /socks=[::1]:9050 for an IPv6 loopback proxy
50 /tor[=HOST:PORT]
51 Tor mode for outbound Nostr, .btf rendezvous, and direct onion peer dials;
52 defaults to the local Tor SOCKS5 proxy at 127.0.0.1:9050
53 /managedtor[=PATH]
54 start Tor, create a v3 hidden service for this node, route outbound
55 discovery through it, and advertise the generated onion endpoint
56 Windows release packages that include tor/tor.exe start this mode
57 automatically; use /nomanagedtor to force the legacy relay path
45 /solomine mine without requiring a connected peer (local testing) 58 /solomine mine without requiring a connected peer (local testing)
46 /operator operator mode (run pool server) 59 /operator operator mode (run pool server)
47 /participant=ADDR 60 /participant=ADDR
48 participant mode; mine to pool .btf address ADDR 61 participant mode; mine to pool .btf address ADDR
62 /stratumbridge=ADDR
63 listen on 127.0.0.1:3333 and bridge external Stratum miners
64 to pool .btf address ADDR
65 /stratumbridgeport=N
66 local Stratum bridge port (default 3333)
49 /poolname=NAME operator announcement pool name 67 /poolname=NAME operator announcement pool name
50 /poolfee=PCT operator announcement fee percent (example: 0.75) 68 /poolfee=PCT operator announcement fee percent (example: 0.75)
51 /pooldashboard=URL 69 /pooldashboard=URL
52 operator announcement dashboard URL 70 operator announcement dashboard URL
71 /poolstatusfile=PATH
72 write pool status JSON for dashboards
73 /poolroundsfile=PATH
74 write public pool round proofs JSON
53 /connectbtf=ADDR 75 /connectbtf=ADDR
54 keep an outbound .btf connection to ADDR 76 keep an outbound .btf connection to ADDR
55 /rvrelay=HOST:PORT 77 /rvrelay=HOST:PORT
56 override meeting relay list with one relay 78 override meeting relay list with one relay
57 /announcerelay=HOST:PORT 79 /announcerelay=HOST:PORT
58 announce this relay on Nostr for discovery 80 announce this relay on Nostr for discovery
81 /onionservice=HOST.onion:PORT
82 advertise this node's Tor hidden service in its signed .btf descriptor
59 83
60 84
61Roadmap 85Roadmap
diff --git a/src/rpc.cpp b/src/rpc.cpp
index 03d1f44..3827cd5 100644
--- a/src/rpc.cpp
+++ b/src/rpc.cpp
@@ -31,10 +31,15 @@
31#include <netinet/in.h> 31#include <netinet/in.h>
32#include <arpa/inet.h> 32#include <arpa/inet.h>
33#include <unistd.h> 33#include <unistd.h>
34#include <fcntl.h>
35#include <dirent.h>
36#else
37#include <io.h>
34#endif 38#endif
35 39
36#include <atomic> 40#include <atomic>
37#include <mutex> 41#include <mutex>
42#include <thread>
38#include <map> 43#include <map>
39#include <set> 44#include <set>
40#include <deque> 45#include <deque>
@@ -110,15 +115,28 @@ static uint256 ShareTargetFromDifficulty(double difficulty)
110{ 115{
111 if (difficulty <= 0.0) difficulty = 1.0; 116 if (difficulty <= 0.0) difficulty = 1.0;
112 static const uint256 diff1 = CBigNum(bnProofOfWorkLimit).getuint256(); 117 static const uint256 diff1 = CBigNum(bnProofOfWorkLimit).getuint256();
118 // Representability floor. shareTarget = diff1 / difficulty, and diff1 is
119 // ~uint256(0) >> 12, i.e. a hair under 2^244 -- so the true target stops
120 // fitting in a uint256 once difficulty reaches 2^-12 (= 1/4096), and the
121 // fixed-point divide below breaks even earlier in a second way: the
122 // (uint64)(difficulty * 65536.0) divisor truncates to zero for any
123 // difficulty under 2^-16. Both thresholds sit far below VARDIFF_MIN
124 // (0.001), so live operation never comes near this, but a caller that
125 // bypasses vardiff (or a future VARDIFF_MIN) must not divide by zero or
126 // hand back getuint256()'s low-256-bit truncation of an oversized
127 // quotient -- a wrapped target is not merely wrong, it is arbitrarily
128 // HARDER than difficulty 1. At these difficulties every hash is meant to
129 // be a share, so the honest answer is the easiest representable target.
130 // (This replaces a dead branch that read `if (t < diff1 && difficulty <
131 // 1.0)`: t = diff1/difficulty, so difficulty < 1.0 implies t > diff1 and
132 // the two conditions could never both hold.)
133 if (difficulty < 1.0 / 4096.0) return ~uint256(0);
113 // shareTarget = diff1 / difficulty 134 // shareTarget = diff1 / difficulty
114 CBigNum bn; bn.setuint256(diff1); 135 CBigNum bn; bn.setuint256(diff1);
115 // Multiply by 2^16 to keep precision, divide, shift back 136 // Multiply by 2^16 to keep precision, divide, shift back
116 bn <<= 16; 137 bn <<= 16;
117 bn /= (uint64)(difficulty * 65536.0); 138 bn /= (uint64)(difficulty * 65536.0);
118 uint256 t = bn.getuint256(); 139 return bn.getuint256();
119 // Cap at all-ones (maximum, easiest possible)
120 if (t < diff1 && difficulty < 1.0) t = ~uint256(0);
121 return t;
122} 140}
123 141
124// Expected number of hashes to find one share at a given Stratum difficulty, 142// Expected number of hashes to find one share at a given Stratum difficulty,
@@ -218,6 +236,29 @@ static bool RebuildJob()
218 // From the pool, so the template's payout key is in wallet.dat before 236 // From the pool, so the template's payout key is in wallet.dat before
219 // any miner is handed work against it. 237 // any miner is handed work against it.
220 vector<unsigned char> vchPubKey = GetKeyFromPool(); 238 vector<unsigned char> vchPubKey = GetKeyFromPool();
239 if (vchPubKey.empty()) {
240 // Empty means the wallet could not produce a key: encrypted and
241 // locked, or key generation itself failed. Serialised, an empty
242 // vector is a zero-length push, so the coinbase would become
243 // `<empty> OP_CHECKSIG` -- a script nobody can satisfy. Every
244 // block mined against such a job burns its whole subsidy, while
245 // QueuePayouts still records the full obligation and the miners
246 // are then paid out of coin that block never contributed.
247 // BitcoinMiner refuses to mine in this state; refuse to hand the
248 // pool's miners work in it too. printf and not LogPrint for the
249 // reason given above the deaf-node refusal, and throttled the
250 // same way -- the no-job watchdog retries every few seconds for
251 // as long as the condition lasts.
252 static int64 nLastSaidNoKey = 0;
253 if (GetTime() - nLastSaidNoKey > 60) {
254 nLastSaidNoKey = GetTime();
255 printf("RebuildJob: no wallet key available for the coinbase "
256 "(wallet locked, or key generation failed -- the "
257 "GetKeyFromPool line above says which) -- not issuing "
258 "jobs\n");
259 }
260 return false;
261 }
221 static std::atomic<uint32_t> sExtra{0}; 262 static std::atomic<uint32_t> sExtra{0};
222 uint32_t extraNonce = ++sExtra; 263 uint32_t extraNonce = ++sExtra;
223 264
@@ -236,35 +277,96 @@ static bool RebuildJob()
236 CBlock block; 277 CBlock block;
237 block.vtx.push_back(txNew); 278 block.vtx.push_back(txNew);
238 279
280 // Fill the block the same way BitcoinMiner() does, because this is the
281 // same job: a block that gets mined and submitted for real.
282 //
283 // It used to differ in two ways, and both were bugs.
284 //
285 // One pass over mapTransactions is not enough. That map is keyed by
286 // hash, so it has no dependency order, and a transaction spending the
287 // still-unconfirmed output of another (change from a recent send, most
288 // commonly) fails ConnectInputs whenever its parent happens to sit
289 // later in map order. A single pass then drops it -- and because map
290 // order does not change between rebuilds, every later rebuild drops it
291 // again at the same point. Deterministic exclusion, not bad luck: if
292 // pool operators mine most of the blocks, a transaction of that shape
293 // can stay unconfirmed for as long as its parent does.
294 //
295 // Diagnosis and fix both come from Vic-Nas, who found it in his fork
296 // of this code and described it exactly. The fork has since been
297 // deleted, so this carries it back rather than losing it.
298 int64 nFees = 0;
239 { 299 {
240 CTxDB txdb("r"); 300 CTxDB txdb("r");
241 CRITICAL_BLOCK(cs_mapTransactions) 301 CRITICAL_BLOCK(cs_mapTransactions)
242 { 302 {
243 map<uint256, CTxIndex> pool; 303 map<uint256, CTxIndex> pool;
244 unsigned int sz = 0; 304 vector<char> vfAlreadyAdded(mapTransactions.size());
245 for (auto& kv : mapTransactions) { 305 bool fFoundSomething = true;
246 CTransaction& tx = kv.second; 306 // The cap is MAX_BLOCK_SIZE_GEN, not MAX_SIZE/2. MAX_SIZE is
247 if (tx.IsCoinBase() || !tx.IsFinal()) continue; 307 // the 32 MB serialisation ceiling, so half of it is 16 MB --
248 int64 nFees = 0; 308 // this builder would happily assemble a template that its own
249 map<uint256, CTxIndex> tmp(pool); 309 // CheckBlock() rejects at the 1 MB consensus cap. This is the
250 if (!tx.ConnectInputs(txdb, tmp, CDiskTxPos(1,1,1), 0, nFees, 310 // pool's job builder, so a block over the cap is not one
251 false, true, tx.GetMinFee(block.vtx.size() < 100))) 311 // wasted solve, it is every miner on the pool working a
252 continue; 312 // template the network will reject.
253 pool = tmp; 313 //
254 block.vtx.push_back(tx); 314 // The inner break used to bound the overshoot to one
255 sz += ::GetSerializeSize(tx, SER_NETWORK); 315 // transaction per pass. One transaction is still too many:
256 // Must stay under the consensus cap CheckBlock() enforces. 316 // nothing caps how big a single transaction may be, so that
257 // This is the pool's job builder, so a block over the cap 317 // bound was worth up to MAX_SIZE. Asking whether each one fits
258 // is not just one wasted solve -- it is every miner on the 318 // before appending it bounds the overshoot at zero instead.
259 // pool working a template the network will reject. The 319 //
260 // transaction is already pushed above, so this can overshoot 320 // Seeded with the header, tx count and coinbase, because
261 // by one; half the cap leaves room for that. 321 // CheckBlock() weighs the whole serialized block.
262 if (sz > MAX_BLOCK_SIZE / 2) break; 322 unsigned int sz = ::GetSerializeSize(block, SER_DISK);
323 // And the sigop budget, which CheckBlock() enforces and this
324 // builder never counted against. Same counting function
325 // CheckBlock() sums, seeded from the coinbase because it sums
326 // over all of vtx.
327 unsigned int nSigOps = block.vtx[0].GetSigOpCount();
328 while (fFoundSomething && sz < MAX_BLOCK_SIZE_GEN)
329 {
330 fFoundSomething = false;
331 unsigned int n = 0;
332 for (map<uint256, CTransaction>::iterator mi = mapTransactions.begin();
333 mi != mapTransactions.end(); ++mi, ++n)
334 {
335 if (vfAlreadyAdded[n])
336 continue;
337 CTransaction& tx = (*mi).second;
338 if (tx.IsCoinBase() || !tx.IsFinal()) continue;
339 // Skip rather than stop: one transaction too big for
340 // the room left does not mean the next is. Ordered
341 // before ConnectInputs because ConnectInputs credits
342 // nFees by reference on success, and a transaction we
343 // then decline would leave the coinbase claiming a fee
344 // this block does not collect.
345 unsigned int nTxSize = ::GetSerializeSize(tx, SER_NETWORK);
346 unsigned int nTxSigOps = tx.GetSigOpCount();
347 if (!BlockTemplateFits(sz, nSigOps, nTxSize, nTxSigOps))
348 continue;
349 map<uint256, CTxIndex> tmp(pool);
350 if (!tx.ConnectInputs(txdb, tmp, CDiskTxPos(1,1,1), 0, nFees,
351 false, true, tx.GetMinFee(block.vtx.size() < 100)))
352 continue;
353 swap(pool, tmp);
354 block.vtx.push_back(tx);
355 sz += nTxSize;
356 nSigOps += nTxSigOps;
357 vfAlreadyAdded[n] = true;
358 fFoundSomething = true;
359 }
263 } 360 }
264 } 361 }
265 } 362 }
266 363
267 block.vtx[0].vout[0].nValue = block.GetBlockValue(pindexPrev->nHeight + 1, 0); 364 // And the fees actually go to the coinbase now. nFees was declared
365 // inside the scan loop, so every transaction reset it and the total was
366 // thrown away, while this line passed a hardcoded 0. A pool operator
367 // collected fees into a block and paid none of them to itself. Solo
368 // mining has always passed nFees here; only this builder did not.
369 block.vtx[0].vout[0].nValue = block.GetBlockValue(pindexPrev->nHeight + 1, nFees);
268 block.hashPrevBlock = pindexPrev->GetBlockHash(); 370 block.hashPrevBlock = pindexPrev->GetBlockHash();
269 block.hashMerkleRoot = block.BuildMerkleTree(); 371 block.hashMerkleRoot = block.BuildMerkleTree();
270 block.nTime = max((unsigned int)(pindexPrev->GetMedianTimePast() + 1), 372 block.nTime = max((unsigned int)(pindexPrev->GetMedianTimePast() + 1),
@@ -297,26 +399,406 @@ struct PendingPayout {
297 // supposed to pay it: a 280-block reorg once left ~14.4k BTF of debt 399 // supposed to pay it: a 280-block reorg once left ~14.4k BTF of debt
298 // owed against blocks that no longer existed, which then drained the 400 // owed against blocks that no longer existed, which then drained the
299 // wallet paying fork-era miners out of real-chain coin. 401 // wallet paying fork-era miners out of real-chain coin.
402 //
403 // Upstream keys these on the string "block-<height>". That is the same
404 // bug this fork already paid for, and height keying additionally lets
405 // FindRecordedPoolPayout match the *previous* round's transaction after a
406 // reorg and silently skip a miner, so their `id` is deliberately dropped.
300 int blockHeight = 0; 407 int blockHeight = 0;
301 // Must be explicitly zeroed: uint256's default constructor is empty 408 // Must be explicitly zeroed: uint256's default constructor is empty
302 // (uint256.h:514), so leaving this unassigned gives every ledger entry 409 // (uint256.h:514), so leaving this unassigned gives every ledger entry
303 // loaded from an older file a garbage hash, which then matches no block 410 // loaded from an older file a garbage hash, which then matches no block
304 // and gets voided as "orphaned". That mistake voided a live payout. 411 // and gets voided as "orphaned". That mistake voided a live payout.
305 uint256 blockHash = 0; // zero = entry predates hash keying 412 uint256 blockHash = 0; // zero = entry predates hash keying
306 int matureAtHeight; 413 int matureAtHeight = 0;
307 std::map<std::string, int64> amounts; // address -> satoshis 414 std::map<std::string, int64> amounts; // address -> satoshis
308 int retries = 0; // exec-thread only: SendMoney attempts so far 415 // From upstream: which recipients already have a transaction. This is what
309 int deferrals = 0; // exec-thread only: waits for spendable coin 416 // lets an entry stay on disk until every address in `amounts` is paid,
310 int64 retryAfter = 0; // exec-thread only: unix time, 0 = ready now 417 // instead of being erased the moment the batch is handed off.
418 std::map<std::string, std::string> paidTxids; // address -> txid
419 // Retry and backoff state does NOT live here. It used to be declared here
420 // as well as on PayingRecord, and the copy on this side was never written:
421 // `retryAfter` was declared, serialised, loaded and tested at the flush
422 // gate, and the only assignment to it anywhere was `= 0`, so the gate could
423 // not fire. `retries` and `deferrals` were the same story. The real state
424 // is per (round, recipient) -- which is the granularity a payment can fail
425 // at -- and it lives on PayingRecord, which is now what persists it.
311}; 426};
312static std::vector<PendingPayout> gPendingPayouts; 427static std::vector<PendingPayout> gPendingPayouts;
428static std::mutex gPayoutLedgerMutex;
429// Bumped on every serialisation, under gPayoutLedgerMutex. See PublishVersioned.
430static uint64 gPayoutLedgerSeq = 0;
431// Serialises writers of pending_payouts.json, and guards the sequence number
432// of the image currently on disk. Never taken while a data mutex is held.
433static std::mutex gPayoutLedgerFileMutex;
434static uint64 gPayoutLedgerPublishedSeq = 0;
435
436struct PoolRoundProof {
437 std::string id;
438 int blockHeight;
439 int matureAtHeight;
440 int64 createdAt;
441 std::string blockHash;
442 std::string coinbaseTxid;
443 std::string foundBy;
444 double feePercent;
445 uint64 totalShares;
446 int64 coinbaseValue;
447 int64 subsidy;
448 int64 minerReward;
449 int64 operatorReward;
450 int skippedDust;
451 int64 forfeitedBadAddress;
452 std::map<std::string, uint64> shares;
453 std::map<std::string, int64> payouts;
454 std::map<std::string, std::string> paidTxids;
455 // Recipients the ledger has written off, and why. Without this the public
456 // round proof goes on advertising an address as owed with "txid": null
457 // forever, while the ledger has already dropped the obligation -- the two
458 // records of the same round permanently disagreeing, with the operator
459 // holding the money.
460 std::map<std::string, std::string> forfeited; // address -> reason
461
462 PoolRoundProof()
463 : blockHeight(0), matureAtHeight(0), createdAt(0), feePercent(0.0),
464 totalShares(0), coinbaseValue(0), subsidy(0), minerReward(0), operatorReward(0),
465 skippedDust(0), forfeitedBadAddress(0) {}
466};
467static std::vector<PoolRoundProof> gPoolRounds;
468static std::mutex gPoolRoundsMutex;
469static uint64 gPoolRoundsSeq = 0; // guarded by gPoolRoundsMutex
470static std::mutex gPoolRoundsFileMutex;
471static uint64 gPoolRoundsPublishedSeq = 0; // guarded by gPoolRoundsFileMutex
472static const size_t MAX_POOL_ROUNDS = 200;
313 473
314static std::string PayoutFilePath() 474static std::string PayoutFilePath()
315{ 475{
316 return GetAppDir() + "/pending_payouts.json"; 476 return GetAppDir() + "/pending_payouts.json";
317} 477}
318 478
319static void SavePendingPayouts() 479static std::string PoolRoundsFilePath()
480{
481 if (!strPoolRoundsFile.empty())
482 return strPoolRoundsFile;
483 return GetAppDir() + "/pool_rounds.json";
484}
485
486// ---------------------------------------------------------------------------
487// Atomic, ordered publication of the payout state files
488// ---------------------------------------------------------------------------
489//
490// Every one of these files is a full dump of an in-memory structure, written
491// tmp-then-rename so a crash cannot truncate it. Two things have to be true
492// for that to actually be safe, and only the first was:
493//
494// 1. the rename must be atomic (it is), and
495// 2. exactly one thread at a time may be writing the temp file, and a write
496// must never publish an OLDER image over a newer one.
497//
498// (2) was broken for pending_payouts.json. The event loop wrote it with
499// gPayoutLedgerMutex held, the payout thread wrote it with the mutex
500// released, and nothing serialised the two: both opened the same
501// "<path>.tmp" with independent handles and both renamed it. The interleaved
502// file that lands is either a mixture of two images (LoadPendingPayouts then
503// throws mid-loop and comes up with a nearly empty ledger -- every obligation
504// not yet exploded, gone) or a whole newly-queued round silently overwritten
505// by an older snapshot that predates it.
506//
507// The fix is a per-file write mutex, held only across the I/O and never
508// across a data-structure mutex, plus a version on the snapshot. Serialising
509// happens under the data mutex and stamps a monotonically increasing sequence
510// number; publishing happens with the data mutex released and refuses any
511// blob whose sequence number is not newer than what is already on disk. So
512// writers can queue in any order and finish in any order -- the file only
513// ever moves forwards.
514//
515// The other half of the rule is that no thread waits for I/O while holding
516// gPayoutLedgerMutex, gPayoutExecMutex or gPoolRoundsMutex, which is the
517// contention the merge was right to remove from the event loop. That was
518// claimed here before it was true: pool_status.json is written by the same
519// WriteFileAtomic and RefreshStats held gPayoutLedgerMutex at function scope
520// across the call, so once the writes gained fsync() the event loop was doing
521// two of them every ten seconds with the payout thread's mutex held -- worse
522// than the buffered write the merge replaced. The status file now goes through
523// the same serialise-under-the-lock / publish-outside-it split as the ledgers,
524// and the rule holds for all four files. It is checked, not asserted: see the
525// "no payout mutex is held across a publish" section of the payout-ledger
526// self-test.
527
528// A serialised file image plus the version of the in-memory state it was
529// taken from.
530struct VersionedBlob {
531 std::string blob;
532 uint64 seq = 0;
533};
534
535// Push a file's contents out of the C library's buffer AND out of the kernel's
536// page cache, so the bytes exist on the device before anything points at them.
537static bool SyncFileHandle(FILE* f)
538{
539 if (fflush(f) != 0)
540 return false;
541#ifdef _WIN32
542 return _commit(_fileno(f)) == 0;
543#else
544 return fsync(fileno(f)) == 0;
545#endif
546}
547
548// Make the directory entry that rename() created durable too.
549//
550// A tmp+rename is only atomic with respect to a crash of the process. Against
551// a power cut or a hard VM reset the rename can reach the platter while the
552// data it points at has not, and the file that comes back is short or empty.
553// For pending_payouts.json that is not a corrupt file, it is a silent one:
554// LoadPendingPayouts catches the parse error, logs a single warning, and the
555// node comes up owing nobody anything.
556//
557// Best effort -- a filesystem that does not allow the directory to be opened
558// for this is not a reason to refuse to write the ledger.
559static void SyncDirectoryOf(const std::string& path)
560{
561#ifndef _WIN32
562 size_t slash = path.find_last_of('/');
563 std::string dir = (slash == std::string::npos) ? std::string(".")
564 : path.substr(0, slash);
565 if (dir.empty())
566 dir = "/";
567 int fd = open(dir.c_str(), O_RDONLY);
568 if (fd < 0)
569 return;
570 fsync(fd);
571 close(fd);
572#endif
573}
574
575// Replace `path` with `data` atomically and durably. Caller must hold the
576// write mutex for `path`: the temp name is derived from the target, so two
577// threads in here at once for one path is precisely the race this exists to
578// prevent.
579//
580// Every failure removes the temp file before returning. It used to be left
581// where it fell, so a disk that filled up once left a short, valid-looking
582// "pending_payouts.json.tmp" next to the good ledger for as long as the node
583// ran. That file is a trap: it has the name of a half-finished save, and an
584// operator cleaning up after an incident is exactly the person who will move
585// it into place.
586//
587// Failures print unconditionally rather than through LogPrint(). These three
588// files are written under two different debug categories ("payout" for the
589// ledgers, "pool" for the status file), and a write failure is not a trace of
590// normal operation -- it is the file the operator is about to read not being
591// there. Gating it behind a category that had to be chosen before the failure
592// happened is how "could not replace pool_status.json" went missing.
593static bool WriteFileAtomic(const std::string& path, const std::string& data,
594 const char* what)
595{
596 std::string tmp = path + ".tmp";
597 FILE* f = fopen(tmp.c_str(), "w");
598 if (!f) {
599 printf("[payout] WARNING: could not write %s (%s)\n",
600 tmp.c_str(), what);
601 return false;
602 }
603 if (!data.empty() && fwrite(data.c_str(), 1, data.size(), f) != data.size()) {
604 printf("[payout] WARNING: short write to %s (%s)\n",
605 tmp.c_str(), what);
606 fclose(f);
607 remove(tmp.c_str());
608 return false;
609 }
610 // Before the rename, not after: publishing a name that points at nothing
611 // is the failure this ordering exists to prevent. If the data cannot be
612 // made durable, do not rename -- the previous file is still intact and
613 // still correct, and the next write will try again.
614 if (!SyncFileHandle(f)) {
615 printf("[payout] WARNING: could not flush %s to disk (%s)"
616 " -- keeping the previous file\n", tmp.c_str(), what);
617 fclose(f);
618 remove(tmp.c_str());
619 return false;
620 }
621 // fclose() is where buffered data actually reaches the OS, so a failure
622 // here means the .tmp is short -- renaming it over the real file would
623 // destroy the good copy, which is exactly what these files exist to
624 // prevent.
625 if (fclose(f) != 0) {
626 printf("[payout] WARNING: could not close %s (%s)\n",
627 tmp.c_str(), what);
628 remove(tmp.c_str());
629 return false;
630 }
631#ifdef _WIN32
632 // rename() on Windows will not replace an existing file.
633 remove(path.c_str());
634#endif
635 if (rename(tmp.c_str(), path.c_str()) != 0) {
636 printf("[payout] WARNING: could not replace %s (%s)\n",
637 path.c_str(), what);
638 remove(tmp.c_str());
639 return false;
640 }
641 SyncDirectoryOf(path);
642 return true;
643}
644
645// Paths whose loader threw part-way through, and which must therefore never be
646// written again by this process.
647//
648// All three loaders push entries into the live structure INSIDE the try, so an
649// exception half way down a file leaves a PARTIAL ledger in memory -- the
650// entries before the bad one, and nothing after it. The first publish after
651// that then writes the partial state over the file, and the only record of
652// what was owed is gone. ReconcilePayouts publishes unconditionally at
653// startup, so "the first publish" is a few milliseconds later, before an
654// operator could possibly intervene. a8e2f58 named this failure mode in its
655// own commit message and then only fixed the write side.
656//
657// The file is renamed out of the way instead of being left in place, so the
658// evidence survives even if the operator restarts into the same fault, and so
659// a node that comes up owing nobody anything is visibly missing its ledger
660// rather than quietly appearing to have none.
661static std::set<std::string> gRefusedFiles;
662static std::mutex gRefusedFilesMutex;
663
664static bool FileIsRefused(const std::string& path)
665{
666 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
667 return gRefusedFiles.count(path) > 0;
668}
669
670// Quarantine `path` and refuse to write it for the rest of the process.
671//
672// Prints unconditionally: an operator who has just lost a payout ledger must
673// be told at whatever log settings they were already running with, and the
674// name it was moved to is the only way back to the data.
675static void RefuseFileAfterLoadFailure(const std::string& path, const char* what,
676 const char* why)
677{
678 {
679 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
680 gRefusedFiles.insert(path);
681 }
682 std::string bad = strprintf("%s.bad.%lld", path.c_str(), (long long)GetTime());
683 bool moved = (rename(path.c_str(), bad.c_str()) == 0);
684 printf("[payout] ERROR: %s (%s) did not parse: %s\n"
685 "[payout] ERROR: what loaded before the error is a PARTIAL picture,"
686 " so this file will not be written again by this process --"
687 " restart once the cause is fixed\n"
688 "[payout] ERROR: the file %s\n",
689 path.c_str(), what, why,
690 moved ? strprintf("has been kept as %s", bad.c_str()).c_str()
691 : "could NOT be renamed out of the way -- copy it now");
692}
693
694// Publish a versioned image of some in-memory state to `path`.
695//
696// `fileMutex` serialises writers of `path` and guards `publishedSeq`. A blob
697// older than what is already published is dropped: these files are whole-state
698// dumps, so a higher sequence number is by construction a later state of the
699// same structure and publishing it loses nothing. On a write failure
700// `publishedSeq` is deliberately left alone, so the next (newer) write still
701// gets a chance to land.
702static void PublishVersioned(const std::string& path, const VersionedBlob& v,
703 std::mutex& fileMutex, uint64& publishedSeq,
704 const char* what)
705{
706 // A default-constructed blob is not a snapshot of anything; it is a
707 // VersionedBlob nobody filled in. Today they are already dropped, because
708 // every serialiser stamps ++gSeq and so never produces 0, and 0 fails the
709 // seq <= publishedSeq test below -- but only for as long as publishedSeq
710 // starts at 0 and nobody writes `seq = gSeq++`. One character, and the
711 // ledger file becomes "[]". Say it out loud instead of relying on it.
712 if (!v.seq)
713 return;
714 // A file whose loader threw is not a file this process knows the contents
715 // of. Writing it would replace a full record of what was owed with
716 // whatever happened to parse before the error.
717 if (FileIsRefused(path))
718 return;
719 std::lock_guard<std::mutex> lk(fileMutex);
720 if (v.seq <= publishedSeq)
721 return;
722 if (!WriteFileAtomic(path, v.blob, what))
723 return;
724 publishedSeq = v.seq;
725}
726
727// Identity of a payout round. Used as the pool-round proof id and, more
728// importantly, as the `pool_payout_id` written into wallet transaction
729// metadata and matched by FindRecordedPoolPayout().
730//
731// Derived from the block HASH, not from its height alone. Upstream uses
732// "block-<height>", which names two different blocks identically across a
733// reorg -- and because this id is what the crash-recovery probe matches on, a
734// height-keyed id lets that probe find the *previous* round's transaction and
735// conclude a miner has already been paid when they have not. The height stays
736// in the string for readability only.
737//
738// A zero hash means the entry predates hash keying; those keep the old form so
739// anything already recorded against them still matches.
740static std::string PayoutIdForBlock(int blockHeight, const uint256& blockHash)
741{
742 if (blockHash == 0)
743 return strprintf("block-%d", blockHeight);
744 return strprintf("block-%d-%s", blockHeight,
745 blockHash.ToString().substr(0, 16).c_str());
746}
747
748// Do these two references name the same payout round?
749//
750// The block hash alone does not decide it, and every ledger lookup used to
751// assume it did. Entries grandfathered in from a file written before payouts
752// were keyed by hash all carry hash 0, so under a hash-only comparison the
753// first zero-hash entry answers for every other one: RecordPayoutTxid writes
754// a txid against the wrong round (which is then erased as "fully paid" while
755// the real debt still stands), and ReconcilePayouts drops a live in-flight
756// record because a *different* zero-hash round happened to pay that address.
757//
758// A non-zero hash is unique and decides on its own -- that is the whole point
759// of keying on it. A zero hash falls back to the height, which
760// LoadPendingPayouts supplies for any entry that can name its own maturity.
761// Mixed hash-ness never matches: refusing to match is the safe direction for
762// every caller here, because each of them treats a match as "this debt is
763// already dealt with".
764static bool SamePayoutRound(int aHeight, const uint256& aHash,
765 int bHeight, const uint256& bHash)
766{
767 if (aHash != bHash)
768 return false;
769 if (aHash != 0)
770 return true;
771 return aHeight == bHeight;
772}
773
774// An entry with neither a hash nor a height cannot be told apart from any
775// other entry in the same state, so nothing may be recorded against it: a
776// txid written to the wrong one reports a debt as settled that is still owed.
777// Such an entry is never paid and is logged instead -- refusing to act is the
778// safe failure, and no file this tree has ever written can produce one.
779static bool PayoutRoundIdentifiable(int height, const uint256& hash)
780{
781 return hash != 0 || height != 0;
782}
783
784static bool PayoutFullyPaid(const PendingPayout& pp)
785{
786 for (const auto& kv : pp.amounts)
787 if (pp.paidTxids.count(kv.first) == 0)
788 return false;
789 return true;
790}
791
792// Serialise the ledger and stamp it with the version it was taken at. Caller
793// must hold gPayoutLedgerMutex.
794//
795// Split from the file write so no thread holds the ledger mutex across the
796// I/O. Upstream writes the file inside the per-recipient loop with
797// gPayoutLedgerMutex held, while the event loop's FlushMaturePayouts blocks on
798// that same mutex -- one slow fsync per recipient, on the thread the miners'
799// I/O shares. The sequence number is what lets the write happen later, and
800// possibly out of order, without a stale image reaching the disk.
801static VersionedBlob SerializePendingPayoutsLocked()
320{ 802{
321 nlohmann::json arr = nlohmann::json::array(); 803 nlohmann::json arr = nlohmann::json::array();
322 for (const auto& pp : gPendingPayouts) { 804 for (const auto& pp : gPendingPayouts) {
@@ -324,21 +806,56 @@ static void SavePendingPayouts()
324 obj["blockHeight"] = pp.blockHeight; 806 obj["blockHeight"] = pp.blockHeight;
325 obj["blockHash"] = pp.blockHash.ToString(); 807 obj["blockHash"] = pp.blockHash.ToString();
326 obj["matureAtHeight"] = pp.matureAtHeight; 808 obj["matureAtHeight"] = pp.matureAtHeight;
809 // No retryAfter: it was never written with anything but zero, and the
810 // retry clock that is actually used is on PayingRecord. Files that
811 // still carry the key load fine (it is ignored), and a build that
812 // expects it defaults an absent one to 0, which means "ready now" --
813 // the same answer it always got.
327 nlohmann::json amounts = nlohmann::json::object(); 814 nlohmann::json amounts = nlohmann::json::object();
328 for (const auto& kv : pp.amounts) 815 for (const auto& kv : pp.amounts)
329 amounts[kv.first] = kv.second; 816 amounts[kv.first] = kv.second;
330 obj["amounts"] = amounts; 817 obj["amounts"] = amounts;
818 nlohmann::json paid = nlohmann::json::object();
819 for (const auto& kv : pp.paidTxids)
820 paid[kv.first] = kv.second;
821 obj["paidTxids"] = paid;
331 arr.push_back(obj); 822 arr.push_back(obj);
332 } 823 }
333 std::string path = PayoutFilePath(); 824 VersionedBlob v;
334 FILE* f = fopen(path.c_str(), "w"); 825 v.blob = arr.dump(2);
335 if (f) { 826 v.seq = ++gPayoutLedgerSeq;
336 std::string s = arr.dump(2); 827 return v;
337 fwrite(s.c_str(), 1, s.size(), f); 828}
338 fclose(f); 829
339 } else { 830// Publish a ledger snapshot. Holds no data mutex; safe -- and intended -- to
340 LogPrint("payout", "[payout] WARNING: could not write %s\n", path.c_str()); 831// call with gPayoutLedgerMutex released.
341 } 832static void PublishPendingPayouts(const VersionedBlob& v)
833{
834 PublishVersioned(PayoutFilePath(), v, gPayoutLedgerFileMutex,
835 gPayoutLedgerPublishedSeq, "pending payout ledger");
836}
837
838// Parse a persisted block-hash field, refusing to guess. SetHex() reads
839// whatever hex prefix it finds and stops at the first character it does not
840// recognise, so a corrupt value does not fail to parse -- it parses as a
841// DIFFERENT hash. Both shapes of that are wrong in a dangerous direction: a
842// value with a hex-looking prefix ("deadbeef...") becomes a hash no block
843// ever had, which PayoutBlockStillOnChain reads as an orphaned block and
844// FlushMaturePayouts then VOIDS the debt; a value with no hex prefix at all
845// becomes 0, which is grandfathered past the orphan check on the strength
846// of garbage. A field that is not the 64-digit hex string ToString() wrote
847// is corruption, and corruption is a load failure like any other parse
848// error -- the file is quarantined, not read around.
849static uint256 ParsePayoutBlockHash(const std::string& str)
850{
851 if (str.size() != 64 ||
852 str.find_first_not_of("0123456789abcdefABCDEF") != std::string::npos)
853 throw std::runtime_error(
854 strprintf("blockHash \"%s\" is not a 64-digit hex string",
855 str.c_str()));
856 uint256 h;
857 h.SetHex(str);
858 return h;
342} 859}
343 860
344static void LoadPendingPayouts() 861static void LoadPendingPayouts()
@@ -354,6 +871,7 @@ static void LoadPendingPayouts()
354 fclose(f); 871 fclose(f);
355 try { 872 try {
356 nlohmann::json arr = nlohmann::json::parse(s); 873 nlohmann::json arr = nlohmann::json::parse(s);
874 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
357 for (const auto& obj : arr) { 875 for (const auto& obj : arr) {
358 PendingPayout pp; 876 PendingPayout pp;
359 pp.matureAtHeight = obj["matureAtHeight"].get<int>(); 877 pp.matureAtHeight = obj["matureAtHeight"].get<int>();
@@ -361,18 +879,33 @@ static void LoadPendingPayouts()
361 // field. Leave blockHash zero and they are grandfathered past the 879 // field. Leave blockHash zero and they are grandfathered past the
362 // orphan check rather than voided on a guess. 880 // orphan check rather than voided on a guess.
363 if (obj.contains("blockHash") && obj["blockHash"].is_string()) 881 if (obj.contains("blockHash") && obj["blockHash"].is_string())
364 pp.blockHash.SetHex(obj["blockHash"].get<std::string>()); 882 pp.blockHash =
883 ParsePayoutBlockHash(obj["blockHash"].get<std::string>());
365 if (obj.contains("blockHeight") && obj["blockHeight"].is_number()) 884 if (obj.contains("blockHeight") && obj["blockHeight"].is_number())
366 pp.blockHeight = obj["blockHeight"].get<int>(); 885 pp.blockHeight = obj["blockHeight"].get<int>();
886 // Upstream's tolerance for an entry with no usable height, kept
887 // because it costs nothing and an entry that cannot name its block
888 // is worse than one derived from its maturity. COINBASE_MATURITY
889 // and not COINBASE_SPENDABLE_DEPTH: this reverses whatever the
890 // writer of *that* file used, and only files old enough to lack
891 // blockHeight are affected -- they were written when maturity was
892 // still COINBASE_MATURITY.
893 if (pp.blockHeight == 0 && pp.matureAtHeight > COINBASE_MATURITY)
894 pp.blockHeight = pp.matureAtHeight - COINBASE_MATURITY;
895 // "retryAfter" is deliberately not read. Files written by the
896 // merged build carry it, always as 0, and there is nowhere left to
897 // put it -- the retry clock is PayingRecord's.
367 for (auto it = obj["amounts"].begin(); it != obj["amounts"].end(); ++it) 898 for (auto it = obj["amounts"].begin(); it != obj["amounts"].end(); ++it)
368 pp.amounts[it.key()] = it.value().get<int64>(); 899 pp.amounts[it.key()] = it.value().get<int64>();
900 if (obj.contains("paidTxids") && obj["paidTxids"].is_object())
901 for (auto it = obj["paidTxids"].begin(); it != obj["paidTxids"].end(); ++it)
902 pp.paidTxids[it.key()] = it.value().get<std::string>();
369 gPendingPayouts.push_back(pp); 903 gPendingPayouts.push_back(pp);
370 } 904 }
371 LogPrint("payout", "[payout] loaded %zu pending payout(s) from disk\n", 905 LogPrint("payout", "[payout] loaded %zu pending payout(s) from disk\n",
372 gPendingPayouts.size()); 906 gPendingPayouts.size());
373 } catch (const std::exception& e) { 907 } catch (const std::exception& e) {
374 LogPrint("payout", "[payout] WARNING: failed to parse %s: %s\n", 908 RefuseFileAfterLoadFailure(path, "pending payout ledger", e.what());
375 path.c_str(), e.what());
376 } 909 }
377} 910}
378 911
@@ -388,6 +921,18 @@ static void LoadPendingPayouts()
388// and reconciling against the wallet on startup, is what makes a restart 921// and reconciling against the wallet on startup, is what makes a restart
389// survivable. 922// survivable.
390struct PayingRecord { 923struct PayingRecord {
924 // Identity of this SLOT, distinct from the identity of the payment it
925 // describes. ForgetPayment and ReschedulePayment used to find their record
926 // by comparing every field, which is only an identity while no two records
927 // can be equal -- and a restart used to produce byte-identical duplicates
928 // (see FlushMaturePayouts), so the first match absorbed every update and
929 // the second was never touched again.
930 //
931 // Process-local and deliberately not persisted: gPaying is rebuilt from
932 // disk at every start, so an id only ever has to name a slot within one
933 // run, and reading ids back would mean trusting a file to contain no
934 // duplicates. Assigned under gPayoutExecMutex from gPayingNextId.
935 uint64 id = 0;
391 int blockHeight = 0; 936 int blockHeight = 0;
392 uint256 blockHash = 0; // zero = predates hash keying, see PendingPayout 937 uint256 blockHash = 0; // zero = predates hash keying, see PendingPayout
393 int matureAtHeight = 0; 938 int matureAtHeight = 0;
@@ -396,18 +941,66 @@ struct PayingRecord {
396 int retries = 0; 941 int retries = 0;
397 int deferrals = 0; 942 int deferrals = 0;
398 int64 retryAfter = 0; 943 int64 retryAfter = 0;
944 // Ask the wallet whether this payment already went out, once, before the
945 // first send attempt. Upstream asks on every attempt: FindRecordedPoolPayout
946 // is a linear scan of mapWallet under cs_mapWallet, per recipient, so with
947 // 2000 rotating payout addresses and a retry budget of 1440 that is a
948 // scan per address per minute for a day. The question it answers -- "did a
949 // previous process send this before it died?" -- can only change across a
950 // restart, and every restart re-sets this flag (ReconcilePayouts), so
951 // asking once per record is the whole of the answer.
952 //
953 // Not persisted: a record read back from disk is by definition a restart.
954 bool fProbeWallet = true;
399}; 955};
400static std::vector<PayingRecord> gPaying; // guarded by gPayoutExecMutex 956static std::vector<PayingRecord> gPaying; // guarded by gPayoutExecMutex
401static std::mutex gPayoutExecMutex; 957static std::mutex gPayoutExecMutex;
958static uint64 gPayingNextId = 0; // guarded by gPayoutExecMutex
959static uint64 gPayingSeq = 0; // guarded by gPayoutExecMutex
960static std::mutex gPayingFileMutex;
961static uint64 gPayingPublishedSeq = 0; // guarded by gPayingFileMutex
402static std::atomic<bool> gPayoutThreadRunning{false}; 962static std::atomic<bool> gPayoutThreadRunning{false};
403 963
964// Upstream's gPayoutExecQueue/gPayoutExecMutex/gPayoutThreadRunning are
965// declared above with gPaying: the in-flight list IS the exec queue in this
966// tree, at per-recipient rather than per-round granularity, so a second queue
967// would be a second answer to "who is owed money". Only the dedupe set is new.
968//
969// Ids of rounds currently exploded into gPaying, so a flush cannot queue the
970// same round twice. Guarded by gPayoutExecMutex. Seeded from the restored
971// in-flight list by ReconcilePayouts -- it is memory-only, and leaving it
972// empty at startup was half of why a restart duplicated everything in flight.
973static std::set<std::string> gQueuedPayouts;
974
975// Is this exact payment already in flight? gPaying is keyed on
976// (round, address): the ledger holds one amount per address per round, so two
977// records for that pair are two standing orders to send the same money.
978// Caller must hold gPayoutExecMutex.
979static bool PaymentInFlightLocked(int blockHeight, const uint256& blockHash,
980 const std::string& address)
981{
982 for (const PayingRecord& e : gPaying)
983 if (e.address == address
984 && SamePayoutRound(e.blockHeight, e.blockHash, blockHeight, blockHash))
985 return true;
986 return false;
987}
988
404static std::string PayingFilePath() 989static std::string PayingFilePath()
405{ 990{
406 return GetAppDir() + "/paying_payouts.json"; 991 return GetAppDir() + "/paying_payouts.json";
407} 992}
408 993
409// Caller must hold gPayoutExecMutex. 994// Serialise the in-flight list. Caller must hold gPayoutExecMutex.
410static void SavePayingLocked() 995//
996// gPayoutExecMutex already serialises every writer of this file, so it never
997// had the interleaving race pending_payouts.json had. It is still split the
998// same way for a different reason: FlushMaturePayouts runs on the event loop
999// and holds gPayoutExecMutex, so writing (and, since this series, fsyncing)
1000// the file under that mutex parks miner I/O behind the payout thread's disk.
1001// The version stamp keeps the file monotonic now that the write is outside
1002// the mutex that orders the writers.
1003static VersionedBlob SerializePayingLocked()
411{ 1004{
412 nlohmann::json arr = nlohmann::json::array(); 1005 nlohmann::json arr = nlohmann::json::array();
413 for (const auto& r : gPaying) { 1006 for (const auto& r : gPaying) {
@@ -419,24 +1012,24 @@ static void SavePayingLocked()
419 obj["amount"] = r.amount; 1012 obj["amount"] = r.amount;
420 obj["retries"] = r.retries; 1013 obj["retries"] = r.retries;
421 obj["deferrals"] = r.deferrals; 1014 obj["deferrals"] = r.deferrals;
1015 // The live retry clock, which used not to be written at all: every
1016 // restart handed every backing-off payment a retryAfter of 0 and put
1017 // the whole set into the very next 500 ms batch, so a node restarted
1018 // in a loop retried continuously instead of once a minute.
1019 obj["retryAfter"] = r.retryAfter;
422 arr.push_back(obj); 1020 arr.push_back(obj);
423 } 1021 }
424 std::string path = PayingFilePath(); 1022 VersionedBlob v;
425 // Write to a temporary file and rename, so a crash mid-write cannot leave 1023 v.blob = arr.dump(2);
426 // a truncated list of who is still owed money. 1024 v.seq = ++gPayingSeq;
427 std::string tmp = path + ".tmp"; 1025 return v;
428 FILE* f = fopen(tmp.c_str(), "w"); 1026}
429 if (!f) { 1027
430 LogPrint("payout", "[payout] WARNING: could not write %s\n", tmp.c_str()); 1028// Publish an in-flight-list snapshot. Call with gPayoutExecMutex released.
431 return; 1029static void PublishPaying(const VersionedBlob& v)
432 } 1030{
433 std::string s = arr.dump(2); 1031 PublishVersioned(PayingFilePath(), v, gPayingFileMutex,
434 fwrite(s.c_str(), 1, s.size(), f); 1032 gPayingPublishedSeq, "in-flight payment list");
435 fflush(f);
436 fclose(f);
437 if (rename(tmp.c_str(), path.c_str()) != 0)
438 LogPrint("payout", "[payout] WARNING: could not rename %s -> %s\n",
439 tmp.c_str(), path.c_str());
440} 1033}
441 1034
442static void LoadPaying() 1035static void LoadPaying()
@@ -452,26 +1045,209 @@ static void LoadPaying()
452 fclose(f); 1045 fclose(f);
453 try { 1046 try {
454 nlohmann::json arr = nlohmann::json::parse(s); 1047 nlohmann::json arr = nlohmann::json::parse(s);
1048 // gPaying and gPayingNextId are both annotated as guarded by
1049 // gPayoutExecMutex and this was mutating them without it. Safe today
1050 // -- LoadPaying only runs at startup, before the payout thread is
1051 // spawned -- but "safe because of where it is called from" is a fact
1052 // about the caller, not about this function, and the annotation is
1053 // what every other reader will trust. The mutex is uncontended here by
1054 // construction, so taking it costs nothing.
1055 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
455 for (const auto& obj : arr) { 1056 for (const auto& obj : arr) {
456 PayingRecord r; 1057 PayingRecord r;
457 r.address = obj["address"].get<std::string>(); 1058 r.address = obj["address"].get<std::string>();
458 r.amount = obj["amount"].get<int64>(); 1059 r.amount = obj["amount"].get<int64>();
459 r.matureAtHeight = obj["matureAtHeight"].get<int>(); 1060 r.matureAtHeight = obj["matureAtHeight"].get<int>();
460 if (obj.contains("blockHash") && obj["blockHash"].is_string()) 1061 if (obj.contains("blockHash") && obj["blockHash"].is_string())
461 r.blockHash.SetHex(obj["blockHash"].get<std::string>()); 1062 r.blockHash =
1063 ParsePayoutBlockHash(obj["blockHash"].get<std::string>());
462 if (obj.contains("blockHeight") && obj["blockHeight"].is_number()) 1064 if (obj.contains("blockHeight") && obj["blockHeight"].is_number())
463 r.blockHeight = obj["blockHeight"].get<int>(); 1065 r.blockHeight = obj["blockHeight"].get<int>();
464 if (obj.contains("retries") && obj["retries"].is_number()) 1066 if (obj.contains("retries") && obj["retries"].is_number())
465 r.retries = obj["retries"].get<int>(); 1067 r.retries = obj["retries"].get<int>();
466 if (obj.contains("deferrals") && obj["deferrals"].is_number()) 1068 if (obj.contains("deferrals") && obj["deferrals"].is_number())
467 r.deferrals = obj["deferrals"].get<int>(); 1069 r.deferrals = obj["deferrals"].get<int>();
1070 // Absent in files written before this change; 0 means ready now,
1071 // which is exactly the behaviour those files used to get.
1072 if (obj.contains("retryAfter") && obj["retryAfter"].is_number())
1073 r.retryAfter = obj["retryAfter"].get<int64>();
1074 // Collapse duplicates as they are read. A build without the
1075 // ReconcilePayouts seeding below wrote a second (and after another
1076 // restart, a third) byte-identical record for every unpaid
1077 // recipient, so a file written by one of those has to be cleaned
1078 // up here or the pathology is inherited. The retry and deferral
1079 // counts are merged upwards so collapsing does not hand the
1080 // survivor a fresh budget.
1081 PayingRecord* dup = NULL;
1082 for (PayingRecord& e : gPaying)
1083 if (e.address == r.address
1084 && SamePayoutRound(e.blockHeight, e.blockHash,
1085 r.blockHeight, r.blockHash)) {
1086 dup = &e;
1087 break;
1088 }
1089 if (dup) {
1090 dup->retries = std::max(dup->retries, r.retries);
1091 dup->deferrals = std::max(dup->deferrals, r.deferrals);
1092 dup->retryAfter = std::max(dup->retryAfter, r.retryAfter);
1093 LogPrint("payout", "[payout] collapsing duplicate in-flight"
1094 " record for %s to %s -- one payment, one record\n",
1095 FormatMoney(r.amount).c_str(), r.address.c_str());
1096 continue;
1097 }
1098 r.id = ++gPayingNextId;
468 gPaying.push_back(r); 1099 gPaying.push_back(r);
469 } 1100 }
470 LogPrint("payout", "[payout] loaded %zu in-flight payment(s) from disk\n", 1101 LogPrint("payout", "[payout] loaded %zu in-flight payment(s) from disk\n",
471 gPaying.size()); 1102 gPaying.size());
472 } catch (const std::exception& e) { 1103 } catch (const std::exception& e) {
473 LogPrint("payout", "[payout] WARNING: failed to parse %s: %s\n", 1104 RefuseFileAfterLoadFailure(path, "in-flight payment list", e.what());
474 path.c_str(), e.what()); 1105 }
1106}
1107
1108// ---------------------------------------------------------------------------
1109// Pool round proofs (upstream cbdd06d). Disjoint from the in-flight ledger
1110// above -- this is the operator's published record of how a round was split,
1111// not an obligation to pay. Both sides added code at this offset; both are
1112// kept.
1113// ---------------------------------------------------------------------------
1114static nlohmann::json PoolRoundToJson(const PoolRoundProof& r)
1115{
1116 nlohmann::json j;
1117 j["id"] = r.id;
1118 j["blockHeight"] = r.blockHeight;
1119 j["matureAtHeight"] = r.matureAtHeight;
1120 j["createdAt"] = r.createdAt;
1121 j["blockHash"] = r.blockHash;
1122 j["coinbaseTxid"] = r.coinbaseTxid;
1123 j["foundBy"] = r.foundBy;
1124 j["feePercent"] = r.feePercent;
1125 j["totalShares"] = r.totalShares;
1126 j["coinbaseValueSatoshis"] = r.coinbaseValue;
1127 j["coinbaseValueBtf"] = FormatMoney(r.coinbaseValue);
1128 j["subsidySatoshis"] = r.subsidy;
1129 j["subsidyBtf"] = FormatMoney(r.subsidy);
1130 j["minerRewardSatoshis"] = r.minerReward;
1131 j["minerRewardBtf"] = FormatMoney(r.minerReward);
1132 j["operatorRewardSatoshis"] = r.operatorReward;
1133 j["operatorRewardBtf"] = FormatMoney(r.operatorReward);
1134 j["skippedDust"] = r.skippedDust;
1135 j["forfeitedBadAddressSatoshis"] = r.forfeitedBadAddress;
1136 j["forfeitedBadAddressBtf"] = FormatMoney(r.forfeitedBadAddress);
1137
1138 j["shares"] = nlohmann::json::object();
1139 for (const auto& kv : r.shares)
1140 j["shares"][kv.first] = kv.second;
1141
1142 j["payouts"] = nlohmann::json::object();
1143 for (const auto& kv : r.payouts) {
1144 nlohmann::json p;
1145 p["amountSatoshis"] = kv.second;
1146 p["amountBtf"] = FormatMoney(kv.second);
1147 std::map<std::string, std::string>::const_iterator tx = r.paidTxids.find(kv.first);
1148 if (tx != r.paidTxids.end())
1149 p["txid"] = tx->second;
1150 else
1151 p["txid"] = nullptr;
1152 // Only emitted when it happened, so the common shape of the file is
1153 // unchanged for anything already parsing it.
1154 std::map<std::string, std::string>::const_iterator fo = r.forfeited.find(kv.first);
1155 if (fo != r.forfeited.end()) {
1156 p["forfeited"] = true;
1157 p["forfeitReason"] = fo->second;
1158 }
1159 j["payouts"][kv.first] = p;
1160 }
1161 return j;
1162}
1163
1164// Serialise the published round proofs. Caller must hold gPoolRoundsMutex.
1165//
1166// Like the in-flight list, gPoolRoundsMutex already serialises the writers
1167// (RecordPoolRound on the event loop, MarkPoolRoundPaid on the payout thread),
1168// so this file never had the interleaving race either -- but the event loop
1169// should not be waiting on the payout thread's fsync to publish a round, so
1170// the write moves outside the mutex on the same versioned terms.
1171static VersionedBlob SerializePoolRoundsLocked()
1172{
1173 nlohmann::json root;
1174 root["schema"] = "bitflash-pool-rounds-1";
1175 root["updatedAt"] = GetTime();
1176 root["rounds"] = nlohmann::json::array();
1177 for (const PoolRoundProof& r : gPoolRounds)
1178 root["rounds"].push_back(PoolRoundToJson(r));
1179
1180 VersionedBlob v;
1181 v.blob = root.dump(2);
1182 v.seq = ++gPoolRoundsSeq;
1183 return v;
1184}
1185
1186// Publish a round-proof snapshot. Call with gPoolRoundsMutex released.
1187static void PublishPoolRounds(const VersionedBlob& v)
1188{
1189 PublishVersioned(PoolRoundsFilePath(), v, gPoolRoundsFileMutex,
1190 gPoolRoundsPublishedSeq, "pool round proofs");
1191}
1192
1193static void LoadPoolRounds()
1194{
1195 std::string path = PoolRoundsFilePath();
1196 FILE* f = fopen(path.c_str(), "r");
1197 if (!f) return;
1198 fseek(f, 0, SEEK_END);
1199 long sz = ftell(f);
1200 fseek(f, 0, SEEK_SET);
1201 std::string s(sz, '\0');
1202 fread(&s[0], 1, sz, f);
1203 fclose(f);
1204
1205 try {
1206 nlohmann::json root = nlohmann::json::parse(s);
1207 nlohmann::json arr = root.is_array() ? root : root["rounds"];
1208 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1209 for (const auto& obj : arr) {
1210 PoolRoundProof r;
1211 if (obj.contains("id") && obj["id"].is_string()) r.id = obj["id"].get<std::string>();
1212 if (obj.contains("blockHeight")) r.blockHeight = obj["blockHeight"].get<int>();
1213 if (obj.contains("matureAtHeight")) r.matureAtHeight = obj["matureAtHeight"].get<int>();
1214 if (obj.contains("createdAt")) r.createdAt = obj["createdAt"].get<int64>();
1215 if (obj.contains("blockHash") && obj["blockHash"].is_string()) r.blockHash = obj["blockHash"].get<std::string>();
1216 if (obj.contains("coinbaseTxid") && obj["coinbaseTxid"].is_string()) r.coinbaseTxid = obj["coinbaseTxid"].get<std::string>();
1217 if (obj.contains("foundBy") && obj["foundBy"].is_string()) r.foundBy = obj["foundBy"].get<std::string>();
1218 if (obj.contains("feePercent")) r.feePercent = obj["feePercent"].get<double>();
1219 if (obj.contains("totalShares")) r.totalShares = obj["totalShares"].get<uint64>();
1220 if (obj.contains("coinbaseValueSatoshis")) r.coinbaseValue = obj["coinbaseValueSatoshis"].get<int64>();
1221 if (obj.contains("subsidySatoshis")) r.subsidy = obj["subsidySatoshis"].get<int64>();
1222 if (obj.contains("minerRewardSatoshis")) r.minerReward = obj["minerRewardSatoshis"].get<int64>();
1223 if (obj.contains("operatorRewardSatoshis")) r.operatorReward = obj["operatorRewardSatoshis"].get<int64>();
1224 if (obj.contains("skippedDust")) r.skippedDust = obj["skippedDust"].get<int>();
1225 if (obj.contains("forfeitedBadAddressSatoshis")) r.forfeitedBadAddress = obj["forfeitedBadAddressSatoshis"].get<int64>();
1226 if (obj.contains("shares") && obj["shares"].is_object())
1227 for (auto it = obj["shares"].begin(); it != obj["shares"].end(); ++it)
1228 r.shares[it.key()] = it.value().get<uint64>();
1229 if (obj.contains("payouts") && obj["payouts"].is_object())
1230 for (auto it = obj["payouts"].begin(); it != obj["payouts"].end(); ++it) {
1231 if (it.value().is_object()) {
1232 r.payouts[it.key()] = it.value().value("amountSatoshis", (int64)0);
1233 if (it.value().contains("txid") && it.value()["txid"].is_string())
1234 r.paidTxids[it.key()] = it.value()["txid"].get<std::string>();
1235 if (it.value().value("forfeited", false))
1236 r.forfeited[it.key()] =
1237 it.value().value("forfeitReason", std::string("unspecified"));
1238 } else if (it.value().is_number_integer()) {
1239 r.payouts[it.key()] = it.value().get<int64>();
1240 }
1241 }
1242 if (!r.id.empty())
1243 gPoolRounds.push_back(r);
1244 }
1245 while (gPoolRounds.size() > MAX_POOL_ROUNDS)
1246 gPoolRounds.erase(gPoolRounds.begin());
1247 LogPrint("payout", "[payout] loaded %zu pool round proof(s) from disk\n",
1248 gPoolRounds.size());
1249 } catch (const std::exception& e) {
1250 RefuseFileAfterLoadFailure(path, "pool round proofs", e.what());
475 } 1251 }
476} 1252}
477 1253
@@ -494,105 +1270,487 @@ static bool PayoutBlockStillOnChain(const uint256& blockHash)
494 return false; 1270 return false;
495} 1271}
496 1272
497static std::string PayoutComment(int blockHeight) 1273// The comment every pre-upgrade payout transaction carries, and therefore the
1274// ONLY needle the legacy fallback in FindRecordedPoolPayout may search with.
1275// This format is frozen: change it and every payment those builds made stops
1276// matching, every one of those debts looks unpaid, and reconciliation pays
1277// them all a second time.
1278static std::string LegacyPayoutComment(int blockHeight)
498{ 1279{
499 return strprintf("Pool payout height %d", blockHeight); 1280 return strprintf("Pool payout height %d", blockHeight);
500} 1281}
501 1282
502// Did we already send this payment before dying? Looks for a wallet 1283// The comment written on new sends. It names the funding block, because the
503// transaction of ours carrying this payout's comment and paying exactly this 1284// height alone cannot: a height that is won, lost to a reorg, and won again
504// amount to this address. 1285// yields two rounds whose height-only comments are identical -- and identical
1286// amounts too, whenever the same miner held both rounds. The structured
1287// pool_payout_* metadata already keys the payment by hash for the code; this
1288// puts the same fact where an operator reading the wallet can see it, and
1289// makes a new transaction unmatchable by the legacy needle even if the
1290// structured keys are ever lost.
505// 1291//
506// Biased towards reporting "already paid" on any doubt: skipping a payment a 1292// A zero hash keeps the legacy form: those records predate hash keying, and
507// miner is owed is visible and recoverable from the log, while paying twice 1293// there is no hash to name.
508// is neither. 1294static std::string PayoutComment(int blockHeight, const uint256& blockHash)
509static bool WalletAlreadyPaid(const PayingRecord& r)
510{ 1295{
511 uint160 h160; 1296 if (blockHash == 0)
512 if (!AddressToHash160(r.address, h160)) 1297 return LegacyPayoutComment(blockHeight);
513 return false; 1298 return strprintf("Pool payout height %d block %s", blockHeight,
514 CScript want; 1299 blockHash.ToString().c_str());
515 want << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; 1300}
516 const std::string comment = PayoutComment(r.blockHeight);
517 1301
518 CRITICAL_BLOCK(cs_mapWallet) 1302// Startup reconciliation.
1303//
1304// This used to be the crash-recovery mechanism: it scanned the wallet with
1305// WalletAlreadyPaid() and then dropped ledger entries that had already been
1306// exploded into in-flight records. Neither job is right any more.
1307//
1308// The ledger is now authoritative and an entry survives until every recipient
1309// has a txid recorded against it (upstream cbdd06d), so dropping ledger
1310// entries here would reintroduce exactly the loss this merge is adopting that
1311// model to prevent. And the wallet scan is now FindRecordedPoolPayout(), done
1312// once per record on its first attempt, which is the same question asked in
1313// the one place that can act on the answer. Running both would be two
1314// authorities for the same debt.
1315//
1316// What is left is the useful part: the in-flight list restored from disk may
1317// contain records the ledger already shows as paid (we died between the send
1318// and the in-flight file being rewritten). The ledger wins.
1319//
1320// And the part that was missing: gQueuedPayouts is memory-only, so it came up
1321// empty while gPaying came up full. FlushMaturePayouts deduped against that
1322// empty set and against the ledger's paidTxids, and against nothing else --
1323// so a round that was exploded but not yet paid got a SECOND record per
1324// unpaid recipient on the first flush after every restart, deterministically.
1325// (Observed live: two byte-identical records for one (block, address).) The
1326// surviving in-flight list is exactly the statement of which rounds are
1327// already exploded, so it is what the set is rebuilt from.
1328static void ReconcilePayouts()
1329{
1330 VersionedBlob payingSnap;
519 { 1331 {
520 for (std::map<uint256, CWalletTx>::iterator it = mapWallet.begin(); 1332 std::lock_guard<std::mutex> exec(gPayoutExecMutex);
521 it != mapWallet.end(); ++it) { 1333 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
522 CWalletTx& wtx = it->second; 1334
523 // The comment is written only by the payout path, so its presence 1335 size_t settled = 0;
524 // already identifies the transaction as one of ours. 1336 int64 settledAmount = 0;
525 std::map<std::string, std::string>::const_iterator ci = 1337 for (auto it = gPaying.begin(); it != gPaying.end(); ) {
526 wtx.mapValue.find("comment"); 1338 bool paid = false;
527 if (ci == wtx.mapValue.end() || ci->second != comment) 1339 for (const PendingPayout& pp : gPendingPayouts)
1340 if (SamePayoutRound(pp.blockHeight, pp.blockHash,
1341 it->blockHeight, it->blockHash)
1342 && pp.paidTxids.count(it->address)) {
1343 paid = true;
1344 break;
1345 }
1346 if (paid) {
1347 settledAmount += it->amount;
1348 settled++;
1349 it = gPaying.erase(it);
1350 } else {
1351 // Every restored record gets one wallet probe before it is
1352 // allowed to send: this is precisely the window where the
1353 // wallet may hold a transaction the ledger has not recorded.
1354 it->fProbeWallet = true;
1355 ++it;
1356 }
1357 }
1358 if (settled) {
1359 LogPrint("payout", "[payout] %zu in-flight payment(s) totalling %s"
1360 " are already recorded paid in the ledger -- not"
1361 " resending\n",
1362 settled, FormatMoney(settledAmount).c_str());
1363 }
1364
1365 // Rebuild the "already exploded" set from what survived. Without this
1366 // the first flush after a restart re-explodes every round in flight.
1367 gQueuedPayouts.clear();
1368 for (const PayingRecord& r : gPaying)
1369 gQueuedPayouts.insert(PayoutIdForBlock(r.blockHeight, r.blockHash));
1370
1371 payingSnap = SerializePayingLocked();
1372
1373 if (!gPaying.empty()) {
1374 int64 owed = 0;
1375 for (const auto& r : gPaying) owed += r.amount;
1376 LogPrint("payout", "[payout] resuming %zu in-flight payment(s)"
1377 " totalling %s\n",
1378 gPaying.size(), FormatMoney(owed).c_str());
1379 }
1380 }
1381 PublishPaying(payingSnap);
1382}
1383
1384static void RecordPoolRound(const PoolRoundProof& round)
1385{
1386 VersionedBlob snap;
1387 {
1388 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1389 bool replaced = false;
1390 for (PoolRoundProof& existing : gPoolRounds) {
1391 if (existing.id == round.id) {
1392 existing = round;
1393 replaced = true;
1394 break;
1395 }
1396 }
1397 if (!replaced) {
1398 gPoolRounds.push_back(round);
1399 while (gPoolRounds.size() > MAX_POOL_ROUNDS)
1400 gPoolRounds.erase(gPoolRounds.begin());
1401 }
1402 snap = SerializePoolRoundsLocked();
1403 }
1404 PublishPoolRounds(snap);
1405}
1406
1407static void MarkPoolRoundPaid(const std::string& id,
1408 const std::string& address,
1409 const std::string& txid)
1410{
1411 VersionedBlob snap;
1412 {
1413 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1414 bool found = false;
1415 for (PoolRoundProof& r : gPoolRounds) {
1416 if (r.id != id)
528 continue; 1417 continue;
529 foreach (const CTxOut& txout, wtx.vout) 1418 r.paidTxids[address] = txid;
530 if (txout.nValue == r.amount && txout.scriptPubKey == want) 1419 found = true;
531 return true; 1420 break;
1421 }
1422 if (!found)
1423 return;
1424 snap = SerializePoolRoundsLocked();
1425 }
1426 PublishPoolRounds(snap);
1427}
1428
1429// Publish the fact that a recipient was written off, so the round proof and
1430// the ledger agree. The ledger erases the amount (it is not owed any more and
1431// must not be reported as sent); the proof keeps the amount visible and marks
1432// it forfeited with a reason, because a miner who was promised a share is
1433// entitled to see what happened to it.
1434static void MarkPoolRoundForfeited(const std::string& id,
1435 const std::string& address,
1436 const std::string& reason)
1437{
1438 VersionedBlob snap;
1439 {
1440 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1441 bool found = false;
1442 for (PoolRoundProof& r : gPoolRounds) {
1443 if (r.id != id)
1444 continue;
1445 r.forfeited[address] = reason;
1446 found = true;
1447 break;
532 } 1448 }
1449 if (!found)
1450 return;
1451 snap = SerializePoolRoundsLocked();
533 } 1452 }
1453 PublishPoolRounds(snap);
1454}
1455
1456// Does this round have a published round proof?
1457//
1458// Existence answers a temporal question the legacy fallback below needs:
1459// which builds could have paid this round. RecordPoolRound writes the proof
1460// when the round is created, and structured payout metadata (upstream
1461// cbdd06d) is an ancestor of round proofs (upstream 83ba822) -- no build
1462// able to write a proof has ever sent a payment that only a comment can
1463// identify. A round with a proof therefore cannot have a comment-only
1464// payment; one that matches it is another round's, colliding on the height.
1465static bool PoolRoundHasProof(const std::string& id)
1466{
1467 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
1468 for (const PoolRoundProof& r : gPoolRounds)
1469 if (r.id == id)
1470 return true;
534 return false; 1471 return false;
535} 1472}
536 1473
537// Startup reconciliation. Drops in-flight records the wallet shows were 1474// Which wallet transaction, if any, already paid this?
538// already sent, and drops ledger entries that were already exploded into 1475//
539// records (which happens if we died between writing the records and rewriting 1476// Two kinds of evidence, in order of quality.
540// the ledger -- both files then describe the same debt). 1477//
541static void ReconcilePayouts() 1478// Structured (pool_payout_id / pool_payout_address / pool_payout_amount,
1479// plus pool_payout_block since this series) is upstream's, extended. The id
1480// is derived from the block hash, but only from the FIRST sixteen hex digits
1481// of it -- and proof-of-work concentrates zeros exactly there, so the id can
1482// name two same-height rounds identically. New sends therefore record the
1483// full hash as pool_payout_block, and a structured match must agree with it
1484// wherever both sides have one; a transaction without the key was sent
1485// before it existed, and for those the id is the best evidence there is.
1486// Every transaction this build sends carries all four keys.
1487//
1488// But NO transaction this pool has ever sent carries it. Every payout in the
1489// existing wallet has only a "comment". So on the first boot after this
1490// upgrade -- precisely the boot where the binary is swapped after an unclean
1491// shutdown with payments in flight -- a structured-only probe finds nothing
1492// and every one of those recipients is paid a second time. The deleted
1493// WalletAlreadyPaid() matched on what the payment WAS, and that evidence is
1494// still there; it just has to be read.
1495//
1496// The legacy fallback is therefore restored, with three constraints that
1497// together stop it answering for a round that is not this one:
1498//
1499// 1. it is only consulted when the structured lookup found nothing;
1500// 2. it ignores any transaction carrying pool_payout_id. That key means this
1501// build wrote the transaction, and this build's transactions are
1502// adjudicated by the exact path -- so a NEW round's payment can never be
1503// offered as evidence for an old one. Only pre-upgrade transactions,
1504// which is exactly the population the fallback exists for, are eligible;
1505// 3. `excluded` holds every txid already credited to some OTHER
1506// (round, address) pair -- from the ledger's paidTxids, which is durable,
1507// and from what this process has already claimed. One legacy transaction
1508// can therefore settle at most one debt, so N transactions can never
1509// satisfy more than N obligations;
1510//
1511// 4. it is not consulted at all for a round that has a round proof
1512// (fAllowLegacy, decided by the caller from PoolRoundHasProof). Proofs
1513// are written when a round is created, and only by builds whose every
1514// payment names its round exactly -- so a comment-only transaction
1515// cannot be a payment for such a round, only a same-height collision
1516// from an older one, and skipping the fallback cannot hide a real
1517// payment: the structured probe above already found any that exist.
1518//
1519// What is left unresolved by construction is a comment-only transaction and
1520// a PROOFLESS round at the same height paying the same address the same
1521// amount: two grandfathered zero-hash twins, or a twin pair from the builds
1522// that keyed the ledger by hash before any metadata reached the wallet. The
1523// comment carries only the height, so nothing ties the transaction to one
1524// twin rather than the other. That needs a reorg.
1525//
1526// This used to say the losing twin's ledger entry is voided by the orphan
1527// check before it can be paid. That is wrong: PayoutBlockStillOnChain()
1528// returns true for a zero hash, and EVERY pre-upgrade entry has a zero hash,
1529// so the orphan check is a no-op for exactly the population this fallback
1530// serves.
1531//
1532// The outcome is still safe, by a different mechanism. SamePayoutRound()
1533// treats two zero-hash entries at the same height as the SAME round, so the
1534// pair is indistinguishable everywhere, not only here: FlushMaturePayouts
1535// explodes one of them and skips the other as already queued,
1536// PaymentInFlightLocked refuses a second record for that (round, address), and
1537// RecordPayoutTxid settles against the first match. The pair therefore
1538// presents as ONE debt and can be paid at most once -- which is the direction
1539// that matters for a fallback whose failure mode is treating one payment as
1540// proof of two.
1541//
1542// The cost of that mechanism, stated plainly: only one of a genuine
1543// same-height pair would ever be paid. It takes a reorg AND both rounds
1544// predating hash keying, and no file this tree has written since can produce
1545// one, so it is a property of history rather than a live risk.
1546//
1547// Where the evidence genuinely runs out the fallback simply does not match,
1548// and the payment is made -- paying a debt we cannot prove was settled is the
1549// correct default for a pool.
1550enum PayoutProbeResult {
1551 PAYOUT_PROBE_NONE = 0, // the wallet holds no record of this payment
1552 PAYOUT_PROBE_STRUCTURED, // matched this build's pool_payout_* metadata
1553 PAYOUT_PROBE_LEGACY // matched a pre-upgrade comment and output
1554};
1555
1556static PayoutProbeResult FindRecordedPoolPayout(const std::string& id,
1557 int blockHeight,
1558 const uint256& blockHash,
1559 const std::string& address,
1560 int64 amount,
1561 bool fAllowLegacy,
1562 const std::set<std::string>& excluded,
1563 std::string& txidRet)
542{ 1564{
543 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 1565 // Built before cs_mapWallet is taken: an invalid address has no script to
1566 // compare against, and the caller has already rejected those.
1567 uint160 h160;
1568 bool fHaveScript = AddressToHash160(address, h160);
1569 CScript want;
1570 if (fHaveScript)
1571 want << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
1572 const std::string comment = LegacyPayoutComment(blockHeight);
1573
1574 // Candidate legacy matches, adjudicated after cs_mapWallet is released.
1575 //
1576 // The depth check cannot happen inside the scan: GetDepthInMainChain()
1577 // reads mapBlockIndex and pindexBest, which belong to cs_main, and cs_main
1578 // is taken BEFORE cs_mapWallet everywhere else in this tree
1579 // (CreateTransaction, ConnectBlock). Taking it the other way round here
1580 // would be a lock-order inversion in the payout thread. There is at most a
1581 // handful of these: a candidate has to carry this exact height's comment
1582 // and an output of exactly this amount to this address.
1583 std::vector<uint256> legacyCandidates;
1584
1585 // One pass, not two: this scan runs under cs_mapWallet, which the whole
1586 // node contends on, and the merge is already paying for it once per record.
1587 CRITICAL_BLOCK(cs_mapWallet)
1588 {
1589 for (map<uint256, CWalletTx>::const_iterator it = mapWallet.begin();
1590 it != mapWallet.end(); ++it)
1591 {
1592 const CWalletTx& wtx = it->second;
1593 map<string, string>::const_iterator idIt = wtx.mapValue.find("pool_payout_id");
1594 if (idIt != wtx.mapValue.end()) {
1595 // Written by this build. Adjudicated exactly, and never
1596 // offered to the fallback below.
1597 if (idIt->second != id)
1598 continue;
1599 map<string, string>::const_iterator addrIt = wtx.mapValue.find("pool_payout_address");
1600 if (addrIt == wtx.mapValue.end() || addrIt->second != address)
1601 continue;
1602 map<string, string>::const_iterator amtIt = wtx.mapValue.find("pool_payout_amount");
1603 if (amtIt == wtx.mapValue.end() || amtIt->second != strprintf("%lld", amount))
1604 continue;
1605 // The id alone cannot tell twin rounds at one height apart
1606 // (see above); the full hash can, where the transaction
1607 // recorded one. Where it did not, the id decides, as it
1608 // always has -- refusing those transactions would make every
1609 // payment sent before the key existed look unpaid and pay it
1610 // again.
1611 if (blockHash != 0) {
1612 map<string, string>::const_iterator blkIt =
1613 wtx.mapValue.find("pool_payout_block");
1614 if (blkIt != wtx.mapValue.end()
1615 && blkIt->second != blockHash.ToString())
1616 continue; // a twin round's payment, not this round's
1617 }
1618 txidRet = wtx.GetHash().GetHex();
1619 return PAYOUT_PROBE_STRUCTURED;
1620 }
544 1621
545 size_t settled = 0; 1622 if (!fAllowLegacy)
546 int64 settledAmount = 0; 1623 continue; // comment-only evidence is inadmissible: rule 4
547 for (auto it = gPaying.begin(); it != gPaying.end(); ) { 1624 if (!fHaveScript)
548 if (WalletAlreadyPaid(*it)) { 1625 continue;
549 settledAmount += it->amount; 1626 // The comment is written only by the payout path, so its presence
550 settled++; 1627 // already identifies the transaction as one of ours.
551 it = gPaying.erase(it); 1628 map<string, string>::const_iterator ci = wtx.mapValue.find("comment");
552 } else { 1629 if (ci == wtx.mapValue.end() || ci->second != comment)
553 ++it; 1630 continue;
1631 if (excluded.count(wtx.GetHash().GetHex()))
1632 continue; // already credited to a different debt
1633 foreach (const CTxOut& txout, wtx.vout)
1634 if (txout.nValue == amount && txout.scriptPubKey == want) {
1635 legacyCandidates.push_back(it->first);
1636 break;
1637 }
554 } 1638 }
555 } 1639 }
556 if (settled) {
557 LogPrint("payout", "[payout] %zu in-flight payment(s) totalling %s were"
558 " already sent before the last shutdown -- not resending\n",
559 settled, FormatMoney(settledAmount).c_str());
560 SavePayingLocked();
561 }
562 1640
563 size_t dropped = 0; 1641 // A legacy match must have CONFIRMED.
564 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { 1642 //
565 bool exploded = false; 1643 // The structured path does not need this and deliberately does not have
566 for (const auto& r : gPaying) 1644 // it: this process sent that transaction moments ago, and a send that
567 if (r.matureAtHeight == it->matureAtHeight && r.blockHash == it->blockHash) { 1645 // failed leaves nothing behind, because SendMoney calls
568 exploded = true; 1646 // RollbackTransactionSpent, which calls EraseFromWallet.
569 break; 1647 //
1648 // The legacy path is the opposite case in every respect. Those
1649 // transactions are days or weeks old, so whether they confirmed is a
1650 // settled fact rather than an open question, and a pre-upgrade payout that
1651 // never confirmed is evidence of NON-payment about to be read as payment.
1652 // This chain has had a 280-block reorg (see FlushMaturePayouts): a payout
1653 // invalidated by it is still sitting in mapWallet with its comment intact,
1654 // and the first boot after this upgrade is exactly when it gets matched.
1655 // The cost of being wrong here is a miner who is never paid and no record
1656 // that they were owed.
1657 CRITICAL_BLOCK(cs_main)
1658 CRITICAL_BLOCK(cs_mapWallet)
1659 {
1660 for (const uint256& hash : legacyCandidates) {
1661 map<uint256, CWalletTx>::const_iterator it = mapWallet.find(hash);
1662 if (it == mapWallet.end())
1663 continue; // erased under us; it is not evidence of anything
1664 int depth = it->second.GetDepthInMainChain();
1665 if (depth < 1) {
1666 // Loud: this is the difference between a miner having been
1667 // paid and a miner about to be paid twice, decided on a fact
1668 // that is easy to check and impossible to recover afterwards.
1669 LogPrint("payout", "[payout] wallet tx %s carries the payout"
1670 " comment for height %d and an output of exactly %s to"
1671 " %s, but it is NOT in the main chain (depth %d) --"
1672 " refusing to read it as payment; this recipient will"
1673 " be paid\n",
1674 hash.GetHex().c_str(), blockHeight,
1675 FormatMoney(amount).c_str(), address.c_str(), depth);
1676 continue;
570 } 1677 }
571 if (exploded) { 1678 txidRet = hash.GetHex();
572 LogPrint("payout", "[payout] ledger entry for block %d is already" 1679 return PAYOUT_PROBE_LEGACY;
573 " in flight -- dropping the duplicate\n", it->blockHeight);
574 it = gPendingPayouts.erase(it);
575 dropped++;
576 } else {
577 ++it;
578 } 1680 }
579 } 1681 }
580 if (dropped) SavePendingPayouts(); 1682 return PAYOUT_PROBE_NONE;
1683}
581 1684
582 if (!gPaying.empty()) { 1685// txid -> "<round id>|<address>", for payments this run has already credited
583 int64 owed = 0; 1686// to a debt on legacy evidence. Stops one pre-upgrade transaction being read
584 for (const auto& r : gPaying) owed += r.amount; 1687// as proof that two different obligations were met. The ledger's paidTxids
585 LogPrint("payout", "[payout] resuming %zu in-flight payment(s) totalling %s\n", 1688// covers the same ground durably; this covers the window before the recovered
586 gPaying.size(), FormatMoney(owed).c_str()); 1689// txid reaches the ledger, and the case where the ledger entry has already
1690// been retired.
1691static std::map<std::string, std::string> gLegacyClaimedTxids;
1692static std::mutex gClaimedTxidMutex;
1693
1694// Every txid already spoken for by some payment OTHER than this one. Collected
1695// before cs_mapWallet is taken: the ledger mutex must not be acquired under a
1696// wallet lock.
1697static void CollectForeignPayoutTxids(int blockHeight, const uint256& blockHash,
1698 const std::string& address,
1699 const std::string& claimKey,
1700 std::set<std::string>& out)
1701{
1702 {
1703 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1704 for (const PendingPayout& pp : gPendingPayouts) {
1705 bool sameRound = SamePayoutRound(pp.blockHeight, pp.blockHash,
1706 blockHeight, blockHash);
1707 for (const auto& kv : pp.paidTxids)
1708 if (!(sameRound && kv.first == address))
1709 out.insert(kv.second);
1710 }
1711 }
1712 {
1713 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
1714 for (const auto& kv : gLegacyClaimedTxids)
1715 if (kv.second != claimKey)
1716 out.insert(kv.first);
587 } 1717 }
588} 1718}
589 1719
1720static void ClaimLegacyPayoutTxid(const std::string& txid, const std::string& claimKey)
1721{
1722 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
1723 gLegacyClaimedTxids[txid] = claimKey;
1724}
1725
1726// blockHash stays a uint256 and not upstream's display string: it is the key
1727// this fork's payout ledger is built on, and a key that has been through
1728// ToString() is a key that can be compared wrong. It is converted to text only
1729// where the round proof needs text. The other three arguments are upstream's,
1730// carried for the round-proof data.
590static void QueuePayouts(int blockHeight, 1731static void QueuePayouts(int blockHeight,
591 const uint256& blockHash, 1732 const uint256& blockHash,
1733 const std::string& coinbaseTxid,
1734 int64 coinbaseValue,
1735 const std::string& foundBy,
592 const std::map<std::string, uint64>& shareCount, 1736 const std::map<std::string, uint64>& shareCount,
593 uint64 total) 1737 uint64 total)
594{ 1738{
1739 PoolRoundProof round;
1740 round.id = PayoutIdForBlock(blockHeight, blockHash);
1741 round.blockHeight = blockHeight;
1742 round.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH;
1743 round.createdAt = GetTime();
1744 round.blockHash = blockHash.ToString();
1745 round.coinbaseTxid = coinbaseTxid;
1746 round.coinbaseValue = coinbaseValue;
1747 round.foundBy = foundBy;
1748 round.feePercent = dPoolFeePercent;
1749 round.totalShares = total;
1750 round.shares = shareCount;
1751
595 if (total == 0) { 1752 if (total == 0) {
1753 RecordPoolRound(round);
596 LogPrint("payout", "[payout] block %d: zero shares recorded -- nothing to pay out\n", 1754 LogPrint("payout", "[payout] block %d: zero shares recorded -- nothing to pay out\n",
597 blockHeight); 1755 blockHeight);
598 return; 1756 return;
@@ -608,6 +1766,10 @@ static void QueuePayouts(int blockHeight,
608 int64 nSubsidy = 50 * COIN; 1766 int64 nSubsidy = 50 * COIN;
609 nSubsidy >>= (blockHeight / 210000); // halving schedule 1767 nSubsidy >>= (blockHeight / 210000); // halving schedule
610 int64 minerReward = (int64)((double)nSubsidy * (1.0 - feeFrac)); 1768 int64 minerReward = (int64)((double)nSubsidy * (1.0 - feeFrac));
1769 int64 operatorCut = nSubsidy - minerReward;
1770 round.subsidy = nSubsidy;
1771 round.minerReward = minerReward;
1772 round.operatorReward = operatorCut;
611 1773
612 PendingPayout pp; 1774 PendingPayout pp;
613 pp.blockHeight = blockHeight; 1775 pp.blockHeight = blockHeight;
@@ -621,7 +1783,9 @@ static void QueuePayouts(int blockHeight,
621 // alone on a private fork paying itself, so it says nothing about a pool 1783 // alone on a private fork paying itself, so it says nothing about a pool
622 // competing on the real chain. Waiting the extra 20 blocks makes each 1784 // competing on the real chain. Waiting the extra 20 blocks makes each
623 // payout funded by construction, and puts the maturity gate past the 1785 // payout funded by construction, and puts the maturity gate past the
624 // reorg depth that orphaned 280 blocks of obligations. 1786 // reorg depth that orphaned 280 blocks of obligations. Upstream uses
1787 // COINBASE_MATURITY here; this is a considered disagreement, not an
1788 // oversight -- see the merge commit message.
625 pp.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH; 1789 pp.matureAtHeight = blockHeight + COINBASE_SPENDABLE_DEPTH;
626 1790
627 int skipped = 0; 1791 int skipped = 0;
@@ -643,12 +1807,22 @@ static void QueuePayouts(int blockHeight,
643 continue; 1807 continue;
644 } 1808 }
645 pp.amounts[kv.first] = amount; 1809 pp.amounts[kv.first] = amount;
1810 round.payouts[kv.first] = amount;
646 } 1811 }
1812 round.skippedDust = skipped;
1813 round.forfeitedBadAddress = burned;
647 1814
648 gPendingPayouts.push_back(pp); 1815 {
649 SavePendingPayouts(); 1816 VersionedBlob snap;
1817 {
1818 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1819 gPendingPayouts.push_back(pp);
1820 snap = SerializePendingPayoutsLocked();
1821 }
1822 PublishPendingPayouts(snap);
1823 }
1824 RecordPoolRound(round);
650 1825
651 int64 operatorCut = nSubsidy - minerReward;
652 LogPrint("payout", 1826 LogPrint("payout",
653 "[payout] block %d: subsidy %s BTF, miners get %s" 1827 "[payout] block %d: subsidy %s BTF, miners get %s"
654 " (fee %.2f%% = %s to operator)," 1828 " (fee %.2f%% = %s to operator),"
@@ -664,101 +1838,337 @@ static void QueuePayouts(int blockHeight,
664 : ""); 1838 : "");
665} 1839}
666 1840
667// Called from event loop: move matured entries into the in-flight list (fast). 1841// Called from the event loop: explode matured ledger entries into per-recipient
1842// in-flight records (fast; no wallet or disk work beyond one small write).
1843//
1844// The ledger entry SURVIVES here. That is upstream's model and it is the
1845// crash-safe one: this fork used to erase the entry as soon as its payments
1846// were handed off, so a death in that window left the debt described by
1847// nothing at all -- a container restart on 2026-07-30 silently dropped
1848// ~1,156 BTF exactly that way. An entry is removed only once every recipient
1849// in it has a txid recorded (PayoutFullyPaid), which the payout thread does.
668// 1850//
669// Ordering matters and is deliberate: the in-flight records are written and 1851// gPaying is kept on top of that as the in-flight working set: it carries the
670// fsync'd to their own file *before* the ledger entry is erased, so the debt 1852// per-recipient retry/deferral state that a per-round `nextAttempt` cannot
671// is always described by at least one of the two files. Dying in the window 1853// express, and it is what the payout thread iterates. It is a cache of work
672// where both describe it is handled at startup by ReconcilePayouts, which 1854// derived from the ledger, never an independent record of debt -- if it were
673// drops the ledger entry it can see is already in flight. 1855// lost entirely the ledger would rebuild it on the next flush.
674static void FlushMaturePayouts() 1856static void FlushMaturePayouts()
675{ 1857{
676 // Void obligations whose funding block lost a reorg. The coin that was 1858 // Which funding blocks are still on the main chain? Asked with no payout
677 // meant to pay them does not exist, so paying anyway spends real-chain 1859 // lock held: PayoutBlockStillOnChain takes cs_main, and cs_main is held by
678 // coin the pool owes to somebody else. 1860 // ProcessBlock and SendMoney for long stretches. Holding the ledger mutex
679 bool voided = false; 1861 // across it would let the event loop park the payout thread behind a chain
680 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { 1862 // lock it has no interest in. (It is not a deadlock either way -- nothing
681 if (PayoutBlockStillOnChain(it->blockHash)) { ++it; continue; } 1863 // takes the ledger mutex while holding cs_main -- but it is exactly the
682 int64 total = 0; 1864 // kind of stall this merge is removing from the ledger path.)
683 for (auto& kv : it->amounts) total += kv.second; 1865 std::vector<uint256> orphaned;
684 LogPrint("payout", "[payout] block %d (%s) was orphaned -- voiding %zu" 1866 {
685 " payment(s) totalling %s\n", 1867 std::vector<uint256> hashes;
686 it->blockHeight, it->blockHash.ToString().substr(0, 14).c_str(),
687 it->amounts.size(), FormatMoney(total).c_str());
688 it = gPendingPayouts.erase(it);
689 voided = true;
690 }
691 if (voided) SavePendingPayouts();
692
693 std::vector<PendingPayout> ready;
694 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) {
695 if (nBestHeight < it->matureAtHeight) { ++it; continue; }
696 LogPrint("payout", "[payout] block height %d matured -- queuing %zu payments\n",
697 it->matureAtHeight, it->amounts.size());
698 ready.push_back(*it); // copy first; erase after save
699 ++it;
700 }
701 if (!ready.empty()) {
702 // Record the individual payments first, and only then drop the ledger
703 // entries that produced them.
704 { 1868 {
705 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 1869 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
706 for (const auto& pp : ready) { 1870 for (const PendingPayout& pp : gPendingPayouts)
707 for (const auto& kv : pp.amounts) { 1871 hashes.push_back(pp.blockHash);
708 PayingRecord r;
709 r.blockHeight = pp.blockHeight;
710 r.blockHash = pp.blockHash;
711 r.matureAtHeight = pp.matureAtHeight;
712 r.address = kv.first;
713 r.amount = kv.second;
714 gPaying.push_back(r);
715 }
716 }
717 SavePayingLocked();
718 } 1872 }
1873 for (const uint256& h : hashes)
1874 if (!PayoutBlockStillOnChain(h))
1875 orphaned.push_back(h);
1876 }
1877
1878 std::vector<PendingPayout> ready;
1879 VersionedBlob ledgerSnap;
1880 {
1881 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
1882
1883 // Void obligations whose funding block lost a reorg. The coin that was
1884 // meant to pay them does not exist, so paying anyway spends real-chain
1885 // coin the pool owes to somebody else.
1886 bool voided = false;
719 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) { 1887 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ) {
720 if (nBestHeight >= it->matureAtHeight) 1888 if (std::find(orphaned.begin(), orphaned.end(), it->blockHash) == orphaned.end()) {
721 it = gPendingPayouts.erase(it);
722 else
723 ++it; 1889 ++it;
1890 continue;
1891 }
1892 int64 total = 0;
1893 for (auto& kv : it->amounts) total += kv.second;
1894 // Always: one line per voided entry, never a collapsed summary.
1895 // A reorg voids these in a burst from this one source line.
1896 LogPrintAlways("payout", "[payout] block %d (%s) was orphaned -- voiding %zu"
1897 " payment(s) totalling %s\n",
1898 it->blockHeight, it->blockHash.ToString().substr(0, 14).c_str(),
1899 it->amounts.size(), FormatMoney(total).c_str());
1900 it = gPendingPayouts.erase(it);
1901 voided = true;
1902 }
1903 if (voided) ledgerSnap = SerializePendingPayoutsLocked();
1904
1905 for (const PendingPayout& pp : gPendingPayouts) {
1906 if (nBestHeight < pp.matureAtHeight)
1907 continue;
1908 if (PayoutFullyPaid(pp))
1909 continue;
1910 // An entry with neither a hash nor a height cannot be told apart
1911 // from another in the same state, so a txid recorded against it
1912 // could settle the wrong debt. Paying it is the unsafe move;
1913 // leaving it standing and saying so is not.
1914 if (!PayoutRoundIdentifiable(pp.blockHeight, pp.blockHash)) {
1915 LogPrint("payout", "[payout] WARNING: ledger entry maturing at"
1916 " height %d names neither a block hash nor a height --"
1917 " refusing to pay %zu recipient(s) that cannot be told"
1918 " apart from another round's\n",
1919 pp.matureAtHeight, pp.amounts.size());
1920 continue;
1921 }
1922 ready.push_back(pp);
724 } 1923 }
725 SavePendingPayouts();
726 } 1924 }
727} 1925 // Outside the ledger mutex: publishing is I/O and the event loop is the
1926 // one thread that must never wait on the payout thread's disk.
1927 PublishPendingPayouts(ledgerSnap);
728 1928
729// Identity of a payment, for finding its record again after SendMoney. 1929 if (ready.empty())
730// amounts is a map keyed by address, so (block, address) is already unique. 1930 return;
731static bool SamePayment(const PayingRecord& a, const PayingRecord& b) 1931
732{ 1932 VersionedBlob payingSnap;
733 return a.matureAtHeight == b.matureAtHeight 1933 {
734 && a.blockHash == b.blockHash 1934 std::lock_guard<std::mutex> queue(gPayoutExecMutex);
735 && a.address == b.address 1935 size_t added = 0;
736 && a.amount == b.amount; 1936 for (const PendingPayout& pp : ready) {
1937 const std::string id = PayoutIdForBlock(pp.blockHeight, pp.blockHash);
1938 if (gQueuedPayouts.count(id))
1939 continue; // already exploded into gPaying and still in flight
1940 size_t n = 0;
1941 for (const auto& kv : pp.amounts) {
1942 // Upstream's per-recipient dedupe: a recipient already carrying
1943 // a txid is done, even though the entry as a whole is not.
1944 if (pp.paidTxids.count(kv.first))
1945 continue;
1946 // The invariant, checked against the thing it is an invariant
1947 // of: at most one in-flight record per (round, address). The
1948 // round-level set above is an optimisation that can be wrong --
1949 // it is memory only, it is rebuilt from gPaying at startup, and
1950 // a stale `ready` snapshot can race a round finishing --
1951 // whereas gPaying is the list being protected. A second record
1952 // here is a second standing order to send the same money.
1953 if (PaymentInFlightLocked(pp.blockHeight, pp.blockHash, kv.first))
1954 continue;
1955 PayingRecord r;
1956 r.id = ++gPayingNextId;
1957 r.blockHeight = pp.blockHeight;
1958 r.blockHash = pp.blockHash;
1959 r.matureAtHeight = pp.matureAtHeight;
1960 r.address = kv.first;
1961 r.amount = kv.second;
1962 gPaying.push_back(r);
1963 n++;
1964 }
1965 gQueuedPayouts.insert(id);
1966 added += n;
1967 LogPrint("payout", "[payout] block height %d matured -- queuing"
1968 " payout %s (%zu of %zu recipients still owed)\n",
1969 pp.matureAtHeight, id.c_str(), n, pp.amounts.size());
1970 }
1971 if (added) payingSnap = SerializePayingLocked();
1972 }
1973 PublishPaying(payingSnap);
737} 1974}
738 1975
739// Settle a payment: it is either done or abandoned, either way it stops being 1976// Settle a payment: it is either done or abandoned, either way it stops being
740// owed. Removing the record and rewriting the file is what makes the payment 1977// in flight. Note this only retires the working record -- whether the miner is
741// not happen twice after a restart. 1978// still OWED the money is the ledger's business, and the ledger is updated
1979// separately by RecordPayoutTxid().
1980//
1981// Erases by record id, not by comparing fields. Value-matching was only an
1982// identity for as long as no two records could be equal, and the restart bug
1983// above produced records that were equal in every field: the first match then
1984// absorbed every erase and every state update, and the second was never
1985// touched again. The id names the slot the caller took its copy from, so the
1986// update lands where it belongs whatever else is in the list.
742static void ForgetPayment(const PayingRecord& r) 1987static void ForgetPayment(const PayingRecord& r)
743{ 1988{
744 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 1989 VersionedBlob snap;
745 for (auto it = gPaying.begin(); it != gPaying.end(); ++it) 1990 {
746 if (SamePayment(*it, r)) { gPaying.erase(it); break; } 1991 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
747 SavePayingLocked(); 1992 for (auto it = gPaying.begin(); it != gPaying.end(); ++it)
1993 if (it->id == r.id) { gPaying.erase(it); break; }
1994
1995 // When the last record for a round is gone the round is no longer in
1996 // flight, so a later flush may explode it again if the ledger still
1997 // shows recipients owed. That is the intended behaviour and it is now
1998 // the only way a stalled payment ever leaves this list: a payment that
1999 // keeps failing is retried forever rather than given up on, so its
2000 // record stays here, and the flush that would re-explode it is stopped
2001 // by PaymentInFlightLocked.
2002 bool stillInFlight = false;
2003 for (const auto& e : gPaying)
2004 if (SamePayoutRound(e.blockHeight, e.blockHash,
2005 r.blockHeight, r.blockHash)) {
2006 stillInFlight = true;
2007 break;
2008 }
2009 if (!stillInFlight)
2010 gQueuedPayouts.erase(PayoutIdForBlock(r.blockHeight, r.blockHash));
2011
2012 snap = SerializePayingLocked();
2013 }
2014 PublishPaying(snap);
748} 2015}
749 2016
750// Persist an updated attempt count / backoff for a payment still owed. 2017// Persist an updated attempt count / backoff for a payment still owed.
751static void ReschedulePayment(const PayingRecord& r) 2018static void ReschedulePayment(const PayingRecord& r)
752{ 2019{
753 std::lock_guard<std::mutex> lk(gPayoutExecMutex); 2020 VersionedBlob snap;
754 for (auto& e : gPaying) 2021 {
755 if (SamePayment(e, r)) { 2022 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
756 e.retries = r.retries; 2023 for (auto& e : gPaying)
757 e.deferrals = r.deferrals; 2024 if (e.id == r.id) {
758 e.retryAfter = r.retryAfter; 2025 e.retries = r.retries;
2026 e.deferrals = r.deferrals;
2027 e.retryAfter = r.retryAfter;
2028 // Carry the cleared probe flag back, or every retry re-scans
2029 // the whole wallet -- the cost this merge is specifically
2030 // removing.
2031 e.fProbeWallet = r.fProbeWallet;
2032 break;
2033 }
2034 snap = SerializePayingLocked();
2035 }
2036 PublishPaying(snap);
2037}
2038
2039// Record that a recipient has been paid, in the ledger, which is the only
2040// durable statement that the debt is settled.
2041//
2042// The file write happens with gPayoutLedgerMutex RELEASED. Upstream writes the
2043// file inside the per-recipient loop with the mutex held, so every recipient
2044// costs the event loop a full serialise+fsync it has to wait for in
2045// FlushMaturePayouts. Ordering against the event loop's own writes is the
2046// snapshot version's job, not the mutex's -- see PublishVersioned.
2047static void RecordPayoutTxid(const PayingRecord& r, const std::string& txid)
2048{
2049 VersionedBlob blob;
2050 bool complete = false;
2051 {
2052 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
2053 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ++it) {
2054 if (!SamePayoutRound(it->blockHeight, it->blockHash,
2055 r.blockHeight, r.blockHash))
2056 continue;
2057 it->paidTxids[r.address] = txid;
2058 if (PayoutFullyPaid(*it)) {
2059 LogPrint("payout", "[payout] payout for block %d complete --"
2060 " removing from ledger\n", it->blockHeight);
2061 gPendingPayouts.erase(it);
2062 complete = true;
2063 }
759 break; 2064 break;
760 } 2065 }
761 SavePayingLocked(); 2066 blob = SerializePendingPayoutsLocked();
2067 }
2068 PublishPendingPayouts(blob);
2069 // The round proof records a txid per address, so this is per recipient and
2070 // not only on completion.
2071 MarkPoolRoundPaid(PayoutIdForBlock(r.blockHeight, r.blockHash), r.address, txid);
2072 if (complete)
2073 LogPrint("payout", "[payout] block %d fully settled\n", r.blockHeight);
2074}
2075
2076// Write a recipient off.
2077//
2078// Only ever called for a state that is DECIDABLE -- currently one: the ledger
2079// names an address that is not a valid address, so no transaction paying it
2080// can be constructed by this or any other build. It is not called, and must
2081// not be, because a payment has failed some number of times. "The wallet said
2082// no N times" is not a decidable state, it is a description of how long we
2083// have been asking; a debt written off on the strength of it is a debt the
2084// operator keeps, and the arithmetic that decided WHY the wallet said no was
2085// wrong for a year (see PayoutThreadFn).
2086//
2087// The amount is removed from the entry rather than marked paid, so the money
2088// is not reported as sent. Once nothing is left owed the entry retires. The
2089// round proof is told separately, because dropping the obligation from the
2090// ledger while pool_rounds.json goes on publishing the address as owed with
2091// "txid": null leaves the pool's two records of the same round permanently
2092// disagreeing.
2093static void ForfeitPayoutRecipient(const PayingRecord& r, const std::string& reason)
2094{
2095 LogPrintAlways("payout", "[payout] writing off %s owed to %s for block %d: %s\n",
2096 FormatMoney(r.amount).c_str(), r.address.c_str(), r.blockHeight,
2097 reason.c_str());
2098 VersionedBlob blob;
2099 {
2100 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
2101 for (auto it = gPendingPayouts.begin(); it != gPendingPayouts.end(); ++it) {
2102 if (!SamePayoutRound(it->blockHeight, it->blockHash,
2103 r.blockHeight, r.blockHash))
2104 continue;
2105 it->amounts.erase(r.address);
2106 // PayoutFullyPaid() returns true for an empty amounts map -- there
2107 // is no recipient in it without a txid -- so the
2108 // `amounts.empty()` disjunct that used to be here decided nothing.
2109 if (PayoutFullyPaid(*it))
2110 gPendingPayouts.erase(it);
2111 break;
2112 }
2113 blob = SerializePendingPayoutsLocked();
2114 }
2115 PublishPendingPayouts(blob);
2116 MarkPoolRoundForfeited(PayoutIdForBlock(r.blockHeight, r.blockHash),
2117 r.address, reason);
2118}
2119
2120// What a failed send means for the payment's retry budget.
2121//
2122// A deferral is "the wallet's condition is temporarily wrong": wait, and do
2123// not hold it against this payment. A fault is "the wallet is funded and
2124// unlocked and still would not do it", which is worth logging loudly and
2125// retrying with backoff.
2126//
2127// This used to be decided by re-reading GetBalance() after the failure and
2128// comparing it against `amount + nTransactionFee + CENT`. Every term of that
2129// was wrong:
2130//
2131// * nTransactionFee is 0 in production. It is only ever assigned from the
2132// GUI preferences dialog (gui.cpp), which a headless pool node does not
2133// have, so the whole reserve was the CENT.
2134//
2135// * The fee the reserve is for is GetMinFee(fDiscount=true), a STEP
2136// function of the serialised size: 0 below 10,000 bytes, and
2137// (1 + bytes/1000) * CENT at or above it -- so 11 * CENT at the step.
2138// There is no required-fee value between 1 and 11 CENT. A one-CENT
2139// reserve is therefore never the right number: below the step nothing
2140// needs reserving, and at the step it is short by a factor of eleven.
2141//
2142// * A pool wallet close to broke selects many small inputs BECAUSE it is
2143// close to broke, and ~56 signed P2PKH inputs is 10 KB. The trigger and
2144// the failure are positively correlated: the guess is wrong precisely
2145// when the pool is nearly out of money.
2146//
2147// Concretely, with a balance of amount + 0.05: the pre-send gate passes,
2148// CreateTransaction builds a >10 KB transaction, the fee becomes 0.11,
2149// SelectCoins(amount + 0.11) fails, SendMoney returns false -- and the old
2150// arithmetic said (amount + 0.05) < (amount + 0.01) is false, so "funded",
2151// so a fault. The condition is stable, so it recurred every 60 s, and 1440 of
2152// them erased the debt with the coin still in the operator's wallet.
2153//
2154// So stop guessing. The wallet is asked to say what it did, and this only
2155// classifies the answer. WALLET_SEND_INSUFFICIENT_FUNDS is coin selection
2156// failing to cover amount plus whatever fee that size of transaction
2157// required, whatever that turned out to be -- no reserve to get wrong.
2158enum PayoutFailureClass {
2159 PAYOUT_FAILURE_DEFER = 0, // conditions are wrong; wait for them to change
2160 PAYOUT_FAILURE_FAULT // funded and unlocked, and it still would not go
2161};
2162
2163static PayoutFailureClass ClassifyPayoutSendFailure(WalletSendResult result,
2164 bool fWalletLocked)
2165{
2166 // A locked wallet cannot sign. Nothing about this payment is wrong.
2167 if (fWalletLocked)
2168 return PAYOUT_FAILURE_DEFER;
2169 if (result == WALLET_SEND_INSUFFICIENT_FUNDS)
2170 return PAYOUT_FAILURE_DEFER;
2171 return PAYOUT_FAILURE_FAULT;
762} 2172}
763 2173
764// Payout execution thread: works through gPaying, calls SendMoney. 2174// Payout execution thread: works through gPaying, calls SendMoney.
@@ -780,11 +2190,14 @@ static void PayoutThreadFn(void*)
780 } 2190 }
781 2191
782 for (PayingRecord r : batch) { 2192 for (PayingRecord r : batch) {
2193 const std::string id = PayoutIdForBlock(r.blockHeight, r.blockHash);
2194
783 // Re-check here and not only at maturity: a payment can sit in this 2195 // Re-check here and not only at maturity: a payment can sit in this
784 // list for hours across retries, and the reorg that kills its 2196 // list for hours across retries, and the reorg that kills its
785 // funding block can land at any point in that window. 2197 // funding block can land at any point in that window. Upstream has
2198 // no equivalent -- it decides once, when the entry matures.
786 if (!PayoutBlockStillOnChain(r.blockHash)) { 2199 if (!PayoutBlockStillOnChain(r.blockHash)) {
787 LogPrint("payout", "[payout] block %d (%s) orphaned while queued" 2200 LogPrintAlways("payout", "[payout] block %d (%s) orphaned while queued"
788 " -- dropping %s to %s\n", 2201 " -- dropping %s to %s\n",
789 r.blockHeight, r.blockHash.ToString().substr(0, 14).c_str(), 2202 r.blockHeight, r.blockHash.ToString().substr(0, 14).c_str(),
790 FormatMoney(r.amount).c_str(), r.address.c_str()); 2203 FormatMoney(r.amount).c_str(), r.address.c_str());
@@ -792,72 +2205,244 @@ static void PayoutThreadFn(void*)
792 continue; 2205 continue;
793 } 2206 }
794 2207
2208 // Upstream's per-recipient dedupe, against the ledger rather than
2209 // against the batch: if a txid is already recorded for this
2210 // address, the money went out and nothing more is owed.
2211 {
2212 bool alreadyPaid = false;
2213 {
2214 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
2215 for (const PendingPayout& pp : gPendingPayouts)
2216 if (SamePayoutRound(pp.blockHeight, pp.blockHash,
2217 r.blockHeight, r.blockHash)
2218 && pp.paidTxids.count(r.address)) {
2219 alreadyPaid = true;
2220 break;
2221 }
2222 }
2223 if (alreadyPaid) {
2224 ForgetPayment(r);
2225 continue;
2226 }
2227 }
2228
795 uint160 h160; 2229 uint160 h160;
796 if (!AddressToHash160(r.address, h160)) { 2230 if (!AddressToHash160(r.address, h160)) {
797 // Should not happen -- bad addresses are filtered in QueuePayouts 2231 // Should not happen -- bad addresses are filtered in
798 LogPrint("payout", "[payout] SKIPPED bad address '%s'\n", 2232 // QueuePayouts. If one is here anyway it is decidable and
799 r.address.c_str()); 2233 // permanent: there is no transaction that pays a string which
2234 // is not an address, so retrying is not waiting for anything.
2235 //
2236 // This used to only ForgetPayment(), which drops the in-flight
2237 // record and leaves the ledger entry claiming the debt and
2238 // pool_rounds.json advertising the recipient as unpaid
2239 // forever, with the next flush re-exploding it. Write it off
2240 // properly and say why.
2241 const std::string reason =
2242 strprintf("'%s' is not a valid address, so no transaction"
2243 " can pay it", r.address.c_str());
2244 LogPrintAlways("payout", "[payout] SKIPPED bad address '%s' --"
2245 " %s owed for block %d cannot be paid\n",
2246 r.address.c_str(), FormatMoney(r.amount).c_str(),
2247 r.blockHeight);
2248 ForfeitPayoutRecipient(r, reason);
800 ForgetPayment(r); 2249 ForgetPayment(r);
801 continue; 2250 continue;
802 } 2251 }
803 CScript sc; 2252 CScript sc;
804 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG; 2253 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
805 2254
2255 // Did a previous process send this and die before recording it?
2256 // Only asked once per record -- see PayingRecord::fProbeWallet.
2257 if (r.fProbeWallet) {
2258 std::string recovered;
2259 const std::string claimKey = id + "|" + r.address;
2260 std::set<std::string> excluded;
2261 CollectForeignPayoutTxids(r.blockHeight, r.blockHash, r.address,
2262 claimKey, excluded);
2263 // A round with a published proof was created by a build whose
2264 // every payment names its round exactly, so only structured
2265 // evidence can settle it -- a comment match would be another
2266 // round's payment colliding on the height.
2267 const bool fAllowLegacy = !PoolRoundHasProof(id);
2268 r.fProbeWallet = false;
2269 PayoutProbeResult probe = FindRecordedPoolPayout(
2270 id, r.blockHeight, r.blockHash, r.address, r.amount,
2271 fAllowLegacy, excluded, recovered);
2272 if (probe != PAYOUT_PROBE_NONE) {
2273 if (probe == PAYOUT_PROBE_LEGACY) {
2274 // Loud on purpose: this is the upgrade path, where the
2275 // only evidence a payment went out is a comment written
2276 // by a binary that is no longer running.
2277 LogPrint("payout", "[payout] %s to %s was already paid by"
2278 " a pre-upgrade build (wallet tx %s carries the"
2279 " payout comment for height %d and an output of"
2280 " exactly this amount, and no structured payout"
2281 " metadata) -- not resending\n",
2282 FormatMoney(r.amount).c_str(), r.address.c_str(),
2283 recovered.c_str(), r.blockHeight);
2284 ClaimLegacyPayoutTxid(recovered, claimKey);
2285 } else {
2286 LogPrint("payout", "[payout] recovered recorded payout %s to %s"
2287 " from wallet tx %s -- not resending\n",
2288 FormatMoney(r.amount).c_str(), r.address.c_str(),
2289 recovered.c_str());
2290 }
2291 RecordPayoutTxid(r, recovered);
2292 ForgetPayment(r);
2293 continue;
2294 }
2295 }
2296
806 // Waiting for coin to mature is not a failure, so it must not 2297 // Waiting for coin to mature is not a failure, so it must not
807 // consume the retry budget. Burning a retry every 60s means a 2298 // consume the retry budget. Burning a retry every 60s means a
808 // payment the pool fully intends to honour can hit GIVING UP 2299 // payment the pool fully intends to honour can hit GIVING UP
809 // 24h later purely because the subsidy funding it had not 2300 // 24h later purely because the subsidy funding it had not
810 // matured yet -- the miner is then silently never paid. 2301 // matured yet -- the miner is then silently never paid.
2302 //
2303 // The comparison includes the fee. It used to be against the
2304 // amount alone, so a balance anywhere between amount and
2305 // amount+fee sailed past this gate, failed in SendMoney, and was
2306 // counted as a send FAILURE rather than a wait -- 1440 of those is
2307 // 24 hours, and the recipient was erased with the money left in
2308 // the operator's wallet and nothing on disk saying it was owed.
2309 //
2310 // nTransactionFee is the part of the fee that can be known before
2311 // the transaction exists, and in production it is 0 -- it is only
2312 // ever assigned from the GUI preferences dialog, which a pool node
2313 // does not have -- so including it here changes nothing on this
2314 // pool and is kept only because a GUI operator can set it. The
2315 // rest is GetMinFee(), a function of the serialised size and
2316 // therefore of how many inputs SelectCoins ends up picking, which
2317 // cannot be predicted from here -- so this gate deliberately does
2318 // NOT try to reserve it. Refusing to attempt a payment the wallet
2319 // could actually have made would be its own way of not paying a
2320 // miner. That residual is handled after the fact, by asking the
2321 // wallet why it refused -- see ClassifyPayoutSendFailure.
811 int64 nSpendable = GetBalance(); 2322 int64 nSpendable = GetBalance();
812 if (nSpendable < r.amount) { 2323 if (nSpendable < r.amount + nTransactionFee) {
813 r.deferrals++; 2324 r.deferrals++;
814 r.retryAfter = GetTime() + 60; 2325 r.retryAfter = GetTime() + 60;
815 // Hourly, not every minute: a dry pool would otherwise 2326 // Hourly, not every minute: a dry pool would otherwise
816 // write one line per payment per minute forever. 2327 // write one line per payment per minute forever.
817 if (r.deferrals == 1 || r.deferrals % 60 == 0) 2328 if (r.deferrals == 1 || r.deferrals % 60 == 0)
818 LogPrint("payout", "[payout] deferring %s to %s -- only %s" 2329 LogPrint("payout", "[payout] deferring %s to %s -- only %s"
819 " spendable (waiting %d min so far)\n", 2330 " spendable, need %s with fee (waiting %d min so far)\n",
820 FormatMoney(r.amount).c_str(), r.address.c_str(), 2331 FormatMoney(r.amount).c_str(), r.address.c_str(),
821 FormatMoney(nSpendable).c_str(), r.deferrals); 2332 FormatMoney(nSpendable).c_str(),
2333 FormatMoney(r.amount + nTransactionFee).c_str(),
2334 r.deferrals);
822 ReschedulePayment(r); 2335 ReschedulePayment(r);
823 continue; 2336 continue;
824 } 2337 }
825 2338
826 CWalletTx wtx; 2339 // Likewise a locked wallet: it cannot sign, so every attempt fails
827 wtx.mapValue["comment"] = PayoutComment(r.blockHeight); 2340 // for a reason that has nothing to do with this payment being
2341 // payable. Left to the retry budget, an operator who locks the
2342 // wallet overnight comes back to erased debts.
2343 if (IsWalletLocked()) {
2344 r.deferrals++;
2345 r.retryAfter = GetTime() + 60;
2346 if (r.deferrals == 1 || r.deferrals % 60 == 0)
2347 LogPrint("payout", "[payout] deferring %s to %s -- wallet is"
2348 " locked (waiting %d min so far)\n",
2349 FormatMoney(r.amount).c_str(), r.address.c_str(),
2350 r.deferrals);
2351 ReschedulePayment(r);
2352 continue;
2353 }
828 2354
829 if (SendMoney(sc, r.amount, wtx)) { 2355 CWalletTx wtx;
2356 wtx.mapValue["comment"] = PayoutComment(r.blockHeight, r.blockHash);
2357 // Upstream's structured metadata. This is what makes the wallet
2358 // probe above possible at all -- the old comment-only form could
2359 // not distinguish two rounds paying the same address the same
2360 // amount. Keyed by the hash-derived id, so a reorg cannot make one
2361 // round's transaction answer for another's.
2362 wtx.mapValue["pool_payout_id"] = id;
2363 wtx.mapValue["pool_payout_address"] = r.address;
2364 wtx.mapValue["pool_payout_amount"] = strprintf("%lld", r.amount);
2365 // The full hash, not only the id's sixteen-digit prefix: the
2366 // prefix of a proof-of-work hash is where the zeros live, and
2367 // the probe needs to tell twin rounds at one height apart.
2368 if (r.blockHash != 0)
2369 wtx.mapValue["pool_payout_block"] = r.blockHash.ToString();
2370
2371 SendMoneyOutcome outcome;
2372 if (SendMoney(sc, r.amount, wtx, outcome)) {
830 // The wallet holds the transaction before this returns, so a 2373 // The wallet holds the transaction before this returns, so a
831 // crash here is recoverable: ReconcilePayouts finds it by 2374 // crash between here and the ledger write is recoverable: the
832 // comment+address+amount and does not resend. 2375 // probe above finds it by id+address+amount and does not
2376 // resend. Ledger first, in-flight record second -- the debt is
2377 // described by at least one of them at every instant.
2378 RecordPayoutTxid(r, wtx.GetHash().GetHex());
833 ForgetPayment(r); 2379 ForgetPayment(r);
834 LogPrint("payout", "[payout] paid %s to %s (tx %s)\n", 2380 LogPrintAlways("payout", "[payout] paid %s to %s (tx %s)\n",
835 FormatMoney(r.amount).c_str(), r.address.c_str(), 2381 FormatMoney(r.amount).c_str(), r.address.c_str(),
836 wtx.GetHash().ToString().substr(0, 12).c_str()); 2382 wtx.GetHash().ToString().substr(0, 12).c_str());
837 } else if (r.retries < 1440) {
838 // Transient failures (reorg, locked wallet) must not silently
839 // burn the payment -- retry with backoff. Insufficient funds
840 // never lands here; it defers above.
841 r.retries++;
842 r.retryAfter = GetTime() + 60;
843 ReschedulePayment(r);
844 LogPrint("payout", "[payout] FAILED to pay %s to %s"
845 " (%s spendable at the time) -- retry %d/1440 in 60s\n",
846 FormatMoney(r.amount).c_str(), r.address.c_str(),
847 FormatMoney(nSpendable).c_str(), r.retries);
848 } else { 2383 } else {
849 // Reaching here means 1440 attempts that were NOT 2384 // Why did it fail? The answer used to decide whether the
850 // insufficient-funds waits (those defer instead) against a 2385 // payment eventually got erased, and it was decided by
851 // block still on the main chain -- i.e. a real wallet fault, 2386 // guessing from a balance snapshot -- see
852 // not a maturity or reorg question. 2387 // ClassifyPayoutSendFailure for the arithmetic and what it
853 LogPrint("payout", "[payout] GIVING UP on %s to %s after %d" 2388 // cost. The wallet now says what it did, and nothing here
854 " retries -- block %d is still on-chain and %s was" 2389 // erases anything.
855 " spendable, so this is a wallet fault, not maturity" 2390 //
856 " -- payment forfeited\n", 2391 // The lock state is re-read rather than trusted from before
857 FormatMoney(r.amount).c_str(), r.address.c_str(), 2392 // the attempt: SendMoney takes cs_main and can block for a
858 r.retries, r.blockHeight, 2393 // long time, so it may have moved underneath us. The funding
859 FormatMoney(nSpendable).c_str()); 2394 // answer does not need re-reading -- it is a statement about
860 ForgetPayment(r); 2395 // the attempt that was actually made.
2396 bool fLocked = IsWalletLocked();
2397 int64 nAfter = GetBalance();
2398
2399 if (ClassifyPayoutSendFailure(outcome.result, fLocked)
2400 == PAYOUT_FAILURE_DEFER) {
2401 // Conditions are temporarily wrong. Wait.
2402 r.deferrals++;
2403 r.retryAfter = GetTime() + 60;
2404 if (r.deferrals == 1 || r.deferrals % 60 == 0)
2405 LogPrint("payout", "[payout] deferring %s to %s -- send"
2406 " failed with %s (%s spendable; the wallet"
2407 " needed a fee of %s for a transaction this"
2408 " size); waiting %d min so far\n",
2409 FormatMoney(r.amount).c_str(), r.address.c_str(),
2410 fLocked ? "the wallet locked" : "too little coin",
2411 FormatMoney(nAfter).c_str(),
2412 FormatMoney(outcome.nFeeRequired).c_str(),
2413 r.deferrals);
2414 ReschedulePayment(r);
2415 } else {
2416 // Funded, unlocked, block on the main chain, and the
2417 // wallet still would not build, sign or accept the
2418 // transaction. That is worth shouting about; it is not
2419 // grounds to keep the money.
2420 //
2421 // There is deliberately no give-up branch. It used to
2422 // forfeit the debt after 1440 of these, which is not a
2423 // decidable state -- it is a count of how long we have
2424 // been asking, and it was reached by misclassifying a
2425 // funding shortfall as a fault. A debt that is never
2426 // written off cannot be written off wrongly. The payment
2427 // stays in flight, keeps its ledger entry, keeps being
2428 // reported in pool_status.json as owed, and keeps
2429 // retrying once a minute until the operator or the wallet
2430 // makes it possible.
2431 r.retries++;
2432 r.retryAfter = GetTime() + 60;
2433 ReschedulePayment(r);
2434 // Every minute at first, then hourly: a fault that is not
2435 // going away must not turn the log into one line a minute
2436 // forever, and must not go quiet either.
2437 if (r.retries <= 5 || r.retries % 60 == 0)
2438 LogPrintAlways("payout", "[payout] FAILED to pay %s to %s"
2439 " (%s spendable, wallet unlocked, block %d on"
2440 " the main chain) -- attempt %d, retrying in"
2441 " 60s; this debt stands until it is paid\n",
2442 FormatMoney(r.amount).c_str(), r.address.c_str(),
2443 FormatMoney(nAfter).c_str(), r.blockHeight,
2444 r.retries);
2445 }
861 } 2446 }
862 } 2447 }
863 2448
@@ -881,6 +2466,7 @@ static StatsSnapshot gStats;
881static std::vector<PoolWorkerStatView> gWorkerStats; 2466static std::vector<PoolWorkerStatView> gWorkerStats;
882static std::vector<PendingPayoutView> gPayoutViews; 2467static std::vector<PendingPayoutView> gPayoutViews;
883static std::mutex gStatsMutex; 2468static std::mutex gStatsMutex;
2469static int64 gLastPoolStatusWrite = 0;
884 2470
885void GetPoolOperatorStats(int& authorizedMiners, int& blocksFound, uint64& roundShares, double& totalHashRate) 2471void GetPoolOperatorStats(int& authorizedMiners, int& blocksFound, uint64& roundShares, double& totalHashRate)
886{ 2472{
@@ -903,6 +2489,135 @@ void GetPendingPayouts(std::vector<PendingPayoutView>& out)
903 out = gPayoutViews; 2489 out = gPayoutViews;
904} 2490}
905 2491
2492static std::string PoolStatusFilePath()
2493{
2494 if (!strPoolStatusFile.empty())
2495 return strPoolStatusFile;
2496 return GetAppDir() + "/pool_status.json";
2497}
2498
2499// Serialise the operator status file. Caller must hold gStatsMutex (this reads
2500// gStats / gWorkerStats / gPayoutViews); gPoolRoundsMutex is taken and
2501// released inside.
2502//
2503// Split from the write for the same reason as the three ledgers, and for a
2504// worse instance of the same problem. This used to be called from RefreshStats
2505// with gPayoutLedgerMutex held at function scope, and once the ledger writes
2506// gained fsync() that call became two fsyncs -- one on the file, one on its
2507// directory -- performed by the event loop, the thread that owns every miner's
2508// socket, with the mutex the payout thread needs in its hand. Measured on the
2509// build host (ext4 on LVM on SSD): 3.78 ms per publish, every 10 seconds, and
2510// unbounded on a shared virtual disk under load.
2511//
2512// Nothing here needs the ledger mutex at all: the pending-payout rows were
2513// already copied into gPayoutViews under it, further up RefreshStats.
2514static std::string SerializePoolStatusLocked()
2515{
2516 nlohmann::json j;
2517 j["schema"] = "bitflash-pool-status-1";
2518 j["network"] = "bitflash";
2519 j["updatedAt"] = GetTime();
2520
2521 j["operator"] = {
2522 {"name", strPoolName},
2523 {"btfAddress", BtfLocalAddress()},
2524 {"feePercent", dPoolFeePercent},
2525 {"dashboardUrl", strPoolDashboardUrl},
2526 {"roundsFile", "pool_rounds.json"}
2527 };
2528
2529 j["node"] = {
2530 {"height", nBestHeight},
2531 {"poolRunning", (bool)gPoolRunning},
2532 {"mode", nMineMode}
2533 };
2534
2535 j["pool"] = {
2536 {"authorizedMiners", gStats.authorizedMiners},
2537 {"blocksFoundSession", gStats.blocksFound},
2538 {"roundShares", gStats.roundShares},
2539 {"hashRate", gStats.totalHashRate}
2540 };
2541
2542 j["workers"] = nlohmann::json::array();
2543 for (const PoolWorkerStatView& row : gWorkerStats) {
2544 j["workers"].push_back({
2545 {"address", row.address},
2546 {"worker", row.worker},
2547 {"totalShares", row.totalShares},
2548 {"roundShares", row.roundShares},
2549 {"lastSeen", row.lastSeen},
2550 {"hashRate", row.hashRate}
2551 });
2552 }
2553
2554 j["pendingPayouts"] = nlohmann::json::array();
2555 for (const PendingPayoutView& row : gPayoutViews) {
2556 j["pendingPayouts"].push_back({
2557 {"matureAtHeight", row.matureAtHeight},
2558 {"recipients", row.recipients},
2559 {"totalAmountSatoshis", row.totalAmount},
2560 {"totalAmountBtf", FormatMoney(row.totalAmount)}
2561 });
2562 }
2563
2564 j["recentRounds"] = nlohmann::json::array();
2565 {
2566 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
2567 size_t start = gPoolRounds.size() > 10 ? gPoolRounds.size() - 10 : 0;
2568 for (size_t i = start; i < gPoolRounds.size(); i++) {
2569 const PoolRoundProof& r = gPoolRounds[i];
2570 // A forfeited recipient is not "unpaid and pending" -- the ledger
2571 // has already dropped the obligation. Counting it as unpaid is
2572 // what made the round proof and the ledger disagree for good.
2573 int unpaid = 0;
2574 int forfeited = 0;
2575 for (const auto& kv : r.payouts) {
2576 if (r.paidTxids.count(kv.first))
2577 continue;
2578 if (r.forfeited.count(kv.first))
2579 forfeited++;
2580 else
2581 unpaid++;
2582 }
2583 j["recentRounds"].push_back({
2584 {"id", r.id},
2585 {"blockHeight", r.blockHeight},
2586 {"blockHash", r.blockHash},
2587 {"coinbaseTxid", r.coinbaseTxid},
2588 {"totalShares", r.totalShares},
2589 {"payouts", (int)r.payouts.size()},
2590 {"unpaidPayouts", unpaid},
2591 {"forfeitedPayouts", forfeited},
2592 {"coinbaseValueBtf", FormatMoney(r.coinbaseValue)},
2593 {"minerRewardBtf", FormatMoney(r.minerReward)}
2594 });
2595 }
2596 }
2597
2598 return j.dump(2);
2599}
2600
2601// Publish the operator status file. Call with every payout mutex released.
2602//
2603// Still no version guard and no write mutex, deliberately, and the split does
2604// not change the argument: RefreshStats remains the only caller of either
2605// half, in production it runs only on the event loop, and it now calls this
2606// after leaving the scope that serialised -- so the serialise and the write
2607// stay strictly ordered on one thread and pool_status.json still has a single
2608// writer by construction. It shares WriteFileAtomic so the durability rules
2609// stay in one place.
2610//
2611// "In production" is doing real work in that sentence. The payout-ledger
2612// self-test also calls RefreshStats, from one thread at a time and with the
2613// pool not running, which is why it is still a single writer there too. A
2614// second concurrent caller anywhere would invalidate this paragraph and want
2615// the same versioned publish the ledgers use.
2616static void PublishPoolStatus(const std::string& blob)
2617{
2618 WriteFileAtomic(PoolStatusFilePath(), blob, "pool status");
2619}
2620
906// --------------------------------------------------------------------------- 2621// ---------------------------------------------------------------------------
907// Miner state -- event loop thread only 2622// Miner state -- event loop thread only
908// --------------------------------------------------------------------------- 2623// ---------------------------------------------------------------------------
@@ -928,9 +2643,35 @@ static const double VARDIFF_MAX = 1e12;
928static const int64 VARDIFF_STALL_SECS = 45; // 3x share target -- reacts faster without false-triggering on normal variance 2643static const int64 VARDIFF_STALL_SECS = 45; // 3x share target -- reacts faster without false-triggering on normal variance
929static const double VARDIFF_STALL_CUT = 4.0; // divide difficulty by this much 2644static const double VARDIFF_STALL_CUT = 4.0; // divide difficulty by this much
930 2645
2646// #7: m->hashRate is an EWMA sampled only when a share arrives, so a miner
2647// that stops hashing keeps its last estimate forever and RefreshStats sums
2648// it into the announced pool total for as long as the connection sits open
2649// -- the published pool hashrate ran several times over ground truth
2650// because of exactly this. MinerReportedHashRate() below applies the
2651// policy: a miner gets a grace of HASHRATE_IDLE_MISSES expected share
2652// intervals (expected interval = expected hashes per share at its
2653// difficulty, divided by the estimate itself) with no share, after which
2654// the figure we report falls off with a half-life of one more such
2655// interval. The interval is clamped to [HASHRATE_MIN_INTERVAL,
2656// HASHRATE_MAX_INTERVAL] so a fast miner at the vardiff floor (expected
2657// interval of milliseconds) is not punished for ordinary share-delivery
2658// jitter, and a very slow miner's grace does not stretch past the point
2659// where its estimate means anything. At the vardiff target of one share
2660// per 15s that is a minute of grace, half the rate reported 75s in, and
2661// nothing reported after about seventeen minutes of silence.
2662static const double HASHRATE_IDLE_MISSES = 4.0; // grace, in expected share intervals
2663static const double HASHRATE_MIN_INTERVAL = 1.0; // seconds; floor for the expected interval
2664static const double HASHRATE_MAX_INTERVAL = 3600.0; // seconds; ceiling for the expected interval
2665// Half-lives past the grace beyond which we report a flat zero. 2^-64 of the
2666// original estimate is not a number anyone needs, and stopping here keeps
2667// pow() from walking down into the denormals for a miner that has been quiet
2668// for hours.
2669static const double HASHRATE_DECAY_CUTOFF = 64.0;
2670
931struct Miner { 2671struct Miner {
932 btf_socket_t fd; 2672 btf_socket_t fd;
933 bool authorised = false; 2673 bool authorised = false;
2674 bool subscribed = false; // completed mining.subscribe (#19 census)
934 std::string address; 2675 std::string address;
935 std::string worker; 2676 std::string worker;
936 std::string readBuf; 2677 std::string readBuf;
@@ -954,11 +2695,80 @@ struct Miner {
954 int64 lastShareTime = 0; // GetTime() of the previous accepted share 2695 int64 lastShareTime = 0; // GetTime() of the previous accepted share
955 double hashRate = 0.0; // smoothed H/s estimate 2696 double hashRate = 0.0; // smoothed H/s estimate
956 2697
2698 // Replay guard for mining.submit (#8): nonces already credited on this
2699 // connection for the job it is currently working. Keyed to the job id in
2700 // seenNoncesJob and cleared by SendJob whenever that id changes, so a
2701 // resubmission under a NEW job is never turned away, while a replayed
2702 // (job_id, nonce) pair under the same job is refused before it can be
2703 // credited twice into roundShareCount -- the figure the payout split
2704 // divides by.
2705 std::set<uint32_t> seenNonces;
2706 std::string seenNoncesJob;
2707
957 explicit Miner(btf_socket_t f) 2708 explicit Miner(btf_socket_t f)
958 : fd(f), connectTime(GetTime()), 2709 : fd(f), connectTime(GetTime()),
959 extranonce2(gNextExtranonce2.fetch_add(1)) {} 2710 extranonce2(gNextExtranonce2.fetch_add(1)) {}
960}; 2711};
961 2712
2713// What we publish as a miner's H/s: the last EWMA sample, faded out if shares
2714// have stopped arriving (#7, policy at HASHRATE_IDLE_MISSES).
2715//
2716// This reads m->hashRate and never writes it. That is the whole design. The
2717// first attempt decayed the stored estimate in place from the event loop's
2718// sweep, which put a running accumulator behind a figure that is only ever
2719// displayed, and the accumulator misbehaved in two ways:
2720//
2721// - `expected` was recomputed from the already-decayed estimate on every
2722// pass, so each decay raised `expected`, which raised `grace`, which is
2723// the very thing gating the next decay. `grace` simply chased `silent`
2724// upward and the reported rate settled into a 1/t crawl instead of the
2725// half-life curve the comment above promises -- 291x too high at ten
2726// half-lives past the grace. Issue #7 is about the pool over-reporting;
2727// that under-corrected it for hours.
2728// - Once `expected` reached HASHRATE_MAX_INTERVAL it could not rise any
2729// further, the feedback that had been gating each pass was gone, and the
2730// decay began compounding once a second against the total elapsed
2731// silence -- taking a miner from 0.4% of its baseline to 1e-11 of it in
2732// the next hundred seconds, four hours in, for no reason a reader of
2733// either the code or the announced number could see.
2734//
2735// Computing the value instead of accumulating it removes the failure mode
2736// rather than tuning it: with no stored decay state there is nothing for a
2737// tick rate, a missed pass or a duplicated pass to perturb, and the answer
2738// depends only on how long the miner has actually been quiet. It also keeps
2739// the fade out of the estimator -- the EWMA on the share path still blends
2740// against the value it last computed, not against a faded copy of it, so a
2741// miner that resumes recovers on exactly the curve it would have without any
2742// of this.
2743//
2744// `now` is a parameter rather than a GetTime() call so that one refresh fades
2745// every miner to the same instant, and so the policy is directly testable.
2746static double MinerReportedHashRate(const Miner* m, int64 now)
2747{
2748 // Nothing sampled yet: a fresh connection, or one that has found a single
2749 // share and so has no interval to have been timed over. Nothing to fade,
2750 // and nothing to divide by below.
2751 if (m->hashRate <= 0.0 || m->lastShareTime <= 0) return m->hashRate;
2752
2753 double expected = ExpectedHashesForDifficulty(m->difficulty) / m->hashRate;
2754 // Written so that a NaN would land on the floor rather than sail through:
2755 // the clamp is what guarantees `expected` is finite and strictly positive
2756 // below, whatever the division produced.
2757 if (!(expected > HASHRATE_MIN_INTERVAL)) expected = HASHRATE_MIN_INTERVAL;
2758 if (expected > HASHRATE_MAX_INTERVAL) expected = HASHRATE_MAX_INTERVAL;
2759
2760 double silent = (double)(now - m->lastShareTime);
2761 double grace = HASHRATE_IDLE_MISSES * expected;
2762 // grace is at least HASHRATE_IDLE_MISSES seconds, so a clock that has
2763 // stepped backwards gives a negative `silent`, takes this branch, and
2764 // reports the sample unchanged. Nothing here can scale an estimate up.
2765 if (!(silent > grace)) return m->hashRate;
2766
2767 double halflives = (silent - grace) / expected;
2768 if (halflives >= HASHRATE_DECAY_CUTOFF) return 0.0;
2769 return m->hashRate * pow(0.5, halflives);
2770}
2771
962static std::vector<Miner*> gMiners; 2772static std::vector<Miner*> gMiners;
963 2773
964struct FoundBlock { int height; int64 ts; std::string addr; }; 2774struct FoundBlock { int height; int64 ts; std::string addr; };
@@ -1009,6 +2819,18 @@ static void LocalAcceptFn(void*)
1009 break; 2819 break;
1010 continue; 2820 continue;
1011 } 2821 }
2822 // #6: let the OS probe a peer that has gone silent, so a half-open
2823 // session eventually errors out of recv and lands in the event
2824 // loop's existing reap path instead of sitting in gMiners forever.
2825 // This is a backstop, not the mechanism: the default cadence before
2826 // the first probe is hours on most stacks, so the idle timeout in
2827 // PoolEventLoop fires long before keepalive does. Miners arriving
2828 // via a relay ride a loopback socket pair whose remote transport is
2829 // the rendezvous socket owned by the tunnel pump threads; keepalive
2830 // cannot reach across that, which is part of why the idle timeout
2831 // exists.
2832 int keepalive = 1;
2833 setsockopt(c, SOL_SOCKET, SO_KEEPALIVE, (char*)&keepalive, sizeof(keepalive));
1012 LogPrint("pool", "[pool] miner arrived via loopback -- pushing to event loop\n"); 2834 LogPrint("pool", "[pool] miner arrived via loopback -- pushing to event loop\n");
1013 PushReady((btf_socket_t)c); 2835 PushReady((btf_socket_t)c);
1014 } 2836 }
@@ -1039,6 +2861,30 @@ static bool SendLine(btf_socket_t fd, const json& j)
1039static const size_t MAX_STRATUM_LINE = 64 * 1024; 2861static const size_t MAX_STRATUM_LINE = 64 * 1024;
1040static const size_t MAX_POOL_MINERS = 1000; 2862static const size_t MAX_POOL_MINERS = 1000;
1041 2863
2864// #19 instrumentation: the "broadcast to N miner(s)" line counts connected
2865// sockets, which says nothing about how many are actually mining. These split
2866// the connection census into connected / subscribed / authorized / recently
2867// active, so an operator can tell a real workforce from a pile of half-open
2868// sessions at a glance.
2869static const int64 POOL_CENSUS_SECS = 60; // how often to log the breakdown
2870static const int64 POOL_ACTIVE_WINDOW = 300; // a miner is "active" if it
2871 // authorized and was seen within
2872 // this window (one payout-rotate
2873 // cycle)
2874static const int64 POOL_UNAUTH_GRACE_SECS = 60; // a connection still not
2875 // authorized after this long is
2876 // idle, not merely starting up
2877// #6: a Miner used to leave gMiners only when RecvLine reported a clean
2878// disconnect. A half-open session (miner host died, NAT state expired,
2879// cable pulled) sends no FIN and reports no error, so it sat in the table
2880// forever -- authorize outpaced disconnect 2:1 on the live pool, and the
2881// table grew toward MAX_POOL_MINERS, past which real miners are refused.
2882// lastSeen is stamped on every complete line received; a miner that has
2883// said nothing for this long is gone in every way that matters. Ten
2884// minutes is far beyond the vardiff target (a share every 15s) and the
2885// stall watchdog (45s), so no miner that is actually hashing comes close.
2886static const int64 POOL_IDLE_TIMEOUT_SECS = 600;
2887
1042// Non-blocking recv: pull whatever bytes are available now into buf, then 2888// Non-blocking recv: pull whatever bytes are available now into buf, then
1043// return one complete line if one is ready. Three outcomes: 2889// return one complete line if one is ready. Three outcomes:
1044// true -- line is filled; buf may still contain more lines 2890// true -- line is filled; buf may still contain more lines
@@ -1122,6 +2968,17 @@ static json MakeNotifyParams(const StratumJob& job, const CBlock& b,
1122static void SendJob(Miner* m, bool clean) 2968static void SendJob(Miner* m, bool clean)
1123{ 2969{
1124 if (!gHaveJob) return; 2970 if (!gHaveJob) return;
2971 // Every path that hands a miner a new job funnels through here, so this
2972 // is the one place the replay guard needs to follow the job id: nonces
2973 // credited under the previous job mean nothing under the new one. Keying
2974 // the clear to the id (rather than clearing unconditionally) keeps a
2975 // same-job re-notify -- the vardiff retarget and stall watchdog both
2976 // re-send the CURRENT job -- from reopening the duplicate window it
2977 // exists to close.
2978 if (m->seenNoncesJob != gCurrentJob.jobId) {
2979 m->seenNonces.clear();
2980 m->seenNoncesJob = gCurrentJob.jobId;
2981 }
1125 CBlock mb = BuildMinerBlock(gCurrentJob, m->extranonce2); 2982 CBlock mb = BuildMinerBlock(gCurrentJob, m->extranonce2);
1126 // Use this miner's own shareTarget (set by vardiff) in the notify params. 2983 // Use this miner's own shareTarget (set by vardiff) in the notify params.
1127 // The participant miner reads p[3] as its share target, and HandleLine 2984 // The participant miner reads p[3] as its share target, and HandleLine
@@ -1147,6 +3004,65 @@ static void BroadcastJob(bool clean)
1147} 3004}
1148 3005
1149// --------------------------------------------------------------------------- 3006// ---------------------------------------------------------------------------
3007// #19: connection census. "broadcast to N miner(s)" only counts authorized
3008// sockets that got a job; it says nothing about the rest of the connection
3009// table. This logs the full breakdown so an operator can tell a real workforce
3010// apart from half-open / never-authorized / idle sessions, and flags when the
3011// single select() loop is running out of room (issue #26).
3012//
3013// "Out of room" is not the same question on both platforms, and FD_SETSIZE
3014// answers a different one on each. On POSIX it bounds the *value* of any
3015// descriptor handed to FD_SET, so what matters is the highest fd we hold. On
3016// Windows an fd_set is a counted array of SOCKET handles whose numeric values
3017// are arbitrary, so what matters is how many we hold, not how large they are.
3018// FD_SETSIZE comes from <sys/select.h> or winsock2.h; this file must not define
3019// it, because redefining it after those headers changes nothing.
3020// ---------------------------------------------------------------------------
3021static void LogPoolConnectionCensus()
3022{
3023 int connected = 0, subscribed = 0, authorized = 0, active = 0, neverAuthIdle = 0;
3024 int64 now = GetTime();
3025#ifndef _WIN32
3026 int maxfd = 0;
3027#endif
3028 for (Miner* m : gMiners) {
3029 connected++;
3030#ifndef _WIN32
3031 if ((int)m->fd > maxfd) maxfd = (int)m->fd;
3032#endif
3033 if (m->subscribed) subscribed++;
3034 if (m->authorised) {
3035 authorized++;
3036 if (m->lastSeen > 0 && now - m->lastSeen <= POOL_ACTIVE_WINDOW)
3037 active++;
3038 } else if (now - m->connectTime >= POOL_UNAUTH_GRACE_SECS) {
3039 neverAuthIdle++; // past the grace period and still not authorized
3040 }
3041 }
3042
3043#ifndef _WIN32
3044 int headroom = (int)FD_SETSIZE - maxfd;
3045 LogPrint("pool", "[pool] census: connected=%d subscribed=%d authorized=%d"
3046 " active(last %llds)=%d never-auth-idle=%d"
3047 " (cap=%zu, maxfd=%d, FD_SETSIZE=%d)\n",
3048 connected, subscribed, authorized, (long long)POOL_ACTIVE_WINDOW,
3049 active, neverAuthIdle, MAX_POOL_MINERS, maxfd, (int)FD_SETSIZE);
3050#else
3051 int headroom = (int)FD_SETSIZE - connected;
3052 LogPrint("pool", "[pool] census: connected=%d subscribed=%d authorized=%d"
3053 " active(last %llds)=%d never-auth-idle=%d"
3054 " (cap=%zu, FD_SETSIZE=%d sockets)\n",
3055 connected, subscribed, authorized, (long long)POOL_ACTIVE_WINDOW,
3056 active, neverAuthIdle, MAX_POOL_MINERS, (int)FD_SETSIZE);
3057#endif
3058 if (headroom <= 16)
3059 LogPrint("pool", "[pool] WARNING: %d slots left before FD_SETSIZE=%d --"
3060 " the select() loop is about to walk into undefined behaviour"
3061 " (#26). Refuse new miners or rebuild with poll/epoll.\n",
3062 headroom, (int)FD_SETSIZE);
3063}
3064
3065// ---------------------------------------------------------------------------
1150// Stratum message handler -- called for each complete line from a miner 3066// Stratum message handler -- called for each complete line from a miner
1151// Returns false if the miner should be disconnected immediately. 3067// Returns false if the miner should be disconnected immediately.
1152// --------------------------------------------------------------------------- 3068// ---------------------------------------------------------------------------
@@ -1214,6 +3130,7 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1214 // ------------------------------------------------------------------ 3130 // ------------------------------------------------------------------
1215 if (method == "mining.subscribe") { 3131 if (method == "mining.subscribe") {
1216 std::string sid = ToHex(&m->fd, 4); 3132 std::string sid = ToHex(&m->fd, 4);
3133 m->subscribed = true;
1217 LogPrint("worker", "[worker->pool] subscribe from fd=%d sid=%s\n", 3134 LogPrint("worker", "[worker->pool] subscribe from fd=%d sid=%s\n",
1218 (int)m->fd, sid.c_str()); 3135 (int)m->fd, sid.c_str());
1219 reply(json::array({json::array({json::array({"mining.notify",sid})}),sid,4})); 3136 reply(json::array({json::array({json::array({"mining.notify",sid})}),sid,4}));
@@ -1320,6 +3237,22 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1320 } 3237 }
1321 unsigned int nNonce; memcpy(&nNonce, nb.data(), 4); 3238 unsigned int nNonce; memcpy(&nNonce, nb.data(), 4);
1322 3239
3240 // Reject a replayed (job, nonce) before it costs a rebuild and a
3241 // RandomX hash: nothing about the work below distinguishes a second
3242 // submission of a valid share from the first, so without this check
3243 // it passes every test and is credited again -- and roundShareCount
3244 // is what the payout split divides by, so each replay multiplies the
3245 // address's cut of the round. The set is per-connection and cleared
3246 // by SendJob on every job change, so only a same-job repeat is
3247 // refused.
3248 if (m->seenNonces.count(nNonce)) {
3249 LogPrint("worker", "[worker->pool] duplicate share from %s"
3250 " nonce=%u job=%s -- already credited, not counting again\n",
3251 m->address.c_str(), nNonce, submitJobId.c_str());
3252 reply(false, "Duplicate share");
3253 return true;
3254 }
3255
1323 // Rebuild this miner's exact block (unique merkle via extranonce2) 3256 // Rebuild this miner's exact block (unique merkle via extranonce2)
1324 CBlock b = BuildMinerBlock(gCurrentJob, m->extranonce2); 3257 CBlock b = BuildMinerBlock(gCurrentJob, m->extranonce2);
1325 b.nNonce = nNonce; 3258 b.nNonce = nNonce;
@@ -1347,6 +3280,7 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1347 m->roundShares++; 3280 m->roundShares++;
1348 m->totalShares++; 3281 m->totalShares++;
1349 m->vardiffShares++; 3282 m->vardiffShares++;
3283 m->seenNonces.insert(nNonce); // replay guard: credit each nonce once per job
1350 int64 shareTime = GetTime(); 3284 int64 shareTime = GetTime();
1351 m->lastSeen = shareTime; 3285 m->lastSeen = shareTime;
1352 roundShareCount[m->address]++; 3286 roundShareCount[m->address]++;
@@ -1453,7 +3387,11 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1453 int foundHeight = gCurrentJob.height; 3387 int foundHeight = gCurrentJob.height;
1454 // b is the block we submitted, untouched by ProcessBlock (which 3388 // b is the block we submitted, untouched by ProcessBlock (which
1455 // took a copy), so its hash is what the payout must be keyed on. 3389 // took a copy), so its hash is what the payout must be keyed on.
1456 QueuePayouts(foundHeight, b.GetHash(), roundShareCount, roundShareTotal); 3390 // Passed as a uint256, not hex: the ledger key must stay a real
3391 // hash. The remaining arguments are upstream's round-proof data.
3392 QueuePayouts(foundHeight, b.GetHash(), b.vtx[0].GetHash().GetHex(),
3393 b.vtx[0].vout[0].nValue, m->address,
3394 roundShareCount, roundShareTotal);
1457 // Reset round shares 3395 // Reset round shares
1458 roundShareCount.clear(); 3396 roundShareCount.clear();
1459 roundShareTotal = 0; 3397 roundShareTotal = 0;
@@ -1480,49 +3418,1892 @@ static bool HandleLine(Miner* m, const std::string& rawLine,
1480} 3418}
1481 3419
1482// --------------------------------------------------------------------------- 3420// ---------------------------------------------------------------------------
1483// Stats refresh -- called every 2s from event loop 3421// payout-ledger self-test
3422//
3423// Regression cover for the payout defects fixed in this series. Everything
3424// here runs against a scratch data directory and an in-memory wallet map; it
3425// never opens wallet.dat, never touches a real datadir, and never sends
3426// anything.
1484// --------------------------------------------------------------------------- 3427// ---------------------------------------------------------------------------
1485static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal) 3428
3429static bool PayoutTestReadFile(const std::string& path, std::string& out)
1486{ 3430{
1487 std::lock_guard<std::mutex> lk(gStatsMutex); 3431 FILE* f = fopen(path.c_str(), "r");
3432 if (!f) return false;
3433 fseek(f, 0, SEEK_END);
3434 long sz = ftell(f);
3435 fseek(f, 0, SEEK_SET);
3436 out.assign((size_t)(sz > 0 ? sz : 0), '\0');
3437 if (sz > 0 && fread(&out[0], 1, (size_t)sz, f) != (size_t)sz) {
3438 fclose(f);
3439 return false;
3440 }
3441 fclose(f);
3442 return true;
3443}
1488 3444
1489 gStats.authorizedMiners = 0; 3445// Delete the scratch data directory and everything in it. A sweep and not a
1490 gStats.blocksFound = blocksFoundThisSession; 3446// list of known names: LogPrint binds debug.log to whatever GetAppDir() is
1491 gStats.roundShares = roundShareTotal; 3447// when it first fires, so the test's own log lands in here too, and a
1492 gStats.totalHashRate = 0.0; 3448// directory left behind in src/ after `make tests` is litter.
1493 for (Miner* m : gMiners) 3449static void PayoutTestRemoveDir(const std::string& dir)
1494 if (m->authorised) gStats.authorizedMiners++; 3450{
3451#ifdef _WIN32
3452 WIN32_FIND_DATAA findData;
3453 HANDLE hFind = FindFirstFileA((dir + "\\*").c_str(), &findData);
3454 if (hFind != INVALID_HANDLE_VALUE) {
3455 do {
3456 std::string name = findData.cFileName;
3457 if (name == "." || name == "..") continue;
3458 DeleteFileA((dir + "\\" + name).c_str());
3459 } while (FindNextFileA(hFind, &findData));
3460 FindClose(hFind);
3461 }
3462 RemoveDirectoryA(dir.c_str());
3463#else
3464 // One level of recursion, because CreateTransaction() opens a Berkeley DB
3465 // environment and BDB puts its logs in a "database" subdirectory. Without
3466 // this the scratch directory cannot be removed and the test leaves one
3467 // behind on every run, which is what d5b3503 was for.
3468 DIR* d = opendir(dir.c_str());
3469 if (d) {
3470 struct dirent* ent;
3471 while ((ent = readdir(d)) != NULL) {
3472 std::string name = ent->d_name;
3473 if (name == "." || name == "..") continue;
3474 const std::string path = dir + "/" + name;
3475 if (unlink(path.c_str()) != 0)
3476 PayoutTestRemoveDir(path);
3477 }
3478 closedir(d);
3479 }
3480 rmdir(dir.c_str());
3481#endif
3482}
1495 3483
1496 gWorkerStats.clear(); 3484static bool PayoutTestWriteFile(const std::string& path, const std::string& data)
1497 for (Miner* m : gMiners) { 3485{
1498 if (!m->authorised || m->lastSeen <= 0) continue; 3486 FILE* f = fopen(path.c_str(), "w");
1499 PoolWorkerStatView v; 3487 if (!f) return false;
1500 v.address = m->address; 3488 bool ok = data.empty() || fwrite(data.c_str(), 1, data.size(), f) == data.size();
1501 v.worker = m->worker; 3489 return fclose(f) == 0 && ok;
1502 v.roundShares = m->roundShares; 3490}
1503 v.totalShares = m->totalShares; 3491
1504 v.lastSeen = m->lastSeen; 3492// Wipe every piece of payout state this test touches, so each section starts
1505 v.hashRate = m->hashRate; 3493// from the same place and a "restart" can be simulated by reloading from disk.
1506 gWorkerStats.push_back(v); 3494static void PayoutTestResetState()
1507 gStats.totalHashRate += m->hashRate; 3495{
1508 } 3496 {
1509 std::sort(gWorkerStats.begin(), gWorkerStats.end(), 3497 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
1510 [](const PoolWorkerStatView& a, const PoolWorkerStatView& b) { 3498 gPendingPayouts.clear();
1511 return a.roundShares != b.roundShares ? a.roundShares > b.roundShares 3499 }
1512 : a.lastSeen > b.lastSeen; 3500 {
3501 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3502 gPaying.clear();
3503 gQueuedPayouts.clear();
3504 }
3505 {
3506 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
3507 gPoolRounds.clear();
3508 }
3509 {
3510 std::lock_guard<std::mutex> lk(gClaimedTxidMutex);
3511 gLegacyClaimedTxids.clear();
3512 }
3513 {
3514 // A quarantine is meant to last the life of the process; each section
3515 // here is a fresh process as far as the code under test is concerned.
3516 std::lock_guard<std::mutex> lk(gRefusedFilesMutex);
3517 gRefusedFiles.clear();
3518 }
3519}
3520
3521// A BTF address whose hash160 is entirely under the test's control, so the
3522// output script the probe compares against can be built without a wallet.
3523static std::string PayoutTestAddress(unsigned char seed)
3524{
3525 uint160 h = 0;
3526 for (int i = 0; i < 20; i++)
3527 ((unsigned char*)&h)[i] = (unsigned char)(seed + i);
3528 return Hash160ToAddress(h);
3529}
3530
3531static CScript PayoutTestScriptFor(const std::string& address)
3532{
3533 uint160 h160;
3534 CScript sc;
3535 if (AddressToHash160(address, h160))
3536 sc << OP_DUP << OP_HASH160 << h160 << OP_EQUALVERIFY << OP_CHECKSIG;
3537 return sc;
3538}
3539
3540// Block index entries fabricated by PayoutTestAddWalletTx, so they can be
3541// freed and unhooked when the test finishes. gPayoutTestChainTip is the end of
3542// that chain: blocks are linked onto it rather than each replacing the tip,
3543// because IsInMainChain() is "pnext is set, or I am the tip".
3544static std::vector<CBlockIndex*> gPayoutTestBlockIndexes;
3545static CBlockIndex* gPayoutTestChainTip = NULL;
3546
3547static void PayoutTestClearBlockIndex()
3548{
3549 CRITICAL_BLOCK(cs_main)
3550 {
3551 for (CBlockIndex* p : gPayoutTestBlockIndexes) {
3552 for (std::map<uint256, CBlockIndex*>::iterator it = mapBlockIndex.begin();
3553 it != mapBlockIndex.end(); ) {
3554 if (it->second == p) mapBlockIndex.erase(it++);
3555 else ++it;
3556 }
3557 delete p;
3558 }
3559 gPayoutTestBlockIndexes.clear();
3560 gPayoutTestChainTip = NULL;
3561 pindexBest = NULL;
3562 }
3563}
3564
3565// Put a transaction in the wallet map that looks exactly like one this pool
3566// sent. `nonce` only exists to keep the hashes distinct.
3567//
3568// fConfirmed decides what GetDepthInMainChain() will answer for it, which is
3569// what the legacy probe now gates on. A confirmed one is given a block index
3570// entry of its own: an empty merkle branch hashes to the transaction itself,
3571// so a block whose merkle root IS that hash accepts it at index 0, and
3572// pointing pindexBest at that block puts it in the main chain one deep. An
3573// unconfirmed one keeps hashBlock 0, which is depth 0 -- the shape a payout
3574// left behind by a reorg has.
3575static std::string PayoutTestAddWalletTx(const std::string& comment,
3576 const std::string& address,
3577 int64 amount,
3578 int nonce,
3579 const std::string& structuredId,
3580 bool fConfirmed = true,
3581 const std::string& structuredBlock = "")
3582{
3583 CWalletTx wtx;
3584 wtx.vin.resize(1);
3585 wtx.vin[0].prevout.SetNull();
3586 wtx.vin[0].scriptSig << nonce;
3587 wtx.vout.resize(1);
3588 wtx.vout[0].nValue = amount;
3589 wtx.vout[0].scriptPubKey = PayoutTestScriptFor(address);
3590 if (!comment.empty())
3591 wtx.mapValue["comment"] = comment;
3592 if (!structuredId.empty()) {
3593 wtx.mapValue["pool_payout_id"] = structuredId;
3594 wtx.mapValue["pool_payout_address"] = address;
3595 wtx.mapValue["pool_payout_amount"] = strprintf("%lld", amount);
3596 if (!structuredBlock.empty())
3597 wtx.mapValue["pool_payout_block"] = structuredBlock;
3598 }
3599 uint256 hash = wtx.GetHash();
3600 if (fConfirmed) {
3601 // A block hash distinct from the transaction's own.
3602 uint256 blockHash = hash;
3603 ((unsigned char*)&blockHash)[0] ^= 0xff;
3604 CRITICAL_BLOCK(cs_main)
3605 {
3606 CBlockIndex* pindex = new CBlockIndex();
3607 pindex->hashMerkleRoot = hash;
3608 // Linked onto the end of the fabricated chain rather than simply
3609 // becoming the new tip: IsInMainChain() is "pnext is set, or I am
3610 // the tip", so a second confirmed transaction would otherwise take
3611 // the first one out of the chain again.
3612 pindex->pprev = gPayoutTestChainTip;
3613 pindex->nHeight = gPayoutTestChainTip ? gPayoutTestChainTip->nHeight + 1 : 1;
3614 if (gPayoutTestChainTip)
3615 gPayoutTestChainTip->pnext = pindex;
3616 gPayoutTestChainTip = pindex;
3617 std::map<uint256, CBlockIndex*>::iterator mi =
3618 mapBlockIndex.insert(std::make_pair(blockHash, pindex)).first;
3619 pindex->phashBlock = &mi->first;
3620 pindexBest = pindex;
3621 gPayoutTestBlockIndexes.push_back(pindex);
3622 }
3623 wtx.hashBlock = blockHash;
3624 wtx.nIndex = 0;
3625 }
3626 CRITICAL_BLOCK(cs_mapWallet)
3627 mapWallet[hash] = wtx;
3628 return hash.GetHex();
3629}
3630
3631// A real key, in the keystore, so a test wallet's outputs are IsMine() and can
3632// actually be signed -- both are needed for CreateTransaction to build the
3633// large transaction the fee schedule is being tested against. Inserted
3634// straight into the key maps rather than through AddKey(), which would also
3635// write it to a wallet.dat.
3636static std::string PayoutTestAddSpendableKey()
3637{
3638 CKey key;
3639 key.MakeNewKey();
3640 std::vector<unsigned char> vchPubKey = key.GetPubKey();
3641 CRITICAL_BLOCK(cs_mapKeys)
3642 {
3643 mapKeys[vchPubKey] = key.GetPrivKey();
3644 mapPubKeys[Hash160(vchPubKey)] = vchPubKey;
3645 }
3646 return Hash160ToAddress(Hash160(vchPubKey));
3647}
3648
3649// Give the wallet `n` spendable outputs of `each`, one transaction apiece --
3650// which is the granularity SelectCoins works at, and therefore the number of
3651// inputs the transaction spending them will carry.
3652//
3653// Deliberately not coinbases: a null prevout would make CWalletTx::GetCredit
3654// return 0 until COINBASE_SPENDABLE_DEPTH, and there is no chain here.
3655static void PayoutTestFundWallet(int n, int64 each, const std::string& address)
3656{
3657 const CScript sc = PayoutTestScriptFor(address);
3658 CRITICAL_BLOCK(cs_mapWallet)
3659 {
3660 for (int i = 0; i < n; i++) {
3661 CWalletTx wtx;
3662 wtx.vin.resize(1);
3663 wtx.vin[0].prevout.hash = (uint64)(i + 1);
3664 wtx.vin[0].prevout.n = 0;
3665 wtx.vout.resize(1);
3666 wtx.vout[0].nValue = each;
3667 wtx.vout[0].scriptPubKey = sc;
3668 mapWallet[wtx.GetHash()] = wtx;
3669 }
3670 }
3671}
3672
3673// Defined below with the event loop; the status-file publication test drives
3674// it directly.
3675static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal);
3676
3677int RunPayoutLedgerSelfTest()
3678{
3679 int nFail = 0;
3680 auto check = [&](bool ok, const char* msg) {
3681 fprintf(stdout, " %s %s\n", ok ? "ok" : "FAIL", msg);
3682 if (!ok) nFail++;
3683 };
3684
3685 fprintf(stdout, "payout-ledger self-test\n");
3686
3687 const std::string savedDataDir = strSetDataDir;
3688 const int savedBestHeight = nBestHeight;
3689
3690 // Scratch data directory in the current working directory. GetAppDir()
3691 // follows strSetDataDir, so this is what PayoutFilePath() and
3692 // PayingFilePath() resolve against for the rest of the test.
3693#ifdef _WIN32
3694 unsigned long pid = (unsigned long)GetCurrentProcessId();
3695#else
3696 unsigned long pid = (unsigned long)getpid();
3697#endif
3698 // Absolute, and the process moves into it.
3699 //
3700 // Both are needed by the CreateTransaction section: Berkeley DB resolves
3701 // its log directory relative to the environment home, so a relative
3702 // datadir makes dbenv.open() fail with DB_RUNRECOVERY; and BDB opens
3703 // db.log relative to the working directory, which is src/ during `make
3704 // tests`. Moving in puts it inside the scratch directory, where it dies
3705 // with everything else.
3706 std::string cwd = ".";
3707 {
3708#ifdef _WIN32
3709 char buf[MAX_PATH];
3710 if (GetCurrentDirectoryA(MAX_PATH, buf) > 0) cwd = buf;
3711#else
3712 char buf[4096];
3713 if (getcwd(buf, sizeof(buf)) != NULL) cwd = buf;
3714#endif
3715 }
3716 std::string dir = cwd + "/" +
3717 strprintf("payout_selftest_%d_%lu", (int)GetTime(), pid);
3718#ifdef _WIN32
3719 CreateDirectoryA(dir.c_str(), NULL);
3720 SetCurrentDirectoryA(dir.c_str());
3721#else
3722 mkdir(dir.c_str(), 0700);
3723 if (chdir(dir.c_str()) != 0) { /* stay put; the paths are absolute anyway */ }
3724#endif
3725 strSetDataDir = dir;
3726 fprintf(stdout, " scratch datadir: %s\n", dir.c_str());
3727
3728 const std::string ledgerPath = PayoutFilePath();
3729 const std::string payingPath = PayingFilePath();
3730 const std::string addrA = PayoutTestAddress(0x11);
3731 const std::string addrB = PayoutTestAddress(0x55);
3732
3733 // -- 1. A restart must not duplicate an in-flight record -----------------
3734 //
3735 // The ledger entry is grandfathered (zero hash) on purpose: it keeps
3736 // PayoutBlockStillOnChain out of the picture, which needs a block index,
3737 // and it exercises the height fallback at the same time.
3738 {
3739 PayoutTestResetState();
3740 remove(ledgerPath.c_str());
3741 remove(payingPath.c_str());
3742
3743 PendingPayout pp;
3744 pp.blockHeight = 4242;
3745 pp.blockHash = 0;
3746 pp.matureAtHeight = 4242;
3747 pp.amounts[addrA] = 10 * COIN;
3748 pp.amounts[addrB] = 20 * COIN;
3749 VersionedBlob snap;
3750 {
3751 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3752 gPendingPayouts.push_back(pp);
3753 snap = SerializePendingPayoutsLocked();
3754 }
3755 PublishPendingPayouts(snap);
3756
3757 nBestHeight = 5000;
3758 FlushMaturePayouts();
3759 size_t afterFirstFlush = 0;
3760 {
3761 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3762 afterFirstFlush = gPaying.size();
3763 }
3764 check(afterFirstFlush == 2,
3765 "a matured round explodes into one record per unpaid recipient");
3766
3767 // Restart: drop everything in memory and come back from the files.
3768 for (int restart = 1; restart <= 2; restart++) {
3769 PayoutTestResetState();
3770 LoadPendingPayouts();
3771 LoadPaying();
3772 ReconcilePayouts();
3773
3774 size_t queued = 0;
3775 {
3776 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3777 queued = gQueuedPayouts.size();
3778 }
3779 if (restart == 1)
3780 check(queued == 1,
3781 "startup rebuilds the exploded-round set from the restored"
3782 " in-flight list");
3783
3784 FlushMaturePayouts();
3785 size_t n = 0;
3786 {
3787 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3788 n = gPaying.size();
3789 }
3790 check(n == 2, restart == 1
3791 ? "a restart does not duplicate an in-flight record"
3792 : "a second restart does not duplicate one either");
3793 }
3794
3795 // The other half of the guard: even with the round-level dedupe set
3796 // deliberately emptied, the per-recipient check against gPaying must
3797 // stop a second standing order for the same money.
3798 {
3799 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3800 gQueuedPayouts.clear();
3801 }
3802 FlushMaturePayouts();
3803 size_t n = 0;
3804 {
3805 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3806 n = gPaying.size();
3807 }
3808 check(n == 2,
3809 "the flush refuses a second record for a (round, address) already"
3810 " in flight even with the round set lost");
3811 }
3812
3813 // -- 2. Duplicates already on disk are collapsed as they are read --------
3814 {
3815 PayoutTestResetState();
3816 nlohmann::json arr = nlohmann::json::array();
3817 for (int i = 0; i < 2; i++) {
3818 nlohmann::json o;
3819 o["blockHeight"] = 4242;
3820 o["blockHash"] = uint256(0).ToString();
3821 o["matureAtHeight"] = 4242;
3822 o["address"] = addrA;
3823 o["amount"] = (int64)(10 * COIN);
3824 o["retries"] = i == 0 ? 7 : 0;
3825 o["deferrals"] = i == 0 ? 0 : 3;
3826 arr.push_back(o);
3827 }
3828 check(PayoutTestWriteFile(payingPath, arr.dump(2)),
3829 "wrote a paying file containing the duplicate the old build made");
3830 LoadPaying();
3831 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3832 check(gPaying.size() == 1,
3833 "loading collapses a duplicate in-flight record from disk");
3834 check(gPaying.size() == 1 && gPaying[0].retries == 7 && gPaying[0].deferrals == 3,
3835 "collapsing merges the attempt counts upwards, not resetting them");
3836 }
3837
3838 // -- 3. State updates address a slot, not a value -----------------------
3839 //
3840 // Build the exact pathology the old code produced -- two records equal in
3841 // every field -- and prove the update lands on the one asked for. Under
3842 // value matching the first record absorbed everything and the second was
3843 // untouchable.
3844 {
3845 PayoutTestResetState();
3846 PayingRecord a;
3847 a.blockHeight = 4242; a.blockHash = 0; a.matureAtHeight = 4242;
3848 a.address = addrA; a.amount = 10 * COIN;
3849 PayingRecord b = a;
3850 {
3851 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3852 a.id = ++gPayingNextId;
3853 b.id = ++gPayingNextId;
3854 gPaying.push_back(a);
3855 gPaying.push_back(b);
3856 }
3857 PayingRecord upd = b;
3858 upd.retries = 5;
3859 upd.retryAfter = 1234567;
3860 ReschedulePayment(upd);
3861 {
3862 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3863 check(gPaying.size() == 2 &&
3864 gPaying[0].retries == 0 && gPaying[1].retries == 5,
3865 "rescheduling updates the record it was given, not the first"
3866 " one that looks like it");
3867 }
3868 ForgetPayment(a);
3869 {
3870 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3871 check(gPaying.size() == 1 && gPaying[0].id == b.id,
3872 "forgetting erases the record it was given, not the first"
3873 " one that looks like it");
3874 }
3875 }
3876
3877 // -- 4. retryAfter survives a restart ------------------------------------
3878 {
3879 PayoutTestResetState();
3880 remove(payingPath.c_str());
3881 PayingRecord r;
3882 r.blockHeight = 77; r.blockHash = 0; r.matureAtHeight = 77;
3883 r.address = addrA; r.amount = 3 * COIN;
3884 r.retryAfter = 1700000000LL;
3885 r.deferrals = 9;
3886 VersionedBlob snap;
3887 {
3888 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3889 r.id = ++gPayingNextId;
3890 gPaying.push_back(r);
3891 snap = SerializePayingLocked();
3892 }
3893 PublishPaying(snap);
3894 PayoutTestResetState();
3895 LoadPaying();
3896 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
3897 check(gPaying.size() == 1 && gPaying[0].retryAfter == 1700000000LL &&
3898 gPaying[0].deferrals == 9,
3899 "the retry clock that is actually used survives a restart");
3900 }
3901
3902 // -- 5. The ledger file cannot be published from a stale snapshot --------
3903 {
3904 PayoutTestResetState();
3905 remove(ledgerPath.c_str());
3906
3907 PendingPayout one;
3908 one.blockHeight = 100; one.blockHash = 0; one.matureAtHeight = 100;
3909 one.amounts[addrA] = 1 * COIN;
3910 VersionedBlob older, newer;
3911 {
3912 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3913 gPendingPayouts.push_back(one);
3914 older = SerializePendingPayoutsLocked();
3915 PendingPayout two;
3916 two.blockHeight = 200; two.blockHash = 0; two.matureAtHeight = 200;
3917 two.amounts[addrB] = 2 * COIN;
3918 gPendingPayouts.push_back(two);
3919 newer = SerializePendingPayoutsLocked();
3920 }
3921 check(newer.seq > older.seq,
3922 "a later serialisation carries a strictly higher version");
3923
3924 // The order the race produced: the payout thread's older image
3925 // arriving after the event loop's newer one.
3926 PublishPendingPayouts(newer);
3927 PublishPendingPayouts(older);
3928 std::string onDisk;
3929 check(PayoutTestReadFile(ledgerPath, onDisk) && onDisk == newer.blob,
3930 "an older snapshot cannot overwrite a newer published ledger");
3931
3932 // And the ordinary direction still works.
3933 VersionedBlob newest;
3934 {
3935 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
3936 gPendingPayouts.clear();
3937 newest = SerializePendingPayoutsLocked();
3938 }
3939 PublishPendingPayouts(newest);
3940 check(PayoutTestReadFile(ledgerPath, onDisk) && onDisk == newest.blob,
3941 "a newer snapshot does replace the published ledger");
3942 }
3943
3944 // -- 6. Concurrent publishers cannot interleave a file -------------------
3945 //
3946 // Two threads opening and renaming the same "<path>.tmp" over the same
3947 // target was the original defect. This test was mutation-tested and did
3948 // not catch it: with the file mutex removed the suite still went green 2
3949 // runs in 5, and the "never leave it unparseable" assertion fired 0 times
3950 // in 5. Two reasons, both fixed here.
3951 //
3952 // First, every image had the same structure and near enough the same
3953 // length, so an interleaving of two of them was still valid JSON. The
3954 // images now alternate between ONE entry and FIVE THOUSAND, so a write
3955 // that lands on top of another leaves either a truncated array or a small
3956 // image followed by the tail of a large one -- neither of which parses.
3957 //
3958 // Second, and worse, almost nothing was being written. Each thread walked
3959 // a permutation of the images, and PublishVersioned drops any blob that is
3960 // not newer than what is on disk, so only the record highs in each
3961 // permutation ever reached the disk -- on the order of ten writes in
3962 // total, four threads, no contention to speak of. Every publish here is
3963 // stamped from a shared counter at the moment it is made, so every one of
3964 // them is newer than the last and every one of them writes.
3965 //
3966 // And the file is watched while the writers run, not only inspected after
3967 // they finish. "The published file is never observed unparseable" is the
3968 // actual invariant -- rename(2) is atomic, so a reader can only ever see a
3969 // completely written image -- and checking it once at the end samples the
3970 // one moment when a torn file is least likely to still be there.
3971 {
3972 const std::string racePath = dir + "/publish_race_selftest.json";
3973 std::mutex raceMutex;
3974 uint64 racePublished = 0;
3975
3976 std::string tiny, huge;
3977 {
3978 nlohmann::json one = nlohmann::json::array();
3979 nlohmann::json o;
3980 o["image"] = 0;
3981 o["entry"] = 0;
3982 o["address"] = addrA;
3983 o["amount"] = (int64)COIN;
3984 one.push_back(o);
3985 tiny = one.dump(2);
3986
3987 nlohmann::json many = nlohmann::json::array();
3988 for (int k = 0; k < 5000; k++) {
3989 nlohmann::json e;
3990 e["image"] = 1;
3991 e["entry"] = k;
3992 e["address"] = addrA;
3993 e["amount"] = (int64)(k + 1) * COIN;
3994 many.push_back(e);
3995 }
3996 huge = many.dump(2);
3997 }
3998
3999 const int kThreads = 4;
4000 const int kPerThread = 100;
4001 const uint64 kTotal = (uint64)kThreads * kPerThread;
4002
4003 std::atomic<uint64> nextSeq(0);
4004 std::atomic<bool> racing(true);
4005 std::atomic<int> tornReads(0);
4006 std::atomic<int> goodReads(0);
4007
4008 std::thread watcher([&]() {
4009 bool seenOnce = false;
4010 while (racing) {
4011 std::string s;
4012 if (!PayoutTestReadFile(racePath, s)) {
4013 // Only meaningful once the file exists at all.
4014 if (seenOnce) tornReads++;
4015 continue;
4016 }
4017 seenOnce = true;
4018 try {
4019 if (nlohmann::json::parse(s).is_array()) goodReads++;
4020 else tornReads++;
4021 } catch (...) {
4022 tornReads++;
4023 }
4024 }
1513 }); 4025 });
1514 4026
1515 // gPendingPayouts is event-loop-only data (no lock needed here since 4027 std::vector<std::thread> threads;
1516 // RefreshStats is always called from the event loop thread). 4028 for (int t = 0; t < kThreads; t++) {
1517 gPayoutViews.clear(); 4029 threads.push_back(std::thread([&]() {
1518 for (const PendingPayout& pp : gPendingPayouts) { 4030 for (int i = 0; i < kPerThread; i++) {
1519 PendingPayoutView pv; 4031 VersionedBlob v;
1520 pv.matureAtHeight = pp.matureAtHeight; 4032 v.seq = ++nextSeq;
1521 pv.recipients = (int)pp.amounts.size(); 4033 v.blob = (v.seq & 1) ? tiny : huge;
1522 pv.totalAmount = 0; 4034 PublishVersioned(racePath, v, raceMutex, racePublished,
1523 for (auto& kv : pp.amounts) pv.totalAmount += kv.second; 4035 "publish race selftest");
1524 gPayoutViews.push_back(pv); 4036 }
4037 }));
4038 }
4039 for (std::thread& th : threads)
4040 th.join();
4041 racing = false;
4042 watcher.join();
4043
4044 check(goodReads > 0,
4045 "the race test actually observed the file while it was being"
4046 " written");
4047 check(tornReads == 0,
4048 "four threads publishing the same file never leave it"
4049 " unparseable, at any instant while they are doing it");
4050
4051 // The highest sequence number issued is the one the file must settle
4052 // on: PublishVersioned refuses anything older, so whichever order the
4053 // writes complete in, that image is the last one that can land.
4054 const std::string& expected = (kTotal & 1) ? tiny : huge;
4055 std::string onDisk;
4056 bool read = PayoutTestReadFile(racePath, onDisk);
4057 check(read && onDisk == expected,
4058 "the published file settles on the highest version, whatever the"
4059 " completion order");
4060 remove(racePath.c_str());
4061 remove((racePath + ".tmp").c_str());
4062 }
4063
4064 // -- 7. Grandfathered zero-hash rounds do not answer for each other ------
4065 {
4066 uint256 h1 = 0, h2 = 0;
4067 h1.SetHex("00000000000000000000000000000000000000000000000000000000000000aa");
4068 h2.SetHex("00000000000000000000000000000000000000000000000000000000000000bb");
4069 check(!SamePayoutRound(100, 0, 200, 0),
4070 "two zero-hash entries at different heights are different rounds");
4071 check(SamePayoutRound(100, 0, 100, 0),
4072 "two zero-hash entries at the same height are the same round");
4073 check(SamePayoutRound(100, h1, 200, h1),
4074 "a real hash identifies the round on its own");
4075 check(!SamePayoutRound(100, h1, 100, h2),
4076 "different hashes at one height are different rounds");
4077 check(!SamePayoutRound(100, h1, 100, 0),
4078 "a grandfathered entry never matches a hash-keyed one");
4079 check(!PayoutRoundIdentifiable(0, 0) && PayoutRoundIdentifiable(0, h1) &&
4080 PayoutRoundIdentifiable(7, 0),
4081 "an entry with neither hash nor height is not adjudicable");
4082
4083 // What actually makes a grandfathered same-height pair safe, since the
4084 // comment on FindRecordedPoolPayout used to claim it was the orphan
4085 // check. It is not: the orphan check passes every zero-hash entry, and
4086 // every pre-upgrade entry has a zero hash.
4087 check(PayoutBlockStillOnChain(0),
4088 "the orphan check passes every grandfathered entry, so it cannot"
4089 " be what voids a losing twin");
4090 PayoutTestResetState();
4091 remove(ledgerPath.c_str());
4092 remove(payingPath.c_str());
4093 {
4094 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4095 PendingPayout twin;
4096 twin.blockHeight = 300; twin.blockHash = 0; twin.matureAtHeight = 300;
4097 twin.amounts[addrA] = 1 * COIN;
4098 gPendingPayouts.push_back(twin);
4099 gPendingPayouts.push_back(twin); // the losing side of a reorg
4100 }
4101 nBestHeight = 5000;
4102 FlushMaturePayouts();
4103 {
4104 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
4105 check(gPaying.size() == 1,
4106 "two grandfathered rounds at one height are one round, so the"
4107 " pair presents as one debt and is paid at most once");
4108 }
4109
4110 // End to end: recording a payment against one grandfathered round must
4111 // not settle the other's debt.
4112 PayoutTestResetState();
4113 remove(ledgerPath.c_str());
4114 {
4115 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4116 PendingPayout first;
4117 first.blockHeight = 100; first.blockHash = 0; first.matureAtHeight = 100;
4118 first.amounts[addrA] = 1 * COIN;
4119 PendingPayout second;
4120 second.blockHeight = 200; second.blockHash = 0; second.matureAtHeight = 200;
4121 second.amounts[addrA] = 1 * COIN;
4122 gPendingPayouts.push_back(first);
4123 gPendingPayouts.push_back(second);
4124 }
4125 PayingRecord r;
4126 r.blockHeight = 200; r.blockHash = 0; r.matureAtHeight = 200;
4127 r.address = addrA; r.amount = 1 * COIN;
4128 RecordPayoutTxid(r, "txid-for-the-second-round");
4129 {
4130 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4131 // The second round was fully paid and retired; the first is
4132 // untouched and still owed.
4133 check(gPendingPayouts.size() == 1 &&
4134 gPendingPayouts[0].blockHeight == 100 &&
4135 gPendingPayouts[0].paidTxids.empty(),
4136 "a txid recorded against one zero-hash round leaves the other"
4137 " still owed");
4138 }
4139 }
4140
4141 // -- 8. A payment with only a legacy comment is recognised as paid -------
4142 {
4143 PayoutTestResetState();
4144 const int height = 8080;
4145 const int64 amount = 7 * COIN;
4146 const std::string id = PayoutIdForBlock(height, 0);
4147 const std::string comment = LegacyPayoutComment(height);
4148 std::set<std::string> none;
4149 std::string found;
4150 // No round proof exists for these rounds, so legacy evidence is
4151 // admissible -- computed exactly as the payout thread computes it.
4152 const bool fAllowLegacy = !PoolRoundHasProof(id);
4153
4154 check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, none, found)
4155 == PAYOUT_PROBE_NONE,
4156 "an empty wallet is not evidence that anything was paid");
4157
4158 const std::string legacyTxid =
4159 PayoutTestAddWalletTx(comment, addrA, amount, 1, "");
4160 check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, none, found)
4161 == PAYOUT_PROBE_LEGACY && found == legacyTxid,
4162 "a payment carrying only a pre-upgrade comment is recognised as"
4163 " already paid");
4164
4165 check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found)
4166 == PAYOUT_PROBE_NONE,
4167 "the legacy match requires the output to pay THIS recipient");
4168 check(FindRecordedPoolPayout(id, height, 0, addrA, amount + COIN, fAllowLegacy, none, found)
4169 == PAYOUT_PROBE_NONE,
4170 "the legacy match requires the output to pay exactly this amount");
4171 check(FindRecordedPoolPayout(id, height + 1, 0, addrA, amount, fAllowLegacy, none, found)
4172 == PAYOUT_PROBE_NONE,
4173 "the legacy match requires the comment for this block height");
4174
4175 std::set<std::string> spent;
4176 spent.insert(legacyTxid);
4177 check(FindRecordedPoolPayout(id, height, 0, addrA, amount, fAllowLegacy, spent, found)
4178 == PAYOUT_PROBE_NONE,
4179 "one legacy transaction cannot settle a second debt");
4180
4181 // A transaction written by THIS build carries pool_payout_id, and must
4182 // never be offered as legacy evidence for a different round -- that is
4183 // exactly what the structured metadata exists to prevent.
4184 const std::string otherId = PayoutIdForBlock(height, 0) + "-other";
4185 PayoutTestAddWalletTx(comment, addrB, amount, 2, otherId);
4186 check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found)
4187 == PAYOUT_PROBE_NONE,
4188 "another round's structured payment is not legacy evidence for"
4189 " this one");
4190
4191 // The structured path still wins where it applies.
4192 const std::string structuredTxid =
4193 PayoutTestAddWalletTx(comment, addrB, amount, 3, id);
4194 check(FindRecordedPoolPayout(id, height, 0, addrB, amount, fAllowLegacy, none, found)
4195 == PAYOUT_PROBE_STRUCTURED && found == structuredTxid,
4196 "structured payout metadata is matched in preference to a comment");
4197
4198 // A comment-matching transaction that never made it into the chain is
4199 // evidence of NON-payment. This tree has seen a 280-block reorg; a
4200 // pre-upgrade payout invalidated by one is still in mapWallet with its
4201 // comment intact, and the first boot after the upgrade is exactly when
4202 // it would be matched.
4203 CRITICAL_BLOCK(cs_mapWallet)
4204 mapWallet.clear();
4205 PayoutTestClearBlockIndex();
4206 const std::string addrC = PayoutTestAddress(0x33);
4207 PayoutTestAddWalletTx(comment, addrC, amount, 4, "", false);
4208 check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, none, found)
4209 == PAYOUT_PROBE_NONE,
4210 "a pre-upgrade payout that never confirmed is not read as"
4211 " payment");
4212
4213 // The same transaction, confirmed, is.
4214 CRITICAL_BLOCK(cs_mapWallet)
4215 mapWallet.clear();
4216 const std::string confirmedTxid =
4217 PayoutTestAddWalletTx(comment, addrC, amount, 4, "", true);
4218 check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, none, found)
4219 == PAYOUT_PROBE_LEGACY && found == confirmedTxid,
4220 "the same payout, confirmed, is");
4221
4222 // And an unconfirmed one does not shadow a confirmed one that would
4223 // have matched: the probe used to stop at the first comment match.
4224 const std::string alsoConfirmed =
4225 PayoutTestAddWalletTx(comment, addrC, amount, 6, "", true);
4226 PayoutTestAddWalletTx(comment, addrC, amount, 5, "", false);
4227 std::set<std::string> spentFirst;
4228 spentFirst.insert(confirmedTxid);
4229 check(FindRecordedPoolPayout(id, height, 0, addrC, amount, fAllowLegacy, spentFirst, found)
4230 == PAYOUT_PROBE_LEGACY && found == alsoConfirmed,
4231 "an unconfirmed transaction does not hide a confirmed one behind"
4232 " it");
4233
4234 CRITICAL_BLOCK(cs_mapWallet)
4235 mapWallet.clear();
4236 PayoutTestClearBlockIndex();
1525 } 4237 }
4238
4239 // -- 9. On-disk compatibility with the live production ledger ------------
4240 {
4241 PayoutTestResetState();
4242 // Exactly the shape of the file on the pool node: amounts, blockHash,
4243 // blockHeight, matureAtHeight, and nothing else.
4244 const char* live =
4245 "[{\"amounts\":{\"%s\":1234500000},"
4246 "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000cd\","
4247 "\"blockHeight\":1000,"
4248 "\"matureAtHeight\":1120}]";
4249 check(PayoutTestWriteFile(ledgerPath, strprintf(live, addrA.c_str())),
4250 "wrote a ledger file in the live production shape");
4251 LoadPendingPayouts();
4252 {
4253 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4254 check(gPendingPayouts.size() == 1 &&
4255 gPendingPayouts[0].blockHeight == 1000 &&
4256 gPendingPayouts[0].matureAtHeight == 1120 &&
4257 gPendingPayouts[0].blockHash != 0 &&
4258 gPendingPayouts[0].amounts.size() == 1 &&
4259 gPendingPayouts[0].amounts[addrA] == 1234500000LL,
4260 "the live production pending_payouts.json still loads");
4261 }
4262
4263 // And the shape the merged build writes, retryAfter included, which
4264 // this build no longer writes but must still read past.
4265 PayoutTestResetState();
4266 const char* merged =
4267 "[{\"amounts\":{\"%s\":100000000},"
4268 "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000ef\","
4269 "\"blockHeight\":2000,\"matureAtHeight\":2120,\"retryAfter\":0,"
4270 "\"paidTxids\":{\"%s\":\"abc123\"}}]";
4271 check(PayoutTestWriteFile(ledgerPath,
4272 strprintf(merged, addrA.c_str(), addrA.c_str())),
4273 "wrote a ledger file in the merged build's shape");
4274 LoadPendingPayouts();
4275 {
4276 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4277 check(gPendingPayouts.size() == 1 &&
4278 gPendingPayouts[0].blockHeight == 2000 &&
4279 gPendingPayouts[0].paidTxids.size() == 1 &&
4280 gPendingPayouts[0].paidTxids[addrA] == "abc123",
4281 "a ledger file carrying retryAfter and paidTxids still loads");
4282 }
4283
4284 // A round trip through this build must not lose anything either.
4285 VersionedBlob snap;
4286 {
4287 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4288 snap = SerializePendingPayoutsLocked();
4289 }
4290 PublishPendingPayouts(snap);
4291 PayoutTestResetState();
4292 LoadPendingPayouts();
4293 {
4294 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4295 check(gPendingPayouts.size() == 1 &&
4296 gPendingPayouts[0].blockHeight == 2000 &&
4297 gPendingPayouts[0].matureAtHeight == 2120 &&
4298 gPendingPayouts[0].paidTxids[addrA] == "abc123",
4299 "what this build writes, this build reads back unchanged");
4300 }
4301 }
4302
4303 // -- 10. A forfeit is published, not just dropped ------------------------
4304 {
4305 PayoutTestResetState();
4306 remove(ledgerPath.c_str());
4307 const std::string roundsPath = dir + "/pool_rounds_selftest.json";
4308 const std::string savedRoundsFile = strPoolRoundsFile;
4309 strPoolRoundsFile = roundsPath;
4310 remove(roundsPath.c_str());
4311
4312 const int height = 606;
4313 PoolRoundProof round;
4314 round.id = PayoutIdForBlock(height, 0);
4315 round.blockHeight = height;
4316 round.matureAtHeight = height;
4317 round.payouts[addrA] = 5 * COIN;
4318 round.payouts[addrB] = 6 * COIN;
4319 RecordPoolRound(round);
4320
4321 {
4322 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4323 PendingPayout pp;
4324 pp.blockHeight = height; pp.blockHash = 0; pp.matureAtHeight = height;
4325 pp.amounts[addrA] = 5 * COIN;
4326 pp.amounts[addrB] = 6 * COIN;
4327 gPendingPayouts.push_back(pp);
4328 }
4329
4330 PayingRecord r;
4331 r.blockHeight = height; r.blockHash = 0; r.matureAtHeight = height;
4332 r.address = addrA; r.amount = 5 * COIN;
4333 ForfeitPayoutRecipient(r, "selftest reason");
4334
4335 {
4336 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4337 check(gPendingPayouts.size() == 1 &&
4338 gPendingPayouts[0].amounts.count(addrA) == 0 &&
4339 gPendingPayouts[0].amounts.count(addrB) == 1,
4340 "a forfeit removes only that recipient from the ledger entry");
4341 }
4342
4343 std::string onDisk;
4344 bool proofOk = false;
4345 if (PayoutTestReadFile(roundsPath, onDisk)) {
4346 try {
4347 nlohmann::json root = nlohmann::json::parse(onDisk);
4348 const nlohmann::json& p = root["rounds"][0]["payouts"];
4349 proofOk = p[addrA].value("forfeited", false) &&
4350 p[addrA].value("forfeitReason", "") == "selftest reason" &&
4351 p[addrA]["txid"].is_null() &&
4352 !p[addrB].contains("forfeited");
4353 } catch (...) {
4354 proofOk = false;
4355 }
4356 }
4357 check(proofOk,
4358 "the published round proof records the forfeit instead of"
4359 " advertising the address as still owed");
4360
4361 // And it survives a reload of the proof file.
4362 {
4363 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
4364 gPoolRounds.clear();
4365 }
4366 LoadPoolRounds();
4367 {
4368 std::lock_guard<std::mutex> lk(gPoolRoundsMutex);
4369 check(gPoolRounds.size() == 1 &&
4370 gPoolRounds[0].forfeited.count(addrA) == 1 &&
4371 gPoolRounds[0].forfeited[addrA] == "selftest reason",
4372 "the forfeit is read back from pool_rounds.json");
4373 }
4374
4375 remove(roundsPath.c_str());
4376 remove((roundsPath + ".tmp").c_str());
4377 strPoolRoundsFile = savedRoundsFile;
4378 }
4379
4380 // -- 11. A ledger that failed to load is never written back --------------
4381 //
4382 // The loaders push entries into the live vector inside the try, so an
4383 // exception part way down a file leaves a PARTIAL ledger in memory: the
4384 // entries before the bad one and nothing after it. Publishing that -- and
4385 // ReconcilePayouts publishes unconditionally, milliseconds into startup --
4386 // writes the partial state over the file and destroys the only record of
4387 // what was owed.
4388 {
4389 PayoutTestResetState();
4390 remove(ledgerPath.c_str());
4391
4392 // Two entries, the second with a matureAtHeight that is not a number.
4393 // The first loads; the second throws where it is read.
4394 const std::string corrupt = strprintf(
4395 "[{\"amounts\":{\"%s\":100000000},\"blockHash\":\"%s\","
4396 "\"blockHeight\":10,\"matureAtHeight\":10},"
4397 "{\"amounts\":{\"%s\":900000000},\"blockHash\":\"%s\","
4398 "\"blockHeight\":20,\"matureAtHeight\":\"twenty\"}]",
4399 addrA.c_str(), uint256(0).ToString().c_str(),
4400 addrB.c_str(), uint256(0).ToString().c_str());
4401 check(PayoutTestWriteFile(ledgerPath, corrupt),
4402 "wrote a ledger whose second entry does not parse");
4403
4404 const int64 t0 = GetTime();
4405 LoadPendingPayouts();
4406 const int64 t1 = GetTime();
4407
4408 size_t loaded = 0;
4409 {
4410 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4411 loaded = gPendingPayouts.size();
4412 }
4413 check(loaded == 1,
4414 "a mid-file parse error does leave a partial ledger in memory --"
4415 " which is the reason for everything below");
4416
4417 std::string gone;
4418 check(!PayoutTestReadFile(ledgerPath, gone),
4419 "the file that did not parse is moved out of the way");
4420
4421 // The rename stamps GetTime(); it was taken between t0 and t1.
4422 std::string kept, keptPath;
4423 for (int64 t = t0; t <= t1 && keptPath.empty(); t++) {
4424 const std::string candidate =
4425 strprintf("%s.bad.%lld", ledgerPath.c_str(), (long long)t);
4426 if (PayoutTestReadFile(candidate, kept))
4427 keptPath = candidate;
4428 }
4429 check(!keptPath.empty() && kept == corrupt,
4430 "it is kept, byte for byte, as <path>.bad.<timestamp>");
4431
4432 // The publish that used to destroy it.
4433 VersionedBlob partial;
4434 {
4435 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4436 partial = SerializePendingPayoutsLocked();
4437 }
4438 PublishPendingPayouts(partial);
4439 check(!PayoutTestReadFile(ledgerPath, gone),
4440 "and the partial ledger is not published over it");
4441
4442 // The refusal is per path and lasts the process: a second attempt,
4443 // with more state, is refused too.
4444 {
4445 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4446 PendingPayout pp;
4447 pp.blockHeight = 30; pp.blockHash = 0; pp.matureAtHeight = 30;
4448 pp.amounts[addrA] = 5 * COIN;
4449 gPendingPayouts.push_back(pp);
4450 partial = SerializePendingPayoutsLocked();
4451 }
4452 PublishPendingPayouts(partial);
4453 check(!PayoutTestReadFile(ledgerPath, gone),
4454 "the refusal is not a one-shot -- the path stays unwritable");
4455
4456 if (!keptPath.empty())
4457 remove(keptPath.c_str());
4458 PayoutTestResetState();
4459 remove(ledgerPath.c_str());
4460
4461 // And a file that parses is not quarantined, so the ordinary path is
4462 // still the ordinary path.
4463 check(PayoutTestWriteFile(ledgerPath, "[]"),
4464 "wrote a ledger that parses");
4465 LoadPendingPayouts();
4466 {
4467 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4468 PendingPayout pp;
4469 pp.blockHeight = 40; pp.blockHash = 0; pp.matureAtHeight = 40;
4470 pp.amounts[addrA] = 2 * COIN;
4471 gPendingPayouts.push_back(pp);
4472 partial = SerializePendingPayoutsLocked();
4473 }
4474 PublishPendingPayouts(partial);
4475 std::string written;
4476 check(PayoutTestReadFile(ledgerPath, written) && written == partial.blob,
4477 "a ledger that loaded cleanly is still published normally");
4478 }
4479
4480 // -- 12. A send that failed for want of coin is not a fault --------------
4481 //
4482 // The defect this replaces: the classifier decided "was this a funding
4483 // problem?" by re-reading GetBalance() and allowing amount +
4484 // nTransactionFee + CENT. nTransactionFee is 0 on a headless node, and
4485 // GetMinFee() is a step function -- 0 below 10,000 bytes, 11 * CENT at it
4486 // -- so the allowance was either unnecessary or eleven times too small,
4487 // and the ledger's own retry budget then wrote the debt off after 1440
4488 // repeats of the same stable condition.
4489 //
4490 // This is tested against a real wallet and the real CreateTransaction, not
4491 // against injected values: the honest question is not "does my enum
4492 // switch work" but "does a wallet holding many small coins actually report
4493 // the state I think it does". 60 outputs of 1 BTF give ~180 signed P2PKH
4494 // bytes each, which is where the fee schedule steps. Nothing is committed
4495 // or broadcast -- CreateTransaction only builds.
4496 {
4497 PayoutTestResetState();
4498 CRITICAL_BLOCK(cs_mapWallet)
4499 mapWallet.clear();
4500
4501 // CreateTransaction opens CTxDB("r"), which throws if blkindex.dat is
4502 // not there. Create it in the scratch datadir first.
4503 bool dbOk = true;
4504 try {
4505 CTxDB txdb("cr");
4506 } catch (const std::exception&) {
4507 dbOk = false;
4508 }
4509 check(dbOk, "a scratch block index exists for CreateTransaction to open");
4510
4511 const std::string walletAddr = PayoutTestAddSpendableKey();
4512 const std::string payee = PayoutTestAddress(0x77);
4513 const CScript payeeScript = PayoutTestScriptFor(payee);
4514
4515 // -- the >= 10 KB case: 60 small coins, balance = amount + 0.05 ------
4516 PayoutTestFundWallet(60, 1 * COIN, walletAddr);
4517 const int64 balance = GetBalance();
4518 check(balance == 60 * COIN,
4519 "the test wallet holds 60 separately spendable coins");
4520
4521 const int64 amount = balance - 5 * CENT;
4522 // The pre-send gate in PayoutThreadFn passes: the balance covers the
4523 // amount and the only fee that can be known in advance.
4524 check(balance >= amount + nTransactionFee,
4525 "the pre-send gate would let this payment through");
4526
4527 CWalletTx wtx;
4528 SendMoneyOutcome outcome;
4529 int64 feeRequired = 0;
4530 WalletSendResult result = WALLET_SEND_OK;
4531 bool built = true;
4532 if (dbOk) {
4533 try {
4534 built = CreateTransaction(payeeScript, amount, wtx, feeRequired,
4535 &result);
4536 } catch (const std::exception&) {
4537 built = true; // report as a failure below
4538 }
4539 }
4540 check(dbOk && !built,
4541 "a wallet of many small coins cannot fund amount + the fee its"
4542 " own size demands");
4543 check(dbOk && result == WALLET_SEND_INSUFFICIENT_FUNDS,
4544 "the wallet reports that as a funding shortfall, not as a"
4545 " failure to build");
4546 // GetMinFee(fDiscount) returns 0 below 10,000 bytes and
4547 // (1 + bytes/1000) * CENT at or above, so a required fee of 11 CENT or
4548 // more IS the statement that the transaction crossed 10 KB. Measured
4549 // through the real serialiser rather than asserted.
4550 check(dbOk && feeRequired >= 11 * CENT,
4551 "the transaction it built crossed 10,000 bytes, so the fee"
4552 " stepped from 0 to at least 11 CENT");
4553
4554 // The defect, stated as arithmetic. The old classifier asked whether
4555 // the balance covered amount + nTransactionFee + CENT; it did, so it
4556 // called a permanent funding shortfall a fault and started counting
4557 // towards a write-off.
4558 check(!(balance < amount + nTransactionFee + CENT),
4559 "the reserve this replaces would have judged this send a fault");
4560 check(balance < amount + feeRequired,
4561 "while the fee actually required was more than the balance had");
4562
4563 outcome.result = result;
4564 outcome.nFeeRequired = feeRequired;
4565 check(dbOk && ClassifyPayoutSendFailure(outcome.result, false)
4566 == PAYOUT_FAILURE_DEFER,
4567 "a >= 10 KB send that ran out of coin is deferred, not counted"
4568 " against the payment");
4569
4570 // -- the < 10 KB case: no reserve is needed at all -------------------
4571 //
4572 // The other half of the arithmetic. Below the step the required fee is
4573 // 0, so a reserve of any size is a reason to refuse a payment the
4574 // wallet could have made -- which is its own way of not paying a
4575 // miner.
4576 CRITICAL_BLOCK(cs_mapWallet)
4577 mapWallet.clear();
4578 PayoutTestFundWallet(3, 1 * COIN, walletAddr);
4579 const int64 smallAmount = 3 * COIN - 5 * CENT;
4580 CWalletTx smallTx;
4581 int64 smallFee = -1;
4582 WalletSendResult smallResult = WALLET_SEND_FAILED;
4583 bool smallBuilt = false;
4584 if (dbOk) {
4585 try {
4586 smallBuilt = CreateTransaction(payeeScript, smallAmount, smallTx,
4587 smallFee, &smallResult);
4588 } catch (const std::exception&) {
4589 smallBuilt = false;
4590 }
4591 }
4592 check(dbOk && smallBuilt && smallResult == WALLET_SEND_OK,
4593 "the same balance pays a small transaction with no headroom at"
4594 " all");
4595 check(dbOk && smallFee == 0,
4596 "because below 10,000 bytes the required fee is zero");
4597
4598 // -- and the classification of every other answer --------------------
4599 check(ClassifyPayoutSendFailure(WALLET_SEND_INSUFFICIENT_FUNDS, false)
4600 == PAYOUT_FAILURE_DEFER &&
4601 ClassifyPayoutSendFailure(WALLET_SEND_INSUFFICIENT_FUNDS, true)
4602 == PAYOUT_FAILURE_DEFER &&
4603 ClassifyPayoutSendFailure(WALLET_SEND_FAILED, true)
4604 == PAYOUT_FAILURE_DEFER,
4605 "short of coin or unable to sign is always a deferral");
4606 check(ClassifyPayoutSendFailure(WALLET_SEND_FAILED, false)
4607 == PAYOUT_FAILURE_FAULT,
4608 "funded and unlocked and still refused is a fault");
4609
4610 CRITICAL_BLOCK(cs_mapWallet)
4611 mapWallet.clear();
4612 }
4613
4614 // -- 13. No payout mutex is held across the status-file write ------------
4615 //
4616 // RefreshStats runs on the event loop -- the thread that owns every
4617 // miner's socket -- and used to hold gStatsMutex and gPayoutLedgerMutex at
4618 // function scope across WriteFileAtomic. Once that write gained fsync() on
4619 // the file and on its directory, publishing pool_status.json parked the
4620 // payout thread behind two disk flushes every ten seconds.
4621 //
4622 // Timing assertions would be flaky, so this does not measure anything. It
4623 // makes the write block for as long as the test likes: strPoolStatusFile
4624 // points at a path whose ".tmp" is a FIFO, and open(2) for writing on a
4625 // FIFO does not return until a reader arrives. While RefreshStats is
4626 // parked in there, every mutex it must NOT be holding has to be
4627 // acquirable. Against the previous code this test does not merely fail, it
4628 // cannot pass: the writer is inside fopen() with both mutexes held and
4629 // nothing will release them until the test lets the write finish.
4630#ifndef _WIN32
4631 {
4632 PayoutTestResetState();
4633 const std::string statusPath = dir + "/pool_status_selftest.json";
4634 const std::string statusTmp = statusPath + ".tmp";
4635 const std::string savedStatusFile = strPoolStatusFile;
4636 const int64 savedLastWrite = gLastPoolStatusWrite;
4637 remove(statusPath.c_str());
4638 remove(statusTmp.c_str());
4639 strPoolStatusFile = statusPath;
4640 gLastPoolStatusWrite = 0; // due a publish on the next refresh
4641
4642 // Give the status file something to report, so the ledger mutex is
4643 // genuinely taken on the way through rather than skipped over.
4644 {
4645 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4646 PendingPayout pp;
4647 pp.blockHeight = 909; pp.blockHash = 0; pp.matureAtHeight = 909;
4648 pp.amounts[addrA] = 3 * COIN;
4649 gPendingPayouts.push_back(pp);
4650 }
4651
4652 bool fifoOk = (mkfifo(statusTmp.c_str(), 0600) == 0);
4653 check(fifoOk, "a FIFO stands in for the status file's temp file, so the"
4654 " write can be held open");
4655 if (fifoOk) {
4656 std::atomic<bool> done(false);
4657 std::thread writer([&done]() {
4658 RefreshStats(0, 0);
4659 done = true;
4660 });
4661
4662 // Settle: the work before the write is a handful of vector
4663 // operations, so anything past this point is inside fopen(). The
4664 // !done check below is what actually proves it -- with no reader
4665 // there is no other way for this thread to still be running.
4666 Sleep(500);
4667 bool blocked = !done;
4668 bool gotLedger = false, gotStats = false, gotExec = false,
4669 gotRounds = false;
4670 for (int i = 0; i < 1000 && blocked; i++) {
4671 if (!gotLedger && gPayoutLedgerMutex.try_lock()) {
4672 gotLedger = true; gPayoutLedgerMutex.unlock();
4673 }
4674 if (!gotStats && gStatsMutex.try_lock()) {
4675 gotStats = true; gStatsMutex.unlock();
4676 }
4677 if (!gotExec && gPayoutExecMutex.try_lock()) {
4678 gotExec = true; gPayoutExecMutex.unlock();
4679 }
4680 if (!gotRounds && gPoolRoundsMutex.try_lock()) {
4681 gotRounds = true; gPoolRoundsMutex.unlock();
4682 }
4683 if (gotLedger && gotStats && gotExec && gotRounds)
4684 break;
4685 Sleep(2);
4686 }
4687
4688 check(blocked,
4689 "the status-file write is held open for the duration of this"
4690 " test");
4691 check(!blocked || gotLedger,
4692 "gPayoutLedgerMutex is free while pool_status.json is being"
4693 " written");
4694 check(!blocked || (gotExec && gotRounds),
4695 "gPayoutExecMutex and gPoolRoundsMutex are free while"
4696 " pool_status.json is being written");
4697 check(!blocked || gotStats,
4698 "gStatsMutex is free while pool_status.json is being written");
4699
4700 // Let the write finish. Drain to EOF rather than closing early:
4701 // the writer is still filling the pipe and a closed read end would
4702 // deliver it SIGPIPE.
4703 std::thread reader([&statusTmp]() {
4704 FILE* rf = fopen(statusTmp.c_str(), "r");
4705 if (!rf) return;
4706 char buf[4096];
4707 while (fread(buf, 1, sizeof(buf), rf) > 0) { }
4708 fclose(rf);
4709 });
4710 writer.join();
4711 reader.join();
4712 }
4713 remove(statusTmp.c_str());
4714
4715 // And the ordinary path still produces the file it always did.
4716 gLastPoolStatusWrite = 0;
4717 RefreshStats(0, 0);
4718 std::string onDisk;
4719 bool statusOk = false;
4720 if (PayoutTestReadFile(statusPath, onDisk)) {
4721 try {
4722 nlohmann::json j = nlohmann::json::parse(onDisk);
4723 statusOk = j.value("schema", "") == "bitflash-pool-status-1" &&
4724 j["pendingPayouts"].is_array() &&
4725 j["pendingPayouts"].size() == 1 &&
4726 j["pendingPayouts"][0].value("matureAtHeight", 0) == 909 &&
4727 j["pendingPayouts"][0].value("totalAmountSatoshis",
4728 (int64)0) == 3 * COIN;
4729 } catch (...) {
4730 statusOk = false;
4731 }
4732 }
4733 check(statusOk,
4734 "splitting the serialise from the write leaves pool_status.json"
4735 " saying the same thing");
4736
4737 remove(statusPath.c_str());
4738 remove((statusPath + ".tmp").c_str());
4739 strPoolStatusFile = savedStatusFile;
4740 gLastPoolStatusWrite = savedLastWrite;
4741 }
4742#endif
4743
4744 // -- 14. The payout comment names the funding block ----------------------
4745 //
4746 // Same height, two hashes: the shape a reorg leaves when the pool wins a
4747 // height, loses the block, and wins the height again. The legacy needle is
4748 // frozen at the pre-upgrade format, so pre-upgrade payments keep matching;
4749 // a new-format comment never matches it, so a new send can never be read
4750 // as a pre-upgrade payment for a twin round.
4751 {
4752 PayoutTestResetState();
4753 CRITICAL_BLOCK(cs_mapWallet)
4754 mapWallet.clear();
4755 PayoutTestClearBlockIndex();
4756
4757 const int height = 9090;
4758 const int64 amount = 3 * COIN;
4759 uint256 hashA = 0, hashB = 0;
4760 hashA.SetHex("00000000000000000000000000000000000000000000000000000000000000aa");
4761 hashB.SetHex("00000000000000000000000000000000000000000000000000000000000000bb");
4762
4763 check(LegacyPayoutComment(height) ==
4764 strprintf("Pool payout height %d", height),
4765 "the legacy needle is frozen at the format pre-upgrade builds"
4766 " wrote");
4767 check(PayoutComment(height, 0) == LegacyPayoutComment(height),
4768 "a grandfathered zero-hash record still writes the legacy form");
4769 check(PayoutComment(height, hashA) ==
4770 strprintf("Pool payout height %d block %s", height,
4771 hashA.ToString().c_str()),
4772 "a hash-keyed record's comment names its funding block");
4773 check(PayoutComment(height, hashA) != PayoutComment(height, hashB),
4774 "twin rounds at one height carry distinguishable comments");
4775
4776 // A transaction wearing the NEW comment format and -- hypothetically
4777 // -- no structured metadata must not be matched by the legacy needle:
4778 // the fallback exists for pre-upgrade transactions, and the new format
4779 // proves this is not one.
4780 const std::string idA = PayoutIdForBlock(height, hashA);
4781 const std::string idB = PayoutIdForBlock(height, hashB);
4782 std::set<std::string> none;
4783 std::string found;
4784 // Neither twin has a round proof here (section 15 covers the gate),
4785 // and their ids collide, so one answer serves both.
4786 const bool fAllowLegacy = !PoolRoundHasProof(idA);
4787 PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount, 1, "");
4788 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found)
4789 == PAYOUT_PROBE_NONE,
4790 "a new-format comment is never legacy evidence for a twin round");
4791 check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found)
4792 == PAYOUT_PROBE_NONE,
4793 "nor a match channel for its own round -- structured metadata is");
4794
4795 // Round A's own payment, sent by this build, is found by its
4796 // structured id.
4797 CRITICAL_BLOCK(cs_mapWallet)
4798 mapWallet.clear();
4799 PayoutTestClearBlockIndex();
4800 const std::string paidA =
4801 PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount,
4802 2, idA);
4803 check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found)
4804 == PAYOUT_PROBE_STRUCTURED && found == paidA,
4805 "the winning twin's own payment is found by its structured id");
4806
4807 // The id is derived from the FIRST sixteen hex digits of the hash --
4808 // the end proof-of-work fills with zeros -- so twin hashes that agree
4809 // there collide in id, and a payment carrying no pool_payout_block
4810 // answers for either twin. That is the accepted trade for payments
4811 // sent before the key existed, pinned here so it stays a decision.
4812 check(idA == idB,
4813 "twin hashes differing only past the sixteenth hex digit collide"
4814 " in id -- the reason the full hash must decide");
4815 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found)
4816 == PAYOUT_PROBE_STRUCTURED,
4817 "a payment without the block key answers for a colliding twin --"
4818 " the accepted residual for pre-key transactions");
4819
4820 // With the full hash recorded -- as every send now records it -- the
4821 // twins are told apart and the re-won height is paid.
4822 CRITICAL_BLOCK(cs_mapWallet)
4823 mapWallet.clear();
4824 PayoutTestClearBlockIndex();
4825 const std::string keyedA =
4826 PayoutTestAddWalletTx(PayoutComment(height, hashA), addrA, amount,
4827 3, idA, true, hashA.ToString());
4828 check(FindRecordedPoolPayout(idA, height, hashA, addrA, amount, fAllowLegacy, none, found)
4829 == PAYOUT_PROBE_STRUCTURED && found == keyedA,
4830 "a payment carrying the full block hash settles its own round");
4831 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount, fAllowLegacy, none, found)
4832 == PAYOUT_PROBE_NONE,
4833 "and is no evidence for the twin, so the re-won height is paid");
4834
4835 CRITICAL_BLOCK(cs_mapWallet)
4836 mapWallet.clear();
4837 PayoutTestClearBlockIndex();
4838 }
4839
4840 // -- 15. A proof-bearing round rejects comment-only evidence -------------
4841 //
4842 // The surviving half of the same-height collision: a pre-upgrade
4843 // transaction carries only "Pool payout height H", so if H is won again
4844 // after a reorg, it matches the new round's (height, address, amount)
4845 // exactly and would settle a debt it never paid. The new round has a
4846 // round proof -- written at creation, and only by builds whose payments
4847 // all name their round -- so for it, comment-only evidence is
4848 // inadmissible and the miner is paid. A round without a proof keeps
4849 // today's bias: the evidence is admissible and doubt resolves to
4850 // "already paid".
4851 {
4852 PayoutTestResetState();
4853 CRITICAL_BLOCK(cs_mapWallet)
4854 mapWallet.clear();
4855 PayoutTestClearBlockIndex();
4856 const std::string roundsPath = dir + "/pool_rounds_gate_selftest.json";
4857 const std::string savedRoundsFile = strPoolRoundsFile;
4858 strPoolRoundsFile = roundsPath;
4859 remove(roundsPath.c_str());
4860
4861 const int height = 7171;
4862 const int64 amount = 4 * COIN;
4863 uint256 hashB = 0;
4864 hashB.SetHex("00000000000000000000000000000000000000000000000000000000000000cc");
4865 const std::string idB = PayoutIdForBlock(height, hashB);
4866 std::set<std::string> none;
4867 std::string found;
4868
4869 // The colliding pre-upgrade payment: confirmed, comment-only, same
4870 // height, same address, same amount -- the strongest wrong evidence
4871 // there is.
4872 PayoutTestAddWalletTx(LegacyPayoutComment(height), addrA, amount, 1, "");
4873
4874 check(!PoolRoundHasProof(idB),
4875 "a round with no recorded proof does not claim one");
4876 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount,
4877 !PoolRoundHasProof(idB), none, found)
4878 == PAYOUT_PROBE_LEGACY,
4879 "without a proof the doubt is genuine and still resolves to"
4880 " already-paid");
4881
4882 PoolRoundProof round;
4883 round.id = idB;
4884 round.blockHeight = height;
4885 round.matureAtHeight = height + COINBASE_SPENDABLE_DEPTH;
4886 round.payouts[addrA] = amount;
4887 RecordPoolRound(round);
4888 check(PoolRoundHasProof(idB),
4889 "recording the round proof marks the round as this lineage's");
4890 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount,
4891 !PoolRoundHasProof(idB), none, found)
4892 == PAYOUT_PROBE_NONE,
4893 "with a proof, a same-height comment-only transaction is not"
4894 " evidence, and the re-won height is paid");
4895
4896 // Structured evidence still is: the round's own payment, recovered.
4897 const std::string paidB =
4898 PayoutTestAddWalletTx(PayoutComment(height, hashB), addrA, amount,
4899 2, idB, true, hashB.ToString());
4900 check(FindRecordedPoolPayout(idB, height, hashB, addrA, amount,
4901 !PoolRoundHasProof(idB), none, found)
4902 == PAYOUT_PROBE_STRUCTURED && found == paidB,
4903 "the gate does not blind the probe to the round's own structured"
4904 " payment");
4905
4906 CRITICAL_BLOCK(cs_mapWallet)
4907 mapWallet.clear();
4908 PayoutTestClearBlockIndex();
4909 remove(roundsPath.c_str());
4910 remove((roundsPath + ".tmp").c_str());
4911 strPoolRoundsFile = savedRoundsFile;
4912 }
4913
4914 // -- 16. A malformed block hash is corruption, not a hash ----------------
4915 //
4916 // SetHex() parses whatever hex prefix it finds and stops, so a corrupt
4917 // blockHash did not fail to load -- it loaded as a DIFFERENT hash.
4918 // "deadbeef..." became 0xdeadbeef, a hash no block ever had, which the
4919 // orphan check reads as an orphaned block and the next flush VOIDS the
4920 // debt; a value with no hex prefix became 0, which is grandfathered past
4921 // the orphan check instead. Either way a corrupt file silently changed
4922 // what the pool owed. It is a load failure now, handled like every other
4923 // parse error: quarantine, and never rewrite.
4924 {
4925 PayoutTestResetState();
4926 remove(ledgerPath.c_str());
4927 remove(payingPath.c_str());
4928
4929 const std::string badLedger = strprintf(
4930 "[{\"amounts\":{\"%s\":100000000},"
4931 "\"blockHash\":\"deadbeef oops, not a hash\","
4932 "\"blockHeight\":10,\"matureAtHeight\":10}]", addrA.c_str());
4933 check(PayoutTestWriteFile(ledgerPath, badLedger),
4934 "wrote a ledger whose blockHash is not a 64-digit hex string");
4935 LoadPendingPayouts();
4936 {
4937 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4938 check(gPendingPayouts.empty(),
4939 "no entry is loaded with a guessed hash");
4940 }
4941 std::string gone;
4942 check(!PayoutTestReadFile(ledgerPath, gone),
4943 "the ledger with the corrupt hash is quarantined");
4944
4945 const std::string badPaying = strprintf(
4946 "[{\"address\":\"%s\",\"amount\":100000000,"
4947 "\"blockHash\":\"00000000cafe\",\"blockHeight\":10,"
4948 "\"matureAtHeight\":10}]", addrA.c_str());
4949 check(PayoutTestWriteFile(payingPath, badPaying),
4950 "wrote an in-flight list whose blockHash is truncated");
4951 LoadPaying();
4952 {
4953 std::lock_guard<std::mutex> lk(gPayoutExecMutex);
4954 check(gPaying.empty(),
4955 "no in-flight record is loaded with a guessed hash");
4956 }
4957 check(!PayoutTestReadFile(payingPath, gone),
4958 "the in-flight list with the corrupt hash is quarantined");
4959
4960 // The two spellings every real file uses still load: the 64-digit
4961 // hash of a keyed round, and the 64 zeros this build writes for a
4962 // grandfathered entry.
4963 PayoutTestResetState();
4964 remove(ledgerPath.c_str());
4965 const std::string goodLedger = strprintf(
4966 "[{\"amounts\":{\"%s\":100000000},"
4967 "\"blockHash\":\"00000000000000000000000000000000000000000000000000000000000000cd\","
4968 "\"blockHeight\":10,\"matureAtHeight\":10},"
4969 "{\"amounts\":{\"%s\":200000000},"
4970 "\"blockHash\":\"%s\","
4971 "\"blockHeight\":20,\"matureAtHeight\":20}]",
4972 addrA.c_str(), addrB.c_str(), uint256(0).ToString().c_str());
4973 check(PayoutTestWriteFile(ledgerPath, goodLedger),
4974 "wrote a ledger in both real hash spellings");
4975 LoadPendingPayouts();
4976 {
4977 std::lock_guard<std::mutex> lk(gPayoutLedgerMutex);
4978 check(gPendingPayouts.size() == 2 &&
4979 gPendingPayouts[0].blockHash != 0 &&
4980 gPendingPayouts[1].blockHash == 0,
4981 "a real hash and a serialised zero hash both still load");
4982 }
4983 remove(ledgerPath.c_str());
4984 }
4985
4986 // -- clean up ------------------------------------------------------------
4987 PayoutTestResetState();
4988 CRITICAL_BLOCK(cs_mapWallet)
4989 mapWallet.clear();
4990 PayoutTestClearBlockIndex();
4991 // Out of the scratch directory before removing it, and out of the key maps
4992 // too: the keys this test generated are process-local, but leaving them in
4993 // the keystore would let a later test think a script was its own.
4994 CRITICAL_BLOCK(cs_mapKeys)
4995 {
4996 mapKeys.clear();
4997 mapPubKeys.clear();
4998 }
4999#ifdef _WIN32
5000 SetCurrentDirectoryA(cwd.c_str());
5001#else
5002 if (chdir(cwd.c_str()) != 0) { /* nothing useful to do about it */ }
5003#endif
5004 PayoutTestRemoveDir(dir);
5005 strSetDataDir = savedDataDir;
5006 nBestHeight = savedBestHeight;
5007
5008 fprintf(stdout, "%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
5009 nFail, nFail == 1 ? "" : "s");
5010 fflush(stdout);
5011 return nFail == 0 ? 0 : 1;
5012}
5013
5014int RunPoolStratumSelfTest()
5015{
5016 int nFail = 0;
5017 auto check = [&](bool ok, const char* msg) {
5018 fprintf(stdout, " %s %s\n", ok ? "ok" : "FAIL", msg);
5019 if (!ok) nFail++;
5020 };
5021
5022 fprintf(stdout, "pool-stratum self-test\n");
5023
5024 StratumJob savedJob = gCurrentJob;
5025 bool savedHaveJob = gHaveJob;
5026
5027 CTransaction coinbase;
5028 coinbase.vin.resize(1);
5029 coinbase.vin[0].prevout.SetNull();
5030 coinbase.vin[0].scriptSig << 1 << (CBigNum)1;
5031 coinbase.vout.resize(1);
5032 coinbase.vout[0].nValue = 50 * COIN;
5033
5034 CBlock block;
5035 block.vtx.push_back(coinbase);
5036 block.nVersion = 1;
5037 block.nBits = 0x1f0fffff;
5038 block.nTime = 1;
5039 block.nNonce = 1;
5040 block.hashMerkleRoot = block.BuildMerkleTree();
5041
5042 gCurrentJob.jobId = "selftest";
5043 gCurrentJob.block = block;
5044 gCurrentJob.target = 0; // accepted share, not a block candidate
5045 gCurrentJob.height = 1;
5046 gHaveJob = true;
5047
5048 Miner miner(INVALID_SOCKET);
5049 miner.authorised = true;
5050 miner.address = "selftest-address";
5051 miner.worker = "worker";
5052 miner.shareTarget = ~uint256(0); // every RandomX hash is a valid share
5053 miner.difficulty = 0.001;
5054 miner.vardiffWindowStart = GetTime();
5055
5056 std::map<std::string, uint64> roundShareCount;
5057 uint64 roundShareTotal = 0;
5058 int blocksFound = 0;
5059
5060 uint256 targetFromNotify;
5061 CBlock minerBlock = BuildMinerBlock(gCurrentJob, miner.extranonce2);
5062 json params = MakeNotifyParams(gCurrentJob, minerBlock, miner.shareTarget, true);
5063 std::vector<unsigned char> targetBytes = FromHex(params[3].get<std::string>());
5064 if (targetBytes.size() == 32)
5065 memcpy(&targetFromNotify, &targetBytes[0], 32);
5066 check(targetFromNotify == miner.shareTarget,
5067 "mining.notify exposes the same share target the pool validates");
5068
5069 json submit = {
5070 {"id", 7},
5071 {"method", "mining.submit"},
5072 {"params", json::array({miner.address, "selftest", ToHex(&minerBlock.nNonce, 4)})}
5073 };
5074 HandleLine(&miner, submit.dump(), roundShareCount, roundShareTotal, blocksFound);
5075
5076 check(roundShareTotal == 1, "fake low-difficulty share increments the round total");
5077 check(roundShareCount[miner.address] == 1, "fake share is credited to the submitting address");
5078 check(miner.sessionShares == 1 && miner.roundShares == 1 && miner.totalShares == 1,
5079 "miner share counters advance together");
5080 check(blocksFound == 0, "share below block target is not counted as a found block");
5081
5082 json stale = {
5083 {"id", 8},
5084 {"method", "mining.submit"},
5085 {"params", json::array({miner.address, "stale", ToHex(&minerBlock.nNonce, 4)})}
5086 };
5087 HandleLine(&miner, stale.dump(), roundShareCount, roundShareTotal, blocksFound);
5088 check(roundShareTotal == 1, "stale submit does not increment shares");
5089
5090 std::vector<PoolRoundProof> savedRounds = gPoolRounds;
5091 std::string savedRoundsFile = strPoolRoundsFile;
5092 std::string proofPath = "pool_rounds_selftest.json";
5093 remove(proofPath.c_str());
5094 remove((proofPath + ".tmp").c_str());
5095 gPoolRounds.clear();
5096 strPoolRoundsFile = proofPath;
5097
5098 PoolRoundProof proof;
5099 proof.id = "selftest-round";
5100 proof.blockHeight = 42;
5101 proof.matureAtHeight = 42 + COINBASE_MATURITY;
5102 proof.createdAt = GetTime();
5103 proof.blockHash = block.GetHash().GetHex();
5104 proof.coinbaseTxid = coinbase.GetHash().GetHex();
5105 proof.foundBy = miner.address;
5106 proof.feePercent = 1.25;
5107 proof.totalShares = 3;
5108 proof.coinbaseValue = 50 * COIN;
5109 proof.subsidy = 50 * COIN;
5110 proof.minerReward = 49 * COIN;
5111 proof.operatorReward = 1 * COIN;
5112 proof.shares[miner.address] = 3;
5113 proof.payouts[miner.address] = 49 * COIN;
5114 RecordPoolRound(proof);
5115 MarkPoolRoundPaid(proof.id, miner.address, "selftest-txid");
5116
5117 bool proofOk = false;
5118 FILE* pf = fopen(proofPath.c_str(), "r");
5119 if (pf) {
5120 fseek(pf, 0, SEEK_END);
5121 long sz = ftell(pf);
5122 fseek(pf, 0, SEEK_SET);
5123 std::string s(sz, '\0');
5124 fread(&s[0], 1, sz, pf);
5125 fclose(pf);
5126 try {
5127 json root = json::parse(s);
5128 proofOk =
5129 root.value("schema", "") == "bitflash-pool-rounds-1" &&
5130 root["rounds"].is_array() &&
5131 root["rounds"].size() == 1 &&
5132 root["rounds"][0].value("id", "") == proof.id &&
5133 root["rounds"][0]["shares"].value(miner.address, (uint64)0) == 3 &&
5134 root["rounds"][0]["payouts"][miner.address].value("txid", "") == "selftest-txid";
5135 } catch (...) {
5136 proofOk = false;
5137 }
5138 }
5139 check(proofOk, "pool round proof ledger is written and paid txid is recorded");
5140
5141 remove(proofPath.c_str());
5142 remove((proofPath + ".tmp").c_str());
5143 strPoolRoundsFile = savedRoundsFile;
5144 gPoolRounds = savedRounds;
5145
5146 // -- hashrate decay (#7) --------------------------------------------------
5147 // The first version of this decayed m->hashRate in place from the event
5148 // loop's sweep, which made the reported figure a function of how the loop
5149 // happened to run rather than of how long the miner had been quiet. These
5150 // pin the properties that fixes: sampling changes nothing, and the answer
5151 // depends only on elapsed silence.
5152 {
5153 // A miner exactly on the vardiff target: 4096 hashes per share at
5154 // difficulty 1.0, one share per 15s, so 4096/15 H/s. Its expected
5155 // interval is 15s and its grace is 60s.
5156 const double base = 4096.0 / 15.0;
5157 const int64 t0 = 1766000000; // any plausible non-zero wall clock
5158 Miner decayed(INVALID_SOCKET);
5159 decayed.difficulty = 1.0;
5160 decayed.hashRate = base;
5161 decayed.lastShareTime = t0;
5162
5163 check(MinerReportedHashRate(&decayed, t0 + 60) == base,
5164 "hashrate decay: a miner inside its grace period reports its full estimate");
5165
5166 // Sampling must not consume the estimate. Twenty passes is one second
5167 // of the event loop's old sweep; a thousand is nearly a minute of it.
5168 double once = MinerReportedHashRate(&decayed, t0 + 61);
5169 bool idempotent = (decayed.hashRate == base);
5170 for (int i = 0; i < 1000 && idempotent; i++)
5171 idempotent = (MinerReportedHashRate(&decayed, t0 + 61) == once) &&
5172 (decayed.hashRate == base);
5173 check(idempotent,
5174 "hashrate decay: repeated sampling returns one value and leaves the estimate alone");
5175
5176 // One second past a 60s grace is 1/15th of a half-life: ~4.5% off, not
5177 // the estimate gone. A miner at the vardiff target overshoots its grace
5178 // roughly 1.8% of the time (e^-4) purely by share-timing variance, and
5179 // must not be zeroed for it.
5180 check(once > 0.94 * base && once < 0.96 * base,
5181 "hashrate decay: one second past grace costs a few percent, not the estimate");
5182 double three = MinerReportedHashRate(&decayed, t0 + 63);
5183 check(three > 0.86 * base && three < 0.88 * base,
5184 "hashrate decay: three seconds past grace costs ~13%");
5185
5186 // One half-life past the grace is exactly half.
5187 double half = MinerReportedHashRate(&decayed, t0 + 75);
5188 check(half > 0.499 * base && half < 0.501 * base,
5189 "hashrate decay: one expected interval past grace halves the estimate");
5190
5191 // A miner that has genuinely stopped ends up reported as nothing --
5192 // the whole point of #7. The old in-place version could not get here:
5193 // its decay fed back into its own grace and settled into a 1/t crawl.
5194 check(MinerReportedHashRate(&decayed, t0 + 3600) == 0.0,
5195 "hashrate decay: a miner silent for an hour is reported as zero");
5196
5197 // A clock that steps backwards must never inflate what we announce.
5198 check(MinerReportedHashRate(&decayed, t0 - 3600) == base,
5199 "hashrate decay: a backwards clock reports the estimate unchanged, never higher");
5200
5201 // Fresh connection, and a connection with one share and so no interval
5202 // to have timed: both must report nothing rather than divide by zero.
5203 Miner fresh(INVALID_SOCKET);
5204 check(MinerReportedHashRate(&fresh, t0) == 0.0,
5205 "hashrate decay: a miner that never submitted a share reports zero");
5206 Miner single(INVALID_SOCKET);
5207 single.lastShareTime = t0;
5208 check(MinerReportedHashRate(&single, t0 + 86400) == 0.0,
5209 "hashrate decay: a miner with no rate sample yet reports zero");
5210
5211 // Nothing in the swept range may produce a NaN, an infinity, a negative
5212 // rate, or a rate above the estimate it started from.
5213 bool finite = true;
5214 for (int64 s = -100; s < 200000 && finite; s += 37) {
5215 double v = MinerReportedHashRate(&decayed, t0 + s);
5216 finite = !std::isnan(v) && !std::isinf(v) && v >= 0.0 && v <= base;
5217 }
5218 check(finite,
5219 "hashrate decay: stays finite, non-negative and non-amplifying across a long silence");
5220 }
5221
5222 gCurrentJob = savedJob;
5223 gHaveJob = savedHaveJob;
5224
5225 fprintf(stdout, "%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
5226 nFail, nFail == 1 ? "" : "s");
5227 fflush(stdout);
5228 return nFail == 0 ? 0 : 1;
5229}
5230
5231// ---------------------------------------------------------------------------
5232// Stats refresh -- called every 2s from event loop
5233// ---------------------------------------------------------------------------
5234static void RefreshStats(int blocksFoundThisSession, uint64 roundShareTotal)
5235{
5236 std::string statusBlob;
5237 bool fPublish = false;
5238 {
5239 std::lock_guard<std::mutex> lk(gStatsMutex);
5240
5241 gStats.authorizedMiners = 0;
5242 gStats.blocksFound = blocksFoundThisSession;
5243 gStats.roundShares = roundShareTotal;
5244 gStats.totalHashRate = 0.0;
5245 for (Miner* m : gMiners)
5246 if (m->authorised) gStats.authorizedMiners++;
5247
5248 gWorkerStats.clear();
5249 // One timestamp for the whole sweep (#7): every miner is faded to the
5250 // same instant, so the per-worker rows and the total they are summed
5251 // into describe one moment rather than a smear across the loop.
5252 int64 nowStats = GetTime();
5253 for (Miner* m : gMiners) {
5254 if (!m->authorised || m->lastSeen <= 0) continue;
5255 PoolWorkerStatView v;
5256 v.address = m->address;
5257 v.worker = m->worker;
5258 v.roundShares = m->roundShares;
5259 v.totalShares = m->totalShares;
5260 v.lastSeen = m->lastSeen;
5261 // Not m->hashRate: a miner that has stopped submitting shares must
5262 // not keep contributing its last sample to the announced total.
5263 v.hashRate = MinerReportedHashRate(m, nowStats);
5264 gWorkerStats.push_back(v);
5265 gStats.totalHashRate += v.hashRate;
5266 }
5267 std::sort(gWorkerStats.begin(), gWorkerStats.end(),
5268 [](const PoolWorkerStatView& a, const PoolWorkerStatView& b) {
5269 return a.roundShares != b.roundShares ? a.roundShares > b.roundShares
5270 : a.lastSeen > b.lastSeen;
5271 });
5272
5273 // Scoped, not held to the end of the function. At function scope this
5274 // lock covered the status-file write below, so the event loop did two
5275 // fsyncs holding the mutex the payout thread takes to record that a
5276 // miner has been paid.
5277 gPayoutViews.clear();
5278 {
5279 std::lock_guard<std::mutex> ledger(gPayoutLedgerMutex);
5280 for (const PendingPayout& pp : gPendingPayouts) {
5281 PendingPayoutView pv;
5282 pv.matureAtHeight = pp.matureAtHeight;
5283 pv.recipients = 0;
5284 pv.totalAmount = 0;
5285 for (auto& kv : pp.amounts) {
5286 if (pp.paidTxids.count(kv.first))
5287 continue;
5288 pv.recipients++;
5289 pv.totalAmount += kv.second;
5290 }
5291 if (pv.recipients > 0)
5292 gPayoutViews.push_back(pv);
5293 }
5294 }
5295
5296 int64 now = GetTime();
5297 if (now - gLastPoolStatusWrite >= 10) {
5298 statusBlob = SerializePoolStatusLocked();
5299 gLastPoolStatusWrite = now;
5300 fPublish = true;
5301 }
5302 }
5303 // Every payout mutex is released here, and gStatsMutex with them. This is
5304 // where the two fsyncs happen, and no other thread is waiting behind them.
5305 if (fPublish)
5306 PublishPoolStatus(statusBlob);
1526} 5307}
1527 5308
1528// --------------------------------------------------------------------------- 5309// ---------------------------------------------------------------------------
@@ -1533,6 +5314,7 @@ static void PoolEventLoop()
1533 int lastHeight = -1; 5314 int lastHeight = -1;
1534 int64 lastRefresh = GetTime(); 5315 int64 lastRefresh = GetTime();
1535 int64 lastStatsRefresh = 0; 5316 int64 lastStatsRefresh = 0;
5317 int64 lastCensus = 0; // #19 connection census throttle
1536 int blocksFoundThisSession = 0; 5318 int blocksFoundThisSession = 0;
1537 int64 noJobSince = 0; // 0 = have a job 5319 int64 noJobSince = 0; // 0 = have a job
1538 bool warnedNoJob = false; 5320 bool warnedNoJob = false;
@@ -1561,6 +5343,31 @@ static void PoolEventLoop()
1561 sock_close(fd); 5343 sock_close(fd);
1562 continue; 5344 continue;
1563 } 5345 }
5346 // #26: select() + FD_SET goes undefined once we exceed what an
5347 // fd_set can carry, and MAX_POOL_MINERS (1000) alone does not
5348 // prevent it -- a Miner is only removed on clean disconnect, so
5349 // half-open zombies accumulate and the OS can hand out fds past
5350 // the limit long before the count reaches 1000. Refuse rather
5351 // than corrupt the fd_set. Which limit applies is platform
5352 // specific: POSIX bounds the descriptor value, Windows the
5353 // number of SOCKET handles an fd_set can hold.
5354#ifndef _WIN32
5355 if ((int)fd >= (int)FD_SETSIZE) {
5356 LogPrint("worker", "[pool] refusing miner fd=%d -- at/over"
5357 " FD_SETSIZE=%d, select() would be undefined (#26)\n",
5358 (int)fd, (int)FD_SETSIZE);
5359 sock_close(fd);
5360 continue;
5361 }
5362#else
5363 if (gMiners.size() >= (size_t)FD_SETSIZE) {
5364 LogPrint("worker", "[pool] refusing miner -- %zu sockets held,"
5365 " an fd_set carries at most FD_SETSIZE=%d (#26)\n",
5366 gMiners.size(), (int)FD_SETSIZE);
5367 sock_close(fd);
5368 continue;
5369 }
5370#endif
1564 gMiners.push_back(new Miner(fd)); 5371 gMiners.push_back(new Miner(fd));
1565 LogPrint("worker", "[pool] miner fd=%d entered event loop" 5372 LogPrint("worker", "[pool] miner fd=%d entered event loop"
1566 " -- awaiting subscribe/authorize\n", (int)fd); 5373 " -- awaiting subscribe/authorize\n", (int)fd);
@@ -1688,6 +5495,35 @@ static void PoolEventLoop()
1688 } 5495 }
1689 } 5496 }
1690 5497
5498 // 3.6. Idle reap (#6). Mirror the clean-disconnect path exactly --
5499 // log, close, erase from gMiners via std::find, delete -- just with
5500 // a different reason. Shares the miner already earned stay in
5501 // roundShareCount, exactly as they do when a miner cleanly
5502 // disconnects mid-round. connectTime stands in for lastSeen on a
5503 // connection that never completed a line. Erasing inside the loop
5504 // above would invalidate the iteration, so collect then reap.
5505 {
5506 int64 now = GetTime();
5507 std::vector<Miner*> idle;
5508 for (Miner* m : gMiners) {
5509 int64 last = (m->lastSeen > 0) ? m->lastSeen : m->connectTime;
5510 if (now - last > POOL_IDLE_TIMEOUT_SECS)
5511 idle.push_back(m);
5512 }
5513 for (Miner* m : idle) {
5514 LogPrint("worker", "[pool] dropping idle miner %s -- nothing"
5515 " received in over %llds (session shares: %llu,"
5516 " round shares: %llu)\n",
5517 m->worker.empty() ? "(never authorized)" : m->worker.c_str(),
5518 (long long)POOL_IDLE_TIMEOUT_SECS,
5519 (unsigned long long)m->sessionShares,
5520 (unsigned long long)m->roundShares);
5521 sock_close(m->fd);
5522 gMiners.erase(std::find(gMiners.begin(), gMiners.end(), m));
5523 delete m;
5524 }
5525 }
5526
1691 // 4. Select over miner sockets 5527 // 4. Select over miner sockets
1692 if (!gMiners.empty()) { 5528 if (!gMiners.empty()) {
1693 fd_set fds; FD_ZERO(&fds); 5529 fd_set fds; FD_ZERO(&fds);
@@ -1710,6 +5546,11 @@ static void PoolEventLoop()
1710 // terminate the whole node -- not just the pool. Every 5546 // terminate the whole node -- not just the pool. Every
1711 // line arrives from an unauthenticated socket. 5547 // line arrives from an unauthenticated socket.
1712 while (RecvLine(m->fd, m->readBuf, line, disc)) { 5548 while (RecvLine(m->fd, m->readBuf, line, disc)) {
5549 // Any complete line is proof of life (#6): authorize
5550 // and valid shares already stamp lastSeen themselves,
5551 // but a miner that only ever subscribes (or sends
5552 // unrecognised methods) must count as alive too.
5553 m->lastSeen = GetTime();
1713 try { 5554 try {
1714 HandleLine(m, line, roundShareCount, 5555 HandleLine(m, line, roundShareCount,
1715 roundShareTotal, blocksFoundThisSession); 5556 roundShareTotal, blocksFoundThisSession);
@@ -1750,6 +5591,14 @@ static void PoolEventLoop()
1750 RefreshStats(blocksFoundThisSession, roundShareTotal); 5591 RefreshStats(blocksFoundThisSession, roundShareTotal);
1751 lastStatsRefresh = GetTime(); 5592 lastStatsRefresh = GetTime();
1752 } 5593 }
5594
5595 // 6. #19: log the connection census periodically. Cheap (one pass over
5596 // gMiners) and it is the single number an operator reads to judge pool
5597 // health, so it should not be an overstatement.
5598 if (GetTime() - lastCensus >= POOL_CENSUS_SECS) {
5599 LogPoolConnectionCensus();
5600 lastCensus = GetTime();
5601 }
1753 } 5602 }
1754 5603
1755 for (Miner* m : gMiners) { sock_close(m->fd); delete m; } 5604 for (Miner* m : gMiners) { sock_close(m->fd); delete m; }
@@ -1778,13 +5627,19 @@ static void AcceptOneFn(void* arg)
1778 std::string host = relay.substr(0, colon); 5627 std::string host = relay.substr(0, colon);
1779 int port = atoi(relay.substr(colon + 1).c_str()); 5628 int port = atoi(relay.substr(colon + 1).c_str());
1780 5629
1781 LogPrint("pool", "[pool] accept thread started for relay %s\n", relay.c_str()); 5630 LogPrint("pool", "[pool] accept thread started for relay %s%s\n", relay.c_str(),
5631 fBtfRelayPinned ? " (/rvrelay pin)" : "");
5632
5633 // Registration health for this relay. A pin has no fallback, so a pin that
5634 // never registers means the pool is simply unreachable -- and that has to
5635 // be loud. The 2026-07-30 incident went unnoticed because this loop logged
5636 // failures at one flat line and success not at all, so "which relay is this
5637 // pool actually registered at" could not be answered from the log.
5638 int nConsecFail = 0;
5639 int64 nFirstFail = 0;
5640 bool fEverRegistered = false;
1782 5641
1783 while (!fShutdown && gPoolRunning) { 5642 while (!fShutdown && gPoolRunning) {
1784 // Advertise this relay as the pool's meeting node BEFORE registering,
1785 // so that miners which resolve our descriptor find a relay where the
1786 // pool (not the node's peer-accept) holds the registration.
1787 BtfSetActiveRelay(relay);
1788 // Returns as soon as the relay lists us -- registered, not yet paired. 5643 // Returns as soon as the relay lists us -- registered, not yet paired.
1789 btf::RvSocket rv = btf::RvServiceRegister(host.c_str(), (unsigned short)port, pk); 5644 btf::RvSocket rv = btf::RvServiceRegister(host.c_str(), (unsigned short)port, pk);
1790 5645
@@ -1793,12 +5648,36 @@ static void AcceptOneFn(void* arg)
1793 break; 5648 break;
1794 } 5649 }
1795 if (rv == btf::RV_INVALID) { 5650 if (rv == btf::RV_INVALID) {
1796 LogPrint("pool", "[pool] relay %s: RvServiceRegister failed" 5651 nConsecFail++;
1797 " -- backing off 30s before retry\n", relay.c_str()); 5652 if (nFirstFail == 0) nFirstFail = GetTime();
5653 LogPrint("pool", "[pool] relay %s: registration failed (attempt %d,"
5654 " failing for %llds) -- backing off 30s before retry\n",
5655 relay.c_str(), nConsecFail, (long long)(GetTime() - nFirstFail));
5656 if (fBtfRelayPinned && (nConsecFail == 3 || nConsecFail % 10 == 0))
5657 LogPrint("pool", "[pool] WARNING: pinned relay %s refused registration"
5658 " %d times running (%llds); /rvrelay leaves no fallback, so"
5659 " miners cannot reach this pool%s\n",
5660 relay.c_str(), nConsecFail, (long long)(GetTime() - nFirstFail),
5661 fEverRegistered ? "" : " -- it has never registered once");
1798 for (int i = 0; i < 30 && !fShutdown && gPoolRunning; i++) Sleep(1000); 5662 for (int i = 0; i < 30 && !fShutdown && gPoolRunning; i++) Sleep(1000);
1799 continue; 5663 continue;
1800 } 5664 }
1801 5665
5666 // Advertise THIS relay only now. Setting it before the call published a
5667 // meeting node we were not yet listed at, so a miner resolving the
5668 // descriptor inside that window burned its dial on a relay holding no
5669 // registration for us. net.cpp's peer-accept has always ordered it this
5670 // way round; the pool had not.
5671 BtfSetActiveRelay(relay);
5672 if (!fEverRegistered || nConsecFail > 0)
5673 LogPrint("pool", "[pool] registered at relay %s%s%s -- advertising it as"
5674 " the pool's meeting node\n", relay.c_str(),
5675 fBtfRelayPinned ? " (/rvrelay pin)" : "",
5676 nConsecFail > 0 ? ", recovered" : "");
5677 fEverRegistered = true;
5678 nConsecFail = 0;
5679 nFirstFail = 0;
5680
1802 // Block for a miner to actually arrive. RvServiceRegister used to do 5681 // Block for a miner to actually arrive. RvServiceRegister used to do
1803 // this itself and returned with the relay's 0x01 pairing byte already 5682 // this itself and returned with the relay's 0x01 pairing byte already
1804 // consumed; it no longer does. Without waiting here, BtfServiceWrap 5683 // consumed; it no longer does. Without waiting here, BtfServiceWrap
@@ -1853,9 +5732,21 @@ static void AcceptLoopFn(void*)
1853 LogPrint("pool", "[pool] .btf identity ready after %ds\n", waitSecs * 2); 5732 LogPrint("pool", "[pool] .btf identity ready after %ds\n", waitSecs * 2);
1854 5733
1855 std::set<std::string> spawned; // relays we've already started a thread for 5734 std::set<std::string> spawned; // relays we've already started a thread for
5735 std::set<std::string> skipped; // discovered relays the pin excluded, logged once each
1856 5736
1857 while (!fShutdown && gPoolRunning) { 5737 while (!fShutdown && gPoolRunning) {
5738 // BtfAllRelays() already honours the pin: with /rvrelay set it returns
5739 // vBtfMeetingRelays alone, so no thread is ever spawned for a
5740 // discovered relay. Name the ones it drops -- a pool registering at a
5741 // stranger's relay is exactly the thing that should not be silent.
1858 vector<string> relays = BtfAllRelays(); 5742 vector<string> relays = BtfAllRelays();
5743 if (fBtfRelayPinned) {
5744 for (const string& r : BtfDiscoveredRelays())
5745 if (std::find(relays.begin(), relays.end(), r) == relays.end() &&
5746 skipped.insert(r).second)
5747 LogPrint("pool", "[pool] not registering at discovered relay %s"
5748 " -- /rvrelay pins the relay list\n", r.c_str());
5749 }
1859 if (relays.empty()) { 5750 if (relays.empty()) {
1860 LogPrint("pool", "[pool] no relays available -- miners cannot connect yet\n"); 5751 LogPrint("pool", "[pool] no relays available -- miners cannot connect yet\n");
1861 } else { 5752 } else {
@@ -1871,8 +5762,9 @@ static void AcceptLoopFn(void*)
1871 relay.c_str()); 5762 relay.c_str());
1872 } else { 5763 } else {
1873 spawned.insert(relay); 5764 spawned.insert(relay);
1874 LogPrint("pool", "[pool] accept thread spawned for relay %s" 5765 LogPrint("pool", "[pool] accept thread spawned for relay %s%s"
1875 " (%zu total)\n", relay.c_str(), spawned.size()); 5766 " (%zu total)\n", relay.c_str(),
5767 fBtfRelayPinned ? " (/rvrelay pin)" : "", spawned.size());
1876 } 5768 }
1877 } 5769 }
1878 } 5770 }
@@ -1901,16 +5793,24 @@ void ThreadRPCServer(void*)
1901 } 5793 }
1902 5794
1903 LoadPendingPayouts(); 5795 LoadPendingPayouts();
5796 LoadPoolRounds();
1904 LoadPaying(); 5797 LoadPaying();
1905 // Must run before the payout thread starts, so nothing is resent that the 5798 // Must run before the payout thread starts, so nothing is resent that the
1906 // wallet already shows as sent, and nothing is owed twice because we died 5799 // ledger already shows as sent, and every restored in-flight record is
1907 // between recording the payments and rewriting the ledger. 5800 // armed for its one wallet probe before it is allowed to send.
1908 ReconcilePayouts(); 5801 ReconcilePayouts();
1909 5802
1910 _beginthread(AcceptLoopFn, 0, NULL); 5803 // _beginthread cannot throw (it reports -1), but an unchecked failure
1911 _beginthread(LocalAcceptFn, 0, NULL); 5804 // leaves the pool running deaf: nobody accepts, announces, or pays, and
1912 _beginthread(ThreadBtfPoolAnnouncer, 0, NULL); 5805 // nothing in the log says why.
1913 _beginthread(PayoutThreadFn, 0, NULL); 5806 if (_beginthread(AcceptLoopFn, 0, NULL) == (uintptr_t)-1)
5807 LogPrint("pool", "[pool] FATAL: accept loop thread failed to spawn -- miners cannot connect\n");
5808 if (_beginthread(LocalAcceptFn, 0, NULL) == (uintptr_t)-1)
5809 LogPrint("pool", "[pool] local accept thread failed to spawn\n");
5810 if (_beginthread(ThreadBtfPoolAnnouncer, 0, NULL) == (uintptr_t)-1)
5811 LogPrint("pool", "[pool] announcer thread failed to spawn -- pool will not be advertised\n");
5812 if (_beginthread(PayoutThreadFn, 0, NULL) == (uintptr_t)-1)
5813 LogPrint("pool", "[pool] FATAL: payout thread failed to spawn -- no one gets paid\n");
1914 5814
1915 PoolEventLoop(); // blocks until gPoolRunning = false or fShutdown 5815 PoolEventLoop(); // blocks until gPoolRunning = false or fShutdown
1916 5816
diff --git a/src/selftest.cpp b/src/selftest.cpp
index ff0289b..357d532 100644
--- a/src/selftest.cpp
+++ b/src/selftest.cpp
@@ -6,9 +6,20 @@
6// and against a temporary data directory. 6// and against a temporary data directory.
7 7
8#include "headers_core.h" 8#include "headers_core.h"
9#include "btfaddr.h"
9#include "bip32.h" 10#include "bip32.h"
11#include "proxy.h"
10#include "selftest.h" 12#include "selftest.h"
13#ifndef _WIN32
14#include <sys/wait.h> // WIFEXITED/WEXITSTATUS for the child-process helper
15#endif
16#include "sockcount.h"
17#include "tor.h"
11#include "walletcmd.h" 18#include "walletcmd.h"
19#include "wallet_sqlite.h"
20
21extern int RunPoolStratumSelfTest();
22extern int RunPayoutLedgerSelfTest();
12 23
13// Test results go to the terminal, not to debug.log. 24// Test results go to the terminal, not to debug.log.
14// 25//
@@ -28,6 +39,7 @@
28#undef printf 39#undef printf
29 40
30#include <mutex> 41#include <mutex>
42#include <cerrno>
31#include <stdexcept> 43#include <stdexcept>
32#include <thread> 44#include <thread>
33 45
@@ -55,6 +67,11 @@ static std::string HexStrLocal(const std::vector<unsigned char>& v)
55 return out; 67 return out;
56} 68}
57 69
70static vector<unsigned char> DataStreamBytes(const CDataStream& ss)
71{
72 return vector<unsigned char>(ss.begin(), ss.end());
73}
74
58static bool MakeTempDir(std::string& out) 75static bool MakeTempDir(std::string& out)
59{ 76{
60#ifdef _WIN32 77#ifdef _WIN32
@@ -348,17 +365,17 @@ static int RunWalletHDSelfTest()
348 nFail += Check(HaveHDSeed(), "the wallet reports having a seed") ? 0 : 1; 365 nFail += Check(HaveHDSeed(), "the wallet reports having a seed") ? 0 : 1;
349 nFail += Check(nHDKeySchema == HD_SCHEMA_BIP44, 366 nFail += Check(nHDKeySchema == HD_SCHEMA_BIP44,
350 "a new recovery phrase records the BIP44 HD schema") ? 0 : 1; 367 "a new recovery phrase records the BIP44 HD schema") ? 0 : 1;
351 nFail += Check(nHDCoinType == HD_BIP44_COIN_TYPE_BITFLASH_PROVISIONAL, 368 nFail += Check(nHDCoinType == HD_BIP44_COIN_TYPE_BITFLASH,
352 "a new recovery phrase records the provisional BIP44 coin type") ? 0 : 1; 369 "a new recovery phrase records the SLIP-0044 BIP44 coin type") ? 0 : 1;
353 nFail += Check(nHDReceiveNext == 1 && nHDChangeNext == 0, 370 nFail += Check(nHDReceiveNext == 1 && nHDChangeNext == 0,
354 "BIP44 receive/change counters reserve the default receive key") ? 0 : 1; 371 "BIP44 receive/change counters reserve the default receive key") ? 0 : 1;
355 std::vector<unsigned int> vBIP44Path = HDBIP44Path(HD_BIP44_COIN_TYPE_BITFLASH_PROVISIONAL, 372 std::vector<unsigned int> vBIP44Path = HDBIP44Path(HD_BIP44_COIN_TYPE_BITFLASH,
356 HD_BIP44_ACCOUNT, 373 HD_BIP44_ACCOUNT,
357 HD_BIP44_CHAIN_RECEIVE, 374 HD_BIP44_CHAIN_RECEIVE,
358 0); 375 0);
359 nFail += Check(vBIP44Path.size() == 5 && 376 nFail += Check(vBIP44Path.size() == 5 &&
360 vBIP44Path[0] == (HD_BIP44_PURPOSE | bitflash::BIP32_HARDENED) && 377 vBIP44Path[0] == (HD_BIP44_PURPOSE | bitflash::BIP32_HARDENED) &&
361 vBIP44Path[1] == (HD_BIP44_COIN_TYPE_BITFLASH_PROVISIONAL | bitflash::BIP32_HARDENED) && 378 vBIP44Path[1] == (HD_BIP44_COIN_TYPE_BITFLASH | bitflash::BIP32_HARDENED) &&
362 vBIP44Path[2] == (HD_BIP44_ACCOUNT | bitflash::BIP32_HARDENED) && 379 vBIP44Path[2] == (HD_BIP44_ACCOUNT | bitflash::BIP32_HARDENED) &&
363 vBIP44Path[3] == HD_BIP44_CHAIN_RECEIVE && 380 vBIP44Path[3] == HD_BIP44_CHAIN_RECEIVE &&
364 vBIP44Path[4] == 0, 381 vBIP44Path[4] == 0,
@@ -380,7 +397,7 @@ static int RunWalletHDSelfTest()
380 hdParent.chainCode = vchHDChainCode; 397 hdParent.chainCode = vchHDChainCode;
381 bitflash::BIP32PrivateNode hdBIP44ReceiveChild; 398 bitflash::BIP32PrivateNode hdBIP44ReceiveChild;
382 if (!bitflash::BIP32DerivePath(hdParent, 399 if (!bitflash::BIP32DerivePath(hdParent,
383 HDBIP44Path(HD_BIP44_COIN_TYPE_BITFLASH_PROVISIONAL, 400 HDBIP44Path(HD_BIP44_COIN_TYPE_BITFLASH,
384 HD_BIP44_ACCOUNT, 401 HD_BIP44_ACCOUNT,
385 HD_BIP44_CHAIN_RECEIVE, 402 HD_BIP44_CHAIN_RECEIVE,
386 0), 403 0),
@@ -568,7 +585,7 @@ static int RunWalletHDSelfTest()
568 nFail += Check(audit.fHaveSeed, "the recovery audit reports the phrase") ? 0 : 1; 585 nFail += Check(audit.fHaveSeed, "the recovery audit reports the phrase") ? 0 : 1;
569 nFail += Check(audit.nSchema == HD_SCHEMA_BIP44, 586 nFail += Check(audit.nSchema == HD_SCHEMA_BIP44,
570 "the recovery audit reports the derivation schema") ? 0 : 1; 587 "the recovery audit reports the derivation schema") ? 0 : 1;
571 nFail += Check(audit.nCoinType == HD_BIP44_COIN_TYPE_BITFLASH_PROVISIONAL, 588 nFail += Check(audit.nCoinType == HD_BIP44_COIN_TYPE_BITFLASH,
572 "the recovery audit reports the BIP44 coin type") ? 0 : 1; 589 "the recovery audit reports the BIP44 coin type") ? 0 : 1;
573 nFail += Check(audit.nReceiveNext == nHDReceiveNext && audit.nChangeNext == 1, 590 nFail += Check(audit.nReceiveNext == nHDReceiveNext && audit.nChangeNext == 1,
574 "the recovery audit reports receive/change counters") ? 0 : 1; 591 "the recovery audit reports receive/change counters") ? 0 : 1;
@@ -582,6 +599,34 @@ static int RunWalletHDSelfTest()
582 "the recovery audit finds wallet.dat-only immature mining rewards") ? 0 : 1; 599 "the recovery audit finds wallet.dat-only immature mining rewards") ? 0 : 1;
583 nFail += Check(audit.nLegacyImmatureTx == 1, 600 nFail += Check(audit.nLegacyImmatureTx == 1,
584 "the recovery audit counts wallet.dat-only immature mining rewards") ? 0 : 1; 601 "the recovery audit counts wallet.dat-only immature mining rewards") ? 0 : 1;
602
603 CKey keyLegacyHD11;
604 nSchemaForLegacyAudit = nHDKeySchema;
605 nHDKeySchema = HD_SCHEMA_LEGACY;
606 if (!DeriveHDKey(11, keyLegacyHD11, strError))
607 throw std::runtime_error("legacy compatibility index 11 derivation failed: " + strError);
608 nHDKeySchema = nSchemaForLegacyAudit;
609 if (!AddKey(keyLegacyHD11))
610 throw std::runtime_error("could not store the legacy compatibility index 11 key");
611
612 std::map<unsigned int, std::vector<unsigned char> > mapLegacyProbe;
613 mapLegacyProbe[0] = keyAuditLegacyHD.GetPubKey();
614 mapLegacyProbe[11] = keyLegacyHD11.GetPubKey();
615
616 CWalletTx wtxLegacyHD11;
617 wtxLegacyHD11.vout.push_back(CTxOut(19 * COIN, CScript() << keyLegacyHD11.GetPubKey() << OP_CHECKSIG));
618 CRITICAL_BLOCK(cs_mapWallet)
619 {
620 mapWallet.clear();
621 mapWallet[wtxLegacyHD11.GetHash()] = wtxLegacyHD11;
622 }
623 nFail += Check(WalletLastUsedPubKeyIndexNext(mapLegacyProbe) == 12,
624 "restore collapses legacy hdnext to the last used compatibility index plus one") ? 0 : 1;
625
626 CRITICAL_BLOCK(cs_mapWallet)
627 mapWallet.clear();
628 nFail += Check(WalletLastUsedPubKeyIndexNext(mapLegacyProbe) == 0,
629 "restore does not persist legacy hdnext at the scan depth when no compatibility key was used") ? 0 : 1;
585 } 630 }
586 catch (const std::exception& e) 631 catch (const std::exception& e)
587 { 632 {
@@ -701,6 +746,1017 @@ static int RunWalletFormatSelfTest()
701 return nFail == 0 ? 0 : 1; 746 return nFail == 0 ? 0 : 1;
702} 747}
703 748
749static bool FileContainsText(const string& strPath, const string& strNeedle);
750static bool MakeDirLocal(const string& strPath);
751static int RunBitflashChild(const string& strExe,
752 const vector<string>& vArgs,
753 const string* pStdinFile,
754 const string* pOutputFile);
755
756class CWalletStorageSanityRawDB : public CWalletDB
757{
758public:
759 CWalletStorageSanityRawDB(const char* pszMode="r+")
760 : CWalletDB(pszMode) { }
761
762 bool WriteUnknownRecord()
763 {
764 return Write(string("storage-sanity-unknown"), 1);
765 }
766
767 bool WriteMalformedKeyRecord()
768 {
769 if (!pdb)
770 return false;
771 unsigned char chKey = 0xff;
772 unsigned char chValue = 0x01;
773 Dbt datKey(&chKey, 1);
774 Dbt datValue(&chValue, 1);
775 return pdb->put(GetTxn(), &datKey, &datValue, 0) == 0;
776 }
777
778 bool MakePlainHDSeedIncomplete()
779 {
780 return Erase(string("hdchaincode"));
781 }
782
783 bool AddEncryptedMarkerToPlainWallet()
784 {
785 CWalletMasterKey kMasterKey;
786 kMasterKey.vchCryptedKey.resize(48);
787 for (size_t i = 0; i < kMasterKey.vchCryptedKey.size(); i++)
788 kMasterKey.vchCryptedKey[i] = (unsigned char)(0x80 + (i & 0x3f));
789 for (size_t i = 0; i < kMasterKey.vchSalt.size(); i++)
790 kMasterKey.vchSalt[i] = (unsigned char)(0x40 + i);
791 return WriteWalletMinVersion(WALLET_FORMAT_ENCRYPTED) &&
792 WriteMasterKey(1, kMasterKey);
793 }
794};
795
796static const char* SELFTEST_MUTATE_WALLET_MARKER =
797 ".bitflash-selftest-wallet-mutate-ok";
798static const char* SELFTEST_MUTATE_WALLET_MARKER_TEXT =
799 "bitflash destructive wallet storage self-test\n";
800
801static string SelfTestMutationMarkerPath()
802{
803 return GetAppDir() + "/" + SELFTEST_MUTATE_WALLET_MARKER;
804}
805
806static bool WriteSelfTestMutationMarker(const string& strDir)
807{
808 string strPath = strDir + "/" + SELFTEST_MUTATE_WALLET_MARKER;
809 FILE* pf = fopen(strPath.c_str(), "wb");
810 if (!pf)
811 return false;
812 bool fOk = fwrite(SELFTEST_MUTATE_WALLET_MARKER_TEXT, 1,
813 strlen(SELFTEST_MUTATE_WALLET_MARKER_TEXT), pf) ==
814 strlen(SELFTEST_MUTATE_WALLET_MARKER_TEXT);
815 if (fclose(pf) != 0)
816 fOk = false;
817 return fOk;
818}
819
820int RunSelfTestMutateWallet(const std::string& name)
821{
822 if (!FileContainsText(SelfTestMutationMarkerPath(),
823 SELFTEST_MUTATE_WALLET_MARKER_TEXT))
824 {
825 fprintf(stderr, "Refusing destructive wallet storage mutation outside "
826 "a marked self-test datadir.\n");
827 return 1;
828 }
829
830 bool fOk = false;
831 try
832 {
833 CWalletStorageSanityRawDB walletdb;
834 if (name == "unknown-record")
835 fOk = walletdb.WriteUnknownRecord();
836 else if (name == "malformed-key")
837 fOk = walletdb.WriteMalformedKeyRecord();
838 else if (name == "incomplete-plain-hd")
839 fOk = walletdb.MakePlainHDSeedIncomplete();
840 else if (name == "encrypted-marker-with-plain-keys")
841 fOk = walletdb.AddEncryptedMarkerToPlainWallet();
842 else
843 {
844 fprintf(stderr, "Unknown wallet storage mutation '%s'\n", name.c_str());
845 return 1;
846 }
847 }
848 catch (const std::exception& e)
849 {
850 fprintf(stderr, "Cannot mutate wallet storage: %s\n", e.what());
851 return 1;
852 }
853 catch (...)
854 {
855 fprintf(stderr, "Cannot mutate wallet storage: unknown exception\n");
856 return 1;
857 }
858
859 DBFlush(true);
860 return fOk ? 0 : 1;
861}
862
863static int RunWalletStorageSanitySelfTest()
864{
865 fflush(stdout);
866 printf("wallet-storage-sanity self-test\n");
867
868 std::string tmp;
869 std::string cwd;
870 if (!MakeTempDir(tmp))
871 {
872 printf(" FAIL could not create a temporary data directory\n");
873 return 1;
874 }
875 if (!GetCurrentDir(cwd))
876 {
877 printf(" FAIL could not read current directory\n");
878 RemoveTree(tmp);
879 return 1;
880 }
881
882 int nFail = 0;
883 printf(" temp root: %s\n", tmp.c_str());
884
885 try
886 {
887#ifdef _WIN32
888 string strExe = cwd + "\\bitflash.exe";
889#else
890 string strExe = cwd + "/bitflash-node";
891#endif
892
893 struct StorageScenario
894 {
895 const char* pszName;
896 const char* pszExpectedFailure;
897 };
898
899 StorageScenario scenarios[] = {
900 { "unknown-record", "wallet.dat contains unknown records" },
901 { "malformed-key", "wallet.dat contains malformed records" },
902 { "incomplete-plain-hd", "plain HD seed is incomplete" },
903 { "encrypted-marker-with-plain-keys",
904 "encrypted wallet still contains plain private key records" },
905 };
906
907 for (size_t i = 0; i < sizeof(scenarios)/sizeof(scenarios[0]); i++)
908 {
909 string strScenarioDir = tmp + "/" + scenarios[i].pszName;
910 nFail += Check(MakeDirLocal(strScenarioDir),
911 "scenario datadir can be created") ? 0 : 1;
912
913 string strCreateOut = tmp + "/" + scenarios[i].pszName + "-create.txt";
914 vector<string> vCreateArgs;
915 vCreateArgs.push_back("-datadir=" + strScenarioDir);
916 vCreateArgs.push_back("-nomanagedtor");
917 vCreateArgs.push_back("-nogui");
918 vCreateArgs.push_back("-walletbackend=bdb"); // this suite is about wallet.dat
919 vCreateArgs.push_back("-newphrase");
920 int nCreateRet = RunBitflashChild(strExe, vCreateArgs, NULL, &strCreateOut);
921 nFail += Check(nCreateRet == 0 &&
922 FileContainsText(strCreateOut, "Write these twelve words down"),
923 "scenario phrase wallet can be created") ? 0 : 1;
924
925 if (i == 0)
926 {
927 string strBlockedOut = tmp + "/" + scenarios[i].pszName + "-blocked-mutate.txt";
928 vector<string> vBlockedArgs;
929 vBlockedArgs.push_back("-datadir=" + strScenarioDir);
930 vBlockedArgs.push_back("-nomanagedtor");
931 vBlockedArgs.push_back("-nogui");
932 vBlockedArgs.push_back("-selftestmutatewallet=" + string(scenarios[i].pszName));
933 int nBlockedRet = RunBitflashChild(strExe, vBlockedArgs, NULL, &strBlockedOut);
934 nFail += Check(nBlockedRet != 0 &&
935 FileContainsText(strBlockedOut, "Refusing destructive wallet storage mutation"),
936 "wallet mutation helper is blocked without marker") ? 0 : 1;
937 }
938
939 nFail += Check(WriteSelfTestMutationMarker(strScenarioDir),
940 "scenario mutation marker can be written") ? 0 : 1;
941
942 string strMutateOut = tmp + "/" + scenarios[i].pszName + "-mutate.txt";
943 vector<string> vMutateArgs;
944 vMutateArgs.push_back("-datadir=" + strScenarioDir);
945 vMutateArgs.push_back("-nomanagedtor");
946 vMutateArgs.push_back("-nogui");
947 vMutateArgs.push_back("-selftestmutatewallet=" + string(scenarios[i].pszName));
948 int nMutateRet = RunBitflashChild(strExe, vMutateArgs, NULL, &strMutateOut);
949 nFail += Check(nMutateRet == 0, scenarios[i].pszName) ? 0 : 1;
950
951 string strCheckOut = tmp + "/" + scenarios[i].pszName + "-check.txt";
952 vector<string> vCheckArgs;
953 vCheckArgs.push_back("-datadir=" + strScenarioDir);
954 vCheckArgs.push_back("-nomanagedtor");
955 vCheckArgs.push_back("-nogui");
956 vCheckArgs.push_back("-walletstoragecheck");
957 int nCheckRet = RunBitflashChild(strExe, vCheckArgs, NULL, &strCheckOut);
958 nFail += Check(nCheckRet == 2 &&
959 FileContainsText(strCheckOut, "storage sanity: failed") &&
960 FileContainsText(strCheckOut, scenarios[i].pszExpectedFailure),
961 scenarios[i].pszExpectedFailure) ? 0 : 1;
962 }
963 }
964 catch (const std::exception& e)
965 {
966 printf(" FAIL exception: %s\n", e.what());
967 nFail++;
968 }
969 catch (...)
970 {
971 printf(" FAIL unknown exception\n");
972 nFail++;
973 }
974
975 RemoveTree(tmp);
976 printf("%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
977 nFail, nFail == 1 ? "" : "s");
978 fflush(stdout);
979 return nFail == 0 ? 0 : 1;
980}
981
982static int RunDbEnvReopenSelfTest()
983{
984 fflush(stdout);
985 printf("db-env-reopen self-test\n");
986
987 std::string tmp;
988 if (!MakeTempDir(tmp))
989 {
990 printf(" FAIL could not create a temporary data directory\n");
991 return 1;
992 }
993
994 int nFail = 0;
995 std::string strOldDataDir = strSetDataDir;
996 strSetDataDir = tmp;
997 printf(" temp datadir: %s\n", tmp.c_str());
998
999 try
1000 {
1001 for (int i = 0; i < 5; i++)
1002 {
1003 int64 nValue = 1000 + i;
1004 {
1005 CWalletDB walletdb("cr+");
1006 nFail += Check(walletdb.WriteSetting("db-env-reopen-cycle", nValue),
1007 "wallet setting can be written before shutdown") ? 0 : 1;
1008 }
1009
1010 DBFlush(true);
1011
1012 int64 nRead = 0;
1013 {
1014 CWalletDB walletdb("r");
1015 nFail += Check(walletdb.ReadSetting("db-env-reopen-cycle", nRead),
1016 "wallet setting can be read after env reopen") ? 0 : 1;
1017 }
1018 nFail += Check(nRead == nValue,
1019 "reopened environment reads the last committed value") ? 0 : 1;
1020
1021 DBFlush(true);
1022 }
1023 }
1024 catch (const std::exception& e)
1025 {
1026 printf(" FAIL exception: %s\n", e.what());
1027 nFail++;
1028 }
1029 catch (...)
1030 {
1031 printf(" FAIL unknown exception\n");
1032 nFail++;
1033 }
1034
1035 DBFlush(true);
1036 strSetDataDir = strOldDataDir;
1037 RemoveTree(tmp);
1038 printf("%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
1039 nFail, nFail == 1 ? "" : "s");
1040 fflush(stdout);
1041 return nFail == 0 ? 0 : 1;
1042}
1043
1044static int RunWalletSQLiteSelfTest()
1045{
1046 fflush(stdout);
1047 printf("wallet-sqlite self-test\n");
1048
1049 std::string tmp;
1050 if (!MakeTempDir(tmp))
1051 {
1052 printf(" FAIL could not create a temporary data directory\n");
1053 return 1;
1054 }
1055
1056 int nFail = 0;
1057 string strPath = tmp + "/wallet.sqlite";
1058 string strError;
1059 string strOldRuntimeDataDir;
1060 bool fRestoreRuntimeDataDir = false;
1061
1062 try
1063 {
1064 CDataStream ssKeySetting(SER_DISK);
1065 ssKeySetting << make_pair(string("setting"), string("sqlite-roundtrip"));
1066 CDataStream ssValueSetting(SER_DISK);
1067 ssValueSetting << (int64)424242;
1068
1069 vector<unsigned char> vchPubKey;
1070 vchPubKey.push_back(0x04);
1071 for (int i = 0; i < 64; i++)
1072 vchPubKey.push_back((unsigned char)i);
1073 CDataStream ssKeyDefault(SER_DISK);
1074 ssKeyDefault << string("defaultkey");
1075 CDataStream ssValueDefault(SER_DISK);
1076 ssValueDefault << vchPubKey;
1077
1078 {
1079 CWalletDBSQLite db;
1080 nFail += Check(db.Open(strPath, strError),
1081 "SQLite wallet opens and creates schema") ? 0 : 1;
1082 nFail += Check(db.BeginTransaction(strError),
1083 "SQLite wallet begins a write transaction") ? 0 : 1;
1084 nFail += Check(db.WriteRecord(DataStreamBytes(ssKeySetting),
1085 DataStreamBytes(ssValueSetting),
1086 strError),
1087 "SQLite wallet writes a serialized setting record") ? 0 : 1;
1088 nFail += Check(db.WriteRecord(DataStreamBytes(ssKeyDefault),
1089 DataStreamBytes(ssValueDefault),
1090 strError),
1091 "SQLite wallet writes a serialized defaultkey record") ? 0 : 1;
1092 nFail += Check(db.CommitTransaction(strError),
1093 "SQLite wallet commits a write transaction") ? 0 : 1;
1094 int nCount = 0;
1095 nFail += Check(db.CountRecords(nCount, strError) && nCount == 2,
1096 "SQLite wallet counts raw records") ? 0 : 1;
1097 }
1098
1099 {
1100 CWalletDBSQLite db;
1101 nFail += Check(db.Open(strPath, strError),
1102 "SQLite wallet reopens") ? 0 : 1;
1103 vector<unsigned char> vchRead;
1104 nFail += Check(db.ReadRecord(DataStreamBytes(ssKeySetting),
1105 vchRead,
1106 strError),
1107 "SQLite wallet reads a serialized setting record") ? 0 : 1;
1108 int64 nReadValue = 0;
1109 CDataStream ssRead(vchRead, SER_DISK);
1110 ssRead >> nReadValue;
1111 nFail += Check(!ssRead.fail() && nReadValue == 424242,
1112 "SQLite wallet preserves serialized values byte-for-byte") ? 0 : 1;
1113
1114 vchRead.clear();
1115 nFail += Check(db.ReadRecord(DataStreamBytes(ssKeyDefault),
1116 vchRead,
1117 strError),
1118 "SQLite wallet reads a serialized defaultkey record") ? 0 : 1;
1119 vector<unsigned char> vchReadPubKey;
1120 CDataStream ssReadDefault(vchRead, SER_DISK);
1121 ssReadDefault >> vchReadPubKey;
1122 nFail += Check(!ssReadDefault.fail() && vchReadPubKey == vchPubKey,
1123 "SQLite wallet preserves public key blobs") ? 0 : 1;
1124 }
1125
1126 {
1127 CWalletDBSQLite db;
1128 nFail += Check(db.Open(strPath, strError),
1129 "SQLite wallet reopens for typed records") ? 0 : 1;
1130
1131 nFail += Check(db.WriteTypedRecord(make_pair(string("setting"),
1132 string("sqlite-typed")),
1133 (int64)777, strError),
1134 "SQLite wallet writes a typed setting record") ? 0 : 1;
1135
1136 int64 nTypedRead = 0;
1137 nFail += Check(db.ReadTypedRecord(make_pair(string("setting"),
1138 string("sqlite-typed")),
1139 nTypedRead, strError) &&
1140 nTypedRead == 777,
1141 "SQLite wallet reads a typed setting record") ? 0 : 1;
1142
1143 nFail += Check(!db.WriteTypedRecord(make_pair(string("setting"),
1144 string("sqlite-typed")),
1145 (int64)888, strError, false),
1146 "SQLite wallet refuses typed overwrite when asked") ? 0 : 1;
1147
1148 nTypedRead = 0;
1149 nFail += Check(db.ReadTypedRecord(make_pair(string("setting"),
1150 string("sqlite-typed")),
1151 nTypedRead, strError) &&
1152 nTypedRead == 777,
1153 "SQLite wallet preserves the old typed value after overwrite refusal") ? 0 : 1;
1154
1155 nFail += Check(db.EraseTypedRecord(make_pair(string("setting"),
1156 string("sqlite-typed")),
1157 strError),
1158 "SQLite wallet erases a typed record") ? 0 : 1;
1159
1160 nTypedRead = 0;
1161 nFail += Check(!db.ReadTypedRecord(make_pair(string("setting"),
1162 string("sqlite-typed")),
1163 nTypedRead, strError),
1164 "SQLite wallet typed erase removes the record") ? 0 : 1;
1165
1166 nFail += Check(db.WriteTypedRecord(string("defaultkey"), vchPubKey,
1167 strError),
1168 "SQLite wallet writes a typed defaultkey record") ? 0 : 1;
1169
1170 vector<unsigned char> vchTypedPubKey;
1171 nFail += Check(db.ReadTypedRecord(string("defaultkey"), vchTypedPubKey,
1172 strError) &&
1173 vchTypedPubKey == vchPubKey,
1174 "SQLite wallet reads a typed defaultkey record") ? 0 : 1;
1175 }
1176
1177 {
1178 string strRuntimePath = tmp + "/runtime-wallet.sqlite";
1179 string strRuntimeDir = tmp + "/runtime-datadir";
1180 nFail += Check(MakeDirLocal(strRuntimeDir),
1181 "SQLite runtime datadir can be created") ? 0 : 1;
1182
1183 {
1184 CWalletDBSQLite db;
1185 nFail += Check(db.Open(strRuntimePath, strError),
1186 "SQLite runtime wallet file can be created") ? 0 : 1;
1187 }
1188
1189 string strOldDataDir = strSetDataDir;
1190 strSetDataDir = strRuntimeDir;
1191 strOldRuntimeDataDir = strOldDataDir;
1192 fRestoreRuntimeDataDir = true;
1193 bool fRuntimeOpened = WalletSQLiteRuntimeOpen(strRuntimePath, strError);
1194 nFail += Check(fRuntimeOpened,
1195 "SQLite runtime wallet can be activated") ? 0 : 1;
1196
1197 if (fRuntimeOpened)
1198 {
1199 CKey key;
1200 key.MakeNewKey();
1201 vector<unsigned char> vchRuntimePubKey = key.GetPubKey();
1202 CPrivKey vchRuntimePrivKey = key.GetPrivKey();
1203 string strRuntimeAddress = PubKeyToAddress(vchRuntimePubKey);
1204
1205 nFail += Check(CWalletDB().WriteSetting("runtime-sqlite-setting",
1206 (int64)909090) &&
1207 CWalletDB().WriteDefaultKey(vchRuntimePubKey) &&
1208 CWalletDB().WriteKey(vchRuntimePubKey,
1209 vchRuntimePrivKey) &&
1210 CWalletDB().WriteName(strRuntimeAddress,
1211 "SQLite Runtime Address"),
1212 "CWalletDB writes route to active SQLite runtime") ? 0 : 1;
1213
1214 int64 nRuntimeSetting = 0;
1215 vector<unsigned char> vchRuntimeDefaultKey;
1216 CPrivKey vchRuntimeReadPrivKey;
1217 string strRuntimeName;
1218 nFail += Check(CWalletDB("r").ReadSetting("runtime-sqlite-setting",
1219 nRuntimeSetting) &&
1220 nRuntimeSetting == 909090 &&
1221 CWalletDB("r").ReadDefaultKey(vchRuntimeDefaultKey) &&
1222 vchRuntimeDefaultKey == vchRuntimePubKey &&
1223 CWalletDB("r").ReadKey(vchRuntimePubKey,
1224 vchRuntimeReadPrivKey) &&
1225 vchRuntimeReadPrivKey == vchRuntimePrivKey &&
1226 CWalletDB("r").ReadName(strRuntimeAddress,
1227 strRuntimeName) &&
1228 strRuntimeName == "SQLite Runtime Address",
1229 "CWalletDB reads route to active SQLite runtime") ? 0 : 1;
1230
1231 nFail += Check(CWalletDB().EraseName(strRuntimeAddress) &&
1232 !CWalletDB("r").ReadName(strRuntimeAddress,
1233 strRuntimeName),
1234 "CWalletDB erases route to active SQLite runtime") ? 0 : 1;
1235
1236 nFail += Check(!FileExists((strRuntimeDir + "/wallet.dat").c_str()),
1237 "active SQLite runtime does not create wallet.dat") ? 0 : 1;
1238
1239 // Transaction bracket: a rolled-back group leaves nothing, a
1240 // committed group persists. This is the all-or-nothing that
1241 // WriteHDMaster relies on so a half-written seed cannot survive
1242 // a failure between its two records.
1243 bool fTxnBegan = WalletSQLiteRuntimeBeginTxn();
1244 bool fTxnWrote = fTxnBegan &&
1245 CWalletDB().WriteSetting("runtime-txn-rollback", (int64)111);
1246 if (fTxnBegan)
1247 WalletSQLiteRuntimeRollbackTxn();
1248 nFail += Check(fTxnBegan && fTxnWrote,
1249 "a SQLite runtime transaction opens and rolls back") ? 0 : 1;
1250 int64 nRolledBack = 0;
1251 nFail += Check(!CWalletDB("r").ReadSetting("runtime-txn-rollback",
1252 nRolledBack),
1253 "a rolled-back SQLite runtime write leaves no record") ? 0 : 1;
1254
1255 bool fCommitOk = WalletSQLiteRuntimeBeginTxn() &&
1256 CWalletDB().WriteSetting("runtime-txn-commit", (int64)222) &&
1257 WalletSQLiteRuntimeCommitTxn();
1258 int64 nCommitted = 0;
1259 nFail += Check(fCommitOk &&
1260 CWalletDB("r").ReadSetting("runtime-txn-commit",
1261 nCommitted) &&
1262 nCommitted == 222,
1263 "a committed SQLite runtime write persists") ? 0 : 1;
1264
1265 // WriteHDMaster brackets its two records in that transaction;
1266 // both halves of the seed must land.
1267 vector<unsigned char> vchTxnMaster(32, 0x5a);
1268 vector<unsigned char> vchTxnChainCode(32, 0xa5);
1269 vector<unsigned char> vchReadMaster, vchReadChainCode;
1270 nFail += Check(CWalletDB().WriteHDMaster(vchTxnMaster, vchTxnChainCode) &&
1271 CWalletDB("r").ReadHDMaster(vchReadMaster, vchReadChainCode) &&
1272 vchReadMaster == vchTxnMaster &&
1273 vchReadChainCode == vchTxnChainCode,
1274 "WriteHDMaster commits both halves of the seed under SQLite") ? 0 : 1;
1275
1276 // Durability: close the backend (checkpoints the WAL), reopen
1277 // the same file, and confirm what CWalletDB wrote is still
1278 // there. This is the write-survives-a-restart property the real
1279 // -walletbackend=sqlite node depends on.
1280 WalletSQLiteRuntimeClose();
1281 bool fReopened =
1282 WalletSQLiteRuntimeOpen(strRuntimePath, strError);
1283 nFail += Check(fReopened,
1284 "SQLite runtime wallet reopens after close") ? 0 : 1;
1285 if (fReopened)
1286 {
1287 int64 nPersisted = 0;
1288 CPrivKey vchPersistedPrivKey;
1289 vector<unsigned char> vchPersistedDefaultKey;
1290 nFail += Check(CWalletDB("r").ReadSetting("runtime-sqlite-setting",
1291 nPersisted) &&
1292 nPersisted == 909090 &&
1293 CWalletDB("r").ReadDefaultKey(vchPersistedDefaultKey) &&
1294 vchPersistedDefaultKey == vchRuntimePubKey &&
1295 CWalletDB("r").ReadKey(vchRuntimePubKey,
1296 vchPersistedPrivKey) &&
1297 vchPersistedPrivKey == vchRuntimePrivKey,
1298 "SQLite runtime writes persist across a close and reopen") ? 0 : 1;
1299 }
1300 }
1301
1302 WalletSQLiteRuntimeClose();
1303 strSetDataDir = strOldDataDir;
1304 fRestoreRuntimeDataDir = false;
1305 }
1306 }
1307 catch (const std::exception& e)
1308 {
1309 WalletSQLiteRuntimeClose();
1310 if (fRestoreRuntimeDataDir)
1311 strSetDataDir = strOldRuntimeDataDir;
1312 printf(" FAIL exception: %s\n", e.what());
1313 nFail++;
1314 }
1315 catch (...)
1316 {
1317 WalletSQLiteRuntimeClose();
1318 if (fRestoreRuntimeDataDir)
1319 strSetDataDir = strOldRuntimeDataDir;
1320 printf(" FAIL unknown exception\n");
1321 nFail++;
1322 }
1323
1324 RemoveTree(tmp);
1325 printf("%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
1326 nFail, nFail == 1 ? "" : "s");
1327 fflush(stdout);
1328 return nFail == 0 ? 0 : 1;
1329}
1330
1331namespace {
1332
1333class CWalletRecordMapVisitor : public CWalletRecordVisitor
1334{
1335public:
1336 std::map<vector<unsigned char>, vector<unsigned char> >& mapRecords;
1337
1338 explicit CWalletRecordMapVisitor(
1339 std::map<vector<unsigned char>, vector<unsigned char> >& mapRecordsIn)
1340 : mapRecords(mapRecordsIn) { }
1341
1342 bool VisitWalletRecord(const CDataStream& ssKey,
1343 const CDataStream& ssValue,
1344 string& strErrorRet)
1345 {
1346 vector<unsigned char> vchKey = DataStreamBytes(ssKey);
1347 if (mapRecords.count(vchKey))
1348 {
1349 strErrorRet = "duplicate serialized wallet key";
1350 return false;
1351 }
1352 mapRecords[vchKey] = DataStreamBytes(ssValue);
1353 return true;
1354 }
1355};
1356
1357} // namespace
1358
1359static bool CreateSQLiteWalletWithEmptyKey(const string& strPath)
1360{
1361 sqlite3* pdb = NULL;
1362 int ret = sqlite3_open_v2(strPath.c_str(), &pdb,
1363 SQLITE_OPEN_READWRITE |
1364 SQLITE_OPEN_CREATE |
1365 SQLITE_OPEN_FULLMUTEX,
1366 NULL);
1367 if (ret != SQLITE_OK || !pdb)
1368 {
1369 if (pdb)
1370 sqlite3_close(pdb);
1371 return false;
1372 }
1373
1374 char* pszErr = NULL;
1375 bool fOk =
1376 sqlite3_exec(pdb,
1377 "CREATE TABLE wallet_records ("
1378 "key BLOB PRIMARY KEY NOT NULL,"
1379 "value BLOB NOT NULL"
1380 ");",
1381 NULL, NULL, &pszErr) == SQLITE_OK &&
1382 sqlite3_exec(pdb,
1383 "INSERT INTO wallet_records(key, value) "
1384 "VALUES(x'', x'01');",
1385 NULL, NULL, &pszErr) == SQLITE_OK;
1386 if (pszErr)
1387 sqlite3_free(pszErr);
1388 sqlite3_close(pdb);
1389 return fOk;
1390}
1391
1392static bool CreateSQLiteWalletWithFutureMinVersion(const string& strPath)
1393{
1394 string strError;
1395 CWalletDBSQLite db;
1396 CDataStream ssKey(SER_DISK);
1397 ssKey << string("walletminversion");
1398 CDataStream ssValue(SER_DISK);
1399 ssValue << (WALLET_FORMAT_SUPPORTED + 1);
1400
1401 if (!db.Open(strPath, strError))
1402 return false;
1403 if (!db.BeginTransaction(strError))
1404 return false;
1405 if (!db.WriteRecord(DataStreamBytes(ssKey), DataStreamBytes(ssValue), strError))
1406 return false;
1407 if (!db.CommitTransaction(strError))
1408 return false;
1409 if (!db.Checkpoint(strError))
1410 return false;
1411 return true;
1412}
1413
1414static bool CreateTypedSQLiteWallet(const string& strPath)
1415{
1416 string strError;
1417 CWalletDBSQLite db;
1418 CKey key;
1419 key.MakeNewKey();
1420
1421 vector<unsigned char> vchPubKey = key.GetPubKey();
1422 CPrivKey vchPrivKey = key.GetPrivKey();
1423 string strAddress = PubKeyToAddress(vchPubKey);
1424
1425 if (!db.Open(strPath, strError))
1426 return false;
1427 if (!db.BeginTransaction(strError))
1428 return false;
1429 if (!db.WriteTypedRecord(make_pair(string("key"), vchPubKey),
1430 vchPrivKey, strError, false))
1431 return false;
1432 if (!db.WriteTypedRecord(string("defaultkey"), vchPubKey, strError))
1433 return false;
1434 if (!db.WriteTypedRecord(make_pair(string("name"), strAddress),
1435 string("Typed SQLite Address"), strError))
1436 return false;
1437 if (!db.CommitTransaction(strError))
1438 return false;
1439 if (!db.Checkpoint(strError))
1440 return false;
1441 return true;
1442}
1443
1444static int RunWalletSQLiteMigrationSelfTest()
1445{
1446 fflush(stdout);
1447 printf("wallet-sqlite-migration self-test\n");
1448
1449 std::string tmp;
1450 std::string cwd;
1451 if (!MakeTempDir(tmp))
1452 {
1453 printf(" FAIL could not create a temporary data directory\n");
1454 return 1;
1455 }
1456 if (!GetCurrentDir(cwd))
1457 {
1458 printf(" FAIL could not read current directory\n");
1459 RemoveTree(tmp);
1460 return 1;
1461 }
1462
1463 int nFail = 0;
1464 printf(" temp root: %s\n", tmp.c_str());
1465
1466 try
1467 {
1468#ifdef _WIN32
1469 string strExe = cwd + "\\bitflash.exe";
1470#else
1471 string strExe = cwd + "/bitflash-node";
1472#endif
1473 string strWalletDir = tmp + "/wallet";
1474 nFail += Check(MakeDirLocal(strWalletDir),
1475 "temporary wallet directory can be created") ? 0 : 1;
1476
1477 string strCreateOut = tmp + "/create.txt";
1478 vector<string> vCreateArgs;
1479 vCreateArgs.push_back("-datadir=" + strWalletDir);
1480 vCreateArgs.push_back("-nomanagedtor");
1481 vCreateArgs.push_back("-nogui");
1482 vCreateArgs.push_back("-walletbackend=bdb"); // migrate FROM a Berkeley DB wallet.dat
1483 vCreateArgs.push_back("-newphrase");
1484 int nCreateRet = RunBitflashChild(strExe, vCreateArgs, NULL, &strCreateOut);
1485 nFail += Check(nCreateRet == 0 &&
1486 FileContainsText(strCreateOut, "Write these twelve words down"),
1487 "a phrase wallet can be created for SQLite export") ? 0 : 1;
1488
1489 string strSQLite = tmp + "/wallet.sqlite";
1490 string strExportOut = tmp + "/sqlite-export.txt";
1491 vector<string> vExportArgs;
1492 vExportArgs.push_back("-datadir=" + strWalletDir);
1493 vExportArgs.push_back("-nomanagedtor");
1494 vExportArgs.push_back("-nogui");
1495 vExportArgs.push_back("-walletsqliteexport=" + strSQLite);
1496 int nExportRet = RunBitflashChild(strExe, vExportArgs, NULL, &strExportOut);
1497 nFail += Check(nExportRet == 0 &&
1498 FileExists(strSQLite.c_str()) &&
1499 FileContainsText(strExportOut, "SQLite wallet export written"),
1500 "wallet.dat can be exported to SQLite") ? 0 : 1;
1501
1502 string strVerifyOut = tmp + "/sqlite-verify.txt";
1503 vector<string> vVerifyArgs;
1504 vVerifyArgs.push_back("-datadir=" + strWalletDir);
1505 vVerifyArgs.push_back("-nomanagedtor");
1506 vVerifyArgs.push_back("-nogui");
1507 vVerifyArgs.push_back("-walletsqliteverify=" + strSQLite);
1508 int nVerifyRet = RunBitflashChild(strExe, vVerifyArgs, NULL, &strVerifyOut);
1509 nFail += Check(nVerifyRet == 0 &&
1510 FileContainsText(strVerifyOut, "verification: ok"),
1511 "SQLite wallet export verifies against wallet.dat") ? 0 : 1;
1512
1513 string strLoadCheckOut = tmp + "/sqlite-loadcheck.txt";
1514 vector<string> vLoadCheckArgs;
1515 vLoadCheckArgs.push_back("-datadir=" + strWalletDir);
1516 vLoadCheckArgs.push_back("-nomanagedtor");
1517 vLoadCheckArgs.push_back("-nogui");
1518 vLoadCheckArgs.push_back("-walletsqliteloadcheck=" + strSQLite);
1519 int nLoadCheckRet =
1520 RunBitflashChild(strExe, vLoadCheckArgs, NULL, &strLoadCheckOut);
1521 nFail += Check(nLoadCheckRet == 0 &&
1522 FileContainsText(strLoadCheckOut,
1523 "load check: ok"),
1524 "SQLite wallet export passes the loader compatibility check") ? 0 : 1;
1525
1526 string strRuntimeOut = tmp + "/sqlite-runtime.txt";
1527 vector<string> vRuntimeArgs;
1528 vRuntimeArgs.push_back("-datadir=" + strWalletDir);
1529 vRuntimeArgs.push_back("-nomanagedtor");
1530 vRuntimeArgs.push_back("-nogui");
1531 vRuntimeArgs.push_back("-walletsqlite=" + strSQLite);
1532 int nRuntimeRet =
1533 RunBitflashChild(strExe, vRuntimeArgs, NULL, &strRuntimeOut);
1534 nFail += Check(nRuntimeRet == 0 &&
1535 FileContainsText(strRuntimeOut,
1536 "SQLite wallet loaded: read-only staging backend"),
1537 "SQLite wallet export loads through the runtime startup path") ? 0 : 1;
1538
1539 string strTypedSQLite = tmp + "/typed-wallet.sqlite";
1540 nFail += Check(CreateTypedSQLiteWallet(strTypedSQLite),
1541 "typed SQLite wallet can be created without Berkeley DB") ? 0 : 1;
1542
1543 string strTypedLoadCheckOut = tmp + "/sqlite-typed-loadcheck.txt";
1544 vector<string> vTypedLoadCheckArgs;
1545 vTypedLoadCheckArgs.push_back("-datadir=" + strWalletDir);
1546 vTypedLoadCheckArgs.push_back("-nomanagedtor");
1547 vTypedLoadCheckArgs.push_back("-nogui");
1548 vTypedLoadCheckArgs.push_back("-walletsqliteloadcheck=" + strTypedSQLite);
1549 int nTypedLoadCheckRet =
1550 RunBitflashChild(strExe, vTypedLoadCheckArgs, NULL,
1551 &strTypedLoadCheckOut);
1552 nFail += Check(nTypedLoadCheckRet == 0 &&
1553 FileContainsText(strTypedLoadCheckOut,
1554 "load check: ok"),
1555 "typed SQLite wallet passes the loader compatibility check") ? 0 : 1;
1556
1557 string strTypedRuntimeDir = tmp + "/typed-runtime-wallet";
1558 nFail += Check(MakeDirLocal(strTypedRuntimeDir),
1559 "typed SQLite runtime directory can be created") ? 0 : 1;
1560 string strTypedRuntimeOut = tmp + "/sqlite-typed-runtime.txt";
1561 vector<string> vTypedRuntimeArgs;
1562 vTypedRuntimeArgs.push_back("-datadir=" + strTypedRuntimeDir);
1563 vTypedRuntimeArgs.push_back("-nomanagedtor");
1564 vTypedRuntimeArgs.push_back("-nogui");
1565 vTypedRuntimeArgs.push_back("-walletsqlite=" + strTypedSQLite);
1566 int nTypedRuntimeRet =
1567 RunBitflashChild(strExe, vTypedRuntimeArgs, NULL,
1568 &strTypedRuntimeOut);
1569 nFail += Check(nTypedRuntimeRet == 0 &&
1570 FileContainsText(strTypedRuntimeOut,
1571 "SQLite wallet loaded: read-only staging backend") &&
1572 !FileExists((strTypedRuntimeDir + "/wallet.dat").c_str()),
1573 "typed SQLite wallet loads through runtime without creating wallet.dat") ? 0 : 1;
1574
1575 string strRestoredDir = tmp + "/restored-wallet";
1576 nFail += Check(MakeDirLocal(strRestoredDir),
1577 "restored wallet directory can be created") ? 0 : 1;
1578 string strRestoreOut = tmp + "/sqlite-restore.txt";
1579 vector<string> vRestoreArgs;
1580 vRestoreArgs.push_back("-datadir=" + strRestoredDir);
1581 vRestoreArgs.push_back("-nomanagedtor");
1582 vRestoreArgs.push_back("-nogui");
1583 vRestoreArgs.push_back("-walletsqliterestore=" + strSQLite);
1584 int nRestoreRet = RunBitflashChild(strExe, vRestoreArgs, NULL, &strRestoreOut);
1585 nFail += Check(nRestoreRet == 0 &&
1586 FileContainsText(strRestoreOut, "records restored:"),
1587 "SQLite wallet export restores into a fresh wallet.dat") ? 0 : 1;
1588
1589 string strVerifyRestoredOut = tmp + "/sqlite-verify-restored.txt";
1590 vector<string> vVerifyRestoredArgs;
1591 vVerifyRestoredArgs.push_back("-datadir=" + strRestoredDir);
1592 vVerifyRestoredArgs.push_back("-nomanagedtor");
1593 vVerifyRestoredArgs.push_back("-nogui");
1594 vVerifyRestoredArgs.push_back("-walletsqliteverify=" + strSQLite);
1595 int nVerifyRestoredRet =
1596 RunBitflashChild(strExe, vVerifyRestoredArgs, NULL, &strVerifyRestoredOut);
1597 nFail += Check(nVerifyRestoredRet == 0 &&
1598 FileContainsText(strVerifyRestoredOut,
1599 "verification: ok"),
1600 "restored wallet.dat verifies against the SQLite export") ? 0 : 1;
1601
1602 string strRestoreOverwriteOut = tmp + "/sqlite-restore-overwrite.txt";
1603 int nRestoreOverwriteRet =
1604 RunBitflashChild(strExe, vRestoreArgs, NULL, &strRestoreOverwriteOut);
1605 nFail += Check(nRestoreOverwriteRet == 1 &&
1606 FileContainsText(strRestoreOverwriteOut,
1607 "Refusing to overwrite existing wallet.dat"),
1608 "SQLite wallet restore refuses to overwrite wallet.dat") ? 0 : 1;
1609
1610 string strBadSQLite = tmp + "/bad-empty-key.sqlite";
1611 nFail += Check(CreateSQLiteWalletWithEmptyKey(strBadSQLite),
1612 "bad SQLite export with an empty key can be created") ? 0 : 1;
1613 string strBadRestoreDir = tmp + "/bad-restore-wallet";
1614 nFail += Check(MakeDirLocal(strBadRestoreDir),
1615 "bad restore wallet directory can be created") ? 0 : 1;
1616 string strBadRestoreOut = tmp + "/sqlite-restore-bad.txt";
1617 vector<string> vBadRestoreArgs;
1618 vBadRestoreArgs.push_back("-datadir=" + strBadRestoreDir);
1619 vBadRestoreArgs.push_back("-nomanagedtor");
1620 vBadRestoreArgs.push_back("-nogui");
1621 vBadRestoreArgs.push_back("-walletsqliterestore=" + strBadSQLite);
1622 int nBadRestoreRet =
1623 RunBitflashChild(strExe, vBadRestoreArgs, NULL, &strBadRestoreOut);
1624 nFail += Check(nBadRestoreRet == 1 &&
1625 FileContainsText(strBadRestoreOut,
1626 "Cannot restore SQLite wallet export") &&
1627 !FileExists((strBadRestoreDir + "/wallet.dat").c_str()),
1628 "failed SQLite wallet restore removes incomplete wallet.dat") ? 0 : 1;
1629
1630 string strFutureSQLite = tmp + "/future-wallet.sqlite";
1631 nFail += Check(CreateSQLiteWalletWithFutureMinVersion(strFutureSQLite),
1632 "SQLite export with a future wallet format can be created") ? 0 : 1;
1633 string strFutureLoadCheckOut = tmp + "/sqlite-loadcheck-future.txt";
1634 vector<string> vFutureLoadCheckArgs;
1635 vFutureLoadCheckArgs.push_back("-datadir=" + strWalletDir);
1636 vFutureLoadCheckArgs.push_back("-nomanagedtor");
1637 vFutureLoadCheckArgs.push_back("-nogui");
1638 vFutureLoadCheckArgs.push_back("-walletsqliteloadcheck=" + strFutureSQLite);
1639 int nFutureLoadCheckRet =
1640 RunBitflashChild(strExe, vFutureLoadCheckArgs, NULL,
1641 &strFutureLoadCheckOut);
1642 nFail += Check(nFutureLoadCheckRet == 2 &&
1643 FileContainsText(strFutureLoadCheckOut,
1644 "unsupported future wallet format") &&
1645 FileContainsText(strFutureLoadCheckOut,
1646 "load check: failed"),
1647 "SQLite wallet load check rejects future wallet formats") ? 0 : 1;
1648
1649 string strFutureRuntimeOut = tmp + "/sqlite-runtime-future.txt";
1650 vector<string> vFutureRuntimeArgs;
1651 vFutureRuntimeArgs.push_back("-datadir=" + strWalletDir);
1652 vFutureRuntimeArgs.push_back("-nomanagedtor");
1653 vFutureRuntimeArgs.push_back("-nogui");
1654 vFutureRuntimeArgs.push_back("-walletsqlite=" + strFutureSQLite);
1655 int nFutureRuntimeRet =
1656 RunBitflashChild(strExe, vFutureRuntimeArgs, NULL,
1657 &strFutureRuntimeOut);
1658 nFail += Check(nFutureRuntimeRet == 1 &&
1659 FileContainsText(strFutureRuntimeOut,
1660 "Cannot open SQLite wallet export") &&
1661 FileContainsText(strFutureRuntimeOut,
1662 "newer version of Bitflash"),
1663 "SQLite runtime startup rejects future wallet formats") ? 0 : 1;
1664
1665 string strOverwriteOut = tmp + "/sqlite-export-overwrite.txt";
1666 int nOverwriteRet =
1667 RunBitflashChild(strExe, vExportArgs, NULL, &strOverwriteOut);
1668 nFail += Check(nOverwriteRet != 0 &&
1669 FileContainsText(strOverwriteOut,
1670 "Refusing to overwrite existing SQLite wallet export"),
1671 "SQLite wallet export refuses to overwrite") ? 0 : 1;
1672 remove((strSQLite + "-wal").c_str());
1673 remove((strSQLite + "-shm").c_str());
1674
1675 std::map<vector<unsigned char>, vector<unsigned char> > mapBDB;
1676 string strOldDataDir = strSetDataDir;
1677 string strScanError;
1678 strSetDataDir = strWalletDir;
1679 CWalletRecordMapVisitor visitor(mapBDB);
1680 bool fScanOk = ScanWalletRecords(visitor, strScanError);
1681 DBFlush(true);
1682 strSetDataDir = strOldDataDir;
1683 nFail += Check(fScanOk && !mapBDB.empty(),
1684 "wallet.dat records can be scanned for comparison") ? 0 : 1;
1685
1686 CWalletDBSQLite db;
1687 string strError;
1688 nFail += Check(db.Open(strSQLite, strError),
1689 "exported SQLite wallet opens") ? 0 : 1;
1690 int nSQLiteRecords = 0;
1691 nFail += Check(db.CountRecords(nSQLiteRecords, strError) &&
1692 nSQLiteRecords == (int)mapBDB.size(),
1693 "SQLite export has the same record count as wallet.dat") ? 0 : 1;
1694
1695 bool fAllMatch = true;
1696 for (std::map<vector<unsigned char>, vector<unsigned char> >::const_iterator it =
1697 mapBDB.begin(); it != mapBDB.end(); ++it)
1698 {
1699 vector<unsigned char> vchSQLiteValue;
1700 if (!db.ReadRecord(it->first, vchSQLiteValue, strError) ||
1701 vchSQLiteValue != it->second)
1702 {
1703 fAllMatch = false;
1704 break;
1705 }
1706 }
1707 nFail += Check(fAllMatch,
1708 "SQLite export preserves every wallet.dat key/value byte-for-byte") ? 0 : 1;
1709
1710 std::map<vector<unsigned char>, vector<unsigned char> > mapSQLite;
1711 CWalletRecordMapVisitor sqliteVisitor(mapSQLite);
1712 nFail += Check(db.ScanRecords(sqliteVisitor, strError) &&
1713 mapSQLite == mapBDB,
1714 "SQLite wallet scanner streams every exported record byte-for-byte") ? 0 : 1;
1715
1716 db.Close();
1717 CWalletDBSQLite dbCorrupt;
1718 vector<unsigned char> vchBadValue;
1719 vchBadValue.push_back(0xba);
1720 vchBadValue.push_back(0xad);
1721 nFail += Check(!mapBDB.empty() &&
1722 dbCorrupt.Open(strSQLite, strError) &&
1723 dbCorrupt.BeginTransaction(strError) &&
1724 dbCorrupt.WriteRecord(mapBDB.begin()->first,
1725 vchBadValue,
1726 strError) &&
1727 dbCorrupt.CommitTransaction(strError) &&
1728 dbCorrupt.Checkpoint(strError),
1729 "SQLite export can be deliberately corrupted for verifier test") ? 0 : 1;
1730 dbCorrupt.Close();
1731
1732 string strVerifyBadOut = tmp + "/sqlite-verify-bad.txt";
1733 int nVerifyBadRet =
1734 RunBitflashChild(strExe, vVerifyArgs, NULL, &strVerifyBadOut);
1735 nFail += Check(nVerifyBadRet == 2 &&
1736 FileContainsText(strVerifyBadOut,
1737 "record value mismatches: 1") &&
1738 FileContainsText(strVerifyBadOut,
1739 "verification: failed"),
1740 "SQLite wallet verifier detects a mismatched record value") ? 0 : 1;
1741 }
1742 catch (const std::exception& e)
1743 {
1744 printf(" FAIL exception: %s\n", e.what());
1745 nFail++;
1746 }
1747 catch (...)
1748 {
1749 printf(" FAIL unknown exception\n");
1750 nFail++;
1751 }
1752
1753 RemoveTree(tmp);
1754 printf("%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
1755 nFail, nFail == 1 ? "" : "s");
1756 fflush(stdout);
1757 return nFail == 0 ? 0 : 1;
1758}
1759
704static void ClearWalletRuntimeForTest() 1760static void ClearWalletRuntimeForTest()
705{ 1761{
706 CRITICAL_BLOCK(cs_mapKeys) 1762 CRITICAL_BLOCK(cs_mapKeys)
@@ -891,6 +1947,90 @@ static bool WriteTextFile(const string& strPath, const string& strText)
891 return fOk; 1947 return fOk;
892} 1948}
893 1949
1950static bool MakeDirLocal(const string& strPath)
1951{
1952#ifdef _WIN32
1953 return CreateDirectoryA(strPath.c_str(), NULL) || GetLastError() == ERROR_ALREADY_EXISTS;
1954#else
1955 return mkdir(strPath.c_str(), 0700) == 0 || errno == EEXIST;
1956#endif
1957}
1958
1959static bool CopyFileLocal(const string& strSrc, const string& strDst)
1960{
1961 FILE* in = fopen(strSrc.c_str(), "rb");
1962 if (!in)
1963 return false;
1964 FILE* out = fopen(strDst.c_str(), "wb");
1965 if (!out)
1966 {
1967 fclose(in);
1968 return false;
1969 }
1970
1971 bool fOk = true;
1972 char buf[65536];
1973 size_t n = 0;
1974 while ((n = fread(buf, 1, sizeof(buf), in)) > 0)
1975 {
1976 if (fwrite(buf, 1, n, out) != n)
1977 {
1978 fOk = false;
1979 break;
1980 }
1981 }
1982 if (ferror(in))
1983 fOk = false;
1984 if (fclose(out) != 0)
1985 fOk = false;
1986 fclose(in);
1987 return fOk;
1988}
1989
1990static bool DirectoryHasFileWithPrefix(const string& strDir, const string& strPrefix)
1991{
1992#ifdef _WIN32
1993 WIN32_FIND_DATAA findData;
1994 string pattern = strDir + "\\*";
1995 HANDLE hFind = FindFirstFileA(pattern.c_str(), &findData);
1996 if (hFind == INVALID_HANDLE_VALUE)
1997 return false;
1998 bool fFound = false;
1999 do
2000 {
2001 if (!(findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY))
2002 {
2003 string name = findData.cFileName;
2004 if (name.compare(0, strPrefix.size(), strPrefix) == 0)
2005 {
2006 fFound = true;
2007 break;
2008 }
2009 }
2010 }
2011 while (FindNextFileA(hFind, &findData));
2012 FindClose(hFind);
2013 return fFound;
2014#else
2015 DIR* dir = opendir(strDir.c_str());
2016 if (!dir)
2017 return false;
2018 bool fFound = false;
2019 struct dirent* ent;
2020 while ((ent = readdir(dir)) != NULL)
2021 {
2022 string name = ent->d_name;
2023 if (name.compare(0, strPrefix.size(), strPrefix) == 0)
2024 {
2025 fFound = true;
2026 break;
2027 }
2028 }
2029 closedir(dir);
2030 return fFound;
2031#endif
2032}
2033
894static string QuoteCommandArg(const string& str) 2034static string QuoteCommandArg(const string& str)
895{ 2035{
896 string out = "\""; 2036 string out = "\"";
@@ -907,7 +2047,8 @@ static string QuoteCommandArg(const string& str)
907 2047
908static int RunBitflashChild(const string& strExe, 2048static int RunBitflashChild(const string& strExe,
909 const vector<string>& vArgs, 2049 const vector<string>& vArgs,
910 const string* pStdinFile = NULL) 2050 const string* pStdinFile = NULL,
2051 const string* pOutputFile = NULL)
911{ 2052{
912#ifdef _WIN32 2053#ifdef _WIN32
913 vector<char*> argv; 2054 vector<char*> argv;
@@ -920,8 +2061,16 @@ static int RunBitflashChild(const string& strExe,
920 int nOldErr = _dup(2); 2061 int nOldErr = _dup(2);
921 int nOldIn = _dup(0); 2062 int nOldIn = _dup(0);
922 int nNull = _open("NUL", _O_WRONLY); 2063 int nNull = _open("NUL", _O_WRONLY);
2064 int nOut = -1;
923 int nIn = -1; 2065 int nIn = -1;
924 if (nNull >= 0) 2066 if (pOutputFile)
2067 nOut = _open(pOutputFile->c_str(), _O_WRONLY|_O_CREAT|_O_TRUNC|_O_BINARY, _S_IREAD|_S_IWRITE);
2068 if (nOut >= 0)
2069 {
2070 _dup2(nOut, 1);
2071 _dup2(nOut, 2);
2072 }
2073 else if (nNull >= 0)
925 { 2074 {
926 _dup2(nNull, 1); 2075 _dup2(nNull, 1);
927 _dup2(nNull, 2); 2076 _dup2(nNull, 2);
@@ -935,6 +2084,8 @@ static int RunBitflashChild(const string& strExe,
935 int nRet = _spawnv(_P_WAIT, strExe.c_str(), &argv[0]); 2084 int nRet = _spawnv(_P_WAIT, strExe.c_str(), &argv[0]);
936 if (nIn >= 0) 2085 if (nIn >= 0)
937 _close(nIn); 2086 _close(nIn);
2087 if (nOut >= 0)
2088 _close(nOut);
938 if (nNull >= 0) 2089 if (nNull >= 0)
939 _close(nNull); 2090 _close(nNull);
940 if (nOldIn >= 0) 2091 if (nOldIn >= 0)
@@ -959,8 +2110,17 @@ static int RunBitflashChild(const string& strExe,
959 strCmd += " " + QuoteCommandArg(vArgs[i]); 2110 strCmd += " " + QuoteCommandArg(vArgs[i]);
960 if (pStdinFile) 2111 if (pStdinFile)
961 strCmd += " < " + QuoteCommandArg(*pStdinFile); 2112 strCmd += " < " + QuoteCommandArg(*pStdinFile);
962 strCmd += " > /dev/null 2>&1"; 2113 if (pOutputFile)
963 return system(strCmd.c_str()); 2114 strCmd += " > " + QuoteCommandArg(*pOutputFile) + " 2>&1";
2115 else
2116 strCmd += " > /dev/null 2>&1";
2117 // system() returns a wait status, not the exit code; unwrap it so callers
2118 // see the same value the Windows _spawnv path returns (e.g. exit code 2 is
2119 // 2, not 512). Without this every "== 2" assertion fails only on Linux.
2120 int nSysRet = system(strCmd.c_str());
2121 if (nSysRet != -1 && WIFEXITED(nSysRet))
2122 return WEXITSTATUS(nSysRet);
2123 return nSysRet;
964#endif 2124#endif
965} 2125}
966 2126
@@ -1001,6 +2161,13 @@ static int RunWalletEncryptSelfTest()
1001 TopUpKeyPool(); 2161 TopUpKeyPool();
1002 2162
1003 vector<unsigned char> vchDefaultPubKey = keyUser.GetPubKey(); 2163 vector<unsigned char> vchDefaultPubKey = keyUser.GetPubKey();
2164 CKey publicLabelKey;
2165 publicLabelKey.MakeNewKey();
2166 string strPublicLabelAddress = PubKeyToAddress(publicLabelKey.GetPubKey());
2167 nFail += Check(CWalletDB().WriteName(strPublicLabelAddress, "rewrite-public-name") &&
2168 CWalletDB().WriteSetting("rewrite-public-setting", (int64)424242),
2169 "public wallet records can be written before encryption") ? 0 : 1;
2170
1004 CPrivKey vchDefaultPrivKey; 2171 CPrivKey vchDefaultPrivKey;
1005 nFail += Check(GetWalletPrivKey(vchDefaultPubKey, vchDefaultPrivKey, strError), 2172 nFail += Check(GetWalletPrivKey(vchDefaultPubKey, vchDefaultPrivKey, strError),
1006 "the default private key is readable before encryption") ? 0 : 1; 2173 "the default private key is readable before encryption") ? 0 : 1;
@@ -1028,16 +2195,31 @@ static int RunWalletEncryptSelfTest()
1028#else 2195#else
1029 string strExe = cwd + "/bitflash-node"; 2196 string strExe = cwd + "/bitflash-node";
1030#endif 2197#endif
2198 string strStorageAudit = tmp + "/encrypted-storage-audit.txt";
1031 string strWrongDump = tmp + "/wrong-pass-dump.txt"; 2199 string strWrongDump = tmp + "/wrong-pass-dump.txt";
1032 string strRightDump = tmp + "/right-pass-dump.txt"; 2200 string strRightDump = tmp + "/right-pass-dump.txt";
1033 string strLiteralDump = tmp + "/literal-pass-dump.txt"; 2201 string strLiteralDump = tmp + "/literal-pass-dump.txt";
1034 string strStdinDump = tmp + "/stdin-pass-dump.txt"; 2202 string strStdinDump = tmp + "/stdin-pass-dump.txt";
2203 string strLockedAudit = tmp + "/locked-recovery-audit.txt";
2204 string strUnlockedAudit = tmp + "/unlocked-recovery-audit.txt";
1035 string strWrongPassFile = tmp + "/wrong-pass.txt"; 2205 string strWrongPassFile = tmp + "/wrong-pass.txt";
1036 string strRightPassFile = tmp + "/right-pass.txt"; 2206 string strRightPassFile = tmp + "/right-pass.txt";
1037 nFail += Check(WriteTextFile(strWrongPassFile, "wrong-passphrase\n") && 2207 nFail += Check(WriteTextFile(strWrongPassFile, "wrong-passphrase\n") &&
1038 WriteTextFile(strRightPassFile, "btf-test-passphrase\n"), 2208 WriteTextFile(strRightPassFile, "btf-test-passphrase\n"),
1039 "passphrase files can be written for restarted commands") ? 0 : 1; 2209 "passphrase files can be written for restarted commands") ? 0 : 1;
1040 2210
2211 vector<string> vStorageAuditArgs;
2212 vStorageAuditArgs.push_back("-datadir=" + tmp);
2213 vStorageAuditArgs.push_back("-walletstorageaudit");
2214 vStorageAuditArgs.push_back("-nogui");
2215 int nStorageAuditRet = RunBitflashChild(strExe, vStorageAuditArgs, NULL, &strStorageAudit);
2216 nFail += Check(nStorageAuditRet == 0 &&
2217 FileContainsText(strStorageAudit, "address book labels: 3"),
2218 "the encrypted rewrite keeps address-book records") ? 0 : 1;
2219 nFail += Check(nStorageAuditRet == 0 &&
2220 FileContainsText(strStorageAudit, "settings: 1"),
2221 "the encrypted rewrite keeps wallet setting records") ? 0 : 1;
2222
1041 vector<string> vWrongArgs; 2223 vector<string> vWrongArgs;
1042 vWrongArgs.push_back("-datadir=" + tmp); 2224 vWrongArgs.push_back("-datadir=" + tmp);
1043 vWrongArgs.push_back("-walletpassphrase=@" + strWrongPassFile); 2225 vWrongArgs.push_back("-walletpassphrase=@" + strWrongPassFile);
@@ -1047,6 +2229,17 @@ static int RunWalletEncryptSelfTest()
1047 nFail += Check(nWrongRet != 0 && !FileExists(strWrongDump.c_str()), 2229 nFail += Check(nWrongRet != 0 && !FileExists(strWrongDump.c_str()),
1048 "a restarted wallet rejects the wrong-passphrase") ? 0 : 1; 2230 "a restarted wallet rejects the wrong-passphrase") ? 0 : 1;
1049 2231
2232 vector<string> vLockedAuditArgs;
2233 vLockedAuditArgs.push_back("-datadir=" + tmp);
2234 vLockedAuditArgs.push_back("-recoveryaudit");
2235 vLockedAuditArgs.push_back("-nogui");
2236 int nLockedAuditRet = RunBitflashChild(strExe, vLockedAuditArgs, NULL, &strLockedAudit);
2237 nFail += Check(nLockedAuditRet == 2 &&
2238 FileContainsText(strLockedAudit, "recovery phrase: encrypted, unlock wallet to audit") &&
2239 FileContainsText(strLockedAudit, "recovery coverage: unavailable while wallet is locked") &&
2240 FileContainsText(strLockedAudit, "Unlock the wallet with /walletpassphrase"),
2241 "a locked encrypted wallet reports that phrase coverage needs unlock") ? 0 : 1;
2242
1050 vector<string> vLiteralArgs; 2243 vector<string> vLiteralArgs;
1051 vLiteralArgs.push_back("-datadir=" + tmp); 2244 vLiteralArgs.push_back("-datadir=" + tmp);
1052 vLiteralArgs.push_back("-walletpassphrase=btf-test-passphrase"); 2245 vLiteralArgs.push_back("-walletpassphrase=btf-test-passphrase");
@@ -1067,6 +2260,17 @@ static int RunWalletEncryptSelfTest()
1067 nFail += Check(FileContainsText(strRightDump, HexStrLocal(vchDefaultPrivBytes)), 2260 nFail += Check(FileContainsText(strRightDump, HexStrLocal(vchDefaultPrivBytes)),
1068 "the restarted wallet can decrypt and dump the original key") ? 0 : 1; 2261 "the restarted wallet can decrypt and dump the original key") ? 0 : 1;
1069 2262
2263 vector<string> vUnlockedAuditArgs;
2264 vUnlockedAuditArgs.push_back("-datadir=" + tmp);
2265 vUnlockedAuditArgs.push_back("-walletpassphrase=@" + strRightPassFile);
2266 vUnlockedAuditArgs.push_back("-recoveryaudit");
2267 vUnlockedAuditArgs.push_back("-nogui");
2268 int nUnlockedAuditRet = RunBitflashChild(strExe, vUnlockedAuditArgs, NULL, &strUnlockedAudit);
2269 nFail += Check(nUnlockedAuditRet == 0 &&
2270 FileContainsText(strUnlockedAudit, "recovery phrase: present") &&
2271 !FileContainsText(strUnlockedAudit, "recovery phrase: not installed"),
2272 "an unlocked encrypted wallet audits the installed recovery phrase") ? 0 : 1;
2273
1070 vector<string> vStdinArgs; 2274 vector<string> vStdinArgs;
1071 vStdinArgs.push_back("-datadir=" + tmp); 2275 vStdinArgs.push_back("-datadir=" + tmp);
1072 vStdinArgs.push_back("-walletpassphrase"); 2276 vStdinArgs.push_back("-walletpassphrase");
@@ -1097,6 +2301,737 @@ static int RunWalletEncryptSelfTest()
1097 return nFail == 0 ? 0 : 1; 2301 return nFail == 0 ? 0 : 1;
1098} 2302}
1099 2303
2304// Encrypting a wallet while it runs on the SQLite backend. Unlike the Berkeley
2305// DB encrypt test above, this drives the real command-line path end to end in
2306// child processes: build a Berkeley DB wallet, export it to SQLite, then run
2307// -walletbackend=sqlite -encryptwallet and prove the SQLite wallet came out
2308// encrypted, unlockable, and with wallet.dat left untouched.
2309static int RunWalletSQLiteEncryptSelfTest()
2310{
2311 fflush(stdout);
2312 printf("wallet-sqlite-encrypt self-test\n");
2313
2314 std::string tmp;
2315 std::string cwd;
2316 if (!MakeTempDir(tmp))
2317 {
2318 printf(" FAIL could not create a temporary data directory\n");
2319 return 1;
2320 }
2321 if (!GetCurrentDir(cwd) || !SetCurrentDir(tmp))
2322 {
2323 printf(" FAIL could not move into the temporary data directory\n");
2324 RemoveTree(tmp);
2325 return 1;
2326 }
2327
2328 int nFail = 0;
2329 strSetDataDir = tmp;
2330 printf(" temp datadir: %s\n", tmp.c_str());
2331
2332 try
2333 {
2334 // Build a normal Berkeley DB wallet with a recovery phrase, a key pool,
2335 // and a couple of public records, then release it. Everything after this
2336 // runs the real command-line paths in child processes.
2337 if (!LoadWallet())
2338 throw std::runtime_error("LoadWallet failed");
2339
2340 string strError;
2341 const string strMnemonic =
2342 "abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about";
2343 nFail += Check(SetHDSeedFromMnemonic(strMnemonic, strError),
2344 "a phrase can be installed before encryption") ? 0 : 1;
2345 TopUpKeyPool();
2346
2347 vector<unsigned char> vchDefaultPubKey = keyUser.GetPubKey();
2348 CKey publicLabelKey;
2349 publicLabelKey.MakeNewKey();
2350 string strPublicLabelAddress = PubKeyToAddress(publicLabelKey.GetPubKey());
2351 nFail += Check(CWalletDB().WriteName(strPublicLabelAddress, "rewrite-public-name") &&
2352 CWalletDB().WriteSetting("rewrite-public-setting", (int64)424242),
2353 "public wallet records can be written before encryption") ? 0 : 1;
2354
2355 CPrivKey vchDefaultPrivKey;
2356 nFail += Check(GetWalletPrivKey(vchDefaultPubKey, vchDefaultPrivKey, strError),
2357 "the default private key is readable before encryption") ? 0 : 1;
2358 vector<unsigned char> vchDefaultPrivBytes(vchDefaultPrivKey.begin(), vchDefaultPrivKey.end());
2359 vector<unsigned char> vchPlainHDMaster = vchHDMaster;
2360 vector<unsigned char> vchPlainHDChainCode = vchHDChainCode;
2361
2362 // Release wallet.dat so a child process can read it for the export.
2363 DBFlush(true);
2364
2365#ifdef _WIN32
2366 string strExe = cwd + "\\bitflash.exe";
2367#else
2368 string strExe = cwd + "/bitflash-node";
2369#endif
2370 string strSQLitePath = tmp + "/wallet.sqlite";
2371 string strWalletDatPath = tmp + "/wallet.dat";
2372 string strRightPassFile = tmp + "/right-pass.txt";
2373 string strWrongPassFile = tmp + "/wrong-pass.txt";
2374 nFail += Check(WriteTextFile(strRightPassFile, "btf-test-passphrase\n") &&
2375 WriteTextFile(strWrongPassFile, "wrong-passphrase\n"),
2376 "passphrase files can be written for restarted commands") ? 0 : 1;
2377
2378 // Step 1: export the Berkeley DB wallet to SQLite.
2379 vector<string> vExportArgs;
2380 vExportArgs.push_back("-datadir=" + tmp);
2381 vExportArgs.push_back("-walletsqliteexport=" + strSQLitePath);
2382 vExportArgs.push_back("-nogui");
2383 int nExportRet = RunBitflashChild(strExe, vExportArgs);
2384 nFail += Check(nExportRet == 0 && FileExists(strSQLitePath.c_str()),
2385 "the wallet exports to a SQLite file") ? 0 : 1;
2386 nFail += Check(FileContainsBytes(strSQLitePath, vchDefaultPrivBytes),
2387 "the exported SQLite wallet holds the plaintext key before encryption") ? 0 : 1;
2388
2389 // Step 2: encrypt through the real SQLite backend command path.
2390 string strEncryptOut = tmp + "/sqlite-encrypt-out.txt";
2391 vector<string> vEncryptArgs;
2392 vEncryptArgs.push_back("-datadir=" + tmp);
2393 vEncryptArgs.push_back("-walletbackend=sqlite");
2394 vEncryptArgs.push_back("-encryptwallet=@" + strRightPassFile);
2395 vEncryptArgs.push_back("-nogui");
2396 int nEncryptRet = RunBitflashChild(strExe, vEncryptArgs, NULL, &strEncryptOut);
2397 nFail += Check(nEncryptRet == 0,
2398 "the SQLite wallet can be encrypted through -walletbackend=sqlite") ? 0 : 1;
2399 nFail += Check(FileContainsText(strEncryptOut, "Wallet backend: SQLite"),
2400 "encryption took the SQLite persistence path") ? 0 : 1;
2401 nFail += Check(FileContainsText(strEncryptOut, "wallet.sqlite.before-encrypt"),
2402 "the plaintext SQLite wallet is preserved as an explicit backup") ? 0 : 1;
2403
2404 // The encrypted SQLite wallet must no longer hold the plaintext secrets.
2405 nFail += Check(!FileContainsBytes(strSQLitePath, vchDefaultPrivBytes),
2406 "the encrypted SQLite wallet no longer contains the default private key bytes") ? 0 : 1;
2407 nFail += Check(!FileContainsBytes(strSQLitePath, vchPlainHDMaster),
2408 "the encrypted SQLite wallet no longer contains the HD master key bytes") ? 0 : 1;
2409 nFail += Check(!FileContainsBytes(strSQLitePath, vchPlainHDChainCode),
2410 "the encrypted SQLite wallet no longer contains the HD chain-code bytes") ? 0 : 1;
2411
2412 // The Berkeley DB wallet.dat is a separate file: the SQLite encrypt path
2413 // must never touch it, so its plaintext key is still exactly there.
2414 nFail += Check(FileContainsBytes(strWalletDatPath, vchDefaultPrivBytes),
2415 "the original Berkeley DB wallet.dat is left untouched by the SQLite encrypt") ? 0 : 1;
2416
2417 // Step 3: a locked encrypted SQLite wallet reports that phrase coverage
2418 // needs an unlock, exactly as the Berkeley DB path does (exit code 2).
2419 string strLockedAudit = tmp + "/locked-recovery-audit.txt";
2420 vector<string> vLockedArgs;
2421 vLockedArgs.push_back("-datadir=" + tmp);
2422 vLockedArgs.push_back("-walletbackend=sqlite");
2423 vLockedArgs.push_back("-recoveryaudit");
2424 vLockedArgs.push_back("-nogui");
2425 int nLockedRet = RunBitflashChild(strExe, vLockedArgs, NULL, &strLockedAudit);
2426 nFail += Check(nLockedRet == 2 &&
2427 FileContainsText(strLockedAudit, "recovery phrase: encrypted, unlock wallet to audit"),
2428 "a locked encrypted SQLite wallet reports that phrase coverage needs unlock") ? 0 : 1;
2429
2430 // The wrong passphrase is rejected and reveals nothing.
2431 string strWrongDump = tmp + "/wrong-dump.txt";
2432 vector<string> vWrongArgs;
2433 vWrongArgs.push_back("-datadir=" + tmp);
2434 vWrongArgs.push_back("-walletbackend=sqlite");
2435 vWrongArgs.push_back("-walletpassphrase=@" + strWrongPassFile);
2436 vWrongArgs.push_back("-dumpwallet=" + strWrongDump);
2437 vWrongArgs.push_back("-nogui");
2438 int nWrongRet = RunBitflashChild(strExe, vWrongArgs);
2439 nFail += Check(nWrongRet != 0 && !FileExists(strWrongDump.c_str()),
2440 "a restarted SQLite wallet rejects the wrong passphrase") ? 0 : 1;
2441
2442 // The right passphrase unlocks and can dump the original key.
2443 string strRightDump = tmp + "/right-dump.txt";
2444 vector<string> vRightArgs;
2445 vRightArgs.push_back("-datadir=" + tmp);
2446 vRightArgs.push_back("-walletbackend=sqlite");
2447 vRightArgs.push_back("-walletpassphrase=@" + strRightPassFile);
2448 vRightArgs.push_back("-dumpwallet=" + strRightDump);
2449 vRightArgs.push_back("-nogui");
2450 int nRightRet = RunBitflashChild(strExe, vRightArgs);
2451 nFail += Check(nRightRet == 0 && FileExists(strRightDump.c_str()),
2452 "a restarted SQLite wallet unlocks with the right passphrase") ? 0 : 1;
2453 nFail += Check(FileContainsText(strRightDump, HexStrLocal(vchDefaultPrivBytes)),
2454 "the unlocked SQLite wallet decrypts and dumps the original key") ? 0 : 1;
2455
2456 // The recovery phrase survives the encryption.
2457 string strAudit = tmp + "/recovery-audit.txt";
2458 vector<string> vAuditArgs;
2459 vAuditArgs.push_back("-datadir=" + tmp);
2460 vAuditArgs.push_back("-walletbackend=sqlite");
2461 vAuditArgs.push_back("-walletpassphrase=@" + strRightPassFile);
2462 vAuditArgs.push_back("-recoveryaudit");
2463 vAuditArgs.push_back("-nogui");
2464 int nAuditRet = RunBitflashChild(strExe, vAuditArgs, NULL, &strAudit);
2465 nFail += Check(nAuditRet == 0 &&
2466 FileContainsText(strAudit, "recovery phrase: present"),
2467 "the encrypted SQLite wallet still has its recovery phrase") ? 0 : 1;
2468
2469 // One-shot storage commands honour -walletbackend. The SQLite wallet is
2470 // encrypted; the wallet.dat beside it is still plaintext. Same datadir,
2471 // opposite answers -> the flag routes the diagnostic to the right file.
2472 string strSqliteAuditJson = tmp + "/sqlite-audit.json";
2473 vector<string> vSqliteAuditArgs;
2474 vSqliteAuditArgs.push_back("-datadir=" + tmp);
2475 vSqliteAuditArgs.push_back("-walletbackend=sqlite");
2476 vSqliteAuditArgs.push_back("-walletstorageauditjson=" + strSqliteAuditJson);
2477 vSqliteAuditArgs.push_back("-nogui");
2478 int nSqliteAuditRet = RunBitflashChild(strExe, vSqliteAuditArgs);
2479 nFail += Check(nSqliteAuditRet == 0 &&
2480 FileContainsText(strSqliteAuditJson, "\"encrypted_hd_seed\": \"complete\"") &&
2481 FileContainsText(strSqliteAuditJson, "\"plain_hd_seed\": \"none\""),
2482 "-walletstorageaudit under SQLite reads the encrypted wallet.sqlite") ? 0 : 1;
2483
2484 string strBdbAuditJson = tmp + "/bdb-audit.json";
2485 vector<string> vBdbAuditArgs;
2486 vBdbAuditArgs.push_back("-datadir=" + tmp);
2487 vBdbAuditArgs.push_back("-walletstorageauditjson=" + strBdbAuditJson);
2488 vBdbAuditArgs.push_back("-nogui");
2489 int nBdbAuditRet = RunBitflashChild(strExe, vBdbAuditArgs);
2490 nFail += Check(nBdbAuditRet == 0 &&
2491 FileContainsText(strBdbAuditJson, "\"plain_hd_seed\": \"complete\"") &&
2492 FileContainsText(strBdbAuditJson, "\"encrypted_hd_seed\": \"none\""),
2493 "-walletstorageaudit without the flag still reads the plaintext wallet.dat") ? 0 : 1;
2494
2495 string strSqliteCheckOut = tmp + "/sqlite-check.txt";
2496 vector<string> vSqliteCheckArgs;
2497 vSqliteCheckArgs.push_back("-datadir=" + tmp);
2498 vSqliteCheckArgs.push_back("-walletbackend=sqlite");
2499 vSqliteCheckArgs.push_back("-walletstoragecheck");
2500 vSqliteCheckArgs.push_back("-nogui");
2501 int nSqliteCheckRet = RunBitflashChild(strExe, vSqliteCheckArgs, NULL, &strSqliteCheckOut);
2502 nFail += Check(nSqliteCheckRet == 0,
2503 "-walletstoragecheck under SQLite passes on the encrypted wallet.sqlite") ? 0 : 1;
2504
2505 // Backup under SQLite copies wallet.sqlite, not the plaintext wallet.dat.
2506 // The copy is a standalone SQLite wallet (loadcheck accepts it) and holds
2507 // none of the plaintext key that wallet.dat still contains.
2508 string strBackupDest = tmp + "/backup.sqlite";
2509 vector<string> vBackupArgs;
2510 vBackupArgs.push_back("-datadir=" + tmp);
2511 vBackupArgs.push_back("-walletbackend=sqlite");
2512 vBackupArgs.push_back("-backupwallet=" + strBackupDest);
2513 vBackupArgs.push_back("-nogui");
2514 int nBackupRet = RunBitflashChild(strExe, vBackupArgs);
2515 nFail += Check(nBackupRet == 0 && FileExists(strBackupDest.c_str()),
2516 "-backupwallet under SQLite writes a backup file") ? 0 : 1;
2517 nFail += Check(!FileContainsBytes(strBackupDest, vchDefaultPrivBytes),
2518 "the SQLite backup holds no plaintext key (it copied wallet.sqlite, not wallet.dat)") ? 0 : 1;
2519
2520 string strBackupLoadCheck = tmp + "/backup-loadcheck.txt";
2521 vector<string> vLoadCheckArgs;
2522 vLoadCheckArgs.push_back("-datadir=" + tmp);
2523 vLoadCheckArgs.push_back("-walletsqliteloadcheck=" + strBackupDest);
2524 vLoadCheckArgs.push_back("-nogui");
2525 int nBackupLoadRet = RunBitflashChild(strExe, vLoadCheckArgs, NULL, &strBackupLoadCheck);
2526 nFail += Check(nBackupLoadRet == 0,
2527 "the SQLite backup is a valid standalone SQLite wallet") ? 0 : 1;
2528 }
2529 catch (const std::exception& e)
2530 {
2531 printf(" FAIL exception: %s\n", e.what());
2532 nFail++;
2533 }
2534 catch (...)
2535 {
2536 printf(" FAIL unknown exception\n");
2537 nFail++;
2538 }
2539
2540 DBFlush(true);
2541 SetCurrentDir(cwd);
2542 RemoveTree(tmp);
2543 printf("%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
2544 nFail, nFail == 1 ? "" : "s");
2545 fflush(stdout);
2546 return nFail == 0 ? 0 : 1;
2547}
2548
2549// Backend resolution when no -walletbackend flag is given: a fresh install
2550// starts on SQLite (the new default), an existing wallet.dat stays on Berkeley
2551// DB, the persistent marker overrides the default, and an explicit flag
2552// overrides everything. Each scenario runs a real child process.
2553static int RunWalletBackendDefaultSelfTest()
2554{
2555 fflush(stdout);
2556 printf("wallet-backend-default self-test\n");
2557
2558 std::string tmp, cwd;
2559 if (!MakeTempDir(tmp))
2560 {
2561 printf(" FAIL could not create a temporary data directory\n");
2562 return 1;
2563 }
2564 if (!GetCurrentDir(cwd) || !SetCurrentDir(tmp))
2565 {
2566 printf(" FAIL could not move into the temporary data directory\n");
2567 RemoveTree(tmp);
2568 return 1;
2569 }
2570
2571 int nFail = 0;
2572 strSetDataDir = tmp;
2573 printf(" temp datadir: %s\n", tmp.c_str());
2574
2575#ifdef _WIN32
2576 string strExe = cwd + "\\bitflash.exe";
2577#else
2578 string strExe = cwd + "/bitflash-node";
2579#endif
2580
2581 try
2582 {
2583 // 1. Fresh datadir, no flag -> this fork refuses, where upstream creates.
2584 // An empty datadir on a headless node is far more often a mount or
2585 // backup fault than a first run, and creating a wallet also pins the
2586 // backend marker, after which restoring the real wallet is a no-op.
2587 // Nothing may be left behind: no wallet of either kind, no marker.
2588 string d1 = tmp + "/fresh";
2589 nFail += Check(MakeDirLocal(d1), "fresh scenario dir can be created") ? 0 : 1;
2590 vector<string> a1;
2591 a1.push_back("-datadir=" + d1);
2592 a1.push_back("-nomanagedtor");
2593 a1.push_back("-nogui");
2594 a1.push_back("-newaddress");
2595 int r0 = RunBitflashChild(strExe, a1);
2596 nFail += Check(r0 != 0 &&
2597 !FileExists((d1 + "/wallet.sqlite").c_str()) &&
2598 !FileExists((d1 + "/wallet.dat").c_str()) &&
2599 !FileExists((d1 + "/wallet-backend").c_str()),
2600 "a flagless headless start refuses to create a wallet") ? 0 : 1;
2601
2602 // 1a. -createwallet says it was meant: new wallet.sqlite, marker
2603 // "sqlite", no wallet.dat.
2604 vector<string> a1c = a1;
2605 a1c.push_back("-createwallet");
2606 int r1 = RunBitflashChild(strExe, a1c);
2607 nFail += Check(r1 == 0 &&
2608 FileExists((d1 + "/wallet.sqlite").c_str()) &&
2609 !FileExists((d1 + "/wallet.dat").c_str()) &&
2610 FileContainsText(d1 + "/wallet-backend", "sqlite"),
2611 "-createwallet starts a fresh install on SQLite and records the marker") ? 0 : 1;
2612
2613 // 1b. Second run reuses it without the flag -- the refusal only guards
2614 // creation, so an existing wallet still starts flagless.
2615 int r1b = RunBitflashChild(strExe, a1);
2616 nFail += Check(r1b == 0 &&
2617 FileExists((d1 + "/wallet.sqlite").c_str()) &&
2618 !FileExists((d1 + "/wallet.dat").c_str()),
2619 "the recorded SQLite choice survives a flagless restart") ? 0 : 1;
2620
2621 // 2. Existing wallet.dat, no flag -> stays on Berkeley DB.
2622 string d2 = tmp + "/existing-bdb";
2623 nFail += Check(MakeDirLocal(d2), "existing-bdb scenario dir can be created") ? 0 : 1;
2624 vector<string> a2c;
2625 a2c.push_back("-datadir=" + d2);
2626 a2c.push_back("-nomanagedtor");
2627 a2c.push_back("-nogui");
2628 a2c.push_back("-walletbackend=bdb");
2629 a2c.push_back("-newphrase");
2630 RunBitflashChild(strExe, a2c);
2631 vector<string> a2;
2632 a2.push_back("-datadir=" + d2);
2633 a2.push_back("-nomanagedtor");
2634 a2.push_back("-nogui");
2635 a2.push_back("-newaddress");
2636 int r2 = RunBitflashChild(strExe, a2);
2637 nFail += Check(r2 == 0 &&
2638 FileExists((d2 + "/wallet.dat").c_str()) &&
2639 !FileExists((d2 + "/wallet.sqlite").c_str()),
2640 "an existing wallet.dat keeps a flagless start on Berkeley DB") ? 0 : 1;
2641
2642 // 3. Marker "bdb" forces Berkeley DB even on a fresh datadir.
2643 string d3 = tmp + "/marker-bdb";
2644 nFail += Check(MakeDirLocal(d3), "marker-bdb scenario dir can be created") ? 0 : 1;
2645 nFail += Check(WriteTextFile(d3 + "/wallet-backend", "bdb\n"),
2646 "a bdb marker can be written") ? 0 : 1;
2647 vector<string> a3;
2648 a3.push_back("-datadir=" + d3);
2649 a3.push_back("-nomanagedtor");
2650 a3.push_back("-nogui");
2651 a3.push_back("-newaddress");
2652 int r3 = RunBitflashChild(strExe, a3);
2653 nFail += Check(r3 == 0 &&
2654 FileExists((d3 + "/wallet.dat").c_str()) &&
2655 !FileExists((d3 + "/wallet.sqlite").c_str()),
2656 "a bdb marker forces Berkeley DB on a fresh datadir") ? 0 : 1;
2657
2658 // 4. Explicit -walletbackend=bdb on a fresh datadir -> wallet.dat.
2659 string d4 = tmp + "/explicit-bdb";
2660 nFail += Check(MakeDirLocal(d4), "explicit-bdb scenario dir can be created") ? 0 : 1;
2661 vector<string> a4;
2662 a4.push_back("-datadir=" + d4);
2663 a4.push_back("-nomanagedtor");
2664 a4.push_back("-nogui");
2665 a4.push_back("-walletbackend=bdb");
2666 a4.push_back("-newaddress");
2667 int r4 = RunBitflashChild(strExe, a4);
2668 nFail += Check(r4 == 0 &&
2669 FileExists((d4 + "/wallet.dat").c_str()) &&
2670 !FileExists((d4 + "/wallet.sqlite").c_str()),
2671 "an explicit -walletbackend=bdb creates wallet.dat on a fresh datadir") ? 0 : 1;
2672 }
2673 catch (const std::exception& e)
2674 {
2675 printf(" FAIL exception: %s\n", e.what());
2676 nFail++;
2677 }
2678 catch (...)
2679 {
2680 printf(" FAIL unknown exception\n");
2681 nFail++;
2682 }
2683
2684 DBFlush(true);
2685 SetCurrentDir(cwd);
2686 RemoveTree(tmp);
2687 printf("%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
2688 nFail, nFail == 1 ? "" : "s");
2689 fflush(stdout);
2690 return nFail == 0 ? 0 : 1;
2691}
2692
2693// The wizard's conversion action, headless: build a Berkeley DB wallet, run
2694// DoConvertWalletToSQLite() (what the GUI button calls), and prove it produced a
2695// usable wallet.sqlite + backend marker while leaving wallet.dat untouched, then
2696// that a flagless restart opens the converted SQLite wallet.
2697static int RunWalletConvertSelfTest()
2698{
2699 fflush(stdout);
2700 printf("wallet-convert self-test\n");
2701
2702 std::string tmp, cwd;
2703 if (!MakeTempDir(tmp))
2704 {
2705 printf(" FAIL could not create a temporary data directory\n");
2706 return 1;
2707 }
2708 if (!GetCurrentDir(cwd) || !SetCurrentDir(tmp))
2709 {
2710 printf(" FAIL could not move into the temporary data directory\n");
2711 RemoveTree(tmp);
2712 return 1;
2713 }
2714
2715 int nFail = 0;
2716 strSetDataDir = tmp;
2717 printf(" temp datadir: %s\n", tmp.c_str());
2718
2719#ifdef _WIN32
2720 string strExe = cwd + "\\bitflash.exe";
2721#else
2722 string strExe = cwd + "/bitflash-node";
2723#endif
2724
2725 try
2726 {
2727 if (!LoadWallet())
2728 throw std::runtime_error("LoadWallet failed");
2729
2730 string strError;
2731 nFail += Check(SetHDSeedFromMnemonic(
2732 "abandon abandon abandon abandon abandon abandon abandon "
2733 "abandon abandon abandon abandon about", strError),
2734 "a phrase can be installed before conversion") ? 0 : 1;
2735 TopUpKeyPool();
2736
2737 vector<unsigned char> vchDefaultPubKey = keyUser.GetPubKey();
2738 CPrivKey vchDefaultPrivKey;
2739 nFail += Check(GetWalletPrivKey(vchDefaultPubKey, vchDefaultPrivKey, strError),
2740 "the default private key is readable before conversion") ? 0 : 1;
2741 vector<unsigned char> vchDefaultPrivBytes(vchDefaultPrivKey.begin(),
2742 vchDefaultPrivKey.end());
2743
2744 // The conversion action the GUI button invokes.
2745 int nCopied = 0;
2746 string strConvertError;
2747 bool fConverted = DoConvertWalletToSQLite(nCopied, strConvertError);
2748 nFail += Check(fConverted && nCopied > 0,
2749 "DoConvertWalletToSQLite exports the wallet") ? 0 : 1;
2750 nFail += Check(FileExists((tmp + "/wallet.sqlite").c_str()) &&
2751 FileContainsText(tmp + "/wallet-backend", "sqlite"),
2752 "conversion writes wallet.sqlite and records the sqlite marker") ? 0 : 1;
2753 nFail += Check(FileContainsBytes(tmp + "/wallet.dat", vchDefaultPrivBytes),
2754 "wallet.dat is left untouched by the conversion") ? 0 : 1;
2755
2756 // Idempotence guard: a second conversion refuses rather than overwrite.
2757 int nCopied2 = 0;
2758 string strSecondError;
2759 nFail += Check(!DoConvertWalletToSQLite(nCopied2, strSecondError),
2760 "a second conversion refuses to overwrite wallet.sqlite") ? 0 : 1;
2761
2762 DBFlush(true);
2763
2764 // A flagless restart now opens the converted SQLite wallet (the marker
2765 // routes it) and finds the same recovery phrase.
2766 string strAudit = tmp + "/converted-recovery-audit.txt";
2767 vector<string> vAuditArgs;
2768 vAuditArgs.push_back("-datadir=" + tmp);
2769 vAuditArgs.push_back("-nomanagedtor");
2770 vAuditArgs.push_back("-nogui");
2771 vAuditArgs.push_back("-recoveryaudit");
2772 int nAuditRet = RunBitflashChild(strExe, vAuditArgs, NULL, &strAudit);
2773 nFail += Check(nAuditRet == 0 &&
2774 FileContainsText(strAudit, "recovery phrase: present"),
2775 "a flagless restart opens the converted SQLite wallet") ? 0 : 1;
2776 }
2777 catch (const std::exception& e)
2778 {
2779 printf(" FAIL exception: %s\n", e.what());
2780 nFail++;
2781 }
2782 catch (...)
2783 {
2784 printf(" FAIL unknown exception\n");
2785 nFail++;
2786 }
2787
2788 DBFlush(true);
2789 SetCurrentDir(cwd);
2790 RemoveTree(tmp);
2791 printf("%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
2792 nFail, nFail == 1 ? "" : "s");
2793 fflush(stdout);
2794 return nFail == 0 ? 0 : 1;
2795}
2796
2797static int RunWalletPortabilitySelfTest()
2798{
2799 fflush(stdout);
2800 printf("wallet-portability self-test\n");
2801
2802 std::string tmp;
2803 std::string cwd;
2804 if (!MakeTempDir(tmp))
2805 {
2806 printf(" FAIL could not create a temporary data directory\n");
2807 return 1;
2808 }
2809 if (!GetCurrentDir(cwd))
2810 {
2811 printf(" FAIL could not read current directory\n");
2812 RemoveTree(tmp);
2813 return 1;
2814 }
2815
2816 int nFail = 0;
2817 printf(" temp root: %s\n", tmp.c_str());
2818
2819 try
2820 {
2821#ifdef _WIN32
2822 string strExe = cwd + "\\bitflash.exe";
2823#else
2824 string strExe = cwd + "/bitflash-node";
2825#endif
2826 string strOriginal = tmp + "/original";
2827 string strRawCopy = tmp + "/raw-copy";
2828 string strBackupCopy = tmp + "/backup-copy";
2829 nFail += Check(MakeDirLocal(strOriginal) &&
2830 MakeDirLocal(strRawCopy) &&
2831 MakeDirLocal(strBackupCopy),
2832 "temporary wallet directories can be created") ? 0 : 1;
2833
2834 string strCreateOut = tmp + "/create.txt";
2835 vector<string> vCreateArgs;
2836 vCreateArgs.push_back("-datadir=" + strOriginal);
2837 vCreateArgs.push_back("-nomanagedtor");
2838 vCreateArgs.push_back("-nogui");
2839 vCreateArgs.push_back("-walletbackend=bdb"); // this suite is about wallet.dat portability
2840 vCreateArgs.push_back("-newphrase");
2841 int nCreateRet = RunBitflashChild(strExe, vCreateArgs, NULL, &strCreateOut);
2842 nFail += Check(nCreateRet == 0 &&
2843 FileContainsText(strCreateOut, "Write these twelve words down"),
2844 "a phrase wallet can be created in a fresh datadir") ? 0 : 1;
2845
2846 string strOriginalAudit = tmp + "/original-audit.txt";
2847 vector<string> vOriginalAuditArgs;
2848 vOriginalAuditArgs.push_back("-datadir=" + strOriginal);
2849 vOriginalAuditArgs.push_back("-nomanagedtor");
2850 vOriginalAuditArgs.push_back("-nogui");
2851 vOriginalAuditArgs.push_back("-recoveryaudit");
2852 int nOriginalAuditRet = RunBitflashChild(strExe, vOriginalAuditArgs, NULL, &strOriginalAudit);
2853 nFail += Check(nOriginalAuditRet == 0 &&
2854 FileContainsText(strOriginalAudit, "recovery phrase: present") &&
2855 FileContainsText(strOriginalAudit, "SLIP-0044 BITFLASH"),
2856 "the original wallet reports its recovery phrase") ? 0 : 1;
2857
2858 string strOriginalStorageAudit = tmp + "/original-storage-audit.txt";
2859 vector<string> vOriginalStorageAuditArgs;
2860 vOriginalStorageAuditArgs.push_back("-datadir=" + strOriginal);
2861 vOriginalStorageAuditArgs.push_back("-nomanagedtor");
2862 vOriginalStorageAuditArgs.push_back("-nogui");
2863 vOriginalStorageAuditArgs.push_back("-walletstorageaudit");
2864 int nOriginalStorageAuditRet =
2865 RunBitflashChild(strExe, vOriginalStorageAuditArgs, NULL, &strOriginalStorageAudit);
2866 nFail += Check(nOriginalStorageAuditRet == 0 &&
2867 FileContainsText(strOriginalStorageAudit, "Wallet storage audit") &&
2868 FileContainsText(strOriginalStorageAudit, "plain HD seed: complete") &&
2869 FileContainsText(strOriginalStorageAudit, "encrypted HD seed: none") &&
2870 FileContainsText(strOriginalStorageAudit, "malformed records: 0"),
2871 "the storage audit recognizes a plaintext phrase wallet") ? 0 : 1;
2872
2873 string strOriginalStorageCheck = tmp + "/original-storage-check.txt";
2874 vector<string> vOriginalStorageCheckArgs;
2875 vOriginalStorageCheckArgs.push_back("-datadir=" + strOriginal);
2876 vOriginalStorageCheckArgs.push_back("-nomanagedtor");
2877 vOriginalStorageCheckArgs.push_back("-nogui");
2878 vOriginalStorageCheckArgs.push_back("-walletstoragecheck");
2879 int nOriginalStorageCheckRet =
2880 RunBitflashChild(strExe, vOriginalStorageCheckArgs, NULL, &strOriginalStorageCheck);
2881 nFail += Check(nOriginalStorageCheckRet == 0 &&
2882 FileContainsText(strOriginalStorageCheck, "storage sanity: ok") &&
2883 FileContainsText(strOriginalStorageCheck, "plain HD seed: complete"),
2884 "the storage sanity check accepts a plaintext phrase wallet") ? 0 : 1;
2885
2886 string strOriginalStorageAuditJson = tmp + "/original-storage-audit.json";
2887 vector<string> vOriginalStorageAuditJsonArgs;
2888 vOriginalStorageAuditJsonArgs.push_back("-datadir=" + strOriginal);
2889 vOriginalStorageAuditJsonArgs.push_back("-nomanagedtor");
2890 vOriginalStorageAuditJsonArgs.push_back("-nogui");
2891 vOriginalStorageAuditJsonArgs.push_back("-walletstorageauditjson=" +
2892 strOriginalStorageAuditJson);
2893 int nOriginalStorageAuditJsonRet =
2894 RunBitflashChild(strExe, vOriginalStorageAuditJsonArgs);
2895 nFail += Check(nOriginalStorageAuditJsonRet == 0 &&
2896 FileContainsText(strOriginalStorageAuditJson,
2897 "\"format\": \"bitflash-wallet-storage-audit-v1\"") &&
2898 FileContainsText(strOriginalStorageAuditJson,
2899 "\"plain_hd_seed\": \"complete\"") &&
2900 FileContainsText(strOriginalStorageAuditJson,
2901 "\"encrypted_hd_seed\": \"none\""),
2902 "the storage audit can write plaintext-wallet JSON") ? 0 : 1;
2903
2904 string strPortableWallet = tmp + "/portable-wallet.dat";
2905 string strBackupOut = tmp + "/backupwallet.txt";
2906 vector<string> vBackupArgs;
2907 vBackupArgs.push_back("-datadir=" + strOriginal);
2908 vBackupArgs.push_back("-nomanagedtor");
2909 vBackupArgs.push_back("-nogui");
2910 vBackupArgs.push_back("-backupwallet=" + strPortableWallet);
2911 int nBackupRet = RunBitflashChild(strExe, vBackupArgs, NULL, &strBackupOut);
2912 nFail += Check(nBackupRet == 0 && FileExists(strPortableWallet.c_str()),
2913 "/backupwallet writes a portable wallet.dat") ? 0 : 1;
2914
2915 string strBackupOut2 = tmp + "/backupwallet-overwrite.txt";
2916 int nBackupRet2 = RunBitflashChild(strExe, vBackupArgs, NULL, &strBackupOut2);
2917 nFail += Check(nBackupRet2 == 0 && FileExists(strPortableWallet.c_str()) &&
2918 !DirectoryHasFileWithPrefix(tmp, "portable-wallet.dat.tmp.") &&
2919 !DirectoryHasFileWithPrefix(tmp, "portable-wallet.dat.old."),
2920 "/backupwallet replaces an existing backup without leaving temp files") ? 0 : 1;
2921
2922 nFail += Check(CopyFileLocal(strOriginal + "/wallet.dat",
2923 strRawCopy + "/wallet.dat"),
2924 "a raw wallet.dat can be copied without database logs") ? 0 : 1;
2925 string strRawAudit = tmp + "/raw-copy-audit.txt";
2926 vector<string> vRawAuditArgs;
2927 vRawAuditArgs.push_back("-datadir=" + strRawCopy);
2928 vRawAuditArgs.push_back("-nomanagedtor");
2929 vRawAuditArgs.push_back("-nogui");
2930 vRawAuditArgs.push_back("-recoveryaudit");
2931 int nRawAuditRet = RunBitflashChild(strExe, vRawAuditArgs, NULL, &strRawAudit);
2932 nFail += Check(nRawAuditRet == 0 &&
2933 FileContainsText(strRawAudit, "recovery phrase: present") &&
2934 !FileContainsText(strRawAudit, "Cannot open wallet.dat"),
2935 "a raw wallet.dat copy opens in a new datadir") ? 0 : 1;
2936
2937 nFail += Check(CopyFileLocal(strPortableWallet,
2938 strBackupCopy + "/wallet.dat"),
2939 "the portable backup can be installed as wallet.dat") ? 0 : 1;
2940 string strBackupAudit = tmp + "/backup-copy-audit.txt";
2941 vector<string> vBackupAuditArgs;
2942 vBackupAuditArgs.push_back("-datadir=" + strBackupCopy);
2943 vBackupAuditArgs.push_back("-nomanagedtor");
2944 vBackupAuditArgs.push_back("-nogui");
2945 vBackupAuditArgs.push_back("-recoveryaudit");
2946 int nBackupAuditRet = RunBitflashChild(strExe, vBackupAuditArgs, NULL, &strBackupAudit);
2947 nFail += Check(nBackupAuditRet == 0 &&
2948 FileContainsText(strBackupAudit, "recovery phrase: present") &&
2949 FileContainsText(strBackupAudit, "SLIP-0044 BITFLASH"),
2950 "a /backupwallet copy opens in a new datadir") ? 0 : 1;
2951
2952 string strPassFile = tmp + "/encrypt-pass.txt";
2953 string strEncryptOut = tmp + "/encryptwallet.txt";
2954 nFail += Check(WriteTextFile(strPassFile, "portable-test-passphrase\n"),
2955 "an encryption passphrase file can be written") ? 0 : 1;
2956 vector<string> vEncryptArgs;
2957 vEncryptArgs.push_back("-datadir=" + strOriginal);
2958 vEncryptArgs.push_back("-nomanagedtor");
2959 vEncryptArgs.push_back("-nogui");
2960 vEncryptArgs.push_back("-encryptwallet=@" + strPassFile);
2961 int nEncryptRet = RunBitflashChild(strExe, vEncryptArgs, NULL, &strEncryptOut);
2962 nFail += Check(nEncryptRet == 0 &&
2963 !DirectoryHasFileWithPrefix(strOriginal + "/database", "log."),
2964 "the encryptwallet command purges Berkeley DB environment logs") ? 0 : 1;
2965
2966 string strEncryptedStorageAudit = tmp + "/encrypted-storage-audit.txt";
2967 vector<string> vEncryptedStorageAuditArgs;
2968 vEncryptedStorageAuditArgs.push_back("-datadir=" + strOriginal);
2969 vEncryptedStorageAuditArgs.push_back("-nomanagedtor");
2970 vEncryptedStorageAuditArgs.push_back("-nogui");
2971 vEncryptedStorageAuditArgs.push_back("-walletstorageaudit");
2972 int nEncryptedStorageAuditRet =
2973 RunBitflashChild(strExe, vEncryptedStorageAuditArgs, NULL, &strEncryptedStorageAudit);
2974 nFail += Check(nEncryptedStorageAuditRet == 0 &&
2975 FileContainsText(strEncryptedStorageAudit, "plain private keys: 0") &&
2976 FileContainsText(strEncryptedStorageAudit, "encrypted private keys:") &&
2977 FileContainsText(strEncryptedStorageAudit, "plain HD seed: none") &&
2978 FileContainsText(strEncryptedStorageAudit, "encrypted HD seed: complete") &&
2979 FileContainsText(strEncryptedStorageAudit, "wallet minimum version: present") &&
2980 FileContainsText(strEncryptedStorageAudit, "malformed records: 0"),
2981 "the storage audit recognizes an encrypted wallet") ? 0 : 1;
2982
2983 string strEncryptedStorageCheck = tmp + "/encrypted-storage-check.txt";
2984 vector<string> vEncryptedStorageCheckArgs;
2985 vEncryptedStorageCheckArgs.push_back("-datadir=" + strOriginal);
2986 vEncryptedStorageCheckArgs.push_back("-nomanagedtor");
2987 vEncryptedStorageCheckArgs.push_back("-nogui");
2988 vEncryptedStorageCheckArgs.push_back("-walletstoragecheck");
2989 int nEncryptedStorageCheckRet =
2990 RunBitflashChild(strExe, vEncryptedStorageCheckArgs, NULL, &strEncryptedStorageCheck);
2991 nFail += Check(nEncryptedStorageCheckRet == 0 &&
2992 FileContainsText(strEncryptedStorageCheck, "storage sanity: ok") &&
2993 FileContainsText(strEncryptedStorageCheck, "plain private keys: 0") &&
2994 FileContainsText(strEncryptedStorageCheck, "encrypted HD seed: complete"),
2995 "the storage sanity check accepts an encrypted wallet") ? 0 : 1;
2996
2997 string strEncryptedStorageAuditJson = tmp + "/encrypted-storage-audit.json";
2998 vector<string> vEncryptedStorageAuditJsonArgs;
2999 vEncryptedStorageAuditJsonArgs.push_back("-datadir=" + strOriginal);
3000 vEncryptedStorageAuditJsonArgs.push_back("-nomanagedtor");
3001 vEncryptedStorageAuditJsonArgs.push_back("-nogui");
3002 vEncryptedStorageAuditJsonArgs.push_back("-walletstorageauditjson=" +
3003 strEncryptedStorageAuditJson);
3004 int nEncryptedStorageAuditJsonRet =
3005 RunBitflashChild(strExe, vEncryptedStorageAuditJsonArgs);
3006 nFail += Check(nEncryptedStorageAuditJsonRet == 0 &&
3007 FileContainsText(strEncryptedStorageAuditJson,
3008 "\"plain_private_keys\": 0") &&
3009 FileContainsText(strEncryptedStorageAuditJson,
3010 "\"plain_hd_seed\": \"none\"") &&
3011 FileContainsText(strEncryptedStorageAuditJson,
3012 "\"encrypted_hd_seed\": \"complete\"") &&
3013 FileContainsText(strEncryptedStorageAuditJson,
3014 "\"wallet_minimum_version\": \"present\""),
3015 "the storage audit can write encrypted-wallet JSON") ? 0 : 1;
3016 }
3017 catch (const std::exception& e)
3018 {
3019 printf(" FAIL exception: %s\n", e.what());
3020 nFail++;
3021 }
3022 catch (...)
3023 {
3024 printf(" FAIL unknown exception\n");
3025 nFail++;
3026 }
3027
3028 RemoveTree(tmp);
3029 printf("%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
3030 nFail, nFail == 1 ? "" : "s");
3031 fflush(stdout);
3032 return nFail == 0 ? 0 : 1;
3033}
3034
1100static int RunNetMessageSelfTest() 3035static int RunNetMessageSelfTest()
1101{ 3036{
1102 fflush(stdout); 3037 fflush(stdout);
@@ -1105,6 +3040,13 @@ static int RunNetMessageSelfTest()
1105 int nFail = 0; 3040 int nFail = 0;
1106 try 3041 try
1107 { 3042 {
3043 // A peer that has not yet been kept waiting must not look like one that
3044 // has been waiting since the epoch. This is the whole bug: the stall
3045 // clock lives in CNode, so only the constructor can start it at zero.
3046 CNode opened(INVALID_SOCKET, CAddress("127.0.0.1"));
3047 nFail += Check(opened.nMsgWaitSince == 0 && opened.nMsgWaitSize == 0,
3048 "a new peer starts with its stall clock stopped") ? 0 : 1;
3049
1108 CNode complete(INVALID_SOCKET, CAddress("127.0.0.1")); 3050 CNode complete(INVALID_SOCKET, CAddress("127.0.0.1"));
1109 complete.nVersion = VERSION; 3051 complete.nVersion = VERSION;
1110 complete.vRecv << CMessageHeader("ping", 0); 3052 complete.vRecv << CMessageHeader("ping", 0);
@@ -1119,20 +3061,38 @@ static int RunNetMessageSelfTest()
1119 nFail += Check(ProcessMessages(&partial), "an incomplete message returns cleanly") ? 0 : 1; 3061 nFail += Check(ProcessMessages(&partial), "an incomplete message returns cleanly") ? 0 : 1;
1120 nFail += Check(partial.vRecv.size() == nPartialBefore, 3062 nFail += Check(partial.vRecv.size() == nPartialBefore,
1121 "an incomplete message keeps one header in the buffer") ? 0 : 1; 3063 "an incomplete message keeps one header in the buffer") ? 0 : 1;
1122 nFail += Check(partial.nIncompleteMessageStart != 0, 3064 nFail += Check(partial.nMsgWaitSince != 0,
1123 "an incomplete message starts a timeout clock") ? 0 : 1; 3065 "an incomplete message starts the stall clock") ? 0 : 1;
3066 nFail += Check(partial.nMsgWaitSize == 0,
3067 "the stall clock records how much payload had arrived") ? 0 : 1;
1124 nFail += Check(!partial.fDisconnect, 3068 nFail += Check(!partial.fDisconnect,
1125 "a fresh incomplete message does not disconnect immediately") ? 0 : 1; 3069 "a fresh incomplete message does not disconnect immediately") ? 0 : 1;
1126 3070
3071 // Still arriving, just slowly: more payload is here than when the clock
3072 // was started, so the clock restarts however old it was. Timing the
3073 // wait instead of the stall would drop this peer.
3074 CNode slow(INVALID_SOCKET, CAddress("127.0.0.1"));
3075 slow.nVersion = VERSION;
3076 slow.vRecv << CMessageHeader("block", 100);
3077 slow.vRecv.insert(slow.vRecv.end(), 50, '\0');
3078 slow.nMsgWaitSince = GetTime() - BTF_MSG_WAIT_MAX_SECS - 1;
3079 slow.nMsgWaitSize = 10;
3080 ProcessMessages(&slow);
3081 nFail += Check(!slow.fDisconnect,
3082 "a peer still sending payload is not dropped") ? 0 : 1;
3083 nFail += Check(slow.nMsgWaitSize == 50,
3084 "progress restarts the stall clock") ? 0 : 1;
3085
3086 // Nothing more has arrived since the clock started, and the clock is
3087 // older than the limit.
1127 CNode stale(INVALID_SOCKET, CAddress("127.0.0.1")); 3088 CNode stale(INVALID_SOCKET, CAddress("127.0.0.1"));
1128 stale.nVersion = VERSION; 3089 stale.nVersion = VERSION;
1129 stale.vRecv << CMessageHeader("block", 100); 3090 stale.vRecv << CMessageHeader("block", 100);
1130 stale.nIncompleteMessageStart = GetTime() - BTF_INCOMPLETE_MESSAGE_TIMEOUT_SECS - 1; 3091 stale.nMsgWaitSince = GetTime() - BTF_MSG_WAIT_MAX_SECS - 1;
1131 stale.nIncompleteMessageSize = 100; 3092 stale.nMsgWaitSize = 0;
1132 stale.strIncompleteMessageCommand = "block";
1133 ProcessMessages(&stale); 3093 ProcessMessages(&stale);
1134 nFail += Check(stale.fDisconnect, 3094 nFail += Check(stale.fDisconnect,
1135 "a stale incomplete message disconnects the peer") ? 0 : 1; 3095 "a stalled incomplete message disconnects the peer") ? 0 : 1;
1136 3096
1137 CNode oversized(INVALID_SOCKET, CAddress("127.0.0.1")); 3097 CNode oversized(INVALID_SOCKET, CAddress("127.0.0.1"));
1138 oversized.nVersion = VERSION; 3098 oversized.nVersion = VERSION;
@@ -1158,6 +3118,87 @@ static int RunNetMessageSelfTest()
1158 return nFail == 0 ? 0 : 1; 3118 return nFail == 0 ? 0 : 1;
1159} 3119}
1160 3120
3121static int RunNetworkParamsSelfTest()
3122{
3123 fflush(stdout);
3124 printf("network-params self-test\n");
3125
3126 int nFail = 0;
3127 std::string strOldDataDir = strSetDataDir;
3128 try
3129 {
3130 SelectChainParams(false);
3131 nFail += Check(!IsTestNet(), "starts on mainnet after explicit selection") ? 0 : 1;
3132 nFail += Check(ntohs(GetDefaultPort()) == MAINNET_PORT,
3133 "mainnet default P2P port is 8433") ? 0 : 1;
3134 CAddress mainAddr("127.0.0.1");
3135 nFail += Check(mainAddr.port == htons(MAINNET_PORT),
3136 "mainnet address parsing uses the mainnet default port") ? 0 : 1;
3137 CMessageHeader mainHeader("ping", 0);
3138 char mainMagic[4];
3139 memcpy(mainMagic, mainHeader.pchMessageStart, sizeof(mainMagic));
3140 uint256 mainGenesis = hashGenesisBlock;
3141 uint256 mainMerkle = hashGenesisMerkleRoot;
3142 unsigned int mainNonce = GENESIS_NONCE;
3143
3144 SelectChainParams(true);
3145 nFail += Check(IsTestNet(), "testnet selection is sticky") ? 0 : 1;
3146 nFail += Check(ntohs(GetDefaultPort()) == TESTNET_PORT,
3147 "testnet default P2P port is 18433") ? 0 : 1;
3148 CAddress testAddr("127.0.0.1");
3149 nFail += Check(testAddr.port == htons(TESTNET_PORT),
3150 "testnet address parsing uses the testnet default port") ? 0 : 1;
3151 CMessageHeader testHeader("ping", 0);
3152 nFail += Check(memcmp(mainMagic, testHeader.pchMessageStart, sizeof(mainMagic)) != 0,
3153 "testnet message magic differs from mainnet") ? 0 : 1;
3154 nFail += Check(hashGenesisBlock != mainGenesis &&
3155 hashGenesisMerkleRoot != mainMerkle &&
3156 GENESIS_NONCE != mainNonce,
3157 "testnet genesis constants differ from mainnet") ? 0 : 1;
3158 nFail += Check(hashGenesisBlock == uint256("0xa9d11c6d697bcb7aea0653bc088b12dcf9f44e732df205162cf085fd9d9963eb") &&
3159 hashGenesisMerkleRoot == uint256("0x510b293105e3ff85b8d4c3920e357f85d1a86bf199eb4e7d11f04f380030ec5c") &&
3160 GENESIS_NONCE == 3199,
3161 "testnet genesis constants are pinned") ? 0 : 1;
3162
3163 std::string tmp;
3164 if (!MakeTempDir(tmp))
3165 {
3166 nFail += Check(false, "creates a temporary data directory") ? 0 : 1;
3167 }
3168 else
3169 {
3170 strSetDataDir = tmp;
3171 SelectChainParams(false);
3172 std::string mainDir = GetAppDir();
3173 SelectChainParams(true);
3174 std::string testDir = GetAppDir();
3175 nFail += Check(mainDir == tmp,
3176 "mainnet uses the selected data directory") ? 0 : 1;
3177 nFail += Check(testDir == tmp + "/testnet",
3178 "testnet stores data below a testnet subdirectory") ? 0 : 1;
3179 RemoveTree(tmp);
3180 }
3181 }
3182 catch (const std::exception& e)
3183 {
3184 printf(" FAIL exception: %s\n", e.what());
3185 nFail++;
3186 }
3187 catch (...)
3188 {
3189 printf(" FAIL unknown exception\n");
3190 nFail++;
3191 }
3192
3193 strSetDataDir = strOldDataDir;
3194 SelectChainParams(false);
3195
3196 printf("%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
3197 nFail, nFail == 1 ? "" : "s");
3198 fflush(stdout);
3199 return nFail == 0 ? 0 : 1;
3200}
3201
1161static CBlock MakeSizedConsensusBlock(size_t nScriptBytes) 3202static CBlock MakeSizedConsensusBlock(size_t nScriptBytes)
1162{ 3203{
1163 CTransaction tx; 3204 CTransaction tx;
@@ -1175,6 +3216,74 @@ static CBlock MakeSizedConsensusBlock(size_t nScriptBytes)
1175 return block; 3216 return block;
1176} 3217}
1177 3218
3219// A distinct transaction of a chosen weight and verification cost. nUnique
3220// only keeps the txids apart, so a mempool built from these is not a pile of
3221// duplicates a block could never carry anyway.
3222static CTransaction MakeSizedTx(size_t nPadBytes, unsigned int nSigOps, int nUnique)
3223{
3224 CTransaction tx;
3225 tx.vin.push_back(CTxIn(COutPoint(), CScript() << (int64)nUnique << 1));
3226 tx.vout.push_back(CTxOut(0, CScript()));
3227 for (unsigned int i = 0; i < nSigOps; i++)
3228 tx.vout[0].scriptPubKey << OP_CHECKSIG;
3229 tx.vout[0].scriptPubKey.insert(tx.vout[0].scriptPubKey.end(), nPadBytes, 0);
3230 return tx;
3231}
3232
3233static CBlock MakeCoinbaseOnlyTemplate()
3234{
3235 CTransaction txNew;
3236 txNew.vin.resize(1);
3237 txNew.vin[0].prevout.SetNull();
3238 txNew.vin[0].scriptSig << 1 << 1 << 1;
3239 txNew.vout.resize(1);
3240 txNew.vout[0].scriptPubKey << OP_CHECKSIG;
3241
3242 CBlock block;
3243 block.vtx.push_back(txNew);
3244 block.nVersion = 1;
3245 block.nTime = GetAdjustedTime();
3246 block.nBits = bnProofOfWorkLimit.GetCompact();
3247 block.nNonce = 0;
3248 return block;
3249}
3250
3251// The sum CheckBlock() takes, spelled the same way.
3252static unsigned int BlockSigOpCount(const CBlock& block)
3253{
3254 unsigned int n = 0;
3255 for (int i = 0; i < (int)block.vtx.size(); i++)
3256 n += block.vtx[i].GetSigOpCount();
3257 return n;
3258}
3259
3260// Drive the selection rule the way both assemblers drive it: ask
3261// BlockTemplateFits before appending, skip what does not fit, never stop early
3262// on one that does not.
3263//
3264// This mirrors the loops in BitcoinMiner() and RebuildJob() rather than calling
3265// them -- one needs a CTxDB, a wallet key and cs_main, the other is static in
3266// rpc.cpp, so neither is reachable from here. What that leaves proven is the
3267// rule the two of them share and the property it is meant to guarantee, which
3268// is where the defect actually lived: the budget was right and the enforcement
3269// was not.
3270static void SelectIntoTemplate(CBlock& block, const vector<CTransaction>& vMempool)
3271{
3272 unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK);
3273 unsigned int nBlockSigOps = block.vtx[0].GetSigOpCount();
3274 for (int i = 0; i < (int)vMempool.size(); i++)
3275 {
3276 const CTransaction& tx = vMempool[i];
3277 unsigned int nTxSize = ::GetSerializeSize(tx, SER_NETWORK);
3278 unsigned int nTxSigOps = tx.GetSigOpCount();
3279 if (!BlockTemplateFits(nBlockSize, nBlockSigOps, nTxSize, nTxSigOps))
3280 continue;
3281 block.vtx.push_back(tx);
3282 nBlockSize += nTxSize;
3283 nBlockSigOps += nTxSigOps;
3284 }
3285}
3286
1178static int RunConsensusLimitsSelfTest() 3287static int RunConsensusLimitsSelfTest()
1179{ 3288{
1180 fflush(stdout); 3289 fflush(stdout);
@@ -1198,6 +3307,95 @@ static int RunConsensusLimitsSelfTest()
1198 "the test block sits between 1 MB and the old 32 MB cap") ? 0 : 1; 3307 "the test block sits between 1 MB and the old 32 MB cap") ? 0 : 1;
1199 nFail += Check(!oversized.CheckSizeLimits(), 3308 nFail += Check(!oversized.CheckSizeLimits(),
1200 "a block above 1 MB is outside the consensus size limit") ? 0 : 1; 3309 "a block above 1 MB is outside the consensus size limit") ? 0 : 1;
3310
3311 // The other half of the cap. Everything above asks what validation
3312 // refuses; nothing asked what assembly produces, which is why a miner
3313 // that could overshoot the cap by an entire mempool went unnoticed.
3314 nFail += Check(MAX_BLOCK_SIZE_GEN < MAX_BLOCK_SIZE,
3315 "the assembly budget stays under the consensus cap") ? 0 : 1;
3316
3317 // The rule itself, at its edges. The sigop comparison has to match
3318 // CheckBlock(), which fails on `> MAX_BLOCK_SIGOPS` and so accepts a
3319 // block sitting exactly on the limit.
3320 nFail += Check(BlockTemplateFits(MAX_BLOCK_SIZE_GEN - 100, 0, 100, 0),
3321 "a transaction that exactly fills the size budget fits") ? 0 : 1;
3322 nFail += Check(!BlockTemplateFits(MAX_BLOCK_SIZE_GEN - 100, 0, 101, 0),
3323 "one byte past the size budget does not fit") ? 0 : 1;
3324 nFail += Check(BlockTemplateFits(0, 0, 0, MAX_BLOCK_SIGOPS),
3325 "sigops exactly on the limit fit, as CheckBlock() allows") ? 0 : 1;
3326 nFail += Check(!BlockTemplateFits(0, 0, 0, MAX_BLOCK_SIGOPS + 1),
3327 "one sigop past the limit does not fit") ? 0 : 1;
3328
3329 // A mempool several times the consensus cap, in ordinary-sized
3330 // transactions. The old miner tested its budget only on the outer
3331 // loop, so a single pass appended all of this.
3332 {
3333 CBlock block = MakeCoinbaseOnlyTemplate();
3334 vector<CTransaction> vMempool;
3335 for (int i = 0; i < 300; i++)
3336 vMempool.push_back(MakeSizedTx(10000, 1, i));
3337 SelectIntoTemplate(block, vMempool);
3338 block.hashMerkleRoot = block.BuildMerkleTree();
3339
3340 nFail += Check(block.vtx.size() > 1,
3341 "a full mempool still fills a template") ? 0 : 1;
3342 nFail += Check(::GetSerializeSize(block, SER_DISK) <= MAX_BLOCK_SIZE_GEN,
3343 "a template built from a 3 MB mempool respects the budget") ? 0 : 1;
3344 nFail += Check(block.CheckSizeLimits(),
3345 "that template is one CheckBlock() would accept on size") ? 0 : 1;
3346 }
3347
3348 // One transaction too big for the budget. Nothing on the path into the
3349 // mempool caps transaction size, so this is a transaction a peer can
3350 // simply send. It must be skipped and the rest still taken -- skipped,
3351 // not stopped on.
3352 {
3353 CBlock block = MakeCoinbaseOnlyTemplate();
3354 vector<CTransaction> vMempool;
3355 vMempool.push_back(MakeSizedTx(1500000, 1, 9999));
3356 for (int i = 0; i < 10; i++)
3357 vMempool.push_back(MakeSizedTx(1000, 1, i));
3358
3359 unsigned int nHugeSize = ::GetSerializeSize(vMempool[0], SER_NETWORK);
3360 nFail += Check(nHugeSize > MAX_BLOCK_SIZE && nHugeSize <= MAX_SIZE,
3361 "the outsized transaction is one the mempool would hold") ? 0 : 1;
3362
3363 SelectIntoTemplate(block, vMempool);
3364 block.hashMerkleRoot = block.BuildMerkleTree();
3365
3366 bool fTookHuge = false;
3367 for (int i = 0; i < (int)block.vtx.size(); i++)
3368 if (block.vtx[i].GetHash() == vMempool[0].GetHash())
3369 fTookHuge = true;
3370
3371 nFail += Check(!fTookHuge,
3372 "a transaction larger than the budget is left out") ? 0 : 1;
3373 nFail += Check(block.vtx.size() == 11,
3374 "the ten that do fit are taken anyway") ? 0 : 1;
3375 nFail += Check(block.CheckSizeLimits(),
3376 "one outsized transaction cannot invalidate a template") ? 0 : 1;
3377 }
3378
3379 // Transactions that are cheap in bytes and expensive to verify. Size
3380 // alone does not bound what a block costs, and CheckBlock() refuses on
3381 // the count regardless of how small the block is.
3382 {
3383 CBlock block = MakeCoinbaseOnlyTemplate();
3384 vector<CTransaction> vMempool;
3385 for (int i = 0; i < 100; i++)
3386 vMempool.push_back(MakeSizedTx(0, 1000, i));
3387 SelectIntoTemplate(block, vMempool);
3388 block.hashMerkleRoot = block.BuildMerkleTree();
3389
3390 nFail += Check(block.vtx.size() > 1,
3391 "sigop-heavy transactions are still selected") ? 0 : 1;
3392 nFail += Check(BlockSigOpCount(block) <= MAX_BLOCK_SIGOPS,
3393 "selection stops at the sigop cap CheckBlock() enforces") ? 0 : 1;
3394 nFail += Check(::GetSerializeSize(block, SER_DISK) < MAX_BLOCK_SIZE_GEN,
3395 "and stops on sigops while still well inside the size budget") ? 0 : 1;
3396 nFail += Check(block.CheckSizeLimits(),
3397 "the sigop-bounded template is within the size limit too") ? 0 : 1;
3398 }
1201 } 3399 }
1202 catch (const std::exception& e) 3400 catch (const std::exception& e)
1203 { 3401 {
@@ -1216,6 +3414,357 @@ static int RunConsensusLimitsSelfTest()
1216 return nFail == 0 ? 0 : 1; 3414 return nFail == 0 ? 0 : 1;
1217} 3415}
1218 3416
3417static int RunParseMoneySelfTest()
3418{
3419 printf("parse-money self-test\n");
3420 int nFail = 0;
3421 struct Case { const char* in; bool ok; int64 val; };
3422 Case cases[] = {
3423 {"0.5", true, 50000000LL},
3424 {"1", true, 100000000LL},
3425 {"1.23456789", true, 123456789LL},
3426 {"0.00000001", true, 1LL},
3427 {"10.00000000", true, 1000000000LL},
3428 {"1,000", true, 100000000000LL},
3429 {"1.5 ", true, 150000000LL},
3430 {"0.123456789", false, 0LL},
3431 {"1.2.3", false, 0LL},
3432 {"abc", false, 0LL},
3433 {"-1", false, 0LL},
3434 {"1e5", false, 0LL},
3435 };
3436 for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++)
3437 {
3438 int64 v = -1;
3439 bool r = ParseMoney(cases[i].in, v);
3440 bool pass = (r == cases[i].ok) && (!r || v == cases[i].val);
3441 char desc[160];
3442 snprintf(desc, sizeof(desc), "ParseMoney(\"%s\") %s and value matches",
3443 cases[i].in, cases[i].ok ? "accepts" : "rejects");
3444 nFail += Check(pass, desc) ? 0 : 1;
3445 }
3446 printf("%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
3447 nFail, nFail == 1 ? "" : "s");
3448 fflush(stdout);
3449 return nFail == 0 ? 0 : 1;
3450}
3451
3452static bool SelfTestReadN(SOCKET s, void* buf, int n)
3453{
3454 char* p = (char*)buf;
3455 int off = 0;
3456 while (off < n)
3457 {
3458 int r = recv(s, p + off, n - off, 0);
3459 if (r <= 0)
3460 return false;
3461 off += r;
3462 }
3463 return true;
3464}
3465
3466static bool SelfTestWriteN(SOCKET s, const void* buf, int n)
3467{
3468 const char* p = (const char*)buf;
3469 int off = 0;
3470 while (off < n)
3471 {
3472 int r = send(s, p + off, n - off, 0);
3473 if (r <= 0)
3474 return false;
3475 off += r;
3476 }
3477 return true;
3478}
3479
3480static void SelfTestSetSocketTimeout(SOCKET s, int nTimeoutSecs)
3481{
3482#ifdef _WIN32
3483 DWORD tv = (DWORD)nTimeoutSecs * 1000;
3484 setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tv, sizeof(tv));
3485 setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, (const char*)&tv, sizeof(tv));
3486#else
3487 struct timeval tv;
3488 tv.tv_sec = nTimeoutSecs;
3489 tv.tv_usec = 0;
3490 setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
3491 setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
3492#endif
3493}
3494
3495#ifdef _WIN32
3496struct SelfTestWinsock
3497{
3498 bool fStarted;
3499 SelfTestWinsock() : fStarted(false)
3500 {
3501 WSADATA wsadata;
3502 fStarted = (WSAStartup(MAKEWORD(2,2), &wsadata) == 0);
3503 }
3504 ~SelfTestWinsock()
3505 {
3506 if (fStarted)
3507 WSACleanup();
3508 }
3509};
3510#endif
3511
3512static bool RunSocks5HandshakeProbe(std::string& errOut)
3513{
3514 errOut.clear();
3515#ifdef _WIN32
3516 SelfTestWinsock winsock;
3517 if (!winsock.fStarted)
3518 {
3519 errOut = "could not start Winsock";
3520 return false;
3521 }
3522#endif
3523
3524 SOCKET listener = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
3525 if (listener == INVALID_SOCKET)
3526 {
3527 errOut = "could not create listener";
3528 return false;
3529 }
3530 SelfTestSetSocketTimeout(listener, 5);
3531
3532 struct sockaddr_in addr;
3533 memset(&addr, 0, sizeof(addr));
3534 addr.sin_family = AF_INET;
3535 addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
3536 addr.sin_port = 0;
3537 if (bind(listener, (struct sockaddr*)&addr, sizeof(addr)) != 0 ||
3538 listen(listener, 1) != 0)
3539 {
3540 closesocket(listener);
3541 errOut = "could not bind/listen on loopback";
3542 return false;
3543 }
3544
3545#ifdef _WIN32
3546 int addrLen = sizeof(addr);
3547#else
3548 socklen_t addrLen = sizeof(addr);
3549#endif
3550 if (getsockname(listener, (struct sockaddr*)&addr, &addrLen) != 0)
3551 {
3552 closesocket(listener);
3553 errOut = "could not read listener port";
3554 return false;
3555 }
3556 unsigned short proxyPort = ntohs(addr.sin_port);
3557
3558 bool fServerOk = false;
3559 std::string serverErr;
3560 std::string requestedHost;
3561 unsigned short requestedPort = 0;
3562 std::thread server([&]() {
3563 SOCKET s = accept(listener, NULL, NULL);
3564 if (s == INVALID_SOCKET)
3565 {
3566 serverErr = "accept failed";
3567 return;
3568 }
3569 SelfTestSetSocketTimeout(s, 5);
3570
3571 unsigned char hello[3] = {0,0,0};
3572 if (!SelfTestReadN(s, hello, sizeof(hello)) ||
3573 hello[0] != 0x05 || hello[1] != 0x01 || hello[2] != 0x00)
3574 {
3575 serverErr = "bad SOCKS5 greeting";
3576 closesocket(s);
3577 return;
3578 }
3579 unsigned char choice[2] = {0x05, 0x00};
3580 if (!SelfTestWriteN(s, choice, sizeof(choice)))
3581 {
3582 serverErr = "could not write method choice";
3583 closesocket(s);
3584 return;
3585 }
3586
3587 unsigned char hdr[5] = {0,0,0,0,0};
3588 if (!SelfTestReadN(s, hdr, sizeof(hdr)) ||
3589 hdr[0] != 0x05 || hdr[1] != 0x01 || hdr[2] != 0x00 || hdr[3] != 0x03)
3590 {
3591 serverErr = "CONNECT did not use domain-name address type";
3592 closesocket(s);
3593 return;
3594 }
3595
3596 unsigned char len = hdr[4];
3597 std::vector<unsigned char> rest((size_t)len + 2);
3598 if (!SelfTestReadN(s, &rest[0], (int)rest.size()))
3599 {
3600 serverErr = "could not read CONNECT target";
3601 closesocket(s);
3602 return;
3603 }
3604 requestedHost.assign((const char*)&rest[0], (size_t)len);
3605 requestedPort = ((unsigned short)rest[len] << 8) | rest[len + 1];
3606
3607 unsigned char reply[10] = {0x05, 0x00, 0x00, 0x01, 0,0,0,0, 0,0};
3608 if (!SelfTestWriteN(s, reply, sizeof(reply)))
3609 {
3610 serverErr = "could not write CONNECT reply";
3611 closesocket(s);
3612 return;
3613 }
3614 fServerOk = true;
3615 closesocket(s);
3616 });
3617
3618 std::string err;
3619 BtfClearSocks5Proxy();
3620 bool fClientOk = BtfSetSocks5Proxy(strprintf("127.0.0.1:%u", (unsigned)proxyPort), err);
3621 SOCKET client = INVALID_SOCKET;
3622 if (fClientOk)
3623 client = BtfConnectSocket("relay.example", 443, SOCK_NOSTR, 5);
3624 if (client != INVALID_SOCKET)
3625 BtfCloseSocket(client);
3626 BtfClearSocks5Proxy();
3627 closesocket(listener);
3628 server.join();
3629
3630 if (!fClientOk)
3631 errOut = strprintf("could not enable proxy: %s", err.c_str());
3632 else if (client == INVALID_SOCKET)
3633 errOut = "client SOCKS5 connect failed";
3634 else if (!fServerOk)
3635 errOut = serverErr.empty() ? "mock server failed" : serverErr;
3636 else if (requestedHost != "relay.example" || requestedPort != 443)
3637 errOut = strprintf("unexpected CONNECT target %s:%u", requestedHost.c_str(), (unsigned)requestedPort);
3638 else
3639 return true;
3640 return false;
3641}
3642
3643static int RunSocks5ProxySelfTest()
3644{
3645 printf("socks5-proxy self-test\n");
3646 int nFail = 0;
3647 std::string host, err;
3648 unsigned short port = 0;
3649
3650 nFail += Check(BtfParseSocks5Proxy("127.0.0.1:9050", host, port, err) &&
3651 host == "127.0.0.1" && port == 9050,
3652 "parses local Tor proxy endpoint") ? 0 : 1;
3653 nFail += Check(BtfParseSocks5Proxy("localhost:9050", host, port, err) &&
3654 host == "localhost" && port == 9050,
3655 "parses hostname proxy endpoint") ? 0 : 1;
3656 nFail += Check(BtfParseSocks5Proxy("[::1]:9050", host, port, err) &&
3657 host == "::1" && port == 9050,
3658 "parses bracketed IPv6 proxy endpoint") ? 0 : 1;
3659 nFail += Check(!BtfParseSocks5Proxy("::1:9050", host, port, err),
3660 "rejects unbracketed IPv6 proxy endpoint") ? 0 : 1;
3661 nFail += Check(!BtfParseSocks5Proxy("localhost", host, port, err),
3662 "rejects missing port") ? 0 : 1;
3663 nFail += Check(!BtfParseSocks5Proxy("localhost:0", host, port, err),
3664 "rejects zero port") ? 0 : 1;
3665 nFail += Check(!BtfParseSocks5Proxy("localhost:70000", host, port, err),
3666 "rejects out-of-range port") ? 0 : 1;
3667 nFail += Check(!BtfParseSocks5Proxy("localhost:999999999999999999999999999999", host, port, err),
3668 "rejects overflowing port") ? 0 : 1;
3669 nFail += Check(!BtfParseSocks5Proxy("user:pass@localhost:9050", host, port, err),
3670 "rejects unsupported authenticated proxy syntax") ? 0 : 1;
3671 nFail += Check(BtfIsTorOnionHost("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcd.onion") &&
3672 BtfIsTorOnionHost("Relay.Example.ONION.") &&
3673 !BtfIsTorOnionHost("relay.example") &&
3674 !BtfIsTorOnionHost(".onion"),
3675 "detects .onion hosts without accepting lookalikes") ? 0 : 1;
3676 std::string onion;
3677 nFail += Check(btf::NormalizeOnionEndpoint("ABCDEFGHIJKLMNOPQRSTUVWXYZABCDEFGHIJKLMNOPQRSTUVWXYZABCD.onion.:8433", onion) &&
3678 onion == "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcd.onion:8433",
3679 "normalizes v3 onion peer endpoint") ? 0 : 1;
3680 nFail += Check(!btf::NormalizeOnionEndpoint("short.onion:8433", onion) &&
3681 !btf::NormalizeOnionEndpoint("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcd.onion:0", onion) &&
3682 !btf::NormalizeOnionEndpoint("abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabc1.onion:8433", onion),
3683 "rejects malformed onion peer endpoints") ? 0 : 1;
3684
3685 BtfClearSocks5Proxy();
3686 nFail += Check(!BtfSocks5ProxyEnabled(), "starts disabled after clear") ? 0 : 1;
3687 nFail += Check(BtfSetSocks5Proxy("127.0.0.1:9050", err) &&
3688 BtfSocks5ProxyEnabled() &&
3689 BtfSocks5ProxyName() == "127.0.0.1:9050",
3690 "enables parsed SOCKS5 proxy") ? 0 : 1;
3691 BtfClearSocks5Proxy();
3692 nFail += Check(!BtfSocks5ProxyEnabled(), "clear disables proxy") ? 0 : 1;
3693
3694 nFail += Check(BtfSetSocks5Proxy("[::1]:9050", err) &&
3695 BtfSocks5ProxyEnabled() &&
3696 BtfSocks5ProxyName() == "[::1]:9050",
3697 "enables bracketed IPv6 proxy endpoint") ? 0 : 1;
3698 BtfClearSocks5Proxy();
3699
3700 nFail += Check(BtfEnableTorProxy("", err) &&
3701 BtfSocks5ProxyEnabled() &&
3702 BtfTorProxyEnabled() &&
3703 BtfSocks5ProxyName() == "127.0.0.1:9050",
3704 "enables Tor mode on the default local SOCKS5 endpoint") ? 0 : 1;
3705 BtfClearSocks5Proxy();
3706
3707 nFail += Check(BtfEnableTorProxy("[::1]:9050", err) &&
3708 BtfSocks5ProxyEnabled() &&
3709 BtfTorProxyEnabled() &&
3710 BtfSocks5ProxyName() == "[::1]:9050",
3711 "enables Tor mode on a custom IPv6 endpoint") ? 0 : 1;
3712 BtfClearSocks5Proxy();
3713
3714 nFail += Check(BtfSetSocks5Proxy("127.0.0.1:9051", err) &&
3715 BtfSocks5ProxyEnabled() &&
3716 !BtfTorProxyEnabled(),
3717 "plain SOCKS5 mode is distinct from Tor mode") ? 0 : 1;
3718 BtfClearSocks5Proxy();
3719
3720 std::string handshakeErr;
3721 bool fHandshakeOk = RunSocks5HandshakeProbe(handshakeErr);
3722 nFail += Check(fHandshakeOk, handshakeErr.empty() ?
3723 "completes SOCKS5 domain-name CONNECT handshake" :
3724 strprintf("completes SOCKS5 domain-name CONNECT handshake (%s)",
3725 handshakeErr.c_str()).c_str()) ? 0 : 1;
3726
3727 printf("%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
3728 nFail, nFail == 1 ? "" : "s");
3729 fflush(stdout);
3730 return nFail == 0 ? 0 : 1;
3731}
3732
3733static int RunManagedTorSelfTest()
3734{
3735 printf("managed-tor self-test\n");
3736 int nFail = 0;
3737
3738 std::string torrc = BtfBuildManagedTorrcForTest("C:/Bitflash Managed Tor/data",
3739 "C:/Bitflash Managed Tor/onion-service",
3740 19050, 19051, 8433);
3741 nFail += Check(torrc.find("DataDirectory \"C:/Bitflash Managed Tor/data\"") != std::string::npos,
3742 "quotes the Tor data directory") ? 0 : 1;
3743 nFail += Check(torrc.find("SocksPort 127.0.0.1:19050") != std::string::npos,
3744 "binds SOCKS5 to localhost only") ? 0 : 1;
3745 nFail += Check(torrc.find("ControlPort 127.0.0.1:19051") != std::string::npos &&
3746 torrc.find("CookieAuthentication 1") != std::string::npos,
3747 "binds an authenticated Tor control port to localhost only") ? 0 : 1;
3748 nFail += Check(torrc.find("IsolateSOCKSAuth") != std::string::npos &&
3749 torrc.find("IsolateClientAddr") != std::string::npos &&
3750 torrc.find("IsolateDestAddr") != std::string::npos &&
3751 torrc.find("IsolateDestPort") != std::string::npos,
3752 "enables strict Tor stream isolation") ? 0 : 1;
3753 nFail += Check(torrc.find("HiddenServiceDir \"C:/Bitflash Managed Tor/onion-service\"") != std::string::npos,
3754 "writes a hidden service directory") ? 0 : 1;
3755 nFail += Check(torrc.find("HiddenServiceVersion 3") != std::string::npos,
3756 "requests a v3 onion service") ? 0 : 1;
3757 nFail += Check(torrc.find("HiddenServicePort 8433 127.0.0.1:8433") != std::string::npos,
3758 "maps the onion service to the Bitflash P2P listener") ? 0 : 1;
3759 nFail += Check(BtfManagedTorStatus() == "disabled",
3760 "managed Tor starts disabled") ? 0 : 1;
3761
3762 printf("%s (%d failure%s)\n", nFail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED",
3763 nFail, nFail == 1 ? "" : "s");
3764 fflush(stdout);
3765 return nFail == 0 ? 0 : 1;
3766}
3767
1219int RunSelfTest(const std::string& name) 3768int RunSelfTest(const std::string& name)
1220{ 3769{
1221 AttachTerminal(); 3770 AttachTerminal();
@@ -1226,16 +3775,44 @@ int RunSelfTest(const std::string& name)
1226 return RunWalletHDSelfTest(); 3775 return RunWalletHDSelfTest();
1227 if (name == "wallet-format") 3776 if (name == "wallet-format")
1228 return RunWalletFormatSelfTest(); 3777 return RunWalletFormatSelfTest();
3778 if (name == "wallet-storage-sanity")
3779 return RunWalletStorageSanitySelfTest();
3780 if (name == "db-env-reopen")
3781 return RunDbEnvReopenSelfTest();
3782 if (name == "wallet-sqlite")
3783 return RunWalletSQLiteSelfTest();
3784 if (name == "wallet-sqlite-migration")
3785 return RunWalletSQLiteMigrationSelfTest();
1229 if (name == "wallet-crypto") 3786 if (name == "wallet-crypto")
1230 return RunWalletCryptoSelfTest(); 3787 return RunWalletCryptoSelfTest();
1231 if (name == "wallet-encrypt") 3788 if (name == "wallet-encrypt")
1232 return RunWalletEncryptSelfTest(); 3789 return RunWalletEncryptSelfTest();
3790 if (name == "wallet-sqlite-encrypt")
3791 return RunWalletSQLiteEncryptSelfTest();
3792 if (name == "wallet-backend-default")
3793 return RunWalletBackendDefaultSelfTest();
3794 if (name == "wallet-convert")
3795 return RunWalletConvertSelfTest();
3796 if (name == "wallet-portability")
3797 return RunWalletPortabilitySelfTest();
1233 if (name == "net-message") 3798 if (name == "net-message")
1234 return RunNetMessageSelfTest(); 3799 return RunNetMessageSelfTest();
3800 if (name == "network-params")
3801 return RunNetworkParamsSelfTest();
1235 if (name == "consensus-limits") 3802 if (name == "consensus-limits")
1236 return RunConsensusLimitsSelfTest(); 3803 return RunConsensusLimitsSelfTest();
3804 if (name == "pool-stratum")
3805 return RunPoolStratumSelfTest();
3806 if (name == "payout-ledger")
3807 return RunPayoutLedgerSelfTest();
3808 if (name == "parse-money")
3809 return RunParseMoneySelfTest();
3810 if (name == "socks5-proxy")
3811 return RunSocks5ProxySelfTest();
3812 if (name == "managed-tor")
3813 return RunManagedTorSelfTest();
1237 3814
1238 printf("Unknown self-test '%s'\n", name.c_str()); 3815 printf("Unknown self-test '%s'\n", name.c_str());
1239 printf("Known self-tests: wallet-keypool, wallet-hd, wallet-format, wallet-crypto, wallet-encrypt, net-message, consensus-limits\n"); 3816 printf("Known self-tests: wallet-keypool, wallet-hd, wallet-format, wallet-storage-sanity, db-env-reopen, wallet-sqlite, wallet-sqlite-migration, wallet-crypto, wallet-encrypt, wallet-sqlite-encrypt, wallet-backend-default, wallet-convert, wallet-portability, net-message, network-params, consensus-limits, pool-stratum, payout-ledger, parse-money, socks5-proxy, managed-tor\n");
1240 return 1; 3817 return 1;
1241} 3818}
diff --git a/src/selftest.h b/src/selftest.h
index d3699c6..0ffe232 100644
--- a/src/selftest.h
+++ b/src/selftest.h
@@ -7,5 +7,6 @@
7#include <string> 7#include <string>
8 8
9int RunSelfTest(const std::string& name); 9int RunSelfTest(const std::string& name);
10int RunSelfTestMutateWallet(const std::string& name);
10 11
11#endif 12#endif
diff --git a/src/sockcount.h b/src/sockcount.h
index 96b6efb..a63c6be 100644
--- a/src/sockcount.h
+++ b/src/sockcount.h
@@ -63,6 +63,8 @@ enum
63 SOCK_PAIR_APP, // loopback pair: the app end handed to CNode 63 SOCK_PAIR_APP, // loopback pair: the app end handed to CNode
64 SOCK_PAIR_PUMP, // loopback pair: the pump end 64 SOCK_PAIR_PUMP, // loopback pair: the pump end
65 SOCK_NOSTR, // outbound to a Nostr relay 65 SOCK_NOSTR, // outbound to a Nostr relay
66 SOCK_ONION_PEER, // direct outbound to a peer's Tor hidden service
67 SOCK_STRATUM_BRIDGE, // local Stratum bridge listener/miner sockets
66 SOCK_SITES 68 SOCK_SITES
67}; 69};
68 70
diff --git a/src/test_btfaddr.cpp b/src/test_btfaddr.cpp
index b40dcc1..9d33d63 100644
--- a/src/test_btfaddr.cpp
+++ b/src/test_btfaddr.cpp
@@ -86,6 +86,7 @@ int main()
86 } 86 }
87 87
88 const std::string ENC_HEX = "aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899"; 88 const std::string ENC_HEX = "aabbccddeeff00112233445566778899aabbccddeeff00112233445566778899";
89 const std::string ONION = "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcd.onion:8433";
89 90
90 // helper: fresh keypair -> (seckey, xonly pubkey) 91 // helper: fresh keypair -> (seckey, xonly pubkey)
91 auto newKey = [&](unsigned char sk[32], unsigned char pk[32]) { 92 auto newKey = [&](unsigned char sk[32], unsigned char pk[32]) {
@@ -108,6 +109,17 @@ int main()
108 CHECK(!btf::VerifyDescriptor(ctx, d, pk2, bad), "different key rejected"); 109 CHECK(!btf::VerifyDescriptor(ctx, d, pk2, bad), "different key rejected");
109 } 110 }
110 111
112 printf("descriptor_signs_optional_onion_endpoint\n");
113 {
114 unsigned char sk[32], pk[32]; newKey(sk, pk);
115 std::string d = btf::SignDescriptor(ctx, sk, ENC_HEX, "meeting-node-abc", 1000, ONION);
116 CHECK(!d.empty(), "sign produced onion descriptor");
117 btf::Descriptor got;
118 CHECK(btf::VerifyDescriptor(ctx, d, pk, got), "onion descriptor verifies");
119 CHECK(got.meeting_node == "meeting-node-abc" && got.enc == ENC_HEX &&
120 got.created == 1000 && got.onion == ONION, "onion fields match");
121 }
122
111 printf("descriptor_rejects_tampering\n"); 123 printf("descriptor_rejects_tampering\n");
112 { 124 {
113 unsigned char sk[32], pk[32]; newKey(sk, pk); 125 unsigned char sk[32], pk[32]; newKey(sk, pk);
@@ -117,6 +129,27 @@ int main()
117 CHECK(!btf::VerifyDescriptor(ctx, j.dump(), pk, o), "tampered meeting_node rejected"); 129 CHECK(!btf::VerifyDescriptor(ctx, j.dump(), pk, o), "tampered meeting_node rejected");
118 json j2 = json::parse(d); j2["enc"] = "00000000000000000000000000000000000000000000000000000000deadbeef"; 130 json j2 = json::parse(d); j2["enc"] = "00000000000000000000000000000000000000000000000000000000deadbeef";
119 CHECK(!btf::VerifyDescriptor(ctx, j2.dump(), pk, o), "tampered enc rejected"); 131 CHECK(!btf::VerifyDescriptor(ctx, j2.dump(), pk, o), "tampered enc rejected");
132
133 std::string d3 = btf::SignDescriptor(ctx, sk, ENC_HEX, "node-1", 5, ONION);
134 json j3 = json::parse(d3);
135 j3["onion"] = "bcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcde.onion:8433";
136 CHECK(!btf::VerifyDescriptor(ctx, j3.dump(), pk, o),
137 "tampered onion rejects the descriptor");
138 json j4 = json::parse(d3);
139 j4["sig3"] = std::string(128, '0');
140 CHECK(!btf::VerifyDescriptor(ctx, j4.dump(), pk, o),
141 "bad onion signature rejects the descriptor");
142 json j5 = json::parse(d3);
143 j5.erase("sig3");
144 CHECK(!btf::VerifyDescriptor(ctx, j5.dump(), pk, o),
145 "unsigned onion rejects the descriptor");
146 // Removing both fields still verifies: the result is indistinguishable
147 // from a peer that never had an onion endpoint. That residual strip
148 // needs the onion bound into the primary sig, an upstream change.
149 json j6 = json::parse(d3);
150 j6.erase("onion"); j6.erase("sig3");
151 CHECK(btf::VerifyDescriptor(ctx, j6.dump(), pk, o) && o.onion.empty(),
152 "full strip indistinguishable from no-onion (documented residual)");
120 } 153 }
121 154
122 printf("full_resolve_chain_from_address\n"); 155 printf("full_resolve_chain_from_address\n");
diff --git a/src/test_compactblock.cpp b/src/test_compactblock.cpp
new file mode 100644
index 0000000..f4b5583
--- /dev/null
+++ b/src/test_compactblock.cpp
@@ -0,0 +1,133 @@
1// Standalone test of compactblock.cpp's core logic: build a compact block,
2// reconstruct it from a simulated mempool, including the missing-tx path.
3// Not wired into the P2P layer -- proves BuildCompactBlock/
4// TryReconstructBlock/FillMissingAndFinish are correct in isolation.
5
6#include "headers.h"
7#include "compactblock.h"
8#include <cstdio>
9#include <cassert>
10
11// main.cpp defines this; we're not linking main.cpp, so define it here.
12map<uint256, CTransaction> mapTransactions;
13string GetAppDir() { return "."; } // util.h's debug-log path helper needs this
14
15static int g_fail = 0;
16#define CHECK(cond, msg) do { \
17 if (cond) printf(" ok %s\n", msg); \
18 else { printf(" FAIL %s\n", msg); g_fail++; } \
19} while (0)
20
21// Build a syntactically-valid-enough CTransaction that GetHash()s uniquely
22// per `nSeed` -- doesn't need to be consensus-valid, this test only
23// exercises short-ID computation and reconstruction, not script validation.
24static CTransaction MakeTx(int nSeed)
25{
26 CTransaction tx;
27 tx.nVersion = 1;
28 tx.nLockTime = nSeed; // cheapest way to make each tx's serialization (and hash) unique
29 return tx;
30}
31
32int main()
33{
34 // Build a block: coinbase + 20 regular transactions.
35 CBlock block;
36 block.nVersion = 1;
37 block.hashPrevBlock = uint256("0x00000000000000000000000000000000000000000000000000000000000abc");
38 block.nTime = 1234567890;
39 block.nBits = 0x1f0fffff;
40 block.nNonce = 42;
41 block.vtx.push_back(MakeTx(0)); // coinbase
42 for (int i = 1; i <= 20; i++)
43 block.vtx.push_back(MakeTx(i));
44 block.hashMerkleRoot = block.BuildMerkleTree();
45
46 printf("build_and_reconstruct_full_mempool\n");
47 {
48 CCmpctBlock cmpct;
49 CHECK(BuildCompactBlock(block, cmpct), "BuildCompactBlock succeeds");
50 CHECK(cmpct.vPrefilledIndex.size() == 1 && cmpct.vPrefilledIndex[0] == 0, "only the coinbase is prefilled");
51 CHECK(cmpct.vShortTxId.size() == 20, "one short ID per remaining tx");
52
53 // Simulate a peer with every transaction already in its mempool.
54 mapTransactions.clear();
55 for (size_t i = 1; i < block.vtx.size(); i++)
56 mapTransactions[block.vtx[i].GetHash()] = block.vtx[i];
57
58 CBlock rebuilt;
59 vector<unsigned int> vMissing;
60 CHECK(TryReconstructBlock(cmpct, rebuilt, vMissing), "fully reconstructs when mempool has everything");
61 CHECK(vMissing.empty(), "nothing reported missing");
62 CHECK(rebuilt.GetHash() == block.GetHash(), "reconstructed block hash matches original exactly");
63 CHECK(rebuilt.vtx.size() == block.vtx.size(), "same transaction count");
64 bool fAllMatch = true;
65 for (size_t i = 0; i < block.vtx.size(); i++)
66 if (rebuilt.vtx[i].GetHash() != block.vtx[i].GetHash()) fAllMatch = false;
67 CHECK(fAllMatch, "every transaction in the right position");
68 }
69
70 printf("partial_mempool_then_getblocktxn_completes_it\n");
71 {
72 CCmpctBlock cmpct;
73 BuildCompactBlock(block, cmpct);
74
75 // Simulate a peer missing 3 of the 20 non-coinbase transactions.
76 mapTransactions.clear();
77 vector<unsigned int> expectMissing;
78 for (size_t i = 1; i < block.vtx.size(); i++)
79 {
80 if (i == 5 || i == 12 || i == 19) { expectMissing.push_back((unsigned int)i); continue; }
81 mapTransactions[block.vtx[i].GetHash()] = block.vtx[i];
82 }
83
84 CBlock rebuilt;
85 vector<unsigned int> vMissing;
86 CHECK(!TryReconstructBlock(cmpct, rebuilt, vMissing), "reports incomplete when mempool is missing some tx");
87 CHECK(vMissing == expectMissing, "missing indices are exactly the ones not in mempool");
88
89 // Simulate the getblocktxn round trip: peer sends back the missing txs.
90 vector<CTransaction> vTxn;
91 foreach(unsigned int idx, vMissing)
92 vTxn.push_back(block.vtx[idx]);
93
94 CHECK(FillMissingAndFinish(rebuilt, vMissing, vTxn), "FillMissingAndFinish succeeds with matching response");
95 CHECK(rebuilt.GetHash() == block.GetHash(), "block hash matches after filling in the missing tx");
96 }
97
98 printf("mismatched_blocktxn_response_rejected\n");
99 {
100 CCmpctBlock cmpct;
101 BuildCompactBlock(block, cmpct);
102 mapTransactions.clear();
103 mapTransactions[block.vtx[1].GetHash()] = block.vtx[1]; // only one in mempool
104
105 CBlock rebuilt;
106 vector<unsigned int> vMissing;
107 TryReconstructBlock(cmpct, rebuilt, vMissing);
108 CHECK(vMissing.size() == 19, "19 missing when mempool has only 1 of 20");
109
110 vector<CTransaction> vWrongCount;
111 vWrongCount.push_back(block.vtx[2]); // way fewer than vMissing.size()
112 CHECK(!FillMissingAndFinish(rebuilt, vMissing, vWrongCount), "rejects a response with the wrong tx count");
113 }
114
115 printf("different_blocks_give_different_short_ids_for_same_tx\n");
116 {
117 // Same transaction, two different (simulated) blocks -- short IDs
118 // must differ, since they're keyed per-block (hashPrevBlock +
119 // fresh nonce each time), not just derived from the txid alone.
120 CTransaction tx = MakeTx(99);
121 uint256 prevA = uint256("0x1111111111111111111111111111111111111111111111111111111111111111");
122 uint256 prevB = uint256("0x2222222222222222222222222222222222222222222222222222222222222222");
123 uint64 id1 = ComputeShortTxId(prevA, 0xAAAABBBBCCCCDDDDULL, tx.GetHash());
124 uint64 id2 = ComputeShortTxId(prevB, 0xAAAABBBBCCCCDDDDULL, tx.GetHash());
125 uint64 id3 = ComputeShortTxId(prevA, 0x1111222233334444ULL, tx.GetHash());
126 CHECK(id1 != id2, "different hashPrevBlock -> different short ID");
127 CHECK(id1 != id3, "different nonce -> different short ID");
128 CHECK(id1 <= 0xFFFFFFFFFFFFULL, "short ID fits in 48 bits");
129 }
130
131 printf("\n%s (%d failures)\n", g_fail == 0 ? "ALL TESTS PASSED" : "TESTS FAILED", g_fail);
132 return g_fail == 0 ? 0 : 1;
133}
diff --git a/src/tor.cpp b/src/tor.cpp
new file mode 100644
index 0000000..fea6912
--- /dev/null
+++ b/src/tor.cpp
@@ -0,0 +1,892 @@
1// Copyright (c) 2026 Bitflash developers
2// Distributed under the MIT/X11 software license.
3
4#include "headers_core.h"
5#include "tor.h"
6#include "nostr.h"
7#include "proxy.h"
8
9#include <errno.h>
10
11#ifndef _WIN32
12#include <signal.h>
13#include <sys/wait.h>
14#endif
15
16static CCriticalSection cs_managedTor;
17static bool g_fManagedTor = false;
18static bool g_fManagedTorStopping = false;
19static unsigned short g_managedTorSocksPort = 0;
20static unsigned short g_managedTorControlPort = 0;
21static unsigned short g_managedTorP2PPort = 0;
22static std::string g_managedTorPath;
23static std::string g_managedTorDataDir;
24static std::string g_managedTorHiddenServiceDir;
25static std::string g_managedTorOnion;
26static std::string g_managedTorStatus;
27#ifdef _WIN32
28static PROCESS_INFORMATION g_managedTorProcess;
29#else
30static pid_t g_managedTorPid = -1;
31#endif
32
33static std::string PathJoin(const std::string& a, const std::string& b)
34{
35 if (a.empty())
36 return b;
37 char last = a[a.size() - 1];
38 if (last == '/' || last == '\\')
39 return a + b;
40 return a + "/" + b;
41}
42
43static std::string NormalizeTorrcPath(std::string path)
44{
45 for (size_t i = 0; i < path.size(); i++)
46 if (path[i] == '\\')
47 path[i] = '/';
48 return path;
49}
50
51static std::string QuoteTorrcPath(const std::string& path)
52{
53 std::string out = "\"";
54 for (size_t i = 0; i < path.size(); i++)
55 {
56 if (path[i] == '"' || path[i] == '\\')
57 out += '\\';
58 out += path[i];
59 }
60 out += "\"";
61 return out;
62}
63
64static std::string QuoteCommandArg(const std::string& arg)
65{
66 std::string out = "\"";
67 for (size_t i = 0; i < arg.size(); i++)
68 {
69 if (arg[i] == '"')
70 out += '\\';
71 out += arg[i];
72 }
73 out += "\"";
74 return out;
75}
76
77static bool EnsureDir(const std::string& path, std::string& errOut)
78{
79 if (path.empty())
80 {
81 errOut = "directory path is empty";
82 return false;
83 }
84#ifdef _WIN32
85 DWORD attr = GetFileAttributesA(path.c_str());
86 if (attr != INVALID_FILE_ATTRIBUTES)
87 {
88 if (attr & FILE_ATTRIBUTE_DIRECTORY)
89 return true;
90 errOut = strprintf("%s exists and is not a directory", path.c_str());
91 return false;
92 }
93 if (CreateDirectoryA(path.c_str(), NULL))
94 return true;
95 DWORD err = GetLastError();
96 if (err == ERROR_ALREADY_EXISTS)
97 return true;
98 errOut = strprintf("could not create %s (error %lu)", path.c_str(), (unsigned long)err);
99 return false;
100#else
101 struct stat st;
102 if (stat(path.c_str(), &st) == 0)
103 {
104 if (S_ISDIR(st.st_mode))
105 {
106 chmod(path.c_str(), 0700);
107 return true;
108 }
109 errOut = strprintf("%s exists and is not a directory", path.c_str());
110 return false;
111 }
112 if (mkdir(path.c_str(), 0700) == 0)
113 return true;
114 if (errno == EEXIST)
115 {
116 if (stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode))
117 {
118 chmod(path.c_str(), 0700);
119 return true;
120 }
121 }
122 errOut = strprintf("could not create %s: %s", path.c_str(), strerror(errno));
123 return false;
124#endif
125}
126
127static bool WriteTextFile(const std::string& path, const std::string& data,
128 std::string& errOut)
129{
130 std::string tmp = path + ".tmp";
131 FILE* f = fopen(tmp.c_str(), "wb");
132 if (!f)
133 {
134 errOut = strprintf("could not write %s", tmp.c_str());
135 return false;
136 }
137 bool ok = fwrite(data.data(), 1, data.size(), f) == data.size();
138 if (fclose(f) != 0)
139 ok = false;
140 if (!ok)
141 {
142 remove(tmp.c_str());
143 errOut = strprintf("could not finish writing %s", tmp.c_str());
144 return false;
145 }
146#ifdef _WIN32
147 if (!MoveFileExA(tmp.c_str(), path.c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH))
148 {
149 remove(tmp.c_str());
150 errOut = strprintf("could not install %s atomically (error %lu)",
151 path.c_str(), (unsigned long)GetLastError());
152 return false;
153 }
154#else
155 if (rename(tmp.c_str(), path.c_str()) != 0)
156 {
157 remove(tmp.c_str());
158 errOut = strprintf("could not install %s atomically: %s",
159 path.c_str(), strerror(errno));
160 return false;
161 }
162#endif
163 return true;
164}
165
166static bool ReadBinaryFile(const std::string& path, std::vector<unsigned char>& data,
167 std::string& errOut)
168{
169 data.clear();
170 FILE* f = fopen(path.c_str(), "rb");
171 if (!f)
172 {
173 errOut = strprintf("could not read %s", path.c_str());
174 return false;
175 }
176 unsigned char buf[256];
177 for (;;)
178 {
179 size_t n = fread(buf, 1, sizeof(buf), f);
180 if (n > 0)
181 data.insert(data.end(), buf, buf + n);
182 if (n < sizeof(buf))
183 {
184 if (ferror(f))
185 {
186 fclose(f);
187 errOut = strprintf("could not finish reading %s", path.c_str());
188 return false;
189 }
190 break;
191 }
192 if (data.size() > 1024)
193 {
194 fclose(f);
195 errOut = strprintf("%s is unexpectedly large", path.c_str());
196 return false;
197 }
198 }
199 fclose(f);
200 if (data.empty())
201 {
202 errOut = strprintf("%s is empty", path.c_str());
203 return false;
204 }
205 return true;
206}
207
208static void SetSocketTimeouts(SOCKET s, int millis)
209{
210#ifdef _WIN32
211 DWORD timeout = millis;
212 setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, (const char*)&timeout, sizeof(timeout));
213 setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, (const char*)&timeout, sizeof(timeout));
214#else
215 struct timeval tv;
216 tv.tv_sec = millis / 1000;
217 tv.tv_usec = (millis % 1000) * 1000;
218 setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv));
219 setsockopt(s, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv));
220#endif
221}
222
223static bool SendAll(SOCKET s, const std::string& data)
224{
225 const char* p = data.data();
226 int left = (int)data.size();
227 while (left > 0)
228 {
229 int n = send(s, p, left, 0);
230 if (n <= 0)
231 return false;
232 p += n;
233 left -= n;
234 }
235 return true;
236}
237
238static bool RecvControlLine(SOCKET s, std::string& line)
239{
240 line.clear();
241 while (line.size() < 4096)
242 {
243 char ch = 0;
244 int n = recv(s, &ch, 1, 0);
245 if (n <= 0)
246 return false;
247 if (ch == '\n')
248 return true;
249 if (ch != '\r')
250 line += ch;
251 }
252 return false;
253}
254
255static SOCKET ConnectLoopback(unsigned short port, std::string& errOut)
256{
257#ifdef _WIN32
258 WSADATA wsadata;
259 int ret = WSAStartup(MAKEWORD(2,2), &wsadata);
260 if (ret != NO_ERROR)
261 {
262 errOut = strprintf("could not start Winsock for Tor control shutdown (WSAStartup returned %d)", ret);
263 return INVALID_SOCKET;
264 }
265#endif
266 SOCKET s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
267 if (s == INVALID_SOCKET)
268 {
269 errOut = "could not create Tor control socket";
270#ifdef _WIN32
271 WSACleanup();
272#endif
273 return INVALID_SOCKET;
274 }
275 SetSocketTimeouts(s, 3000);
276 struct sockaddr_in addr;
277 memset(&addr, 0, sizeof(addr));
278 addr.sin_family = AF_INET;
279 addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
280 addr.sin_port = htons(port);
281 if (connect(s, (struct sockaddr*)&addr, sizeof(addr)) != 0)
282 {
283 closesocket(s);
284 errOut = strprintf("could not connect to Tor control port 127.0.0.1:%u", (unsigned)port);
285#ifdef _WIN32
286 WSACleanup();
287#endif
288 return INVALID_SOCKET;
289 }
290 return s;
291}
292
293static void CloseLoopback(SOCKET s)
294{
295 if (s != INVALID_SOCKET)
296 closesocket(s);
297#ifdef _WIN32
298 WSACleanup();
299#endif
300}
301
302static bool ReadPositiveTorControlReply(SOCKET s, std::string& errOut)
303{
304 std::string line;
305 if (!RecvControlLine(s, line))
306 {
307 errOut = "Tor control port closed before replying";
308 return false;
309 }
310 if (line.size() >= 3 && line.substr(0, 3) == "250")
311 return true;
312 errOut = strprintf("Tor control command failed: %s", line.c_str());
313 return false;
314}
315
316static bool RequestManagedTorShutdown(std::string& errOut)
317{
318 std::string dataDir;
319 unsigned short controlPort = 0;
320 CRITICAL_BLOCK(cs_managedTor)
321 {
322 dataDir = g_managedTorDataDir;
323 controlPort = g_managedTorControlPort;
324 }
325 if (controlPort == 0 || dataDir.empty())
326 {
327 errOut = "Tor control port is not available";
328 return false;
329 }
330
331 std::vector<unsigned char> cookie;
332 if (!ReadBinaryFile(PathJoin(dataDir, "control_auth_cookie"), cookie, errOut))
333 return false;
334
335 SOCKET s = ConnectLoopback(controlPort, errOut);
336 if (s == INVALID_SOCKET)
337 return false;
338
339 bool ok = false;
340 std::string auth = "AUTHENTICATE " + HexStr(cookie.begin(), cookie.end(), false) + "\r\n";
341 if (!SendAll(s, auth) || !ReadPositiveTorControlReply(s, errOut))
342 goto done;
343
344 if (!SendAll(s, "SIGNAL SHUTDOWN\r\n"))
345 {
346 errOut = "could not send Tor shutdown signal";
347 goto done;
348 }
349
350 {
351 std::string reply;
352 if (RecvControlLine(s, reply) && !(reply.size() >= 3 && reply.substr(0, 3) == "250"))
353 {
354 errOut = strprintf("Tor rejected shutdown signal: %s", reply.c_str());
355 goto done;
356 }
357 }
358 ok = true;
359
360done:
361 CloseLoopback(s);
362 return ok;
363}
364
365static bool ReadOneLine(const std::string& path, std::string& out)
366{
367 out.clear();
368 FILE* f = fopen(path.c_str(), "rb");
369 if (!f)
370 return false;
371 char buf[256] = {0};
372 if (!fgets(buf, sizeof(buf), f))
373 {
374 fclose(f);
375 return false;
376 }
377 fclose(f);
378 out = buf;
379 while (!out.empty() && (out[out.size() - 1] == '\n' || out[out.size() - 1] == '\r' ||
380 out[out.size() - 1] == ' ' || out[out.size() - 1] == '\t'))
381 out.erase(out.size() - 1);
382 return !out.empty();
383}
384
385static bool PickLocalPort(unsigned short& portOut, std::string& errOut)
386{
387 portOut = 0;
388 bool ok = false;
389#ifdef _WIN32
390 WSADATA wsadata;
391 int ret = WSAStartup(MAKEWORD(2,2), &wsadata);
392 if (ret != NO_ERROR)
393 {
394 errOut = strprintf("could not start Winsock for local port probe (WSAStartup returned %d)", ret);
395 return false;
396 }
397#endif
398 struct sockaddr_in addr;
399 socklen_t len = sizeof(addr);
400 SOCKET s = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
401 if (s == INVALID_SOCKET)
402 {
403 errOut = "could not create local port probe socket";
404 goto done;
405 }
406 memset(&addr, 0, sizeof(addr));
407 addr.sin_family = AF_INET;
408 addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
409 addr.sin_port = 0;
410 if (bind(s, (struct sockaddr*)&addr, sizeof(addr)) != 0)
411 {
412 errOut = "could not bind local port probe socket";
413 goto done;
414 }
415 if (getsockname(s, (struct sockaddr*)&addr, &len) != 0)
416 {
417 errOut = "could not read local port probe socket";
418 goto done;
419 }
420 portOut = ntohs(addr.sin_port);
421 if (portOut == 0)
422 {
423 errOut = "local port probe returned port zero";
424 goto done;
425 }
426 ok = true;
427done:
428 if (s != INVALID_SOCKET)
429 closesocket(s);
430#ifdef _WIN32
431 WSACleanup();
432#endif
433 return ok;
434}
435
436static std::string ExecutableDir()
437{
438#ifdef _WIN32
439 char path[MAX_PATH + 1] = {0};
440 DWORD n = GetModuleFileNameA(NULL, path, MAX_PATH);
441 if (n == 0 || n >= MAX_PATH)
442 return ".";
443 std::string s = path;
444 size_t slash = s.find_last_of("\\/");
445 if (slash == std::string::npos)
446 return ".";
447 return s.substr(0, slash);
448#else
449 char path[4096] = {0};
450 ssize_t n = readlink("/proc/self/exe", path, sizeof(path) - 1);
451 if (n <= 0)
452 return ".";
453 path[n] = 0;
454 std::string s = path;
455 size_t slash = s.find_last_of('/');
456 if (slash == std::string::npos)
457 return ".";
458 return s.substr(0, slash);
459#endif
460}
461
462static bool FileIsExecutableCandidate(const std::string& path)
463{
464#ifdef _WIN32
465 DWORD attr = GetFileAttributesA(path.c_str());
466 return attr != INVALID_FILE_ATTRIBUTES && !(attr & FILE_ATTRIBUTE_DIRECTORY);
467#else
468 return access(path.c_str(), X_OK) == 0;
469#endif
470}
471
472static std::string ResolveTorPath(const std::string& torPathOpt)
473{
474 if (!torPathOpt.empty())
475 return torPathOpt;
476
477 std::vector<std::string> candidates;
478 std::string exeDir = ExecutableDir();
479#ifdef _WIN32
480 candidates.push_back(PathJoin(exeDir, "tor/tor.exe"));
481 candidates.push_back(PathJoin(exeDir, "tor.exe"));
482 candidates.push_back("tor.exe");
483#else
484 candidates.push_back(PathJoin(exeDir, "tor/tor"));
485 candidates.push_back(PathJoin(exeDir, "tor"));
486 candidates.push_back("tor");
487#endif
488 for (size_t i = 0; i < candidates.size(); i++)
489 {
490 if (candidates[i].find('/') == std::string::npos &&
491 candidates[i].find('\\') == std::string::npos)
492 return candidates[i]; // Let the OS search PATH.
493 if (FileIsExecutableCandidate(candidates[i]))
494 return candidates[i];
495 }
496#ifdef _WIN32
497 return "tor.exe";
498#else
499 return "tor";
500#endif
501}
502
503bool BtfBundledTorPath(std::string& torPathOut)
504{
505 torPathOut.clear();
506 std::string exeDir = ExecutableDir();
507#ifdef _WIN32
508 std::string path = PathJoin(exeDir, "tor/tor.exe");
509#else
510 std::string path = PathJoin(exeDir, "tor/tor");
511#endif
512 if (!FileIsExecutableCandidate(path))
513 return false;
514 torPathOut = path;
515 return true;
516}
517
518std::string BtfBuildManagedTorrcForTest(const std::string& dataDir,
519 const std::string& hiddenServiceDir,
520 unsigned short socksPort,
521 unsigned short controlPort,
522 unsigned short p2pPort)
523{
524 std::string torData = NormalizeTorrcPath(dataDir);
525 std::string hsDir = NormalizeTorrcPath(hiddenServiceDir);
526 std::string s;
527 s += "# Generated by Bitflash. Edit only while Bitflash is not managing Tor.\n";
528 s += "DataDirectory " + QuoteTorrcPath(torData) + "\n";
529 s += strprintf("SocksPort 127.0.0.1:%u IsolateSOCKSAuth IsolateClientAddr IsolateDestAddr IsolateDestPort\n",
530 (unsigned)socksPort);
531 s += strprintf("ControlPort 127.0.0.1:%u\n", (unsigned)controlPort);
532 s += "CookieAuthentication 1\n";
533 s += "HiddenServiceDir " + QuoteTorrcPath(hsDir) + "\n";
534 s += "HiddenServiceVersion 3\n";
535 s += strprintf("HiddenServicePort %u 127.0.0.1:%u\n",
536 (unsigned)p2pPort, (unsigned)p2pPort);
537 return s;
538}
539
540static bool StartTorProcess(const std::string& torPath, const std::string& torrcPath,
541 std::string& errOut)
542{
543#ifdef _WIN32
544 memset(&g_managedTorProcess, 0, sizeof(g_managedTorProcess));
545 STARTUPINFOA si;
546 memset(&si, 0, sizeof(si));
547 si.cb = sizeof(si);
548 std::string cmd = QuoteCommandArg(torPath) + " -f " + QuoteCommandArg(torrcPath);
549 std::vector<char> mutableCmd(cmd.begin(), cmd.end());
550 mutableCmd.push_back(0);
551 if (!CreateProcessA(NULL, &mutableCmd[0], NULL, NULL, FALSE, CREATE_NO_WINDOW,
552 NULL, NULL, &si, &g_managedTorProcess))
553 {
554 errOut = strprintf("could not start Tor process %s (error %lu)",
555 torPath.c_str(), (unsigned long)GetLastError());
556 return false;
557 }
558 return true;
559#else
560 g_managedTorPid = fork();
561 if (g_managedTorPid < 0)
562 {
563 errOut = strprintf("could not fork Tor process: %s", strerror(errno));
564 return false;
565 }
566 if (g_managedTorPid == 0)
567 {
568 execlp(torPath.c_str(), torPath.c_str(), "-f", torrcPath.c_str(), (char*)NULL);
569 _exit(127);
570 }
571 return true;
572#endif
573}
574
575static bool ManagedTorProcessAlive(bool& fExited, int& nExitCode)
576{
577 fExited = false;
578 nExitCode = 0;
579#ifdef _WIN32
580 HANDLE hProcess = NULL;
581 CRITICAL_BLOCK(cs_managedTor)
582 hProcess = g_managedTorProcess.hProcess;
583 if (!hProcess)
584 return false;
585 DWORD code = 0;
586 if (!GetExitCodeProcess(hProcess, &code))
587 return false;
588 if (code != STILL_ACTIVE)
589 {
590 fExited = true;
591 nExitCode = (int)code;
592 return false;
593 }
594 return true;
595#else
596 pid_t pid = -1;
597 CRITICAL_BLOCK(cs_managedTor)
598 pid = g_managedTorPid;
599 if (pid <= 0)
600 return false;
601 int status = 0;
602 pid_t r = waitpid(pid, &status, WNOHANG);
603 if (r == 0)
604 return true;
605 if (r == pid)
606 {
607 fExited = true;
608 if (WIFEXITED(status))
609 nExitCode = WEXITSTATUS(status);
610 else if (WIFSIGNALED(status))
611 nExitCode = 128 + WTERMSIG(status);
612 CRITICAL_BLOCK(cs_managedTor)
613 if (g_managedTorPid == pid)
614 g_managedTorPid = -1;
615 }
616 return false;
617#endif
618}
619
620static bool ValidateManagedTorStartup(std::string& errOut)
621{
622 Sleep(500);
623 bool fExited = false;
624 int nExitCode = 0;
625 if (ManagedTorProcessAlive(fExited, nExitCode))
626 return true;
627 if (fExited)
628 errOut = strprintf("Tor exited during startup with code %d", nExitCode);
629 else
630 errOut = "Tor process is not running after startup";
631 return false;
632}
633
634static void MarkManagedTorDead(int nExitCode)
635{
636 BtfStopManagedTor();
637 CRITICAL_BLOCK(cs_managedTor)
638 g_managedTorStatus = strprintf("stopped unexpectedly, Tor exit code %d", nExitCode);
639}
640
641static void ThreadManagedTorWatchdog(void*)
642{
643 for (;;)
644 {
645 Sleep(5000);
646 CRITICAL_BLOCK(cs_managedTor)
647 {
648 if (!g_fManagedTor || g_fManagedTorStopping || fShutdown)
649 return;
650 }
651
652 bool fExited = false;
653 int nExitCode = 0;
654 if (!ManagedTorProcessAlive(fExited, nExitCode))
655 {
656 if (fExited)
657 MarkManagedTorDead(nExitCode);
658 else
659 {
660 BtfStopManagedTor();
661 CRITICAL_BLOCK(cs_managedTor)
662 g_managedTorStatus = "stopped unexpectedly, Tor process handle is gone";
663 }
664 return;
665 }
666 }
667}
668
669static void ThreadManagedTorHostname(void*)
670{
671 std::string hostnamePath;
672 unsigned short p2pPort = 0;
673 CRITICAL_BLOCK(cs_managedTor)
674 {
675 hostnamePath = PathJoin(g_managedTorHiddenServiceDir, "hostname");
676 p2pPort = g_managedTorP2PPort;
677 }
678
679 for (int i = 0; i < 90 && !fShutdown; i++)
680 {
681 std::string onion;
682 if (ReadOneLine(hostnamePath, onion))
683 {
684 std::string endpoint = strprintf("%s:%u", onion.c_str(), (unsigned)p2pPort);
685 std::string err;
686 if (BtfSetLocalOnionEndpoint(endpoint, err))
687 {
688 CRITICAL_BLOCK(cs_managedTor)
689 {
690 g_managedTorOnion = BtfLocalOnionEndpoint();
691 g_managedTorStatus = strprintf("running, SOCKS5 127.0.0.1:%u, onion %s",
692 (unsigned)g_managedTorSocksPort,
693 g_managedTorOnion.c_str());
694 }
695 printf("Managed Tor hidden service ready: %s\n", BtfLocalOnionEndpoint().c_str());
696 return;
697 }
698 CRITICAL_BLOCK(cs_managedTor)
699 g_managedTorStatus = strprintf("running, but generated onion endpoint was rejected: %s",
700 err.c_str());
701 return;
702 }
703 Sleep(1000);
704 }
705
706 BtfStopManagedTor();
707 CRITICAL_BLOCK(cs_managedTor)
708 g_managedTorStatus = "failed: Tor did not create a hidden service hostname within 90s";
709}
710
711bool BtfStartManagedTor(const std::string& torPathOpt, std::string& errOut)
712{
713 CRITICAL_BLOCK(cs_managedTor)
714 {
715 if (g_fManagedTor)
716 {
717 errOut = "managed Tor is already running";
718 return false;
719 }
720 }
721
722 std::string root = PathJoin(GetAppDir(), "managed-tor");
723 std::string dataDir = PathJoin(root, "data");
724 std::string hsDir = PathJoin(root, "onion-service");
725 std::string torrcPath = PathJoin(root, "torrc");
726 if (!EnsureDir(root, errOut) || !EnsureDir(dataDir, errOut) || !EnsureDir(hsDir, errOut))
727 return false;
728
729 unsigned short socksPort = 0;
730 if (!PickLocalPort(socksPort, errOut))
731 return false;
732 unsigned short controlPort = 0;
733 if (!PickLocalPort(controlPort, errOut))
734 return false;
735
736 unsigned short p2pPort = ntohs(nListenPort);
737 std::string torrc = BtfBuildManagedTorrcForTest(dataDir, hsDir, socksPort,
738 controlPort, p2pPort);
739 if (!WriteTextFile(torrcPath, torrc, errOut))
740 return false;
741
742 std::string torPath = ResolveTorPath(torPathOpt);
743 {
744 CRITICAL_BLOCK(cs_managedTor)
745 {
746 g_fManagedTor = true;
747 g_fManagedTorStopping = false;
748 g_managedTorPath = torPath;
749 g_managedTorDataDir = dataDir;
750 g_managedTorHiddenServiceDir = hsDir;
751 g_managedTorSocksPort = socksPort;
752 g_managedTorControlPort = controlPort;
753 g_managedTorP2PPort = p2pPort;
754 g_managedTorOnion.clear();
755 g_managedTorStatus = strprintf("starting %s, SOCKS5 127.0.0.1:%u",
756 torPath.c_str(), (unsigned)socksPort);
757 }
758 }
759
760 if (!StartTorProcess(torPath, torrcPath, errOut))
761 {
762 CRITICAL_BLOCK(cs_managedTor)
763 {
764 g_fManagedTor = false;
765 g_managedTorStatus = strprintf("failed: %s", errOut.c_str());
766 }
767 return false;
768 }
769 if (!ValidateManagedTorStartup(errOut))
770 {
771 BtfStopManagedTor();
772 CRITICAL_BLOCK(cs_managedTor)
773 g_managedTorStatus = strprintf("failed: %s", errOut.c_str());
774 return false;
775 }
776
777 std::string proxyErr;
778 if (!BtfEnableTorProxy(strprintf("127.0.0.1:%u", (unsigned)socksPort), proxyErr))
779 {
780 BtfStopManagedTor();
781 errOut = strprintf("could not enable managed Tor SOCKS5 proxy: %s", proxyErr.c_str());
782 return false;
783 }
784
785 if (_beginthread(ThreadManagedTorHostname, 0, NULL) == (uintptr_t)-1)
786 {
787 BtfStopManagedTor();
788 errOut = "could not start managed Tor hostname watcher";
789 return false;
790 }
791 if (_beginthread(ThreadManagedTorWatchdog, 0, NULL) == (uintptr_t)-1)
792 {
793 BtfStopManagedTor();
794 errOut = "could not start managed Tor watchdog";
795 return false;
796 }
797
798 return true;
799}
800
801void BtfStopManagedTor()
802{
803 bool fWasRunning = false;
804 CRITICAL_BLOCK(cs_managedTor)
805 {
806 fWasRunning = g_fManagedTor;
807 if (!fWasRunning)
808 return;
809 g_fManagedTorStopping = true;
810 g_fManagedTor = false;
811 g_managedTorStatus = "stopping";
812 }
813#ifdef _WIN32
814 if (g_managedTorProcess.hProcess)
815 {
816 std::string shutdownErr;
817 bool fGraceful = RequestManagedTorShutdown(shutdownErr);
818 DWORD waitResult = WaitForSingleObject(g_managedTorProcess.hProcess,
819 fGraceful ? 10000 : 1000);
820 if (waitResult != WAIT_OBJECT_0)
821 {
822 TerminateProcess(g_managedTorProcess.hProcess, 0);
823 WaitForSingleObject(g_managedTorProcess.hProcess, 5000);
824 }
825 CloseHandle(g_managedTorProcess.hProcess);
826 CloseHandle(g_managedTorProcess.hThread);
827 memset(&g_managedTorProcess, 0, sizeof(g_managedTorProcess));
828 }
829#else
830 // Snapshot the pid under the lock. The watchdog's ManagedTorProcessAlive
831 // reaps the same child concurrently and resets g_managedTorPid under
832 // cs_managedTor; reading the global unlocked here could leave us waiting
833 // on ECHILD and then SIGKILLing a recycled PID.
834 pid_t pid = -1;
835 CRITICAL_BLOCK(cs_managedTor)
836 pid = g_managedTorPid;
837 if (pid > 0)
838 {
839 std::string shutdownErr;
840 if (!RequestManagedTorShutdown(shutdownErr))
841 kill(pid, SIGTERM);
842 bool fExited = false;
843 for (int i = 0; i < 100; i++)
844 {
845 int status = 0;
846 pid_t r = waitpid(pid, &status, WNOHANG);
847 // ECHILD means the watchdog already reaped it -- that is exit,
848 // not a reason to signal anything.
849 if (r == pid || (r == -1 && errno == ECHILD))
850 {
851 fExited = true;
852 break;
853 }
854 Sleep(100);
855 }
856 if (!fExited)
857 {
858 kill(pid, SIGKILL);
859 waitpid(pid, NULL, 0);
860 }
861 CRITICAL_BLOCK(cs_managedTor)
862 if (g_managedTorPid == pid)
863 g_managedTorPid = -1;
864 }
865#endif
866 CRITICAL_BLOCK(cs_managedTor)
867 {
868 g_managedTorStatus = "stopped";
869 g_managedTorOnion.clear();
870 g_managedTorSocksPort = 0;
871 g_managedTorControlPort = 0;
872 g_managedTorP2PPort = 0;
873 }
874}
875
876bool BtfManagedTorEnabled()
877{
878 CRITICAL_BLOCK(cs_managedTor)
879 return g_fManagedTor;
880 return false;
881}
882
883std::string BtfManagedTorStatus()
884{
885 CRITICAL_BLOCK(cs_managedTor)
886 {
887 if (!g_fManagedTor && g_managedTorStatus.empty())
888 return "disabled";
889 return g_managedTorStatus;
890 }
891 return "disabled";
892}
diff --git a/src/tor.h b/src/tor.h
new file mode 100644
index 0000000..cfb87a5
--- /dev/null
+++ b/src/tor.h
@@ -0,0 +1,24 @@
1// Copyright (c) 2026 Bitflash developers
2// Distributed under the MIT/X11 software license.
3//
4// Managed Tor process support. This is transport bootstrapping only: it does
5// not touch consensus, wallet keys, mining, or the .btf identity.
6
7#ifndef BITFLASH_TOR_H
8#define BITFLASH_TOR_H
9
10#include <string>
11
12bool BtfStartManagedTor(const std::string& torPathOpt, std::string& errOut);
13void BtfStopManagedTor();
14bool BtfManagedTorEnabled();
15std::string BtfManagedTorStatus();
16bool BtfBundledTorPath(std::string& torPathOut);
17
18std::string BtfBuildManagedTorrcForTest(const std::string& dataDir,
19 const std::string& hiddenServiceDir,
20 unsigned short socksPort,
21 unsigned short controlPort,
22 unsigned short p2pPort);
23
24#endif
diff --git a/src/util.cpp b/src/util.cpp
index d52ca88..ab29178 100644
--- a/src/util.cpp
+++ b/src/util.cpp
@@ -7,6 +7,12 @@
7 7
8 8
9bool fDebug = false; 9bool fDebug = false;
10bool fDebugRelay = false;
11
12// In-process cap for debug.log, in bytes. Default matches the 1G trigger of
13// the shipped logrotate policy so a long-running node stays bounded even
14// without it; -maxdebugfilesize=N (MB) tunes it, 0 disables.
15int64 nMaxDebugFileSize = 1024 * 1024 * (int64)1024;
10 16
11// --------------------------------------------------------------------------- 17// ---------------------------------------------------------------------------
12// Categorized, burst-collapsing logging -- see the LogPrint doc comment in 18// Categorized, burst-collapsing logging -- see the LogPrint doc comment in
@@ -26,10 +32,22 @@ static bool LogCategoryDefaultOn(const char* category)
26 strcmp(category, "payout") == 0; 32 strcmp(category, "payout") == 0;
27} 33}
28 34
35// Categories so noisy they drown out everything else even under -debug:
36// per-message send traces, wallet relay rebroadcasts, raw CBlock dumps and
37// per-address discovery lines made up ~90% of a 13-hour production capture
38// (issue #35). They need /debugrelay, not merely /debug.
39static bool LogCategoryVerboseOnly(const char* category)
40{
41 return strcmp(category, "relay") == 0 ||
42 strcmp(category, "nostrdiscovery") == 0;
43}
44
29bool LogAcceptsCategory(const char* category) 45bool LogAcceptsCategory(const char* category)
30{ 46{
31 if (LogCategoryDefaultOn(category)) 47 if (LogCategoryDefaultOn(category))
32 return true; 48 return true;
49 if (LogCategoryVerboseOnly(category))
50 return fDebugRelay;
33 return fDebug; 51 return fDebug;
34} 52}
35 53
@@ -47,6 +65,61 @@ struct LogDedupState {
47static std::map<std::string, LogDedupState> g_logDedup; 65static std::map<std::string, LogDedupState> g_logDedup;
48static CCriticalSection cs_logDedup; 66static CCriticalSection cs_logDedup;
49 67
68// A pending "(N similar)" summary, lifted out of the table while the lock is
69// held and written after it is released.
70struct LogDedupFlush {
71 std::string category;
72 int suppressed;
73 int64 elapsed; // how long the window that counted them ran
74 std::string lastMsg;
75};
76
77static int64 g_logDedupLastSweep = 0;
78
79static void EmitDedupSummary(const LogDedupFlush& f)
80{
81 OutputDebugStringF("[%s] (%d similar in %llds, last: %s)\n",
82 f.category.c_str(), f.suppressed,
83 (long long)f.elapsed, f.lastMsg.c_str());
84}
85
86// A window is otherwise only closed out when its own call site fires again, so
87// a burst that simply stops -- 100 ledger entries voided in one pass and then
88// silence -- leaves its count sitting in the table and the summary is never
89// written at all. That is how 99 of 100 voids went unreported, with no "(N
90// similar)" line to say anything was missing (issue #3). Sweep every expired
91// window instead, so a finished burst is summarised as soon as anything at all
92// is logged.
93//
94// Expired entries are erased rather than reopened: the window is over, so there
95// is no state left worth keeping, and the next call from that site then takes
96// the "first time" path and prints directly -- exactly what the rollover branch
97// below would have done. It also stops the table growing without bound.
98//
99// Caller holds cs_logDedup; the summaries are written after it is released.
100static void SweepExpiredWindowsLocked(int64 now, std::vector<LogDedupFlush>& out)
101{
102 for (std::map<std::string, LogDedupState>::iterator it = g_logDedup.begin();
103 it != g_logDedup.end(); )
104 {
105 if (now - it->second.windowStart < DEDUP_WINDOW_SECS)
106 {
107 ++it;
108 continue;
109 }
110 if (it->second.suppressed > 0)
111 {
112 LogDedupFlush f;
113 f.category = it->first.substr(0, it->first.find('|'));
114 f.suppressed = it->second.suppressed;
115 f.elapsed = now - it->second.windowStart;
116 f.lastMsg = it->second.lastMsg;
117 out.push_back(f);
118 }
119 g_logDedup.erase(it++);
120 }
121}
122
50void LogPrintDedup(const char* category, const char* file, int line, const std::string& msgIn) 123void LogPrintDedup(const char* category, const char* file, int line, const std::string& msgIn)
51{ 124{
52 // Call sites follow the existing printf() convention of ending their 125 // Call sites follow the existing printf() convention of ending their
@@ -61,11 +134,19 @@ void LogPrintDedup(const char* category, const char* file, int line, const std::
61 134
62 int64 now = GetTime(); 135 int64 now = GetTime();
63 bool fEmitNow = false; 136 bool fEmitNow = false;
64 int suppressedToFlush = 0; 137 std::vector<LogDedupFlush> flushes;
65 std::string flushedMsg;
66 138
67 CRITICAL_BLOCK(cs_logDedup) 139 CRITICAL_BLOCK(cs_logDedup)
68 { 140 {
141 // At most once a second. The table holds one entry per call site that
142 // has fired in the last few seconds, so the walk is short, but it must
143 // not run once per line of a burst.
144 if (now != g_logDedupLastSweep)
145 {
146 g_logDedupLastSweep = now;
147 SweepExpiredWindowsLocked(now, flushes);
148 }
149
69 std::map<std::string, LogDedupState>::iterator it = g_logDedup.find(key); 150 std::map<std::string, LogDedupState>::iterator it = g_logDedup.find(key);
70 if (it == g_logDedup.end()) 151 if (it == g_logDedup.end())
71 { 152 {
@@ -76,10 +157,18 @@ void LogPrintDedup(const char* category, const char* file, int line, const std::
76 } 157 }
77 else if (now - it->second.windowStart >= DEDUP_WINDOW_SECS) 158 else if (now - it->second.windowStart >= DEDUP_WINDOW_SECS)
78 { 159 {
79 // Window rolled over: flush what was suppressed during it (if 160 // Window rolled over within the same second as the last sweep:
80 // anything), then start a new window with this call printed. 161 // flush what was suppressed during it (if anything), then start a
81 suppressedToFlush = it->second.suppressed; 162 // new window with this call printed.
82 flushedMsg = it->second.lastMsg; 163 if (it->second.suppressed > 0)
164 {
165 LogDedupFlush f;
166 f.category = category;
167 f.suppressed = it->second.suppressed;
168 f.elapsed = now - it->second.windowStart;
169 f.lastMsg = it->second.lastMsg;
170 flushes.push_back(f);
171 }
83 it->second.windowStart = now; 172 it->second.windowStart = now;
84 it->second.suppressed = 0; 173 it->second.suppressed = 0;
85 it->second.lastMsg = msg; 174 it->second.lastMsg = msg;
@@ -94,13 +183,23 @@ void LogPrintDedup(const char* category, const char* file, int line, const std::
94 } 183 }
95 } 184 }
96 185
97 if (suppressedToFlush > 0) 186 for (size_t i = 0; i < flushes.size(); i++)
98 OutputDebugStringF("[%s] (%d similar in %llds, last: %s)\n", 187 EmitDedupSummary(flushes[i]);
99 category, suppressedToFlush, (long long)DEDUP_WINDOW_SECS, flushedMsg.c_str());
100 if (fEmitNow) 188 if (fEmitNow)
101 OutputDebugStringF("[%s] %s\n", category, msg.c_str()); 189 OutputDebugStringF("[%s] %s\n", category, msg.c_str());
102} 190}
103 191
192// LogPrintAlways() -- see the doc comment in util.h. No window, no counting,
193// no table entry: every call writes its own line.
194void LogPrintNoCollapse(const char* category, const std::string& msgIn)
195{
196 std::string msg = msgIn;
197 while (!msg.empty() && (msg.back() == '\n' || msg.back() == '\r'))
198 msg.pop_back();
199
200 OutputDebugStringF("[%s] %s\n", category, msg.c_str());
201}
202
104 203
105 204
106 205
@@ -337,7 +436,7 @@ string FormatMoney(int64 n, bool fPlus)
337bool ParseMoney(const char* pszIn, int64& nRet) 436bool ParseMoney(const char* pszIn, int64& nRet)
338{ 437{
339 string strWhole; 438 string strWhole;
340 int64 nCents = 0; 439 int64 nUnits = 0; // fractional part, scaled to COIN (8 decimals)
341 const char* p = pszIn; 440 const char* p = pszIn;
342 while (isspace(*p)) 441 while (isspace(*p))
343 p++; 442 p++;
@@ -348,12 +447,16 @@ bool ParseMoney(const char* pszIn, int64& nRet)
348 if (*p == '.') 447 if (*p == '.')
349 { 448 {
350 p++; 449 p++;
351 if (!isdigit(p[0]) || !isdigit(p[1])) 450 int64 nMult = COIN / 10; // weight of the first decimal digit
352 return false; 451 while (isdigit(*p) && nMult > 0)
353 nCents = atoi64(p); 452 {
354 if (nCents < 0 || nCents > 99) 453 nUnits += nMult * (*p - '0');
454 nMult /= 10;
455 p++;
456 }
457 // reject more than 8 fractional digits rather than silently truncate
458 if (isdigit(*p))
355 return false; 459 return false;
356 p += 2;
357 break; 460 break;
358 } 461 }
359 if (isspace(*p)) 462 if (isspace(*p))
@@ -365,13 +468,12 @@ bool ParseMoney(const char* pszIn, int64& nRet)
365 for (; *p; p++) 468 for (; *p; p++)
366 if (!isspace(*p)) 469 if (!isspace(*p))
367 return false; 470 return false;
368 if (strWhole.size() > 17) 471 if (strWhole.size() > 10) // 10 whole digits * COIN stays within int64
369 return false; 472 return false;
370 int64 nWhole = atoi64(strWhole); 473 int64 nWhole = atoi64(strWhole);
371 int64 nValue = nWhole * 100 + nCents; 474 int64 nValue = nWhole * COIN + nUnits;
372 if (nValue / 100 != nWhole) 475 if (nValue < 0 || (nValue - nUnits) / COIN != nWhole)
373 return false; 476 return false;
374 nValue *= CENT;
375 nRet = nValue; 477 nRet = nValue;
376 return true; 478 return true;
377} 479}
@@ -394,13 +496,37 @@ bool FileExists(const char* psz)
394#endif 496#endif
395} 497}
396 498
397int GetFilesize(FILE* file) 499#ifndef _WIN32
500// If off_t is 32 bits wide, ftello() truncates exactly like the int this
501// function used to return. Fail the build rather than ship the silent
502// version of the bug (32-bit hosts need -D_FILE_OFFSET_BITS=64).
503static_assert(sizeof(off_t) >= 8, "GetFilesize needs a 64-bit off_t");
504#endif
505
506// Returns the file's size in bytes, or -1 if it cannot be determined.
507//
508// This used to hold ftell()'s long in an int, so a file of 2 GiB or more came
509// back negative. Its only caller is the debug.log roll in util.h, which skips
510// the roll on a negative size -- i.e. rotation switched itself off for good at
511// exactly the size the roll exists to prevent, and any cap of 2048 MB or more
512// could never be exceeded at all. Both failures were silent.
513int64 GetFilesize(FILE* file)
398{ 514{
399 int nSavePos = ftell(file); 515#ifdef _WIN32
400 int nFilesize = -1; 516 __int64 nSavePos = _ftelli64(file);
401 if (fseek(file, 0, SEEK_END) == 0) 517 int64 nFilesize = -1;
402 nFilesize = ftell(file); 518 if (nSavePos >= 0 && _fseeki64(file, 0, SEEK_END) == 0)
403 fseek(file, nSavePos, SEEK_SET); 519 nFilesize = _ftelli64(file);
520 if (nSavePos >= 0)
521 _fseeki64(file, nSavePos, SEEK_SET);
522#else
523 off_t nSavePos = ftello(file);
524 int64 nFilesize = -1;
525 if (nSavePos >= 0 && fseeko(file, 0, SEEK_END) == 0)
526 nFilesize = ftello(file);
527 if (nSavePos >= 0)
528 fseeko(file, nSavePos, SEEK_SET);
529#endif
404 return nFilesize; 530 return nFilesize;
405} 531}
406 532
diff --git a/src/util.h b/src/util.h
index d72518b..dfe7511 100644
--- a/src/util.h
+++ b/src/util.h
@@ -87,6 +87,15 @@ inline T& REF(const T& val)
87 87
88extern bool fDebug; 88extern bool fDebug;
89 89
90// /debugrelay: the per-message firehose ("relay" and "nostrdiscovery"
91// categories) that even -debug keeps off. See LogAcceptsCategory().
92extern bool fDebugRelay;
93
94// debug.log is rolled to debug.log.old once it grows past this many bytes
95// (set from -maxdebugfilesize, in MB; 0 disables the check). See the roll
96// site in OutputDebugStringF() below.
97extern int64 nMaxDebugFileSize;
98
90// --------------------------------------------------------------------------- 99// ---------------------------------------------------------------------------
91// Categorized, burst-collapsing logging 100// Categorized, burst-collapsing logging
92// 101//
@@ -106,15 +115,29 @@ extern bool fDebug;
106// 2. Burst collapse: repeated LogPrint() calls from the *same source line* 115// 2. Burst collapse: repeated LogPrint() calls from the *same source line*
107// only emit one line per DEDUP_WINDOW_SECS; if more calls land inside 116// only emit one line per DEDUP_WINDOW_SECS; if more calls land inside
108// that window they're counted instead of reprinted, and the count is 117// that window they're counted instead of reprinted, and the count is
109// flushed as a single "(xN similar in Ws)" line the moment the window 118// flushed as a single "(xN similar in Ws)" line once the window expires
110// rolls over. A loop discovering 500 peers by address now produces a 119// -- whether or not that call site ever fires again. A loop discovering
111// small, bounded number of lines instead of 500 -- log growth is capped 120// 500 peers by address now produces a small, bounded number of lines
112// by elapsed time, not by how many similar events happen to fire. 121// instead of 500 -- log growth is capped by elapsed time, not by how
122// many similar events happen to fire.
123//
124// Use LogPrintAlways() instead for anything that reports an irreversible
125// change to money: an obligation voided, a payment sent, a payment that
126// failed. One line standing in for an unknown number of identical events is
127// a tidy report of peer chatter and a false report of a ledger. A reorg
128// voiding 100 entries hits one source line 100 times inside one window;
129// under LogPrint() that read as a single block losing a single 49.49 BTF
130// payment, while 4,950 BTF of obligations were actually destroyed, and the
131// discrepancy was only found by counting entries in pending_payouts.json
132// (issue #3). Bounded log growth is not worth that ambiguity here.
113// --------------------------------------------------------------------------- 133// ---------------------------------------------------------------------------
114bool LogAcceptsCategory(const char* category); 134bool LogAcceptsCategory(const char* category);
115void LogPrintDedup(const char* category, const char* file, int line, const std::string& msg); 135void LogPrintDedup(const char* category, const char* file, int line, const std::string& msg);
136void LogPrintNoCollapse(const char* category, const std::string& msg);
116#define LogPrint(category, ...) \ 137#define LogPrint(category, ...) \
117 do { if (LogAcceptsCategory(category)) LogPrintDedup((category), __FILE__, __LINE__, strprintf(__VA_ARGS__)); } while (0) 138 do { if (LogAcceptsCategory(category)) LogPrintDedup((category), __FILE__, __LINE__, strprintf(__VA_ARGS__)); } while (0)
139#define LogPrintAlways(category, ...) \
140 do { if (LogAcceptsCategory(category)) LogPrintNoCollapse((category), strprintf(__VA_ARGS__)); } while (0)
118 141
119// Every format string in this project goes through one of the four functions 142// Every format string in this project goes through one of the four functions
120// below -- `printf` itself is remapped to OutputDebugStringF just above, and 143// below -- `printf` itself is remapped to OutputDebugStringF just above, and
@@ -145,7 +168,7 @@ void ParseString(const string& str, char c, vector<string>& v);
145string FormatMoney(int64 n, bool fPlus=false); 168string FormatMoney(int64 n, bool fPlus=false);
146bool ParseMoney(const char* pszIn, int64& nRet); 169bool ParseMoney(const char* pszIn, int64& nRet);
147bool FileExists(const char* psz); 170bool FileExists(const char* psz);
148int GetFilesize(FILE* file); 171int64 GetFilesize(FILE* file);
149uint64 GetRand(uint64 nMax); 172uint64 GetRand(uint64 nMax);
150int64 GetTime(); 173int64 GetTime();
151int64 GetAdjustedTime(); 174int64 GetAdjustedTime();
@@ -199,7 +222,7 @@ protected:
199public: 222public:
200 char* pszFile; 223 char* pszFile;
201 int nLine; 224 int nLine;
202 explicit CCriticalSection() { InitializeCriticalSection(&cs); } 225 explicit CCriticalSection() : pszFile(NULL), nLine(0) { InitializeCriticalSection(&cs); }
203 ~CCriticalSection() { DeleteCriticalSection(&cs); } 226 ~CCriticalSection() { DeleteCriticalSection(&cs); }
204 void Enter() { EnterCriticalSection(&cs); } 227 void Enter() { EnterCriticalSection(&cs); }
205 void Leave() { LeaveCriticalSection(&cs); } 228 void Leave() { LeaveCriticalSection(&cs); }
@@ -390,7 +413,65 @@ inline int OutputDebugStringF(const char* pszFormat, ...)
390 else 413 else
391 strDebugFile = "debug.log"; 414 strDebugFile = "debug.log";
392 } 415 }
416 // Latched by a failed rename in the roll below, which cannot be
417 // retried usefully and must not be retried endlessly.
418 static bool fRollDisabled = false;
393 FILE* fileout = fopen(strDebugFile.c_str(), "a"); 419 FILE* fileout = fopen(strDebugFile.c_str(), "a");
420 if (fileout && nMaxDebugFileSize > 0 && !fRollDisabled)
421 {
422 // Roll an overgrown debug.log before adding to it. Nothing in the
423 // tree shrinks this file once the process is running (there is no
424 // ShrinkDebugFile, and even Bitcoin's only ran at startup), so a
425 // node that stays up for months grows the log without bound --
426 // production hit ~2 GiB. The file is opened and closed per line,
427 // so a rename is safe here: the next line simply reopens the
428 // path. One generation is kept as debug.log.old; logrotate
429 // (contrib/logrotate/) handles anything longer-term.
430 int64 nSize = GetFilesize(fileout);
431 if (nSize >= 0 && nSize > nMaxDebugFileSize)
432 {
433 fclose(fileout);
434 string strOldFile = strDebugFile + ".old";
435 remove(strOldFile.c_str()); // rename() won't clobber on Windows
436 bool fRolled = (rename(strDebugFile.c_str(), strOldFile.c_str()) == 0);
437 int nRenameErrno = errno; // before fopen() can overwrite it
438 fileout = fopen(strDebugFile.c_str(), "a");
439 if (fRolled)
440 {
441 if (fileout)
442 fprintf(fileout,
443 "debug.log grew past %lld MB; previous contents moved to debug.log.old (-maxdebugfilesize)\n",
444 (long long)(nMaxDebugFileSize / 1024 / 1024));
445 }
446 else if (nRenameErrno == ENOENT)
447 {
448 // The file went away between the size check and the
449 // rename, so an external rotator (logrotate) got there
450 // first. Nothing left to roll and nothing wrong: the
451 // fopen above has already recreated it. Latching on this
452 // would let a benign race with logrotate switch the cap
453 // off for the rest of the run.
454 }
455 else
456 {
457 // The file is still over the cap, so this branch would be
458 // reached again on the next line and every line after it.
459 // Unlatched, the complaint about failing to roll becomes
460 // the runaway growth the roll exists to prevent. Say it
461 // once and stop trying -- on stderr too, because whatever
462 // blocked the rename (read-only mount, permissions, EXDEV
463 // across a bind mount) may equally block the log write.
464 fRollDisabled = true;
465 fprintf(stderr,
466 "cannot roll %s (rename to .old failed: %s); -maxdebugfilesize disabled for this run\n",
467 strDebugFile.c_str(), strerror(nRenameErrno));
468 if (fileout)
469 fprintf(fileout,
470 "cannot roll debug.log (rename to .old failed: %s); -maxdebugfilesize disabled for this run\n",
471 strerror(nRenameErrno));
472 }
473 }
474 }
394 if (fileout) 475 if (fileout)
395 { 476 {
396 va_list arg_ptr; 477 va_list arg_ptr;
diff --git a/src/wallet_sqlite.cpp b/src/wallet_sqlite.cpp
new file mode 100644
index 0000000..f4b4b37
--- /dev/null
+++ b/src/wallet_sqlite.cpp
@@ -0,0 +1,374 @@
1// Copyright (c) 2026 Bitflash developers
2// Distributed under the MIT/X11 software license.
3
4#include "headers.h"
5#include "wallet_sqlite.h"
6
7static void SetSQLiteError(sqlite3* pdb, const string& strContext,
8 string& strError)
9{
10 if (pdb)
11 strError = strContext + ": " + sqlite3_errmsg(pdb);
12 else
13 strError = strContext;
14}
15
16CWalletDBSQLite::CWalletDBSQLite() : pdb(NULL)
17{
18}
19
20CWalletDBSQLite::~CWalletDBSQLite()
21{
22 Close();
23}
24
25bool CWalletDBSQLite::Open(const string& strPath, string& strError)
26{
27 Close();
28
29 int ret = sqlite3_open_v2(strPath.c_str(), &pdb,
30 SQLITE_OPEN_READWRITE |
31 SQLITE_OPEN_CREATE |
32 SQLITE_OPEN_FULLMUTEX,
33 NULL);
34 if (ret != SQLITE_OK)
35 {
36 SetSQLiteError(pdb, "cannot open SQLite wallet", strError);
37 Close();
38 return false;
39 }
40
41 sqlite3_busy_timeout(pdb, 5000);
42
43 if (!Exec("PRAGMA journal_mode=WAL;", strError))
44 {
45 Close();
46 return false;
47 }
48 if (!Exec("PRAGMA synchronous=FULL;", strError))
49 {
50 Close();
51 return false;
52 }
53 if (!InitSchema(strError))
54 {
55 Close();
56 return false;
57 }
58 return true;
59}
60
61bool CWalletDBSQLite::OpenReadOnly(const string& strPath, string& strError)
62{
63 Close();
64
65 int ret = sqlite3_open_v2(strPath.c_str(), &pdb,
66 SQLITE_OPEN_READONLY |
67 SQLITE_OPEN_FULLMUTEX,
68 NULL);
69 if (ret != SQLITE_OK)
70 {
71 SetSQLiteError(pdb, "cannot open SQLite wallet read-only", strError);
72 Close();
73 return false;
74 }
75
76 sqlite3_busy_timeout(pdb, 5000);
77 return true;
78}
79
80void CWalletDBSQLite::Close()
81{
82 if (pdb)
83 {
84 sqlite3_close(pdb);
85 pdb = NULL;
86 }
87}
88
89bool CWalletDBSQLite::BeginTransaction(string& strError)
90{
91 if (!pdb)
92 {
93 strError = "SQLite wallet is not open";
94 return false;
95 }
96 return Exec("BEGIN IMMEDIATE;", strError);
97}
98
99bool CWalletDBSQLite::CommitTransaction(string& strError)
100{
101 if (!pdb)
102 {
103 strError = "SQLite wallet is not open";
104 return false;
105 }
106 return Exec("COMMIT;", strError);
107}
108
109bool CWalletDBSQLite::RollbackTransaction(string& strError)
110{
111 if (!pdb)
112 {
113 strError = "SQLite wallet is not open";
114 return false;
115 }
116 return Exec("ROLLBACK;", strError);
117}
118
119bool CWalletDBSQLite::Checkpoint(string& strError)
120{
121 if (!pdb)
122 {
123 strError = "SQLite wallet is not open";
124 return false;
125 }
126 return Exec("PRAGMA wal_checkpoint(TRUNCATE);", strError);
127}
128
129bool CWalletDBSQLite::Exec(const char* pszSql, string& strError)
130{
131 char* pszErr = NULL;
132 int ret = sqlite3_exec(pdb, pszSql, NULL, NULL, &pszErr);
133 if (ret != SQLITE_OK)
134 {
135 if (pszErr)
136 {
137 strError = pszErr;
138 sqlite3_free(pszErr);
139 }
140 else
141 SetSQLiteError(pdb, "SQLite exec failed", strError);
142 return false;
143 }
144 return true;
145}
146
147bool CWalletDBSQLite::InitSchema(string& strError)
148{
149 return Exec("CREATE TABLE IF NOT EXISTS wallet_records ("
150 "key BLOB PRIMARY KEY NOT NULL,"
151 "value BLOB NOT NULL"
152 ");",
153 strError) &&
154 Exec("PRAGMA user_version=1;", strError);
155}
156
157bool CWalletDBSQLite::WriteRecord(const vector<unsigned char>& vchKey,
158 const vector<unsigned char>& vchValue,
159 string& strError,
160 bool fOverwrite)
161{
162 if (!pdb)
163 {
164 strError = "SQLite wallet is not open";
165 return false;
166 }
167 if (vchKey.empty())
168 {
169 strError = "SQLite wallet record key is empty";
170 return false;
171 }
172
173 sqlite3_stmt* stmt = NULL;
174 const char* pszSql = fOverwrite
175 ? "INSERT OR REPLACE INTO wallet_records(key, value) VALUES(?, ?);"
176 : "INSERT INTO wallet_records(key, value) VALUES(?, ?);";
177 int ret = sqlite3_prepare_v2(pdb, pszSql, -1, &stmt, NULL);
178 if (ret != SQLITE_OK)
179 {
180 SetSQLiteError(pdb, "cannot prepare SQLite wallet write", strError);
181 return false;
182 }
183
184 ret = sqlite3_bind_blob(stmt, 1, &vchKey[0], (int)vchKey.size(),
185 SQLITE_TRANSIENT);
186 if (ret == SQLITE_OK)
187 ret = sqlite3_bind_blob(stmt, 2,
188 vchValue.empty() ? NULL : &vchValue[0],
189 (int)vchValue.size(),
190 SQLITE_TRANSIENT);
191 if (ret == SQLITE_OK)
192 ret = sqlite3_step(stmt);
193
194 bool fOk = (ret == SQLITE_DONE);
195 if (!fOk)
196 SetSQLiteError(pdb, "cannot write SQLite wallet record", strError);
197 sqlite3_finalize(stmt);
198 return fOk;
199}
200
201bool CWalletDBSQLite::ReadRecord(const vector<unsigned char>& vchKey,
202 vector<unsigned char>& vchValueRet,
203 string& strError)
204{
205 vchValueRet.clear();
206 if (!pdb)
207 {
208 strError = "SQLite wallet is not open";
209 return false;
210 }
211 if (vchKey.empty())
212 {
213 strError = "SQLite wallet record key is empty";
214 return false;
215 }
216
217 sqlite3_stmt* stmt = NULL;
218 const char* pszSql = "SELECT value FROM wallet_records WHERE key = ?;";
219 int ret = sqlite3_prepare_v2(pdb, pszSql, -1, &stmt, NULL);
220 if (ret != SQLITE_OK)
221 {
222 SetSQLiteError(pdb, "cannot prepare SQLite wallet read", strError);
223 return false;
224 }
225
226 ret = sqlite3_bind_blob(stmt, 1, &vchKey[0], (int)vchKey.size(),
227 SQLITE_TRANSIENT);
228 if (ret != SQLITE_OK)
229 {
230 SetSQLiteError(pdb, "cannot bind SQLite wallet key", strError);
231 sqlite3_finalize(stmt);
232 return false;
233 }
234
235 ret = sqlite3_step(stmt);
236 if (ret == SQLITE_ROW)
237 {
238 const unsigned char* pch =
239 (const unsigned char*)sqlite3_column_blob(stmt, 0);
240 int nBytes = sqlite3_column_bytes(stmt, 0);
241 if (nBytes > 0 && pch)
242 vchValueRet.assign(pch, pch + nBytes);
243 sqlite3_finalize(stmt);
244 return true;
245 }
246
247 if (ret == SQLITE_DONE)
248 strError = "SQLite wallet record not found";
249 else
250 SetSQLiteError(pdb, "cannot read SQLite wallet record", strError);
251 sqlite3_finalize(stmt);
252 return false;
253}
254
255bool CWalletDBSQLite::EraseRecord(const vector<unsigned char>& vchKey,
256 string& strError)
257{
258 if (!pdb)
259 {
260 strError = "SQLite wallet is not open";
261 return false;
262 }
263 if (vchKey.empty())
264 {
265 strError = "SQLite wallet record key is empty";
266 return false;
267 }
268
269 sqlite3_stmt* stmt = NULL;
270 const char* pszSql = "DELETE FROM wallet_records WHERE key = ?;";
271 int ret = sqlite3_prepare_v2(pdb, pszSql, -1, &stmt, NULL);
272 if (ret != SQLITE_OK)
273 {
274 SetSQLiteError(pdb, "cannot prepare SQLite wallet erase", strError);
275 return false;
276 }
277
278 ret = sqlite3_bind_blob(stmt, 1, &vchKey[0], (int)vchKey.size(),
279 SQLITE_TRANSIENT);
280 if (ret == SQLITE_OK)
281 ret = sqlite3_step(stmt);
282
283 bool fOk = (ret == SQLITE_DONE);
284 if (!fOk)
285 SetSQLiteError(pdb, "cannot erase SQLite wallet record", strError);
286 sqlite3_finalize(stmt);
287 return fOk;
288}
289
290bool CWalletDBSQLite::ScanRecords(CWalletRecordVisitor& visitor,
291 string& strError)
292{
293 if (!pdb)
294 {
295 strError = "SQLite wallet is not open";
296 return false;
297 }
298
299 sqlite3_stmt* stmt = NULL;
300 const char* pszSql = "SELECT key, value FROM wallet_records ORDER BY rowid;";
301 int ret = sqlite3_prepare_v2(pdb, pszSql, -1, &stmt, NULL);
302 if (ret != SQLITE_OK)
303 {
304 SetSQLiteError(pdb, "cannot prepare SQLite wallet scan", strError);
305 return false;
306 }
307
308 while ((ret = sqlite3_step(stmt)) == SQLITE_ROW)
309 {
310 const unsigned char* pchKey =
311 (const unsigned char*)sqlite3_column_blob(stmt, 0);
312 int nKeyBytes = sqlite3_column_bytes(stmt, 0);
313 const unsigned char* pchValue =
314 (const unsigned char*)sqlite3_column_blob(stmt, 1);
315 int nValueBytes = sqlite3_column_bytes(stmt, 1);
316
317 vector<unsigned char> vchKey;
318 vector<unsigned char> vchValue;
319 if (nKeyBytes > 0 && pchKey)
320 vchKey.assign(pchKey, pchKey + nKeyBytes);
321 if (nValueBytes > 0 && pchValue)
322 vchValue.assign(pchValue, pchValue + nValueBytes);
323
324 CDataStream ssKey(vchKey, SER_DISK);
325 CDataStream ssValue(vchValue, SER_DISK);
326 if (!visitor.VisitWalletRecord(ssKey, ssValue, strError))
327 {
328 sqlite3_finalize(stmt);
329 return false;
330 }
331 }
332
333 if (ret != SQLITE_DONE)
334 {
335 SetSQLiteError(pdb, "cannot scan SQLite wallet records", strError);
336 sqlite3_finalize(stmt);
337 return false;
338 }
339
340 sqlite3_finalize(stmt);
341 return true;
342}
343
344bool CWalletDBSQLite::CountRecords(int& nRecordsRet, string& strError)
345{
346 nRecordsRet = 0;
347 if (!pdb)
348 {
349 strError = "SQLite wallet is not open";
350 return false;
351 }
352
353 sqlite3_stmt* stmt = NULL;
354 int ret = sqlite3_prepare_v2(pdb,
355 "SELECT COUNT(*) FROM wallet_records;",
356 -1, &stmt, NULL);
357 if (ret != SQLITE_OK)
358 {
359 SetSQLiteError(pdb, "cannot prepare SQLite wallet count", strError);
360 return false;
361 }
362
363 ret = sqlite3_step(stmt);
364 if (ret != SQLITE_ROW)
365 {
366 SetSQLiteError(pdb, "cannot count SQLite wallet records", strError);
367 sqlite3_finalize(stmt);
368 return false;
369 }
370
371 nRecordsRet = sqlite3_column_int(stmt, 0);
372 sqlite3_finalize(stmt);
373 return true;
374}
diff --git a/src/wallet_sqlite.h b/src/wallet_sqlite.h
new file mode 100644
index 0000000..d27e13a
--- /dev/null
+++ b/src/wallet_sqlite.h
@@ -0,0 +1,120 @@
1// Copyright (c) 2026 Bitflash developers
2// Distributed under the MIT/X11 software license.
3
4#ifndef BITFLASH_WALLET_SQLITE_H
5#define BITFLASH_WALLET_SQLITE_H
6
7#include <sqlite3.h>
8
9class CWalletRecordVisitor;
10
11class CWalletDBSQLite
12{
13public:
14 CWalletDBSQLite();
15 ~CWalletDBSQLite();
16
17 bool Open(const string& strPath, string& strError);
18 bool OpenReadOnly(const string& strPath, string& strError);
19 void Close();
20
21 bool BeginTransaction(string& strError);
22 bool CommitTransaction(string& strError);
23 bool RollbackTransaction(string& strError);
24 bool Checkpoint(string& strError);
25
26 bool WriteRecord(const vector<unsigned char>& vchKey,
27 const vector<unsigned char>& vchValue,
28 string& strError,
29 bool fOverwrite=true);
30 bool ReadRecord(const vector<unsigned char>& vchKey,
31 vector<unsigned char>& vchValueRet,
32 string& strError);
33 bool EraseRecord(const vector<unsigned char>& vchKey,
34 string& strError);
35 bool ScanRecords(CWalletRecordVisitor& visitor, string& strError);
36 bool CountRecords(int& nRecordsRet, string& strError);
37
38 template<typename K, typename T>
39 bool WriteTypedRecord(const K& key, const T& value, string& strError,
40 bool fOverwrite=true)
41 {
42 CDataStream ssKey(SER_DISK);
43 ssKey.reserve(1000);
44 ssKey << key;
45
46 CDataStream ssValue(SER_DISK);
47 ssValue.reserve(10000);
48 ssValue << value;
49
50 vector<unsigned char> vchKey(ssKey.begin(), ssKey.end());
51 vector<unsigned char> vchValue(ssValue.begin(), ssValue.end());
52 bool fOk = WriteRecord(vchKey, vchValue, strError, fOverwrite);
53
54 if (!vchKey.empty())
55 memset(&vchKey[0], 0, vchKey.size());
56 if (!vchValue.empty())
57 memset(&vchValue[0], 0, vchValue.size());
58 if (!ssKey.empty())
59 memset(&ssKey[0], 0, ssKey.size());
60 if (!ssValue.empty())
61 memset(&ssValue[0], 0, ssValue.size());
62 return fOk;
63 }
64
65 template<typename K, typename T>
66 bool ReadTypedRecord(const K& key, T& valueRet, string& strError)
67 {
68 CDataStream ssKey(SER_DISK);
69 ssKey.reserve(1000);
70 ssKey << key;
71 vector<unsigned char> vchKey(ssKey.begin(), ssKey.end());
72
73 vector<unsigned char> vchValue;
74 bool fOk = ReadRecord(vchKey, vchValue, strError);
75 if (!vchKey.empty())
76 memset(&vchKey[0], 0, vchKey.size());
77 if (!ssKey.empty())
78 memset(&ssKey[0], 0, ssKey.size());
79 if (!fOk)
80 return false;
81
82 CDataStream ssValue(vchValue, SER_DISK);
83 ssValue >> valueRet;
84 bool fReadOk = !ssValue.fail();
85 if (!vchValue.empty())
86 memset(&vchValue[0], 0, vchValue.size());
87 if (!ssValue.empty())
88 memset(&ssValue[0], 0, ssValue.size());
89 if (!fReadOk)
90 strError = "SQLite wallet record could not be deserialized";
91 return fReadOk;
92 }
93
94 template<typename K>
95 bool EraseTypedRecord(const K& key, string& strError)
96 {
97 CDataStream ssKey(SER_DISK);
98 ssKey.reserve(1000);
99 ssKey << key;
100 vector<unsigned char> vchKey(ssKey.begin(), ssKey.end());
101
102 bool fOk = EraseRecord(vchKey, strError);
103 if (!vchKey.empty())
104 memset(&vchKey[0], 0, vchKey.size());
105 if (!ssKey.empty())
106 memset(&ssKey[0], 0, ssKey.size());
107 return fOk;
108 }
109
110private:
111 sqlite3* pdb;
112
113 bool Exec(const char* pszSql, string& strError);
114 bool InitSchema(string& strError);
115
116 CWalletDBSQLite(const CWalletDBSQLite&);
117 void operator=(const CWalletDBSQLite&);
118};
119
120#endif
diff --git a/src/walletcmd.cpp b/src/walletcmd.cpp
index 06f35a5..f59fc0c 100644
--- a/src/walletcmd.cpp
+++ b/src/walletcmd.cpp
@@ -12,6 +12,7 @@
12#include "headers_core.h" 12#include "headers_core.h"
13#include "bip32.h" 13#include "bip32.h"
14#include "walletcmd.h" 14#include "walletcmd.h"
15#include "wallet_sqlite.h"
15 16
16#undef printf 17#undef printf
17 18
@@ -54,6 +55,37 @@ bool RestoreScanReachedDepth(int nSchema,
54 return nLegacyNext >= nDepth; 55 return nLegacyNext >= nDepth;
55} 56}
56 57
58unsigned int WalletLastUsedPubKeyIndexNext(
59 const std::map<unsigned int, std::vector<unsigned char> >& mapPubKeysByIndex)
60{
61 std::map<std::vector<unsigned char>, unsigned int> mapIndexByPubKey;
62 for (std::map<unsigned int, std::vector<unsigned char> >::const_iterator it =
63 mapPubKeysByIndex.begin();
64 it != mapPubKeysByIndex.end(); ++it)
65 mapIndexByPubKey[it->second] = it->first;
66
67 unsigned int nNext = 0;
68 CRITICAL_BLOCK(cs_mapWallet)
69 {
70 for (map<uint256, CWalletTx>::iterator it = mapWallet.begin();
71 it != mapWallet.end(); ++it)
72 {
73 CWalletTx& wtx = (*it).second;
74 for (int i = 0; i < (int)wtx.vout.size(); i++)
75 {
76 vector<unsigned char> vchPubKey;
77 if (!ExtractPubKey(wtx.vout[i].scriptPubKey, false, vchPubKey))
78 continue;
79 std::map<std::vector<unsigned char>, unsigned int>::const_iterator mi =
80 mapIndexByPubKey.find(vchPubKey);
81 if (mi != mapIndexByPubKey.end() && mi->second + 1 > nNext)
82 nNext = mi->second + 1;
83 }
84 }
85 }
86 return nNext;
87}
88
57int CmdNewPhrase() 89int CmdNewPhrase()
58{ 90{
59 AttachTerminal(); 91 AttachTerminal();
@@ -132,7 +164,10 @@ bool RestoreFromPhrase(const std::string& strMnemonic,
132 // Derive forward in batches, scanning after each, until a whole batch turns 164 // Derive forward in batches, scanning after each, until a whole batch turns
133 // up nothing. Every derived key is written to the wallet before the scan, 165 // up nothing. Every derived key is written to the wallet before the scan,
134 // because the scan asks the wallet what belongs to it. 166 // because the scan asks the wallet what belongs to it.
135 int nTotalDerived = nHDKeySchema == HD_SCHEMA_BIP44 167 const bool fRestoreBIP44 = nHDKeySchema == HD_SCHEMA_BIP44;
168 unsigned int nLegacyScanNext = nHDNext;
169 std::map<unsigned int, vector<unsigned char> > mapLegacyScanPubKeys;
170 int nTotalDerived = fRestoreBIP44
136 ? (int)(nHDReceiveNext + nHDChangeNext) 171 ? (int)(nHDReceiveNext + nHDChangeNext)
137 : (int)nHDNext; 172 : (int)nHDNext;
138 int nStopDepth = max(nMinDepth, RESTORE_MIN_SCAN); 173 int nStopDepth = max(nMinDepth, RESTORE_MIN_SCAN);
@@ -151,7 +186,7 @@ bool RestoreFromPhrase(const std::string& strMnemonic,
151 nWalletBefore = mapWallet.size(); 186 nWalletBefore = mapWallet.size();
152 187
153 std::string strDeriveError; 188 std::string strDeriveError;
154 if (nHDKeySchema == HD_SCHEMA_BIP44) 189 if (fRestoreBIP44)
155 { 190 {
156 for (int i = 0; i < RESTORE_BATCH; i++) 191 for (int i = 0; i < RESTORE_BATCH; i++)
157 { 192 {
@@ -182,15 +217,15 @@ bool RestoreFromPhrase(const std::string& strMnemonic,
182 for (int i = 0; i < RESTORE_BATCH; i++) 217 for (int i = 0; i < RESTORE_BATCH; i++)
183 { 218 {
184 CKey key; 219 CKey key;
185 if (!DeriveHDKey(nHDNext, key, strDeriveError)) 220 if (!DeriveHDKey(nLegacyScanNext, key, strDeriveError))
186 break; 221 break;
187 if (!AddKey(key)) 222 if (!AddKey(key))
188 break; 223 break;
189 nHDNext++; 224 mapLegacyScanPubKeys[nLegacyScanNext] = key.GetPubKey();
225 nLegacyScanNext++;
190 nTotalDerived++; 226 nTotalDerived++;
191 } 227 }
192 nHDKeySchema = nSavedSchema; 228 nHDKeySchema = nSavedSchema;
193 CWalletDB().WriteHDNext(nHDNext);
194 } 229 }
195 else 230 else
196 { 231 {
@@ -209,6 +244,22 @@ bool RestoreFromPhrase(const std::string& strMnemonic,
209 244
210 ScanForWalletTransactions(pindexGenesisBlock); 245 ScanForWalletTransactions(pindexGenesisBlock);
211 246
247 if (fRestoreBIP44)
248 {
249 // The legacy branch is scanned only for compatibility with phrases
250 // created before BIP44. Persist the last used legacy index + 1, not
251 // the lookahead depth. Otherwise a restore with no legacy hits can
252 // leave hdnext at RESTORE_MAX and make future compatibility scans
253 // look exhausted even though no legacy key was ever used.
254 unsigned int nLegacyUsedNext =
255 WalletLastUsedPubKeyIndexNext(mapLegacyScanPubKeys);
256 if (nLegacyUsedNext != nHDNext)
257 {
258 nHDNext = nLegacyUsedNext;
259 CWalletDB().WriteHDNext(nHDNext);
260 }
261 }
262
212 size_t nWalletAfter = 0; 263 size_t nWalletAfter = 0;
213 CRITICAL_BLOCK(cs_mapWallet) 264 CRITICAL_BLOCK(cs_mapWallet)
214 nWalletAfter = mapWallet.size(); 265 nWalletAfter = mapWallet.size();
@@ -239,7 +290,7 @@ bool RestoreFromPhrase(const std::string& strMnemonic,
239 RestoreScanReachedDepth(nHDKeySchema, 290 RestoreScanReachedDepth(nHDKeySchema,
240 nHDReceiveNext, 291 nHDReceiveNext,
241 nHDChangeNext, 292 nHDChangeNext,
242 nHDNext, 293 fRestoreBIP44 ? nLegacyScanNext : nHDNext,
243 nStopDepth)) 294 nStopDepth))
244 break; 295 break;
245 } 296 }
@@ -350,11 +401,16 @@ int CmdRecoveryAudit()
350 int64 nImmatureTotal = audit.nRecoverableImmatureCredit + audit.nLegacyImmatureCredit; 401 int64 nImmatureTotal = audit.nRecoverableImmatureCredit + audit.nLegacyImmatureCredit;
351 402
352 printf("Wallet recovery audit\n"); 403 printf("Wallet recovery audit\n");
353 printf(" recovery phrase: %s\n", audit.fHaveSeed ? "present" : "not installed"); 404 if (audit.fHaveSeed)
405 printf(" recovery phrase: present\n");
406 else if (audit.fSeedEncryptedLocked)
407 printf(" recovery phrase: encrypted, unlock wallet to audit\n");
408 else
409 printf(" recovery phrase: not installed\n");
354 if (audit.fHaveSeed) 410 if (audit.fHaveSeed)
355 { 411 {
356 printf(" derivation schema: %s\n", HDKeySchemaName(audit.nSchema).c_str()); 412 printf(" derivation schema: %s\n", HDKeySchemaName(audit.nSchema).c_str());
357 printf(" BIP44 coin type: %u (provisional BITFLASH)\n", audit.nCoinType); 413 printf(" BIP44 coin type: %u (SLIP-0044 BITFLASH)\n", audit.nCoinType);
358 printf(" derived keys known to this wallet: %u\n", audit.nDerivedKnown); 414 printf(" derived keys known to this wallet: %u\n", audit.nDerivedKnown);
359 printf(" receive/change counters: %u/%u\n", 415 printf(" receive/change counters: %u/%u\n",
360 audit.nReceiveNext, audit.nChangeNext); 416 audit.nReceiveNext, audit.nChangeNext);
@@ -362,15 +418,24 @@ int CmdRecoveryAudit()
362 if (!audit.fDeriveComplete) 418 if (!audit.fDeriveComplete)
363 printf(" derivation warning: %s\n", audit.strDeriveError.c_str()); 419 printf(" derivation warning: %s\n", audit.strDeriveError.c_str());
364 printf(" total spendable balance: %s BTF\n", FormatMoney(nTotal).c_str()); 420 printf(" total spendable balance: %s BTF\n", FormatMoney(nTotal).c_str());
421 if (audit.fSeedEncryptedLocked)
422 {
423 printf(" recovery coverage: unavailable while wallet is locked\n");
424 printf(" immature mining rewards: %s BTF\n", FormatMoney(nImmatureTotal).c_str());
425 printf("\n");
426 printf("Unlock the wallet with /walletpassphrase or /walletpassphrase=@FILE to audit phrase coverage.\n");
427 fflush(stdout);
428 return 2;
429 }
365 printf(" covered by recovery phrase: %s BTF (%d transaction(s))\n", 430 printf(" covered by recovery phrase: %s BTF (%d transaction(s))\n",
366 FormatMoney(audit.nRecoverableCredit).c_str(), audit.nRecoverableTx); 431 FormatMoney(audit.nRecoverableCredit).c_str(), audit.nRecoverableTx);
367 printf(" wallet.dat-only balance: %s BTF (%d transaction(s))\n", 432 printf(" not phrase-backed balance: %s BTF (%d transaction(s))\n",
368 FormatMoney(audit.nLegacyCredit).c_str(), audit.nLegacyTx); 433 FormatMoney(audit.nLegacyCredit).c_str(), audit.nLegacyTx);
369 printf(" immature mining rewards: %s BTF\n", FormatMoney(nImmatureTotal).c_str()); 434 printf(" immature mining rewards: %s BTF\n", FormatMoney(nImmatureTotal).c_str());
370 printf(" phrase-backed immature: %s BTF (%d transaction(s))\n", 435 printf(" phrase-backed immature: %s BTF (%d transaction(s))\n",
371 FormatMoney(audit.nRecoverableImmatureCredit).c_str(), 436 FormatMoney(audit.nRecoverableImmatureCredit).c_str(),
372 audit.nRecoverableImmatureTx); 437 audit.nRecoverableImmatureTx);
373 printf(" wallet.dat-only immature: %s BTF (%d transaction(s))\n", 438 printf(" not phrase-backed immature: %s BTF (%d transaction(s))\n",
374 FormatMoney(audit.nLegacyImmatureCredit).c_str(), 439 FormatMoney(audit.nLegacyImmatureCredit).c_str(),
375 audit.nLegacyImmatureTx); 440 audit.nLegacyImmatureTx);
376 441
@@ -392,7 +457,7 @@ int CmdRecoveryAudit()
392 { 457 {
393 printf("\n"); 458 printf("\n");
394 printf("Warning: some coins are on keys the phrase does not reproduce.\n"); 459 printf("Warning: some coins are on keys the phrase does not reproduce.\n");
395 printf("Keep wallet.dat backups until that balance has been moved to a phrase-backed address.\n"); 460 printf("Keep a file backup of this wallet until that balance has been moved to a phrase-backed address.\n");
396 fflush(stdout); 461 fflush(stdout);
397 return 2; 462 return 2;
398 } 463 }
@@ -406,10 +471,952 @@ int CmdRecoveryAudit()
406 return 0; 471 return 0;
407} 472}
408 473
474struct WalletStorageAuditCounts
475{
476 unsigned int nTotal = 0;
477 unsigned int nMalformed = 0;
478 unsigned int nUnknown = 0;
479 unsigned int nVersion = 0;
480 unsigned int nNames = 0;
481 unsigned int nTransactions = 0;
482 unsigned int nPlainKeys = 0;
483 unsigned int nEncryptedKeys = 0;
484 unsigned int nMasterKeys = 0;
485 unsigned int nDefaultKey = 0;
486 unsigned int nPlainHDMaster = 0;
487 unsigned int nPlainHDChainCode = 0;
488 unsigned int nCryptedHDMaster = 0;
489 unsigned int nCryptedHDChainCode = 0;
490 unsigned int nWalletMinVersion = 0;
491 unsigned int nPool = 0;
492 unsigned int nSettings = 0;
493 unsigned int nHDNext = 0;
494 unsigned int nHDSchema = 0;
495 unsigned int nHDCoinType = 0;
496 unsigned int nHDReceiveNext = 0;
497 unsigned int nHDChangeNext = 0;
498};
499
500static std::string HDSeedStorageState(unsigned int nMaster, unsigned int nChain)
501{
502 if (nMaster == 0 && nChain == 0)
503 return "none";
504 if (nMaster > 0 && nChain > 0)
505 return "complete";
506 return "incomplete";
507}
508
509static void CountWalletStorageType(const std::string& strType,
510 WalletStorageAuditCounts& c)
511{
512 if (strType == "version")
513 c.nVersion++;
514 else if (strType == "name")
515 c.nNames++;
516 else if (strType == "tx")
517 c.nTransactions++;
518 else if (strType == "key")
519 c.nPlainKeys++;
520 else if (strType == "ckey")
521 c.nEncryptedKeys++;
522 else if (strType == "mkey")
523 c.nMasterKeys++;
524 else if (strType == "defaultkey")
525 c.nDefaultKey++;
526 else if (strType == "hdmaster")
527 c.nPlainHDMaster++;
528 else if (strType == "hdchaincode")
529 c.nPlainHDChainCode++;
530 else if (strType == "cryptedhdmaster")
531 c.nCryptedHDMaster++;
532 else if (strType == "cryptedhdchaincode")
533 c.nCryptedHDChainCode++;
534 else if (strType == "walletminversion")
535 c.nWalletMinVersion++;
536 else if (strType == "pool")
537 c.nPool++;
538 else if (strType == "setting")
539 c.nSettings++;
540 else if (strType == "hdnext")
541 c.nHDNext++;
542 else if (strType == "hdschema")
543 c.nHDSchema++;
544 else if (strType == "hdcointype")
545 c.nHDCoinType++;
546 else if (strType == "hdreceivenext")
547 c.nHDReceiveNext++;
548 else if (strType == "hdchangenext")
549 c.nHDChangeNext++;
550 else
551 c.nUnknown++;
552}
553
554static bool ReadWalletStorageCounts(WalletStorageAuditCounts& counts,
555 std::string& strError)
556{
557 class CWalletStorageCountVisitor : public CWalletRecordVisitor
558 {
559 public:
560 WalletStorageAuditCounts& counts;
561
562 explicit CWalletStorageCountVisitor(WalletStorageAuditCounts& countsIn)
563 : counts(countsIn) { }
564
565 bool VisitWalletRecord(const CDataStream& ssKeyIn,
566 const CDataStream& ssValue,
567 std::string& strErrorRet)
568 {
569 (void)ssValue;
570 (void)strErrorRet;
571 counts.nTotal++;
572 try
573 {
574 CDataStream ssKey = ssKeyIn;
575 std::string strType;
576 ssKey >> strType;
577 if (ssKey.fail())
578 {
579 counts.nMalformed++;
580 return true;
581 }
582 CountWalletStorageType(strType, counts);
583 }
584 catch (...)
585 {
586 counts.nMalformed++;
587 }
588 return true;
589 }
590 };
591
592 CWalletStorageCountVisitor visitor(counts);
593 return ScanWalletRecords(visitor, strError);
594}
595
596static std::string WalletStorageAuditJson(const WalletStorageAuditCounts& counts)
597{
598 std::string strJson;
599 strJson += "{\n";
600 strJson += " \"format\": \"bitflash-wallet-storage-audit-v1\",\n";
601 strJson += strprintf(" \"records_total\": %u,\n", counts.nTotal);
602 strJson += strprintf(" \"malformed_records\": %u,\n", counts.nMalformed);
603 strJson += strprintf(" \"unknown_records\": %u,\n", counts.nUnknown);
604 strJson += strprintf(" \"plain_private_keys\": %u,\n", counts.nPlainKeys);
605 strJson += strprintf(" \"encrypted_private_keys\": %u,\n", counts.nEncryptedKeys);
606 strJson += strprintf(" \"encryption_master_keys\": %u,\n", counts.nMasterKeys);
607 strJson += strprintf(" \"default_public_key\": \"%s\",\n",
608 counts.nDefaultKey ? "present" : "none");
609 strJson += strprintf(" \"plain_hd_seed\": \"%s\",\n",
610 HDSeedStorageState(counts.nPlainHDMaster,
611 counts.nPlainHDChainCode).c_str());
612 strJson += strprintf(" \"encrypted_hd_seed\": \"%s\",\n",
613 HDSeedStorageState(counts.nCryptedHDMaster,
614 counts.nCryptedHDChainCode).c_str());
615 strJson += strprintf(" \"wallet_minimum_version\": \"%s\",\n",
616 counts.nWalletMinVersion ? "present" : "none");
617 strJson += strprintf(" \"keypool_entries\": %u,\n", counts.nPool);
618 strJson += strprintf(" \"wallet_transactions\": %u,\n", counts.nTransactions);
619 strJson += strprintf(" \"address_book_labels\": %u,\n", counts.nNames);
620 strJson += strprintf(" \"settings\": %u,\n", counts.nSettings);
621 strJson += strprintf(" \"hd_metadata_records\": %u,\n",
622 counts.nHDNext + counts.nHDSchema + counts.nHDCoinType +
623 counts.nHDReceiveNext + counts.nHDChangeNext);
624 strJson += strprintf(" \"database_version_records\": %u\n", counts.nVersion);
625 strJson += "}\n";
626 return strJson;
627}
628
629static bool WriteAuditTextFile(const std::string& strPath,
630 const std::string& strText,
631 std::string& strError)
632{
633 FILE* pf = fopen(strPath.c_str(), "wb");
634 if (!pf)
635 {
636 strError = "cannot open output file";
637 return false;
638 }
639 size_t nWritten = fwrite(strText.data(), 1, strText.size(), pf);
640 bool fCloseOk = fclose(pf) == 0;
641 if (nWritten != strText.size() || !fCloseOk)
642 {
643 strError = "could not write the complete output file";
644 return false;
645 }
646 return true;
647}
648
649static void AddWalletStorageFailure(std::vector<std::string>& vFailures,
650 const std::string& strFailure)
651{
652 vFailures.push_back(strFailure);
653}
654
655static void BuildWalletStorageSanityFailures(const WalletStorageAuditCounts& counts,
656 std::vector<std::string>& vFailures)
657{
658 bool fPlainHD = counts.nPlainHDMaster > 0 || counts.nPlainHDChainCode > 0;
659 bool fCryptedHD = counts.nCryptedHDMaster > 0 || counts.nCryptedHDChainCode > 0;
660 bool fEncryptedRecords = counts.nEncryptedKeys > 0 ||
661 counts.nMasterKeys > 0 ||
662 fCryptedHD;
663
664 if (counts.nMalformed > 0)
665 AddWalletStorageFailure(vFailures, "wallet.dat contains malformed records");
666 if (counts.nUnknown > 0)
667 AddWalletStorageFailure(vFailures, "wallet.dat contains unknown records");
668 if ((counts.nPlainHDMaster > 0) != (counts.nPlainHDChainCode > 0))
669 AddWalletStorageFailure(vFailures, "plain HD seed is incomplete");
670 if ((counts.nCryptedHDMaster > 0) != (counts.nCryptedHDChainCode > 0))
671 AddWalletStorageFailure(vFailures, "encrypted HD seed is incomplete");
672 if (fPlainHD && fCryptedHD)
673 AddWalletStorageFailure(vFailures, "wallet.dat contains both plain and encrypted HD seed records");
674 if (counts.nDefaultKey > 1)
675 AddWalletStorageFailure(vFailures, "wallet.dat contains more than one default public key record");
676 if (counts.nWalletMinVersion > 1)
677 AddWalletStorageFailure(vFailures, "wallet.dat contains more than one minimum-version record");
678
679 if (fEncryptedRecords)
680 {
681 if (counts.nPlainKeys > 0)
682 AddWalletStorageFailure(vFailures, "encrypted wallet still contains plain private key records");
683 if (fPlainHD)
684 AddWalletStorageFailure(vFailures, "encrypted wallet still contains plain HD seed records");
685 if (counts.nMasterKeys == 0)
686 AddWalletStorageFailure(vFailures, "encrypted wallet has no encryption master key record");
687 if (counts.nWalletMinVersion == 0)
688 AddWalletStorageFailure(vFailures, "encrypted wallet has no minimum-version record");
689 }
690}
691
692int CmdWalletStorageCheck()
693{
694 AttachTerminal();
695
696 WalletStorageAuditCounts counts;
697 std::string strError;
698 if (!ReadWalletStorageCounts(counts, strError))
699 {
700 fprintf(stderr, "Cannot check wallet storage: %s\n", strError.c_str());
701 return 1;
702 }
703
704 std::vector<std::string> vFailures;
705 BuildWalletStorageSanityFailures(counts, vFailures);
706
707 printf("Wallet storage sanity check\n");
708 printf(" records total: %u\n", counts.nTotal);
709 printf(" encrypted records: %s\n",
710 (counts.nEncryptedKeys || counts.nMasterKeys ||
711 counts.nCryptedHDMaster || counts.nCryptedHDChainCode) ? "yes" : "no");
712 printf(" plain private keys: %u\n", counts.nPlainKeys);
713 printf(" encrypted private keys: %u\n", counts.nEncryptedKeys);
714 printf(" plain HD seed: %s\n",
715 HDSeedStorageState(counts.nPlainHDMaster,
716 counts.nPlainHDChainCode).c_str());
717 printf(" encrypted HD seed: %s\n",
718 HDSeedStorageState(counts.nCryptedHDMaster,
719 counts.nCryptedHDChainCode).c_str());
720 if (vFailures.empty())
721 {
722 printf(" storage sanity: ok\n");
723 fflush(stdout);
724 return 0;
725 }
726
727 printf(" storage sanity: failed\n");
728 for (std::vector<std::string>::const_iterator it = vFailures.begin();
729 it != vFailures.end(); ++it)
730 printf(" failure: %s\n", it->c_str());
731 fflush(stdout);
732 return 2;
733}
734
735int CmdWalletStorageAudit(const std::string& strJsonOut)
736{
737 AttachTerminal();
738
739 WalletStorageAuditCounts counts;
740 std::string strError;
741 if (!ReadWalletStorageCounts(counts, strError))
742 {
743 fprintf(stderr, "Cannot audit wallet storage: %s\n", strError.c_str());
744 return 1;
745 }
746
747 if (!strJsonOut.empty())
748 {
749 std::string strWriteError;
750 if (!WriteAuditTextFile(strJsonOut,
751 WalletStorageAuditJson(counts),
752 strWriteError))
753 {
754 fprintf(stderr, "Cannot write wallet storage audit JSON: %s\n",
755 strWriteError.c_str());
756 return 1;
757 }
758 printf("Wallet storage audit written to %s\n", strJsonOut.c_str());
759 fflush(stdout);
760 return counts.nMalformed > 0 ? 2 : 0;
761 }
762
763 printf("Wallet storage audit\n");
764 printf(" records total: %u\n", counts.nTotal);
765 printf(" malformed records: %u\n", counts.nMalformed);
766 printf(" unknown records: %u\n", counts.nUnknown);
767 printf(" plain private keys: %u\n", counts.nPlainKeys);
768 printf(" encrypted private keys: %u\n", counts.nEncryptedKeys);
769 printf(" encryption master keys: %u\n", counts.nMasterKeys);
770 printf(" default public key: %s\n", counts.nDefaultKey ? "present" : "none");
771 printf(" plain HD seed: %s\n",
772 HDSeedStorageState(counts.nPlainHDMaster,
773 counts.nPlainHDChainCode).c_str());
774 printf(" encrypted HD seed: %s\n",
775 HDSeedStorageState(counts.nCryptedHDMaster,
776 counts.nCryptedHDChainCode).c_str());
777 printf(" wallet minimum version: %s\n",
778 counts.nWalletMinVersion ? "present" : "none");
779 printf(" keypool entries: %u\n", counts.nPool);
780 printf(" wallet transactions: %u\n", counts.nTransactions);
781 printf(" address book labels: %u\n", counts.nNames);
782 printf(" settings: %u\n", counts.nSettings);
783 printf(" HD metadata records: %u\n",
784 counts.nHDNext + counts.nHDSchema + counts.nHDCoinType +
785 counts.nHDReceiveNext + counts.nHDChangeNext);
786 printf(" database version records: %u\n", counts.nVersion);
787 fflush(stdout);
788
789 if (counts.nMalformed > 0)
790 return 2;
791 return 0;
792}
793
794static vector<unsigned char> DataStreamBytesLocal(const CDataStream& ss)
795{
796 return vector<unsigned char>(ss.begin(), ss.end());
797}
798
799namespace {
800
801static void RemoveSQLiteExportFiles(const string& strPath)
802{
803 remove(strPath.c_str());
804 remove((strPath + "-wal").c_str());
805 remove((strPath + "-shm").c_str());
806}
807
808static void RemoveFailedRestoreWallet(const string& strWalletPath)
809{
810 DBFlush(true);
811 remove(strWalletPath.c_str());
812}
813
814class CWalletSQLiteExportVisitor : public CWalletRecordVisitor
815{
816public:
817 CWalletDBSQLite& db;
818 int& nCopied;
819
820 CWalletSQLiteExportVisitor(CWalletDBSQLite& dbIn, int& nCopiedIn)
821 : db(dbIn), nCopied(nCopiedIn) { }
822
823 bool VisitWalletRecord(const CDataStream& ssKey,
824 const CDataStream& ssValue,
825 string& strErrorRet)
826 {
827 if (!db.WriteRecord(DataStreamBytesLocal(ssKey),
828 DataStreamBytesLocal(ssValue),
829 strErrorRet))
830 return false;
831 nCopied++;
832 return true;
833 }
834};
835
836class CWalletRecordMapVisitor : public CWalletRecordVisitor
837{
838public:
839 std::map<vector<unsigned char>, vector<unsigned char> >& mapRecords;
840
841 explicit CWalletRecordMapVisitor(
842 std::map<vector<unsigned char>, vector<unsigned char> >& mapRecordsIn)
843 : mapRecords(mapRecordsIn) { }
844
845 bool VisitWalletRecord(const CDataStream& ssKey,
846 const CDataStream& ssValue,
847 string& strErrorRet)
848 {
849 vector<unsigned char> vchKey = DataStreamBytesLocal(ssKey);
850 if (mapRecords.count(vchKey))
851 {
852 strErrorRet = "duplicate serialized wallet key";
853 return false;
854 }
855 mapRecords[vchKey] = DataStreamBytesLocal(ssValue);
856 return true;
857 }
858};
859
860class CWalletRawRestoreDB : public CWalletDB
861{
862public:
863 CWalletRawRestoreDB(const char* pszMode="cr+", bool fTxn=true)
864 : CWalletDB(pszMode, fTxn) { }
865
866 bool WriteRaw(CDataStream ssKey, CDataStream ssValue)
867 {
868 if (!pdb || ssKey.empty())
869 return false;
870
871 Dbt datKey((void*)&ssKey[0], ssKey.size());
872 Dbt datValue(ssValue.empty() ? NULL : (void*)&ssValue[0],
873 ssValue.size());
874 int ret = pdb->put(GetTxn(), &datKey, &datValue, 0);
875
876 memset(datKey.get_data(), 0, datKey.get_size());
877 if (datValue.get_data())
878 memset(datValue.get_data(), 0, datValue.get_size());
879 return ret == 0;
880 }
881};
882
883class CWalletSQLiteRestoreVisitor : public CWalletRecordVisitor
884{
885public:
886 CWalletRawRestoreDB& db;
887 int& nCopied;
888
889 CWalletSQLiteRestoreVisitor(CWalletRawRestoreDB& dbIn, int& nCopiedIn)
890 : db(dbIn), nCopied(nCopiedIn) { }
891
892 bool VisitWalletRecord(const CDataStream& ssKey,
893 const CDataStream& ssValue,
894 string& strErrorRet)
895 {
896 if (!db.WriteRaw(ssKey, ssValue))
897 {
898 strErrorRet = "could not write raw wallet record";
899 return false;
900 }
901 nCopied++;
902 return true;
903 }
904};
905
906class CWalletSQLiteLoadCheckVisitor : public CWalletRecordVisitor
907{
908public:
909 int nRecords;
910 int nUnknown;
911 string strFailure;
912
913 CWalletSQLiteLoadCheckVisitor() : nRecords(0), nUnknown(0) { }
914
915 bool Fail(const string& strMessage, string& strErrorRet)
916 {
917 strFailure = strMessage;
918 strErrorRet = strMessage;
919 return false;
920 }
921
922 bool VisitWalletRecord(const CDataStream& ssKeyIn,
923 const CDataStream& ssValueIn,
924 string& strErrorRet)
925 {
926 nRecords++;
927 try
928 {
929 CDataStream ssKey = ssKeyIn;
930 CDataStream ssValue = ssValueIn;
931 string strType;
932 ssKey >> strType;
933 if (ssKey.fail())
934 return Fail("malformed wallet record key", strErrorRet);
935
936 if (strType == "name")
937 {
938 string strAddress;
939 string strName;
940 ssKey >> strAddress;
941 ssValue >> strName;
942 }
943 else if (strType == "tx")
944 {
945 uint256 hash;
946 CWalletTx wtx;
947 ssKey >> hash;
948 ssValue >> wtx;
949 }
950 else if (strType == "key")
951 {
952 vector<unsigned char> vchPubKey;
953 CPrivKey vchPrivKey;
954 ssKey >> vchPubKey;
955 ssValue >> vchPrivKey;
956 }
957 else if (strType == "mkey")
958 {
959 unsigned int nID = 0;
960 CWalletMasterKey kMasterKey;
961 ssKey >> nID;
962 ssValue >> kMasterKey;
963 }
964 else if (strType == "ckey")
965 {
966 vector<unsigned char> vchPubKey;
967 vector<unsigned char> vchCryptedSecret;
968 ssKey >> vchPubKey;
969 ssValue >> vchCryptedSecret;
970 }
971 else if (strType == "defaultkey" ||
972 strType == "hdmaster" ||
973 strType == "hdchaincode" ||
974 strType == "cryptedhdmaster" ||
975 strType == "cryptedhdchaincode")
976 {
977 vector<unsigned char> vch;
978 ssValue >> vch;
979 }
980 else if (strType == "hdnext" ||
981 strType == "hdreceivenext" ||
982 strType == "hdchangenext" ||
983 strType == "hdcointype")
984 {
985 unsigned int nValue = 0;
986 ssValue >> nValue;
987 }
988 else if (strType == "hdschema")
989 {
990 int nSchema = 0;
991 ssValue >> nSchema;
992 if (!ssValue.fail() &&
993 nSchema != HD_SCHEMA_NONE &&
994 nSchema != HD_SCHEMA_LEGACY &&
995 nSchema != HD_SCHEMA_BIP44)
996 return Fail("unsupported deterministic wallet schema",
997 strErrorRet);
998 }
999 else if (strType == "walletminversion")
1000 {
1001 int nMinVersion = 0;
1002 ssValue >> nMinVersion;
1003 if (!ssValue.fail() && nMinVersion > WALLET_FORMAT_SUPPORTED)
1004 return Fail("unsupported future wallet format", strErrorRet);
1005 }
1006 else if (strType == "pool")
1007 {
1008 int64 nIndex = 0;
1009 vector<unsigned char> vchPubKey;
1010 ssKey >> nIndex;
1011 ssValue >> vchPubKey;
1012 }
1013 else if (strType == "setting")
1014 {
1015 string strKey;
1016 ssKey >> strKey;
1017 if (strKey == "nTransactionFee")
1018 {
1019 int64 nValue = 0;
1020 ssValue >> nValue;
1021 }
1022 else if (strKey == "addrIncoming")
1023 {
1024 CAddress addr;
1025 ssValue >> addr;
1026 }
1027 else if (strKey == "nMineMode")
1028 {
1029 int nValue = 0;
1030 ssValue >> nValue;
1031 }
1032 else if (strKey == "fGenerateBitcoins")
1033 {
1034 int fValue = 0;
1035 ssValue >> fValue;
1036 }
1037 else if (strKey == "strParticipantPool")
1038 {
1039 string strValue;
1040 ssValue >> strValue;
1041 }
1042 }
1043 else if (strType == "version")
1044 {
1045 int nVersion = 0;
1046 ssValue >> nVersion;
1047 }
1048 else
1049 {
1050 nUnknown++;
1051 }
1052
1053 if (ssKey.fail())
1054 return Fail("malformed wallet record key", strErrorRet);
1055 if (ssValue.fail())
1056 return Fail("malformed wallet record value", strErrorRet);
1057 }
1058 catch (const std::exception& e)
1059 {
1060 return Fail(strprintf("wallet record parse exception: %s", e.what()),
1061 strErrorRet);
1062 }
1063 catch (...)
1064 {
1065 return Fail("wallet record parse exception", strErrorRet);
1066 }
1067 return true;
1068 }
1069};
1070
1071} // namespace
1072
1073// Export the currently-loaded wallet to a SQLite file. Shared by the
1074// -walletsqliteexport command and the GUI conversion wizard: opens the
1075// destination, copies every record inside one transaction, checkpoints, and
1076// verifies the stored count matches. Returns false with strErrorRet set on any
1077// failure and leaves no partial file behind; the source wallet is only read.
1078bool ExportActiveWalletToSQLite(const std::string& strDest, int& nCopiedRet,
1079 std::string& strErrorRet)
1080{
1081 nCopiedRet = 0;
1082 if (strDest.empty())
1083 {
1084 strErrorRet = "no SQLite export path was given";
1085 return false;
1086 }
1087 if (FileExists(strDest.c_str()))
1088 {
1089 strErrorRet = strprintf("refusing to overwrite existing file: %s", strDest.c_str());
1090 return false;
1091 }
1092
1093 CWalletDBSQLite db;
1094 if (!db.Open(strDest, strErrorRet))
1095 {
1096 RemoveSQLiteExportFiles(strDest);
1097 return false;
1098 }
1099 if (!db.BeginTransaction(strErrorRet))
1100 {
1101 db.Close();
1102 RemoveSQLiteExportFiles(strDest);
1103 return false;
1104 }
1105
1106 CWalletSQLiteExportVisitor visitor(db, nCopiedRet);
1107 if (!ScanWalletRecords(visitor, strErrorRet))
1108 {
1109 string strRollbackError;
1110 db.RollbackTransaction(strRollbackError);
1111 db.Close();
1112 RemoveSQLiteExportFiles(strDest);
1113 return false;
1114 }
1115 if (!db.CommitTransaction(strErrorRet) || !db.Checkpoint(strErrorRet))
1116 {
1117 db.Close();
1118 RemoveSQLiteExportFiles(strDest);
1119 return false;
1120 }
1121
1122 int nCount = 0;
1123 if (!db.CountRecords(nCount, strErrorRet))
1124 {
1125 db.Close();
1126 RemoveSQLiteExportFiles(strDest);
1127 return false;
1128 }
1129 if (nCount != nCopiedRet)
1130 {
1131 strErrorRet = strprintf("export count mismatch: copied %d, stored %d",
1132 nCopiedRet, nCount);
1133 db.Close();
1134 RemoveSQLiteExportFiles(strDest);
1135 return false;
1136 }
1137 db.Close();
1138 return true;
1139}
1140
1141// Convert the running Berkeley DB wallet to SQLite for the GUI wizard: export
1142// wallet.dat to <datadir>/wallet.sqlite, then record the backend marker so the
1143// next start opens SQLite. wallet.dat is never modified -- it stays as the
1144// fallback. Refuses if already on SQLite or if a wallet.sqlite already exists.
1145bool DoConvertWalletToSQLite(int& nCopiedRet, std::string& strErrorRet)
1146{
1147 nCopiedRet = 0;
1148 if (WalletSQLiteRuntimeActive())
1149 {
1150 strErrorRet = "this wallet is already on the SQLite backend";
1151 return false;
1152 }
1153 string strDest = GetAppDir() + "/wallet.sqlite";
1154 if (FileExists(strDest.c_str()))
1155 {
1156 strErrorRet = strprintf("a SQLite wallet already exists at %s", strDest.c_str());
1157 return false;
1158 }
1159 if (!ExportActiveWalletToSQLite(strDest, nCopiedRet, strErrorRet))
1160 return false;
1161 if (!WriteWalletBackendMarker("sqlite"))
1162 {
1163 strErrorRet = "the wallet was exported but the backend choice could not be saved";
1164 return false;
1165 }
1166 return true;
1167}
1168
1169int CmdWalletSQLiteExport(const std::string& strDest)
1170{
1171 AttachTerminal();
1172
1173 // Keep the command's own specific messages for the two conditions the
1174 // shared helper folds into a generic error, so scripts and the tests that
1175 // read this output see the same wording as before.
1176 if (strDest.empty())
1177 {
1178 fprintf(stderr, "Missing SQLite wallet export path.\n");
1179 return 1;
1180 }
1181 if (FileExists(strDest.c_str()))
1182 {
1183 fprintf(stderr, "Refusing to overwrite existing SQLite wallet export: %s\n",
1184 strDest.c_str());
1185 return 1;
1186 }
1187
1188 int nCopied = 0;
1189 string strError;
1190 if (!ExportActiveWalletToSQLite(strDest, nCopied, strError))
1191 {
1192 fprintf(stderr, "Cannot export wallet to SQLite: %s\n", strError.c_str());
1193 return 1;
1194 }
1195
1196 printf("SQLite wallet export written to %s\n", strDest.c_str());
1197 printf(" records copied: %d\n", nCopied);
1198 fflush(stdout);
1199 return 0;
1200}
1201
1202int CmdWalletSQLiteVerify(const std::string& strPath)
1203{
1204 AttachTerminal();
1205
1206 if (strPath.empty())
1207 {
1208 fprintf(stderr, "Missing SQLite wallet export path.\n");
1209 return 1;
1210 }
1211 if (!FileExists(strPath.c_str()))
1212 {
1213 fprintf(stderr, "SQLite wallet export does not exist: %s\n",
1214 strPath.c_str());
1215 return 1;
1216 }
1217
1218 std::map<vector<unsigned char>, vector<unsigned char> > mapBDB;
1219 std::map<vector<unsigned char>, vector<unsigned char> > mapSQLite;
1220 string strError;
1221
1222 CWalletRecordMapVisitor bdbVisitor(mapBDB);
1223 if (!ScanWalletRecords(bdbVisitor, strError))
1224 {
1225 fprintf(stderr, "Cannot scan wallet.dat records: %s\n", strError.c_str());
1226 return 1;
1227 }
1228
1229 CWalletDBSQLite db;
1230 if (!db.Open(strPath, strError))
1231 {
1232 fprintf(stderr, "Cannot open SQLite wallet export: %s\n", strError.c_str());
1233 return 1;
1234 }
1235 CWalletRecordMapVisitor sqliteVisitor(mapSQLite);
1236 if (!db.ScanRecords(sqliteVisitor, strError))
1237 {
1238 fprintf(stderr, "Cannot scan SQLite wallet export: %s\n", strError.c_str());
1239 return 1;
1240 }
1241
1242 int nMissing = 0;
1243 int nMismatched = 0;
1244 for (std::map<vector<unsigned char>, vector<unsigned char> >::const_iterator it =
1245 mapBDB.begin(); it != mapBDB.end(); ++it)
1246 {
1247 std::map<vector<unsigned char>, vector<unsigned char> >::const_iterator sit =
1248 mapSQLite.find(it->first);
1249 if (sit == mapSQLite.end())
1250 nMissing++;
1251 else if (sit->second != it->second)
1252 nMismatched++;
1253 }
1254
1255 int nExtra = 0;
1256 for (std::map<vector<unsigned char>, vector<unsigned char> >::const_iterator it =
1257 mapSQLite.begin(); it != mapSQLite.end(); ++it)
1258 {
1259 if (!mapBDB.count(it->first))
1260 nExtra++;
1261 }
1262
1263 printf("SQLite wallet export verification\n");
1264 printf(" wallet.dat records: %u\n", (unsigned int)mapBDB.size());
1265 printf(" SQLite records: %u\n", (unsigned int)mapSQLite.size());
1266 printf(" missing records: %d\n", nMissing);
1267 printf(" extra records: %d\n", nExtra);
1268 printf(" record value mismatches: %d\n", nMismatched);
1269 if (nMissing == 0 && nExtra == 0 && nMismatched == 0)
1270 {
1271 printf(" verification: ok\n");
1272 fflush(stdout);
1273 return 0;
1274 }
1275
1276 printf(" verification: failed\n");
1277 fflush(stdout);
1278 return 2;
1279}
1280
1281int CmdWalletSQLiteRestore(const std::string& strPath)
1282{
1283 AttachTerminal();
1284
1285 if (strPath.empty())
1286 {
1287 fprintf(stderr, "Missing SQLite wallet export path.\n");
1288 return 1;
1289 }
1290 if (!FileExists(strPath.c_str()))
1291 {
1292 fprintf(stderr, "SQLite wallet export does not exist: %s\n",
1293 strPath.c_str());
1294 return 1;
1295 }
1296
1297 string strWalletPath = GetAppDir() + "/wallet.dat";
1298 if (FileExists(strWalletPath.c_str()))
1299 {
1300 fprintf(stderr, "Refusing to overwrite existing wallet.dat: %s\n",
1301 strWalletPath.c_str());
1302 return 1;
1303 }
1304
1305 string strError;
1306 CWalletDBSQLite sqlite;
1307 if (!sqlite.Open(strPath, strError))
1308 {
1309 fprintf(stderr, "Cannot open SQLite wallet export: %s\n", strError.c_str());
1310 return 1;
1311 }
1312
1313 int nSQLiteRecords = 0;
1314 if (!sqlite.CountRecords(nSQLiteRecords, strError))
1315 {
1316 fprintf(stderr, "Cannot count SQLite wallet export records: %s\n",
1317 strError.c_str());
1318 return 1;
1319 }
1320 if (nSQLiteRecords <= 0)
1321 {
1322 fprintf(stderr, "SQLite wallet export contains no records.\n");
1323 return 2;
1324 }
1325
1326 CWalletRawRestoreDB bdb("cr+", true);
1327 if (!bdb.TxnBegin())
1328 {
1329 fprintf(stderr, "Cannot begin Berkeley DB restore transaction.\n");
1330 return 1;
1331 }
1332
1333 int nCopied = 0;
1334 CWalletSQLiteRestoreVisitor visitor(bdb, nCopied);
1335 if (!sqlite.ScanRecords(visitor, strError))
1336 {
1337 bdb.TxnAbort();
1338 bdb.Close();
1339 RemoveFailedRestoreWallet(strWalletPath);
1340 fprintf(stderr, "Cannot restore SQLite wallet export: %s\n",
1341 strError.c_str());
1342 return 1;
1343 }
1344 if (nCopied != nSQLiteRecords)
1345 {
1346 bdb.TxnAbort();
1347 bdb.Close();
1348 RemoveFailedRestoreWallet(strWalletPath);
1349 fprintf(stderr, "SQLite wallet restore copied %d records, expected %d.\n",
1350 nCopied, nSQLiteRecords);
1351 return 2;
1352 }
1353 if (!bdb.TxnCommit())
1354 {
1355 bdb.Close();
1356 RemoveFailedRestoreWallet(strWalletPath);
1357 fprintf(stderr, "Cannot commit Berkeley DB restore transaction.\n");
1358 return 1;
1359 }
1360
1361 printf("SQLite wallet export restored to wallet.dat\n");
1362 printf(" records restored: %d\n", nCopied);
1363 fflush(stdout);
1364 return 0;
1365}
1366
1367int CmdWalletSQLiteLoadCheck(const std::string& strPath)
1368{
1369 AttachTerminal();
1370
1371 if (strPath.empty())
1372 {
1373 fprintf(stderr, "Missing SQLite wallet export path.\n");
1374 return 1;
1375 }
1376 if (!FileExists(strPath.c_str()))
1377 {
1378 fprintf(stderr, "SQLite wallet export does not exist: %s\n",
1379 strPath.c_str());
1380 return 1;
1381 }
1382
1383 string strError;
1384 CWalletDBSQLite sqlite;
1385 if (!sqlite.Open(strPath, strError))
1386 {
1387 fprintf(stderr, "Cannot open SQLite wallet export: %s\n", strError.c_str());
1388 return 1;
1389 }
1390
1391 CWalletSQLiteLoadCheckVisitor visitor;
1392 if (!sqlite.ScanRecords(visitor, strError))
1393 {
1394 printf("SQLite wallet load check\n");
1395 printf(" records checked: %d\n", visitor.nRecords);
1396 printf(" unknown records: %d\n", visitor.nUnknown);
1397 printf(" load check: failed\n");
1398 printf(" failure: %s\n", strError.c_str());
1399 fflush(stdout);
1400 return 2;
1401 }
1402
1403 printf("SQLite wallet load check\n");
1404 printf(" records checked: %d\n", visitor.nRecords);
1405 printf(" unknown records: %d\n", visitor.nUnknown);
1406 printf(" load check: ok\n");
1407 fflush(stdout);
1408 return 0;
1409}
1410
409int CmdEncryptWallet(const std::string& strPassphrase) 1411int CmdEncryptWallet(const std::string& strPassphrase)
410{ 1412{
411 AttachTerminal(); 1413 AttachTerminal();
412 1414
1415 // Capture the backend now: under SQLite, EncryptWallet() closes the runtime
1416 // store to swap the file, so WalletSQLiteRuntimeActive() reads false by the
1417 // time we would decide whether to purge Berkeley DB logs below.
1418 bool fSQLiteBackend = WalletSQLiteRuntimeActive();
1419
413 string strBackup; 1420 string strBackup;
414 string strError; 1421 string strError;
415 if (!EncryptWallet(strPassphrase, strBackup, strError)) 1422 if (!EncryptWallet(strPassphrase, strBackup, strError))
@@ -420,6 +1427,8 @@ int CmdEncryptWallet(const std::string& strPassphrase)
420 } 1427 }
421 1428
422 printf("Wallet encrypted.\n"); 1429 printf("Wallet encrypted.\n");
1430 if (fSQLiteBackend)
1431 printf("Wallet backend: SQLite (wallet.sqlite).\n");
423 printf("The old unencrypted wallet was moved to:\n"); 1432 printf("The old unencrypted wallet was moved to:\n");
424 printf(" %s\n", strBackup.c_str()); 1433 printf(" %s\n", strBackup.c_str());
425 printf("That backup still contains private keys in plain text. Move it offline or\n"); 1434 printf("That backup still contains private keys in plain text. Move it offline or\n");
@@ -432,7 +1441,13 @@ int CmdEncryptWallet(const std::string& strPassphrase)
432 // them. Closing the environment and dropping the logs is the last step of 1441 // them. Closing the environment and dropping the logs is the last step of
433 // encrypting, not an optimisation -- our own backup advice is to copy the 1442 // encrypting, not an optimisation -- our own backup advice is to copy the
434 // whole directory, and following it otherwise carries the keys along. 1443 // whole directory, and following it otherwise carries the keys along.
435 PurgeDbEnvironmentLogs(); 1444 //
1445 // The SQLite backend has no Berkeley DB environment behind the wallet, and
1446 // EncryptWallet() already closed and swapped the SQLite store; there is
1447 // nothing to purge, and calling into the environment here would only open
1448 // one needlessly.
1449 if (!fSQLiteBackend)
1450 PurgeDbEnvironmentLogs();
436 return 0; 1451 return 0;
437} 1452}
438 1453
diff --git a/src/walletcmd.h b/src/walletcmd.h
index 04d3fb7..7013d44 100644
--- a/src/walletcmd.h
+++ b/src/walletcmd.h
@@ -7,7 +7,9 @@
7#ifndef BITFLASH_WALLETCMD_H 7#ifndef BITFLASH_WALLETCMD_H
8#define BITFLASH_WALLETCMD_H 8#define BITFLASH_WALLETCMD_H
9 9
10#include <map>
10#include <string> 11#include <string>
12#include <vector>
11 13
12// Create a phrase for a wallet that does not have one, install the seed, and 14// Create a phrase for a wallet that does not have one, install the seed, and
13// show the words once. Refuses if a phrase already exists. Returns 0 on success. 15// show the words once. Refuses if a phrase already exists. Returns 0 on success.
@@ -41,6 +43,13 @@ bool RestoreScanReachedDepth(int nSchema,
41 unsigned int nLegacyNext, 43 unsigned int nLegacyNext,
42 int nStopDepth); 44 int nStopDepth);
43 45
46// Given the pubkeys derived while scanning a compatibility branch, return the
47// next index after the highest one that actually appears in wallet
48// transactions. The scan depth can be much larger than this; hdnext must record
49// use, not how far a restore looked ahead.
50unsigned int WalletLastUsedPubKeyIndexNext(
51 const std::map<unsigned int, std::vector<unsigned char> >& mapPubKeysByIndex);
52
44// Take the next address from the key pool and print it. With a recovery phrase 53// Take the next address from the key pool and print it. With a recovery phrase
45// installed the address is derived, so the phrase can bring back whatever is 54// installed the address is derived, so the phrase can bring back whatever is
46// paid to it. 55// paid to it.
@@ -53,6 +62,40 @@ int CmdShowDerived(int nCount);
53// by the installed recovery phrase or still depends on wallet.dat-only keys. 62// by the installed recovery phrase or still depends on wallet.dat-only keys.
54int CmdRecoveryAudit(); 63int CmdRecoveryAudit();
55 64
65// Diagnostic: count wallet.dat record types without printing keys,
66// addresses, transaction ids, labels, or other wallet values.
67int CmdWalletStorageAudit(const std::string& strJsonOut = "");
68
69// Diagnostic: fail closed when wallet.dat contains an internally inconsistent
70// or migration-unsafe mix of storage records.
71int CmdWalletStorageCheck();
72
73// Copy every raw wallet.dat record into a SQLite key/value store, preserving
74// serialized bytes. A migration/scripting tool; it does not change which backend
75// the runtime wallet opens.
76int CmdWalletSQLiteExport(const std::string& strDest);
77
78// Export the currently-loaded wallet to a SQLite file (shared by the command
79// above and the GUI wizard). Returns false with an error on any failure.
80bool ExportActiveWalletToSQLite(const std::string& strDest, int& nCopiedRet,
81 std::string& strErrorRet);
82
83// Convert the running Berkeley DB wallet to SQLite and record the backend
84// marker so the next start opens it. wallet.dat is left untouched as fallback.
85bool DoConvertWalletToSQLite(int& nCopiedRet, std::string& strErrorRet);
86
87// Diagnostic/migration staging: compare a SQLite wallet export with the current
88// wallet.dat raw records. Prints counts only, never record keys or values.
89int CmdWalletSQLiteVerify(const std::string& strPath);
90
91// Diagnostic/migration staging: rebuild wallet.dat from a SQLite export in an
92// empty data directory. Refuses to overwrite an existing wallet.dat.
93int CmdWalletSQLiteRestore(const std::string& strPath);
94
95// Diagnostic/migration staging: parse a SQLite wallet export as the runtime
96// wallet loader would. Prints counts only, never record keys or values.
97int CmdWalletSQLiteLoadCheck(const std::string& strPath);
98
56// Rewrite wallet.dat so private keys and the HD seed are encrypted with the 99// Rewrite wallet.dat so private keys and the HD seed are encrypted with the
57// given passphrase. Returns 0 on success and exits without starting the node. 100// given passphrase. Returns 0 on success and exits without starting the node.
58int CmdEncryptWallet(const std::string& strPassphrase); 101int CmdEncryptWallet(const std::string& strPassphrase);
diff --git a/tests/fixtures/chain-tools/explorer-blocks.json b/tests/fixtures/chain-tools/explorer-blocks.json
new file mode 100644
index 0000000..795d6e4
--- /dev/null
+++ b/tests/fixtures/chain-tools/explorer-blocks.json
@@ -0,0 +1 @@
{"blocks":[{"cbAddr":"BDnXDAAvEameyQk12FrtvBiPBp9MsfRmUA","cbValue":5000000000,"hash":"7c0e819cec25ebe29452049f1b35479526ba679bc2c1b00407417109384dea76","height":2,"nTx":2,"size":331,"time":1753315440},{"cbAddr":"B61KYnjpfXHN2w2e1nR869HneK4P1Gz4va","cbValue":5000000000,"hash":"17bdb455d41592adbff4aa97b7fca2091fef69600e67306baca829ab5a588b30","height":1,"nTx":2,"size":331,"time":1753315320},{"cbAddr":"BDU4zj6apurZ7YWNxsndaTt75PMgzvwbN4","cbValue":5000000000,"hash":"5bd7cb255d814e48cebcdfb72da4dc87b34bd774227f8ceb546c5640f4bdc169","height":0,"nTx":1,"size":278,"time":1753315200}],"count":3,"network":"bitflash","tipHeight":2}
diff --git a/tests/fixtures/chain-tools/fair-launch-report.json b/tests/fixtures/chain-tools/fair-launch-report.json
new file mode 100644
index 0000000..d35f30b
--- /dev/null
+++ b/tests/fixtures/chain-tools/fair-launch-report.json
@@ -0,0 +1 @@
{"attestation_payload_sha256":"bf192bdbbb84c69d15bb68cf133abd474d0e1021600febc68e9f1a9de844b2af","chain":{"coin":100000000,"message_start_hex":"bf205cfd","name":"Bitflash","ticker":"BTF"},"checks":{"computed_merkle_matches_header":true,"genesis_has_single_transaction":true,"genesis_hash_matches_source":true,"genesis_merkle_matches_source":true,"genesis_supply_is_50_btf":true,"genesis_transaction_is_coinbase":true},"coinbase_outputs":[{"block_hash":"5bd7cb255d814e48cebcdfb72da4dc87b34bd774227f8ceb546c5640f4bdc169","coinbase_txid":"1a45b4482532abb29b10e234d3f13132230525a339ecea91658ffa675a5b1325","height":0,"outputs":[{"script_pub_key_hex":"4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac","value_btf":"50.00000000","value_satoshis":5000000000}],"value_btf":"50.00000000","value_satoshis":5000000000},{"block_hash":"17bdb455d41592adbff4aa97b7fca2091fef69600e67306baca829ab5a588b30","coinbase_txid":"d6967c16013f1da136183f962aefd2f15e59b6836043f8ef49aadb92f38dca05","height":1,"outputs":[{"script_pub_key_hex":"76a914111111111111111111111111111111111111111188ac","value_btf":"50.00000000","value_satoshis":5000000000}],"value_btf":"50.00000000","value_satoshis":5000000000},{"block_hash":"7c0e819cec25ebe29452049f1b35479526ba679bc2c1b00407417109384dea76","coinbase_txid":"d55af2c80e8b70a9ae8f362a8aac5448e7925267c7f737d1fa0d227fd472a657","height":2,"outputs":[{"script_pub_key_hex":"76a914666666666666666666666666666666666666666688ac","value_btf":"50.00000000","value_satoshis":5000000000}],"value_btf":"50.00000000","value_satoshis":5000000000}],"deterministic":true,"external_timestamping":{"instructions":["Save the canonical JSON report.","Run: ots stamp bitflash-fair-launch-report.json","Later verify with: ots verify bitflash-fair-launch-report.json.ots"],"recommended":"OpenTimestamps","status":"not_embedded"},"genesis":{"bits":"0x1f0fffff","coinbase_txid":"1a45b4482532abb29b10e234d3f13132230525a339ecea91658ffa675a5b1325","computed_merkle_root":"1a45b4482532abb29b10e234d3f13132230525a339ecea91658ffa675a5b1325","hash":"5bd7cb255d814e48cebcdfb72da4dc87b34bd774227f8ceb546c5640f4bdc169","merkle_root":"1a45b4482532abb29b10e234d3f13132230525a339ecea91658ffa675a5b1325","nonce":3141,"outputs":[{"script_pub_key_hex":"4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac","value_btf":"50.00000000","value_satoshis":5000000000}],"time":1753315200,"time_utc":"2025-07-24T00:00:00Z","tx_count":1},"scanned":{"block_count":3,"first_height":0,"last_height":2,"non_coinbase_transactions":2,"total_coinbase_btf":"150.00000000","total_coinbase_satoshis":15000000000},"schema":"bitflash-fair-launch-report-1","source_constants":{"genesis_bits":"0x1f0fffff","genesis_hash":"5bd7cb255d814e48cebcdfb72da4dc87b34bd774227f8ceb546c5640f4bdc169","genesis_merkle_root":"1a45b4482532abb29b10e234d3f13132230525a339ecea91658ffa675a5b1325","genesis_message":"Bitflash 24/Jul/2026 Fair launch: one CPU one vote, no premine","genesis_nonce":3141,"genesis_time":1753315200,"genesis_time_utc":"2025-07-24T00:00:00Z"},"source_mode":"local-block-files","verdict":"FAIR_LAUNCH_BASELINE_VERIFIED"}
diff --git a/tests/fixtures/chain-tools/utxo-proof-verification.json b/tests/fixtures/chain-tools/utxo-proof-verification.json
new file mode 100644
index 0000000..a40c673
--- /dev/null
+++ b/tests/fixtures/chain-tools/utxo-proof-verification.json
@@ -0,0 +1 @@
{"checks":{"leaf_hash_matches_target":true,"root_matches":true,"schema_is_supported":true},"commitment":{"computed_root":"6e49b47ffccb178b62402235343ae7c55ec4a03effe19debb8c057e8ffc5b3c0","expected_root":"6e49b47ffccb178b62402235343ae7c55ec4a03effe19debb8c057e8ffc5b3c0","leaf_hash":"da964462812e40c918d6349baa57aee1dc838416eca44ccd31b782f0d1542d7b","path_length":3},"deterministic":true,"failures":[],"schema":"bitflash-utxo-proof-verification-1","target":{"created_height":2,"outpoint":"218609b2969dc0b6ae1c378ae255bbed863902987df4c943200e3c291b0e4c2a:1","value_btf":"17.50000000","value_satoshis":1750000000},"verdict":"UTXO_PROOF_VERIFIED"}
diff --git a/tests/fixtures/chain-tools/utxo-proof.json b/tests/fixtures/chain-tools/utxo-proof.json
new file mode 100644
index 0000000..154c45b
--- /dev/null
+++ b/tests/fixtures/chain-tools/utxo-proof.json
@@ -0,0 +1 @@
{"chain":{"coin":100000000,"message_start_hex":"bf205cfd","name":"Bitflash","ticker":"BTF"},"commitment":{"algorithm":"sorted-outpoint-merkle-sha256","leaf_count":5,"leaf_domain":"BTFUTXO1","node_domain":"BTFNODE1","root":"6e49b47ffccb178b62402235343ae7c55ec4a03effe19debb8c057e8ffc5b3c0"},"deterministic":true,"proof":[{"duplicated":false,"hash":"7a4f4f13aaf91f8b6b32d1fb22feaa3d0398782504915e9c156e60fdeca1255d","level":0,"side":"left"},{"duplicated":false,"hash":"5f91d5ba472b7d6ab96e9ce99d689896dad3c264d3abbce4c77f257687608943","level":1,"side":"right"},{"duplicated":false,"hash":"f041ba6a82fbc6759f0632f412dd8aab971ac1317358d6f204af3e57da4d12de","level":2,"side":"right"}],"scanned":{"first_height":0,"last_height":2,"main_chain_block_count":3,"raw_block_count":3,"side_blocks_ignored":0,"tip_hash":"7c0e819cec25ebe29452049f1b35479526ba679bc2c1b00407417109384dea76","tip_time":1753315440,"tip_time_utc":"2025-07-24T00:04:00Z"},"schema":"bitflash-utxo-inclusion-proof-1","target":{"leaf_hash":"da964462812e40c918d6349baa57aee1dc838416eca44ccd31b782f0d1542d7b","leaf_index":1,"outpoint":"218609b2969dc0b6ae1c378ae255bbed863902987df4c943200e3c291b0e4c2a:1","txid":"218609b2969dc0b6ae1c378ae255bbed863902987df4c943200e3c291b0e4c2a","utxo":{"coinbase":false,"created_block_hash":"7c0e819cec25ebe29452049f1b35479526ba679bc2c1b00407417109384dea76","created_height":2,"created_txid":"218609b2969dc0b6ae1c378ae255bbed863902987df4c943200e3c291b0e4c2a","script_pub_key_hex":"76a914555555555555555555555555555555555555555588ac","txid":"218609b2969dc0b6ae1c378ae255bbed863902987df4c943200e3c291b0e4c2a","value_btf":"17.50000000","value_satoshis":1750000000,"vout":1},"vout":1},"verdict":"UTXO_INCLUSION_PROOF_CREATED","warnings":{"legacy_duplicate_output_overwrites":0}}
diff --git a/tests/fixtures/chain-tools/utxo-report.json b/tests/fixtures/chain-tools/utxo-report.json
new file mode 100644
index 0000000..c6e98ee
--- /dev/null
+++ b/tests/fixtures/chain-tools/utxo-report.json
@@ -0,0 +1 @@
{"chain":{"coin":100000000,"message_start_hex":"bf205cfd","name":"Bitflash","ticker":"BTF"},"checks":{"all_scanned_merkle_roots_match":true,"genesis_hash_matches_source":true,"genesis_merkle_matches_source":true,"no_missing_spends":true},"commitment":{"algorithm":"sorted-outpoint-merkle-sha256","leaf_domain":"BTFUTXO1","node_domain":"BTFNODE1","root":"6e49b47ffccb178b62402235343ae7c55ec4a03effe19debb8c057e8ffc5b3c0"},"deterministic":true,"scanned":{"first_height":0,"last_height":2,"main_chain_block_count":3,"raw_block_count":3,"side_blocks_ignored":0,"tip_hash":"7c0e819cec25ebe29452049f1b35479526ba679bc2c1b00407417109384dea76","tip_time":1753315440,"tip_time_utc":"2025-07-24T00:04:00Z"},"schema":"bitflash-utxo-set-commitment-1","stats":{"coinbase_minus_unspent_btf":"1.50000000","coinbase_minus_unspent_satoshis":150000000,"duplicate_outputs":[],"missing_spends":[],"normal_transactions":2,"spent_outputs":8000000000,"total_coinbase_btf":"150.00000000","total_coinbase_satoshis":15000000000,"total_fees_btf":"1.50000000","total_fees_satoshis":150000000,"total_unspent_btf":"148.50000000","total_unspent_satoshis":14850000000,"utxo_count":5},"utxos":{"218609b2969dc0b6ae1c378ae255bbed863902987df4c943200e3c291b0e4c2a:0":{"coinbase":false,"created_block_hash":"7c0e819cec25ebe29452049f1b35479526ba679bc2c1b00407417109384dea76","created_height":2,"created_txid":"218609b2969dc0b6ae1c378ae255bbed863902987df4c943200e3c291b0e4c2a","script_pub_key_hex":"76a914444444444444444444444444444444444444444488ac","txid":"218609b2969dc0b6ae1c378ae255bbed863902987df4c943200e3c291b0e4c2a","value_btf":"12.00000000","value_satoshis":1200000000,"vout":0},"218609b2969dc0b6ae1c378ae255bbed863902987df4c943200e3c291b0e4c2a:1":{"coinbase":false,"created_block_hash":"7c0e819cec25ebe29452049f1b35479526ba679bc2c1b00407417109384dea76","created_height":2,"created_txid":"218609b2969dc0b6ae1c378ae255bbed863902987df4c943200e3c291b0e4c2a","script_pub_key_hex":"76a914555555555555555555555555555555555555555588ac","txid":"218609b2969dc0b6ae1c378ae255bbed863902987df4c943200e3c291b0e4c2a","value_btf":"17.50000000","value_satoshis":1750000000,"vout":1},"24bddfded9e22c2f98d9369e9ac00a4eccc8dcf05a2d1b9e12d94fd9f62ababc:1":{"coinbase":false,"created_block_hash":"17bdb455d41592adbff4aa97b7fca2091fef69600e67306baca829ab5a588b30","created_height":1,"created_txid":"24bddfded9e22c2f98d9369e9ac00a4eccc8dcf05a2d1b9e12d94fd9f62ababc","script_pub_key_hex":"76a914333333333333333333333333333333333333333388ac","txid":"24bddfded9e22c2f98d9369e9ac00a4eccc8dcf05a2d1b9e12d94fd9f62ababc","value_btf":"19.00000000","value_satoshis":1900000000,"vout":1},"d55af2c80e8b70a9ae8f362a8aac5448e7925267c7f737d1fa0d227fd472a657:0":{"coinbase":true,"created_block_hash":"7c0e819cec25ebe29452049f1b35479526ba679bc2c1b00407417109384dea76","created_height":2,"created_txid":"d55af2c80e8b70a9ae8f362a8aac5448e7925267c7f737d1fa0d227fd472a657","script_pub_key_hex":"76a914666666666666666666666666666666666666666688ac","txid":"d55af2c80e8b70a9ae8f362a8aac5448e7925267c7f737d1fa0d227fd472a657","value_btf":"50.00000000","value_satoshis":5000000000,"vout":0},"d6967c16013f1da136183f962aefd2f15e59b6836043f8ef49aadb92f38dca05:0":{"coinbase":true,"created_block_hash":"17bdb455d41592adbff4aa97b7fca2091fef69600e67306baca829ab5a588b30","created_height":1,"created_txid":"d6967c16013f1da136183f962aefd2f15e59b6836043f8ef49aadb92f38dca05","script_pub_key_hex":"76a914111111111111111111111111111111111111111188ac","txid":"d6967c16013f1da136183f962aefd2f15e59b6836043f8ef49aadb92f38dca05","value_btf":"50.00000000","value_satoshis":5000000000,"vout":0}},"verdict":"UTXO_COMMITMENT_VERIFIED","warnings":{"legacy_duplicate_output_overwrites":0}}
diff --git a/tests/scripts/test_bitflash_tools.py b/tests/scripts/test_bitflash_tools.py
new file mode 100644
index 0000000..3b4c846
--- /dev/null
+++ b/tests/scripts/test_bitflash_tools.py
@@ -0,0 +1,286 @@
1#!/usr/bin/env python3
2"""Regression tests for deterministic Bitflash Python tooling."""
3
4import json
5import os
6import struct
7import subprocess
8import sys
9import tempfile
10from pathlib import Path
11
12
13ROOT = Path(__file__).resolve().parents[2]
14SCRIPTS = ROOT / "scripts"
15FIXTURES = ROOT / "tests" / "fixtures" / "chain-tools"
16sys.path.insert(0, str(SCRIPTS))
17
18import bitflash_chain as chain # noqa: E402
19
20
21def run(args, **kwargs):
22 env = dict(os.environ)
23 env["PYTHONIOENCODING"] = "utf-8"
24 return subprocess.run(
25 [sys.executable] + [str(a) for a in args],
26 cwd=str(ROOT),
27 env=env,
28 check=True,
29 text=True,
30 stdout=subprocess.PIPE,
31 stderr=subprocess.PIPE,
32 **kwargs
33 )
34
35
36def write_block_file(path, blocks):
37 with Path(path).open("wb") as f:
38 f.write(b"noise")
39 f.write(chain.MAGIC)
40 f.write(struct.pack("<I", chain.MAX_BLOCK_PAYLOAD + 1))
41 f.write(b"still-noise")
42 for block in blocks:
43 payload = chain.ser_block(block)
44 f.write(chain.MAGIC)
45 f.write(struct.pack("<I", len(payload)))
46 f.write(payload)
47
48
49def p2pkh_script(byte):
50 return "76a914%s88ac" % (bytes([byte]) * 20).hex()
51
52
53def make_tx(vin, vout):
54 tx = {
55 "version": 1,
56 "vin": vin,
57 "vout": vout,
58 "lock_time": 0,
59 }
60 tx["txid"] = chain.hash_hex(chain.ser_tx(tx))
61 return tx
62
63
64def make_coinbase(label, script_byte, value=50 * chain.COIN):
65 return make_tx(
66 [{
67 "prevout_hash": "0" * 64,
68 "prevout_n": 0xFFFFFFFF,
69 "script_sig_hex": label.encode("ascii").hex(),
70 "sequence": 0xFFFFFFFF,
71 }],
72 [{
73 "value_satoshis": value,
74 "value_btf": chain.format_money(value),
75 "script_pub_key_hex": p2pkh_script(script_byte),
76 }],
77 )
78
79
80def make_spend(prev_txid, prev_n, outputs, label):
81 return make_tx(
82 [{
83 "prevout_hash": prev_txid,
84 "prevout_n": prev_n,
85 "script_sig_hex": label.encode("ascii").hex(),
86 "sequence": 0xFFFFFFFF,
87 }],
88 [{
89 "value_satoshis": value,
90 "value_btf": chain.format_money(value),
91 "script_pub_key_hex": p2pkh_script(script_byte),
92 } for value, script_byte in outputs],
93 )
94
95
96def make_block(prev_hash, txs, ntime, nonce):
97 merkle = chain.merkle_root([tx["txid"] for tx in txs])
98 header = (
99 struct.pack("<i", 1)
100 + bytes.fromhex(prev_hash)[::-1]
101 + bytes.fromhex(merkle)[::-1]
102 + struct.pack("<III", ntime, chain.GENESIS_BITS, nonce)
103 )
104 block = {
105 "hash": chain.hash_hex(header),
106 "version": 1,
107 "previous_hash": prev_hash,
108 "merkle_root": merkle,
109 "computed_merkle_root": merkle,
110 "time": ntime,
111 "time_utc": chain.utc(ntime),
112 "bits": "0x%08x" % chain.GENESIS_BITS,
113 "nonce": nonce,
114 "tx_count": len(txs),
115 "transactions": txs,
116 "source_file": "synthetic",
117 "source_offset": 0,
118 "size": 80 + len(chain.ser_compact_size(len(txs))) + sum(len(chain.ser_tx(tx)) for tx in txs),
119 }
120 return block
121
122
123def fixture_chain():
124 genesis = chain.reconstructed_genesis()
125 spend1 = make_spend(
126 chain.GENESIS_MERKLE,
127 0,
128 [(30 * chain.COIN, 0x22), (19 * chain.COIN, 0x33)],
129 "spend genesis with 1 BTF fee",
130 )
131 block1 = make_block(
132 genesis["hash"],
133 [make_coinbase("fixture coinbase 1", 0x11), spend1],
134 chain.GENESIS_TIME + 120,
135 4001,
136 )
137 spend2 = make_spend(
138 spend1["txid"],
139 0,
140 [(12 * chain.COIN, 0x44), (17 * chain.COIN + 50_000_000, 0x55)],
141 "split spend with 0.5 BTF fee",
142 )
143 block2 = make_block(
144 block1["hash"],
145 [make_coinbase("fixture coinbase 2", 0x66), spend2],
146 chain.GENESIS_TIME + 240,
147 4002,
148 )
149 return [genesis, block1, block2], {
150 "genesis_outpoint": "%s:0" % chain.GENESIS_MERKLE,
151 "first_spend_txid": spend1["txid"],
152 "proof_outpoint": "%s:1" % spend2["txid"],
153 }
154
155
156def compare_golden(name, actual, update):
157 path = FIXTURES / name
158 if update:
159 path.parent.mkdir(parents=True, exist_ok=True)
160 path.write_bytes(actual)
161 return
162 # The tools promise canonical LF-terminated JSON. On Windows checkouts a
163 # fixture may still appear as CRLF in the working tree; normalize only line
164 # endings so content, field order and compact serialization still drift-test.
165 expected = path.read_bytes().replace(b"\r\n", b"\n")
166 if actual.replace(b"\r\n", b"\n") != expected:
167 raise AssertionError("golden mismatch: %s" % name)
168
169
170def assert_true(cond, msg):
171 if not cond:
172 raise AssertionError(msg)
173
174
175def run_genesis_smoke():
176 genesis = chain.reconstructed_genesis()
177 outpoint = "%s:0" % chain.GENESIS_MERKLE
178
179 with tempfile.TemporaryDirectory(prefix="bitflash-tool-tests-") as tmp:
180 tmp = Path(tmp)
181 datadir = tmp / "datadir"
182 datadir.mkdir()
183 blk1 = datadir / "blk0001.dat"
184 blk2 = datadir / "blk0002.dat"
185 write_block_file(blk1, [genesis])
186 blk2.write_bytes(b"")
187
188 fair = run([SCRIPTS / "verify-fair-launch.py", "--datadir", datadir,
189 "--max-blocks", "1", "--json"]).stdout
190 fair_report = json.loads(fair)
191 assert_true(fair_report["verdict"] == "FAIR_LAUNCH_BASELINE_VERIFIED",
192 "fair launch verifier failed synthetic genesis")
193
194 utxo = run([SCRIPTS / "verify-utxo-set.py", "--datadir", datadir,
195 "--json"]).stdout
196 utxo_report = json.loads(utxo)
197 assert_true(utxo_report["verdict"] == "UTXO_COMMITMENT_VERIFIED",
198 "UTXO verifier failed synthetic genesis")
199
200 proof_path = tmp / "proof.json"
201 run([SCRIPTS / "prove-utxo.py", outpoint, "--datadir", datadir,
202 "--out", proof_path, "--json"])
203 proof_check = run([SCRIPTS / "verify-utxo-proof.py", proof_path,
204 "--root", utxo_report["commitment"]["root"], "--json"]).stdout
205 proof_report = json.loads(proof_check)
206 assert_true(proof_report["verdict"] == "UTXO_PROOF_VERIFIED",
207 "UTXO proof verifier rejected synthetic proof")
208
209 explorer_dir = tmp / "explorer"
210 run([SCRIPTS / "build-explorer.py", blk1, blk2, explorer_dir])
211 blocks = json.loads((explorer_dir / "blocks.json").read_text(encoding="ascii"))
212 assert_true(blocks["tipHeight"] == 0 and blocks["count"] == 1,
213 "explorer did not build the synthetic chain")
214 index_html = (explorer_dir / "index.html").read_text(encoding="utf-8")
215 assert_true("innerHTML" not in index_html, "explorer viewer still uses innerHTML")
216
217 try:
218 chain.apply_blocks([genesis, genesis], strict_duplicates=True)
219 except chain.ParseError:
220 pass
221 else:
222 raise AssertionError("strict duplicate output mode did not fail")
223
224
225def run_golden_fixture(update):
226 blocks, meta = fixture_chain()
227
228 with tempfile.TemporaryDirectory(prefix="bitflash-tool-golden-") as tmp:
229 tmp = Path(tmp)
230 datadir = tmp / "datadir"
231 datadir.mkdir()
232 blk1 = datadir / "blk0001.dat"
233 blk2 = datadir / "blk0002.dat"
234 write_block_file(blk1, blocks[:2])
235 write_block_file(blk2, blocks[2:])
236
237 fair = run([SCRIPTS / "verify-fair-launch.py", "--datadir", datadir,
238 "--max-blocks", "3", "--json"]).stdout.encode("ascii")
239 compare_golden("fair-launch-report.json", fair, update)
240 fair_report = json.loads(fair)
241 assert_true(fair_report["scanned"]["non_coinbase_transactions"] == 2,
242 "fair-launch fixture did not count normal transactions")
243
244 utxo = run([SCRIPTS / "verify-utxo-set.py", "--datadir", datadir,
245 "--include-utxos", "--strict-duplicates", "--json"]).stdout.encode("ascii")
246 compare_golden("utxo-report.json", utxo, update)
247 utxo_report = json.loads(utxo)
248 assert_true(utxo_report["stats"]["normal_transactions"] == 2,
249 "UTXO fixture did not apply normal transactions")
250 assert_true(utxo_report["stats"]["total_fees_btf"] == "1.50000000",
251 "UTXO fixture fee total changed")
252 assert_true(meta["genesis_outpoint"] not in utxo_report["utxos"],
253 "spent genesis output still appears in the UTXO map")
254
255 proof_path = tmp / "fixture-proof.json"
256 proof = run([SCRIPTS / "prove-utxo.py", meta["proof_outpoint"],
257 "--datadir", datadir, "--strict-duplicates",
258 "--out", proof_path, "--json"]).stdout.encode("ascii")
259 compare_golden("utxo-proof.json", proof, update)
260
261 proof_check = run([SCRIPTS / "verify-utxo-proof.py", proof_path,
262 "--root", utxo_report["commitment"]["root"], "--json"]).stdout.encode("ascii")
263 compare_golden("utxo-proof-verification.json", proof_check, update)
264 proof_report = json.loads(proof_check)
265 assert_true(proof_report["verdict"] == "UTXO_PROOF_VERIFIED",
266 "fixture UTXO proof did not verify")
267
268 explorer_dir = tmp / "explorer"
269 run([SCRIPTS / "build-explorer.py", blk1, blk2, explorer_dir])
270 compare_golden("explorer-blocks.json",
271 (explorer_dir / "blocks.json").read_bytes(), update)
272 detail = json.loads((explorer_dir / "block" / "1.json").read_text(encoding="ascii"))
273 assert_true(len(detail["txs"]) == 2,
274 "explorer fixture block did not include the normal transaction")
275
276
277def main():
278 update = "--update-goldens" in sys.argv[1:]
279 run_genesis_smoke()
280 run_golden_fixture(update)
281 print("Bitflash script tool regressions passed")
282 return 0
283
284
285if __name__ == "__main__":
286 raise SystemExit(main())