diff --git a/test/regtest_smoke.sh b/test/regtest_smoke.sh index dd01c13..790fc73 100755 --- a/test/regtest_smoke.sh +++ b/test/regtest_smoke.sh @@ -1,16 +1,27 @@ #!/usr/bin/env bash -# Regtest smoke test for kamado-api. +# Regtest smoke test for kamado-api — two phases: # -# Validates the full critical path: -# ckpool log line → log tailer → aggregator → bitcoind RPC enrichment → /api/blocks +# Phase 1 (always): log-inject path +# Injects a "Solved and confirmed block N" line directly into the ckpool log +# and verifies the full tailer → aggregator → bitcoind RPC enrichment → /api/blocks +# chain. Quick, no ckpool binary needed. # -# Prerequisites: bitcoind, bitcoin-cli, jq, curl -# The kamado-api binary is auto-built from source if go is in PATH; -# alternatively set KAMADO_BIN to point at a pre-built binary. +# Phase 2 (auto): full stratum path — the complete revenue-critical path +# A Python stratum miner connects to ckpool, mines a real diff-1 SHA256d hash +# using multiprocessing, submits the winning share, ckpool calls bitcoind +# submitblock, logs "Solved and confirmed block N+1", and the kamado-api +# tailer propagates it to /api/blocks. +# ckpool is auto-built from the pinned source in ckpool/ if not already cached. +# Built binary is cached in ~/.cache/kamado-dev/ (keyed on commit hash) so +# subsequent runs are instant. +# +# Prerequisites: bitcoind, bitcoin-cli, jq, curl, go, python3 +# + gcc/autoconf/automake/libtool/pkg-config/libzmq-dev for ckpool build # # Usage: # test/regtest_smoke.sh # KAMADO_BIN=./api/bin/kamado-api test/regtest_smoke.sh +# KAMADO_BIN=./api/bin/kamado-api CKPOOL_BIN=/usr/local/bin/ckpool test/regtest_smoke.sh set -euo pipefail @@ -21,22 +32,27 @@ API_DIR="$SCRIPT_DIR/../api" log() { printf '[smoke] %s\n' "$*"; } die() { printf '[smoke] FAIL: %s\n' "$*" >&2; exit 1; } -pass() { printf '[smoke] PASS: %s\n' "$*"; exit 0; } WORK_DIR="" BITCOIN_PID="" KAMADO_PID="" +CKPOOL_PID="" +MINER_PID="" +CKPOOL_BUILD_TMP="" cleanup() { - [[ -n "$KAMADO_PID" ]] && kill "$KAMADO_PID" 2>/dev/null || true - [[ -n "$BITCOIN_PID" ]] && kill "$BITCOIN_PID" 2>/dev/null || true - [[ -n "$WORK_DIR" ]] && rm -rf "$WORK_DIR" + [[ -n "$MINER_PID" ]] && kill "$MINER_PID" 2>/dev/null || true + [[ -n "$CKPOOL_PID" ]] && kill "$CKPOOL_PID" 2>/dev/null || true + [[ -n "$KAMADO_PID" ]] && kill "$KAMADO_PID" 2>/dev/null || true + [[ -n "$BITCOIN_PID" ]] && kill "$BITCOIN_PID" 2>/dev/null || true + [[ -n "$WORK_DIR" ]] && rm -rf "$WORK_DIR" + [[ -n "$CKPOOL_BUILD_TMP" ]] && rm -rf "$CKPOOL_BUILD_TMP" } trap cleanup EXIT # ---- prerequisites ---------------------------------------------------------- -for cmd in bitcoind bitcoin-cli jq curl; do +for cmd in bitcoind bitcoin-cli jq curl python3; do command -v "$cmd" >/dev/null 2>&1 || die "required command not found: $cmd" done @@ -57,6 +73,340 @@ fi [[ -n "${WORK_DIR:-}" ]] || WORK_DIR=$(mktemp -d) +# Locate or build ckpool — required for Phase 2. +# Priority: CKPOOL_BIN env var → PATH → cached build → build from source. +CKPOOL_AVAILABLE=false +CKPOOL_DIR="$SCRIPT_DIR/../ckpool" +CKPOOL_REPO=$(cat "$CKPOOL_DIR/CKPOOL_REPO" | tr -d '[:space:]') +CKPOOL_COMMIT=$(cat "$CKPOOL_DIR/CKPOOL_COMMIT" | tr -d '[:space:]') +# Cache keyed on commit so a pinned-commit bump automatically triggers a rebuild. +CKPOOL_CACHE="$HOME/.cache/kamado-dev/ckpool-${CKPOOL_COMMIT:0:12}/ckpool" + +if [[ -n "${CKPOOL_BIN:-}" ]]; then + [[ -x "$CKPOOL_BIN" ]] || die "CKPOOL_BIN=$CKPOOL_BIN is not executable" + CKPOOL_AVAILABLE=true +elif command -v ckpool >/dev/null 2>&1; then + CKPOOL_BIN=$(command -v ckpool) + CKPOOL_AVAILABLE=true +elif [[ -x "$CKPOOL_CACHE" ]]; then + CKPOOL_BIN="$CKPOOL_CACHE" + CKPOOL_AVAILABLE=true + log "ckpool: using cached build at $CKPOOL_BIN" +elif command -v gcc autoconf automake libtool pkg-config git >/dev/null 2>&1 \ + && pkg-config --exists libzmq 2>/dev/null; then + log "ckpool: building from source at commit ${CKPOOL_COMMIT:0:12}…" + log " (this takes ~1-2 min; the result is cached for future runs)" + CKPOOL_BUILD_TMP=$(mktemp -d) # cleaned up by the existing cleanup trap + git clone --quiet "$CKPOOL_REPO" "$CKPOOL_BUILD_TMP/ckpool" \ + || die "ckpool: git clone failed" + ( + cd "$CKPOOL_BUILD_TMP/ckpool" + git checkout --quiet "$CKPOOL_COMMIT" + for patch in "$CKPOOL_DIR/patches/"*.patch; do + [[ -f "$patch" ]] || continue + log "ckpool: applying $(basename "$patch")" + git apply "$patch" + done + ./autogen.sh >/dev/null 2>&1 + CFLAGS="-O2 -Wall -pipe" ./configure --quiet --prefix=/usr/local >/dev/null + make -j"$(nproc)" >/dev/null + ) || die "ckpool: build failed" + mkdir -p "$(dirname "$CKPOOL_CACHE")" + cp "$CKPOOL_BUILD_TMP/ckpool/src/ckpool" "$CKPOOL_CACHE" + chmod +x "$CKPOOL_CACHE" + rm -rf "$CKPOOL_BUILD_TMP" + CKPOOL_BIN="$CKPOOL_CACHE" + CKPOOL_AVAILABLE=true + log "ckpool: build complete → $CKPOOL_BIN" +else + log "ckpool not found and build prerequisites missing — Phase 2 will be skipped" + log " Missing one or more of: gcc, autoconf, automake, libtool, pkg-config, git, libzmq-dev" + log " Or set CKPOOL_BIN=/path/to/ckpool to skip the build" +fi + +# Write the Python stratum miner — no build step needed; python3 is in PATH (checked above). +# The miner uses hashlib SHA256 midstate optimization + multiprocessing so it scales linearly +# with CPU count. Diff-1 requires on average 2^32 ≈ 4.3B hashes; at ~1.5 MH/s per core this +# takes ~43 minutes on a single core but ~3 minutes on 16 cores. +STRATUM_MINER="$WORK_DIR/stratum_miner.py" +cat > "$STRATUM_MINER" << 'PYEOF' +#!/usr/bin/env python3 +""" +ckpool regtest stratum miner. + +Field encoding derived from ckpool source (bitcoin.c / stratifier.c): + version, ntime, nbits: sent as 8-char big-endian hex of the integer value. + → canonical LE header bytes: struct.pack('= 30: + elapsed = now - t0 + eta = max(0, expected_s - elapsed) + log(f'mining: elapsed={elapsed:.0f}s {total_mhs:.1f} MH/s ETA~{eta:.0f}s en2={cur_en2_hex}') + last_log_time = now + + # Drain result queue + try: + while True: + r = out_q.get_nowait() + if r is None: + exhausted += 1 + else: + found_nonce = r + break + except Exception: + pass + + if found_nonce is not None: + break + + # Poll for new stratum messages + try: + chunk_data = sock.recv(4096).decode(errors='replace') + if chunk_data: + buf += chunk_data + while '\n' in buf: + line_str, buf = buf.split('\n', 1) + try: + msg = json.loads(line_str) + if msg.get('method') == 'mining.notify': + pn = msg['params'] + new_job = {'id': pn[0], 'ph': pn[1], 'cb1': pn[2], 'cb2': pn[3], + 'br': pn[4], 'ver': pn[5], 'nb': pn[6], 'nt': pn[7]} + log(f'New job: id={new_job["id"]} ntime={new_job["nt"]}') + job = new_job + # Always restart on any new job. ckpool only keeps + # recent workbases (~120s); mining one job for 300s + # guarantees a stale rejection regardless of clean flag. + clean_job_arrived = True + except (json.JSONDecodeError, IndexError): + pass + except socket.timeout: + pass + + # Stop all workers + stop_ev.set() + for proc in procs: + proc.join(timeout=3) + if proc.is_alive(): + proc.kill() + + if found_nonce is not None: + nonce_hex = f'{found_nonce:08x}' + full_header = h64 + h12 + struct.pack('/dev/null || true +done +sleep 0.5 # give killed processes a moment to release their ports + # Shorthand for bitcoin-cli pointing at the regtest instance. cli() { bitcoin-cli \ @@ -106,21 +468,21 @@ for i in $(seq 1 30); do sleep 1 done $ready || die "bitcoind did not become ready within 30s (see $WORK_DIR/bitcoind.log)" +kill -0 "$BITCOIN_PID" 2>/dev/null \ + || die "bitcoind (pid $BITCOIN_PID) is not running — another process may have answered on port $BTC_RPC_PORT" log "bitcoind ready (pid $BITCOIN_PID)" -# Create a descriptor wallet and mine enough blocks for coinbase maturity, -# then mine the one block we'll claim to have "solved". -log "setting up wallet and generating blocks" +# Create a descriptor wallet and mine enough blocks for coinbase maturity. +log "setting up wallet and generating 101 maturity blocks" cli createwallet smoke >/dev/null 2>&1 || cli loadwallet smoke >/dev/null 2>&1 || true MINE_ADDR=$(cli getnewaddress) - -# 101 maturity blocks so the wallet's coinbase is spendable, then the -# target block that we'll inject into the ckpool log. cli generatetoaddress 101 "$MINE_ADDR" >/dev/null + +# Mine the Phase 1 target block (it exists in bitcoind before we inject the log line). BEFORE_HEIGHT=$(cli getblockcount) cli generatetoaddress 1 "$MINE_ADDR" >/dev/null -TARGET_HEIGHT=$((BEFORE_HEIGHT + 1)) -log "target block at height $TARGET_HEIGHT is live in bitcoind" +P1_HEIGHT=$((BEFORE_HEIGHT + 1)) +log "Phase 1 target block at height $P1_HEIGHT is live in bitcoind" # ---- start kamado-api ------------------------------------------------------- @@ -130,7 +492,7 @@ BITCOIN_RPC_URL="http://127.0.0.1:$BTC_RPC_PORT" \ BITCOIN_RPC_USER="$BTC_USER" \ BITCOIN_RPC_PASSWORD="$BTC_PASS" \ BITCOIN_ZMQ_BLOCK="" \ -CKPOOL_SOCKDIR="$SOCK_DIR" \ +CKPOOL_SOCKDIR="$CKPOOL_SOCK_DIR" \ CKPOOL_LOGFILE="$CKPOOL_LOG" \ DB_PATH="$KAMADO_DB" \ POLL_INTERVAL=2s \ @@ -143,59 +505,224 @@ for i in $(seq 1 30); do if curl -sf "http://127.0.0.1:$KAMADO_PORT/api/snapshot" >/dev/null 2>&1; then ready=true; break fi - # Bail early if the process died. kill -0 "$KAMADO_PID" 2>/dev/null || die "kamado-api exited unexpectedly (see $WORK_DIR/kamado.log)" sleep 1 done $ready || die "kamado-api did not become ready within 30s (see $WORK_DIR/kamado.log)" +kill -0 "$KAMADO_PID" 2>/dev/null \ + || die "kamado-api (pid $KAMADO_PID) is not running — port $KAMADO_PORT may already be in use" log "kamado-api ready (pid $KAMADO_PID)" -# ---- inject the ckpool solve line ------------------------------------------- -# -# The block is already in bitcoind (generated above), so the tailer's -# enrichment call (getblockhash → getblock) will succeed immediately. +# ============================================================ +# Phase 1: basic log injection +# ============================================================ +log "" +log "=== Phase 1: log-injection path ===" -log "writing 'Solved and confirmed block $TARGET_HEIGHT' to ckpool log" -printf 'Solved and confirmed block %d\n' "$TARGET_HEIGHT" >>"$CKPOOL_LOG" +log "writing 'Solved and confirmed block $P1_HEIGHT' to ckpool log" +printf 'Solved and confirmed block %d\n' "$P1_HEIGHT" >> "$CKPOOL_LOG" -# ---- wait for the block to appear in /api/blocks ---------------------------- - -log "polling /api/blocks for height $TARGET_HEIGHT (up to 30s)…" +log "polling /api/blocks for height $P1_HEIGHT (up to 30s)…" found=false for i in $(seq 1 30); do RESP=$(curl -sf "http://127.0.0.1:$KAMADO_PORT/api/blocks" 2>/dev/null || echo "[]") - if echo "$RESP" | jq -e "any(.[]; .height == $TARGET_HEIGHT)" >/dev/null 2>&1; then + if echo "$RESP" | jq -e "any(.[]; .height == $P1_HEIGHT)" >/dev/null 2>&1; then found=true; break fi sleep 1 done - if ! $found; then log "last /api/blocks response: $(curl -sf "http://127.0.0.1:$KAMADO_PORT/api/blocks" || echo "(unreachable)")" log "kamado-api log tail:" tail -20 "$WORK_DIR/kamado.log" | sed 's/^/ /' - die "block $TARGET_HEIGHT did not appear in /api/blocks within 30s" + die "Phase 1: block $P1_HEIGHT did not appear in /api/blocks within 30s" fi -# ---- verify enrichment: block must carry a hash from bitcoind --------------- - -BLOCK_HASH=$( +P1_HASH=$( curl -sf "http://127.0.0.1:$KAMADO_PORT/api/blocks" \ - | jq -r "first(.[] | select(.height == $TARGET_HEIGHT) | .hash // \"\")" + | jq -r "first(.[] | select(.height == $P1_HEIGHT) | .hash // \"\")" ) -if [[ -z "$BLOCK_HASH" || "$BLOCK_HASH" == "null" ]]; then - die "block $TARGET_HEIGHT in /api/blocks but hash enrichment failed (hash=$BLOCK_HASH)" +[[ -n "$P1_HASH" && "$P1_HASH" != "null" ]] \ + || die "Phase 1: block $P1_HEIGHT appeared but hash enrichment failed" +[[ "$P1_HASH" =~ ^[0-9a-f]{64}$ ]] \ + || die "Phase 1: hash looks wrong: $P1_HASH" +P1_CANONICAL=$(cli getblockhash "$P1_HEIGHT") +[[ "$P1_HASH" == "$P1_CANONICAL" ]] \ + || die "Phase 1: hash mismatch: api=$P1_HASH bitcoind=$P1_CANONICAL" + +log "Phase 1 PASS: block $P1_HEIGHT → hash $P1_HASH (matches bitcoind)" + +# ============================================================ +# Phase 2: full stratum path (requires ckpool) +# ============================================================ +if ! $CKPOOL_AVAILABLE; then + log "" + log "=== Phase 2: SKIPPED (ckpool not available) ===" + log "To run Phase 2 set CKPOOL_BIN=" + printf '[smoke] PASS: Phase 1 (log-injection path) verified\n' + exit 0 fi -# Verify the hash looks like a real block hash (64 hex chars). -if [[ ! "$BLOCK_HASH" =~ ^[0-9a-f]{64}$ ]]; then - die "block hash looks wrong: $BLOCK_HASH" +log "" +log "=== Phase 2: stratum path ($(basename "$CKPOOL_BIN")) ===" + +P2_HEIGHT=$((P1_HEIGHT + 1)) # the block ckpool will mine via stratum +# Any valid address works; ckpool-solo uses it for the coinbase output. +POOL_ADDR=$(cli getnewaddress) + +# ---- write ckpool.conf ------------------------------------------------------ +# ckpool parses mindiff/startdiff as int64_t. Regtest network difficulty +# (≈4.65e-10) is clamped to 1 internally, so clients must produce a share +# with sdiff ≥ 1 for acceptance. Set both to 1 (the minimum meaningful +# integer value) so ckpool hands out diff-1 work. Expected solve time: +# ~2^32 hashes per process; at ~1.5 MH/s per Python worker × 8 workers +# ≈ 360s on average. +CKPOOL_CONF="$WORK_DIR/ckpool.conf" +cat > "$CKPOOL_CONF" << CONF_EOF +{ + "btcd": [ + { + "url": "127.0.0.1:$BTC_RPC_PORT", + "auth": "$BTC_USER", + "pass": "$BTC_PASS", + "notify": false + } + ], + "btcaddress": "$POOL_ADDR", + "btcsig": "/KamadoSmoke/", + "blockpoll": 100, + "update_interval": 30, + "serverurl": ["0.0.0.0:$STRATUM_PORT"], + "mindiff": 1, + "startdiff": 1, + "maxdiff": 0, + "logdir": "$WORK_DIR" +} +CONF_EOF + +# ---- start ckpool ----------------------------------------------------------- + +log "starting ckpool (stratum port $STRATUM_PORT, logdir $WORK_DIR)" +"$CKPOOL_BIN" --btcsolo --config "$CKPOOL_CONF" --sockdir "$CKPOOL_SOCK_DIR" \ + >"$WORK_DIR/ckpool-stderr.log" 2>&1 & +CKPOOL_PID=$! + +log "waiting for ckpool stratum port $STRATUM_PORT…" +ready=false +for i in $(seq 1 30); do + (exec 3<>/dev/tcp/127.0.0.1/"$STRATUM_PORT" && exec 3>&-) 2>/dev/null \ + && { ready=true; break; } || true + kill -0 "$CKPOOL_PID" 2>/dev/null \ + || die "ckpool exited unexpectedly (see $WORK_DIR/ckpool-stderr.log)" + sleep 1 +done +$ready || die "ckpool stratum port did not open within 30s (see $WORK_DIR/ckpool-stderr.log)" +kill -0 "$CKPOOL_PID" 2>/dev/null \ + || die "ckpool (pid $CKPOOL_PID) is not running — check $WORK_DIR/ckpool-stderr.log" +log "ckpool ready (pid $CKPOOL_PID)" + +# ---- run Python stratum miner in background --------------------------------- +# ckpool-solo requires the stratum username to be a valid Bitcoin address. +# The Python miner connects via stratum, receives diff-1 work, and mines real +# SHA256d hashes across multiple processes until one finds sdiff ≥ 1. + +MINER_LOG="$WORK_DIR/miner.log" +log "launching Python stratum miner for block $P2_HEIGHT…" +# PYTHONWARNINGS suppresses the "leaked semaphore objects" noise from +# multiprocessing.resource_tracker when daemon workers are killed mid-run. +PYTHONWARNINGS=ignore python3 "$STRATUM_MINER" \ + "127.0.0.1" "$STRATUM_PORT" "$POOL_ADDR" "$MINER_LOG" \ + >>"$MINER_LOG" 2>&1 & +MINER_PID=$! + +# ---- monitor mining progress until ckpool logs the block solve -------------- + +log "waiting up to 1800s for ckpool to log 'Solved and confirmed block $P2_HEIGHT'…" +MINE_START=$(date +%s) +last_report=0 +solved=false +for _i in $(seq 1 600); do + sleep 3 + + if ! kill -0 "$MINER_PID" 2>/dev/null; then + log "miner log (last 20 lines):" + tail -20 "$MINER_LOG" | sed 's/^/ /' + die "Phase 2: miner exited before block was solved" + fi + + if grep -q "Solved and confirmed block $P2_HEIGHT" "$CKPOOL_LOG" 2>/dev/null; then + solved=true + break + fi + + now=$(date +%s) + if (( now - last_report >= 30 )); then + last_report=$now + elapsed=$(( now - MINE_START )) + rate=$(grep -oE '[0-9]+(\.[0-9]+)? [kMG]H/s' "$MINER_LOG" 2>/dev/null | tail -1 || true) + if [[ -n "$rate" ]]; then + rate_val=${rate%% *} + rate_unit=${rate##* } + eta_s=$(awk -v v="$rate_val" -v u="${rate_unit:0:1}" -v e="$elapsed" \ + 'BEGIN { m=(u=="k")?1000:(u=="M")?1e6:(u=="G")?1e9:1; r=4294967296/(v*m)-e; printf "%.0f", (r>0)?r:0 }') + log " mining: elapsed=${elapsed}s rate=$rate ETA~${eta_s}s" + else + log " mining: elapsed=${elapsed}s (hashrate not yet reported)" + fi + fi + + if (( $(date +%s) - MINE_START >= 1800)); then break; fi +done + +kill "$MINER_PID" 2>/dev/null || true +MINER_PID="" + +if ! $solved; then + log "miner log (last 30 lines):" + tail -30 "$MINER_LOG" | sed 's/^/ /' + log "ckpool log (last 20 lines):" + tail -20 "$CKPOOL_LOG" | sed 's/^/ /' + log "ckpool stderr (last 10 lines):" + tail -10 "$WORK_DIR/ckpool-stderr.log" | sed 's/^/ /' + die "Phase 2: block $P2_HEIGHT not solved within 1800s" fi -# Cross-check: the hash must match what bitcoind says is at that height. -CANONICAL=$(cli getblockhash "$TARGET_HEIGHT") -if [[ "$BLOCK_HASH" != "$CANONICAL" ]]; then - die "hash mismatch: kamado has $BLOCK_HASH, bitcoind says $CANONICAL" +log "ckpool logged 'Solved and confirmed block $P2_HEIGHT' ✓" + +# ---- wait for block to appear in /api/blocks -------------------------------- + +log "polling /api/blocks for height $P2_HEIGHT (up to 30s)…" +found=false +for i in $(seq 1 30); do + RESP=$(curl -sf "http://127.0.0.1:$KAMADO_PORT/api/blocks" 2>/dev/null || echo "[]") + if echo "$RESP" | jq -e "any(.[]; .height == $P2_HEIGHT)" >/dev/null 2>&1; then + found=true; break + fi + sleep 1 +done +if ! $found; then + log "last /api/blocks response: $(curl -sf "http://127.0.0.1:$KAMADO_PORT/api/blocks" || echo "(unreachable)")" + log "kamado-api log tail:" + tail -20 "$WORK_DIR/kamado.log" | sed 's/^/ /' + die "Phase 2: block $P2_HEIGHT did not appear in /api/blocks within 30s" fi -pass "block $TARGET_HEIGHT → hash $BLOCK_HASH (matches bitcoind canonical)" +# ---- cross-check hash against bitcoind -------------------------------------- + +P2_HASH=$( + curl -sf "http://127.0.0.1:$KAMADO_PORT/api/blocks" \ + | jq -r "first(.[] | select(.height == $P2_HEIGHT) | .hash // \"\")" +) +[[ -n "$P2_HASH" && "$P2_HASH" != "null" ]] \ + || die "Phase 2: block $P2_HEIGHT in /api/blocks but hash enrichment failed" +[[ "$P2_HASH" =~ ^[0-9a-f]{64}$ ]] \ + || die "Phase 2: hash looks wrong: $P2_HASH" +P2_CANONICAL=$(cli getblockhash "$P2_HEIGHT") +[[ "$P2_HASH" == "$P2_CANONICAL" ]] \ + || die "Phase 2: hash mismatch: api=$P2_HASH bitcoind=$P2_CANONICAL" + +log "Phase 2 PASS: block $P2_HEIGHT → hash $P2_HASH (matches bitcoind)" + +# ---- all done --------------------------------------------------------------- + +printf '[smoke] PASS: Phase 1 (log-injection) + Phase 2 (stratum → submitblock) both verified\n'