Fix config generation error for zmq-enabled on upgrade

The field was moved to the top level but not marked nullable, so
upgrades from older configs (where it lived under advanced) failed
with "Field Is Not Nullable". The entrypoint already falls back to
true via yq's // operator.

Makefile

@@ -6,13 +6,16 @@ TS_FILES := $(shell find ./scripts -name \*.ts 2>/dev/null)
 # Where KamadoPool source lives (buildx passes it as a named context).
 KAMADO_SRC ?= ../KamadoPool
 
+# Buildx builder name. Must support linux/amd64 and linux/arm64.
+BUILDER ?= kamado-multiarch
+
 .DELETE_ON_ERROR:
 
 # Default: build both arches and verify a universal package.
 all: verify
 
-# Single-arch convenience targets — much faster if you don't have
-# qemu-user-static registered for cross-arch emulation.
+# Single-arch convenience targets — much faster when you don't need
+# cross-arch emulation.
 x86: ARCH = x86_64
 x86:
 	@rm -f docker-images/aarch64.tar
@@ -23,6 +26,14 @@ arm:
 	@rm -f docker-images/x86_64.tar
 	$(MAKE) ARCH=aarch64
 
+# Register qemu for cross-arch builds and create a multiplatform builder.
+setup:
+	docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+	-docker buildx rm $(BUILDER) 2>/dev/null
+	docker buildx create --name $(BUILDER) --platform linux/amd64,linux/arm64 --use
+	docker buildx inspect --bootstrap $(BUILDER)
+	@echo "Ready. 'make' will now build universal packages."
+
 verify: $(PKG_ID).s9pk
 	start-sdk verify s9pk $(PKG_ID).s9pk
 	@echo "Package: $(PKG_ID).s9pk ($(shell du -h $(PKG_ID).s9pk | cut -f1))"
@@ -38,7 +49,7 @@ scripts/embassy.js: $(TS_FILES)
 docker-images/x86_64.tar: Dockerfile docker_entrypoint.sh
 ifneq ($(ARCH),aarch64)
 	mkdir -p docker-images
-	docker buildx build \
+	docker buildx build --builder $(BUILDER) --progress=plain \
 		--tag start9/$(PKG_ID)/main:$(PKG_VERSION) \
 		--platform linux/amd64 \
 		--build-arg ARCH=x86_64 \
@@ -50,7 +61,7 @@ endif
 docker-images/aarch64.tar: Dockerfile docker_entrypoint.sh
 ifneq ($(ARCH),x86_64)
 	mkdir -p docker-images
-	docker buildx build \
+	docker buildx build --builder $(BUILDER) --progress=plain \
 		--tag start9/$(PKG_ID)/main:$(PKG_VERSION) \
 		--platform linux/arm64 \
 		--build-arg ARCH=aarch64 \
@@ -60,9 +71,17 @@ ifneq ($(ARCH),x86_64)
 endif
 
 # ── Pack ─────────────────────────────────────────────────────────────
+ifeq ($(ARCH),x86_64)
+DOCKER_DEPS := docker-images/x86_64.tar
+else ifeq ($(ARCH),aarch64)
+DOCKER_DEPS := docker-images/aarch64.tar
+else
+DOCKER_DEPS := docker-images/x86_64.tar docker-images/aarch64.tar
+endif
+
 $(PKG_ID).s9pk: manifest.yaml instructions.md icon.png LICENSE \
                 scripts/embassy.js \
-                docker-images/x86_64.tar docker-images/aarch64.tar
+                $(DOCKER_DEPS)
 ifeq ($(ARCH),x86_64)
 	@echo "start-sdk: packing x86_64 only"
 else ifeq ($(ARCH),aarch64)
@@ -76,4 +95,4 @@ clean:
 	rm -rf docker-images
 	rm -f $(PKG_ID).s9pk image.tar scripts/embassy.js
 
-.PHONY: all x86 arm verify install clean
+.PHONY: all x86 arm verify install clean setup

README.md

@@ -2,25 +2,57 @@
 
 StartOS 0.3.5.1 wrapper for [Kamado Pool](https://github.com/Relaxo143/KamadoPool), a modern solo Bitcoin mining pool built on a patched fork of CKPool-solo with a Go middleware API and Svelte real-time dashboard.
 
-## Build
+## Prerequisites
+
+- **Docker** with [buildx](https://docs.docker.com/build/buildx/) plugin
+- **deno** — bundles the TypeScript embassy procedures
+- **yq** — parses `manifest.yaml` for package metadata
+- **start-sdk** — packs and verifies the `.s9pk` (from the [StartOS SDK](https://docs.start9.com/latest/developer-docs/specification/))
+- **KamadoPool source** — local sibling checkout at `../KamadoPool` (override with `KAMADO_SRC=/path/to/KamadoPool`)
+
+## Building
+
+### Universal package (x86_64 + aarch64)
+
+A universal `.s9pk` runs on any StartOS machine regardless of architecture. This is the default and what you should ship.
+
+**One-time setup** (re-run after each reboot):
+
+```sh
+make setup
+```
+
+This registers qemu binfmt handlers for cross-architecture emulation and creates a multiplatform Docker buildx builder. The arm64 build runs under emulation on x86 hosts, so it is significantly slower (~5-10x) than native.
+
+**Build:**
 
 ```sh
 make
 ```
 
-This runs `deno` to bundle the embassy TypeScript procedures, builds a multi-arch OCI image via `docker buildx`, and packs everything into `kamado-pool.s9pk` using `start-sdk`.
+This bundles the TypeScript procedures, builds Docker images for both architectures, and packs `kamado-pool.s9pk`. The universal package is roughly double the size of a single-arch package since it contains two Docker images.
 
-The build pulls KamadoPool source from a **local sibling checkout** (`../KamadoPool` by default) via a docker buildx named build context — no GitHub clone, no pinned SHA. If your checkout lives elsewhere, override it:
+### Single-arch (development)
+
+For faster iteration when you only need one architecture:
 
 ```sh
+make x86    # x86_64 only
+make arm    # aarch64 only
+```
+
+These skip the other architecture entirely — no emulation overhead.
+
+### Overrides
+
+```sh
+# Use a different KamadoPool source directory
 make KAMADO_SRC=/path/to/KamadoPool
+
+# Use a different buildx builder
+make BUILDER=my-builder
 ```
 
-## Install
-
-```sh
-make install
-```
 
 ## License
 

docker_entrypoint.sh

@@ -47,7 +47,7 @@ export MINDIFF=$(q '.advanced.mindiff // 1000')
 export MAXDIFF=$(q '.advanced.maxdiff // 0')
 export DROPIDLE=$(q '.advanced.dropidle // 0')
 LOG_LEVEL=$(q '.advanced.log-level // "info"')
-ZMQ_ENABLED=$(q '.advanced.zmq-enabled // true')
+ZMQ_ENABLED=$(q '.zmq-enabled // true')
 TLS_ENABLED=$(q '.tls.enabled // "disabled"')
 TLS_PORT=$(q '.tls.port // 3334')
 
@@ -105,6 +105,10 @@ export UPDATE_INTERVAL_S=30
 # disabled the second bind is harmless (nothing connects to it).
 export TLS_INTERNAL_PORT=3437
 
+# CKPool loglevel: 6 = LOG_INFO, required for share-level logging
+# (Accepted/Rejected client lines) used by the stats feature.
+export CKPOOL_LOGLEVEL=6
+
 mkdir -p "${LOGDIR}" "${SOCKET_DIR}" /etc/ckpool
 
 # Render ckpool.conf using the same sed approach as the upstream
@@ -132,10 +136,6 @@ sed \
     -e "s|\${ZMQ_BLOCK}|${ZMQ_BLOCK}|g" \
     "${TEMPLATE}" > "${CONF}"
 
-echo "kamado-entrypoint: starting ckpool (solo, ${BITCOIND_VARIANT}) on port ${STRATUM_PORT}"
-/usr/local/bin/ckpool --btcsolo --config "${CONF}" --sockdir "${SOCKET_DIR}" --log-shares &
-CKPOOL_PID=$!
-
 # DB_PATH must live on the persisted main volume; the default
 # /var/lib/kamado/kamado.db is ephemeral container storage.
 KAMADO_DATA_DIR=/root/.kamado/data
@@ -157,6 +157,42 @@ echo "kamado-entrypoint: starting kamado-api"
 /usr/local/bin/kamado-api &
 API_PID=$!
 
+# wait_for_bitcoind blocks until bitcoind responds to getblockchaininfo.
+# Called before each ckpool start so we don't launch ckpool into a wall.
+wait_for_bitcoind() {
+    local url="http://${BITCOIN_RPC_USER}:${BITCOIN_RPC_PASSWORD}@${BITCOIN_RPC_HOST}:${BITCOIN_RPC_PORT}"
+    local backoff=2
+    while true; do
+        if curl -sf --max-time 5 \
+            -d '{"jsonrpc":"1.0","method":"getblockchaininfo","params":[]}' \
+            -H 'Content-Type: application/json' \
+            "${url}" >/dev/null 2>&1; then
+            return 0
+        fi
+        echo "kamado-entrypoint: waiting for bitcoind (retry in ${backoff}s)..."
+        sleep "${backoff}"
+        backoff=$(( backoff < 30 ? backoff * 2 : 30 ))
+    done
+}
+
+# Supervised ckpool restart loop. When ckpool exits (killed by the API
+# on bitcoind failure, or crashed), we wait for bitcoind to be reachable
+# again before restarting. This keeps ckpool alive when bitcoind is
+# healthy and lets miners failover when it's not — without restart-
+# looping the entire container.
+run_ckpool_loop() {
+    while true; do
+        wait_for_bitcoind
+        echo "kamado-entrypoint: starting ckpool (solo, ${BITCOIND_VARIANT}) on port ${STRATUM_PORT}, loglevel ${CKPOOL_LOGLEVEL}"
+        /usr/local/bin/ckpool --btcsolo --config "${CONF}" --sockdir "${SOCKET_DIR}" --log-shares -l "${CKPOOL_LOGLEVEL}"
+        EXIT_CODE=$?
+        echo "kamado-entrypoint: ckpool exited (code ${EXIT_CODE}), will restart after bitcoind is reachable"
+        sleep 2
+    done
+}
+run_ckpool_loop &
+CKPOOL_LOOP_PID=$!
+
 # Optional TLS stratum via stunnel sidecar.
 STUNNEL_PID=""
 if [[ "${TLS_ENABLED}" == "enabled" ]]; then
@@ -301,16 +337,20 @@ fi
 
 term() {
     echo "kamado-entrypoint: SIGTERM — shutting down"
-    kill -TERM "${API_PID}" "${CKPOOL_PID}" ${STUNNEL_PID:-} 2>/dev/null || true
-    wait "${API_PID}" "${CKPOOL_PID}" ${STUNNEL_PID:-} 2>/dev/null || true
+    kill -TERM "${API_PID}" "${CKPOOL_LOOP_PID}" ${STUNNEL_PID:-} 2>/dev/null || true
+    # Kill any running ckpool process inside the loop.
+    pkill -TERM -f '/usr/local/bin/ckpool' 2>/dev/null || true
+    wait "${API_PID}" "${CKPOOL_LOOP_PID}" ${STUNNEL_PID:-} 2>/dev/null || true
     exit 0
 }
 trap term TERM INT
 
-# shellcheck disable=SC2086
-wait -n ${CKPOOL_PID} ${API_PID} ${STUNNEL_PID:-}
+# The API is the critical process — if it exits, the container should
+# restart. The ckpool loop manages its own lifecycle independently.
+wait "${API_PID}"
 EXIT_CODE=$?
-echo "kamado-entrypoint: a supervised process exited (${EXIT_CODE}), stopping the rest"
-kill -TERM "${API_PID}" "${CKPOOL_PID}" ${STUNNEL_PID:-} 2>/dev/null || true
+echo "kamado-entrypoint: kamado-api exited (${EXIT_CODE}), stopping the rest"
+kill -TERM "${CKPOOL_LOOP_PID}" ${STUNNEL_PID:-} 2>/dev/null || true
+pkill -TERM -f '/usr/local/bin/ckpool' 2>/dev/null || true
 wait || true
 exit "${EXIT_CODE}"

manifest.yaml

@@ -131,9 +131,11 @@ backup:
       - create
       - /mnt/backup
       - /root/.kamado
+      - /root/.ckpool
     mounts:
       BACKUP: /mnt/backup
       main: /root/.kamado
+      ckpool: /root/.ckpool
   restore:
     type: docker
     image: compat
@@ -144,9 +146,38 @@ backup:
       - restore
       - /mnt/backup
       - /root/.kamado
+      - /root/.ckpool
     mounts:
       BACKUP: /mnt/backup
       main: /root/.kamado
+      ckpool: /root/.ckpool
+
+actions:
+  stratum-smoke-test:
+    name: "Pool Status"
+    description: "Displays a full status snapshot: Bitcoin Core sync state, ckpool health, connected miners, hashrate, found blocks, and submit-gap diagnostics."
+    warning: ~
+    allowed-statuses:
+      - running
+    implementation:
+      type: script
+  regen-tls-cert:
+    name: "Regenerate TLS Certificate"
+    description: "Clears the current stratum TLS certificate so a fresh one is generated on the next service start. Use this to rotate an expired or untrusted certificate."
+    warning: "Miners connected via TLS will be disconnected on restart and will need to accept or re-pin the new certificate fingerprint."
+    allowed-statuses:
+      - running
+      - stopped
+    implementation:
+      type: script
+  reset-latency:
+    name: "Reset Block Latency"
+    description: "Zeroes the block-update latency counters (avg, last, wasted work, block count). Use this after tuning ZMQ or ckpool to start fresh measurements."
+    warning: ~
+    allowed-statuses:
+      - running
+    implementation:
+      type: script
 
 migrations:
   from:

scripts/embassy.ts

@@ -4,3 +4,4 @@ export { dependencies } from "./procedures/dependencies.ts";
 export { health } from "./procedures/healthChecks.ts";
 export { migration } from "./procedures/migrations.ts";
 export { properties } from "./procedures/properties.ts";
+export { action } from "./procedures/actions.ts";

scripts/procedures/actions.ts

@@ -0,0 +1,330 @@
+import { types as T } from "../deps.ts";
+
+const POOL_HOST = "kamado-pool.embassy";
+const STRATUM_PORT = 3333; // shown in output; direct TCP not testable from sandbox
+const API_BASE = `http://${POOL_HOST}:8080`;
+
+// ── formatting helpers ────────────────────────────────────────────────────────
+
+function ts(): string { return new Date().toISOString().slice(11, 22); }
+
+function fmtHR(hs: number): string {
+  if (hs <= 0) return "0 H/s";
+  if (hs >= 1e18) return `${(hs / 1e18).toFixed(2)} EH/s`;
+  if (hs >= 1e15) return `${(hs / 1e15).toFixed(2)} PH/s`;
+  if (hs >= 1e12) return `${(hs / 1e12).toFixed(2)} TH/s`;
+  if (hs >= 1e9)  return `${(hs / 1e9 ).toFixed(2)} GH/s`;
+  if (hs >= 1e6)  return `${(hs / 1e6 ).toFixed(2)} MH/s`;
+  if (hs >= 1e3)  return `${(hs / 1e3 ).toFixed(2)} kH/s`;
+  return `${hs.toFixed(0)} H/s`;
+}
+
+function fmtDiff(d: number): string {
+  if (d <= 0) return "0";
+  if (d >= 1e12) return `${(d / 1e12).toFixed(2)}T`;
+  if (d >= 1e9)  return `${(d / 1e9 ).toFixed(2)}G`;
+  if (d >= 1e6)  return `${(d / 1e6 ).toFixed(2)}M`;
+  if (d >= 1e3)  return `${(d / 1e3 ).toFixed(2)}K`;
+  return d.toFixed(2);
+}
+
+function fmtUptime(s: number): string {
+  if (s <= 0) return "0m";
+  const d = Math.floor(s / 86400);
+  const h = Math.floor((s % 86400) / 3600);
+  const m = Math.floor((s % 3600) / 60);
+  if (d > 0) return `${d}d ${h}h ${m}m`;
+  if (h > 0) return `${h}h ${m}m`;
+  return `${m}m`;
+}
+
+function fmtBTC(btc: number): string { return `${btc.toFixed(8)} BTC`; }
+function pad(s: string, w: number): string { return s.padEnd(w); }
+
+const chainNames: Record<string, string> = { main: "mainnet" };
+function displayChain(c: string): string { return chainNames[c] ?? c; }
+
+// ── snapshot shape ────────────────────────────────────────────────────────────
+
+interface PoolStats {
+  workers: number; users: number; accepted: number; rejected: number;
+  shares: number; dsps1: number; dsps5: number; dsps60: number; dsps1440: number;
+}
+interface Chain {
+  chain: string; blocks: number; headers: number; difficulty: number;
+  initialblockdownload: boolean; verificationprogress: number; bestblockhash: string;
+}
+interface Worker { worker: string; dsps1: number; bestdiff: number; bestever: number; idle: boolean; }
+interface Client  { id: number; workername: string; diff: number; dsps1: number; useragent: string; }
+interface BlockRecord {
+  height: number; hash?: string; reward_btc?: number; found_at: string;
+  orphaned_at?: string; chain?: string;
+}
+interface Snapshot {
+  pool: PoolStats | null;
+  uptime_seconds: number;
+  hashrate_hs_1m: number; hashrate_hs_5m: number;
+  hashrate_hs_1h: number; hashrate_hs_24h: number;
+  best_diff: number; cumulative_shares: number;
+  next_block_reward_btc: number; next_difficulty_percent: number;
+  chain: Chain | null; network_hashrate_hs: number;
+  recent_blocks: BlockRecord[];
+  ckpool_ok: boolean; bitcoin_ok: boolean; last_error?: string;
+  block_submit_attempts: number; block_submits_confirmed: number;
+  zmq_enabled: boolean; zmq_stale: boolean;
+  has_last_zmq_event: boolean; last_zmq_event_age: number;
+  workers: Worker[]; clients: Client[];
+}
+
+// ── actions ───────────────────────────────────────────────────────────────────
+
+export const action: T.ExpectedExports.action = {
+
+  // ── Pool status ─────────────────────────────────────────────────────────────
+  "stratum-smoke-test": async (effects) => {
+    const lines: string[] = [];
+    const log = (s: string) => lines.push(`[${ts()}] ${s}`);
+    const sep = () => log("──────────────────────────────────────────");
+    let allPass = true;
+
+    log("Kamado Pool — Status");
+    sep();
+
+    // ── 1. Full snapshot ────────────────────────────────────────────────────
+    log("● Pool & Bitcoin Core status");
+    let snap: Snapshot | null = null;
+    try {
+      const res = await effects.fetch(`${API_BASE}/api/snapshot`);
+      snap = await res.json() as Snapshot;
+    } catch (e) {
+      log(`  API unreachable: ${(e as Error).message}`);
+      allPass = false;
+    }
+
+    if (snap) {
+      const chain = snap.chain;
+      const pool  = snap.pool;
+
+      log(`  Bitcoin Core:     ${snap.bitcoin_ok ? "OK" : "FAIL"}`);
+      if (chain) {
+        const ibd     = chain.initialblockdownload;
+        const syncPct = (chain.verificationprogress * 100).toFixed(2);
+        log(`  Network:          ${chain.chain}  height=${chain.blocks}  headers=${chain.headers}`);
+        log(`  Sync:             ${ibd ? `IBD — ${syncPct}% (pool will not mine until synced)` : `${syncPct}% — fully synced`}`);
+        log(`  Block hash:       ${chain.bestblockhash}`);
+        log(`  Difficulty:       ${fmtDiff(chain.difficulty)}`);
+        log(`  Network hashrate: ${fmtHR(snap.network_hashrate_hs)}`);
+        if (ibd) allPass = false;
+      }
+      if (!snap.bitcoin_ok) allPass = false;
+
+      if (snap.next_block_reward_btc > 0)
+        log(`  Next block reward: ${fmtBTC(snap.next_block_reward_btc)}`);
+      if (snap.next_difficulty_percent !== 0) {
+        const sign = snap.next_difficulty_percent > 0 ? "+" : "";
+        log(`  Next diff adjust:  ${sign}${snap.next_difficulty_percent.toFixed(1)}%`);
+      }
+
+      if (snap.zmq_enabled) {
+        const age = snap.has_last_zmq_event
+          ? `last event ${snap.last_zmq_event_age.toFixed(0)}s ago`
+          : "no event yet since startup";
+        log(`  ZMQ:              ${snap.zmq_stale ? "STALE — " : "OK — "}${age}`);
+        if (snap.zmq_stale) allPass = false;
+      } else {
+        log(`  ZMQ:              disabled`);
+      }
+
+      log("");
+      log(`  ckpool process:   ${snap.ckpool_ok ? "OK" : "FAIL"}`);
+      if (!snap.ckpool_ok) allPass = false;
+      if (snap.last_error) log(`  last_error:       ${snap.last_error}`);
+      if (pool) {
+        log(`  Uptime:           ${fmtUptime(snap.uptime_seconds)}`);
+        log(`  Workers online:   ${pool.workers}  (${pool.users} user${pool.users !== 1 ? "s" : ""})`);
+        log(`  Shares accepted:  ${pool.accepted}  rejected: ${pool.rejected}`);
+        log(`  Hashrate 1m:      ${fmtHR(snap.hashrate_hs_1m)}`);
+        log(`  Hashrate 5m:      ${fmtHR(snap.hashrate_hs_5m)}`);
+        log(`  Hashrate 1h:      ${fmtHR(snap.hashrate_hs_1h)}`);
+        log(`  Hashrate 24h:     ${fmtHR(snap.hashrate_hs_24h)}`);
+        log(`  Best share ever:  diff ${fmtDiff(snap.best_diff)}`);
+        log(`  Cumul. work:      ${fmtDiff(snap.cumulative_shares)} diff-1 shares`);
+      }
+
+      const gap = snap.block_submit_attempts - snap.block_submits_confirmed;
+      log(`  Submit attempts:  ${snap.block_submit_attempts}  confirmed: ${snap.block_submits_confirmed}`);
+      log(`  Submit gap:       ${gap > 0 ? `WARN — ${gap} unconfirmed` : "OK (all confirmed)"}`);
+      if (gap > 0) allPass = false;
+
+      if (snap.clients && snap.clients.length > 0) {
+        log("");
+        log(`● Connected miners (${snap.clients.length})`);
+        for (const c of snap.clients) {
+          const tag = c.useragent ? ` [${c.useragent}]` : "";
+          log(`  ${pad(c.workername || `client#${c.id}`, 30)} diff=${fmtDiff(c.diff)}  ${fmtHR(c.dsps1 * 4294967296)}${tag}`);
+        }
+      } else {
+        log(""); log("● Connected miners: none");
+      }
+
+      if (snap.workers && snap.workers.length > 0) {
+        log(""); log("● Workers — best shares");
+        for (const w of snap.workers) {
+          const ever = w.bestever > 0 ? w.bestever : w.bestdiff;
+          log(`  ${pad(w.worker, 30)} best=${fmtDiff(ever)}${w.idle ? " (idle)" : ""}`);
+        }
+      }
+
+      log("");
+      if (snap.recent_blocks && snap.recent_blocks.length > 0) {
+        const currentNetwork = chain?.chain ?? "";
+        log(`● Blocks found (${snap.recent_blocks.length})`);
+        for (const b of snap.recent_blocks) {
+          let status = "";
+          if (b.orphaned_at) {
+            status = " ORPHANED";
+          } else if (b.chain && currentNetwork && b.chain !== currentNetwork) {
+            status = ` [${displayChain(b.chain)}]`;
+          }
+          const reward = b.reward_btc ? `  ${fmtBTC(b.reward_btc)}` : "";
+          const hash   = b.hash ? `  ${b.hash.slice(0, 16)}…` : "";
+          log(`  height=${b.height}  ${b.found_at.slice(0, 19)}${reward}${hash}${status}`);
+          log(`    chain="${b.chain ?? ""}"  orphaned_at="${b.orphaned_at ?? ""}"  hash="${b.hash ?? ""}"`);
+        }
+      } else {
+        log("● Blocks found: none yet");
+      }
+    }
+
+    // ── 1b. Debug: memory vs DB comparison ─────────────────────────────────
+    try {
+      const dbgRes = await effects.fetch(`${API_BASE}/api/admin/debug-blocks`);
+      const dbg = await dbgRes.json() as {
+        memory: { height: number; hash: string; chain: string; orphaned_at: string; found_at: string }[];
+        db: { height: number; hash: string; chain: string; orphaned_at: string; found_at: string }[];
+      };
+      log(""); log("● Debug: memory vs DB");
+      log(`  Memory blocks: ${dbg.memory?.length ?? 0}  DB blocks: ${dbg.db?.length ?? 0}`);
+      if (dbg.memory && dbg.memory.length > 0) {
+        for (const m of dbg.memory) {
+          const dbRow = dbg.db?.find(d => d.height === m.height);
+          const memOrph = m.orphaned_at || "none";
+          const dbOrph = dbRow?.orphaned_at || "none";
+          const match = memOrph === dbOrph ? "✓" : "MISMATCH";
+          log(`  h=${m.height} mem_orphan="${memOrph}" db_orphan="${dbOrph}" ${match}`);
+          if (memOrph !== dbOrph) {
+            log(`    mem: chain="${m.chain}" hash="${(m.hash || "").slice(0, 20)}…"`);
+            log(`    db:  chain="${dbRow?.chain ?? "?"}" hash="${(dbRow?.hash || "").slice(0, 20)}…"`);
+          }
+        }
+      }
+    } catch (e) {
+      log(`  (debug endpoint unavailable: ${(e as Error).message})`);
+    }
+
+    // ── 2. Stratum ───────────────────────────────────────────────────────────
+    log(""); sep();
+    log(`● Stratum (port ${STRATUM_PORT})`);
+    if (snap?.ckpool_ok) {
+      log("  ckpool process: healthy — stratum port is served by the same process");
+    } else {
+      log("  FAIL — ckpool is not running, stratum port will not accept connections");
+    }
+
+    // ── 3. Overall ──────────────────────────────────────────────────────────
+    log(""); sep();
+    const summary = allPass && (snap?.ckpool_ok ?? false)
+      ? "PASS — pool is healthy"
+      : "FAIL — see details above";
+    log(`Overall: ${summary}`);
+
+    return {
+      result: {
+        version: "0",
+        message: summary,
+        value: lines.join("\n"),
+        copyable: true,
+        qr: false,
+      },
+    };
+  },
+
+  // ── Reset block latency ─────────────────────────────────────────────────────
+  "reset-latency": async (effects) => {
+    try {
+      const res = await effects.fetch(`${API_BASE}/api/admin/reset-latency`, {
+        method: "POST",
+      });
+      if (!res.ok) {
+        const body = await res.text();
+        return {
+          result: {
+            version: "0",
+            message: "Failed to reset latency stats",
+            value: `API returned ${res.status}: ${body}`,
+            copyable: false,
+            qr: false,
+          },
+        };
+      }
+    } catch (e) {
+      return {
+        result: {
+          version: "0",
+          message: "Failed — API unreachable",
+          value: `Could not reach kamado-api: ${(e as Error).message}`,
+          copyable: false,
+          qr: false,
+        },
+      };
+    }
+    return {
+      result: {
+        version: "0",
+        message: "Block latency stats reset to zero",
+        value: "All latency counters (count, avg, last, wasted work) have been cleared.\nNew measurements will accumulate from the next block.",
+        copyable: false,
+        qr: false,
+      },
+    };
+  },
+
+  // ── Regenerate TLS certificate ──────────────────────────────────────────────
+  "regen-tls-cert": async (effects) => {
+    const VOLUME = "main";
+    const TLS_PATH = "tls";
+    const certFiles = ["stratum.crt", "stratum.key", "stratum.pem", "cert_version", "fingerprint.txt"];
+
+    const removed: string[] = [];
+    const missing: string[] = [];
+    for (const f of certFiles) {
+      try {
+        await effects.removeFile({ volumeId: VOLUME, path: `${TLS_PATH}/${f}` });
+        removed.push(f);
+      } catch {
+        missing.push(f);
+      }
+    }
+
+    const hadCert = removed.length > 0;
+    const message = hadCert
+      ? "TLS certificate cleared — restart the service to generate a new one"
+      : "No TLS certificate files found (TLS may not have been enabled)";
+
+    const detail = [
+      hadCert ? `Removed: ${removed.join(", ")}` : "No certificate files were present.",
+      missing.length > 0 ? `Already absent: ${missing.join(", ")}` : "",
+      "",
+      "Next steps:",
+      "  1. Restart Kamado Pool.",
+      "  2. The new certificate fingerprint will appear in the service logs",
+      '     (search for "stratum TLS SHA256 fingerprint").',
+      "  3. Provide the new fingerprint to miners that pin the certificate.",
+    ].filter(Boolean).join("\n");
+
+    return {
+      result: { version: "0", message, value: detail, copyable: false, qr: false },
+    };
+  },
+
+};

scripts/procedures/getConfig.ts

@@ -80,7 +80,7 @@ export const getConfig: T.ExpectedExports.getConfig = compat.getConfig({
       "Optionally accept stratum connections over TLS via an stunnel sidecar. A self-signed certificate is generated once on first start and persisted — miners must connect with TLS verification disabled (stratum+ssl:// plus a skip-verify flag).",
     "tag": {
       "id": "enabled",
-      "name": "Mode",
+      "name": "TLS Mode",
       "description": "Disable or enable TLS termination.",
       "variant-names": {
         "disabled": "Disabled",
@@ -104,18 +104,26 @@ export const getConfig: T.ExpectedExports.getConfig = compat.getConfig({
       },
     },
   },
+  "zmq-enabled": {
+    "type": "boolean",
+    "name": "Enable ZMQ Block Notifications",
+    "description":
+      "Subscribe to Bitcoin Core's hashblock ZMQ topic for sub-second block detection on the dashboard. Requires zmqpubhashblock enabled on bitcoind.",
+    "nullable": true,
+    "default": true,
+  },
   "advanced": {
     "type": "object",
     "name": "Advanced",
     "description":
-      "Fine-tuning knobs for vardiff, logging, and block notifications. Safe to ignore — the defaults work well for Bitaxe-class miners.",
+      "Fine-tuning knobs for vardiff, logging, and block explorer links. Safe to ignore — the defaults work well for Bitaxe-class miners.",
     "spec": {
       "pool-identifier": {
         "type": "string",
         "name": "Coinbase Tag",
         "description":
           "Short string embedded in the coinbase transaction of solved blocks.",
-        "nullable": false,
+        "nullable": true,
         "default": "/Kamado/",
         "masked": false,
         "copyable": false,
@@ -125,7 +133,7 @@ export const getConfig: T.ExpectedExports.getConfig = compat.getConfig({
         "name": "Starting Difficulty",
         "description":
           "Initial vardiff target for new miner connections. Bitaxe-class miners typically land around 16384.",
-        "nullable": false,
+        "nullable": true,
         "default": 16384,
         "range": "[1,*)",
         "integral": true,
@@ -134,7 +142,7 @@ export const getConfig: T.ExpectedExports.getConfig = compat.getConfig({
         "type": "number",
         "name": "Minimum Difficulty",
         "description": "Floor for the vardiff algorithm.",
-        "nullable": false,
+        "nullable": true,
         "default": 1000,
         "range": "[1,*)",
         "integral": true,
@@ -144,7 +152,7 @@ export const getConfig: T.ExpectedExports.getConfig = compat.getConfig({
         "name": "Maximum Difficulty",
         "description":
           "Ceiling for the vardiff algorithm. 0 means no cap.",
-        "nullable": false,
+        "nullable": true,
         "default": 0,
         "range": "[0,*)",
         "integral": true,
@@ -154,18 +162,11 @@ export const getConfig: T.ExpectedExports.getConfig = compat.getConfig({
         "name": "Drop Idle (seconds)",
         "description":
           "Disconnect clients that have not submitted a share in this many seconds. 0 disables the idle disconnect.",
-        "nullable": false,
+        "nullable": true,
         "default": 0,
         "range": "[0,*)",
         "integral": true,
       },
-      "zmq-enabled": {
-        "type": "boolean",
-        "name": "Enable ZMQ Block Notifications",
-        "description":
-          "Subscribe to Bitcoin Core's hashblock ZMQ topic for sub-second chain refresh on the dashboard. Requires zmqpubhashblock on bitcoind.",
-        "default": true,
-      },
       "log-level": {
         "type": "enum",
         "name": "Log Level",
@@ -186,7 +187,7 @@ export const getConfig: T.ExpectedExports.getConfig = compat.getConfig({
           "Where Kamado's UI links should send you when you click a block hash or a user's BTC address. Defaults to mempool.space (the public instance). Switch to 'Custom' to point Kamado at a self-hosted mempool.space mirror — useful if you run mempool as another StartOS service or on the same network.",
         "tag": {
           "id": "type",
-          "name": "Source",
+          "name": "Block explorer for dashboard links",
           "description": "Public mempool.space, or your own instance.",
           "variant-names": {
             "default": "mempool.space (default)",
@@ -199,10 +200,10 @@ export const getConfig: T.ExpectedExports.getConfig = compat.getConfig({
           "custom": {
             "url": {
               "type": "string",
-              "name": "Mempool URL",
+              "name": "URL (e.g. your own Mempool instance's address)",
               "description":
                 "Base URL of your mempool instance, e.g. https://mempool.example.com. Kamado will append /address/<addr> and /block/<hash> to it, so the instance must follow the standard mempool.space URL layout.",
-              "nullable": false,
+              "nullable": true,
               "default": "",
               "pattern": "^https?://[^\\s]+$",
               "pattern-description":

scripts/procedures/healthChecks.ts

@@ -1,6 +1,7 @@
-import { healthUtil, types as T } from "../deps.ts";
+import { healthUtil, util, types as T } from "../deps.ts";
 
 const API_BASE = "http://kamado-pool.embassy:8080";
+const STARTING_GRACE_MS = 20_000;
 
 interface HealthPayload {
   ok: boolean;
@@ -11,65 +12,46 @@ interface HealthPayload {
   last_error?: string;
 }
 
-async function fetchHealth(): Promise<HealthPayload | null> {
+async function fetchHealth(effects: T.Effects): Promise<HealthPayload | null> {
   try {
-    const res = await fetch(`${API_BASE}/api/health`, {
-      signal: AbortSignal.timeout(5_000),
-    });
+    const res = await effects.fetch(`${API_BASE}/api/health`);
     return await res.json() as HealthPayload;
   } catch {
     return null;
   }
 }
 
-// Returns a "starting" result for the first 20 s after launch, then
-// "failing" so a crashed service doesn't look like it's perpetually booting.
-function apiUnreachable(duration: number, label: string): T.HealthProceduralResult {
-  if (duration < 20_000) {
-    return { status: "starting", message: `${label} is starting…` };
-  }
-  return { status: "failing", message: "Kamado API is unreachable — service may be down." };
-}
-
 export const health: T.ExpectedExports.health = {
-  // Overall reachability: uses the SDK helper so StartOS marks the
-  // service degraded (503) whenever ckpool or bitcoind is down.
   "web": async (effects, duration) => {
     return healthUtil.checkWebUrl(`${API_BASE}/api/health`)(effects, duration);
   },
 
-  // Is the stratum server up and accepting miner connections?
-  "ckpool": async (_effects, duration) => {
-    const h = await fetchHealth();
-    if (!h) return apiUnreachable(duration, "Stratum server");
-    if (h.ckpool) return { status: "passing", message: "Stratum server is running." };
+  "ckpool": async (effects, duration) => {
+    if (duration < STARTING_GRACE_MS) return util.errorCode(60, "Stratum server is starting…");
+    const h = await fetchHealth(effects);
+    if (!h) return util.error("Kamado API is unreachable — service may be down.");
+    if (h.ckpool) return util.ok;
     const detail = h.last_error ? ` (${h.last_error})` : "";
-    return { status: "failing", message: `Stratum server is not responding${detail}.` };
+    return util.error(`Stratum server is not responding${detail}.`);
   },
 
-  // Can we reach Bitcoin Core via RPC? (Required for block templates and
-  // block submission — if this is down, mining produces no valid work.)
-  "bitcoin": async (_effects, duration) => {
-    const h = await fetchHealth();
-    if (!h) return apiUnreachable(duration, "Bitcoin Core connection");
-    if (h.bitcoin) return { status: "passing", message: "Connected to Bitcoin Core." };
+  "bitcoin": async (effects, duration) => {
+    if (duration < STARTING_GRACE_MS) return util.errorCode(60, "Bitcoin Core connection is starting…");
+    const h = await fetchHealth(effects);
+    if (!h) return util.error("Kamado API is unreachable — service may be down.");
+    if (h.bitcoin) return util.ok;
     const detail = h.last_error ? ` (${h.last_error})` : "";
-    return { status: "failing", message: `Bitcoin Core RPC is unreachable${detail}.` };
+    return util.error(`Bitcoin Core RPC is unreachable${detail}.`);
   },
 
-  // Did every block submission get confirmed by bitcoind? A non-zero gap
-  // means at least one "Possible block solve" never got a "Solved and
-  // confirmed" response — the reward may have been lost or delayed.
-  "submit-gap": async (_effects, duration) => {
-    const h = await fetchHealth();
-    if (!h) return apiUnreachable(duration, "Block submission check");
+  "submit-gap": async (effects, duration) => {
+    if (duration < STARTING_GRACE_MS) return util.errorCode(60, "Block submission check is starting…");
+    const h = await fetchHealth(effects);
+    if (!h) return util.error("Kamado API is unreachable — service may be down.");
     const gap = h.submit_gap ?? 0;
-    if (gap === 0) return { status: "passing", message: "All block submissions confirmed." };
-    return {
-      status: "failing",
-      message:
-        `${gap} block${gap === 1 ? "" : "s"} submitted to bitcoind but not confirmed — ` +
-        `check Bitcoin Core logs for rejected or missing submissions.`,
-    };
+    if (gap === 0) return util.ok;
+    return util.error(
+      `${gap} block${gap === 1 ? "" : "s"} submitted to bitcoind but not confirmed — check Bitcoin Core logs.`,
+    );
   },
 };