From daa87e23521f980c354afdcd94065a4dbe15aea7 Mon Sep 17 00:00:00 2001 From: satoshi Date: Tue, 19 May 2026 01:29:49 +0300 Subject: [PATCH] Kill ckpool on bitcoind failure so miners can failover When bitcoind is unreachable for 3+ consecutive polls and no block submission is pending, SIGTERM the ckpool process so miners disconnect and failover to backup pools. Adds a red dashboard banner when Bitcoin Core is down. --- api/cmd/kamado-api/main.go | 36 ++++++++++++++++++++++++++++ api/internal/state/aggregator.go | 40 +++++++++++++++++++++++++++++++- ui/src/lib/HealthBanners.svelte | 19 ++++++++++++++- 3 files changed, 93 insertions(+), 2 deletions(-) diff --git a/api/cmd/kamado-api/main.go b/api/cmd/kamado-api/main.go index 3bf27f4..91f19a5 100644 --- a/api/cmd/kamado-api/main.go +++ b/api/cmd/kamado-api/main.go @@ -66,6 +66,7 @@ func main() { agg.Store = blockStore agg.MempoolBaseURL = cfg.MempoolBaseURL agg.LogFilePath = cfg.CKPoolLogFile + agg.KillCKPool = killCKPool(log) // Transaction accelerator (prioritisetransaction). var accSvc *accelerator.Service @@ -164,3 +165,38 @@ func main() { os.Exit(1) } } + +// killCKPool returns a function that finds the ckpool process by name +// and sends it SIGTERM. Used by the aggregator to disconnect miners +// when bitcoind is unreachable so they can failover to other pools. +func killCKPool(log *slog.Logger) func() error { + return func() error { + entries, err := os.ReadDir("/proc") + if err != nil { + return fmt.Errorf("read /proc: %w", err) + } + for _, e := range entries { + if !e.IsDir() { + continue + } + pid, err := strconv.Atoi(e.Name()) + if err != nil { + continue + } + cmdline, err := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid)) + if err != nil { + continue + } + // ckpool's cmdline is NUL-separated; the first arg is the binary path. + if strings.Contains(string(cmdline), "ckpool") { + log.Info("sending SIGTERM to ckpool", "pid", pid) + proc, err := os.FindProcess(pid) + if err != nil { + return fmt.Errorf("find process %d: %w", pid, err) + } + return proc.Signal(syscall.SIGTERM) + } + } + return fmt.Errorf("ckpool process not found") + } +} diff --git a/api/internal/state/aggregator.go b/api/internal/state/aggregator.go index fef9166..02caee3 100644 --- a/api/internal/state/aggregator.go +++ b/api/internal/state/aggregator.go @@ -190,6 +190,12 @@ type Aggregator struct { // refresh. Used by the WebSocket hub to push updates to clients. OnRefresh func(Snapshot) + // KillCKPool, if set, is called when bitcoind has been unreachable + // for several consecutive polls and no block submission is pending. + // Killing ckpool disconnects miners so they can failover to other + // pools instead of mining stale work. + KillCKPool func() error + mu sync.RWMutex snap Snapshot blocks []BlockRecord @@ -231,6 +237,12 @@ type Aggregator struct { // escalate to WARN. ckFailStreak int + // btcFailStreak counts consecutive refreshes where bitcoind was + // unreachable. After a threshold, if no block submission is pending, + // we kill the ckpool process so miners can failover to other pools. + btcFailStreak int + ckpoolKilled bool // true after we sent SIGTERM, reset on bitcoind recovery + // readyOnce + ready closes the Ready() channel exactly once after // the first refresh completes. main blocks briefly on this so the // HTTP server doesn't serve a never-refreshed (all-zeros) snapshot. @@ -424,6 +436,11 @@ func (a *Aggregator) refresh(ctx context.Context) { if bi, err := a.RPC.GetBlockchainInfo(ctx); err == nil { next.Chain = bi next.BitcoinOK = true + if a.btcFailStreak > 0 { + a.Log.Info("bitcoind recovered", "after_failures", a.btcFailStreak) + } + a.btcFailStreak = 0 + a.ckpoolKilled = false // Track when the tip height last changed. if bi.Blocks != a.lastTipHeight { a.lastTipHeight = bi.Blocks @@ -436,7 +453,8 @@ func (a *Aggregator) refresh(ctx context.Context) { next.NetworkHashrateHs = nh } } else { - a.Log.Warn("bitcoind getblockchaininfo failed", "err", err) + a.btcFailStreak++ + a.Log.Warn("bitcoind getblockchaininfo failed", "err", err, "streak", a.btcFailStreak) if next.LastError == "" { next.LastError = err.Error() } @@ -675,6 +693,26 @@ func (a *Aggregator) refresh(ctx context.Context) { cb(pushed) } a.markReady() + + // Kill ckpool when bitcoind is unreachable so miners can failover. + // Conditions: 3+ consecutive failures, no pending block submission, + // haven't already killed it, and a kill callback is configured. + const btcFailThreshold = 3 + submitGap := next.BlockSubmitAttempts - next.BlockSubmitsConfirmed + if a.btcFailStreak >= btcFailThreshold && !a.ckpoolKilled && a.KillCKPool != nil { + if submitGap > 0 { + a.Log.Warn("bitcoind down but block submission pending, keeping ckpool alive", + "submit_gap", submitGap, "streak", a.btcFailStreak) + } else { + a.Log.Warn("bitcoind unreachable, killing ckpool so miners can failover", + "streak", a.btcFailStreak) + if err := a.KillCKPool(); err != nil { + a.Log.Error("failed to kill ckpool", "err", err) + } else { + a.ckpoolKilled = true + } + } + } } // AckBestDiff records the current best_diff as acknowledged so the UI diff --git a/ui/src/lib/HealthBanners.svelte b/ui/src/lib/HealthBanners.svelte index 5f54ee7..438ec00 100644 --- a/ui/src/lib/HealthBanners.svelte +++ b/ui/src/lib/HealthBanners.svelte @@ -25,10 +25,22 @@ // Alarm when ZMQ is 3+ min older than the last tip change. return zmqAge > tipAge + 180; }); + + const bitcoindDown = $derived(snap.data != null && !snap.data.bitcoin_ok); -{#if submitGap > 0 || zmqStale} +{#if bitcoindDown || submitGap > 0 || zmqStale}
+ {#if bitcoindDown} + + {/if} {#if submitGap > 0}