Kill ckpool on bitcoind failure so miners can failover

When bitcoind is unreachable for 3+ consecutive polls and no block
submission is pending, SIGTERM the ckpool process so miners disconnect
and failover to backup pools. Adds a red dashboard banner when
Bitcoin Core is down.

api/cmd/kamado-api/main.go

@@ -66,6 +66,7 @@ func main() {
 	agg.Store = blockStore
 	agg.MempoolBaseURL = cfg.MempoolBaseURL
 	agg.LogFilePath = cfg.CKPoolLogFile
+	agg.KillCKPool = killCKPool(log)
 
 	// Transaction accelerator (prioritisetransaction).
 	var accSvc *accelerator.Service
@@ -164,3 +165,38 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+// killCKPool returns a function that finds the ckpool process by name
+// and sends it SIGTERM. Used by the aggregator to disconnect miners
+// when bitcoind is unreachable so they can failover to other pools.
+func killCKPool(log *slog.Logger) func() error {
+	return func() error {
+		entries, err := os.ReadDir("/proc")
+		if err != nil {
+			return fmt.Errorf("read /proc: %w", err)
+		}
+		for _, e := range entries {
+			if !e.IsDir() {
+				continue
+			}
+			pid, err := strconv.Atoi(e.Name())
+			if err != nil {
+				continue
+			}
+			cmdline, err := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid))
+			if err != nil {
+				continue
+			}
+			// ckpool's cmdline is NUL-separated; the first arg is the binary path.
+			if strings.Contains(string(cmdline), "ckpool") {
+				log.Info("sending SIGTERM to ckpool", "pid", pid)
+				proc, err := os.FindProcess(pid)
+				if err != nil {
+					return fmt.Errorf("find process %d: %w", pid, err)
+				}
+				return proc.Signal(syscall.SIGTERM)
+			}
+		}
+		return fmt.Errorf("ckpool process not found")
+	}
+}

api/internal/state/aggregator.go

@@ -190,6 +190,12 @@ type Aggregator struct {
 	// refresh. Used by the WebSocket hub to push updates to clients.
 	OnRefresh func(Snapshot)
 
+	// KillCKPool, if set, is called when bitcoind has been unreachable
+	// for several consecutive polls and no block submission is pending.
+	// Killing ckpool disconnects miners so they can failover to other
+	// pools instead of mining stale work.
+	KillCKPool func() error
+
 	mu     sync.RWMutex
 	snap   Snapshot
 	blocks []BlockRecord
@@ -231,6 +237,12 @@ type Aggregator struct {
 	// escalate to WARN.
 	ckFailStreak int
 
+	// btcFailStreak counts consecutive refreshes where bitcoind was
+	// unreachable. After a threshold, if no block submission is pending,
+	// we kill the ckpool process so miners can failover to other pools.
+	btcFailStreak    int
+	ckpoolKilled     bool // true after we sent SIGTERM, reset on bitcoind recovery
+
 	// readyOnce + ready closes the Ready() channel exactly once after
 	// the first refresh completes. main blocks briefly on this so the
 	// HTTP server doesn't serve a never-refreshed (all-zeros) snapshot.
@@ -424,6 +436,11 @@ func (a *Aggregator) refresh(ctx context.Context) {
 	if bi, err := a.RPC.GetBlockchainInfo(ctx); err == nil {
 		next.Chain = bi
 		next.BitcoinOK = true
+		if a.btcFailStreak > 0 {
+			a.Log.Info("bitcoind recovered", "after_failures", a.btcFailStreak)
+		}
+		a.btcFailStreak = 0
+		a.ckpoolKilled = false
 		// Track when the tip height last changed.
 		if bi.Blocks != a.lastTipHeight {
 			a.lastTipHeight = bi.Blocks
@@ -436,7 +453,8 @@ func (a *Aggregator) refresh(ctx context.Context) {
 			next.NetworkHashrateHs = nh
 		}
 	} else {
-		a.Log.Warn("bitcoind getblockchaininfo failed", "err", err)
+		a.btcFailStreak++
+		a.Log.Warn("bitcoind getblockchaininfo failed", "err", err, "streak", a.btcFailStreak)
 		if next.LastError == "" {
 			next.LastError = err.Error()
 		}
@@ -675,6 +693,26 @@ func (a *Aggregator) refresh(ctx context.Context) {
 		cb(pushed)
 	}
 	a.markReady()
+
+	// Kill ckpool when bitcoind is unreachable so miners can failover.
+	// Conditions: 3+ consecutive failures, no pending block submission,
+	// haven't already killed it, and a kill callback is configured.
+	const btcFailThreshold = 3
+	submitGap := next.BlockSubmitAttempts - next.BlockSubmitsConfirmed
+	if a.btcFailStreak >= btcFailThreshold && !a.ckpoolKilled && a.KillCKPool != nil {
+		if submitGap > 0 {
+			a.Log.Warn("bitcoind down but block submission pending, keeping ckpool alive",
+				"submit_gap", submitGap, "streak", a.btcFailStreak)
+		} else {
+			a.Log.Warn("bitcoind unreachable, killing ckpool so miners can failover",
+				"streak", a.btcFailStreak)
+			if err := a.KillCKPool(); err != nil {
+				a.Log.Error("failed to kill ckpool", "err", err)
+			} else {
+				a.ckpoolKilled = true
+			}
+		}
+	}
 }
 
 // AckBestDiff records the current best_diff as acknowledged so the UI

ui/src/lib/HealthBanners.svelte

@@ -25,10 +25,22 @@
     // Alarm when ZMQ is 3+ min older than the last tip change.
     return zmqAge > tipAge + 180;
   });
+
+  const bitcoindDown = $derived(snap.data != null && !snap.data.bitcoin_ok);
 </script>
 
-{#if submitGap > 0 || zmqStale}
+{#if bitcoindDown || submitGap > 0 || zmqStale}
   <div class="banners">
+    {#if bitcoindDown}
+      <div class="banner error">
+        <span class="icon">!</span>
+        <div class="text">
+          <strong>Bitcoin Core is unreachable.</strong>
+          Stratum server has been stopped — miners will failover to backup pools.
+          Service will resume automatically when Bitcoin Core recovers.
+        </div>
+      </div>
+    {/if}
     {#if submitGap > 0}
       <div class="banner warn">
         <span class="icon">!</span>
@@ -75,6 +87,11 @@
     border-color: rgb(220, 170, 60);
     color: rgb(220, 180, 100);
   }
+  .banner.error {
+    background: rgb(50, 20, 20);
+    border-color: rgb(220, 60, 60);
+    color: rgb(230, 120, 120);
+  }
   .icon {
     flex: 0 0 auto;
     width: 1.5em;