Tighten fallback latency + alert UI on degraded states

Submit-first ordering. Patch 0004 now calls generator_submitblock
BEFORE writing the pending-block hex to disk. The happy path adds zero
disk I/O — we only dump when the primary returned false. The same
patch bounds generator_submitblock's "no live current_si" spin to
~3s instead of the original infinite loop, so a permanently-down
primary doesn't pin the stratifier; the bounded spin lets the caller
return false and lets local_block_submit dump for kamado-api to take
over.

Default grace lowered from 30s to 3s. With ckpool's bounded spin and
sub-second sweep cadence, the fallback now reacts within ~4s of a
failed primary submit — fast enough that the work is still relevant
for the current chain tip. The submitter's sweep poll dropped to 1s
to match.

UI HealthBanners. New top-of-page strip surfaces:
  * Fallback used (red banner, 24h after most recent event):
    "primary bitcoind didn't accept; backup X took over Y ago"
  * Submit gap (orange banner, only when no recent fallback):
    "N blocks attempted but unconfirmed — configure backups"
  * ZMQ stale (orange banner): no hashblock frame in 30+ minutes
Operators see degraded-but-not-fatal states without checking logs.

Startup readiness gate. main now waits up to 8s on agg.Ready() before
starting the HTTP server so the very first /api/snapshot doesn't show
all-zero state during the aggregator's first refresh. Capped so a
permanently-down bitcoind can't block startup; /healthz is honest
about the degraded state once we do start serving.

api/cmd/kamado-api/main.go

@@ -134,11 +134,28 @@ func main() {
 				agg.RecordFallbackSubmit(height, viaLabel)
 			},
 		}
-		go sub.Run(ctx, 5*time.Second)
+		// Sweep every second so we react within ~grace+1s of a
+		// failed submit. Reading an empty directory is cheap.
+		go sub.Run(ctx, 1*time.Second)
 	} else {
 		log.Info("blocksubmit fallback disabled (PENDING_BLOCKS_DIR not set)")
 	}
 
+	// Wait briefly for the aggregator's first refresh to complete so
+	// the very first /api/snapshot or /api/health hit doesn't see an
+	// all-zeros snapshot and report bitcoin_ok=false during its own
+	// initialization. Cap the wait so a permanently-down bitcoind
+	// can't block startup forever — /healthz is honest about the
+	// degraded state.
+	select {
+	case <-agg.Ready():
+		log.Info("first snapshot ready")
+	case <-time.After(8 * time.Second):
+		log.Warn("first snapshot not ready within 8s, serving HTTP anyway (snapshot will be partial until backends respond)")
+	case <-ctx.Done():
+		return
+	}
+
 	srv := &http.Server{
 		Addr:              cfg.ListenAddr,
 		Handler:           api.Handler(),

api/internal/blocksubmit/submit.go

@@ -71,7 +71,11 @@ func (s *Submitter) Run(ctx context.Context, poll time.Duration) {
 		return
 	}
 	if s.Grace == 0 {
-		s.Grace = 30 * time.Second
+		// Match the config default — short enough that a failed
+		// submit gets recovered while the work is still relevant,
+		// long enough that we don't spuriously fallback while the
+		// primary's RPC is mid-handshake.
+		s.Grace = 3 * time.Second
 	}
 	if s.MaxAge == 0 {
 		s.MaxAge = 24 * time.Hour

api/internal/config/config.go

@@ -52,8 +52,13 @@ type Config struct {
 	BackupRPCURLs string
 
 	// How long a pending block file must persist before the submitter
-	// will try fallback RPCs. Defaults to 30s — long enough for
-	// ckpool's own retry loop and a fast ckpool->bitcoind round-trip.
+	// will try fallback RPCs. Defaults to 3s — short enough that a
+	// failed submit is recovered while the work is still relevant,
+	// long enough that ckpool's bounded internal wait (~3s for a live
+	// primary server) and a single slow round-trip don't trigger a
+	// spurious fallback. Patch 0004 makes ckpool's submit return false
+	// rather than spinning indefinitely, so we no longer need to wait
+	// for that case to time out.
 	PendingBlocksGrace time.Duration
 }
 
@@ -73,7 +78,7 @@ func FromEnv() (*Config, error) {
 
 		PendingBlocksDir:   os.Getenv("PENDING_BLOCKS_DIR"),
 		BackupRPCURLs:      os.Getenv("BACKUP_RPC_URLS"),
-		PendingBlocksGrace: getenvDuration("PENDING_BLOCKS_GRACE", 30*time.Second),
+		PendingBlocksGrace: getenvDuration("PENDING_BLOCKS_GRACE", 3*time.Second),
 	}
 
 	if cfg.BitcoinRPCURL == "" {

api/internal/state/aggregator.go

@@ -185,6 +185,12 @@ type Aggregator struct {
 	// escalate to WARN.
 	ckFailStreak int
 
+	// readyOnce + ready closes the Ready() channel exactly once after
+	// the first refresh completes. main blocks briefly on this so the
+	// HTTP server doesn't serve a never-refreshed (all-zeros) snapshot.
+	readyOnce sync.Once
+	ready     chan struct{}
+
 	// Submit-attempt vs confirmed counters. Persisted in kv so the
 	// running gap survives restarts. Both are monotonic.
 	blockSubmitAttempts   int64
@@ -210,9 +216,21 @@ func New(ck *ckpool.Client, rpc *bitcoind.RPC, interval time.Duration, log *slog
 		RPC:      rpc,
 		Interval: interval,
 		Log:      log,
+		ready:    make(chan struct{}),
 	}
 }
 
+// Ready returns a channel that's closed once the aggregator has
+// completed its first refresh — used by main to delay HTTP serving
+// until /api/snapshot reflects real state instead of zeros.
+func (a *Aggregator) Ready() <-chan struct{} {
+	return a.ready
+}
+
+func (a *Aggregator) markReady() {
+	a.readyOnce.Do(func() { close(a.ready) })
+}
+
 // Run blocks until ctx is cancelled, refreshing the snapshot every Interval.
 // It runs one immediate refresh at startup so readers don't see an empty
 // snapshot after ctx launches the goroutine. Persisted block history is
@@ -486,6 +504,7 @@ func (a *Aggregator) refresh(ctx context.Context) {
 	if cb != nil {
 		cb(pushed)
 	}
+	a.markReady()
 }
 
 // loadPersistedState restores cumulative work and hashrate history from

ckpool/patches/0004-dump-pending-block-for-fallback.patch

@@ -1,16 +1,48 @@
+diff --git a/src/generator.c b/src/generator.c
+index 22e2e08..438b199 100644
+--- a/src/generator.c
++++ b/src/generator.c
+@@ -351,11 +351,25 @@ bool generator_submitblock(ckpool_t *ckp, const char *buf)
+ 	server_instance_t *si;
+ 	bool warn = false;
+ 	connsock_t *cs;
++	/* Bound the wait for current_si so a permanently-down primary
++	 * bitcoind does not pin the caller forever. After this many 10ms
++	 * sleeps (~3s), return false so the caller (in our fork:
++	 * local_block_submit) can hand off to kamado-api's fallback
++	 * broadcaster instead of blocking the stratifier on a dead RPC.
++	 * Original upstream behavior was to spin indefinitely; we trade
++	 * that for predictable latency. */
++	const int max_no_si_iters = 300;
++	int no_si_iters = 0;
+ 
+ 	while (unlikely(!(si = gdata->current_si))) {
+ 		if (!warn)
+-			LOGWARNING("No live current server in generator_blocksubmit! Resubmitting indefinitely!");
++			LOGWARNING("No live current server in generator_blocksubmit! Waiting up to ~3s before giving up so fallback can take over...");
+ 		warn = true;
++		if (++no_si_iters > max_no_si_iters) {
++			LOGWARNING("generator_submitblock: no live primary server after %d ms, returning false",
++				   no_si_iters * 10);
++			return false;
++		}
+ 		cksleep_ms(10);
+ 	}
+ 	cs = &si->cs;
 diff --git a/src/stratifier.c b/src/stratifier.c
-index 52da790..815eb01 100644
+index 52da790..7caf179 100644
 --- a/src/stratifier.c
 +++ b/src/stratifier.c
-@@ -2069,16 +2069,64 @@ process_block(const workbase_t *wb, const char *coinbase, const int cblen,
+@@ -2069,16 +2069,75 @@ process_block(const workbase_t *wb, const char *coinbase, const int cblen,
  	return gbt_block;
  }
  
+-/* Submit block data locally, absorbing and freeing gbt_block */
 +/* Write the raw block hex to a sidecar file under <logdir>/pending-blocks/
-+ * before submitting, so an external watcher (kamado-api) can re-broadcast
-+ * via fallback RPC nodes if our primary bitcoind doesn't accept it. The
-+ * file is unlinked once generator_submitblock returns success. Best-
-+ * effort: any error here is logged at INFO and never blocks the submit. */
++ * so the external watcher (kamado-api) can re-broadcast via fallback RPC
++ * nodes. Best-effort: any error here is logged at INFO and never blocks
++ * the caller. Called only after the primary submit fails so the happy
++ * path stays disk-free. */
 +static void kamado_dump_pending_block(ckpool_t *ckp, const char *gbt_block,
 +				      const char *rhash, int height,
 +				      char *out_path, size_t out_path_len)
@@ -44,11 +76,17 @@ index 52da790..815eb01 100644
 +		out_path[0] = '\0';
 +		return;
 +	}
-+	LOGNOTICE("kamado: dumped pending block height %d to %s (%zu bytes)",
++	LOGNOTICE("kamado: dumped pending block height %d to %s (%zu bytes) for fallback broadcast",
 +		  height, out_path, blen);
 +}
 +
- /* Submit block data locally, absorbing and freeing gbt_block */
++/* Submit block data locally, absorbing and freeing gbt_block.
++ *
++ * Order matters: we always try ckp's primary bitcoind FIRST so the
++ * happy path adds zero latency or disk I/O. Only when the primary
++ * rejects (or generator_submitblock returns false for any other
++ * reason) do we dump the raw hex to <logdir>/pending-blocks/ so the
++ * kamado-api watcher can re-broadcast via fallback RPC nodes. */
  static bool local_block_submit(ckpool_t *ckp, char *gbt_block, const uchar *flip32, int height)
  {
 -	bool ret = generator_submitblock(ckp, gbt_block);
@@ -60,19 +98,26 @@ index 52da790..815eb01 100644
 -	free(gbt_block);
  	swap_256(swap256, flip32);
  	__bin2hex(rhash, swap256, 32);
-+	kamado_dump_pending_block(ckp, gbt_block, rhash, height,
-+				  pending_path, sizeof(pending_path));
 +
++	/* Primary submit first — this is the latency-critical path. */
 +	ret = generator_submitblock(ckp, gbt_block);
 +
++	/* Only touch disk if the primary didn't accept. */
++	if (!ret) {
++		kamado_dump_pending_block(ckp, gbt_block, rhash, height,
++					  pending_path, sizeof(pending_path));
++	}
++
 +	free(gbt_block);
  	generator_preciousblock(ckp, rhash);
  
  	/* Check failures that may be inconclusive but were submitted via other
-@@ -2099,6 +2147,8 @@ static bool local_block_submit(ckpool_t *ckp, char *gbt_block, const uchar *flip
+@@ -2099,6 +2158,10 @@ static bool local_block_submit(ckpool_t *ckp, char *gbt_block, const uchar *flip
  				   height, ret ? "ACCEPTED" : "REJECTED");
  		}
  	}
++	/* Block ended up on chain (either initial submit or precious-block
++	 * recovery): the dump file is no longer needed. */
 +	if (ret && pending_path[0])
 +		unlink(pending_path);
  	return ret;

ui/src/App.svelte

@@ -3,6 +3,7 @@
   import { connect, snap } from "./stores/snapshot.svelte";
   import { selection } from "./stores/selection.svelte";
   import Header from "./lib/Header.svelte";
+  import HealthBanners from "./lib/HealthBanners.svelte";
   import PoolOverview from "./lib/PoolOverview.svelte";
   import HashrateChart from "./lib/HashrateChart.svelte";
   import MinersTable from "./lib/MinersTable.svelte";
@@ -19,6 +20,10 @@
 <main>
   <Header />
 
+  {#if snap.data}
+    <HealthBanners />
+  {/if}
+
   {#if !snap.data}
     <div class="card placeholder">
       <div class="stat-label">Status</div>

ui/src/lib/HealthBanners.svelte

@@ -0,0 +1,130 @@
+<script lang="ts">
+  import { snap } from "../stores/snapshot.svelte";
+  import { formatAgo } from "../format";
+
+  // submit_gap > 0 means ckpool tried to submit at least one block that
+  // never got the "Solved and confirmed" follow-up — so either bitcoind
+  // rejected the submission or the RPC dropped. The number stays > 0
+  // forever after such an event (these are persistent counters), so we
+  // only show it as a banner when the fallback hasn't covered for it
+  // OR there's been a recent fallback the operator should investigate.
+  const submitGap = $derived.by(() => {
+    const d = snap.data;
+    if (!d) return 0;
+    return Math.max(0, (d.block_submit_attempts ?? 0) - (d.block_submits_confirmed ?? 0));
+  });
+
+  const fallbackCount = $derived(snap.data?.fallback_submits_total ?? 0);
+  const lastFallbackAt = $derived(snap.data?.last_fallback_submit_at ?? 0);
+  const lastFallbackVia = $derived(snap.data?.last_fallback_via ?? "");
+
+  // ZMQ stale = configured but no event in 30+ minutes. Bitcoin's avg
+  // block interval is 10 min; 30 min covers normal variance without
+  // false-alarming on quiet stretches.
+  const zmqStale = $derived.by(() => {
+    const d = snap.data;
+    if (!d || !d.zmq_enabled || !d.has_last_zmq_event) return false;
+    return (d.last_zmq_event_age ?? 0) > 1800;
+  });
+
+  // Show the fallback banner for 24h after the most recent fallback
+  // event so the operator sees the alert during their next check-in
+  // even if the underlying problem auto-resolved.
+  const fallbackRecent = $derived.by(() => {
+    if (!lastFallbackAt) return false;
+    const ageSec = Date.now() / 1000 - lastFallbackAt;
+    return ageSec >= 0 && ageSec < 86400;
+  });
+</script>
+
+{#if submitGap > 0 || fallbackRecent || zmqStale}
+  <div class="banners">
+    {#if fallbackRecent}
+      <div class="banner alert">
+        <span class="icon">!</span>
+        <div class="text">
+          <strong>Fallback broadcaster used</strong>
+          —
+          our primary bitcoind didn't accept a block submission. Backup
+          RPC <code>{lastFallbackVia}</code> took over
+          {formatAgo(lastFallbackAt)}. Total fallbacks since first run: {fallbackCount}.
+          Investigate primary bitcoind health.
+        </div>
+      </div>
+    {/if}
+    {#if submitGap > 0 && !fallbackRecent}
+      <div class="banner warn">
+        <span class="icon">?</span>
+        <div class="text">
+          <strong>Submit gap:</strong>
+          {submitGap} block{submitGap === 1 ? "" : "s"} attempted but not
+          confirmed by bitcoind. Either rejected at submission or the
+          RPC dropped. Configure backup RPC URLs to enable automatic
+          fallback broadcast.
+        </div>
+      </div>
+    {/if}
+    {#if zmqStale}
+      <div class="banner warn">
+        <span class="icon">~</span>
+        <div class="text">
+          <strong>ZMQ subscriber stale.</strong>
+          No <code>hashblock</code> frame from bitcoind in
+          {formatAgo(Date.now() / 1000 - (snap.data?.last_zmq_event_age ?? 0))}.
+          Tip changes will fall back to slower polling; check that
+          bitcoind is reachable on its ZMQ port.
+        </div>
+      </div>
+    {/if}
+  </div>
+{/if}
+
+<style>
+  .banners {
+    display: flex;
+    flex-direction: column;
+    gap: 0.5rem;
+  }
+  .banner {
+    display: flex;
+    align-items: flex-start;
+    gap: 0.6rem;
+    padding: 0.65rem 0.85rem;
+    border-radius: 6px;
+    border: 1px solid transparent;
+    line-height: 1.4;
+  }
+  .banner.alert {
+    background: rgba(220, 80, 80, 0.10);
+    border-color: rgba(220, 80, 80, 0.40);
+    color: rgb(230, 130, 130);
+  }
+  .banner.warn {
+    background: rgba(220, 170, 60, 0.10);
+    border-color: rgba(220, 170, 60, 0.40);
+    color: rgb(220, 180, 100);
+  }
+  .icon {
+    flex: 0 0 auto;
+    width: 1.5em;
+    height: 1.5em;
+    border-radius: 50%;
+    display: inline-flex;
+    align-items: center;
+    justify-content: center;
+    font-weight: 700;
+    font-size: 0.9em;
+    background: currentColor;
+    color: var(--bg, #111);
+  }
+  .text {
+    flex: 1 1 auto;
+    font-size: 0.92em;
+  }
+  .banner code {
+    font-size: 0.9em;
+    padding: 0.05em 0.3em;
+    border-radius: 3px;
+    background: rgba(255, 255, 255, 0.07);
+  }
+</style>