From 99302cf4af68ae34cf918aea6933617fa503a968 Mon Sep 17 00:00:00 2001 From: satoshi Date: Mon, 27 Apr 2026 21:53:23 +0300 Subject: [PATCH] Tighten fallback latency + alert UI on degraded states MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Submit-first ordering. Patch 0004 now calls generator_submitblock BEFORE writing the pending-block hex to disk. The happy path adds zero disk I/O — we only dump when the primary returned false. The same patch bounds generator_submitblock's "no live current_si" spin to ~3s instead of the original infinite loop, so a permanently-down primary doesn't pin the stratifier; the bounded spin lets the caller return false and lets local_block_submit dump for kamado-api to take over. Default grace lowered from 30s to 3s. With ckpool's bounded spin and sub-second sweep cadence, the fallback now reacts within ~4s of a failed primary submit — fast enough that the work is still relevant for the current chain tip. The submitter's sweep poll dropped to 1s to match. UI HealthBanners. New top-of-page strip surfaces: * Fallback used (red banner, 24h after most recent event): "primary bitcoind didn't accept; backup X took over Y ago" * Submit gap (orange banner, only when no recent fallback): "N blocks attempted but unconfirmed — configure backups" * ZMQ stale (orange banner): no hashblock frame in 30+ minutes Operators see degraded-but-not-fatal states without checking logs. Startup readiness gate. main now waits up to 8s on agg.Ready() before starting the HTTP server so the very first /api/snapshot doesn't show all-zero state during the aggregator's first refresh. Capped so a permanently-down bitcoind can't block startup; /healthz is honest about the degraded state once we do start serving. --- api/cmd/kamado-api/main.go | 19 ++- api/internal/blocksubmit/submit.go | 6 +- api/internal/config/config.go | 11 +- api/internal/state/aggregator.go | 19 +++ ...0004-dump-pending-block-for-fallback.patch | 67 +++++++-- ui/src/App.svelte | 5 + ui/src/lib/HealthBanners.svelte | 130 ++++++++++++++++++ 7 files changed, 241 insertions(+), 16 deletions(-) create mode 100644 ui/src/lib/HealthBanners.svelte diff --git a/api/cmd/kamado-api/main.go b/api/cmd/kamado-api/main.go index 1b85f33..a6e82b3 100644 --- a/api/cmd/kamado-api/main.go +++ b/api/cmd/kamado-api/main.go @@ -134,11 +134,28 @@ func main() { agg.RecordFallbackSubmit(height, viaLabel) }, } - go sub.Run(ctx, 5*time.Second) + // Sweep every second so we react within ~grace+1s of a + // failed submit. Reading an empty directory is cheap. + go sub.Run(ctx, 1*time.Second) } else { log.Info("blocksubmit fallback disabled (PENDING_BLOCKS_DIR not set)") } + // Wait briefly for the aggregator's first refresh to complete so + // the very first /api/snapshot or /api/health hit doesn't see an + // all-zeros snapshot and report bitcoin_ok=false during its own + // initialization. Cap the wait so a permanently-down bitcoind + // can't block startup forever — /healthz is honest about the + // degraded state. + select { + case <-agg.Ready(): + log.Info("first snapshot ready") + case <-time.After(8 * time.Second): + log.Warn("first snapshot not ready within 8s, serving HTTP anyway (snapshot will be partial until backends respond)") + case <-ctx.Done(): + return + } + srv := &http.Server{ Addr: cfg.ListenAddr, Handler: api.Handler(), diff --git a/api/internal/blocksubmit/submit.go b/api/internal/blocksubmit/submit.go index 43f91f7..c074fcf 100644 --- a/api/internal/blocksubmit/submit.go +++ b/api/internal/blocksubmit/submit.go @@ -71,7 +71,11 @@ func (s *Submitter) Run(ctx context.Context, poll time.Duration) { return } if s.Grace == 0 { - s.Grace = 30 * time.Second + // Match the config default — short enough that a failed + // submit gets recovered while the work is still relevant, + // long enough that we don't spuriously fallback while the + // primary's RPC is mid-handshake. + s.Grace = 3 * time.Second } if s.MaxAge == 0 { s.MaxAge = 24 * time.Hour diff --git a/api/internal/config/config.go b/api/internal/config/config.go index 18266be..afa9fa7 100644 --- a/api/internal/config/config.go +++ b/api/internal/config/config.go @@ -52,8 +52,13 @@ type Config struct { BackupRPCURLs string // How long a pending block file must persist before the submitter - // will try fallback RPCs. Defaults to 30s — long enough for - // ckpool's own retry loop and a fast ckpool->bitcoind round-trip. + // will try fallback RPCs. Defaults to 3s — short enough that a + // failed submit is recovered while the work is still relevant, + // long enough that ckpool's bounded internal wait (~3s for a live + // primary server) and a single slow round-trip don't trigger a + // spurious fallback. Patch 0004 makes ckpool's submit return false + // rather than spinning indefinitely, so we no longer need to wait + // for that case to time out. PendingBlocksGrace time.Duration } @@ -73,7 +78,7 @@ func FromEnv() (*Config, error) { PendingBlocksDir: os.Getenv("PENDING_BLOCKS_DIR"), BackupRPCURLs: os.Getenv("BACKUP_RPC_URLS"), - PendingBlocksGrace: getenvDuration("PENDING_BLOCKS_GRACE", 30*time.Second), + PendingBlocksGrace: getenvDuration("PENDING_BLOCKS_GRACE", 3*time.Second), } if cfg.BitcoinRPCURL == "" { diff --git a/api/internal/state/aggregator.go b/api/internal/state/aggregator.go index af009af..52a7e3e 100644 --- a/api/internal/state/aggregator.go +++ b/api/internal/state/aggregator.go @@ -185,6 +185,12 @@ type Aggregator struct { // escalate to WARN. ckFailStreak int + // readyOnce + ready closes the Ready() channel exactly once after + // the first refresh completes. main blocks briefly on this so the + // HTTP server doesn't serve a never-refreshed (all-zeros) snapshot. + readyOnce sync.Once + ready chan struct{} + // Submit-attempt vs confirmed counters. Persisted in kv so the // running gap survives restarts. Both are monotonic. blockSubmitAttempts int64 @@ -210,9 +216,21 @@ func New(ck *ckpool.Client, rpc *bitcoind.RPC, interval time.Duration, log *slog RPC: rpc, Interval: interval, Log: log, + ready: make(chan struct{}), } } +// Ready returns a channel that's closed once the aggregator has +// completed its first refresh — used by main to delay HTTP serving +// until /api/snapshot reflects real state instead of zeros. +func (a *Aggregator) Ready() <-chan struct{} { + return a.ready +} + +func (a *Aggregator) markReady() { + a.readyOnce.Do(func() { close(a.ready) }) +} + // Run blocks until ctx is cancelled, refreshing the snapshot every Interval. // It runs one immediate refresh at startup so readers don't see an empty // snapshot after ctx launches the goroutine. Persisted block history is @@ -486,6 +504,7 @@ func (a *Aggregator) refresh(ctx context.Context) { if cb != nil { cb(pushed) } + a.markReady() } // loadPersistedState restores cumulative work and hashrate history from diff --git a/ckpool/patches/0004-dump-pending-block-for-fallback.patch b/ckpool/patches/0004-dump-pending-block-for-fallback.patch index c8ccb16..e4944b6 100644 --- a/ckpool/patches/0004-dump-pending-block-for-fallback.patch +++ b/ckpool/patches/0004-dump-pending-block-for-fallback.patch @@ -1,16 +1,48 @@ +diff --git a/src/generator.c b/src/generator.c +index 22e2e08..438b199 100644 +--- a/src/generator.c ++++ b/src/generator.c +@@ -351,11 +351,25 @@ bool generator_submitblock(ckpool_t *ckp, const char *buf) + server_instance_t *si; + bool warn = false; + connsock_t *cs; ++ /* Bound the wait for current_si so a permanently-down primary ++ * bitcoind does not pin the caller forever. After this many 10ms ++ * sleeps (~3s), return false so the caller (in our fork: ++ * local_block_submit) can hand off to kamado-api's fallback ++ * broadcaster instead of blocking the stratifier on a dead RPC. ++ * Original upstream behavior was to spin indefinitely; we trade ++ * that for predictable latency. */ ++ const int max_no_si_iters = 300; ++ int no_si_iters = 0; + + while (unlikely(!(si = gdata->current_si))) { + if (!warn) +- LOGWARNING("No live current server in generator_blocksubmit! Resubmitting indefinitely!"); ++ LOGWARNING("No live current server in generator_blocksubmit! Waiting up to ~3s before giving up so fallback can take over..."); + warn = true; ++ if (++no_si_iters > max_no_si_iters) { ++ LOGWARNING("generator_submitblock: no live primary server after %d ms, returning false", ++ no_si_iters * 10); ++ return false; ++ } + cksleep_ms(10); + } + cs = &si->cs; diff --git a/src/stratifier.c b/src/stratifier.c -index 52da790..815eb01 100644 +index 52da790..7caf179 100644 --- a/src/stratifier.c +++ b/src/stratifier.c -@@ -2069,16 +2069,64 @@ process_block(const workbase_t *wb, const char *coinbase, const int cblen, +@@ -2069,16 +2069,75 @@ process_block(const workbase_t *wb, const char *coinbase, const int cblen, return gbt_block; } +-/* Submit block data locally, absorbing and freeing gbt_block */ +/* Write the raw block hex to a sidecar file under /pending-blocks/ -+ * before submitting, so an external watcher (kamado-api) can re-broadcast -+ * via fallback RPC nodes if our primary bitcoind doesn't accept it. The -+ * file is unlinked once generator_submitblock returns success. Best- -+ * effort: any error here is logged at INFO and never blocks the submit. */ ++ * so the external watcher (kamado-api) can re-broadcast via fallback RPC ++ * nodes. Best-effort: any error here is logged at INFO and never blocks ++ * the caller. Called only after the primary submit fails so the happy ++ * path stays disk-free. */ +static void kamado_dump_pending_block(ckpool_t *ckp, const char *gbt_block, + const char *rhash, int height, + char *out_path, size_t out_path_len) @@ -44,11 +76,17 @@ index 52da790..815eb01 100644 + out_path[0] = '\0'; + return; + } -+ LOGNOTICE("kamado: dumped pending block height %d to %s (%zu bytes)", ++ LOGNOTICE("kamado: dumped pending block height %d to %s (%zu bytes) for fallback broadcast", + height, out_path, blen); +} + - /* Submit block data locally, absorbing and freeing gbt_block */ ++/* Submit block data locally, absorbing and freeing gbt_block. ++ * ++ * Order matters: we always try ckp's primary bitcoind FIRST so the ++ * happy path adds zero latency or disk I/O. Only when the primary ++ * rejects (or generator_submitblock returns false for any other ++ * reason) do we dump the raw hex to /pending-blocks/ so the ++ * kamado-api watcher can re-broadcast via fallback RPC nodes. */ static bool local_block_submit(ckpool_t *ckp, char *gbt_block, const uchar *flip32, int height) { - bool ret = generator_submitblock(ckp, gbt_block); @@ -60,19 +98,26 @@ index 52da790..815eb01 100644 - free(gbt_block); swap_256(swap256, flip32); __bin2hex(rhash, swap256, 32); -+ kamado_dump_pending_block(ckp, gbt_block, rhash, height, -+ pending_path, sizeof(pending_path)); + ++ /* Primary submit first — this is the latency-critical path. */ + ret = generator_submitblock(ckp, gbt_block); + ++ /* Only touch disk if the primary didn't accept. */ ++ if (!ret) { ++ kamado_dump_pending_block(ckp, gbt_block, rhash, height, ++ pending_path, sizeof(pending_path)); ++ } ++ + free(gbt_block); generator_preciousblock(ckp, rhash); /* Check failures that may be inconclusive but were submitted via other -@@ -2099,6 +2147,8 @@ static bool local_block_submit(ckpool_t *ckp, char *gbt_block, const uchar *flip +@@ -2099,6 +2158,10 @@ static bool local_block_submit(ckpool_t *ckp, char *gbt_block, const uchar *flip height, ret ? "ACCEPTED" : "REJECTED"); } } ++ /* Block ended up on chain (either initial submit or precious-block ++ * recovery): the dump file is no longer needed. */ + if (ret && pending_path[0]) + unlink(pending_path); return ret; diff --git a/ui/src/App.svelte b/ui/src/App.svelte index ef9ee32..f2604ae 100644 --- a/ui/src/App.svelte +++ b/ui/src/App.svelte @@ -3,6 +3,7 @@ import { connect, snap } from "./stores/snapshot.svelte"; import { selection } from "./stores/selection.svelte"; import Header from "./lib/Header.svelte"; + import HealthBanners from "./lib/HealthBanners.svelte"; import PoolOverview from "./lib/PoolOverview.svelte"; import HashrateChart from "./lib/HashrateChart.svelte"; import MinersTable from "./lib/MinersTable.svelte"; @@ -19,6 +20,10 @@
+ {#if snap.data} + + {/if} + {#if !snap.data}
Status
diff --git a/ui/src/lib/HealthBanners.svelte b/ui/src/lib/HealthBanners.svelte new file mode 100644 index 0000000..c8cc14f --- /dev/null +++ b/ui/src/lib/HealthBanners.svelte @@ -0,0 +1,130 @@ + + +{#if submitGap > 0 || fallbackRecent || zmqStale} +
+ {#if fallbackRecent} + + {/if} + {#if submitGap > 0 && !fallbackRecent} + + {/if} + {#if zmqStale} + + {/if} +
+{/if} + +