P1 reliability + block-broadcast fallback path

P1 audits / fixes:

* Bitcoin Core RPC now retries up to 3 times with linear backoff on
  transport errors, 5xx responses, and warm-up/loading RPC errors
  (code -28). Hard "no" answers (block-not-found etc.) bubble up
  immediately so we don't mask real errors.

* WebSocket hub disconnects clients that miss 6 consecutive broadcasts
  (~30s with the default poll cadence). Stuck readers no longer hold
  stale snapshots indefinitely or freeze hub state.

* ZMQ subscriber freshness: aggregator records the last-event
  timestamp, surfaces zmq_enabled / has_last_zmq_event /
  last_zmq_event_age in the snapshot. /healthz flags zmq_stale when
  the gap exceeds 30 minutes.

* /healthz expanded with submit_attempts / submits_confirmed /
  submit_gap, fallback_submits_total + last_fallback_*, and the zmq
  staleness check. Now usable as a real-world ops dashboard signal.

Block-broadcast fallback (new feature):

  * ckpool patch 0004: hooks local_block_submit to write the raw block
    hex to <logdir>/pending-blocks/<height>-<hash16>.hex right before
    invoking generator_submitblock. Unlinks on success. ckpool's normal
    flow is otherwise untouched.

  * api/internal/blocksubmit: watcher polls the dir every 5s. Files
    sitting longer than the grace window (default 30s, configurable)
    are re-broadcast through operator-supplied backup RPC URLs in
    sequence. Treats both null and any "duplicate*" reject reason as
    success (the block landed). Pre-checks the primary chain first so
    a stale file from a successful-but-unlinked submit gets cleaned
    up without bothering fallbacks.

  * Aggregator records each successful fallback submission as a
    persistent counter and surfaces it in the snapshot so the UI can
    show a "primary bitcoind isn't accepting submits" alert.

  * Config: BACKUP_RPC_URLS (comma- or newline-separated, with
    optional inline credentials) plus PENDING_BLOCKS_DIR and
    PENDING_BLOCKS_GRACE. URLs are parsed via net/url so
    https://user:pass@host:port/ works cleanly.

The fallback is opt-in and disabled by default. Once enabled with at
least one URL, a primary bitcoind outage at the moment of solving no
longer means a lost block — kamado-api re-broadcasts via whichever
backup the operator trusts (a second self-hosted node, an
authenticated public RPC service, etc.).

api/internal/state/aggregator.go

@@ -77,9 +77,25 @@ type Snapshot struct {
 	// log lines) and confirmed solves ("Solved and confirmed block").
 	// A growing gap means bitcoind is rejecting our submissions or
 	// dropping the RPC — surface it in the UI as an alert.
-	BlockSubmitAttempts int64 `json:"block_submit_attempts"`
+	BlockSubmitAttempts   int64 `json:"block_submit_attempts"`
 	BlockSubmitsConfirmed int64 `json:"block_submits_confirmed"`
 
+	// Number of blocks the fallback submitter had to broadcast via a
+	// backup RPC URL because the primary bitcoind didn't accept them
+	// in time. Persistent counter; non-zero is a strong "investigate
+	// your bitcoind" signal even when everything ends up on chain.
+	FallbackSubmitsTotal int64  `json:"fallback_submits_total"`
+	LastFallbackSubmitAt int64  `json:"last_fallback_submit_at,omitempty"` // unix seconds
+	LastFallbackVia     string `json:"last_fallback_via,omitempty"`
+
+	// Health diagnostics for /healthz and the UI status badge.
+	// LastZMQEventAge is the seconds-since the last bitcoind hashblock
+	// frame arrived; -1 means no event seen since startup. ZMQEnabled
+	// is whether the user configured an endpoint at all.
+	ZMQEnabled        bool    `json:"zmq_enabled"`
+	LastZMQEventAge   float64 `json:"last_zmq_event_age,omitempty"` // seconds; >=0
+	HasLastZMQEvent   bool    `json:"has_last_zmq_event"`
+
 	// Health
 	CKPoolOK  bool   `json:"ckpool_ok"`
 	BitcoinOK bool   `json:"bitcoin_ok"`
@@ -98,6 +114,7 @@ const (
 
 	kvSubmitAttempts   = "block_submit_attempts"
 	kvSubmitsConfirmed = "block_submits_confirmed"
+	kvFallbackSubmits  = "fallback_submits_total"
 
 	// reconcileInterval is how often we sweep recent blocks looking
 	// for missing hash/reward enrichment and reorg-orphaned hashes.
@@ -173,6 +190,18 @@ type Aggregator struct {
 	blockSubmitAttempts   int64
 	blockSubmitsConfirmed int64
 	lastSubmitCountSave   time.Time
+
+	// Fallback-submitter telemetry. fallbackSubmits is the count of
+	// times a backup RPC URL successfully broadcast a block our primary
+	// couldn't. Persisted; survives restarts.
+	fallbackSubmits      int64
+	lastFallbackSubmitAt time.Time
+	lastFallbackVia      string
+
+	// ZMQ diagnostics: timestamp of the last hashblock frame relayed
+	// from zmqmon. Used by /healthz to flag stale subscriptions.
+	zmqEnabled       bool
+	lastZMQEventTime time.Time
 }
 
 func New(ck *ckpool.Client, rpc *bitcoind.RPC, interval time.Duration, log *slog.Logger) *Aggregator {
@@ -190,6 +219,9 @@ func New(ck *ckpool.Client, rpc *bitcoind.RPC, interval time.Duration, log *slog
 // loaded from the store before the first refresh. If tipEvents is non-nil,
 // each received tip triggers an immediate refresh outside the poll cadence.
 func (a *Aggregator) Run(ctx context.Context, tipEvents <-chan zmqmon.TipEvent) {
+	a.mu.Lock()
+	a.zmqEnabled = tipEvents != nil
+	a.mu.Unlock()
 	a.loadPersistedBlocks()
 	a.loadPersistedState()
 	a.refresh(ctx)
@@ -206,12 +238,35 @@ func (a *Aggregator) Run(ctx context.Context, tipEvents <-chan zmqmon.TipEvent)
 				tipEvents = nil
 				continue
 			}
+			a.mu.Lock()
+			a.lastZMQEventTime = ev.SeenAt
+			a.mu.Unlock()
 			a.Log.Debug("zmq tip, refreshing", "hash", ev.Hash)
 			a.refresh(ctx)
 		}
 	}
 }
 
+// RecordFallbackSubmit is called by blocksubmit.Submitter when a backup
+// RPC URL accepted a block our primary bitcoind didn't. Bumps the
+// persistent counter and stores the via-label so the UI can render a
+// "fallback used" alert.
+func (a *Aggregator) RecordFallbackSubmit(height int64, via string) {
+	a.mu.Lock()
+	a.fallbackSubmits++
+	a.lastFallbackSubmitAt = time.Now()
+	a.lastFallbackVia = via
+	n := a.fallbackSubmits
+	a.mu.Unlock()
+	a.Log.Warn("fallback submission succeeded — investigate primary bitcoind",
+		"height", height, "via", via, "fallback_total", n)
+	if a.Store != nil {
+		if err := a.Store.SetKV(kvFallbackSubmits, strconv.FormatInt(n, 10)); err != nil {
+			a.Log.Warn("fallback counter persist failed", "err", err)
+		}
+	}
+}
+
 // Snapshot returns a copy of the current snapshot.
 func (a *Aggregator) Snapshot() Snapshot {
 	a.mu.RLock()
@@ -413,6 +468,16 @@ func (a *Aggregator) refresh(ctx context.Context) {
 	}
 	next.BlockSubmitAttempts = a.blockSubmitAttempts
 	next.BlockSubmitsConfirmed = a.blockSubmitsConfirmed
+	next.FallbackSubmitsTotal = a.fallbackSubmits
+	if !a.lastFallbackSubmitAt.IsZero() {
+		next.LastFallbackSubmitAt = a.lastFallbackSubmitAt.Unix()
+		next.LastFallbackVia = a.lastFallbackVia
+	}
+	next.ZMQEnabled = a.zmqEnabled
+	if !a.lastZMQEventTime.IsZero() {
+		next.LastZMQEventAge = time.Since(a.lastZMQEventTime).Seconds()
+		next.HasLastZMQEvent = true
+	}
 	a.snap = next
 	cb := a.OnRefresh
 	pushed := next
@@ -456,6 +521,13 @@ func (a *Aggregator) loadPersistedState() {
 			a.mu.Unlock()
 		}
 	}
+	if v, err := a.Store.GetKV(kvFallbackSubmits); err == nil && v != "" {
+		if n, perr := strconv.ParseInt(v, 10, 64); perr == nil {
+			a.mu.Lock()
+			a.fallbackSubmits = n
+			a.mu.Unlock()
+		}
+	}
 
 	cutoff := time.Now().Add(-24 * time.Hour).Unix()
 	if samples, err := a.Store.HashrateSince(cutoff); err != nil {