P1 reliability + block-broadcast fallback path

P1 audits / fixes:

* Bitcoin Core RPC now retries up to 3 times with linear backoff on
  transport errors, 5xx responses, and warm-up/loading RPC errors
  (code -28). Hard "no" answers (block-not-found etc.) bubble up
  immediately so we don't mask real errors.

* WebSocket hub disconnects clients that miss 6 consecutive broadcasts
  (~30s with the default poll cadence). Stuck readers no longer hold
  stale snapshots indefinitely or freeze hub state.

* ZMQ subscriber freshness: aggregator records the last-event
  timestamp, surfaces zmq_enabled / has_last_zmq_event /
  last_zmq_event_age in the snapshot. /healthz flags zmq_stale when
  the gap exceeds 30 minutes.

* /healthz expanded with submit_attempts / submits_confirmed /
  submit_gap, fallback_submits_total + last_fallback_*, and the zmq
  staleness check. Now usable as a real-world ops dashboard signal.

Block-broadcast fallback (new feature):

  * ckpool patch 0004: hooks local_block_submit to write the raw block
    hex to <logdir>/pending-blocks/<height>-<hash16>.hex right before
    invoking generator_submitblock. Unlinks on success. ckpool's normal
    flow is otherwise untouched.

  * api/internal/blocksubmit: watcher polls the dir every 5s. Files
    sitting longer than the grace window (default 30s, configurable)
    are re-broadcast through operator-supplied backup RPC URLs in
    sequence. Treats both null and any "duplicate*" reject reason as
    success (the block landed). Pre-checks the primary chain first so
    a stale file from a successful-but-unlinked submit gets cleaned
    up without bothering fallbacks.

  * Aggregator records each successful fallback submission as a
    persistent counter and surfaces it in the snapshot so the UI can
    show a "primary bitcoind isn't accepting submits" alert.

  * Config: BACKUP_RPC_URLS (comma- or newline-separated, with
    optional inline credentials) plus PENDING_BLOCKS_DIR and
    PENDING_BLOCKS_GRACE. URLs are parsed via net/url so
    https://user:pass@host:port/ works cleanly.

The fallback is opt-in and disabled by default. Once enabled with at
least one URL, a primary bitcoind outage at the moment of solving no
longer means a lost block — kamado-api re-broadcasts via whichever
backup the operator trusts (a second self-hosted node, an
authenticated public RPC service, etc.).

api/internal/bitcoind/rpc.go

@@ -6,9 +6,11 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
+	"strings"
 	"time"
 )
 
@@ -50,8 +52,35 @@ func (e *rpcError) Error() string {
 	return fmt.Sprintf("bitcoind rpc error %d: %s", e.Code, e.Message)
 }
 
-// Call performs a single JSON-RPC request and unmarshals the result.
+// Call performs a JSON-RPC request and unmarshals the result. Transient
+// failures (transport errors, 502/503/504, or RPC errors with an
+// "in warmup"/"loading"/"verifying" message that bitcoind returns
+// during startup) are retried up to twice with a short backoff. RPC
+// errors with semantic codes (e.g. block-not-found) are returned
+// immediately because retrying won't change the answer.
 func (c *RPC) Call(ctx context.Context, method string, params []any, out any) error {
+	const maxAttempts = 3
+	var lastErr error
+	for attempt := 1; attempt <= maxAttempts; attempt++ {
+		err := c.callOnce(ctx, method, params, out)
+		if err == nil {
+			return nil
+		}
+		lastErr = err
+		if !isRetryable(err) || attempt == maxAttempts || ctx.Err() != nil {
+			return err
+		}
+		backoff := time.Duration(attempt) * 200 * time.Millisecond
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(backoff):
+		}
+	}
+	return lastErr
+}
+
+func (c *RPC) callOnce(ctx context.Context, method string, params []any, out any) error {
 	body, err := json.Marshal(rpcRequest{
 		JSONRPC: "1.0",
 		ID:      "kamado",
@@ -78,10 +107,14 @@ func (c *RPC) Call(ctx context.Context, method string, params []any, out any) er
 	if err != nil {
 		return fmt.Errorf("bitcoind rpc: read body: %w", err)
 	}
-	// bitcoind returns 500 on rpc errors but still with a valid JSON body.
+	// 5xx without a parseable body: surface as transport-level error
+	// so isRetryable() can flag it.
 	var rr rpcResponse
-	if err := json.Unmarshal(raw, &rr); err != nil {
-		return fmt.Errorf("bitcoind rpc: unmarshal (status %d): %w: %s", resp.StatusCode, err, string(raw))
+	if jerr := json.Unmarshal(raw, &rr); jerr != nil {
+		if resp.StatusCode >= 500 {
+			return fmt.Errorf("bitcoind rpc: %d: %s", resp.StatusCode, truncate(string(raw), 256))
+		}
+		return fmt.Errorf("bitcoind rpc: unmarshal (status %d): %w: %s", resp.StatusCode, jerr, truncate(string(raw), 256))
 	}
 	if rr.Error != nil {
 		return rr.Error
@@ -92,6 +125,56 @@ func (c *RPC) Call(ctx context.Context, method string, params []any, out any) er
 	return nil
 }
 
+func truncate(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[:n] + "…"
+}
+
+// isRetryable distinguishes "the RPC didn't reach a definitive answer
+// yet" (transport-level errors, 5xx, bitcoind warm-up/loading) from
+// "bitcoind answered, the answer is no" (RPC error with a code, e.g.
+// -5 block not found). We retry the first kind and bubble the second.
+func isRetryable(err error) bool {
+	if err == nil {
+		return false
+	}
+	// rpcError values come from bitcoind itself — check for warmup/
+	// loading messages that mean "ask again in a moment".
+	var rerr *rpcError
+	if errors.As(err, &rerr) {
+		// -28 is RPC_IN_WARMUP per bitcoin/src/rpc/protocol.h
+		if rerr.Code == -28 {
+			return true
+		}
+		msg := strings.ToLower(rerr.Message)
+		if strings.Contains(msg, "warming up") ||
+			strings.Contains(msg, "loading") ||
+			strings.Contains(msg, "verifying") ||
+			strings.Contains(msg, "rewinding") ||
+			strings.Contains(msg, "still busy") {
+			return true
+		}
+		return false
+	}
+	// Wrapped transport / 5xx / read errors all flow through fmt.Errorf
+	// with the "bitcoind rpc:" prefix and no rpcError target.
+	msg := strings.ToLower(err.Error())
+	switch {
+	case strings.Contains(msg, "connection refused"),
+		strings.Contains(msg, "connection reset"),
+		strings.Contains(msg, "no such host"),
+		strings.Contains(msg, "deadline exceeded"),
+		strings.Contains(msg, "i/o timeout"),
+		strings.Contains(msg, "eof"),
+		strings.Contains(msg, "broken pipe"),
+		strings.Contains(msg, " 502"), strings.Contains(msg, " 503"), strings.Contains(msg, " 504"):
+		return true
+	}
+	return false
+}
+
 // ---- typed method wrappers ----
 
 type BlockchainInfo struct {

api/internal/blocksubmit/submit.go

@@ -0,0 +1,271 @@
+// Package blocksubmit watches a sidecar directory that ckpool fills
+// with raw block hex right before it tries to submit to its primary
+// bitcoind, and re-submits via operator-configured fallback RPC URLs
+// if a file persists longer than the grace period.
+//
+// ckpool's local_block_submit (patched in 0004) writes
+// <logdir>/pending-blocks/<height>-<rhash>.hex before calling
+// generator_submitblock, and unlinks on success. So the rule is
+// simple: any file still present after `grace` has likely failed to
+// land, and we should help.
+//
+// Failure modes this protects against:
+//   - bitcoind crashed / OOM-killed at the moment of submission
+//   - bitcoind RPC saturated, returning timeouts
+//   - upstream peering issue making bitcoind unable to relay
+//   - ckpool itself crashes after the file was written but before unlink
+//
+// Each fallback URL is tried in order. submitblock returns either null
+// (accepted), a "duplicate*" reason (already on chain — same outcome),
+// or a hard error. We treat both null and any "duplicate" prefix as
+// success and unlink.
+package blocksubmit
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/kamadopool/kamado-api/internal/bitcoind"
+)
+
+// FallbackTarget is one back-up RPC endpoint we'll try if the primary
+// bitcoind didn't accept a block. URL must be a full http(s):// endpoint;
+// User and Password are optional (some self-hosted nodes accept
+// unauthenticated localhost RPC, public RPC services usually don't).
+type FallbackTarget struct {
+	URL      string
+	User     string
+	Password string
+	Label    string // human-friendly name for logs; defaults to host
+}
+
+// Submitter polls a pending-blocks directory and re-broadcasts blocks
+// that have been sitting too long to its primary bitcoind plus any
+// configured fallbacks. Stateless across iterations — the filesystem
+// is the queue.
+type Submitter struct {
+	Dir       string
+	Grace     time.Duration   // how long we wait before considering a file stale
+	MaxAge    time.Duration   // older than this: log error and remove (operator intervention required)
+	Primary   *bitcoind.RPC   // primary bitcoind for "is the block already on chain?" sanity check
+	Fallbacks []FallbackTarget
+	Log       *slog.Logger
+
+	// OnSuccess, if set, is called with the height and the URL string
+	// that accepted the block, so the aggregator can bump a counter or
+	// emit a UI banner.
+	OnSuccess func(height int64, viaURL string, viaLabel string)
+}
+
+// Run blocks until ctx is cancelled, sweeping the dir every poll.
+func (s *Submitter) Run(ctx context.Context, poll time.Duration) {
+	if s.Dir == "" {
+		s.Log.Info("blocksubmit disabled (no pending-blocks dir)")
+		return
+	}
+	if s.Grace == 0 {
+		s.Grace = 30 * time.Second
+	}
+	if s.MaxAge == 0 {
+		s.MaxAge = 24 * time.Hour
+	}
+	if poll == 0 {
+		poll = 5 * time.Second
+	}
+
+	// Ensure the directory exists so the first sweep doesn't WARN. Don't
+	// fail if we can't create it — ckpool may not have run yet, or perms
+	// may need a moment to settle. The sweep itself logs at DEBUG when
+	// the dir is missing.
+	_ = os.MkdirAll(s.Dir, 0750)
+
+	t := time.NewTicker(poll)
+	defer t.Stop()
+	s.Log.Info("blocksubmit started", "dir", s.Dir, "grace", s.Grace, "fallbacks", len(s.Fallbacks))
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			s.sweep(ctx)
+		}
+	}
+}
+
+func (s *Submitter) sweep(ctx context.Context) {
+	entries, err := os.ReadDir(s.Dir)
+	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			s.Log.Debug("blocksubmit: readdir failed", "dir", s.Dir, "err", err)
+		}
+		return
+	}
+	now := time.Now()
+	for _, e := range entries {
+		if e.IsDir() || !strings.HasSuffix(e.Name(), ".hex") {
+			continue
+		}
+		path := filepath.Join(s.Dir, e.Name())
+		info, err := e.Info()
+		if err != nil {
+			continue
+		}
+		age := now.Sub(info.ModTime())
+		if age < s.Grace {
+			continue
+		}
+		if age > s.MaxAge {
+			s.Log.Error("blocksubmit: pending block too old, abandoning",
+				"path", path, "age", age)
+			_ = os.Remove(path)
+			continue
+		}
+		s.handleFile(ctx, path, e.Name(), age)
+	}
+}
+
+func (s *Submitter) handleFile(ctx context.Context, path, name string, age time.Duration) {
+	height, hashPrefix := parseFilename(name)
+	logger := s.Log.With("path", path, "height", height, "hash_prefix", hashPrefix, "age", age)
+
+	// First check: maybe the primary bitcoind quietly accepted it but
+	// ckpool failed to unlink (process crash between submit and unlink).
+	// If our primary already has a block at this height, we're done —
+	// no need to involve fallbacks.
+	if height > 0 && s.Primary != nil {
+		lookupCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+		hash, err := s.Primary.GetBlockHash(lookupCtx, height)
+		cancel()
+		if err == nil && hash != "" {
+			logger.Info("blocksubmit: block already on primary chain, removing stale pending file",
+				"canonical_hash", hash)
+			_ = os.Remove(path)
+			return
+		}
+	}
+
+	hex, err := os.ReadFile(path)
+	if err != nil {
+		logger.Warn("blocksubmit: read failed", "err", err)
+		return
+	}
+	hexStr := strings.TrimSpace(string(hex))
+	if hexStr == "" {
+		logger.Warn("blocksubmit: empty pending file, removing")
+		_ = os.Remove(path)
+		return
+	}
+
+	if len(s.Fallbacks) == 0 {
+		// No fallbacks configured: surface that the block is still
+		// pending and let the operator deal with it. Don't spam — log
+		// once per minute via a coarse round of `age`.
+		if int(age.Minutes())%1 == 0 {
+			logger.Warn("blocksubmit: pending block has no fallback RPCs configured")
+		}
+		return
+	}
+
+	for _, fb := range s.Fallbacks {
+		label := fb.Label
+		if label == "" {
+			label = fb.URL
+		}
+		submitCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+		err := submitOne(submitCtx, fb, hexStr)
+		cancel()
+		if err == nil {
+			logger.Warn("blocksubmit: fallback accepted block", "via", label)
+			_ = os.Remove(path)
+			if s.OnSuccess != nil {
+				s.OnSuccess(height, fb.URL, label)
+			}
+			return
+		}
+		if isDuplicate(err) {
+			// Block already on chain at the fallback (it was relayed
+			// via P2P from somewhere else). Same outcome — done.
+			logger.Info("blocksubmit: fallback reports duplicate (block on chain), removing pending",
+				"via", label, "err", err)
+			_ = os.Remove(path)
+			if s.OnSuccess != nil {
+				s.OnSuccess(height, fb.URL, label)
+			}
+			return
+		}
+		logger.Warn("blocksubmit: fallback rejected", "via", label, "err", err)
+	}
+
+	logger.Error("blocksubmit: ALL fallbacks failed; block remains pending for next sweep",
+		"fallbacks_tried", len(s.Fallbacks))
+}
+
+// submitOne sends submitblock to one fallback RPC and returns nil on
+// acceptance, a wrapped error otherwise.
+func submitOne(ctx context.Context, fb FallbackTarget, hexBlock string) error {
+	rpc := bitcoind.NewRPC(fb.URL, fb.User, fb.Password, 30*time.Second)
+	// submitblock returns: null on success, or a JSON string with the
+	// reject reason. We have to be a bit clever with json.RawMessage to
+	// distinguish those.
+	var raw json.RawMessage
+	if err := rpc.Call(ctx, "submitblock", []any{hexBlock}, &raw); err != nil {
+		return fmt.Errorf("submitblock rpc: %w", err)
+	}
+	// Trim whitespace from the raw bytes for comparison.
+	body := strings.TrimSpace(string(raw))
+	if body == "" || body == "null" {
+		return nil
+	}
+	// Strip surrounding quotes from the JSON string.
+	if len(body) >= 2 && body[0] == '"' && body[len(body)-1] == '"' {
+		body = body[1 : len(body)-1]
+	}
+	return errors.New(body)
+}
+
+// isDuplicate covers Bitcoin Core's submitblock reject reasons that
+// mean "this block is already in the chain": "duplicate",
+// "duplicate-invalid", "duplicate-inconclusive". From the operator's
+// perspective these are all "we don't need to keep trying".
+func isDuplicate(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := strings.ToLower(err.Error())
+	return strings.Contains(msg, "duplicate")
+}
+
+// parseFilename extracts (height, hashPrefix) from the ckpool-written
+// filename `<height>-<hashprefix16>.hex`. Returns zeros if the format
+// doesn't match — the caller should still attempt the submit; we just
+// can't do the "already on chain" optimization.
+func parseFilename(name string) (int64, string) {
+	base := strings.TrimSuffix(name, ".hex")
+	dash := strings.IndexByte(base, '-')
+	if dash <= 0 {
+		return 0, ""
+	}
+	height, err := parseInt64(base[:dash])
+	if err != nil {
+		return 0, base[dash+1:]
+	}
+	return height, base[dash+1:]
+}
+
+func parseInt64(s string) (int64, error) {
+	var n int64
+	for _, c := range s {
+		if c < '0' || c > '9' {
+			return 0, fmt.Errorf("not a number: %q", s)
+		}
+		n = n*10 + int64(c-'0')
+	}
+	return n, nil
+}

api/internal/config/config.go

@@ -39,6 +39,22 @@ type Config struct {
 	// the UI uses the public mempool.space; non-empty means the user
 	// has pointed Kamado at their own instance via the StartOS config.
 	MempoolBaseURL string
+
+	// Directory ckpool's patched local_block_submit writes raw block
+	// hex into. Empty disables the fallback submitter.
+	PendingBlocksDir string
+
+	// Comma- or newline-separated list of fallback bitcoind RPC URLs
+	// that the submitter will try in order if a block stays pending
+	// past the grace window. Each entry can encode credentials inline:
+	//   https://user:pass@host:8332/
+	// or without (e.g. an unauthenticated localhost backup node).
+	BackupRPCURLs string
+
+	// How long a pending block file must persist before the submitter
+	// will try fallback RPCs. Defaults to 30s — long enough for
+	// ckpool's own retry loop and a fast ckpool->bitcoind round-trip.
+	PendingBlocksGrace time.Duration
 }
 
 func FromEnv() (*Config, error) {
@@ -54,6 +70,10 @@ func FromEnv() (*Config, error) {
 		DBPath:            getenv("DB_PATH", "/var/lib/kamado/kamado.db"),
 		PollInterval:      getenvDuration("POLL_INTERVAL", 5*time.Second),
 		MempoolBaseURL:    os.Getenv("MEMPOOL_BASE_URL"),
+
+		PendingBlocksDir:   os.Getenv("PENDING_BLOCKS_DIR"),
+		BackupRPCURLs:      os.Getenv("BACKUP_RPC_URLS"),
+		PendingBlocksGrace: getenvDuration("PENDING_BLOCKS_GRACE", 30*time.Second),
 	}
 
 	if cfg.BitcoinRPCURL == "" {

api/internal/httpapi/server.go

@@ -90,14 +90,42 @@ func writeJSON(w http.ResponseWriter, status int, v any) {
 func (s *Server) health(w http.ResponseWriter, r *http.Request) {
 	snap := s.Agg.Snapshot()
 	status := http.StatusOK
+
+	// Submit-attempt gap: if we've tried to submit more blocks than have
+	// been confirmed, surface the gap. A non-zero gap is a strong signal
+	// even if everything else looks healthy.
+	submitGap := snap.BlockSubmitAttempts - snap.BlockSubmitsConfirmed
+	if submitGap < 0 {
+		submitGap = 0
+	}
+
+	// ZMQ freshness: only meaningful if the operator enabled it. Stale
+	// = no event in 30 minutes (typical mainnet block interval is 10
+	// min, but spikes happen).
+	zmqStale := false
+	if snap.ZMQEnabled && snap.HasLastZMQEvent && snap.LastZMQEventAge > 1800 {
+		zmqStale = true
+	}
+
+	overallOK := snap.CKPoolOK && snap.BitcoinOK && submitGap == 0 && !zmqStale
 	if !snap.CKPoolOK || !snap.BitcoinOK {
 		status = http.StatusServiceUnavailable
 	}
 	writeJSON(w, status, map[string]any{
-		"ok":         snap.CKPoolOK && snap.BitcoinOK,
-		"ckpool":     snap.CKPoolOK,
-		"bitcoin":    snap.BitcoinOK,
-		"last_error": snap.LastError,
+		"ok":                       overallOK,
+		"ckpool":                   snap.CKPoolOK,
+		"bitcoin":                  snap.BitcoinOK,
+		"submit_attempts":          snap.BlockSubmitAttempts,
+		"submits_confirmed":        snap.BlockSubmitsConfirmed,
+		"submit_gap":               submitGap,
+		"fallback_submits_total":   snap.FallbackSubmitsTotal,
+		"last_fallback_submit_at":  snap.LastFallbackSubmitAt,
+		"last_fallback_via":        snap.LastFallbackVia,
+		"zmq_enabled":              snap.ZMQEnabled,
+		"zmq_event_age_seconds":    snap.LastZMQEventAge,
+		"zmq_has_event":            snap.HasLastZMQEvent,
+		"zmq_stale":                zmqStale,
+		"last_error":               snap.LastError,
 	})
 }
 

api/internal/httpapi/ws.go

@@ -42,8 +42,18 @@ type wsClient struct {
 	conn    net.Conn
 	send    chan []byte // serialized JSON frames pending write
 	writeMu sync.Mutex  // guards writes to conn (writer + reader-pong)
+
+	// dropMisses counts consecutive Broadcast calls where this client's
+	// send channel was full. After maxDropMisses we close the connection
+	// instead of letting a stuck reader hold an old snapshot forever.
+	dropMisses int
 }
 
+// maxDropMisses bounds how many back-to-back broadcasts we let the
+// hub skip for one client before forcibly disconnecting it. With the
+// poll cadence at 5s this is roughly 30s of unresponsiveness.
+const maxDropMisses = 6
+
 // writeFrameLocked writes one frame, serializing writers on the client.
 func (c *wsClient) writeFrameLocked(opcode byte, payload []byte) error {
 	c.writeMu.Lock()
@@ -79,22 +89,33 @@ func (h *Hub) remove(c *wsClient) {
 
 // Broadcast serializes the snapshot once and enqueues it for every
 // subscribed client. Slow clients are dropped rather than blocking
-// the hub.
+// the hub. After maxDropMisses consecutive drops for a single client,
+// we forcibly close its connection so a stuck consumer doesn't hold
+// resources indefinitely or freeze on a stale snapshot.
 func (h *Hub) Broadcast(snap state.Snapshot) {
 	payload, err := json.Marshal(snap)
 	if err != nil {
 		return
 	}
-	h.mu.RLock()
-	defer h.mu.RUnlock()
+	var toClose []*wsClient
+	h.mu.Lock()
 	for c := range h.clients {
 		select {
 		case c.send <- payload:
+			c.dropMisses = 0
 		default:
-			// Drop — the client's reader goroutine will clean up on
-			// the next write failure or close frame.
+			c.dropMisses++
+			if c.dropMisses >= maxDropMisses {
+				toClose = append(toClose, c)
+			}
 		}
 	}
+	h.mu.Unlock()
+	for _, c := range toClose {
+		// Closing the conn unblocks the writer goroutine, which
+		// removes the client from the hub through wsReader's defer.
+		_ = c.conn.Close()
+	}
 }
 
 // handleWS upgrades an HTTP request to a WebSocket connection and

api/internal/state/aggregator.go

@@ -77,9 +77,25 @@ type Snapshot struct {
 	// log lines) and confirmed solves ("Solved and confirmed block").
 	// A growing gap means bitcoind is rejecting our submissions or
 	// dropping the RPC — surface it in the UI as an alert.
-	BlockSubmitAttempts int64 `json:"block_submit_attempts"`
+	BlockSubmitAttempts   int64 `json:"block_submit_attempts"`
 	BlockSubmitsConfirmed int64 `json:"block_submits_confirmed"`
 
+	// Number of blocks the fallback submitter had to broadcast via a
+	// backup RPC URL because the primary bitcoind didn't accept them
+	// in time. Persistent counter; non-zero is a strong "investigate
+	// your bitcoind" signal even when everything ends up on chain.
+	FallbackSubmitsTotal int64  `json:"fallback_submits_total"`
+	LastFallbackSubmitAt int64  `json:"last_fallback_submit_at,omitempty"` // unix seconds
+	LastFallbackVia     string `json:"last_fallback_via,omitempty"`
+
+	// Health diagnostics for /healthz and the UI status badge.
+	// LastZMQEventAge is the seconds-since the last bitcoind hashblock
+	// frame arrived; -1 means no event seen since startup. ZMQEnabled
+	// is whether the user configured an endpoint at all.
+	ZMQEnabled        bool    `json:"zmq_enabled"`
+	LastZMQEventAge   float64 `json:"last_zmq_event_age,omitempty"` // seconds; >=0
+	HasLastZMQEvent   bool    `json:"has_last_zmq_event"`
+
 	// Health
 	CKPoolOK  bool   `json:"ckpool_ok"`
 	BitcoinOK bool   `json:"bitcoin_ok"`
@@ -98,6 +114,7 @@ const (
 
 	kvSubmitAttempts   = "block_submit_attempts"
 	kvSubmitsConfirmed = "block_submits_confirmed"
+	kvFallbackSubmits  = "fallback_submits_total"
 
 	// reconcileInterval is how often we sweep recent blocks looking
 	// for missing hash/reward enrichment and reorg-orphaned hashes.
@@ -173,6 +190,18 @@ type Aggregator struct {
 	blockSubmitAttempts   int64
 	blockSubmitsConfirmed int64
 	lastSubmitCountSave   time.Time
+
+	// Fallback-submitter telemetry. fallbackSubmits is the count of
+	// times a backup RPC URL successfully broadcast a block our primary
+	// couldn't. Persisted; survives restarts.
+	fallbackSubmits      int64
+	lastFallbackSubmitAt time.Time
+	lastFallbackVia      string
+
+	// ZMQ diagnostics: timestamp of the last hashblock frame relayed
+	// from zmqmon. Used by /healthz to flag stale subscriptions.
+	zmqEnabled       bool
+	lastZMQEventTime time.Time
 }
 
 func New(ck *ckpool.Client, rpc *bitcoind.RPC, interval time.Duration, log *slog.Logger) *Aggregator {
@@ -190,6 +219,9 @@ func New(ck *ckpool.Client, rpc *bitcoind.RPC, interval time.Duration, log *slog
 // loaded from the store before the first refresh. If tipEvents is non-nil,
 // each received tip triggers an immediate refresh outside the poll cadence.
 func (a *Aggregator) Run(ctx context.Context, tipEvents <-chan zmqmon.TipEvent) {
+	a.mu.Lock()
+	a.zmqEnabled = tipEvents != nil
+	a.mu.Unlock()
 	a.loadPersistedBlocks()
 	a.loadPersistedState()
 	a.refresh(ctx)
@@ -206,12 +238,35 @@ func (a *Aggregator) Run(ctx context.Context, tipEvents <-chan zmqmon.TipEvent)
 				tipEvents = nil
 				continue
 			}
+			a.mu.Lock()
+			a.lastZMQEventTime = ev.SeenAt
+			a.mu.Unlock()
 			a.Log.Debug("zmq tip, refreshing", "hash", ev.Hash)
 			a.refresh(ctx)
 		}
 	}
 }
 
+// RecordFallbackSubmit is called by blocksubmit.Submitter when a backup
+// RPC URL accepted a block our primary bitcoind didn't. Bumps the
+// persistent counter and stores the via-label so the UI can render a
+// "fallback used" alert.
+func (a *Aggregator) RecordFallbackSubmit(height int64, via string) {
+	a.mu.Lock()
+	a.fallbackSubmits++
+	a.lastFallbackSubmitAt = time.Now()
+	a.lastFallbackVia = via
+	n := a.fallbackSubmits
+	a.mu.Unlock()
+	a.Log.Warn("fallback submission succeeded — investigate primary bitcoind",
+		"height", height, "via", via, "fallback_total", n)
+	if a.Store != nil {
+		if err := a.Store.SetKV(kvFallbackSubmits, strconv.FormatInt(n, 10)); err != nil {
+			a.Log.Warn("fallback counter persist failed", "err", err)
+		}
+	}
+}
+
 // Snapshot returns a copy of the current snapshot.
 func (a *Aggregator) Snapshot() Snapshot {
 	a.mu.RLock()
@@ -413,6 +468,16 @@ func (a *Aggregator) refresh(ctx context.Context) {
 	}
 	next.BlockSubmitAttempts = a.blockSubmitAttempts
 	next.BlockSubmitsConfirmed = a.blockSubmitsConfirmed
+	next.FallbackSubmitsTotal = a.fallbackSubmits
+	if !a.lastFallbackSubmitAt.IsZero() {
+		next.LastFallbackSubmitAt = a.lastFallbackSubmitAt.Unix()
+		next.LastFallbackVia = a.lastFallbackVia
+	}
+	next.ZMQEnabled = a.zmqEnabled
+	if !a.lastZMQEventTime.IsZero() {
+		next.LastZMQEventAge = time.Since(a.lastZMQEventTime).Seconds()
+		next.HasLastZMQEvent = true
+	}
 	a.snap = next
 	cb := a.OnRefresh
 	pushed := next
@@ -456,6 +521,13 @@ func (a *Aggregator) loadPersistedState() {
 			a.mu.Unlock()
 		}
 	}
+	if v, err := a.Store.GetKV(kvFallbackSubmits); err == nil && v != "" {
+		if n, perr := strconv.ParseInt(v, 10, 64); perr == nil {
+			a.mu.Lock()
+			a.fallbackSubmits = n
+			a.mu.Unlock()
+		}
+	}
 
 	cutoff := time.Now().Add(-24 * time.Hour).Unix()
 	if samples, err := a.Store.HashrateSince(cutoff); err != nil {