P1 reliability + block-broadcast fallback path

P1 audits / fixes:

* Bitcoin Core RPC now retries up to 3 times with linear backoff on
  transport errors, 5xx responses, and warm-up/loading RPC errors
  (code -28). Hard "no" answers (block-not-found etc.) bubble up
  immediately so we don't mask real errors.

* WebSocket hub disconnects clients that miss 6 consecutive broadcasts
  (~30s with the default poll cadence). Stuck readers no longer hold
  stale snapshots indefinitely or freeze hub state.

* ZMQ subscriber freshness: aggregator records the last-event
  timestamp, surfaces zmq_enabled / has_last_zmq_event /
  last_zmq_event_age in the snapshot. /healthz flags zmq_stale when
  the gap exceeds 30 minutes.

* /healthz expanded with submit_attempts / submits_confirmed /
  submit_gap, fallback_submits_total + last_fallback_*, and the zmq
  staleness check. Now usable as a real-world ops dashboard signal.

Block-broadcast fallback (new feature):

  * ckpool patch 0004: hooks local_block_submit to write the raw block
    hex to <logdir>/pending-blocks/<height>-<hash16>.hex right before
    invoking generator_submitblock. Unlinks on success. ckpool's normal
    flow is otherwise untouched.

  * api/internal/blocksubmit: watcher polls the dir every 5s. Files
    sitting longer than the grace window (default 30s, configurable)
    are re-broadcast through operator-supplied backup RPC URLs in
    sequence. Treats both null and any "duplicate*" reject reason as
    success (the block landed). Pre-checks the primary chain first so
    a stale file from a successful-but-unlinked submit gets cleaned
    up without bothering fallbacks.

  * Aggregator records each successful fallback submission as a
    persistent counter and surfaces it in the snapshot so the UI can
    show a "primary bitcoind isn't accepting submits" alert.

  * Config: BACKUP_RPC_URLS (comma- or newline-separated, with
    optional inline credentials) plus PENDING_BLOCKS_DIR and
    PENDING_BLOCKS_GRACE. URLs are parsed via net/url so
    https://user:pass@host:port/ works cleanly.

The fallback is opt-in and disabled by default. Once enabled with at
least one URL, a primary bitcoind outage at the moment of solving no
longer means a lost block — kamado-api re-broadcasts via whichever
backup the operator trusts (a second self-hosted node, an
authenticated public RPC service, etc.).

api/internal/bitcoind/rpc.go

@@ -6,9 +6,11 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
+	"strings"
 	"time"
 )
 
@@ -50,8 +52,35 @@ func (e *rpcError) Error() string {
 	return fmt.Sprintf("bitcoind rpc error %d: %s", e.Code, e.Message)
 }
 
-// Call performs a single JSON-RPC request and unmarshals the result.
+// Call performs a JSON-RPC request and unmarshals the result. Transient
+// failures (transport errors, 502/503/504, or RPC errors with an
+// "in warmup"/"loading"/"verifying" message that bitcoind returns
+// during startup) are retried up to twice with a short backoff. RPC
+// errors with semantic codes (e.g. block-not-found) are returned
+// immediately because retrying won't change the answer.
 func (c *RPC) Call(ctx context.Context, method string, params []any, out any) error {
+	const maxAttempts = 3
+	var lastErr error
+	for attempt := 1; attempt <= maxAttempts; attempt++ {
+		err := c.callOnce(ctx, method, params, out)
+		if err == nil {
+			return nil
+		}
+		lastErr = err
+		if !isRetryable(err) || attempt == maxAttempts || ctx.Err() != nil {
+			return err
+		}
+		backoff := time.Duration(attempt) * 200 * time.Millisecond
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(backoff):
+		}
+	}
+	return lastErr
+}
+
+func (c *RPC) callOnce(ctx context.Context, method string, params []any, out any) error {
 	body, err := json.Marshal(rpcRequest{
 		JSONRPC: "1.0",
 		ID:      "kamado",
@@ -78,10 +107,14 @@ func (c *RPC) Call(ctx context.Context, method string, params []any, out any) er
 	if err != nil {
 		return fmt.Errorf("bitcoind rpc: read body: %w", err)
 	}
-	// bitcoind returns 500 on rpc errors but still with a valid JSON body.
+	// 5xx without a parseable body: surface as transport-level error
+	// so isRetryable() can flag it.
 	var rr rpcResponse
-	if err := json.Unmarshal(raw, &rr); err != nil {
-		return fmt.Errorf("bitcoind rpc: unmarshal (status %d): %w: %s", resp.StatusCode, err, string(raw))
+	if jerr := json.Unmarshal(raw, &rr); jerr != nil {
+		if resp.StatusCode >= 500 {
+			return fmt.Errorf("bitcoind rpc: %d: %s", resp.StatusCode, truncate(string(raw), 256))
+		}
+		return fmt.Errorf("bitcoind rpc: unmarshal (status %d): %w: %s", resp.StatusCode, jerr, truncate(string(raw), 256))
 	}
 	if rr.Error != nil {
 		return rr.Error
@@ -92,6 +125,56 @@ func (c *RPC) Call(ctx context.Context, method string, params []any, out any) er
 	return nil
 }
 
+func truncate(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[:n] + "…"
+}
+
+// isRetryable distinguishes "the RPC didn't reach a definitive answer
+// yet" (transport-level errors, 5xx, bitcoind warm-up/loading) from
+// "bitcoind answered, the answer is no" (RPC error with a code, e.g.
+// -5 block not found). We retry the first kind and bubble the second.
+func isRetryable(err error) bool {
+	if err == nil {
+		return false
+	}
+	// rpcError values come from bitcoind itself — check for warmup/
+	// loading messages that mean "ask again in a moment".
+	var rerr *rpcError
+	if errors.As(err, &rerr) {
+		// -28 is RPC_IN_WARMUP per bitcoin/src/rpc/protocol.h
+		if rerr.Code == -28 {
+			return true
+		}
+		msg := strings.ToLower(rerr.Message)
+		if strings.Contains(msg, "warming up") ||
+			strings.Contains(msg, "loading") ||
+			strings.Contains(msg, "verifying") ||
+			strings.Contains(msg, "rewinding") ||
+			strings.Contains(msg, "still busy") {
+			return true
+		}
+		return false
+	}
+	// Wrapped transport / 5xx / read errors all flow through fmt.Errorf
+	// with the "bitcoind rpc:" prefix and no rpcError target.
+	msg := strings.ToLower(err.Error())
+	switch {
+	case strings.Contains(msg, "connection refused"),
+		strings.Contains(msg, "connection reset"),
+		strings.Contains(msg, "no such host"),
+		strings.Contains(msg, "deadline exceeded"),
+		strings.Contains(msg, "i/o timeout"),
+		strings.Contains(msg, "eof"),
+		strings.Contains(msg, "broken pipe"),
+		strings.Contains(msg, " 502"), strings.Contains(msg, " 503"), strings.Contains(msg, " 504"):
+		return true
+	}
+	return false
+}
+
 // ---- typed method wrappers ----
 
 type BlockchainInfo struct {