P1 reliability + block-broadcast fallback path

P1 audits / fixes:

* Bitcoin Core RPC now retries up to 3 times with linear backoff on
  transport errors, 5xx responses, and warm-up/loading RPC errors
  (code -28). Hard "no" answers (block-not-found etc.) bubble up
  immediately so we don't mask real errors.

* WebSocket hub disconnects clients that miss 6 consecutive broadcasts
  (~30s with the default poll cadence). Stuck readers no longer hold
  stale snapshots indefinitely or freeze hub state.

* ZMQ subscriber freshness: aggregator records the last-event
  timestamp, surfaces zmq_enabled / has_last_zmq_event /
  last_zmq_event_age in the snapshot. /healthz flags zmq_stale when
  the gap exceeds 30 minutes.

* /healthz expanded with submit_attempts / submits_confirmed /
  submit_gap, fallback_submits_total + last_fallback_*, and the zmq
  staleness check. Now usable as a real-world ops dashboard signal.

Block-broadcast fallback (new feature):

  * ckpool patch 0004: hooks local_block_submit to write the raw block
    hex to <logdir>/pending-blocks/<height>-<hash16>.hex right before
    invoking generator_submitblock. Unlinks on success. ckpool's normal
    flow is otherwise untouched.

  * api/internal/blocksubmit: watcher polls the dir every 5s. Files
    sitting longer than the grace window (default 30s, configurable)
    are re-broadcast through operator-supplied backup RPC URLs in
    sequence. Treats both null and any "duplicate*" reject reason as
    success (the block landed). Pre-checks the primary chain first so
    a stale file from a successful-but-unlinked submit gets cleaned
    up without bothering fallbacks.

  * Aggregator records each successful fallback submission as a
    persistent counter and surfaces it in the snapshot so the UI can
    show a "primary bitcoind isn't accepting submits" alert.

  * Config: BACKUP_RPC_URLS (comma- or newline-separated, with
    optional inline credentials) plus PENDING_BLOCKS_DIR and
    PENDING_BLOCKS_GRACE. URLs are parsed via net/url so
    https://user:pass@host:port/ works cleanly.

The fallback is opt-in and disabled by default. Once enabled with at
least one URL, a primary bitcoind outage at the moment of solving no
longer means a lost block — kamado-api re-broadcasts via whichever
backup the operator trusts (a second self-hosted node, an
authenticated public RPC service, etc.).

api/cmd/kamado-api/main.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"log/slog"
 	"net/http"
+	"net/url"
 	"os"
 	"os/signal"
 	"strconv"
@@ -19,6 +20,7 @@ import (
 	"time"
 
 	"github.com/kamadopool/kamado-api/internal/bitcoind"
+	"github.com/kamadopool/kamado-api/internal/blocksubmit"
 	"github.com/kamadopool/kamado-api/internal/ckpool"
 	"github.com/kamadopool/kamado-api/internal/config"
 	"github.com/kamadopool/kamado-api/internal/httpapi"
@@ -116,6 +118,27 @@ func main() {
 	go agg.IngestBlockEvents(ctx, tailer.Events)
 	go agg.IngestAttemptEvents(ctx, tailer.Attempts)
 
+	// Fallback block submitter: if ckpool's primary bitcoind doesn't
+	// accept a block, the patched local_block_submit leaves the raw
+	// hex sitting in PENDING_BLOCKS_DIR. This watcher re-broadcasts it
+	// via the operator-configured BACKUP_RPC_URLS list.
+	if cfg.PendingBlocksDir != "" {
+		fallbacks := parseBackupRPCs(cfg.BackupRPCURLs, log)
+		sub := &blocksubmit.Submitter{
+			Dir:       cfg.PendingBlocksDir,
+			Grace:     cfg.PendingBlocksGrace,
+			Primary:   rpc,
+			Fallbacks: fallbacks,
+			Log:       log,
+			OnSuccess: func(height int64, viaURL, viaLabel string) {
+				agg.RecordFallbackSubmit(height, viaLabel)
+			},
+		}
+		go sub.Run(ctx, 5*time.Second)
+	} else {
+		log.Info("blocksubmit fallback disabled (PENDING_BLOCKS_DIR not set)")
+	}
+
 	srv := &http.Server{
 		Addr:              cfg.ListenAddr,
 		Handler:           api.Handler(),
@@ -137,3 +160,41 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+// parseBackupRPCs splits BACKUP_RPC_URLS (newline- or comma-separated)
+// into FallbackTargets. Credentials are accepted inline as
+// https://user:pass@host:port/. Lines beginning with '#' are comments.
+// Whitespace and empty lines are ignored.
+func parseBackupRPCs(raw string, log *slog.Logger) []blocksubmit.FallbackTarget {
+	if raw == "" {
+		return nil
+	}
+	separated := strings.NewReplacer(",", "\n").Replace(raw)
+	var out []blocksubmit.FallbackTarget
+	for _, line := range strings.Split(separated, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		u, err := url.Parse(line)
+		if err != nil {
+			log.Warn("blocksubmit: invalid fallback URL, skipping", "raw", line, "err", err)
+			continue
+		}
+		var user, pass string
+		if u.User != nil {
+			user = u.User.Username()
+			pass, _ = u.User.Password()
+			u.User = nil
+		}
+		clean := u.String()
+		out = append(out, blocksubmit.FallbackTarget{
+			URL:      clean,
+			User:     user,
+			Password: pass,
+			Label:    u.Host,
+		})
+		log.Info("blocksubmit fallback registered", "url", clean, "host", u.Host, "auth", user != "")
+	}
+	return out
+}