diff --git a/api/cmd/kamado-api/main.go b/api/cmd/kamado-api/main.go index a6e82b3..98f56a7 100644 --- a/api/cmd/kamado-api/main.go +++ b/api/cmd/kamado-api/main.go @@ -11,7 +11,6 @@ import ( "fmt" "log/slog" "net/http" - "net/url" "os" "os/signal" "strconv" @@ -20,7 +19,6 @@ import ( "time" "github.com/kamadopool/kamado-api/internal/bitcoind" - "github.com/kamadopool/kamado-api/internal/blocksubmit" "github.com/kamadopool/kamado-api/internal/ckpool" "github.com/kamadopool/kamado-api/internal/config" "github.com/kamadopool/kamado-api/internal/httpapi" @@ -118,29 +116,6 @@ func main() { go agg.IngestBlockEvents(ctx, tailer.Events) go agg.IngestAttemptEvents(ctx, tailer.Attempts) - // Fallback block submitter: if ckpool's primary bitcoind doesn't - // accept a block, the patched local_block_submit leaves the raw - // hex sitting in PENDING_BLOCKS_DIR. This watcher re-broadcasts it - // via the operator-configured BACKUP_RPC_URLS list. - if cfg.PendingBlocksDir != "" { - fallbacks := parseBackupRPCs(cfg.BackupRPCURLs, log) - sub := &blocksubmit.Submitter{ - Dir: cfg.PendingBlocksDir, - Grace: cfg.PendingBlocksGrace, - Primary: rpc, - Fallbacks: fallbacks, - Log: log, - OnSuccess: func(height int64, viaURL, viaLabel string) { - agg.RecordFallbackSubmit(height, viaLabel) - }, - } - // Sweep every second so we react within ~grace+1s of a - // failed submit. Reading an empty directory is cheap. - go sub.Run(ctx, 1*time.Second) - } else { - log.Info("blocksubmit fallback disabled (PENDING_BLOCKS_DIR not set)") - } - // Wait briefly for the aggregator's first refresh to complete so // the very first /api/snapshot or /api/health hit doesn't see an // all-zeros snapshot and report bitcoin_ok=false during its own @@ -177,41 +152,3 @@ func main() { os.Exit(1) } } - -// parseBackupRPCs splits BACKUP_RPC_URLS (newline- or comma-separated) -// into FallbackTargets. Credentials are accepted inline as -// https://user:pass@host:port/. Lines beginning with '#' are comments. -// Whitespace and empty lines are ignored. -func parseBackupRPCs(raw string, log *slog.Logger) []blocksubmit.FallbackTarget { - if raw == "" { - return nil - } - separated := strings.NewReplacer(",", "\n").Replace(raw) - var out []blocksubmit.FallbackTarget - for _, line := range strings.Split(separated, "\n") { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - u, err := url.Parse(line) - if err != nil { - log.Warn("blocksubmit: invalid fallback URL, skipping", "raw", line, "err", err) - continue - } - var user, pass string - if u.User != nil { - user = u.User.Username() - pass, _ = u.User.Password() - u.User = nil - } - clean := u.String() - out = append(out, blocksubmit.FallbackTarget{ - URL: clean, - User: user, - Password: pass, - Label: u.Host, - }) - log.Info("blocksubmit fallback registered", "url", clean, "host", u.Host, "auth", user != "") - } - return out -} diff --git a/api/internal/blocksubmit/submit.go b/api/internal/blocksubmit/submit.go deleted file mode 100644 index c074fcf..0000000 --- a/api/internal/blocksubmit/submit.go +++ /dev/null @@ -1,275 +0,0 @@ -// Package blocksubmit watches a sidecar directory that ckpool fills -// with raw block hex right before it tries to submit to its primary -// bitcoind, and re-submits via operator-configured fallback RPC URLs -// if a file persists longer than the grace period. -// -// ckpool's local_block_submit (patched in 0004) writes -// /pending-blocks/-.hex before calling -// generator_submitblock, and unlinks on success. So the rule is -// simple: any file still present after `grace` has likely failed to -// land, and we should help. -// -// Failure modes this protects against: -// - bitcoind crashed / OOM-killed at the moment of submission -// - bitcoind RPC saturated, returning timeouts -// - upstream peering issue making bitcoind unable to relay -// - ckpool itself crashes after the file was written but before unlink -// -// Each fallback URL is tried in order. submitblock returns either null -// (accepted), a "duplicate*" reason (already on chain — same outcome), -// or a hard error. We treat both null and any "duplicate" prefix as -// success and unlink. -package blocksubmit - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "log/slog" - "os" - "path/filepath" - "strings" - "time" - - "github.com/kamadopool/kamado-api/internal/bitcoind" -) - -// FallbackTarget is one back-up RPC endpoint we'll try if the primary -// bitcoind didn't accept a block. URL must be a full http(s):// endpoint; -// User and Password are optional (some self-hosted nodes accept -// unauthenticated localhost RPC, public RPC services usually don't). -type FallbackTarget struct { - URL string - User string - Password string - Label string // human-friendly name for logs; defaults to host -} - -// Submitter polls a pending-blocks directory and re-broadcasts blocks -// that have been sitting too long to its primary bitcoind plus any -// configured fallbacks. Stateless across iterations — the filesystem -// is the queue. -type Submitter struct { - Dir string - Grace time.Duration // how long we wait before considering a file stale - MaxAge time.Duration // older than this: log error and remove (operator intervention required) - Primary *bitcoind.RPC // primary bitcoind for "is the block already on chain?" sanity check - Fallbacks []FallbackTarget - Log *slog.Logger - - // OnSuccess, if set, is called with the height and the URL string - // that accepted the block, so the aggregator can bump a counter or - // emit a UI banner. - OnSuccess func(height int64, viaURL string, viaLabel string) -} - -// Run blocks until ctx is cancelled, sweeping the dir every poll. -func (s *Submitter) Run(ctx context.Context, poll time.Duration) { - if s.Dir == "" { - s.Log.Info("blocksubmit disabled (no pending-blocks dir)") - return - } - if s.Grace == 0 { - // Match the config default — short enough that a failed - // submit gets recovered while the work is still relevant, - // long enough that we don't spuriously fallback while the - // primary's RPC is mid-handshake. - s.Grace = 3 * time.Second - } - if s.MaxAge == 0 { - s.MaxAge = 24 * time.Hour - } - if poll == 0 { - poll = 5 * time.Second - } - - // Ensure the directory exists so the first sweep doesn't WARN. Don't - // fail if we can't create it — ckpool may not have run yet, or perms - // may need a moment to settle. The sweep itself logs at DEBUG when - // the dir is missing. - _ = os.MkdirAll(s.Dir, 0750) - - t := time.NewTicker(poll) - defer t.Stop() - s.Log.Info("blocksubmit started", "dir", s.Dir, "grace", s.Grace, "fallbacks", len(s.Fallbacks)) - for { - select { - case <-ctx.Done(): - return - case <-t.C: - s.sweep(ctx) - } - } -} - -func (s *Submitter) sweep(ctx context.Context) { - entries, err := os.ReadDir(s.Dir) - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - s.Log.Debug("blocksubmit: readdir failed", "dir", s.Dir, "err", err) - } - return - } - now := time.Now() - for _, e := range entries { - if e.IsDir() || !strings.HasSuffix(e.Name(), ".hex") { - continue - } - path := filepath.Join(s.Dir, e.Name()) - info, err := e.Info() - if err != nil { - continue - } - age := now.Sub(info.ModTime()) - if age < s.Grace { - continue - } - if age > s.MaxAge { - s.Log.Error("blocksubmit: pending block too old, abandoning", - "path", path, "age", age) - _ = os.Remove(path) - continue - } - s.handleFile(ctx, path, e.Name(), age) - } -} - -func (s *Submitter) handleFile(ctx context.Context, path, name string, age time.Duration) { - height, hashPrefix := parseFilename(name) - logger := s.Log.With("path", path, "height", height, "hash_prefix", hashPrefix, "age", age) - - // First check: maybe the primary bitcoind quietly accepted it but - // ckpool failed to unlink (process crash between submit and unlink). - // If our primary already has a block at this height, we're done — - // no need to involve fallbacks. - if height > 0 && s.Primary != nil { - lookupCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - hash, err := s.Primary.GetBlockHash(lookupCtx, height) - cancel() - if err == nil && hash != "" { - logger.Info("blocksubmit: block already on primary chain, removing stale pending file", - "canonical_hash", hash) - _ = os.Remove(path) - return - } - } - - hex, err := os.ReadFile(path) - if err != nil { - logger.Warn("blocksubmit: read failed", "err", err) - return - } - hexStr := strings.TrimSpace(string(hex)) - if hexStr == "" { - logger.Warn("blocksubmit: empty pending file, removing") - _ = os.Remove(path) - return - } - - if len(s.Fallbacks) == 0 { - // No fallbacks configured: surface that the block is still - // pending and let the operator deal with it. Don't spam — log - // once per minute via a coarse round of `age`. - if int(age.Minutes())%1 == 0 { - logger.Warn("blocksubmit: pending block has no fallback RPCs configured") - } - return - } - - for _, fb := range s.Fallbacks { - label := fb.Label - if label == "" { - label = fb.URL - } - submitCtx, cancel := context.WithTimeout(ctx, 30*time.Second) - err := submitOne(submitCtx, fb, hexStr) - cancel() - if err == nil { - logger.Warn("blocksubmit: fallback accepted block", "via", label) - _ = os.Remove(path) - if s.OnSuccess != nil { - s.OnSuccess(height, fb.URL, label) - } - return - } - if isDuplicate(err) { - // Block already on chain at the fallback (it was relayed - // via P2P from somewhere else). Same outcome — done. - logger.Info("blocksubmit: fallback reports duplicate (block on chain), removing pending", - "via", label, "err", err) - _ = os.Remove(path) - if s.OnSuccess != nil { - s.OnSuccess(height, fb.URL, label) - } - return - } - logger.Warn("blocksubmit: fallback rejected", "via", label, "err", err) - } - - logger.Error("blocksubmit: ALL fallbacks failed; block remains pending for next sweep", - "fallbacks_tried", len(s.Fallbacks)) -} - -// submitOne sends submitblock to one fallback RPC and returns nil on -// acceptance, a wrapped error otherwise. -func submitOne(ctx context.Context, fb FallbackTarget, hexBlock string) error { - rpc := bitcoind.NewRPC(fb.URL, fb.User, fb.Password, 30*time.Second) - // submitblock returns: null on success, or a JSON string with the - // reject reason. We have to be a bit clever with json.RawMessage to - // distinguish those. - var raw json.RawMessage - if err := rpc.Call(ctx, "submitblock", []any{hexBlock}, &raw); err != nil { - return fmt.Errorf("submitblock rpc: %w", err) - } - // Trim whitespace from the raw bytes for comparison. - body := strings.TrimSpace(string(raw)) - if body == "" || body == "null" { - return nil - } - // Strip surrounding quotes from the JSON string. - if len(body) >= 2 && body[0] == '"' && body[len(body)-1] == '"' { - body = body[1 : len(body)-1] - } - return errors.New(body) -} - -// isDuplicate covers Bitcoin Core's submitblock reject reasons that -// mean "this block is already in the chain": "duplicate", -// "duplicate-invalid", "duplicate-inconclusive". From the operator's -// perspective these are all "we don't need to keep trying". -func isDuplicate(err error) bool { - if err == nil { - return false - } - msg := strings.ToLower(err.Error()) - return strings.Contains(msg, "duplicate") -} - -// parseFilename extracts (height, hashPrefix) from the ckpool-written -// filename `-.hex`. Returns zeros if the format -// doesn't match — the caller should still attempt the submit; we just -// can't do the "already on chain" optimization. -func parseFilename(name string) (int64, string) { - base := strings.TrimSuffix(name, ".hex") - dash := strings.IndexByte(base, '-') - if dash <= 0 { - return 0, "" - } - height, err := parseInt64(base[:dash]) - if err != nil { - return 0, base[dash+1:] - } - return height, base[dash+1:] -} - -func parseInt64(s string) (int64, error) { - var n int64 - for _, c := range s { - if c < '0' || c > '9' { - return 0, fmt.Errorf("not a number: %q", s) - } - n = n*10 + int64(c-'0') - } - return n, nil -} diff --git a/api/internal/config/config.go b/api/internal/config/config.go index afa9fa7..769049a 100644 --- a/api/internal/config/config.go +++ b/api/internal/config/config.go @@ -39,27 +39,6 @@ type Config struct { // the UI uses the public mempool.space; non-empty means the user // has pointed Kamado at their own instance via the StartOS config. MempoolBaseURL string - - // Directory ckpool's patched local_block_submit writes raw block - // hex into. Empty disables the fallback submitter. - PendingBlocksDir string - - // Comma- or newline-separated list of fallback bitcoind RPC URLs - // that the submitter will try in order if a block stays pending - // past the grace window. Each entry can encode credentials inline: - // https://user:pass@host:8332/ - // or without (e.g. an unauthenticated localhost backup node). - BackupRPCURLs string - - // How long a pending block file must persist before the submitter - // will try fallback RPCs. Defaults to 3s — short enough that a - // failed submit is recovered while the work is still relevant, - // long enough that ckpool's bounded internal wait (~3s for a live - // primary server) and a single slow round-trip don't trigger a - // spurious fallback. Patch 0004 makes ckpool's submit return false - // rather than spinning indefinitely, so we no longer need to wait - // for that case to time out. - PendingBlocksGrace time.Duration } func FromEnv() (*Config, error) { @@ -75,10 +54,6 @@ func FromEnv() (*Config, error) { DBPath: getenv("DB_PATH", "/var/lib/kamado/kamado.db"), PollInterval: getenvDuration("POLL_INTERVAL", 5*time.Second), MempoolBaseURL: os.Getenv("MEMPOOL_BASE_URL"), - - PendingBlocksDir: os.Getenv("PENDING_BLOCKS_DIR"), - BackupRPCURLs: os.Getenv("BACKUP_RPC_URLS"), - PendingBlocksGrace: getenvDuration("PENDING_BLOCKS_GRACE", 3*time.Second), } if cfg.BitcoinRPCURL == "" { diff --git a/api/internal/httpapi/server.go b/api/internal/httpapi/server.go index 999f335..32fd23d 100644 --- a/api/internal/httpapi/server.go +++ b/api/internal/httpapi/server.go @@ -112,20 +112,17 @@ func (s *Server) health(w http.ResponseWriter, r *http.Request) { status = http.StatusServiceUnavailable } writeJSON(w, status, map[string]any{ - "ok": overallOK, - "ckpool": snap.CKPoolOK, - "bitcoin": snap.BitcoinOK, - "submit_attempts": snap.BlockSubmitAttempts, - "submits_confirmed": snap.BlockSubmitsConfirmed, - "submit_gap": submitGap, - "fallback_submits_total": snap.FallbackSubmitsTotal, - "last_fallback_submit_at": snap.LastFallbackSubmitAt, - "last_fallback_via": snap.LastFallbackVia, - "zmq_enabled": snap.ZMQEnabled, - "zmq_event_age_seconds": snap.LastZMQEventAge, - "zmq_has_event": snap.HasLastZMQEvent, - "zmq_stale": zmqStale, - "last_error": snap.LastError, + "ok": overallOK, + "ckpool": snap.CKPoolOK, + "bitcoin": snap.BitcoinOK, + "submit_attempts": snap.BlockSubmitAttempts, + "submits_confirmed": snap.BlockSubmitsConfirmed, + "submit_gap": submitGap, + "zmq_enabled": snap.ZMQEnabled, + "zmq_event_age_seconds": snap.LastZMQEventAge, + "zmq_has_event": snap.HasLastZMQEvent, + "zmq_stale": zmqStale, + "last_error": snap.LastError, }) } diff --git a/api/internal/state/aggregator.go b/api/internal/state/aggregator.go index 52a7e3e..216353b 100644 --- a/api/internal/state/aggregator.go +++ b/api/internal/state/aggregator.go @@ -80,14 +80,6 @@ type Snapshot struct { BlockSubmitAttempts int64 `json:"block_submit_attempts"` BlockSubmitsConfirmed int64 `json:"block_submits_confirmed"` - // Number of blocks the fallback submitter had to broadcast via a - // backup RPC URL because the primary bitcoind didn't accept them - // in time. Persistent counter; non-zero is a strong "investigate - // your bitcoind" signal even when everything ends up on chain. - FallbackSubmitsTotal int64 `json:"fallback_submits_total"` - LastFallbackSubmitAt int64 `json:"last_fallback_submit_at,omitempty"` // unix seconds - LastFallbackVia string `json:"last_fallback_via,omitempty"` - // Health diagnostics for /healthz and the UI status badge. // LastZMQEventAge is the seconds-since the last bitcoind hashblock // frame arrived; -1 means no event seen since startup. ZMQEnabled @@ -114,7 +106,6 @@ const ( kvSubmitAttempts = "block_submit_attempts" kvSubmitsConfirmed = "block_submits_confirmed" - kvFallbackSubmits = "fallback_submits_total" // reconcileInterval is how often we sweep recent blocks looking // for missing hash/reward enrichment and reorg-orphaned hashes. @@ -197,13 +188,6 @@ type Aggregator struct { blockSubmitsConfirmed int64 lastSubmitCountSave time.Time - // Fallback-submitter telemetry. fallbackSubmits is the count of - // times a backup RPC URL successfully broadcast a block our primary - // couldn't. Persisted; survives restarts. - fallbackSubmits int64 - lastFallbackSubmitAt time.Time - lastFallbackVia string - // ZMQ diagnostics: timestamp of the last hashblock frame relayed // from zmqmon. Used by /healthz to flag stale subscriptions. zmqEnabled bool @@ -265,26 +249,6 @@ func (a *Aggregator) Run(ctx context.Context, tipEvents <-chan zmqmon.TipEvent) } } -// RecordFallbackSubmit is called by blocksubmit.Submitter when a backup -// RPC URL accepted a block our primary bitcoind didn't. Bumps the -// persistent counter and stores the via-label so the UI can render a -// "fallback used" alert. -func (a *Aggregator) RecordFallbackSubmit(height int64, via string) { - a.mu.Lock() - a.fallbackSubmits++ - a.lastFallbackSubmitAt = time.Now() - a.lastFallbackVia = via - n := a.fallbackSubmits - a.mu.Unlock() - a.Log.Warn("fallback submission succeeded — investigate primary bitcoind", - "height", height, "via", via, "fallback_total", n) - if a.Store != nil { - if err := a.Store.SetKV(kvFallbackSubmits, strconv.FormatInt(n, 10)); err != nil { - a.Log.Warn("fallback counter persist failed", "err", err) - } - } -} - // Snapshot returns a copy of the current snapshot. func (a *Aggregator) Snapshot() Snapshot { a.mu.RLock() @@ -486,11 +450,6 @@ func (a *Aggregator) refresh(ctx context.Context) { } next.BlockSubmitAttempts = a.blockSubmitAttempts next.BlockSubmitsConfirmed = a.blockSubmitsConfirmed - next.FallbackSubmitsTotal = a.fallbackSubmits - if !a.lastFallbackSubmitAt.IsZero() { - next.LastFallbackSubmitAt = a.lastFallbackSubmitAt.Unix() - next.LastFallbackVia = a.lastFallbackVia - } next.ZMQEnabled = a.zmqEnabled if !a.lastZMQEventTime.IsZero() { next.LastZMQEventAge = time.Since(a.lastZMQEventTime).Seconds() @@ -540,13 +499,6 @@ func (a *Aggregator) loadPersistedState() { a.mu.Unlock() } } - if v, err := a.Store.GetKV(kvFallbackSubmits); err == nil && v != "" { - if n, perr := strconv.ParseInt(v, 10, 64); perr == nil { - a.mu.Lock() - a.fallbackSubmits = n - a.mu.Unlock() - } - } cutoff := time.Now().Add(-24 * time.Hour).Unix() if samples, err := a.Store.HashrateSince(cutoff); err != nil { diff --git a/ckpool/patches/0004-dump-pending-block-for-fallback.patch b/ckpool/patches/0004-dump-pending-block-for-fallback.patch deleted file mode 100644 index e4944b6..0000000 --- a/ckpool/patches/0004-dump-pending-block-for-fallback.patch +++ /dev/null @@ -1,125 +0,0 @@ -diff --git a/src/generator.c b/src/generator.c -index 22e2e08..438b199 100644 ---- a/src/generator.c -+++ b/src/generator.c -@@ -351,11 +351,25 @@ bool generator_submitblock(ckpool_t *ckp, const char *buf) - server_instance_t *si; - bool warn = false; - connsock_t *cs; -+ /* Bound the wait for current_si so a permanently-down primary -+ * bitcoind does not pin the caller forever. After this many 10ms -+ * sleeps (~3s), return false so the caller (in our fork: -+ * local_block_submit) can hand off to kamado-api's fallback -+ * broadcaster instead of blocking the stratifier on a dead RPC. -+ * Original upstream behavior was to spin indefinitely; we trade -+ * that for predictable latency. */ -+ const int max_no_si_iters = 300; -+ int no_si_iters = 0; - - while (unlikely(!(si = gdata->current_si))) { - if (!warn) -- LOGWARNING("No live current server in generator_blocksubmit! Resubmitting indefinitely!"); -+ LOGWARNING("No live current server in generator_blocksubmit! Waiting up to ~3s before giving up so fallback can take over..."); - warn = true; -+ if (++no_si_iters > max_no_si_iters) { -+ LOGWARNING("generator_submitblock: no live primary server after %d ms, returning false", -+ no_si_iters * 10); -+ return false; -+ } - cksleep_ms(10); - } - cs = &si->cs; -diff --git a/src/stratifier.c b/src/stratifier.c -index 52da790..7caf179 100644 ---- a/src/stratifier.c -+++ b/src/stratifier.c -@@ -2069,16 +2069,75 @@ process_block(const workbase_t *wb, const char *coinbase, const int cblen, - return gbt_block; - } - --/* Submit block data locally, absorbing and freeing gbt_block */ -+/* Write the raw block hex to a sidecar file under /pending-blocks/ -+ * so the external watcher (kamado-api) can re-broadcast via fallback RPC -+ * nodes. Best-effort: any error here is logged at INFO and never blocks -+ * the caller. Called only after the primary submit fails so the happy -+ * path stays disk-free. */ -+static void kamado_dump_pending_block(ckpool_t *ckp, const char *gbt_block, -+ const char *rhash, int height, -+ char *out_path, size_t out_path_len) -+{ -+ char dir[512] = {}; -+ int fd; -+ size_t blen; -+ ssize_t w; -+ -+ out_path[0] = '\0'; -+ if (!ckp || !ckp->logdir || !gbt_block) -+ return; -+ snprintf(dir, sizeof(dir), "%spending-blocks", ckp->logdir); -+ if (mkdir(dir, 0750) < 0 && errno != EEXIST) { -+ LOGINFO("kamado: mkdir %s failed: %s", dir, strerror(errno)); -+ return; -+ } -+ snprintf(out_path, out_path_len, "%s/%d-%.16s.hex", dir, height, rhash); -+ fd = open(out_path, O_WRONLY | O_CREAT | O_TRUNC, 0640); -+ if (fd < 0) { -+ LOGINFO("kamado: open %s failed: %s", out_path, strerror(errno)); -+ out_path[0] = '\0'; -+ return; -+ } -+ blen = strlen(gbt_block); -+ w = write(fd, gbt_block, blen); -+ close(fd); -+ if (w != (ssize_t)blen) { -+ LOGINFO("kamado: short write to %s (%zd/%zu)", out_path, w, blen); -+ unlink(out_path); -+ out_path[0] = '\0'; -+ return; -+ } -+ LOGNOTICE("kamado: dumped pending block height %d to %s (%zu bytes) for fallback broadcast", -+ height, out_path, blen); -+} -+ -+/* Submit block data locally, absorbing and freeing gbt_block. -+ * -+ * Order matters: we always try ckp's primary bitcoind FIRST so the -+ * happy path adds zero latency or disk I/O. Only when the primary -+ * rejects (or generator_submitblock returns false for any other -+ * reason) do we dump the raw hex to /pending-blocks/ so the -+ * kamado-api watcher can re-broadcast via fallback RPC nodes. */ - static bool local_block_submit(ckpool_t *ckp, char *gbt_block, const uchar *flip32, int height) - { -- bool ret = generator_submitblock(ckp, gbt_block); - char heighthash[68] = {}, rhash[68] = {}; -+ char pending_path[512] = {}; - uchar swap256[32]; -+ bool ret; - -- free(gbt_block); - swap_256(swap256, flip32); - __bin2hex(rhash, swap256, 32); -+ -+ /* Primary submit first — this is the latency-critical path. */ -+ ret = generator_submitblock(ckp, gbt_block); -+ -+ /* Only touch disk if the primary didn't accept. */ -+ if (!ret) { -+ kamado_dump_pending_block(ckp, gbt_block, rhash, height, -+ pending_path, sizeof(pending_path)); -+ } -+ -+ free(gbt_block); - generator_preciousblock(ckp, rhash); - - /* Check failures that may be inconclusive but were submitted via other -@@ -2099,6 +2158,10 @@ static bool local_block_submit(ckpool_t *ckp, char *gbt_block, const uchar *flip - height, ret ? "ACCEPTED" : "REJECTED"); - } - } -+ /* Block ended up on chain (either initial submit or precious-block -+ * recovery): the dump file is no longer needed. */ -+ if (ret && pending_path[0]) -+ unlink(pending_path); - return ret; - } - diff --git a/ckpool/patches/README.md b/ckpool/patches/README.md index 231af00..15b00d9 100644 --- a/ckpool/patches/README.md +++ b/ckpool/patches/README.md @@ -9,7 +9,7 @@ Patches are applied in alphabetical order by filename. Use a numeric prefix to e ## Current state -Four Kamado patches are applied on top of the pinned upstream commit, in +Three Kamado patches are applied on top of the pinned upstream commit, in alphabetical order: | Patch | What it does | @@ -17,7 +17,6 @@ alphabetical order: | `0001-expose-bestever-in-runtime-json.patch` | Adds `bestever` field to the `users` / `workers` runtime socket JSON | | `0002-enable-socket-api-responses.patch` | Always reply on the listener socket so kamado-api gets responses even with `btcsolo: true` | | `0003-share-error-as-stratum-array.patch` | Maps `share_err` to Stratum spec error codes; emits `[code, msg, null]` per Slush | -| `0004-dump-pending-block-for-fallback.patch` | Writes the raw block hex to `/pending-blocks/` before submit; unlinks on success | ### Why 0001 matters @@ -34,26 +33,6 @@ This patch adds `bestever` to the runtime JSON so the UI can show both "this round" and "all-time" best share side by side. No behavioral change to share validation or block handling. Candidate for upstreaming. -### Why 0004 matters - -Block submission to bitcoind is the most revenue-critical RPC call ckpool -makes. If bitcoind is unreachable when a share meets network difficulty, -ckpool's `generator` thread retries indefinitely against the same single -endpoint — and the raw block data lives only in stratifier memory, so a -ckpool crash before bitcoind comes back permanently loses the block. - -This patch hooks `local_block_submit` to write the raw block hex to -`/pending-blocks/-.hex` *before* invoking -`generator_submitblock`, and unlinks the file on success. `kamado-api` -runs a watcher over that directory: if a file persists past a grace -period (default 30 s), it submits the block via fallback RPC URLs the -operator has configured. Multiple fallbacks are tried in sequence; the -file is unlinked when any fallback returns success or "duplicate" -(meaning the block already landed). - -This is a Kamado-specific integration hook — almost certainly not -upstreamable, but minimal-impact on existing ckpool behavior. - Beyond this patch, the pinned upstream commit (`cfb0f83b`, tagged as version 1.0) already includes every fix that Bassin issue #29 asked to backport, plus several improvements: diff --git a/ui/src/lib/HealthBanners.svelte b/ui/src/lib/HealthBanners.svelte index c8cc14f..0952fbf 100644 --- a/ui/src/lib/HealthBanners.svelte +++ b/ui/src/lib/HealthBanners.svelte @@ -4,20 +4,15 @@ // submit_gap > 0 means ckpool tried to submit at least one block that // never got the "Solved and confirmed" follow-up — so either bitcoind - // rejected the submission or the RPC dropped. The number stays > 0 - // forever after such an event (these are persistent counters), so we - // only show it as a banner when the fallback hasn't covered for it - // OR there's been a recent fallback the operator should investigate. + // rejected the submission or the RPC dropped. These are persistent + // counters so the banner stays visible until the operator clears it + // by inspecting bitcoind's logs. const submitGap = $derived.by(() => { const d = snap.data; if (!d) return 0; return Math.max(0, (d.block_submit_attempts ?? 0) - (d.block_submits_confirmed ?? 0)); }); - const fallbackCount = $derived(snap.data?.fallback_submits_total ?? 0); - const lastFallbackAt = $derived(snap.data?.last_fallback_submit_at ?? 0); - const lastFallbackVia = $derived(snap.data?.last_fallback_via ?? ""); - // ZMQ stale = configured but no event in 30+ minutes. Bitcoin's avg // block interval is 10 min; 30 min covers normal variance without // false-alarming on quiet stretches. @@ -26,41 +21,18 @@ if (!d || !d.zmq_enabled || !d.has_last_zmq_event) return false; return (d.last_zmq_event_age ?? 0) > 1800; }); - - // Show the fallback banner for 24h after the most recent fallback - // event so the operator sees the alert during their next check-in - // even if the underlying problem auto-resolved. - const fallbackRecent = $derived.by(() => { - if (!lastFallbackAt) return false; - const ageSec = Date.now() / 1000 - lastFallbackAt; - return ageSec >= 0 && ageSec < 86400; - }); -{#if submitGap > 0 || fallbackRecent || zmqStale} +{#if submitGap > 0 || zmqStale}
- {#if fallbackRecent} - - {/if} - {#if submitGap > 0 && !fallbackRecent} + {#if submitGap > 0} {/if} @@ -94,11 +66,6 @@ border: 1px solid transparent; line-height: 1.4; } - .banner.alert { - background: rgba(220, 80, 80, 0.10); - border-color: rgba(220, 80, 80, 0.40); - color: rgb(230, 130, 130); - } .banner.warn { background: rgba(220, 170, 60, 0.10); border-color: rgba(220, 170, 60, 0.40); diff --git a/ui/src/types.ts b/ui/src/types.ts index bd6e8f2..79ba898 100644 --- a/ui/src/types.ts +++ b/ui/src/types.ts @@ -132,11 +132,6 @@ export type Snapshot = { // submissions are being rejected by bitcoind. block_submit_attempts: number; block_submits_confirmed: number; - // Fallback submitter telemetry: how often we had to broadcast via a - // backup RPC URL because the primary bitcoind didn't accept in time. - fallback_submits_total: number; - last_fallback_submit_at?: number; - last_fallback_via?: string; // ZMQ subscriber freshness diagnostics. zmq_enabled: boolean; has_last_zmq_event: boolean;