satoshi/KamadoPool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert block-broadcast fallback path

Browse Source 
Removes the entire fallback submitter mechanism: ckpool patch 0004,
the blocksubmit package, the wiring in main.go, the config fields,
the aggregator's fallback counters and snapshot fields, the healthz
fallback fields, and the TS type fields plus the HealthBanners
fallback alert.

Reasoning: ckpool's primary bitcoind submission must remain the
single source of truth, and getting the parallel "race a fallback
during the submit" semantics right is more architectural complexity
than the marginal reliability gain justifies. The original upstream
behavior — submit to bitcoind, retry indefinitely if unavailable —
is what we want.

Kept intact:
  * Submit-attempt vs confirmed counters (block_submit_attempts /
    block_submits_confirmed). Useful on their own as a "did bitcoind
    confirm the submission?" signal.
  * HealthBanners shows submit_gap and zmq_stale only.
  * /healthz exposes submit_gap, zmq_stale, etc.
  * All P0 reliability work (tailer cursor, reconcile loop, reorg
    detection, multi-solve guard) and other P1 (RPC retry, WS
    back-pressure, ZMQ tracking, startup readiness gate).
This commit is contained in:
satoshi
2026-04-28 02:07:17 +03:00
parent 99302cf4af
commit 1dcf087842
9 changed files with 20 additions and 618 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/cmd/kamado-api/main.go
-63
View File
@@ -11,7 +11,6 @@ import (
"fmt" "fmt"
"log/slog" "log/slog"
"net/http" "net/http"
"net/url"
"os" "os"
"os/signal" "os/signal"
"strconv" "strconv"
@@ -20,7 +19,6 @@ import (
"time" "time"
"github.com/kamadopool/kamado-api/internal/bitcoind" "github.com/kamadopool/kamado-api/internal/bitcoind"
"github.com/kamadopool/kamado-api/internal/blocksubmit"
"github.com/kamadopool/kamado-api/internal/ckpool" "github.com/kamadopool/kamado-api/internal/ckpool"
"github.com/kamadopool/kamado-api/internal/config" "github.com/kamadopool/kamado-api/internal/config"
"github.com/kamadopool/kamado-api/internal/httpapi" "github.com/kamadopool/kamado-api/internal/httpapi"
@@ -118,29 +116,6 @@ func main() {
go agg.IngestBlockEvents(ctx, tailer.Events) go agg.IngestBlockEvents(ctx, tailer.Events)
go agg.IngestAttemptEvents(ctx, tailer.Attempts) go agg.IngestAttemptEvents(ctx, tailer.Attempts)
// Fallback block submitter: if ckpool's primary bitcoind doesn't
// accept a block, the patched local_block_submit leaves the raw
// hex sitting in PENDING_BLOCKS_DIR. This watcher re-broadcasts it
// via the operator-configured BACKUP_RPC_URLS list.
if cfg.PendingBlocksDir != "" {
fallbacks := parseBackupRPCs(cfg.BackupRPCURLs, log)
sub := &blocksubmit.Submitter{
Dir: cfg.PendingBlocksDir,
Grace: cfg.PendingBlocksGrace,
Primary: rpc,
Fallbacks: fallbacks,
Log: log,
OnSuccess: func(height int64, viaURL, viaLabel string) {
agg.RecordFallbackSubmit(height, viaLabel)
},
}
// Sweep every second so we react within ~grace+1s of a
// failed submit. Reading an empty directory is cheap.
go sub.Run(ctx, 1*time.Second)
} else {
log.Info("blocksubmit fallback disabled (PENDING_BLOCKS_DIR not set)")
}
// Wait briefly for the aggregator's first refresh to complete so // Wait briefly for the aggregator's first refresh to complete so
// the very first /api/snapshot or /api/health hit doesn't see an // the very first /api/snapshot or /api/health hit doesn't see an
// all-zeros snapshot and report bitcoin_ok=false during its own // all-zeros snapshot and report bitcoin_ok=false during its own
@@ -177,41 +152,3 @@ func main() {
os.Exit(1) os.Exit(1)
} }
} }
// parseBackupRPCs splits BACKUP_RPC_URLS (newline- or comma-separated)
// into FallbackTargets. Credentials are accepted inline as
// https://user:pass@host:port/. Lines beginning with '#' are comments.
// Whitespace and empty lines are ignored.
func parseBackupRPCs(raw string, log *slog.Logger) []blocksubmit.FallbackTarget {
if raw == "" {
return nil
}
separated := strings.NewReplacer(",", "\n").Replace(raw)
var out []blocksubmit.FallbackTarget
for _, line := range strings.Split(separated, "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
u, err := url.Parse(line)
if err != nil {
log.Warn("blocksubmit: invalid fallback URL, skipping", "raw", line, "err", err)
continue
}
var user, pass string
if u.User != nil {
user = u.User.Username()
pass, _ = u.User.Password()
u.User = nil
}
clean := u.String()
out = append(out, blocksubmit.FallbackTarget{
URL: clean,
User: user,
Password: pass,
Label: u.Host,
})
log.Info("blocksubmit fallback registered", "url", clean, "host", u.Host, "auth", user != "")
}
return out
}
api/internal/blocksubmit/submit.go
-275
View File
@@ -1,275 +0,0 @@
// Package blocksubmit watches a sidecar directory that ckpool fills
// with raw block hex right before it tries to submit to its primary
// bitcoind, and re-submits via operator-configured fallback RPC URLs
// if a file persists longer than the grace period.
//
// ckpool's local_block_submit (patched in 0004) writes
// <logdir>/pending-blocks/<height>-<rhash>.hex before calling
// generator_submitblock, and unlinks on success. So the rule is
// simple: any file still present after `grace` has likely failed to
// land, and we should help.
//
// Failure modes this protects against:
// - bitcoind crashed / OOM-killed at the moment of submission
// - bitcoind RPC saturated, returning timeouts
// - upstream peering issue making bitcoind unable to relay
// - ckpool itself crashes after the file was written but before unlink
//
// Each fallback URL is tried in order. submitblock returns either null
// (accepted), a "duplicate*" reason (already on chain — same outcome),
// or a hard error. We treat both null and any "duplicate" prefix as
// success and unlink.
package blocksubmit
import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"os"
"path/filepath"
"strings"
"time"
"github.com/kamadopool/kamado-api/internal/bitcoind"
)
// FallbackTarget is one back-up RPC endpoint we'll try if the primary
// bitcoind didn't accept a block. URL must be a full http(s):// endpoint;
// User and Password are optional (some self-hosted nodes accept
// unauthenticated localhost RPC, public RPC services usually don't).
type FallbackTarget struct {
URL string
User string
Password string
Label string // human-friendly name for logs; defaults to host
}
// Submitter polls a pending-blocks directory and re-broadcasts blocks
// that have been sitting too long to its primary bitcoind plus any
// configured fallbacks. Stateless across iterations — the filesystem
// is the queue.
type Submitter struct {
Dir string
Grace time.Duration // how long we wait before considering a file stale
MaxAge time.Duration // older than this: log error and remove (operator intervention required)
Primary *bitcoind.RPC // primary bitcoind for "is the block already on chain?" sanity check
Fallbacks []FallbackTarget
Log *slog.Logger
// OnSuccess, if set, is called with the height and the URL string
// that accepted the block, so the aggregator can bump a counter or
// emit a UI banner.
OnSuccess func(height int64, viaURL string, viaLabel string)
}
// Run blocks until ctx is cancelled, sweeping the dir every poll.
func (s *Submitter) Run(ctx context.Context, poll time.Duration) {
if s.Dir == "" {
s.Log.Info("blocksubmit disabled (no pending-blocks dir)")
return
}
if s.Grace == 0 {
// Match the config default — short enough that a failed
// submit gets recovered while the work is still relevant,
// long enough that we don't spuriously fallback while the
// primary's RPC is mid-handshake.
s.Grace = 3 * time.Second
}
if s.MaxAge == 0 {
s.MaxAge = 24 * time.Hour
}
if poll == 0 {
poll = 5 * time.Second
}
// Ensure the directory exists so the first sweep doesn't WARN. Don't
// fail if we can't create it — ckpool may not have run yet, or perms
// may need a moment to settle. The sweep itself logs at DEBUG when
// the dir is missing.
_ = os.MkdirAll(s.Dir, 0750)
t := time.NewTicker(poll)
defer t.Stop()
s.Log.Info("blocksubmit started", "dir", s.Dir, "grace", s.Grace, "fallbacks", len(s.Fallbacks))
for {
select {
case <-ctx.Done():
return
case <-t.C:
s.sweep(ctx)
}
}
}
func (s *Submitter) sweep(ctx context.Context) {
entries, err := os.ReadDir(s.Dir)
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
s.Log.Debug("blocksubmit: readdir failed", "dir", s.Dir, "err", err)
}
return
}
now := time.Now()
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".hex") {
continue
}
path := filepath.Join(s.Dir, e.Name())
info, err := e.Info()
if err != nil {
continue
}
age := now.Sub(info.ModTime())
if age < s.Grace {
continue
}
if age > s.MaxAge {
s.Log.Error("blocksubmit: pending block too old, abandoning",
"path", path, "age", age)
_ = os.Remove(path)
continue
}
s.handleFile(ctx, path, e.Name(), age)
}
}
func (s *Submitter) handleFile(ctx context.Context, path, name string, age time.Duration) {
height, hashPrefix := parseFilename(name)
logger := s.Log.With("path", path, "height", height, "hash_prefix", hashPrefix, "age", age)
// First check: maybe the primary bitcoind quietly accepted it but
// ckpool failed to unlink (process crash between submit and unlink).
// If our primary already has a block at this height, we're done —
// no need to involve fallbacks.
if height > 0 && s.Primary != nil {
lookupCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
hash, err := s.Primary.GetBlockHash(lookupCtx, height)
cancel()
if err == nil && hash != "" {
logger.Info("blocksubmit: block already on primary chain, removing stale pending file",
"canonical_hash", hash)
_ = os.Remove(path)
return
}
}
hex, err := os.ReadFile(path)
if err != nil {
logger.Warn("blocksubmit: read failed", "err", err)
return
}
hexStr := strings.TrimSpace(string(hex))
if hexStr == "" {
logger.Warn("blocksubmit: empty pending file, removing")
_ = os.Remove(path)
return
}
if len(s.Fallbacks) == 0 {
// No fallbacks configured: surface that the block is still
// pending and let the operator deal with it. Don't spam — log
// once per minute via a coarse round of `age`.
if int(age.Minutes())%1 == 0 {
logger.Warn("blocksubmit: pending block has no fallback RPCs configured")
}
return
}
for _, fb := range s.Fallbacks {
label := fb.Label
if label == "" {
label = fb.URL
}
submitCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
err := submitOne(submitCtx, fb, hexStr)
cancel()
if err == nil {
logger.Warn("blocksubmit: fallback accepted block", "via", label)
_ = os.Remove(path)
if s.OnSuccess != nil {
s.OnSuccess(height, fb.URL, label)
}
return
}
if isDuplicate(err) {
// Block already on chain at the fallback (it was relayed
// via P2P from somewhere else). Same outcome — done.
logger.Info("blocksubmit: fallback reports duplicate (block on chain), removing pending",
"via", label, "err", err)
_ = os.Remove(path)
if s.OnSuccess != nil {
s.OnSuccess(height, fb.URL, label)
}
return
}
logger.Warn("blocksubmit: fallback rejected", "via", label, "err", err)
}
logger.Error("blocksubmit: ALL fallbacks failed; block remains pending for next sweep",
"fallbacks_tried", len(s.Fallbacks))
}
// submitOne sends submitblock to one fallback RPC and returns nil on
// acceptance, a wrapped error otherwise.
func submitOne(ctx context.Context, fb FallbackTarget, hexBlock string) error {
rpc := bitcoind.NewRPC(fb.URL, fb.User, fb.Password, 30*time.Second)
// submitblock returns: null on success, or a JSON string with the
// reject reason. We have to be a bit clever with json.RawMessage to
// distinguish those.
var raw json.RawMessage
if err := rpc.Call(ctx, "submitblock", []any{hexBlock}, &raw); err != nil {
return fmt.Errorf("submitblock rpc: %w", err)
}
// Trim whitespace from the raw bytes for comparison.
body := strings.TrimSpace(string(raw))
if body == "" || body == "null" {
return nil
}
// Strip surrounding quotes from the JSON string.
if len(body) >= 2 && body[0] == '"' && body[len(body)-1] == '"' {
body = body[1 : len(body)-1]
}
return errors.New(body)
}
// isDuplicate covers Bitcoin Core's submitblock reject reasons that
// mean "this block is already in the chain": "duplicate",
// "duplicate-invalid", "duplicate-inconclusive". From the operator's
// perspective these are all "we don't need to keep trying".
func isDuplicate(err error) bool {
if err == nil {
return false
}
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "duplicate")
}
// parseFilename extracts (height, hashPrefix) from the ckpool-written
// filename `<height>-<hashprefix16>.hex`. Returns zeros if the format
// doesn't match — the caller should still attempt the submit; we just
// can't do the "already on chain" optimization.
func parseFilename(name string) (int64, string) {
base := strings.TrimSuffix(name, ".hex")
dash := strings.IndexByte(base, '-')
if dash <= 0 {
return 0, ""
}
height, err := parseInt64(base[:dash])
if err != nil {
return 0, base[dash+1:]
}
return height, base[dash+1:]
}
func parseInt64(s string) (int64, error) {
var n int64
for _, c := range s {
if c < '0' || c > '9' {
return 0, fmt.Errorf("not a number: %q", s)
}
n = n*10 + int64(c-'0')
}
return n, nil
}
api/internal/config/config.go
-25
View File
@@ -39,27 +39,6 @@ type Config struct {
// the UI uses the public mempool.space; non-empty means the user // the UI uses the public mempool.space; non-empty means the user
// has pointed Kamado at their own instance via the StartOS config. // has pointed Kamado at their own instance via the StartOS config.
MempoolBaseURL string MempoolBaseURL string
// Directory ckpool's patched local_block_submit writes raw block
// hex into. Empty disables the fallback submitter.
PendingBlocksDir string
// Comma- or newline-separated list of fallback bitcoind RPC URLs
// that the submitter will try in order if a block stays pending
// past the grace window. Each entry can encode credentials inline:
// https://user:pass@host:8332/
// or without (e.g. an unauthenticated localhost backup node).
BackupRPCURLs string
// How long a pending block file must persist before the submitter
// will try fallback RPCs. Defaults to 3s — short enough that a
// failed submit is recovered while the work is still relevant,
// long enough that ckpool's bounded internal wait (~3s for a live
// primary server) and a single slow round-trip don't trigger a
// spurious fallback. Patch 0004 makes ckpool's submit return false
// rather than spinning indefinitely, so we no longer need to wait
// for that case to time out.
PendingBlocksGrace time.Duration
} }
func FromEnv() (*Config, error) { func FromEnv() (*Config, error) {
@@ -75,10 +54,6 @@ func FromEnv() (*Config, error) {
DBPath: getenv("DB_PATH", "/var/lib/kamado/kamado.db"), DBPath: getenv("DB_PATH", "/var/lib/kamado/kamado.db"),
PollInterval: getenvDuration("POLL_INTERVAL", 5*time.Second), PollInterval: getenvDuration("POLL_INTERVAL", 5*time.Second),
MempoolBaseURL: os.Getenv("MEMPOOL_BASE_URL"), MempoolBaseURL: os.Getenv("MEMPOOL_BASE_URL"),
PendingBlocksDir: os.Getenv("PENDING_BLOCKS_DIR"),
BackupRPCURLs: os.Getenv("BACKUP_RPC_URLS"),
PendingBlocksGrace: getenvDuration("PENDING_BLOCKS_GRACE", 3*time.Second),
} }
if cfg.BitcoinRPCURL == "" { if cfg.BitcoinRPCURL == "" {
api/internal/httpapi/server.go
-3
View File
@@ -118,9 +118,6 @@ func (s *Server) health(w http.ResponseWriter, r *http.Request) {
"submit_attempts": snap.BlockSubmitAttempts, "submit_attempts": snap.BlockSubmitAttempts,
"submits_confirmed": snap.BlockSubmitsConfirmed, "submits_confirmed": snap.BlockSubmitsConfirmed,
"submit_gap": submitGap, "submit_gap": submitGap,
"fallback_submits_total": snap.FallbackSubmitsTotal,
"last_fallback_submit_at": snap.LastFallbackSubmitAt,
"last_fallback_via": snap.LastFallbackVia,
"zmq_enabled": snap.ZMQEnabled, "zmq_enabled": snap.ZMQEnabled,
"zmq_event_age_seconds": snap.LastZMQEventAge, "zmq_event_age_seconds": snap.LastZMQEventAge,
"zmq_has_event": snap.HasLastZMQEvent, "zmq_has_event": snap.HasLastZMQEvent,
api/internal/state/aggregator.go
-48
View File
@@ -80,14 +80,6 @@ type Snapshot struct {
BlockSubmitAttempts int64 `json:"block_submit_attempts"` BlockSubmitAttempts int64 `json:"block_submit_attempts"`
BlockSubmitsConfirmed int64 `json:"block_submits_confirmed"` BlockSubmitsConfirmed int64 `json:"block_submits_confirmed"`
// Number of blocks the fallback submitter had to broadcast via a
// backup RPC URL because the primary bitcoind didn't accept them
// in time. Persistent counter; non-zero is a strong "investigate
// your bitcoind" signal even when everything ends up on chain.
FallbackSubmitsTotal int64 `json:"fallback_submits_total"`
LastFallbackSubmitAt int64 `json:"last_fallback_submit_at,omitempty"` // unix seconds
LastFallbackVia string `json:"last_fallback_via,omitempty"`
// Health diagnostics for /healthz and the UI status badge. // Health diagnostics for /healthz and the UI status badge.
// LastZMQEventAge is the seconds-since the last bitcoind hashblock // LastZMQEventAge is the seconds-since the last bitcoind hashblock
// frame arrived; -1 means no event seen since startup. ZMQEnabled // frame arrived; -1 means no event seen since startup. ZMQEnabled
@@ -114,7 +106,6 @@ const (
kvSubmitAttempts = "block_submit_attempts" kvSubmitAttempts = "block_submit_attempts"
kvSubmitsConfirmed = "block_submits_confirmed" kvSubmitsConfirmed = "block_submits_confirmed"
kvFallbackSubmits = "fallback_submits_total"
// reconcileInterval is how often we sweep recent blocks looking // reconcileInterval is how often we sweep recent blocks looking
// for missing hash/reward enrichment and reorg-orphaned hashes. // for missing hash/reward enrichment and reorg-orphaned hashes.
@@ -197,13 +188,6 @@ type Aggregator struct {
blockSubmitsConfirmed int64 blockSubmitsConfirmed int64
lastSubmitCountSave time.Time lastSubmitCountSave time.Time
// Fallback-submitter telemetry. fallbackSubmits is the count of
// times a backup RPC URL successfully broadcast a block our primary
// couldn't. Persisted; survives restarts.
fallbackSubmits int64
lastFallbackSubmitAt time.Time
lastFallbackVia string
// ZMQ diagnostics: timestamp of the last hashblock frame relayed // ZMQ diagnostics: timestamp of the last hashblock frame relayed
// from zmqmon. Used by /healthz to flag stale subscriptions. // from zmqmon. Used by /healthz to flag stale subscriptions.
zmqEnabled bool zmqEnabled bool
@@ -265,26 +249,6 @@ func (a *Aggregator) Run(ctx context.Context, tipEvents <-chan zmqmon.TipEvent)
} }
} }
// RecordFallbackSubmit is called by blocksubmit.Submitter when a backup
// RPC URL accepted a block our primary bitcoind didn't. Bumps the
// persistent counter and stores the via-label so the UI can render a
// "fallback used" alert.
func (a *Aggregator) RecordFallbackSubmit(height int64, via string) {
a.mu.Lock()
a.fallbackSubmits++
a.lastFallbackSubmitAt = time.Now()
a.lastFallbackVia = via
n := a.fallbackSubmits
a.mu.Unlock()
a.Log.Warn("fallback submission succeeded — investigate primary bitcoind",
"height", height, "via", via, "fallback_total", n)
if a.Store != nil {
if err := a.Store.SetKV(kvFallbackSubmits, strconv.FormatInt(n, 10)); err != nil {
a.Log.Warn("fallback counter persist failed", "err", err)
}
}
}
// Snapshot returns a copy of the current snapshot. // Snapshot returns a copy of the current snapshot.
func (a *Aggregator) Snapshot() Snapshot { func (a *Aggregator) Snapshot() Snapshot {
a.mu.RLock() a.mu.RLock()
@@ -486,11 +450,6 @@ func (a *Aggregator) refresh(ctx context.Context) {
} }
next.BlockSubmitAttempts = a.blockSubmitAttempts next.BlockSubmitAttempts = a.blockSubmitAttempts
next.BlockSubmitsConfirmed = a.blockSubmitsConfirmed next.BlockSubmitsConfirmed = a.blockSubmitsConfirmed
next.FallbackSubmitsTotal = a.fallbackSubmits
if !a.lastFallbackSubmitAt.IsZero() {
next.LastFallbackSubmitAt = a.lastFallbackSubmitAt.Unix()
next.LastFallbackVia = a.lastFallbackVia
}
next.ZMQEnabled = a.zmqEnabled next.ZMQEnabled = a.zmqEnabled
if !a.lastZMQEventTime.IsZero() { if !a.lastZMQEventTime.IsZero() {
next.LastZMQEventAge = time.Since(a.lastZMQEventTime).Seconds() next.LastZMQEventAge = time.Since(a.lastZMQEventTime).Seconds()
@@ -540,13 +499,6 @@ func (a *Aggregator) loadPersistedState() {
a.mu.Unlock() a.mu.Unlock()
} }
} }
if v, err := a.Store.GetKV(kvFallbackSubmits); err == nil && v != "" {
if n, perr := strconv.ParseInt(v, 10, 64); perr == nil {
a.mu.Lock()
a.fallbackSubmits = n
a.mu.Unlock()
}
}
cutoff := time.Now().Add(-24 * time.Hour).Unix() cutoff := time.Now().Add(-24 * time.Hour).Unix()
if samples, err := a.Store.HashrateSince(cutoff); err != nil { if samples, err := a.Store.HashrateSince(cutoff); err != nil {
ckpool/patches/0004-dump-pending-block-for-fallback.patch
-125
View File
@@ -1,125 +0,0 @@
diff --git a/src/generator.c b/src/generator.c
index 22e2e08..438b199 100644
--- a/src/generator.c
+++ b/src/generator.c
@@ -351,11 +351,25 @@ bool generator_submitblock(ckpool_t *ckp, const char *buf)
server_instance_t *si;
bool warn = false;
connsock_t *cs;
+ /* Bound the wait for current_si so a permanently-down primary
+ * bitcoind does not pin the caller forever. After this many 10ms
+ * sleeps (~3s), return false so the caller (in our fork:
+ * local_block_submit) can hand off to kamado-api's fallback
+ * broadcaster instead of blocking the stratifier on a dead RPC.
+ * Original upstream behavior was to spin indefinitely; we trade
+ * that for predictable latency. */
+ const int max_no_si_iters = 300;
+ int no_si_iters = 0;
while (unlikely(!(si = gdata->current_si))) {
if (!warn)
- LOGWARNING("No live current server in generator_blocksubmit! Resubmitting indefinitely!");
+ LOGWARNING("No live current server in generator_blocksubmit! Waiting up to ~3s before giving up so fallback can take over...");
warn = true;
+ if (++no_si_iters > max_no_si_iters) {
+ LOGWARNING("generator_submitblock: no live primary server after %d ms, returning false",
+ no_si_iters * 10);
+ return false;
+ }
cksleep_ms(10);
}
cs = &si->cs;
diff --git a/src/stratifier.c b/src/stratifier.c
index 52da790..7caf179 100644
--- a/src/stratifier.c
+++ b/src/stratifier.c
@@ -2069,16 +2069,75 @@ process_block(const workbase_t *wb, const char *coinbase, const int cblen,
return gbt_block;
}
-/* Submit block data locally, absorbing and freeing gbt_block */
+/* Write the raw block hex to a sidecar file under <logdir>/pending-blocks/
+ * so the external watcher (kamado-api) can re-broadcast via fallback RPC
+ * nodes. Best-effort: any error here is logged at INFO and never blocks
+ * the caller. Called only after the primary submit fails so the happy
+ * path stays disk-free. */
+static void kamado_dump_pending_block(ckpool_t *ckp, const char *gbt_block,
+ const char *rhash, int height,
+ char *out_path, size_t out_path_len)
+{
+ char dir[512] = {};
+ int fd;
+ size_t blen;
+ ssize_t w;
+
+ out_path[0] = '\0';
+ if (!ckp || !ckp->logdir || !gbt_block)
+ return;
+ snprintf(dir, sizeof(dir), "%spending-blocks", ckp->logdir);
+ if (mkdir(dir, 0750) < 0 && errno != EEXIST) {
+ LOGINFO("kamado: mkdir %s failed: %s", dir, strerror(errno));
+ return;
+ }
+ snprintf(out_path, out_path_len, "%s/%d-%.16s.hex", dir, height, rhash);
+ fd = open(out_path, O_WRONLY | O_CREAT | O_TRUNC, 0640);
+ if (fd < 0) {
+ LOGINFO("kamado: open %s failed: %s", out_path, strerror(errno));
+ out_path[0] = '\0';
+ return;
+ }
+ blen = strlen(gbt_block);
+ w = write(fd, gbt_block, blen);
+ close(fd);
+ if (w != (ssize_t)blen) {
+ LOGINFO("kamado: short write to %s (%zd/%zu)", out_path, w, blen);
+ unlink(out_path);
+ out_path[0] = '\0';
+ return;
+ }
+ LOGNOTICE("kamado: dumped pending block height %d to %s (%zu bytes) for fallback broadcast",
+ height, out_path, blen);
+}
+
+/* Submit block data locally, absorbing and freeing gbt_block.
+ *
+ * Order matters: we always try ckp's primary bitcoind FIRST so the
+ * happy path adds zero latency or disk I/O. Only when the primary
+ * rejects (or generator_submitblock returns false for any other
+ * reason) do we dump the raw hex to <logdir>/pending-blocks/ so the
+ * kamado-api watcher can re-broadcast via fallback RPC nodes. */
static bool local_block_submit(ckpool_t *ckp, char *gbt_block, const uchar *flip32, int height)
{
- bool ret = generator_submitblock(ckp, gbt_block);
char heighthash[68] = {}, rhash[68] = {};
+ char pending_path[512] = {};
uchar swap256[32];
+ bool ret;
- free(gbt_block);
swap_256(swap256, flip32);
__bin2hex(rhash, swap256, 32);
+
+ /* Primary submit first — this is the latency-critical path. */
+ ret = generator_submitblock(ckp, gbt_block);
+
+ /* Only touch disk if the primary didn't accept. */
+ if (!ret) {
+ kamado_dump_pending_block(ckp, gbt_block, rhash, height,
+ pending_path, sizeof(pending_path));
+ }
+
+ free(gbt_block);
generator_preciousblock(ckp, rhash);
/* Check failures that may be inconclusive but were submitted via other
@@ -2099,6 +2158,10 @@ static bool local_block_submit(ckpool_t *ckp, char *gbt_block, const uchar *flip
height, ret ? "ACCEPTED" : "REJECTED");
}
}
+ /* Block ended up on chain (either initial submit or precious-block
+ * recovery): the dump file is no longer needed. */
+ if (ret && pending_path[0])
+ unlink(pending_path);
return ret;
}
ckpool/patches/README.md
+1 -22
View File
@@ -9,7 +9,7 @@ Patches are applied in alphabetical order by filename. Use a numeric prefix to e
## Current state ## Current state
Four Kamado patches are applied on top of the pinned upstream commit, in Three Kamado patches are applied on top of the pinned upstream commit, in
alphabetical order: alphabetical order:
| Patch | What it does | | Patch | What it does |
@@ -17,7 +17,6 @@ alphabetical order:
| `0001-expose-bestever-in-runtime-json.patch` | Adds `bestever` field to the `users` / `workers` runtime socket JSON | | `0001-expose-bestever-in-runtime-json.patch` | Adds `bestever` field to the `users` / `workers` runtime socket JSON |
| `0002-enable-socket-api-responses.patch` | Always reply on the listener socket so kamado-api gets responses even with `btcsolo: true` | | `0002-enable-socket-api-responses.patch` | Always reply on the listener socket so kamado-api gets responses even with `btcsolo: true` |
| `0003-share-error-as-stratum-array.patch` | Maps `share_err` to Stratum spec error codes; emits `[code, msg, null]` per Slush | | `0003-share-error-as-stratum-array.patch` | Maps `share_err` to Stratum spec error codes; emits `[code, msg, null]` per Slush |
| `0004-dump-pending-block-for-fallback.patch` | Writes the raw block hex to `<logdir>/pending-blocks/` before submit; unlinks on success |
### Why 0001 matters ### Why 0001 matters
@@ -34,26 +33,6 @@ This patch adds `bestever` to the runtime JSON so the UI can show both
"this round" and "all-time" best share side by side. No behavioral change "this round" and "all-time" best share side by side. No behavioral change
to share validation or block handling. Candidate for upstreaming. to share validation or block handling. Candidate for upstreaming.
### Why 0004 matters
Block submission to bitcoind is the most revenue-critical RPC call ckpool
makes. If bitcoind is unreachable when a share meets network difficulty,
ckpool's `generator` thread retries indefinitely against the same single
endpoint — and the raw block data lives only in stratifier memory, so a
ckpool crash before bitcoind comes back permanently loses the block.
This patch hooks `local_block_submit` to write the raw block hex to
`<logdir>/pending-blocks/<height>-<rhash>.hex` *before* invoking
`generator_submitblock`, and unlinks the file on success. `kamado-api`
runs a watcher over that directory: if a file persists past a grace
period (default 30 s), it submits the block via fallback RPC URLs the
operator has configured. Multiple fallbacks are tried in sequence; the
file is unlinked when any fallback returns success or "duplicate"
(meaning the block already landed).
This is a Kamado-specific integration hook — almost certainly not
upstreamable, but minimal-impact on existing ckpool behavior.
Beyond this patch, the pinned upstream commit (`cfb0f83b`, tagged as Beyond this patch, the pinned upstream commit (`cfb0f83b`, tagged as
version 1.0) already includes every fix that Bassin issue #29 asked to version 1.0) already includes every fix that Bassin issue #29 asked to
backport, plus several improvements: backport, plus several improvements:
ui/src/lib/HealthBanners.svelte
+8 -41
View File
@@ -4,20 +4,15 @@
// submit_gap > 0 means ckpool tried to submit at least one block that // submit_gap > 0 means ckpool tried to submit at least one block that
// never got the "Solved and confirmed" follow-up — so either bitcoind // never got the "Solved and confirmed" follow-up — so either bitcoind
// rejected the submission or the RPC dropped. The number stays > 0 // rejected the submission or the RPC dropped. These are persistent
// forever after such an event (these are persistent counters), so we // counters so the banner stays visible until the operator clears it
// only show it as a banner when the fallback hasn't covered for it // by inspecting bitcoind's logs.
// OR there's been a recent fallback the operator should investigate.
const submitGap = $derived.by(() => { const submitGap = $derived.by(() => {
const d = snap.data; const d = snap.data;
if (!d) return 0; if (!d) return 0;
return Math.max(0, (d.block_submit_attempts ?? 0) - (d.block_submits_confirmed ?? 0)); return Math.max(0, (d.block_submit_attempts ?? 0) - (d.block_submits_confirmed ?? 0));
}); });
const fallbackCount = $derived(snap.data?.fallback_submits_total ?? 0);
const lastFallbackAt = $derived(snap.data?.last_fallback_submit_at ?? 0);
const lastFallbackVia = $derived(snap.data?.last_fallback_via ?? "");
// ZMQ stale = configured but no event in 30+ minutes. Bitcoin's avg // ZMQ stale = configured but no event in 30+ minutes. Bitcoin's avg
// block interval is 10 min; 30 min covers normal variance without // block interval is 10 min; 30 min covers normal variance without
// false-alarming on quiet stretches. // false-alarming on quiet stretches.
@@ -26,41 +21,18 @@
if (!d || !d.zmq_enabled || !d.has_last_zmq_event) return false; if (!d || !d.zmq_enabled || !d.has_last_zmq_event) return false;
return (d.last_zmq_event_age ?? 0) > 1800; return (d.last_zmq_event_age ?? 0) > 1800;
}); });
// Show the fallback banner for 24h after the most recent fallback
// event so the operator sees the alert during their next check-in
// even if the underlying problem auto-resolved.
const fallbackRecent = $derived.by(() => {
if (!lastFallbackAt) return false;
const ageSec = Date.now() / 1000 - lastFallbackAt;
return ageSec >= 0 && ageSec < 86400;
});
</script> </script>
{#if submitGap > 0 || fallbackRecent || zmqStale} {#if submitGap > 0 || zmqStale}
<div class="banners"> <div class="banners">
{#if fallbackRecent} {#if submitGap > 0}
<div class="banner alert">
<span class="icon">!</span>
<div class="text">
<strong>Fallback broadcaster used</strong>
—
our primary bitcoind didn't accept a block submission. Backup
RPC <code>{lastFallbackVia}</code> took over
{formatAgo(lastFallbackAt)}. Total fallbacks since first run: {fallbackCount}.
Investigate primary bitcoind health.
</div>
</div>
{/if}
{#if submitGap > 0 && !fallbackRecent}
<div class="banner warn"> <div class="banner warn">
<span class="icon">?</span> <span class="icon">!</span>
<div class="text"> <div class="text">
<strong>Submit gap:</strong> <strong>Submit gap:</strong>
{submitGap} block{submitGap === 1 ? "" : "s"} attempted but not {submitGap} block{submitGap === 1 ? "" : "s"} attempted but not
confirmed by bitcoind. Either rejected at submission or the confirmed by bitcoind. Investigate primary bitcoind logs —
RPC dropped. Configure backup RPC URLs to enable automatic submission may have been rejected or the RPC dropped.
fallback broadcast.
</div> </div>
</div> </div>
{/if} {/if}
@@ -94,11 +66,6 @@
border: 1px solid transparent; border: 1px solid transparent;
line-height: 1.4; line-height: 1.4;
} }
.banner.alert {
background: rgba(220, 80, 80, 0.10);
border-color: rgba(220, 80, 80, 0.40);
color: rgb(230, 130, 130);
}
.banner.warn { .banner.warn {
background: rgba(220, 170, 60, 0.10); background: rgba(220, 170, 60, 0.10);
border-color: rgba(220, 170, 60, 0.40); border-color: rgba(220, 170, 60, 0.40);
ui/src/types.ts
-5
View File
@@ -132,11 +132,6 @@ export type Snapshot = {
// submissions are being rejected by bitcoind. // submissions are being rejected by bitcoind.
block_submit_attempts: number; block_submit_attempts: number;
block_submits_confirmed: number; block_submits_confirmed: number;
// Fallback submitter telemetry: how often we had to broadcast via a
// backup RPC URL because the primary bitcoind didn't accept in time.
fallback_submits_total: number;
last_fallback_submit_at?: number;
last_fallback_via?: string;
// ZMQ subscriber freshness diagnostics. // ZMQ subscriber freshness diagnostics.
zmq_enabled: boolean; zmq_enabled: boolean;
has_last_zmq_event: boolean; has_last_zmq_event: boolean;