Harden block-recording pipeline: P0 reliability fixes
Closes the silent-failure modes between "ckpool logs a solve" and "block correctly displayed": * Difficulty estimate matched mempool.space — the projection now uses (inEpoch + 1) intervals so it converges on Bitcoin Core's eventual retarget formula at end-of-epoch instead of undershooting by ~0.05– 0.10 % throughout. * Tailer resumes mid-log on restart — persists (inode, offset) to kv every EOF + on shutdown, and replays the unread tail next time. Any solve line written while kamado-api was down would previously be invisible forever. * Background reconcile loop (60 s) retries hash/reward enrichment for blocks the original RPC missed, so a transient bitcoind-index race no longer permanently leaves a block hashless. * Reorg detection: same loop compares each recent stored hash against getblockhash(height); a mismatch stamps orphaned_at. UI renders these strikethrough with a red "orphaned" tag instead of showing illusory rewards forever. * InsertBlock now reports whether a row was actually inserted; the caller WARN-logs duplicate-height ignores so a re-mined orphaned height can't disappear silently. * Submit-attempt vs confirmed counters surface failed submissions: every "Possible/Submitting block solve" log line increments block_submit_attempts; "Solved and confirmed" increments block_submits_confirmed. A growing gap means bitcoind is rejecting our submissions — previously invisible. * share_err patch refreshed against pinned ckpool source: added SE_NO_JOBID -> 21 and SE_WORKER_MISMATCH -> 24 mappings, kept SE_INVALID_NONCE2 in 20 (it's a malformed-input error, not low-diff). AxeOS users now see actionable Stratum codes instead of "unknown error". UI gets new orphaned_at + block_submit_attempts/confirmed fields on the snapshot type and a strikethrough-with-tag rendering for orphaned blocks in BlocksTable.
This commit is contained in:
@@ -73,6 +73,13 @@ type Snapshot struct {
|
||||
// the StartOS config "Block Explorer" -> "Custom URL".
|
||||
MempoolBaseURL string `json:"mempool_base_url,omitempty"`
|
||||
|
||||
// Counts of share-submit attempts ("Possible/Submitting block solve"
|
||||
// log lines) and confirmed solves ("Solved and confirmed block").
|
||||
// A growing gap means bitcoind is rejecting our submissions or
|
||||
// dropping the RPC — surface it in the UI as an alert.
|
||||
BlockSubmitAttempts int64 `json:"block_submit_attempts"`
|
||||
BlockSubmitsConfirmed int64 `json:"block_submits_confirmed"`
|
||||
|
||||
// Health
|
||||
CKPoolOK bool `json:"ckpool_ok"`
|
||||
BitcoinOK bool `json:"bitcoin_ok"`
|
||||
@@ -88,6 +95,20 @@ const (
|
||||
// diff-1-normalized work, so cumulative_work * 2^32 is real hashes.
|
||||
// The old key is orphaned in the kv table on upgrade; harmless.
|
||||
kvCumulativeWork = "cumulative_work"
|
||||
|
||||
kvSubmitAttempts = "block_submit_attempts"
|
||||
kvSubmitsConfirmed = "block_submits_confirmed"
|
||||
|
||||
// reconcileInterval is how often we sweep recent blocks looking
|
||||
// for missing hash/reward enrichment and reorg-orphaned hashes.
|
||||
// 60s is fast enough to recover from a transient bitcoind hiccup
|
||||
// within a minute, slow enough to never load the RPC.
|
||||
reconcileInterval = 60 * time.Second
|
||||
|
||||
// reconcileLookback caps how far back the reconcile loop looks.
|
||||
// Blocks older than this with empty hash are abandoned; hashes
|
||||
// older than this are assumed deep enough to never reorg.
|
||||
reconcileLookback = 24 * time.Hour
|
||||
)
|
||||
|
||||
// Aggregator refreshes a Snapshot on a ticker.
|
||||
@@ -146,6 +167,12 @@ type Aggregator struct {
|
||||
// DEBUG (likely a transient warm-up or lock hiccup); repeated failures
|
||||
// escalate to WARN.
|
||||
ckFailStreak int
|
||||
|
||||
// Submit-attempt vs confirmed counters. Persisted in kv so the
|
||||
// running gap survives restarts. Both are monotonic.
|
||||
blockSubmitAttempts int64
|
||||
blockSubmitsConfirmed int64
|
||||
lastSubmitCountSave time.Time
|
||||
}
|
||||
|
||||
func New(ck *ckpool.Client, rpc *bitcoind.RPC, interval time.Duration, log *slog.Logger) *Aggregator {
|
||||
@@ -270,7 +297,12 @@ func (a *Aggregator) refresh(ctx context.Context) {
|
||||
}
|
||||
if a.retargetStartUnix > 0 {
|
||||
elapsed := float64(time.Now().Unix() - a.retargetStartUnix)
|
||||
expected := float64(inEpoch) * 600
|
||||
// Match Bitcoin Core / mempool.space: project nActualTimespan
|
||||
// by treating the elapsed window as covering (inEpoch + 1)
|
||||
// block intervals, since at retarget the consensus formula
|
||||
// uses (lastBlock.time - firstBlock.time) across all 2016
|
||||
// blocks of the epoch.
|
||||
expected := float64(inEpoch+1) * 600
|
||||
if elapsed > 0 {
|
||||
factor := expected / elapsed
|
||||
if factor > 4 {
|
||||
@@ -379,6 +411,8 @@ func (a *Aggregator) refresh(ctx context.Context) {
|
||||
next.RecentBlocks = make([]BlockRecord, len(a.blocks))
|
||||
copy(next.RecentBlocks, a.blocks)
|
||||
}
|
||||
next.BlockSubmitAttempts = a.blockSubmitAttempts
|
||||
next.BlockSubmitsConfirmed = a.blockSubmitsConfirmed
|
||||
a.snap = next
|
||||
cb := a.OnRefresh
|
||||
pushed := next
|
||||
@@ -408,6 +442,21 @@ func (a *Aggregator) loadPersistedState() {
|
||||
}
|
||||
}
|
||||
|
||||
if v, err := a.Store.GetKV(kvSubmitAttempts); err == nil && v != "" {
|
||||
if n, perr := strconv.ParseInt(v, 10, 64); perr == nil {
|
||||
a.mu.Lock()
|
||||
a.blockSubmitAttempts = n
|
||||
a.mu.Unlock()
|
||||
}
|
||||
}
|
||||
if v, err := a.Store.GetKV(kvSubmitsConfirmed); err == nil && v != "" {
|
||||
if n, perr := strconv.ParseInt(v, 10, 64); perr == nil {
|
||||
a.mu.Lock()
|
||||
a.blockSubmitsConfirmed = n
|
||||
a.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
cutoff := time.Now().Add(-24 * time.Hour).Unix()
|
||||
if samples, err := a.Store.HashrateSince(cutoff); err != nil {
|
||||
a.Log.Warn("hashrate history load failed", "err", err)
|
||||
|
||||
Reference in New Issue
Block a user