satoshi/KamadoPool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Harden block-recording pipeline: P0 reliability fixes

Browse Source 
Closes the silent-failure modes between "ckpool logs a solve" and
"block correctly displayed":

* Difficulty estimate matched mempool.space — the projection now uses
  (inEpoch + 1) intervals so it converges on Bitcoin Core's eventual
  retarget formula at end-of-epoch instead of undershooting by ~0.05–
  0.10 % throughout.

* Tailer resumes mid-log on restart — persists (inode, offset) to kv
  every EOF + on shutdown, and replays the unread tail next time. Any
  solve line written while kamado-api was down would previously be
  invisible forever.

* Background reconcile loop (60 s) retries hash/reward enrichment for
  blocks the original RPC missed, so a transient bitcoind-index race no
  longer permanently leaves a block hashless.

* Reorg detection: same loop compares each recent stored hash against
  getblockhash(height); a mismatch stamps orphaned_at. UI renders these
  strikethrough with a red "orphaned" tag instead of showing illusory
  rewards forever.

* InsertBlock now reports whether a row was actually inserted; the
  caller WARN-logs duplicate-height ignores so a re-mined orphaned
  height can't disappear silently.

* Submit-attempt vs confirmed counters surface failed submissions:
  every "Possible/Submitting block solve" log line increments
  block_submit_attempts; "Solved and confirmed" increments
  block_submits_confirmed. A growing gap means bitcoind is rejecting
  our submissions — previously invisible.

* share_err patch refreshed against pinned ckpool source: added
  SE_NO_JOBID -> 21 and SE_WORKER_MISMATCH -> 24 mappings, kept
  SE_INVALID_NONCE2 in 20 (it's a malformed-input error, not low-diff).
  AxeOS users now see actionable Stratum codes instead of
  "unknown error".

UI gets new orphaned_at + block_submit_attempts/confirmed fields on
the snapshot type and a strikethrough-with-tag rendering for orphaned
blocks in BlocksTable.
This commit is contained in:
satoshi
2026-04-27 16:30:15 +03:00
parent 81227cb90f
commit df0dbf89e5
8 changed files with 513 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/internal/logmon/tailer.go
+98 -13
View File
@@ -36,24 +36,50 @@ type BlockEvent struct {
ShareDiff float64 `json:"share_diff,omitempty"`
}
// AttemptEvent is emitted when ckpool logs a "Possible block solve"
// or "Submitting possible block solve" line — i.e. ckpool decided a
// share met network difficulty and is attempting to submit it to
// bitcoind. This precedes the "Solved and confirmed" line that drives
// BlockEvent. Counting attempts vs confirmations surfaces submission
// failures (bitcoind rejected, RPC timeout, etc.) that would otherwise
// be invisible.
type AttemptEvent struct {
SeenAt time.Time
ShareDiff float64
RawLine string
}
// Tailer follows a log file, surviving rotation/truncation, and emits
// parsed events. Create with New, then Run in a goroutine.
type Tailer struct {
Path string
Events chan BlockEvent
Attempts chan AttemptEvent
Log *slog.Logger
PollWait time.Duration // how long to sleep between EOF polls
// LoadCursor / SaveCursor, if both non-nil, persist the read
// position across process restarts. LoadCursor returns (inode,
// offset, true) when a saved cursor exists for this path; the
// tailer will resume from offset only if the inode still matches.
// SaveCursor is invoked on a throttled cadence as we read, so a
// crash loses at most ~1s of unread bytes.
LoadCursor func() (inode uint64, offset int64, ok bool)
SaveCursor func(inode uint64, offset int64)
// lastSolveDiff remembers the share diff from the most recent
// "Possible block solve" line so handleLine can attach it to the
// subsequent "Solved and confirmed block" event. Reset after use.
lastSolveDiff float64
lastCursorSave time.Time
}
func New(path string, log *slog.Logger) *Tailer {
return &Tailer{
Path: path,
Events: make(chan BlockEvent, 16),
Attempts: make(chan AttemptEvent, 16),
Log: log,
PollWait: 500 * time.Millisecond,
}
@@ -70,17 +96,24 @@ var (
solveDiffRE = regexp.MustCompile(`(?:Possible|Submitting[^"]*possible).*block solve.*diff\s+([0-9eE.+-]+)`)
)
// Run blocks until ctx is cancelled. It opens the file, seeks to the end,
// and reads new lines as they are appended. If the file is rotated
// (shrinks, or inode changes), it re-opens.
// Run blocks until ctx is cancelled. It opens the file, seeks to either
// the saved cursor (if LoadCursor returns one and the inode still
// matches) or the end on first-ever run, and reads new lines as they
// are appended. If the file is rotated (shrinks, or inode changes), it
// re-opens.
func (t *Tailer) Run(ctx context.Context) {
defer close(t.Events)
defer close(t.Attempts)
var (
f *os.File
reader *bufio.Reader
lastIno uint64
lastPos int64
// firstOpen distinguishes the very first Open of this tailer
// (where we honor LoadCursor and replay the backlog) from
// rotation re-opens (where we always start at 0).
firstOpen = true
)
open := func() error {
@@ -91,23 +124,64 @@ func (t *Tailer) Run(ctx context.Context) {
if err != nil {
return err
}
// Start at end on first open so we don't replay old events.
if _, err := nf.Seek(0, io.SeekEnd); err != nil {
_ = nf.Close()
return err
}
st, err := nf.Stat()
if err != nil {
_ = nf.Close()
return err
}
ino := inodeOf(st)
size := st.Size()
// Decide where to start reading.
var startAt int64
if firstOpen {
if t.LoadCursor != nil {
if savedIno, savedOff, ok := t.LoadCursor(); ok &&
savedIno == ino && savedOff <= size {
startAt = savedOff
if savedOff < size {
t.Log.Info("logmon: resuming from saved cursor",
"path", t.Path, "offset", savedOff, "size", size)
}
} else {
// No cursor, or stale (file was rotated since we
// last saw it, so we have no way to know what we
// already read). Start at end to avoid replaying
// the entire log.
startAt = size
}
} else {
startAt = size
}
firstOpen = false
} else {
// Rotation: read the whole replacement file from the start.
startAt = 0
}
if _, err := nf.Seek(startAt, io.SeekStart); err != nil {
_ = nf.Close()
return err
}
f = nf
reader = bufio.NewReader(f)
lastIno = inodeOf(st)
lastPos, _ = f.Seek(0, io.SeekCurrent)
lastIno = ino
lastPos = startAt
return nil
}
saveCursor := func(force bool) {
if t.SaveCursor == nil || f == nil {
return
}
now := time.Now()
if !force && now.Sub(t.lastCursorSave) < time.Second {
return
}
t.SaveCursor(lastIno, lastPos)
t.lastCursorSave = now
}
// Initial open; retry on failure until the file exists.
for {
if err := open(); err != nil {
@@ -127,6 +201,7 @@ func (t *Tailer) Run(ctx context.Context) {
for {
if ctx.Err() != nil {
saveCursor(true)
return
}
line, err := reader.ReadString('\n')
@@ -139,6 +214,7 @@ func (t *Tailer) Run(ctx context.Context) {
}
if !errors.Is(err, io.EOF) {
t.Log.Warn("logmon: read error, reopening", "err", err)
saveCursor(true)
if !sleep(ctx, t.PollWait) {
return
}
@@ -146,7 +222,9 @@ func (t *Tailer) Run(ctx context.Context) {
continue
}
// EOF: check for rotation (inode changed) or truncation (size < pos).
// EOF: persist where we are (caught up to current end), then
// check for rotation (inode changed) or truncation (size < pos).
saveCursor(false)
if st, statErr := os.Stat(t.Path); statErr == nil {
if inodeOf(st) != lastIno || st.Size() < lastPos {
t.Log.Info("logmon: log rotated, reopening", "path", t.Path)
@@ -164,8 +242,15 @@ func (t *Tailer) Run(ctx context.Context) {
func (t *Tailer) handleLine(line string) {
if m := solveDiffRE.FindStringSubmatch(line); m != nil {
if d, err := strconv.ParseFloat(m[1], 64); err == nil {
t.lastSolveDiff = d
var d float64
if v, err := strconv.ParseFloat(m[1], 64); err == nil {
d = v
t.lastSolveDiff = v
}
select {
case t.Attempts <- AttemptEvent{SeenAt: time.Now(), ShareDiff: d, RawLine: line}:
default:
// Best-effort metric — dropping is fine.
}
return
}