Add best share analysis page, miner column, and UI improvements

- Best share page: hex + binary hash comparison against network target, per-bit coloring showing exactly which bits prevented a valid block, toggle between network diff at time of finding vs current diff - Capture best share hash from ckpool logs with one-time backfill - Persist network difficulty at time of best share for historical accuracy - Add miner (worker) column to blocks table via coinbase address matching - Truncate block hashes in table with full hash on hover - Increase hashrate chart Y-axis to 7 ticks for better readability
2026-05-18 17:19:35 +03:00
parent f1dc0a8a79
commit 1157f3501a
17 changed files with 1499 additions and 81 deletions
@@ -6,6 +6,7 @@ package state

 import (
 	"context"
+	"encoding/json"
 	"log/slog"
 	"strconv"
 	"sync"
@@ -13,6 +14,7 @@ import (

 	"github.com/kamadopool/kamado-api/internal/bitcoind"
 	"github.com/kamadopool/kamado-api/internal/ckpool"
+	"github.com/kamadopool/kamado-api/internal/logmon"
 	"github.com/kamadopool/kamado-api/internal/store"
 	"github.com/kamadopool/kamado-api/internal/zmqmon"
 )
@@ -43,6 +45,10 @@ type Snapshot struct {

 	// Best share difficulty ever seen across all workers.
 	BestDiff float64 `json:"best_diff"`
+	// Block header hash of the best share ever found (hex string).
+	BestShareHash string `json:"best_share_hash,omitempty"`
+	// Network difficulty at the time the best share was found.
+	BestShareNetDiff float64 `json:"best_share_net_diff,omitempty"`
 	// Last best-diff value acknowledged by the user via the UI.
 	// The frontend shows a "new best" glow when best_diff > this.
 	AckedBestDiff float64 `json:"acked_best_diff"`
@@ -99,6 +105,17 @@ type Snapshot struct {
 	AllTimeAccepted int64 `json:"alltime_accepted"`
 	AllTimeRejected int64 `json:"alltime_rejected"`

+	// Share statistics: rejection reasons and difficulty distribution.
+	// Session resets on ckpool restart; AllTime persisted across restarts.
+	RejectReasons    map[string]int64 `json:"reject_reasons_session,omitempty"`
+	RejectReasonsAll map[string]int64 `json:"reject_reasons_alltime,omitempty"`
+	// Difficulty distribution buckets: [<1M, 1M-100M, 100M-1G, 1G-100G, 100G-1T, >1T]
+	DiffDist    [6]int64 `json:"diff_dist_session"`
+	DiffDistAll [6]int64 `json:"diff_dist_alltime"`
+	// Average share difficulty (arithmetic mean of all accepted shares).
+	AvgDiffSession float64 `json:"avg_diff_session"`
+	AvgDiffAlltime float64 `json:"avg_diff_alltime"`
+
 	// Block update latency diagnostics (ZMQ trigger → mining.notify).
 	LatencyCount    int64   `json:"latency_count"`
 	LatencyAvgMs    int64   `json:"latency_avg_ms"`
@@ -129,12 +146,17 @@ const (
 	kvLatencyLastMs   = "latency_last_ms"
 	kvStaleWorkHashes = "stale_work_hashes"

-	kvAckedBestDiff = "acked_best_diff"
-	kvBestDiff      = "best_diff"
+	kvAckedBestDiff  = "acked_best_diff"
+	kvBestDiff       = "best_diff"
+	kvBestShareHash    = "best_share_hash"
+	kvBestShareNetDiff = "best_share_net_diff"

 	kvAllTimeAccepted = "alltime_accepted"
 	kvAllTimeRejected = "alltime_rejected"

+	kvRejectReasonsAll = "reject_reasons_alltime"
+	kvDiffDistAll      = "diff_dist_alltime"
+
 	// reconcileInterval is how often we sweep recent blocks looking
 	// for missing hash/reward enrichment and reorg-orphaned hashes.
 	// 60s is fast enough to recover from a transient bitcoind hiccup
@@ -159,6 +181,11 @@ type Aggregator struct {
 	// MempoolBaseURL leaves the UI on its mempool.space defaults.
 	MempoolBaseURL string

+	// LogFilePath is the ckpool log path, used for one-time backfill
+	// of the best share hash when upgrading from a version that didn't
+	// capture it. Set by main before calling Run.
+	LogFilePath string
+
 	// OnRefresh, if set, is called (non-blocking) after each snapshot
 	// refresh. Used by the WebSocket hub to push updates to clients.
 	OnRefresh func(Snapshot)
@@ -246,17 +273,35 @@ type Aggregator struct {
 	// High-water mark for the best share difficulty ever seen.
 	// Persisted to kv so it survives restarts and transient ckpool gaps.
 	bestDiff float64
+	// Block header hash of the best share ever found. Captured from the
+	// ckpool log "Accepted client ... : <hash>" line for the share that
+	// set the bestDiff record. Persisted alongside bestDiff.
+	bestShareHash    string
+	bestShareNetDiff float64 // network difficulty when best share was found

 	// Last best-diff value acknowledged by the user in the UI.
 	ackedBestDiff float64
+
+	// Share statistics from log parsing.
+	sessionRejectReasons map[string]int64
+	alltimeRejectReasons map[string]int64
+	sessionDiffDist      [6]int64
+	alltimeDiffDist      [6]int64
+	sessionDiffCount     int64   // total accepted shares (session)
+	alltimeDiffCount     int64   // total accepted shares (alltime)
+	sessionDiffSum       float64 // sum of share difficulties (session)
+	alltimeDiffSum       float64 // sum of share difficulties (alltime)
+	lastShareStatsSave   time.Time
 }

 func New(ck *ckpool.Client, rpc *bitcoind.RPC, interval time.Duration, log *slog.Logger) *Aggregator {
 	return &Aggregator{
-		CK:       ck,
-		RPC:      rpc,
-		Interval: interval,
-		Log:      log,
+		CK:                  ck,
+		RPC:                 rpc,
+		Interval:            interval,
+		Log:                 log,
+		sessionRejectReasons: make(map[string]int64),
+		alltimeRejectReasons: make(map[string]int64),
 		ready:    make(chan struct{}),
 	}
 }
@@ -443,6 +488,8 @@ func (a *Aggregator) refresh(ctx context.Context) {
 		}
 	}
 	next.BestDiff = a.bestDiff
+	next.BestShareHash = a.bestShareHash
+	next.BestShareNetDiff = a.bestShareNetDiff
 	next.AckedBestDiff = a.ackedBestDiff
 	now := time.Now()

@@ -483,6 +530,9 @@ func (a *Aggregator) refresh(ctx context.Context) {
 		if a.hasShareCountBaseline {
 			if curAcc >= a.lastPoolAcceptedRaw {
 				a.allTimeAccepted += curAcc - a.lastPoolAcceptedRaw
+			} else {
+				// ckpool restarted — reset session share stats.
+				a.resetSessionShareStats()
 			}
 			if curRej >= a.lastPoolRejectedRaw {
 				a.allTimeRejected += curRej - a.lastPoolRejectedRaw
@@ -569,6 +619,31 @@ func (a *Aggregator) refresh(ctx context.Context) {
 	}
 	next.LatencyLastMs = a.latencyLastMs
 	next.StaleWorkHashes = a.staleWorkHashes
+
+	// Share statistics (from log parsing).
+	if len(a.sessionRejectReasons) > 0 {
+		m := make(map[string]int64, len(a.sessionRejectReasons))
+		for k, v := range a.sessionRejectReasons {
+			m[k] = v
+		}
+		next.RejectReasons = m
+	}
+	if len(a.alltimeRejectReasons) > 0 {
+		m := make(map[string]int64, len(a.alltimeRejectReasons))
+		for k, v := range a.alltimeRejectReasons {
+			m[k] = v
+		}
+		next.RejectReasonsAll = m
+	}
+	next.DiffDist = a.sessionDiffDist
+	next.DiffDistAll = a.alltimeDiffDist
+	if a.sessionDiffCount > 0 {
+		next.AvgDiffSession = a.sessionDiffSum / float64(a.sessionDiffCount)
+	}
+	if a.alltimeDiffCount > 0 {
+		next.AvgDiffAlltime = a.alltimeDiffSum / float64(a.alltimeDiffCount)
+	}
+
 	a.snap = next
 	cb := a.OnRefresh
 	pushed := next
@@ -605,6 +680,126 @@ func (a *Aggregator) ResetAckedBestDiff() {
 	}
 }

+// diffBucket returns the index [0..5] for a share difficulty:
+//
+//	0: < 1M    1: 1M–100M   2: 100M–1G
+//	3: 1G–100G 4: 100G–1T   5: ≥ 1T
+func diffBucket(d float64) int {
+	switch {
+	case d < 1e6:
+		return 0
+	case d < 1e8:
+		return 1
+	case d < 1e9:
+		return 2
+	case d < 1e11:
+		return 3
+	case d < 1e12:
+		return 4
+	default:
+		return 5
+	}
+}
+
+// DiffBucketLabels are the human-readable labels for each difficulty bucket.
+var DiffBucketLabels = [6]string{"< 1M", "1M – 100M", "100M – 1G", "1G – 100G", "100G – 1T", "≥ 1T"}
+
+
+// IngestShareEvents reads individual share events from the log tailer
+// and maintains rejection-reason counts and difficulty-distribution
+// histograms. Session counters reset when ckpool restarts (detected by
+// the aggregator's existing pool.Shares decrease logic). Alltime
+// counters are persisted to the kv store.
+func (a *Aggregator) IngestShareEvents(ctx context.Context, events <-chan logmon.ShareEvent) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case ev, ok := <-events:
+			if !ok {
+				return
+			}
+			a.mu.Lock()
+			if ev.Rejected {
+				reason := ev.Reason
+				if reason == "" {
+					reason = "Unknown"
+				}
+				a.sessionRejectReasons[reason]++
+				a.alltimeRejectReasons[reason]++
+			} else {
+				bucket := diffBucket(ev.Diff)
+				a.sessionDiffDist[bucket]++
+				a.alltimeDiffDist[bucket]++
+				a.sessionDiffCount++
+				a.alltimeDiffCount++
+				a.sessionDiffSum += ev.Diff
+				a.alltimeDiffSum += ev.Diff
+				// Track hash of the best share from log parsing.
+				if ev.Hash != "" && ev.Diff > a.bestDiff {
+					a.bestDiff = ev.Diff
+					a.bestShareHash = ev.Hash
+					// Capture the current network difficulty at time of finding.
+					if a.snap.Chain != nil && a.snap.Chain.Difficulty > 0 {
+						a.bestShareNetDiff = a.snap.Chain.Difficulty
+					}
+					if a.Store != nil {
+						_ = a.Store.SetKV(kvBestDiff, strconv.FormatFloat(ev.Diff, 'f', -1, 64))
+						_ = a.Store.SetKV(kvBestShareHash, ev.Hash)
+						if a.bestShareNetDiff > 0 {
+							_ = a.Store.SetKV(kvBestShareNetDiff, strconv.FormatFloat(a.bestShareNetDiff, 'f', -1, 64))
+						}
+					}
+				}
+			}
+			save := a.Store != nil && time.Since(a.lastShareStatsSave) >= time.Minute
+			if save {
+				a.lastShareStatsSave = time.Now()
+			}
+			a.mu.Unlock()
+			if save {
+				a.persistShareStats()
+			}
+		}
+	}
+}
+
+// ResetSessionShareStats clears session-level share statistics. Called
+// when a ckpool restart is detected (pool.Shares decreases).
+func (a *Aggregator) resetSessionShareStats() {
+	a.sessionRejectReasons = make(map[string]int64)
+	a.sessionDiffDist = [6]int64{}
+	a.sessionDiffCount = 0
+	a.sessionDiffSum = 0
+}
+
+func (a *Aggregator) persistShareStats() {
+	if a.Store == nil {
+		return
+	}
+	a.mu.Lock()
+	reasonsCopy := make(map[string]int64, len(a.alltimeRejectReasons))
+	for k, v := range a.alltimeRejectReasons {
+		reasonsCopy[k] = v
+	}
+	distCopy := a.alltimeDiffDist
+	countCopy := a.alltimeDiffCount
+	sumCopy := a.alltimeDiffSum
+	a.mu.Unlock()
+
+	if data, err := json.Marshal(reasonsCopy); err == nil {
+		_ = a.Store.SetKV(kvRejectReasonsAll, string(data))
+	}
+	type distPersist struct {
+		Buckets [6]int64 `json:"b"`
+		Count   int64    `json:"n"`
+		Sum     float64  `json:"s"`
+	}
+	if data, err := json.Marshal(distPersist{Buckets: distCopy, Count: countCopy, Sum: sumCopy}); err == nil {
+		_ = a.Store.SetKV(kvDiffDistAll, string(data))
+	}
+}
+
 // loadPersistedState restores cumulative work and hashrate history from
 // the store so they survive process restarts. Safe to call with a nil
 // Store — becomes a no-op.
@@ -655,7 +850,7 @@ func (a *Aggregator) loadPersistedState() {
 		}
 	}

-	// Restore best diff high-water mark.
+	// Restore best diff high-water mark and its block header hash.
 	if v, err := a.Store.GetKV(kvBestDiff); err == nil && v != "" {
 		if f, perr := strconv.ParseFloat(v, 64); perr == nil {
 			a.mu.Lock()
@@ -663,6 +858,44 @@ func (a *Aggregator) loadPersistedState() {
 			a.mu.Unlock()
 		}
 	}
+	if v, err := a.Store.GetKV(kvBestShareHash); err == nil && v != "" {
+		a.mu.Lock()
+		a.bestShareHash = v
+		a.mu.Unlock()
+	}
+	if v, err := a.Store.GetKV(kvBestShareNetDiff); err == nil && v != "" {
+		if f, perr := strconv.ParseFloat(v, 64); perr == nil {
+			a.mu.Lock()
+			a.bestShareNetDiff = f
+			a.mu.Unlock()
+		}
+	}
+
+	// One-time backfill: if we have no hash (pre-upgrade), scan the
+	// ckpool log for the highest-difficulty accepted share and use its hash.
+	a.mu.RLock()
+	needsBackfill := a.bestShareHash == "" && a.LogFilePath != ""
+	a.mu.RUnlock()
+	if needsBackfill {
+		if logDiff, h := logmon.FindBestShareHash(a.LogFilePath); h != "" {
+			a.mu.Lock()
+			a.bestShareHash = h
+			// The log may contain the true best — adopt it if higher.
+			if logDiff > a.bestDiff {
+				a.bestDiff = logDiff
+				if a.Store != nil {
+					_ = a.Store.SetKV(kvBestDiff, strconv.FormatFloat(logDiff, 'f', -1, 64))
+				}
+			}
+			a.mu.Unlock()
+			if a.Store != nil {
+				_ = a.Store.SetKV(kvBestShareHash, h)
+			}
+			a.Log.Info("backfilled best share hash from log", "diff", logDiff, "hash", h)
+		} else {
+			a.Log.Info("best share hash backfill: no accepted shares found in log")
+		}
+	}

 	// Restore acknowledged best diff.
 	if v, err := a.Store.GetKV(kvAckedBestDiff); err == nil && v != "" {
@@ -701,6 +934,30 @@ func (a *Aggregator) loadPersistedState() {
 	}
 	a.mu.Unlock()

+	// Restore share statistics.
+	if v, err := a.Store.GetKV(kvRejectReasonsAll); err == nil && v != "" {
+		var m map[string]int64
+		if jerr := json.Unmarshal([]byte(v), &m); jerr == nil {
+			a.mu.Lock()
+			a.alltimeRejectReasons = m
+			a.mu.Unlock()
+		}
+	}
+	if v, err := a.Store.GetKV(kvDiffDistAll); err == nil && v != "" {
+		var dp struct {
+			Buckets [6]int64 `json:"b"`
+			Count   int64    `json:"n"`
+			Sum     float64  `json:"s"`
+		}
+		if jerr := json.Unmarshal([]byte(v), &dp); jerr == nil {
+			a.mu.Lock()
+			a.alltimeDiffDist = dp.Buckets
+			a.alltimeDiffCount = dp.Count
+			a.alltimeDiffSum = dp.Sum
+			a.mu.Unlock()
+		}
+	}
+
 	cutoff := time.Now().Add(-24 * time.Hour).Unix()
 	if samples, err := a.Store.HashrateSince(cutoff); err != nil {
 		a.Log.Warn("hashrate history load failed", "err", err)