Files
KamadoPool/api/internal/bitcoind/rpc.go
T
satoshi a4a894e196 P1 reliability + block-broadcast fallback path
P1 audits / fixes:

* Bitcoin Core RPC now retries up to 3 times with linear backoff on
  transport errors, 5xx responses, and warm-up/loading RPC errors
  (code -28). Hard "no" answers (block-not-found etc.) bubble up
  immediately so we don't mask real errors.

* WebSocket hub disconnects clients that miss 6 consecutive broadcasts
  (~30s with the default poll cadence). Stuck readers no longer hold
  stale snapshots indefinitely or freeze hub state.

* ZMQ subscriber freshness: aggregator records the last-event
  timestamp, surfaces zmq_enabled / has_last_zmq_event /
  last_zmq_event_age in the snapshot. /healthz flags zmq_stale when
  the gap exceeds 30 minutes.

* /healthz expanded with submit_attempts / submits_confirmed /
  submit_gap, fallback_submits_total + last_fallback_*, and the zmq
  staleness check. Now usable as a real-world ops dashboard signal.

Block-broadcast fallback (new feature):

  * ckpool patch 0004: hooks local_block_submit to write the raw block
    hex to <logdir>/pending-blocks/<height>-<hash16>.hex right before
    invoking generator_submitblock. Unlinks on success. ckpool's normal
    flow is otherwise untouched.

  * api/internal/blocksubmit: watcher polls the dir every 5s. Files
    sitting longer than the grace window (default 30s, configurable)
    are re-broadcast through operator-supplied backup RPC URLs in
    sequence. Treats both null and any "duplicate*" reject reason as
    success (the block landed). Pre-checks the primary chain first so
    a stale file from a successful-but-unlinked submit gets cleaned
    up without bothering fallbacks.

  * Aggregator records each successful fallback submission as a
    persistent counter and surfaces it in the snapshot so the UI can
    show a "primary bitcoind isn't accepting submits" alert.

  * Config: BACKUP_RPC_URLS (comma- or newline-separated, with
    optional inline credentials) plus PENDING_BLOCKS_DIR and
    PENDING_BLOCKS_GRACE. URLs are parsed via net/url so
    https://user:pass@host:port/ works cleanly.

The fallback is opt-in and disabled by default. Once enabled with at
least one URL, a primary bitcoind outage at the moment of solving no
longer means a lost block — kamado-api re-broadcasts via whichever
backup the operator trusts (a second self-hosted node, an
authenticated public RPC service, etc.).
2026-04-27 21:25:56 +03:00

279 lines
7.9 KiB
Go

// Package bitcoind provides a minimal Bitcoin Core JSON-RPC client.
// Only the methods kamado-api actually needs are implemented.
package bitcoind
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
)
type RPC struct {
URL string
User string
Password string
HTTP *http.Client
}
func NewRPC(url, user, password string, timeout time.Duration) *RPC {
return &RPC{
URL: url,
User: user,
Password: password,
HTTP: &http.Client{Timeout: timeout},
}
}
type rpcRequest struct {
JSONRPC string `json:"jsonrpc"`
ID string `json:"id"`
Method string `json:"method"`
Params []any `json:"params"`
}
type rpcResponse struct {
Result json.RawMessage `json:"result"`
Error *rpcError `json:"error"`
ID string `json:"id"`
}
type rpcError struct {
Code int `json:"code"`
Message string `json:"message"`
}
func (e *rpcError) Error() string {
return fmt.Sprintf("bitcoind rpc error %d: %s", e.Code, e.Message)
}
// Call performs a JSON-RPC request and unmarshals the result. Transient
// failures (transport errors, 502/503/504, or RPC errors with an
// "in warmup"/"loading"/"verifying" message that bitcoind returns
// during startup) are retried up to twice with a short backoff. RPC
// errors with semantic codes (e.g. block-not-found) are returned
// immediately because retrying won't change the answer.
func (c *RPC) Call(ctx context.Context, method string, params []any, out any) error {
const maxAttempts = 3
var lastErr error
for attempt := 1; attempt <= maxAttempts; attempt++ {
err := c.callOnce(ctx, method, params, out)
if err == nil {
return nil
}
lastErr = err
if !isRetryable(err) || attempt == maxAttempts || ctx.Err() != nil {
return err
}
backoff := time.Duration(attempt) * 200 * time.Millisecond
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(backoff):
}
}
return lastErr
}
func (c *RPC) callOnce(ctx context.Context, method string, params []any, out any) error {
body, err := json.Marshal(rpcRequest{
JSONRPC: "1.0",
ID: "kamado",
Method: method,
Params: params,
})
if err != nil {
return err
}
req, err := http.NewRequestWithContext(ctx, "POST", c.URL, bytes.NewReader(body))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
req.SetBasicAuth(c.User, c.Password)
resp, err := c.HTTP.Do(req)
if err != nil {
return fmt.Errorf("bitcoind rpc: %w", err)
}
defer resp.Body.Close()
raw, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("bitcoind rpc: read body: %w", err)
}
// 5xx without a parseable body: surface as transport-level error
// so isRetryable() can flag it.
var rr rpcResponse
if jerr := json.Unmarshal(raw, &rr); jerr != nil {
if resp.StatusCode >= 500 {
return fmt.Errorf("bitcoind rpc: %d: %s", resp.StatusCode, truncate(string(raw), 256))
}
return fmt.Errorf("bitcoind rpc: unmarshal (status %d): %w: %s", resp.StatusCode, jerr, truncate(string(raw), 256))
}
if rr.Error != nil {
return rr.Error
}
if out != nil {
return json.Unmarshal(rr.Result, out)
}
return nil
}
func truncate(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n] + "…"
}
// isRetryable distinguishes "the RPC didn't reach a definitive answer
// yet" (transport-level errors, 5xx, bitcoind warm-up/loading) from
// "bitcoind answered, the answer is no" (RPC error with a code, e.g.
// -5 block not found). We retry the first kind and bubble the second.
func isRetryable(err error) bool {
if err == nil {
return false
}
// rpcError values come from bitcoind itself — check for warmup/
// loading messages that mean "ask again in a moment".
var rerr *rpcError
if errors.As(err, &rerr) {
// -28 is RPC_IN_WARMUP per bitcoin/src/rpc/protocol.h
if rerr.Code == -28 {
return true
}
msg := strings.ToLower(rerr.Message)
if strings.Contains(msg, "warming up") ||
strings.Contains(msg, "loading") ||
strings.Contains(msg, "verifying") ||
strings.Contains(msg, "rewinding") ||
strings.Contains(msg, "still busy") {
return true
}
return false
}
// Wrapped transport / 5xx / read errors all flow through fmt.Errorf
// with the "bitcoind rpc:" prefix and no rpcError target.
msg := strings.ToLower(err.Error())
switch {
case strings.Contains(msg, "connection refused"),
strings.Contains(msg, "connection reset"),
strings.Contains(msg, "no such host"),
strings.Contains(msg, "deadline exceeded"),
strings.Contains(msg, "i/o timeout"),
strings.Contains(msg, "eof"),
strings.Contains(msg, "broken pipe"),
strings.Contains(msg, " 502"), strings.Contains(msg, " 503"), strings.Contains(msg, " 504"):
return true
}
return false
}
// ---- typed method wrappers ----
type BlockchainInfo struct {
Chain string `json:"chain"`
Blocks int64 `json:"blocks"`
Headers int64 `json:"headers"`
BestBlockHash string `json:"bestblockhash"`
Difficulty float64 `json:"difficulty"`
MedianTime int64 `json:"mediantime"`
VerificationProgress float64 `json:"verificationprogress"`
InitialBlockDownload bool `json:"initialblockdownload"`
}
func (c *RPC) GetBlockchainInfo(ctx context.Context) (*BlockchainInfo, error) {
var out BlockchainInfo
if err := c.Call(ctx, "getblockchaininfo", nil, &out); err != nil {
return nil, err
}
return &out, nil
}
// GetBlockHash returns the block hash at the given height.
func (c *RPC) GetBlockHash(ctx context.Context, height int64) (string, error) {
var out string
if err := c.Call(ctx, "getblockhash", []any{height}, &out); err != nil {
return "", err
}
return out, nil
}
// BlockVerbose2 is the subset of `getblock <hash> 2` we care about.
// Verbosity 2 expands each tx into its full object so we can read the
// coinbase output values without a second getrawtransaction call.
type BlockVerbose2 struct {
Hash string `json:"hash"`
Height int64 `json:"height"`
Confirmations int64 `json:"confirmations"`
Time int64 `json:"time"`
Tx []BlockTx `json:"tx"`
}
type BlockTx struct {
Txid string `json:"txid"`
Vout []BlockVout `json:"vout"`
}
type BlockVout struct {
Value float64 `json:"value"`
N int `json:"n"`
}
// GetBlock returns a verbose-level-2 block decode for the given hash.
func (c *RPC) GetBlock(ctx context.Context, hash string) (*BlockVerbose2, error) {
var out BlockVerbose2
if err := c.Call(ctx, "getblock", []any{hash, 2}, &out); err != nil {
return nil, err
}
return &out, nil
}
// CoinbaseReward sums all outputs of the first transaction in the block
// (the coinbase). In solo mining this is the full subsidy + fees the
// solving worker receives.
func (b *BlockVerbose2) CoinbaseReward() float64 {
if len(b.Tx) == 0 {
return 0
}
var total float64
for _, vout := range b.Tx[0].Vout {
total += vout.Value
}
return total
}
// NetworkHashPS returns the network hashrate at the given block height.
// `blocks` is a window (default 120). Pass -1 to use the default.
func (c *RPC) GetNetworkHashPS(ctx context.Context, blocks, height int) (float64, error) {
var out float64
if err := c.Call(ctx, "getnetworkhashps", []any{blocks, height}, &out); err != nil {
return 0, err
}
return out, nil
}
// BlockTemplate is the subset of getblocktemplate we care about: the
// coinbase value (subsidy + fees) and height of the next block.
type BlockTemplate struct {
CoinbaseValue int64 `json:"coinbasevalue"` // satoshis
Height int64 `json:"height"`
}
// GetBlockTemplate fetches the next block template with segwit rules.
// The call is relatively expensive; rate-limit callers to ~1/min.
func (c *RPC) GetBlockTemplate(ctx context.Context) (*BlockTemplate, error) {
var out BlockTemplate
params := []any{map[string]any{"rules": []string{"segwit"}}}
if err := c.Call(ctx, "getblocktemplate", params, &out); err != nil {
return nil, err
}
return &out, nil
}