Files
satoshi a4a894e196 P1 reliability + block-broadcast fallback path
P1 audits / fixes:

* Bitcoin Core RPC now retries up to 3 times with linear backoff on
  transport errors, 5xx responses, and warm-up/loading RPC errors
  (code -28). Hard "no" answers (block-not-found etc.) bubble up
  immediately so we don't mask real errors.

* WebSocket hub disconnects clients that miss 6 consecutive broadcasts
  (~30s with the default poll cadence). Stuck readers no longer hold
  stale snapshots indefinitely or freeze hub state.

* ZMQ subscriber freshness: aggregator records the last-event
  timestamp, surfaces zmq_enabled / has_last_zmq_event /
  last_zmq_event_age in the snapshot. /healthz flags zmq_stale when
  the gap exceeds 30 minutes.

* /healthz expanded with submit_attempts / submits_confirmed /
  submit_gap, fallback_submits_total + last_fallback_*, and the zmq
  staleness check. Now usable as a real-world ops dashboard signal.

Block-broadcast fallback (new feature):

  * ckpool patch 0004: hooks local_block_submit to write the raw block
    hex to <logdir>/pending-blocks/<height>-<hash16>.hex right before
    invoking generator_submitblock. Unlinks on success. ckpool's normal
    flow is otherwise untouched.

  * api/internal/blocksubmit: watcher polls the dir every 5s. Files
    sitting longer than the grace window (default 30s, configurable)
    are re-broadcast through operator-supplied backup RPC URLs in
    sequence. Treats both null and any "duplicate*" reject reason as
    success (the block landed). Pre-checks the primary chain first so
    a stale file from a successful-but-unlinked submit gets cleaned
    up without bothering fallbacks.

  * Aggregator records each successful fallback submission as a
    persistent counter and surfaces it in the snapshot so the UI can
    show a "primary bitcoind isn't accepting submits" alert.

  * Config: BACKUP_RPC_URLS (comma- or newline-separated, with
    optional inline credentials) plus PENDING_BLOCKS_DIR and
    PENDING_BLOCKS_GRACE. URLs are parsed via net/url so
    https://user:pass@host:port/ works cleanly.

The fallback is opt-in and disabled by default. Once enabled with at
least one URL, a primary bitcoind outage at the moment of solving no
longer means a lost block — kamado-api re-broadcasts via whichever
backup the operator trusts (a second self-hosted node, an
authenticated public RPC service, etc.).
2026-04-27 21:25:56 +03:00

313 lines
8.2 KiB
Go

// Minimal RFC 6455 WebSocket server implementation, stdlib-only.
//
// Scope: push JSON snapshots to subscribed clients. We never expect
// client payloads larger than a ping/pong and we don't negotiate
// extensions (compression, fragmentation). Anything fancier is out of
// scope for Phase 2b — upgrade to a real ws library once we can run
// `go mod tidy` in a real build environment.
package httpapi
import (
"bufio"
"crypto/sha1"
"encoding/base64"
"encoding/binary"
"encoding/json"
"errors"
"io"
"net"
"net/http"
"strings"
"sync"
"time"
"github.com/kamadopool/kamado-api/internal/state"
)
// wsGUID is the fixed magic string from RFC 6455 §1.3 used to compute
// Sec-WebSocket-Accept.
const wsGUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
// WebSocket frame opcodes we care about.
const (
opText byte = 0x1
opBinary byte = 0x2
opClose byte = 0x8
opPing byte = 0x9
opPong byte = 0xA
)
// wsClient is one connected WebSocket subscriber.
type wsClient struct {
conn net.Conn
send chan []byte // serialized JSON frames pending write
writeMu sync.Mutex // guards writes to conn (writer + reader-pong)
// dropMisses counts consecutive Broadcast calls where this client's
// send channel was full. After maxDropMisses we close the connection
// instead of letting a stuck reader hold an old snapshot forever.
dropMisses int
}
// maxDropMisses bounds how many back-to-back broadcasts we let the
// hub skip for one client before forcibly disconnecting it. With the
// poll cadence at 5s this is roughly 30s of unresponsiveness.
const maxDropMisses = 6
// writeFrameLocked writes one frame, serializing writers on the client.
func (c *wsClient) writeFrameLocked(opcode byte, payload []byte) error {
c.writeMu.Lock()
defer c.writeMu.Unlock()
_ = c.conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
return writeFrame(c.conn, opcode, payload)
}
// Hub fans out snapshot updates to all subscribed WebSocket clients.
// Register via Add / Remove from the websocket handler; Broadcast is
// called from the aggregator's OnRefresh hook.
type Hub struct {
mu sync.RWMutex
clients map[*wsClient]struct{}
}
func NewHub() *Hub { return &Hub{clients: make(map[*wsClient]struct{})} }
func (h *Hub) add(c *wsClient) {
h.mu.Lock()
h.clients[c] = struct{}{}
h.mu.Unlock()
}
func (h *Hub) remove(c *wsClient) {
h.mu.Lock()
if _, ok := h.clients[c]; ok {
delete(h.clients, c)
close(c.send)
}
h.mu.Unlock()
}
// Broadcast serializes the snapshot once and enqueues it for every
// subscribed client. Slow clients are dropped rather than blocking
// the hub. After maxDropMisses consecutive drops for a single client,
// we forcibly close its connection so a stuck consumer doesn't hold
// resources indefinitely or freeze on a stale snapshot.
func (h *Hub) Broadcast(snap state.Snapshot) {
payload, err := json.Marshal(snap)
if err != nil {
return
}
var toClose []*wsClient
h.mu.Lock()
for c := range h.clients {
select {
case c.send <- payload:
c.dropMisses = 0
default:
c.dropMisses++
if c.dropMisses >= maxDropMisses {
toClose = append(toClose, c)
}
}
}
h.mu.Unlock()
for _, c := range toClose {
// Closing the conn unblocks the writer goroutine, which
// removes the client from the hub through wsReader's defer.
_ = c.conn.Close()
}
}
// handleWS upgrades an HTTP request to a WebSocket connection and
// subscribes it to the hub. On error before the hijack, it writes a
// plain HTTP 4xx.
func (s *Server) handleWS(w http.ResponseWriter, r *http.Request) {
if !strings.EqualFold(r.Header.Get("Upgrade"), "websocket") ||
!strings.Contains(strings.ToLower(r.Header.Get("Connection")), "upgrade") {
http.Error(w, "expected websocket upgrade", http.StatusBadRequest)
return
}
if r.Header.Get("Sec-WebSocket-Version") != "13" {
w.Header().Set("Sec-WebSocket-Version", "13")
http.Error(w, "unsupported websocket version", http.StatusUpgradeRequired)
return
}
key := r.Header.Get("Sec-WebSocket-Key")
if key == "" {
http.Error(w, "missing Sec-WebSocket-Key", http.StatusBadRequest)
return
}
hj, ok := w.(http.Hijacker)
if !ok {
http.Error(w, "hijack unsupported", http.StatusInternalServerError)
return
}
conn, brw, err := hj.Hijack()
if err != nil {
http.Error(w, "hijack failed", http.StatusInternalServerError)
return
}
// Write handshake response directly to the bufio writer.
accept := wsAcceptKey(key)
resp := "HTTP/1.1 101 Switching Protocols\r\n" +
"Upgrade: websocket\r\n" +
"Connection: Upgrade\r\n" +
"Sec-WebSocket-Accept: " + accept + "\r\n\r\n"
if _, err := brw.WriteString(resp); err != nil {
_ = conn.Close()
return
}
if err := brw.Flush(); err != nil {
_ = conn.Close()
return
}
client := &wsClient{conn: conn, send: make(chan []byte, 8)}
s.Hub.add(client)
s.Log.Info("ws client connected", "remote", conn.RemoteAddr())
// Immediately push the current snapshot so the UI doesn't wait for
// the next tick.
if payload, err := json.Marshal(s.Agg.Snapshot()); err == nil {
select {
case client.send <- payload:
default:
}
}
go s.wsWriter(client)
go s.wsReader(client, brw.Reader)
}
// wsWriter pumps queued payloads as text frames until send is closed
// or a write fails.
func (s *Server) wsWriter(c *wsClient) {
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
for {
select {
case payload, ok := <-c.send:
if !ok {
_ = c.writeFrameLocked(opClose, nil)
_ = c.conn.Close()
return
}
if err := c.writeFrameLocked(opText, payload); err != nil {
_ = c.conn.Close()
return
}
case <-ticker.C:
if err := c.writeFrameLocked(opPing, nil); err != nil {
_ = c.conn.Close()
return
}
}
}
}
// wsReader reads client frames mainly to notice close/ping and to
// drive cleanup when the connection dies. Ignores any app-level
// content since the protocol is server-push only.
func (s *Server) wsReader(c *wsClient, br *bufio.Reader) {
defer s.Hub.remove(c)
for {
op, payload, err := readFrame(br)
if err != nil {
return
}
switch op {
case opClose:
return
case opPing:
_ = c.writeFrameLocked(opPong, payload)
}
}
}
// wsAcceptKey computes Sec-WebSocket-Accept from the client's key per
// RFC 6455 §4.2.2.
func wsAcceptKey(key string) string {
h := sha1.New()
_, _ = io.WriteString(h, key+wsGUID)
return base64.StdEncoding.EncodeToString(h.Sum(nil))
}
// writeFrame writes a single unmasked server frame. Only supports
// payloads up to 2^63-1 bytes which is more than we'll ever send.
func writeFrame(conn net.Conn, opcode byte, payload []byte) error {
var hdr [10]byte
hdr[0] = 0x80 | (opcode & 0x0F) // FIN=1
n := len(payload)
var hdrLen int
switch {
case n < 126:
hdr[1] = byte(n)
hdrLen = 2
case n < 1<<16:
hdr[1] = 126
binary.BigEndian.PutUint16(hdr[2:4], uint16(n))
hdrLen = 4
default:
hdr[1] = 127
binary.BigEndian.PutUint64(hdr[2:10], uint64(n))
hdrLen = 10
}
if _, err := conn.Write(hdr[:hdrLen]); err != nil {
return err
}
if n > 0 {
if _, err := conn.Write(payload); err != nil {
return err
}
}
return nil
}
// readFrame reads one client frame. Clients MUST mask (RFC 6455 §5.3);
// we enforce that and unmask in place.
func readFrame(br *bufio.Reader) (byte, []byte, error) {
var h [2]byte
if _, err := io.ReadFull(br, h[:]); err != nil {
return 0, nil, err
}
opcode := h[0] & 0x0F
masked := h[1]&0x80 != 0
if !masked {
return 0, nil, errors.New("ws: client frame not masked")
}
length := int64(h[1] & 0x7F)
switch length {
case 126:
var ext [2]byte
if _, err := io.ReadFull(br, ext[:]); err != nil {
return 0, nil, err
}
length = int64(binary.BigEndian.Uint16(ext[:]))
case 127:
var ext [8]byte
if _, err := io.ReadFull(br, ext[:]); err != nil {
return 0, nil, err
}
length = int64(binary.BigEndian.Uint64(ext[:]))
}
// Sanity cap on client payloads; we never expect real data from
// clients.
if length > 1<<20 {
return 0, nil, errors.New("ws: client frame too large")
}
var mask [4]byte
if _, err := io.ReadFull(br, mask[:]); err != nil {
return 0, nil, err
}
payload := make([]byte, length)
if _, err := io.ReadFull(br, payload); err != nil {
return 0, nil, err
}
for i := range payload {
payload[i] ^= mask[i%4]
}
return opcode, payload, nil
}