Tighten fallback latency + alert UI on degraded states
Submit-first ordering. Patch 0004 now calls generator_submitblock
BEFORE writing the pending-block hex to disk. The happy path adds zero
disk I/O — we only dump when the primary returned false. The same
patch bounds generator_submitblock's "no live current_si" spin to
~3s instead of the original infinite loop, so a permanently-down
primary doesn't pin the stratifier; the bounded spin lets the caller
return false and lets local_block_submit dump for kamado-api to take
over.
Default grace lowered from 30s to 3s. With ckpool's bounded spin and
sub-second sweep cadence, the fallback now reacts within ~4s of a
failed primary submit — fast enough that the work is still relevant
for the current chain tip. The submitter's sweep poll dropped to 1s
to match.
UI HealthBanners. New top-of-page strip surfaces:
* Fallback used (red banner, 24h after most recent event):
"primary bitcoind didn't accept; backup X took over Y ago"
* Submit gap (orange banner, only when no recent fallback):
"N blocks attempted but unconfirmed — configure backups"
* ZMQ stale (orange banner): no hashblock frame in 30+ minutes
Operators see degraded-but-not-fatal states without checking logs.
Startup readiness gate. main now waits up to 8s on agg.Ready() before
starting the HTTP server so the very first /api/snapshot doesn't show
all-zero state during the aggregator's first refresh. Capped so a
permanently-down bitcoind can't block startup; /healthz is honest
about the degraded state once we do start serving.
This commit is contained in:
@@ -0,0 +1,130 @@
|
||||
<script lang="ts">
|
||||
import { snap } from "../stores/snapshot.svelte";
|
||||
import { formatAgo } from "../format";
|
||||
|
||||
// submit_gap > 0 means ckpool tried to submit at least one block that
|
||||
// never got the "Solved and confirmed" follow-up — so either bitcoind
|
||||
// rejected the submission or the RPC dropped. The number stays > 0
|
||||
// forever after such an event (these are persistent counters), so we
|
||||
// only show it as a banner when the fallback hasn't covered for it
|
||||
// OR there's been a recent fallback the operator should investigate.
|
||||
const submitGap = $derived.by(() => {
|
||||
const d = snap.data;
|
||||
if (!d) return 0;
|
||||
return Math.max(0, (d.block_submit_attempts ?? 0) - (d.block_submits_confirmed ?? 0));
|
||||
});
|
||||
|
||||
const fallbackCount = $derived(snap.data?.fallback_submits_total ?? 0);
|
||||
const lastFallbackAt = $derived(snap.data?.last_fallback_submit_at ?? 0);
|
||||
const lastFallbackVia = $derived(snap.data?.last_fallback_via ?? "");
|
||||
|
||||
// ZMQ stale = configured but no event in 30+ minutes. Bitcoin's avg
|
||||
// block interval is 10 min; 30 min covers normal variance without
|
||||
// false-alarming on quiet stretches.
|
||||
const zmqStale = $derived.by(() => {
|
||||
const d = snap.data;
|
||||
if (!d || !d.zmq_enabled || !d.has_last_zmq_event) return false;
|
||||
return (d.last_zmq_event_age ?? 0) > 1800;
|
||||
});
|
||||
|
||||
// Show the fallback banner for 24h after the most recent fallback
|
||||
// event so the operator sees the alert during their next check-in
|
||||
// even if the underlying problem auto-resolved.
|
||||
const fallbackRecent = $derived.by(() => {
|
||||
if (!lastFallbackAt) return false;
|
||||
const ageSec = Date.now() / 1000 - lastFallbackAt;
|
||||
return ageSec >= 0 && ageSec < 86400;
|
||||
});
|
||||
</script>
|
||||
|
||||
{#if submitGap > 0 || fallbackRecent || zmqStale}
|
||||
<div class="banners">
|
||||
{#if fallbackRecent}
|
||||
<div class="banner alert">
|
||||
<span class="icon">!</span>
|
||||
<div class="text">
|
||||
<strong>Fallback broadcaster used</strong>
|
||||
—
|
||||
our primary bitcoind didn't accept a block submission. Backup
|
||||
RPC <code>{lastFallbackVia}</code> took over
|
||||
{formatAgo(lastFallbackAt)}. Total fallbacks since first run: {fallbackCount}.
|
||||
Investigate primary bitcoind health.
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
{#if submitGap > 0 && !fallbackRecent}
|
||||
<div class="banner warn">
|
||||
<span class="icon">?</span>
|
||||
<div class="text">
|
||||
<strong>Submit gap:</strong>
|
||||
{submitGap} block{submitGap === 1 ? "" : "s"} attempted but not
|
||||
confirmed by bitcoind. Either rejected at submission or the
|
||||
RPC dropped. Configure backup RPC URLs to enable automatic
|
||||
fallback broadcast.
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
{#if zmqStale}
|
||||
<div class="banner warn">
|
||||
<span class="icon">~</span>
|
||||
<div class="text">
|
||||
<strong>ZMQ subscriber stale.</strong>
|
||||
No <code>hashblock</code> frame from bitcoind in
|
||||
{formatAgo(Date.now() / 1000 - (snap.data?.last_zmq_event_age ?? 0))}.
|
||||
Tip changes will fall back to slower polling; check that
|
||||
bitcoind is reachable on its ZMQ port.
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<style>
|
||||
.banners {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
.banner {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 0.6rem;
|
||||
padding: 0.65rem 0.85rem;
|
||||
border-radius: 6px;
|
||||
border: 1px solid transparent;
|
||||
line-height: 1.4;
|
||||
}
|
||||
.banner.alert {
|
||||
background: rgba(220, 80, 80, 0.10);
|
||||
border-color: rgba(220, 80, 80, 0.40);
|
||||
color: rgb(230, 130, 130);
|
||||
}
|
||||
.banner.warn {
|
||||
background: rgba(220, 170, 60, 0.10);
|
||||
border-color: rgba(220, 170, 60, 0.40);
|
||||
color: rgb(220, 180, 100);
|
||||
}
|
||||
.icon {
|
||||
flex: 0 0 auto;
|
||||
width: 1.5em;
|
||||
height: 1.5em;
|
||||
border-radius: 50%;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-weight: 700;
|
||||
font-size: 0.9em;
|
||||
background: currentColor;
|
||||
color: var(--bg, #111);
|
||||
}
|
||||
.text {
|
||||
flex: 1 1 auto;
|
||||
font-size: 0.92em;
|
||||
}
|
||||
.banner code {
|
||||
font-size: 0.9em;
|
||||
padding: 0.05em 0.3em;
|
||||
border-radius: 3px;
|
||||
background: rgba(255, 255, 255, 0.07);
|
||||
}
|
||||
</style>
|
||||
Reference in New Issue
Block a user