Expose granular health indicators to StartOS

Add three new named health checks to the manifest alongside the existing
web check, each implemented as a fetch+parse of /api/health:

  ckpool      — is the stratum server alive? (miners can connect)
  bitcoin     — is Bitcoin Core RPC reachable? (blocks can be submitted)
  submit-gap  — have all block submissions been confirmed by bitcoind?

The web check retains its current role: calling /api/health which returns
503 when any critical subsystem is down, so StartOS marks the overall
service as degraded. The three new checks give operators a drill-down
view of which subsystem is the problem.

Each check returns "starting" for the first 20 s after launch so normal
boot sequencing does not trigger false alarms.

scripts/procedures/healthChecks.ts

@@ -1,10 +1,75 @@
 import { healthUtil, types as T } from "../deps.ts";
 
+const API_BASE = "http://kamado-pool.embassy:8080";
+
+interface HealthPayload {
+  ok: boolean;
+  ckpool: boolean;
+  bitcoin: boolean;
+  submit_gap: number;
+  zmq_stale: boolean;
+  last_error?: string;
+}
+
+async function fetchHealth(): Promise<HealthPayload | null> {
+  try {
+    const res = await fetch(`${API_BASE}/api/health`, {
+      signal: AbortSignal.timeout(5_000),
+    });
+    return await res.json() as HealthPayload;
+  } catch {
+    return null;
+  }
+}
+
+// Returns a "starting" result for the first 20 s after launch, then
+// "failing" so a crashed service doesn't look like it's perpetually booting.
+function apiUnreachable(duration: number, label: string): T.HealthProceduralResult {
+  if (duration < 20_000) {
+    return { status: "starting", message: `${label} is starting…` };
+  }
+  return { status: "failing", message: "Kamado API is unreachable — service may be down." };
+}
+
 export const health: T.ExpectedExports.health = {
+  // Overall reachability: uses the SDK helper so StartOS marks the
+  // service degraded (503) whenever ckpool or bitcoind is down.
   "web": async (effects, duration) => {
-    return healthUtil.checkWebUrl("http://kamado-pool.embassy:8080/api/health")(
-      effects,
-      duration,
-    );
+    return healthUtil.checkWebUrl(`${API_BASE}/api/health`)(effects, duration);
+  },
+
+  // Is the stratum server up and accepting miner connections?
+  "ckpool": async (_effects, duration) => {
+    const h = await fetchHealth();
+    if (!h) return apiUnreachable(duration, "Stratum server");
+    if (h.ckpool) return { status: "passing", message: "Stratum server is running." };
+    const detail = h.last_error ? ` (${h.last_error})` : "";
+    return { status: "failing", message: `Stratum server is not responding${detail}.` };
+  },
+
+  // Can we reach Bitcoin Core via RPC? (Required for block templates and
+  // block submission — if this is down, mining produces no valid work.)
+  "bitcoin": async (_effects, duration) => {
+    const h = await fetchHealth();
+    if (!h) return apiUnreachable(duration, "Bitcoin Core connection");
+    if (h.bitcoin) return { status: "passing", message: "Connected to Bitcoin Core." };
+    const detail = h.last_error ? ` (${h.last_error})` : "";
+    return { status: "failing", message: `Bitcoin Core RPC is unreachable${detail}.` };
+  },
+
+  // Did every block submission get confirmed by bitcoind? A non-zero gap
+  // means at least one "Possible block solve" never got a "Solved and
+  // confirmed" response — the reward may have been lost or delayed.
+  "submit-gap": async (_effects, duration) => {
+    const h = await fetchHealth();
+    if (!h) return apiUnreachable(duration, "Block submission check");
+    const gap = h.submit_gap ?? 0;
+    if (gap === 0) return { status: "passing", message: "All block submissions confirmed." };
+    return {
+      status: "failing",
+      message:
+        `${gap} block${gap === 1 ? "" : "s"} submitted to bitcoind but not confirmed — ` +
+        `check Bitcoin Core logs for rejected or missing submissions.`,
+    };
   },
 };