more stability fixes

OpenPuck/OpenPuck.ino

@@ -291,11 +291,15 @@ void loop()
 	NRF_WDT->RR[0] = WDT_RR_RR_Reload;
 	faultDiagBeat(); // loop heartbeat -- the SOF blob reports ms-since-beat so a wedge is visible live
 	faultDiagStackTick(); // per-task stack headroom (self-gated ~1Hz) -- usbd-overflow hypothesis check
+	faultDiagFlightTick(); // flight recorder: refresh vitals + heartbeat crumb + live CDC vitals line
 	hapticStabTask(); // stability-test keepalive buzz (no-op unless armed via WebUSB)
 	// cross-check HFCLK(micros) vs LFCLK(millis) once a second -- cheap, both builds (clone clock diagnostic)
 	clockDiagTick();
 	if (g_dirty) {
 		g_dirty = false;
+		// A bond flash write erases+programs a page with interrupts effectively stalled for ms -- a known
+		// watchdog-hang class on slow-flash clones, so leave a crumb bracketing it in the recorder.
+		faultDiagTrace(FR_SAVE, 0);
 		saveBonds();
 	}
 #if OPK_LOG

OpenPuck/fault_diag.cpp

@@ -3,6 +3,8 @@
 #include <FreeRTOS.h>
 #include <task.h> // uxTaskGetSystemState -> per-task stack high-water (overflow hypothesis check)
 #include <string.h>
+#include "rf_link.h" // g_pollsps/g_relayps/g_crcps/g_norxps/g_rfStallRecover -- flight-recorder vitals
+#include "haptics.h" // g_ringFault, relayPending() -- relay-load vitals
 
 // GPREGRET2 markers. GPREGRET (id 0) is reserved by the Adafruit bootloader for the DFU magic; GPREGRET2 is
 // free for the application and is retained across soft/watchdog/pin reset, cleared only on power-on/brownout.
@@ -139,6 +141,50 @@ const char *faultDiagHangStageStr()
 	return g_hangStage < STAGE_COUNT ? STAGE_STR[g_hangStage] : "n/a";
 }
 
+// ---- flight recorder (data) --------------------------------------------------------------------------------
+// The live ring lives in .noinit so it survives a watchdog reset (like g_hangFrame). At boot we copy the
+// pre-reset trail into g_flightSaved (ordinary RAM -- no need to survive twice) so the live session can start a
+// fresh ring while the console can still re-dump the old trail on demand. Declared here (above faultDiagBoot)
+// because the boot classifier snapshots + dumps it; the push/tick/dump functions are implemented further down.
+struct FRRec {
+	uint32_t ms; // millis() at the push
+	uint16_t arg; // event-specific
+	uint8_t evt; // FR_*
+	uint8_t stage; // loop stage current at the push (g_curStage)
+};
+#define FR_RING 96u
+#define FR_MAGIC 0x464C4954u // "FLIT"
+struct FlightRec {
+	uint32_t magic;
+	uint16_t head; // next write slot
+	uint16_t count; // total pushed this session (saturating) -> "showing last N of M"
+	FRRec ring[FR_RING];
+	// live vitals, refreshed ~4x/s -- the last values captured before a wedge
+	uint32_t vMs; // millis() at last refresh
+	uint32_t loopPerSec; // loop() iterations in the last full second
+	uint32_t heapUsed; // mallinfo().uordblks (bytes) -- trend up = leak/fragmentation
+	uint16_t usbdStackFree, loopStackFree; // words (0 = overflowed)
+	uint16_t pollsps, relayps, crcps, norxps;
+	uint16_t rfHeal, ringFault;
+	uint8_t curStage;
+	uint8_t stallMs; // ms since last loop beat, capped 255
+};
+__attribute__((section(".noinit"))) static volatile struct FlightRec g_flight;
+static struct FlightRec
+	g_flightSaved; // boot-time copy of the pre-reset trail (BSS)
+static bool g_haveSaved = false;
+bool g_vitals = true; // live per-second CDC vitals line (console "VIT" toggles)
+
+static const char *const FR_STR[] = { "none",  "beat",	  "SET",    "GET",
+				      "relay", "rf-up",	  "rf-DN",  "HEAL!",
+				      "mount", "SUSPEND", "resume", "OFF",
+				      "RINGF", "save" };
+#define FR_STR_COUNT ((uint8_t)(sizeof FR_STR / sizeof FR_STR[0]))
+static const char *frEvtStr(uint8_t e)
+{
+	return e < FR_STR_COUNT ? FR_STR[e] : "?";
+}
+
 void faultDiagBoot()
 {
 	uint32_t rr =
@@ -199,6 +245,227 @@ void faultDiagBoot()
 		Serial.printf("# hang PC=0x%08lX LR=0x%08lX\n",
 			      (unsigned long)g_reportHangPC,
 			      (unsigned long)g_reportHangLR);
+
+	// Flight recorder: if the pre-reset trail survived (.noinit), snapshot it into ordinary RAM so the live
+	// session can start a fresh ring while the console can still re-dump the old trail ("FR"). On a hang, dump
+	// it now -- this is the post-mortem of what the firmware was doing in the seconds before it wedged.
+	if (g_flight.magic == FR_MAGIC) {
+		memcpy(&g_flightSaved, (const void *)&g_flight,
+		       sizeof g_flightSaved);
+		g_haveSaved = true;
+	}
+	g_flight.magic = FR_MAGIC; // start this session's trail fresh
+	g_flight.head = 0;
+	g_flight.count = 0;
+	if (isHang)
+		faultDiagDumpFlight();
+}
+
+// ---- flight recorder (implementation; data declared above faultDiagBoot) ----------------------------------
+void faultDiagTrace(uint8_t evt, uint16_t arg)
+{
+	uint32_t pm = __get_PRIMASK();
+	__disable_irq();
+	if (g_flight.magic !=
+	    FR_MAGIC) { // lazy (re)init: power-on garbage or first push this session
+		g_flight.magic = FR_MAGIC;
+		g_flight.head = 0;
+		g_flight.count = 0;
+	}
+	uint16_t h = g_flight.head;
+	g_flight.ring[h].ms = millis();
+	g_flight.ring[h].arg = arg;
+	g_flight.ring[h].evt = evt;
+	g_flight.ring[h].stage = g_curStage;
+	g_flight.head = (uint16_t)((h + 1u) % FR_RING);
+	if (g_flight.count < 0xFFFFu)
+		g_flight.count++;
+	if (!pm)
+		__enable_irq();
+}
+
+void faultDiagFlightTick(void)
+{
+	static unsigned long beatMs = 0, secMs = 0;
+	static uint32_t loops = 0;
+	loops++;
+	unsigned long now = millis();
+
+	// ~4x/s: refresh the persistent vitals snapshot + drop a heartbeat crumb (its cadence in the trail shows
+	// exactly when loop() stopped advancing).
+	if ((uint32_t)(now - beatMs) >= 250u) {
+		beatMs = now;
+		uint32_t stall = faultDiagStallMs();
+		g_flight.vMs = now;
+		g_flight.usbdStackFree = faultDiagUsbdStackFree();
+		g_flight.loopStackFree = faultDiagLoopStackFree();
+		// NOTE: mallinfo() removed here -- it suspended the FreeRTOS scheduler + walked the heap free list 4x/s,
+		// which is new loop latency and a new wedge surface on a slow clone, for zero payoff (heap was flat). If
+		// a heap-usage trend is ever needed, sample it lazily from loop, not on this hot path.
+		g_flight.heapUsed = 0;
+		g_flight.pollsps = g_pollsps;
+		g_flight.relayps = g_relayps;
+		g_flight.crcps = g_crcps;
+		g_flight.norxps = g_norxps;
+		g_flight.rfHeal = g_rfStallRecover;
+		g_flight.ringFault = g_ringFault;
+		g_flight.curStage = g_curStage;
+		g_flight.stallMs = (uint8_t)(stall > 255 ? 255 : stall);
+		faultDiagTrace(FR_BEAT, (uint16_t)g_flight.stallMs);
+	}
+
+	// once per second: latch loop rate + emit the live CDC vitals line (loop context -> Serial is safe here,
+	// unlike the usbd task). Reads as a trend on a flaky board: usbd stack falling toward 0 = the overflow;
+	// heapUsed climbing = a leak; poll==0 = the RF loop stalled; relay high = Steam flooding the link.
+	if ((uint32_t)(now - secMs) >= 1000u) {
+		g_flight.loopPerSec = loops;
+		loops = 0;
+		secMs = now;
+		// Guard on FIFO space: a CDC write to a host that has the port open but isn't draining blocks the loop --
+		// itself a watchdog-hang source (same failure the WebUSB blob send hit). If there's no room this second,
+		// skip the line rather than stall; the persistent trail still captured the vitals via the beat above.
+		if (g_vitals && Serial && Serial.availableForWrite() >= 200) {
+			// Clock fingerprint on every line: the radio waits time out on micros() (HFCLK-derived). If a clone's
+			// HFCLK drops to the internal RC (or micros() freezes), usPerMs drifts off 1000 and a "bounded" RX/
+			// disable wait can spin forever -- the prime suspect for a healthy-then-sudden hard lock. lf/hf: R=RC,
+			// X=xtal, S=synth, -=stopped.
+			const char clkc[] = { '-', 'R', 'X', 'S' };
+			uint8_t lf = clockLfSrc(), hf = clockHfSrc();
+			Serial.printf(
+				"# vit up=%lus loop=%lu/s stall=%ums stage=%s usbdStk=%u loopStk=%u heapUsed=%lu "
+				"poll=%u relay=%u crc=%u norx=%u heal=%u ringF=%u clk=%c/%c usPerMs=%u\n",
+				(unsigned long)(now / 1000),
+				(unsigned long)g_flight.loopPerSec,
+				(unsigned)g_flight.stallMs,
+				faultDiagStageStr(g_flight.curStage),
+				(unsigned)g_flight.usbdStackFree,
+				(unsigned)g_flight.loopStackFree,
+				(unsigned long)g_flight.heapUsed,
+				(unsigned)g_flight.pollsps,
+				(unsigned)g_flight.relayps,
+				(unsigned)g_flight.crcps,
+				(unsigned)g_flight.norxps,
+				(unsigned)g_flight.rfHeal,
+				(unsigned)g_flight.ringFault,
+				lf < 4 ? clkc[lf] : '?',
+				hf < 4 ? clkc[hf] : '?',
+				(unsigned)clockUsPerMs());
+		}
+	}
+}
+
+// Wait (bounded) for room in the CDC TX FIFO so a multi-line dump to a slow host neither drops lines nor
+// blocks forever. Caps each line's wait at ~20ms -> the whole ~100-line dump can't approach the ~8s watchdog.
+static void frWaitTx(uint16_t need)
+{
+	for (uint8_t w = 0;
+	     w < 20 && Serial && Serial.availableForWrite() < need; w++)
+		delay(1);
+}
+
+void faultDiagDumpFlight(void)
+{
+	if (!Serial) // no host attached (e.g. the automatic boot dump before the panel connects) -- use "FR" later
+		return;
+	if (!g_haveSaved || g_flightSaved.magic != FR_MAGIC) {
+		Serial.println(
+			"# flight: no pre-reset trail (recorder lost across this reset, or last boot wasn't a hang)");
+		return;
+	}
+	const struct FlightRec *f = &g_flightSaved;
+	Serial.printf(
+		"# ---- FLIGHT RECORDER (trail up to the last hang) ----\n"
+		"# vitals@wedge: loop=%lu/s stall=%ums stage=%s usbdStkFree=%u loopStkFree=%u heapUsed=%lu\n"
+		"#               poll=%u relay=%u crc=%u norx=%u heal=%u ringFault=%u  (age %lums before reset)\n",
+		(unsigned long)f->loopPerSec, (unsigned)f->stallMs,
+		faultDiagStageStr(f->curStage), (unsigned)f->usbdStackFree,
+		(unsigned)f->loopStackFree, (unsigned long)f->heapUsed,
+		(unsigned)f->pollsps, (unsigned)f->relayps, (unsigned)f->crcps,
+		(unsigned)f->norxps, (unsigned)f->rfHeal,
+		(unsigned)f->ringFault, (unsigned long)0);
+
+	uint16_t total = f->count;
+	uint16_t cnt = total < FR_RING ? total : (uint16_t)FR_RING;
+	if (!cnt) {
+		Serial.println("# flight: trail empty");
+		return;
+	}
+	uint16_t start = (uint16_t)((f->head + FR_RING - cnt) % FR_RING);
+	uint16_t lastIdx = (uint16_t)((f->head + FR_RING - 1u) % FR_RING);
+	uint32_t last = f->ring[lastIdx].ms;
+	Serial.printf(
+		"# %u events (of %u total); t = ms BEFORE the last recorded crumb:\n",
+		cnt, total);
+	for (uint16_t i = 0; i < cnt; i++) {
+		uint16_t idx = (uint16_t)((start + i) % FR_RING);
+		const FRRec *r = &f->ring[idx];
+		frWaitTx(64);
+		Serial.printf("#  -%6lu  %-8s stage=%-9s arg=0x%04X\n",
+			      (unsigned long)(last - r->ms), frEvtStr(r->evt),
+			      faultDiagStageStr(r->stage), r->arg);
+	}
+	Serial.println("# ---- end flight recorder ----");
+}
+
+// ---- WebUSB panel access to the saved trail ----------------------------------------------------------------
+static uint16_t g_frDrain =
+	0; // pull cursor into g_flightSaved, 0 = oldest saved event
+
+bool faultDiagHaveFlight(void)
+{
+	return g_haveSaved && g_flightSaved.magic == FR_MAGIC;
+}
+uint16_t faultDiagFlightCount(void)
+{
+	if (!faultDiagHaveFlight())
+		return 0;
+	uint16_t t = g_flightSaved.count;
+	return t < FR_RING ? t : (uint16_t)FR_RING;
+}
+uint16_t faultDiagFlightTotal(void)
+{
+	return faultDiagHaveFlight() ? g_flightSaved.count : 0;
+}
+void faultDiagFlightDrainReset(void)
+{
+	g_frDrain = 0;
+}
+bool faultDiagFlightPull(uint32_t *dtMs, uint8_t *evt, uint8_t *stage,
+			 uint16_t *arg)
+{
+	uint16_t cnt = faultDiagFlightCount();
+	if (g_frDrain >= cnt)
+		return false;
+	const struct FlightRec *f = &g_flightSaved;
+	uint16_t start = (uint16_t)((f->head + FR_RING - cnt) % FR_RING);
+	uint16_t idx = (uint16_t)((start + g_frDrain) % FR_RING);
+	uint16_t lastIdx = (uint16_t)((f->head + FR_RING - 1u) % FR_RING);
+	*dtMs = (uint32_t)(f->ring[lastIdx].ms - f->ring[idx].ms);
+	*evt = f->ring[idx].evt;
+	*stage = f->ring[idx].stage;
+	*arg = f->ring[idx].arg;
+	g_frDrain++;
+	return true;
+}
+bool faultDiagFlightVitals(struct FaultFlightVitals *out)
+{
+	if (!faultDiagHaveFlight())
+		return false;
+	const struct FlightRec *f = &g_flightSaved;
+	out->loopPerSec =
+		(uint16_t)(f->loopPerSec > 0xFFFFu ? 0xFFFFu : f->loopPerSec);
+	out->usbdStkFree = f->usbdStackFree;
+	out->loopStkFree = f->loopStackFree;
+	out->heapUsed = f->heapUsed;
+	out->pollsps = f->pollsps;
+	out->relayps = f->relayps;
+	out->crcps = f->crcps;
+	out->norxps = f->norxps;
+	out->rfHeal = f->rfHeal;
+	out->ringFault = f->ringFault;
+	out->curStage = f->curStage;
+	out->stallMs = f->stallMs;
+	return true;
 }
 
 // ---- clock fingerprint -------------------------------------------------------------------------------------

OpenPuck/fault_diag.h

@@ -75,6 +75,67 @@ uint8_t faultDiagReason();
 uint32_t faultDiagResetReas();
 const char *faultDiagReasonStr();
 
+// ---- flight recorder (catch a watchdog hang post-mortem) -------------------------------------------------
+// A .noinit ring of recent high-level events + a live vitals snapshot, BOTH surviving a watchdog reset (same
+// mechanism as the hang PC frame: .noinit RAM isn't zeroed by the C runtime, so it rides through the reset even
+// on clone boards whose bootloader wipes GPREGRET2). A hang drops USB the instant it resets, so live CDC
+// logging can never catch the moment itself -- instead we record the trail continuously and DUMP IT ON THE NEXT
+// BOOT when the last reset was a hang. Every push is a few stores under PRIMASK: safe to call from the fragile
+// 800B usbd task and from ISRs (NO printf, NO allocation). This is the "what was going on when it wedged" the
+// cheap boards (Teyleten hangs ~every 4 min) need. `arg` is event-specific, usually packed (high<<8)|low.
+enum {
+	FR_NONE = 0,
+	FR_BEAT, // periodic loop heartbeat (arg = stall ms) -- the gap between beats localizes the wedge in time
+	FR_SET, // usbd handleSet feature write     (arg = (rid<<8)|cmd)
+	FR_GET, // usbd handleGet feature read      (arg = (rid<<8)|cmd)
+	FR_RELAY, // host->controller relay enqueued (arg = (slot<<8)|rid)
+	FR_RFUP, // RF (re)connect                   (arg = slot)
+	FR_RFDN, // RF link down                     (arg = slot)
+	FR_HEAL, // RF-stall radio self-heal (power-cycle)
+	FR_MOUNT, // USB HID mount/unmount changed   (arg = mounted-slot count)
+	FR_SUSP, // USB bus suspend edge
+	FR_RESUME, // USB bus resume edge
+	FR_OFF, // controller power-off (0x9F) relayed (arg = slot)
+	FR_RINGF, // relay-ring fault caught          (arg = running count)
+	FR_SAVE, // bond flash write (saveBonds)
+};
+
+// Push one event into the flight recorder. Cheap + ISR/usbd-safe (a few stores under PRIMASK, no printf).
+void faultDiagTrace(uint8_t evt, uint16_t arg);
+// Per-loop upkeep: refresh the persistent vitals snapshot + emit a heartbeat (~4x/s), and stream the live
+// per-second vitals line over CDC when g_vitals is on. Call once per loop() iteration.
+void faultDiagFlightTick(void);
+// Print the pre-reset flight trail + vitals-at-wedge over CDC (from the boot-time copy of the surviving ring).
+// Called automatically from faultDiagBoot() after a hang; also re-dumpable on demand from the console ("FR").
+void faultDiagDumpFlight(void);
+
+// ---- WebUSB panel access to the saved (pre-reset) trail ---------------------------------------------------
+// The same boot-time copy the console "FR" dumps, exposed for the panel to stream as 0xA8 frames. Mirrors the
+// hapLog capture drain: reset the cursor to the oldest saved event, then pull entries until exhausted, so a
+// full 96-event trail streams across several panel polls without a single blocking write.
+bool faultDiagHaveFlight(
+	void); // true if a pre-reset trail was captured at this boot
+uint16_t faultDiagFlightCount(void); // saved events available to stream (<= 96)
+uint16_t faultDiagFlightTotal(
+	void); // total events pushed before the reset (for "showing N of M")
+void faultDiagFlightDrainReset(
+	void); // rewind the pull cursor to the oldest saved event
+// Pull the next saved event; false when exhausted. *dtMs = ms the event fired BEFORE the last recorded crumb.
+bool faultDiagFlightPull(uint32_t *dtMs, uint8_t *evt, uint8_t *stage,
+			 uint16_t *arg);
+// Vitals as of the last ~4Hz refresh before the reset (the "@wedge" snapshot). false if no trail was captured.
+struct FaultFlightVitals {
+	uint16_t loopPerSec;
+	uint16_t usbdStkFree, loopStkFree;
+	uint32_t heapUsed;
+	uint16_t pollsps, relayps, crcps, norxps, rfHeal, ringFault;
+	uint8_t curStage, stallMs;
+};
+bool faultDiagFlightVitals(struct FaultFlightVitals *out);
+// 1 = stream the live per-second vitals line over CDC (console "VIT" toggles). Watch usbd stack / heap / relay
+// rate trend on a flaky board in the seconds before it wedges.
+extern bool g_vitals;
+
 // ---- clock fingerprint (clone-board stability diagnostic) -------------------------------------------------
 // nice!nano clones vary in their crystals. The bare-metal radio needs HFXO (32 MHz xtal); millis()/RTC and the
 // watchdog run on LFCLK (32.768 kHz xtal or RC). If a clone runs HFCLK on the internal RC, or its LFCLK is

OpenPuck/haptics.cpp

@@ -4,6 +4,7 @@
 #include "rf_link.h"
 #include "puck_hid.h" // puckLizardActive() -- gate the lizard-suppression keepalive
 
+#include "fault_diag.h" // faultDiagTrace() -- flight recorder
 // USBDevice.suspended() -> autonomous controller power-off on host sleep
 #include <Adafruit_TinyUSB.h>
 #include <Arduino.h>
@@ -51,6 +52,7 @@ void hapticSendShutdown(uint8_t slot)
 	// controller (broadcasting it killed every connected controller when the user turned off one). The
 	// broadcast default (0xFF) remains for the triggers that logically mean "all off": host suspend and
 	// the panel/test power-off button.
+	faultDiagTrace(FR_OFF, slot);
 	for (uint8_t i = 0; i < HAPTIC_SHUTDOWN_SHOTS; i++)
 		relayEnqueue(0x9F, OFF, sizeof OFF, slot);
 }
@@ -125,6 +127,7 @@ bool relayEnqueue(uint8_t rid, const uint8_t *payload, uint8_t plen,
 	if (rid == 0x82 || rid == 0x80)
 		g_haptic82Ms = millis();
 	__set_PRIMASK(pm);
+	faultDiagTrace(FR_RELAY, (uint16_t)((slot << 8) | rid));
 	return true;
 }
 
@@ -259,6 +262,7 @@ static void hapticCancelPendingOn(int slot)
 			if (!guard--) { // desynced/corrupt ring -> recover, don't spin IRQs-off
 				g_rqHead[s] = g_rqTail[s] = 0;
 				g_ringFault++;
+				faultDiagTrace(FR_RINGF, g_ringFault);
 				break;
 			}
 			RelayMsg &m = g_rq[s][i];
@@ -399,6 +403,7 @@ bool rfConnFlushRelay(uint8_t ch, uint8_t s1)
 		if (!guard--) { // desynced/corrupt ring -> recover, don't spin IRQs-off (watchdog-hang class)
 			g_rqHead[cur] = g_rqTail[cur] = 0;
 			g_ringFault++;
+			faultDiagTrace(FR_RINGF, g_ringFault);
 			break;
 		}
 		RelayMsg &m = g_rq[cur][g_rqTail[cur]];
@@ -647,9 +652,13 @@ void hapticTask()
 	if (susp && !wasSusp) {
 		suspSinceMs = millis();
 		suspArmed = true;
+		faultDiagTrace(FR_SUSP, 0);
 	}
-	if (!susp)
+	if (!susp) {
+		if (wasSusp)
+			faultDiagTrace(FR_RESUME, 0);
 		suspArmed = false;
+	}
 	if (suspArmed && vbus && (millis() - suspSinceMs) >= SUSPEND_OFF_MS) {
 		hapticSendShutdown();
 		suspArmed = false; // fire once per suspend

OpenPuck/puck_hid.cpp

@@ -3,6 +3,7 @@
 #include "config.h"
 #include "identity.h"
 #include "haptics.h"
+#include "fault_diag.h"
 #include "rf_link.h"
 #include "triton.h"
 #include "mode_lizard.h"
@@ -270,6 +271,9 @@ static void handleSet(int slot, uint8_t rid, hid_report_type_t type,
 	hapLogAdd((uint8_t)slot, cmd, b, n);
 	// ...and to the serial feature-command capture (diagnostic; drained in loop, see puckCmdLogDrain).
 	fcPush(0, slot, rid, cmd, len, pl, pln > 10 ? 10 : (uint8_t)pln);
+	// ...and to the flight recorder (survives a watchdog reset). This runs ON the fragile usbd task, so the
+	// command flood that (theory) overflows it lands in the post-mortem trail rather than being lost with USB.
+	faultDiagTrace(FR_SET, (uint16_t)((rid << 8) | cmd));
 
 	// settings/haptic/LED report (incl. 0x87 lizard-off heartbeat, SDL Triton lizard-disable)
 	if (cmd >= 0x80 && cmd <= 0x89)
@@ -550,6 +554,9 @@ static uint16_t handleGet(int slot, uint8_t rid, hid_report_type_t type,
 	if (n > reqlen)
 		n = reqlen;
 	memcpy(buf, S.resp, n);
+	// Flight recorder (every GET, un-deduped): a read STORM -- e.g. the "AE x39" identity hammering seen before
+	// the identity fix -- is itself a wedge signal, so we want it filling the post-mortem trail if it happens.
+	faultDiagTrace(FR_GET, (uint16_t)((rid << 8) | S.resp[0]));
 	// Battery diagnostic: in gamepad (Steam) mode battery is read host-side via the feature channel, NOT the
 	// forwarded 0x43 (that path is verbatim-identical to lizard mode, where battery works). Capture what Steam
 	// GETs so a WebUSB-panel (or CDC) capture in Steam mode shows the report id it polls for battery (then we

OpenPuck/rf_link.cpp

@@ -214,8 +214,8 @@ static void rfHostFrameOnce(int slot, bool discovery)
 		g_rfRxCount++;
 		bool crcok = NRF_RADIO->CRCSTATUS & 1;
 		uint8_t len = rfrx[0];
-		// non-blocking: don't stall the loop on CDC backpressure
-		if (Serial.availableForWrite() > 90) {
+		// non-blocking: don't stall the loop on CDC backpressure (whole line ~165B; CDC write() has no timeout)
+		if (Serial.availableForWrite() > 180) {
 			Serial.printf(
 				"*** RESP#%lu ch%u crc%d rxmatch%lu len%u: ",
 				(unsigned long)g_rfRxCount, g_rfCh, crcok,
@@ -318,7 +318,7 @@ uint8_t rfConnTx(uint8_t ch, uint8_t s1, const uint8_t *payload, uint8_t plen,
 					// Lifecycle log (CDC debug boot): distinguish a first-ever connect from a reconnect and
 					// print the silent gap -- lets a long session of connect/disconnect cycles be diffed to
 					// see whether each cycle re-establishes (churn / boot-haptic click) vs stays linked.
-					if (Serial.availableForWrite() > 50) {
+					if (Serial.availableForWrite() > 130) {
 						uint32_t gap =
 							g_connReplyMs[s] ?
 								(uint32_t)(millis() -
@@ -338,6 +338,7 @@ uint8_t rfConnTx(uint8_t ch, uint8_t s1, const uint8_t *payload, uint8_t plen,
 									g_connCooldown));
 					}
 					hapticOnReconnect(s);
+					faultDiagTrace(FR_RFUP, s);
 				}
 				g_connRx++;
 				// link alive -> loop() suppresses the redundant E1 beacon
@@ -368,12 +369,15 @@ uint8_t rfConnTx(uint8_t ch, uint8_t s1, const uint8_t *payload, uint8_t plen,
 					if (i != g_curSlot && g_slot[i].used &&
 					    millis() - g_connReplyMs[i] < 300)
 						others++;
+				faultDiagTrace(FR_RFDN,
+					       (uint16_t)((g_curSlot << 8) |
+							  (others & 0xFF)));
 				if (others == 0)
 					g_connCooldown = millis();
 				// Lifecycle log: the controller sent F2 (disconnect/power-off). Note whether it armed the
 				// 2.5s cooldown (which pauses ALL beacon+poll -> the controller can lose the session and
 				// reboot on the next reconnect = a boot-haptic click). Prime suspect for the connect buzz.
-				if (Serial.availableForWrite() > 50)
+				if (Serial.availableForWrite() > 130)
 					Serial.printf(
 						"# LC t=%lu slot%d F2 DISCONNECT others=%d%s\n",
 						(unsigned long)millis(),
@@ -654,7 +658,7 @@ uint8_t rfConnTx(uint8_t ch, uint8_t s1, const uint8_t *payload, uint8_t plen,
 						static unsigned long lastUnk =
 							0;
 						if (Serial.availableForWrite() >
-							    110 &&
+							    150 &&
 						    millis() - lastUnk >= 200) {
 							lastUnk = millis();
 							Serial.printf(
@@ -722,7 +726,7 @@ uint8_t rfConnTx(uint8_t ch, uint8_t s1, const uint8_t *payload, uint8_t plen,
 				// Serial.print stalls the RF+USB loop -> jaggy input). One line/frame using the last record.
 				if (lastRep && !g_connVerbose &&
 				    !g_cmdCapture &&
-				    Serial.availableForWrite() > 110 &&
+				    Serial.availableForWrite() > 150 &&
 				    millis() - g_lastStream >= 4) {
 					g_lastStream = millis();
 					Serial.print("I45 ");
@@ -732,7 +736,7 @@ uint8_t rfConnTx(uint8_t ch, uint8_t s1, const uint8_t *payload, uint8_t plen,
 					Serial.println();
 				}
 			}
-			if (g_connVerbose) {
+			if (g_connVerbose && Serial.availableForWrite() > 180) {
 				Serial.printf(
 					"%s CRX#%lu txtype%02X ch%u len%u: ",
 					isF1 ? "<<<F1" :
@@ -963,6 +967,7 @@ void rfLinkTask()
 		    (uint32_t)(nowMs2 - lastRecoverMs) > interval) {
 			lastRecoverMs = nowMs2;
 			g_rfStallRecover++;
+			faultDiagTrace(FR_HEAL, g_rfStallRecover);
 			if (consecStall < 255)
 				consecStall++;
 			NRF_RADIO->POWER = 0;
@@ -970,7 +975,7 @@ void rfLinkTask()
 			g_connCooldown = 0;
 			g_connSt = 0;
 			g_connStep = 0;
-			if (Serial.availableForWrite() > 60)
+			if (Serial.availableForWrite() > 110)
 				Serial.printf(
 					"# RF STALL (#%u consec=%u%s) -> radio power-cycle + reconnect\n",
 					g_rfStallRecover, consecStall,
@@ -1048,7 +1053,11 @@ void rfLinkTask()
 			g_pollDtCnt ? (uint16_t)(g_pollDtSum / g_pollDtCnt) : 0;
 		g_pollDtSum = 0;
 		g_pollDtCnt = 0;
-		if (Serial.availableForWrite() > 70)
+		// Require room for the WHOLE line (~85B): CDC write() has NO timeout -- it spins yield() until the host
+		// drains, so a guard smaller than the line lets write() start, fill the FIFO mid-line, then spin loop()
+		// forever if the serial host stalls -> watchdog hang. (This exact line, guarded at >70 for an ~85B line,
+		// was the confirmed diagnostic-induced hang: the capture ended truncated mid-"# stat".)
+		if (Serial.availableForWrite() > 130)
 			Serial.printf(
 				"# stat polls=%lu/s F1=%lu/s new=%lu/s F3=%lu/s(v%d) e7b=%u crcfail=%lu noRx=%lu slot=%d\n",
 				(unsigned long)tPoll, (unsigned long)tF1,

OpenPuck/serial_console.cpp

@@ -61,6 +61,15 @@ void serialConsolePoll()
 				Serial.printf(
 					"# land-all-0x87 (verbatim 0x87 relay) %s\n",
 					g_landAll87 ? "ON" : "off");
+			} else if (!strcmp(line, "FR")) {
+				// re-dump the flight recorder trail captured before the last watchdog hang (also printed
+				// automatically at boot, but CDC may not be attached yet then -- this reprints on demand).
+				faultDiagDumpFlight();
+			} else if (!strcmp(line, "VIT")) {
+				// toggle the live per-second CDC vitals line (loop rate / usbd stack / heap / relay trend)
+				g_vitals = !g_vitals;
+				Serial.printf("# live vitals line %s\n",
+					      g_vitals ? "ON" : "off");
 			} else if (line[0] == 'l')
 				rfListenStart();
 			else if (line[0] == 'B') {

OpenPuck/usb_mount.cpp

@@ -1,5 +1,6 @@
 #include "usb_mount.h"
 #include "bonds.h"
+#include "fault_diag.h" // faultDiagTrace() -- flight recorder
 #include <Arduino.h>
 
 uint8_t g_usbMountCount = 0;
@@ -147,5 +148,6 @@ void usbMountTask()
 	if (maskOfList(tmp, n) == mountedMask())
 		return; // same set already mounted (order is preserved across joins/leaves)
 	commitOrder(tmp, n);
+	faultDiagTrace(FR_MOUNT, g_usbMountCount);
 	usbReenumerate(g_usbMountCount);
 }

OpenPuck/webusb_config.cpp

@@ -269,6 +269,100 @@ static void webusbSendBondExport()
 	}
 }
 
+// Stream the saved (pre-reset) flight-recorder trail to the panel as 0xA8 frames. Always compiled -- the
+// recorder is not OPK_LOG-gated (field hangs happen on release builds too). Frame layout mirrors the 0xA6
+// capture drain so the same drop-on-full discipline applies (a write only issues when the FIFO can take the
+// whole frame -> usb_web.write() never spins -> loop() never stalls here):
+//   header: [0xA8][29][T=2][count u16][total u16][loopPerSec u16][stallMs][curStage][usbdStk u16][loopStk u16]
+//                        [heapUsed u32][pollsps u16][relayps u16][crc u8][norx u8][heal u16][ringF u16]
+//   entry:  [0xA8][9][T=1][dtMs u32][evt][stage][arg u16]     (dtMs = ms before the last recorded crumb)
+//   end:    [0xA8][1][T=0]
+// The panel sends 0x10 with buf[1]=1 to (re)start (emits the header + rewinds the cursor), then 0x10 buf[1]=0
+// repeatedly; each call streams a budgeted chunk and the final call appends the end frame.
+#define WB_FLIGHT_HDR 29
+static void webusbDrainFlight(bool restart)
+{
+	if (!usb_web.connected())
+		return;
+	if (restart) {
+		faultDiagFlightDrainReset();
+		if (tud_vendor_write_available() >= 2 + WB_FLIGHT_HDR) {
+			struct FaultFlightVitals v;
+			bool have = faultDiagFlightVitals(&v);
+			if (!have)
+				memset(&v, 0, sizeof v);
+			uint16_t cnt = faultDiagFlightCount();
+			uint16_t tot = faultDiagFlightTotal();
+			uint8_t f[2 + WB_FLIGHT_HDR];
+			f[0] = 0xA8;
+			f[1] = WB_FLIGHT_HDR;
+			f[2] = 2; // header type
+			f[3] = (uint8_t)cnt;
+			f[4] = (uint8_t)(cnt >> 8);
+			f[5] = (uint8_t)tot;
+			f[6] = (uint8_t)(tot >> 8);
+			f[7] = (uint8_t)v.loopPerSec;
+			f[8] = (uint8_t)(v.loopPerSec >> 8);
+			f[9] = v.stallMs;
+			f[10] = v.curStage;
+			f[11] = (uint8_t)v.usbdStkFree;
+			f[12] = (uint8_t)(v.usbdStkFree >> 8);
+			f[13] = (uint8_t)v.loopStkFree;
+			f[14] = (uint8_t)(v.loopStkFree >> 8);
+			f[15] = (uint8_t)v.heapUsed;
+			f[16] = (uint8_t)(v.heapUsed >> 8);
+			f[17] = (uint8_t)(v.heapUsed >> 16);
+			f[18] = (uint8_t)(v.heapUsed >> 24);
+			f[19] = (uint8_t)v.pollsps;
+			f[20] = (uint8_t)(v.pollsps >> 8);
+			f[21] = (uint8_t)v.relayps;
+			f[22] = (uint8_t)(v.relayps >> 8);
+			f[23] = (uint8_t)(v.crcps > 255 ? 255 : v.crcps);
+			f[24] = (uint8_t)(v.norxps > 255 ? 255 : v.norxps);
+			f[25] = (uint8_t)v.rfHeal;
+			f[26] = (uint8_t)(v.rfHeal >> 8);
+			f[27] = (uint8_t)v.ringFault;
+			f[28] = (uint8_t)(v.ringFault >> 8);
+			f[29] = (uint8_t)0; // reserved / pad to header len
+			f[30] = (uint8_t)0;
+			usb_web.write(f, sizeof f);
+			usb_web.flush();
+		}
+	}
+	uint32_t dt = 0;
+	uint8_t evt = 0, stage = 0;
+	uint16_t arg = 0;
+	uint16_t budget = 64;
+	bool done = false;
+	while (budget--) {
+		if (tud_vendor_write_available() < 2 + 9)
+			break; // FIFO full -> resume on the panel's next 0x10 poll
+		if (!faultDiagFlightPull(&dt, &evt, &stage, &arg)) {
+			done = true;
+			break;
+		}
+		uint8_t f[2 + 9];
+		f[0] = 0xA8;
+		f[1] = 9;
+		f[2] = 1; // entry type
+		f[3] = (uint8_t)dt;
+		f[4] = (uint8_t)(dt >> 8);
+		f[5] = (uint8_t)(dt >> 16);
+		f[6] = (uint8_t)(dt >> 24);
+		f[7] = evt;
+		f[8] = stage;
+		f[9] = (uint8_t)arg;
+		f[10] = (uint8_t)(arg >> 8);
+		usb_web.write(f, sizeof f);
+		usb_web.flush();
+	}
+	if (done && tud_vendor_write_available() >= 3) {
+		uint8_t e[3] = { 0xA8, 1, 0 };
+		usb_web.write(e, 3);
+		usb_web.flush();
+	}
+}
+
 // Runs on the usbd task (registered via usbTxRegisterDrain -> tud_sof_cb). Sends the blob if loop() asked for
 // one. Keeps every usb_web write/flush off the loop task so it can't block on the device event queue.
 static void webusbSofDrain(void)
@@ -361,16 +455,17 @@ void webusbPoll()
 			if (n == 0)
 				break;
 			uint8_t op = buf[0];
-			if (op < 0x01 || op > 0x0F) { // resync: drop one byte
+			if (op < 0x01 || op > 0x10) { // resync: drop one byte
 				memmove(buf, buf + 1, --n);
 				continue;
 			}
-			// Command length (fixed per opcode). 0x0D = write-one-bond-slot (27 B); 0x05/0x0E/0x0F carry
+			// Command length (fixed per opcode). 0x0D = write-one-bond-slot (27 B); 0x05/0x0E/0x0F/0x10 carry
 			// one value byte; 0x02 a field+value; 0x0A a 3-byte magic.
 			uint8_t need = (op == 0x0D) ? 27 :
 				       (op == 0x02) ? 3 :
 				       (op == 0x03 || op == 0x05 ||
-					op == 0x0E || op == 0x0F) ?
+					op == 0x0E || op == 0x0F ||
+					op == 0x10) ?
 						      2 :
 				       (op == 0x0A) ? 4 :
 						      1;
@@ -394,6 +489,12 @@ void webusbPoll()
 				g_e7b = 0; // byte2=0 => host-awake
 				g_e7announce = g_stabTest;
 			}
+			// 0x10: stream the saved flight-recorder trail (0xA8 frames). buf[1]=1 restarts (header + rewind),
+			// buf[1]=0 continues the drain. Drained straight from loop context like the 0x06 capture drain --
+			// webusbDrainFlight is drop-on-full so it never blocks. Always available (recorder isn't OPK_LOG).
+			else if (op == 0x10) {
+				webusbDrainFlight(buf[1] != 0);
+			}
 #if OPK_LOG
 
 			// rewind drain: buf[1]=1 from boot (whole ring), 0 from now

docs/index.html

@@ -204,6 +204,16 @@
       <pre id="hangOut" style="font:12px ui-monospace,Menlo,monospace;color:var(--fg);background:#0e1017;border:1px solid #232a3a;border-radius:8px;padding:10px;margin:0;max-height:260px;overflow:auto;white-space:pre"></pre>
     </div>
 
+    <div class="card wide">
+      <h2>Flight recorder</h2>
+      <p class="note" style="margin-top:0">The trail of events the firmware recorded in the seconds before its most recent <b>watchdog hang</b> — the ring survives the reset in <code>.noinit</code> RAM (like the hang PC), so it's the post-mortem of what wedged the board. Loads automatically after a hang reconnects; click to reload. Same data the console <code>FR</code> command prints. <span id="flightWedge" style="font-family:ui-monospace,Menlo,monospace"></span></p>
+      <div class="row" style="margin:0 0 8px;flex-wrap:wrap">
+        <span class="v" id="flightSummary" style="flex:1">no trail loaded</span>
+        <button id="flightLoad">Load last-hang trail</button>
+      </div>
+      <pre id="flightOut" style="font:12px ui-monospace,Menlo,monospace;color:var(--fg);background:#0e1017;border:1px solid #232a3a;border-radius:8px;padding:10px;margin:0;max-height:320px;overflow:auto;white-space:pre">—</pre>
+    </div>
+
     <div class="card wide logonly">
       <h2>Capture — host → controller commands</h2>
       <p class="note" style="margin-top:0">The firmware logs everything from boot into a big RAM ring: Steam's writes (<code>ifN</code>), the frames we transmit to the controller (<code>TX→ctlr</code>), and RF <code>LINK UP/DOWN</code> edges. For the reconnect buzz: trigger it (it appears moments after the puck boots / the controller reconnects), then connect here and click <b>Dump from boot</b> — the trigger is still in the ring. (Only present in a logging build, <code>-DOPK_LOG=1</code>.)</p>
@@ -446,6 +456,7 @@ function onGone(){
   if(stabArmed && stabStart){ up=(Date.now()-stabStart)/1000; stabLastRun={secs:up}; log("⏱ stability: stayed up "+fmtDur(up)+", then reset"); stabStart=0; }
   // queue a hang-log entry; the reason/PC/stack are filled from the first blob after reconnect
   pendingHang = { t: Date.now(), uptimeSecs: up };
+  window._flightAuto=false; // re-arm the one-shot flight-trail auto-load for the next reconnect
   polling=false; dev=null; tabInited=false; // re-default the config tab to the mode on next connect
   $("#connState").textContent="reconnecting…"; $("#connState").className="pill dn";
   $("#panel").classList.add("hide");
@@ -646,6 +657,10 @@ function applyBlob(p){
         pc:hpc?hx(hpc):"", lr:hlr?hx(hlr):"", usbd:usbdW2 });
       pendingHang=null; renderHangLog();
     }
+    // Auto-load the flight-recorder trail once per connection when the last boot was a real crash (watchdog/
+    // lockup/hardfault) -- the trail is the post-mortem of what wedged the board. Deferred to the poll loop so
+    // it doesn't nest transferIn inside this refresh.
+    if (RR_FAULT.has(code) && !window._flightAuto){ window._flightAuto=true; window._autoFlight=true; }
   }
 
   const gitDirty=(p.length>38?p[38]:0); let gitHash="";
@@ -689,7 +704,7 @@ function setSlider(id,v){ const el=$("#"+id); if(document.activeElement!==el){ e
 
 let inflight=false;
 async function refresh(){
-  if(!dev||inflight||capturing||backupBusy) return; inflight=true;
+  if(!dev||inflight||capturing||backupBusy||flightBusy) return; inflight=true;
   await send([0x01]); const p=await readBlob(); if(p) applyBlob(p);
   inflight=false;
 }
@@ -839,6 +854,73 @@ async function drainTick(){
   draining=false;
   if(liveTimer){ const el=$("#capOut"); const atBottom=el.scrollTop+el.clientHeight>=el.scrollHeight-4; renderCap(); if(atBottom) el.scrollTop=el.scrollHeight; }
 }
+// ---- Flight recorder (0xA8 stream) ----
+// The firmware keeps a .noinit ring of the last high-level events (usbd SET/GET, relays, RF up/down, radio
+// self-heal, USB mount/suspend, bond flash writes, per-250ms heartbeats) that survives a watchdog reset. On
+// 0x10 it streams them: one header (T=2, vitals@wedge + counts), then entries (T=1), then an end (T=0). We
+// pump 0x10 (restart) then 0x10 (continue) until the end frame, accumulating across FIFO-sized batches.
+let flightBusy=false, flightHdr=null, flightEvents=[];
+const FR_EVT=["none","beat","SET","GET","relay","rf-up","rf-DN","HEAL!","mount","SUSPEND","resume","OFF","RINGF","save"];
+function frEvtName(e){ return FR_EVT[e]!==undefined?FR_EVT[e]:("evt"+e); }
+async function loadFlightTrail(){
+  if(!dev||flightBusy) return; flightBusy=true;
+  $("#flightSummary").textContent="loading…";
+  flightHdr=null; flightEvents=[];
+  const STAGE_NAMES=["webusb","ctrl.task","serial","rfdiag","rflink","haptic","led","usbmount","usbtx"];
+  try{
+    let restart=true, done=false, guard=0, acc=new Uint8Array(0);
+    while(!done && guard++<600){
+      await send([0x10, restart?1:0]); restart=false;
+      const r=await dev.transferIn(epIn,192);
+      if(r.status!=="ok") break;
+      const d=new Uint8Array(r.data.buffer);
+      const m=new Uint8Array(acc.length+d.length); m.set(acc); m.set(d,acc.length); acc=m;
+      let i=0;
+      while(i<acc.length){
+        if(acc[i]!==0xA8){ i++; continue; }
+        if(i+2>acc.length) break;
+        const L=acc[i+1]; if(i+2+L>acc.length) break;
+        const f=acc.slice(i+2,i+2+L); i+=2+L;
+        const T=f[0];
+        if(T===0){ done=true; break; }
+        else if(T===2 && L>=27){
+          flightHdr={ count:f[1]|(f[2]<<8), total:f[3]|(f[4]<<8),
+            loopPerSec:f[5]|(f[6]<<8), stallMs:f[7], stage:f[8],
+            usbdStk:f[9]|(f[10]<<8), loopStk:f[11]|(f[12]<<8),
+            heap:((f[13]|(f[14]<<8)|(f[15]<<16)|(f[16]<<24))>>>0),
+            pollsps:f[17]|(f[18]<<8), relayps:f[19]|(f[20]<<8),
+            crc:f[21], norx:f[22], heal:f[23]|(f[24]<<8), ringF:f[25]|(f[26]<<8) };
+        } else if(T===1 && L>=9){
+          const dt=((f[1]|(f[2]<<8)|(f[3]<<16)|(f[4]<<24))>>>0);
+          flightEvents.push({dt, evt:f[5], stage:f[6], arg:f[7]|(f[8]<<8)});
+        }
+      }
+      acc=acc.slice(i);
+    }
+  }catch(e){ log("flight err: "+e.message); }
+  flightBusy=false;
+  renderFlight(STAGE_NAMES);
+}
+function renderFlight(STAGE_NAMES){
+  const h=flightHdr, out=$("#flightOut"), sum=$("#flightSummary"), wedge=$("#flightWedge");
+  const stage=s=>(STAGE_NAMES[s]||("st"+s));
+  if(!h || (h.count===0 && flightEvents.length===0)){
+    sum.textContent="no trail — last boot wasn't a hang, or the recorder didn't survive this board's reset";
+    wedge.textContent=""; out.textContent="—"; return;
+  }
+  const hx=v=>"0x"+v.toString(16).padStart(4,"0");
+  sum.textContent="showing "+flightEvents.length+" of "+h.total+" events before the last hang";
+  // usbd stack free trending to 0 is the leading overflow suspect -> flag it inline.
+  const usbdFlag=(h.usbdStk>0 && h.usbdStk<16)?" ⚠LOW":"";
+  wedge.innerHTML=" &nbsp;<b>@wedge:</b> stuck in "+stage(h.stage)+", stall "+h.stallMs+"ms, loop "+h.loopPerSec+"/s"
+    +" · usbdStk "+h.usbdStk+"w"+usbdFlag+" · loopStk "+h.loopStk+"w · heap "+h.heap+"B"
+    +" · poll "+h.pollsps+" relay "+h.relayps+" crc "+h.crc+" norx "+h.norx+" heal "+h.heal+" ringF "+h.ringF;
+  const pad=(s,n)=>(""+s).padEnd(n);
+  out.textContent =
+    pad("Δms",8)+pad("event",9)+pad("stage",10)+"arg\n"
+    +"".padEnd(30,"─")+"\n"
+    + flightEvents.map(e=>pad("-"+e.dt,8)+pad(frEvtName(e.evt),9)+pad(stage(e.stage),10)+hx(e.arg)).join("\n");
+}
 async function startCapture(){
   if(!dev||capturing) return;
   capturing=true; capLines=[];
@@ -857,7 +939,10 @@ async function stopCapture(){
 }
 async function startPolling(){
   polling=true; await refresh();
-  while(polling && dev){ await new Promise(r=>setTimeout(r,600)); await refresh(); }
+  while(polling && dev){
+    await new Promise(r=>setTimeout(r,600)); await refresh();
+    if(window._autoFlight){ window._autoFlight=false; await loadFlightTrail(); }
+  }
 }
 
 // UI wiring
@@ -876,6 +961,7 @@ $("#stabBtn").onclick=async()=>{
   updateStabUI();
 };
 $("#hangClear").onclick=()=>{ hangLog=[]; renderHangLog(); };
+$("#flightLoad").onclick=loadFlightTrail;
 $("#hangCsv").onclick=()=>{
   const rows=[["time","uptime_s","reason","stage","pc","lr","usbd_free_words"]].concat(
     hangLog.map(e=>[e.time, e.uptime!=null?Math.round(e.uptime):"", e.reason, e.stage, e.pc, e.lr, e.usbd]));