Files
meshcore-tth/tools/radio_stall_watch.py
T

130 lines
4.4 KiB
Python
Executable File

#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Watch a RADIO_DIAG node on its USB console until its receiver goes deaf.
usage: radio_stall_watch.py [--stall-min N] [--log FILE] <serial-port>
Born 31. 10. 2026: bench-sensor stopped receiving (recv frozen, noise_floor
-120, last_rssi 0) and a reboot revived it. A reboot fixes nothing, it only
destroys the evidence -- so this sits on the console and waits for the next one:
* every line the node prints (MESH_DEBUG) goes to the log with a UTC stamp
* every 60 s: `radiostat` + `stats-packets`
* no packet read for --stall-min minutes (default 15; the live CZ mesh
normally delivers a flood every 30-60 s) = STALL:
- three `radiostat` snapshots, 10 s apart (is anything still moving?)
- then the recovery ladder, gentlest first, one step at a time:
clrirq -> rx -> standby -> agc. After each step it waits 3 min for a
packet; the FIRST step that brings recv back names the broken layer
(lost DIO1 edge / RX not re-armed / chip out of RX / AGC-analog).
- if nothing helps, it says so and leaves the node deaf for a human.
It does not reboot the node, ever.
Needs firmware built with -D RADIO_DIAG (radiostat/radiokick). Holds the port
open for as long as it runs, so the node's BLE (dock-quiet) and other console
users are off meanwhile. Nothing is transmitted.
"""
import re
import sys
import time
if {"-h", "--help"} & set(sys.argv[1:]):
print(__doc__.strip())
sys.exit(0)
import serial # noqa: E402
LADDER = ["clrirq", "rx", "standby", "agc"]
RX_RE = re.compile(r"\brx=(\d+)/(\d+)s")
def utc():
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
class Node:
def __init__(self, port, log):
self.s = serial.Serial(port, 115200, timeout=0.2)
self.log = log
self.buf = b""
def write_log(self, line):
self.log.write(f"{utc()} {line}\n")
self.log.flush()
def pump(self, secs):
"""Log everything the node prints for `secs`; return the lines."""
out, end = [], time.time() + secs
while time.time() < end:
self.buf += self.s.read(4096)
while b"\n" in self.buf:
raw, self.buf = self.buf.split(b"\n", 1)
line = raw.decode(errors="replace").rstrip("\r")
if line:
self.write_log(line)
out.append(line)
return out
def cmd(self, c):
self.write_log(f">>> {c}")
self.s.write((c + "\r").encode())
for line in self.pump(1.0):
if line.startswith(c) and "->" in line:
return line.split("->", 1)[1].strip()
return ""
def rx_age(self):
m = RX_RE.search(self.cmd("radiostat"))
return (int(m.group(1)), int(m.group(2))) if m else (None, None)
def main(port, stall_s, logpath):
with open(logpath, "a") as log:
n = Node(port, log)
n.write_log(f"=== watch start, stall after {stall_s} s, port {port}")
n.cmd("ver")
while True:
n.pump(59)
n.cmd("stats-packets")
rx, age = n.rx_age()
if age is None or age < stall_s:
continue
n.write_log(f"!!! STALL: no packet read for {age} s (rx={rx})")
for _ in range(3):
n.cmd("radiostat")
n.pump(10)
for step in LADDER:
n.cmd(f"radiokick {step}")
end = time.time() + 180
while time.time() < end:
n.pump(30)
rx2, _ = n.rx_age()
if rx2 is not None and rx2 > rx:
n.write_log(f"!!! RECOVERED by '{step}' (rx {rx} -> {rx2})")
break
else:
n.write_log(f"... '{step}' did not bring RX back")
continue
break
else:
n.write_log("!!! NOTHING on the ladder revived RX -- left deaf for inspection")
n.pump(10**9)
stall_min, logfile = 15, "radio_stall_watch.log"
it = iter(sys.argv[1:])
args = []
for a in it:
if a == "--stall-min":
stall_min = int(next(it))
elif a == "--log":
logfile = next(it)
elif a.startswith("--"):
sys.exit(__doc__.strip().splitlines()[2])
else:
args.append(a)
if len(args) != 1:
sys.exit(__doc__.strip().splitlines()[2])
main(args[0], stall_min * 60, logfile)