130 lines
4.4 KiB
Python
Executable File
130 lines
4.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: MIT
|
|
"""Watch a RADIO_DIAG node on its USB console until its receiver goes deaf.
|
|
|
|
usage: radio_stall_watch.py [--stall-min N] [--log FILE] <serial-port>
|
|
|
|
Born 31. 10. 2026: bench-sensor stopped receiving (recv frozen, noise_floor
|
|
-120, last_rssi 0) and a reboot revived it. A reboot fixes nothing, it only
|
|
destroys the evidence -- so this sits on the console and waits for the next one:
|
|
|
|
* every line the node prints (MESH_DEBUG) goes to the log with a UTC stamp
|
|
* every 60 s: `radiostat` + `stats-packets`
|
|
* no packet read for --stall-min minutes (default 15; the live CZ mesh
|
|
normally delivers a flood every 30-60 s) = STALL:
|
|
- three `radiostat` snapshots, 10 s apart (is anything still moving?)
|
|
- then the recovery ladder, gentlest first, one step at a time:
|
|
clrirq -> rx -> standby -> agc. After each step it waits 3 min for a
|
|
packet; the FIRST step that brings recv back names the broken layer
|
|
(lost DIO1 edge / RX not re-armed / chip out of RX / AGC-analog).
|
|
- if nothing helps, it says so and leaves the node deaf for a human.
|
|
It does not reboot the node, ever.
|
|
|
|
Needs firmware built with -D RADIO_DIAG (radiostat/radiokick). Holds the port
|
|
open for as long as it runs, so the node's BLE (dock-quiet) and other console
|
|
users are off meanwhile. Nothing is transmitted.
|
|
"""
|
|
|
|
import re
|
|
import sys
|
|
import time
|
|
|
|
if {"-h", "--help"} & set(sys.argv[1:]):
|
|
print(__doc__.strip())
|
|
sys.exit(0)
|
|
|
|
import serial # noqa: E402
|
|
|
|
LADDER = ["clrirq", "rx", "standby", "agc"]
|
|
RX_RE = re.compile(r"\brx=(\d+)/(\d+)s")
|
|
|
|
|
|
def utc():
|
|
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
|
|
|
|
class Node:
|
|
def __init__(self, port, log):
|
|
self.s = serial.Serial(port, 115200, timeout=0.2)
|
|
self.log = log
|
|
self.buf = b""
|
|
|
|
def write_log(self, line):
|
|
self.log.write(f"{utc()} {line}\n")
|
|
self.log.flush()
|
|
|
|
def pump(self, secs):
|
|
"""Log everything the node prints for `secs`; return the lines."""
|
|
out, end = [], time.time() + secs
|
|
while time.time() < end:
|
|
self.buf += self.s.read(4096)
|
|
while b"\n" in self.buf:
|
|
raw, self.buf = self.buf.split(b"\n", 1)
|
|
line = raw.decode(errors="replace").rstrip("\r")
|
|
if line:
|
|
self.write_log(line)
|
|
out.append(line)
|
|
return out
|
|
|
|
def cmd(self, c):
|
|
self.write_log(f">>> {c}")
|
|
self.s.write((c + "\r").encode())
|
|
for line in self.pump(1.0):
|
|
if line.startswith(c) and "->" in line:
|
|
return line.split("->", 1)[1].strip()
|
|
return ""
|
|
|
|
def rx_age(self):
|
|
m = RX_RE.search(self.cmd("radiostat"))
|
|
return (int(m.group(1)), int(m.group(2))) if m else (None, None)
|
|
|
|
|
|
def main(port, stall_s, logpath):
|
|
with open(logpath, "a") as log:
|
|
n = Node(port, log)
|
|
n.write_log(f"=== watch start, stall after {stall_s} s, port {port}")
|
|
n.cmd("ver")
|
|
while True:
|
|
n.pump(59)
|
|
n.cmd("stats-packets")
|
|
rx, age = n.rx_age()
|
|
if age is None or age < stall_s:
|
|
continue
|
|
n.write_log(f"!!! STALL: no packet read for {age} s (rx={rx})")
|
|
for _ in range(3):
|
|
n.cmd("radiostat")
|
|
n.pump(10)
|
|
for step in LADDER:
|
|
n.cmd(f"radiokick {step}")
|
|
end = time.time() + 180
|
|
while time.time() < end:
|
|
n.pump(30)
|
|
rx2, _ = n.rx_age()
|
|
if rx2 is not None and rx2 > rx:
|
|
n.write_log(f"!!! RECOVERED by '{step}' (rx {rx} -> {rx2})")
|
|
break
|
|
else:
|
|
n.write_log(f"... '{step}' did not bring RX back")
|
|
continue
|
|
break
|
|
else:
|
|
n.write_log("!!! NOTHING on the ladder revived RX -- left deaf for inspection")
|
|
n.pump(10**9)
|
|
|
|
|
|
stall_min, logfile = 15, "radio_stall_watch.log"
|
|
it = iter(sys.argv[1:])
|
|
args = []
|
|
for a in it:
|
|
if a == "--stall-min":
|
|
stall_min = int(next(it))
|
|
elif a == "--log":
|
|
logfile = next(it)
|
|
elif a.startswith("--"):
|
|
sys.exit(__doc__.strip().splitlines()[2])
|
|
else:
|
|
args.append(a)
|
|
if len(args) != 1:
|
|
sys.exit(__doc__.strip().splitlines()[2])
|
|
main(args[0], stall_min * 60, logfile)
|