Files
meshcore-tth/tools/bench_soak.py
T

248 lines
9.4 KiB
Python
Executable File

#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Dlouhý běh lavice: kdo koho slyší, v čase, a kdy deska rebootuje nebo ohluchne.
Vzniklo 23. 9. 2026 kvůli second-mast: po OTA na nový firmware přestala přijímat
(`recv 0`) a na zerohop hlásila `sent 0` + `tx_timeout 4`, přestože ob2 vedle ní
slyšel všechny čtyři adverty. Rádio vysílalo, ale IRQ se nedostalo do MCU.
`fleet_test.py` je jednorázový snímek za pár sekund -- tohle je stejná otázka
položená stokrát za sebou, protože chyba přišla až po minutách běhu.
tools/bench_soak.py --boards b1,b2 --hours 8
tools/bench_soak.py --period 30 --out /tmp/soak # all bench boards of the config
Co dělá, v kolech (každých --period s, vysílá jedna deska, dokola):
1. snímek `stats-packets` a `stats-core` na všech deskách,
2. vysílající pošle `advert.zerohop` (nikdo ho nerebroadcastuje, takže
„B slyšel A" nejde splnit přes třetí desku a netřeba sahat na `repeat`),
3. po --settle s druhý snímek.
Orákulum je vždy JINÁ deska: slyšel = `recv` příjemce vzrostl. Vlastní `sent`
vysílajícího se jen zapisuje vedle -- u second-mast lhal (0, zatímco vysílala).
Konzole se drží otevřené celou dobu a KAŽDÝ řádek jde s časem do logu, takže se
zachytí i to, co deska vypíše sama (boot `RESETREAS`, `!!! HARDFAULT`). Když
deska zmizí z USB, nástroj to zapíše a čeká, až se vrátí -- nic nerestartuje.
Nic nemění: žádné `set`, žádné `reboot`. Prefs pro test nastav předem ručně.
PAST (z fleet_test.py): na testované desce nesmí viset BLE klient -- USB a BLE
konzole dělí jeden buffer a prokládají znaky.
"""
import argparse
import glob
import json
import os
import re
import signal
import sys
import threading
import time
from datetime import datetime, timezone
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from board_probe import _open as open_port
from fleet import serial_for
FLAG_RE = re.compile(r"boot:|RESETREAS|HARDFAULT|LOCKUP|WATCHDOG", re.IGNORECASE)
def now():
return datetime.now(timezone.utc).isoformat(timespec="seconds")
class Board:
"""Jedna konzole: čtecí vlákno loguje vše, `cmd()` sbírá odpověď z okna."""
def __init__(self, name, log_fh, events):
self.name = name
self.sn = serial_for(name)
self.log = log_fh
self.events = events
self.s = None
self.lines = [] # řádky od posledního cmd()
self.lock = threading.Lock()
self.stop = False
self.up = False
threading.Thread(target=self._reader, daemon=True).start()
def _path(self):
hits = [p for p in glob.glob("/dev/serial/by-id/*") if self.sn in p]
return hits[0] if len(hits) == 1 else None
def _event(self, kind, **kw):
ev = {"t": now(), "board": self.name, "event": kind, **kw}
self.events.append(ev)
self.log.write(json.dumps(ev, ensure_ascii=False) + "\n")
self.log.flush()
print(f"[{ev['t']}] {self.name}: {kind} {kw if kw else ''}", flush=True)
def _reader(self):
buf = b""
while not self.stop:
if self.s is None:
p = self._path()
if p:
try:
self.s = open_port(p)
if self.up is False and self.events is not None:
self._event("usb_up", port=os.path.basename(p))
self.up = True
except Exception:
self.s = None
time.sleep(1)
continue
try:
d = self.s.read(4096)
except Exception as e:
self._event("usb_gone", err=str(e)[:80])
self.up = False
try:
self.s.close()
except Exception:
pass
self.s = None
continue
if not d:
time.sleep(0.02)
continue
buf += d
while b"\n" in buf:
raw, buf = buf.split(b"\n", 1)
line = raw.decode("utf-8", "replace").rstrip("\r")
if not line.strip():
continue
self.log.write(f"{now()} {self.name} | {line}\n")
if FLAG_RE.search(line):
self._event("flagged_line", line=line[:200])
with self.lock:
self.lines.append(line)
self.log.flush()
def cmd(self, text, window=1.5):
if self.s is None:
return None
with self.lock:
self.lines = []
try:
self.s.write((text + "\r").encode())
except Exception:
return None
time.sleep(window)
with self.lock:
return "\n".join(self.lines)
def stats(self):
out = {}
for c in ("stats-packets", "stats-core"):
r = self.cmd(c)
m = re.search(r"\{.*\}", r or "")
if not m:
return None
try:
out.update(json.loads(m.group(0)))
except ValueError:
return None
return out
def main():
ap = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--boards", default=None,
help="desky z lokalniho configu, čárkou (default: všechny role=bench bez refuse)")
ap.add_argument("--hours", type=float, default=8.0)
ap.add_argument("--period", type=float, default=60.0, help="s mezi koly")
ap.add_argument("--settle", type=float, default=4.0, help="s po advertu")
ap.add_argument("--out", default=".", help="adresář na log + JSONL")
a = ap.parse_args()
if a.boards is None:
from mctools_config import by_role
a.boards = ",".join(n for n, b in by_role("bench").items() if b.get("serial") and not b.get("refuse"))
if not a.boards:
ap.error("no --boards and no role=bench board in the local config (tools/mctools_config.py)")
os.makedirs(a.out, exist_ok=True)
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
log = open(os.path.join(a.out, f"soak-{stamp}.log"), "a")
rounds_fh = open(os.path.join(a.out, f"soak-{stamp}.jsonl"), "a")
events = []
boards = [Board(n.strip(), log, events) for n in a.boards.split(",")]
if len(boards) < 2:
sys.exit("potřeba aspoň dvě desky -- svědkem musí být JINÁ deska")
stop = {"v": False}
signal.signal(signal.SIGINT, lambda *_: stop.update(v=True))
signal.signal(signal.SIGTERM, lambda *_: stop.update(v=True))
time.sleep(3)
last_up = {}
pairs = {} # (tx, rx) -> [heard, tried]
miss_streak = {} # (tx, rx) -> po sobě jdoucí neslyšení
reboots = {b.name: 0 for b in boards}
txto = {b.name: 0 for b in boards}
t_end = time.time() + a.hours * 3600
rnd = 0
while not stop["v"] and time.time() < t_end:
t0 = time.time()
tx = boards[rnd % len(boards)]
before = {b.name: b.stats() for b in boards}
ok = tx.cmd("advert.zerohop", window=1.0)
time.sleep(a.settle)
after = {b.name: b.stats() for b in boards}
rec = {"t": now(), "round": rnd, "tx": tx.name,
"tx_cmd_ok": bool(ok and "OK" in ok), "heard": {}, "boards": {}}
for b in boards:
s0, s1 = before[b.name], after[b.name]
rec["boards"][b.name] = s1
if s1 and "uptime_secs" in s1:
u = s1["uptime_secs"]
if b.name in last_up and u < last_up[b.name]:
reboots[b.name] += 1
b._event("reboot", uptime_before=last_up[b.name], uptime_now=u)
last_up[b.name] = u
if s0 and s1:
d_to = s1.get("tx_timeout", 0) - s0.get("tx_timeout", 0)
if d_to > 0:
txto[b.name] += d_to
b._event("tx_timeout", delta=d_to)
if b is tx or not (s0 and s1):
continue
heard = s1.get("recv", 0) - s0.get("recv", 0) >= 1
rec["heard"][b.name] = heard
key = (tx.name, b.name)
h = pairs.setdefault(key, [0, 0])
h[1] += 1
h[0] += int(heard)
miss_streak[key] = 0 if heard else miss_streak.get(key, 0) + 1
if miss_streak[key] == 3:
b._event("deaf?", tx=tx.name, streak=3,
note="3 kola po sobě neslyšel -- ověř svědkem")
tb, ta = before[tx.name], after[tx.name]
if tb and ta:
rec["tx_sent_delta"] = ta.get("sent", 0) - tb.get("sent", 0)
rounds_fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
rounds_fh.flush()
rnd += 1
time.sleep(max(0.0, a.period - (time.time() - t0)))
for b in boards:
b.stop = True
summary = {
"rounds": rnd,
"pairs": {f"{k[0]}->{k[1]}": f"{v[0]}/{v[1]}" for k, v in sorted(pairs.items())},
"reboots": reboots,
"tx_timeouts": txto,
"events": len(events),
}
print(json.dumps(summary, ensure_ascii=False, indent=1))
rounds_fh.write(json.dumps({"summary": summary}, ensure_ascii=False) + "\n")
bad = any(v for v in reboots.values()) or any(v for v in txto.values()) or \
any(v[0] < v[1] for v in pairs.values())
sys.exit(1 if bad else 0)
if __name__ == "__main__":
main()