248 lines
9.4 KiB
Python
Executable File
248 lines
9.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: MIT
|
|
"""Dlouhý běh lavice: kdo koho slyší, v čase, a kdy deska rebootuje nebo ohluchne.
|
|
|
|
Vzniklo 23. 9. 2026 kvůli second-mast: po OTA na nový firmware přestala přijímat
|
|
(`recv 0`) a na zerohop hlásila `sent 0` + `tx_timeout 4`, přestože ob2 vedle ní
|
|
slyšel všechny čtyři adverty. Rádio vysílalo, ale IRQ se nedostalo do MCU.
|
|
`fleet_test.py` je jednorázový snímek za pár sekund -- tohle je stejná otázka
|
|
položená stokrát za sebou, protože chyba přišla až po minutách běhu.
|
|
|
|
tools/bench_soak.py --boards b1,b2 --hours 8
|
|
tools/bench_soak.py --period 30 --out /tmp/soak # all bench boards of the config
|
|
|
|
Co dělá, v kolech (každých --period s, vysílá jedna deska, dokola):
|
|
1. snímek `stats-packets` a `stats-core` na všech deskách,
|
|
2. vysílající pošle `advert.zerohop` (nikdo ho nerebroadcastuje, takže
|
|
„B slyšel A" nejde splnit přes třetí desku a netřeba sahat na `repeat`),
|
|
3. po --settle s druhý snímek.
|
|
|
|
Orákulum je vždy JINÁ deska: slyšel = `recv` příjemce vzrostl. Vlastní `sent`
|
|
vysílajícího se jen zapisuje vedle -- u second-mast lhal (0, zatímco vysílala).
|
|
|
|
Konzole se drží otevřené celou dobu a KAŽDÝ řádek jde s časem do logu, takže se
|
|
zachytí i to, co deska vypíše sama (boot `RESETREAS`, `!!! HARDFAULT`). Když
|
|
deska zmizí z USB, nástroj to zapíše a čeká, až se vrátí -- nic nerestartuje.
|
|
|
|
Nic nemění: žádné `set`, žádné `reboot`. Prefs pro test nastav předem ručně.
|
|
PAST (z fleet_test.py): na testované desce nesmí viset BLE klient -- USB a BLE
|
|
konzole dělí jeden buffer a prokládají znaky.
|
|
"""
|
|
import argparse
|
|
import glob
|
|
import json
|
|
import os
|
|
import re
|
|
import signal
|
|
import sys
|
|
import threading
|
|
import time
|
|
from datetime import datetime, timezone
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
from board_probe import _open as open_port
|
|
from fleet import serial_for
|
|
|
|
FLAG_RE = re.compile(r"boot:|RESETREAS|HARDFAULT|LOCKUP|WATCHDOG", re.IGNORECASE)
|
|
|
|
|
|
def now():
|
|
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
|
|
|
|
class Board:
|
|
"""Jedna konzole: čtecí vlákno loguje vše, `cmd()` sbírá odpověď z okna."""
|
|
|
|
def __init__(self, name, log_fh, events):
|
|
self.name = name
|
|
self.sn = serial_for(name)
|
|
self.log = log_fh
|
|
self.events = events
|
|
self.s = None
|
|
self.lines = [] # řádky od posledního cmd()
|
|
self.lock = threading.Lock()
|
|
self.stop = False
|
|
self.up = False
|
|
threading.Thread(target=self._reader, daemon=True).start()
|
|
|
|
def _path(self):
|
|
hits = [p for p in glob.glob("/dev/serial/by-id/*") if self.sn in p]
|
|
return hits[0] if len(hits) == 1 else None
|
|
|
|
def _event(self, kind, **kw):
|
|
ev = {"t": now(), "board": self.name, "event": kind, **kw}
|
|
self.events.append(ev)
|
|
self.log.write(json.dumps(ev, ensure_ascii=False) + "\n")
|
|
self.log.flush()
|
|
print(f"[{ev['t']}] {self.name}: {kind} {kw if kw else ''}", flush=True)
|
|
|
|
def _reader(self):
|
|
buf = b""
|
|
while not self.stop:
|
|
if self.s is None:
|
|
p = self._path()
|
|
if p:
|
|
try:
|
|
self.s = open_port(p)
|
|
if self.up is False and self.events is not None:
|
|
self._event("usb_up", port=os.path.basename(p))
|
|
self.up = True
|
|
except Exception:
|
|
self.s = None
|
|
time.sleep(1)
|
|
continue
|
|
try:
|
|
d = self.s.read(4096)
|
|
except Exception as e:
|
|
self._event("usb_gone", err=str(e)[:80])
|
|
self.up = False
|
|
try:
|
|
self.s.close()
|
|
except Exception:
|
|
pass
|
|
self.s = None
|
|
continue
|
|
if not d:
|
|
time.sleep(0.02)
|
|
continue
|
|
buf += d
|
|
while b"\n" in buf:
|
|
raw, buf = buf.split(b"\n", 1)
|
|
line = raw.decode("utf-8", "replace").rstrip("\r")
|
|
if not line.strip():
|
|
continue
|
|
self.log.write(f"{now()} {self.name} | {line}\n")
|
|
if FLAG_RE.search(line):
|
|
self._event("flagged_line", line=line[:200])
|
|
with self.lock:
|
|
self.lines.append(line)
|
|
self.log.flush()
|
|
|
|
def cmd(self, text, window=1.5):
|
|
if self.s is None:
|
|
return None
|
|
with self.lock:
|
|
self.lines = []
|
|
try:
|
|
self.s.write((text + "\r").encode())
|
|
except Exception:
|
|
return None
|
|
time.sleep(window)
|
|
with self.lock:
|
|
return "\n".join(self.lines)
|
|
|
|
def stats(self):
|
|
out = {}
|
|
for c in ("stats-packets", "stats-core"):
|
|
r = self.cmd(c)
|
|
m = re.search(r"\{.*\}", r or "")
|
|
if not m:
|
|
return None
|
|
try:
|
|
out.update(json.loads(m.group(0)))
|
|
except ValueError:
|
|
return None
|
|
return out
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("--boards", default=None,
|
|
help="desky z lokalniho configu, čárkou (default: všechny role=bench bez refuse)")
|
|
ap.add_argument("--hours", type=float, default=8.0)
|
|
ap.add_argument("--period", type=float, default=60.0, help="s mezi koly")
|
|
ap.add_argument("--settle", type=float, default=4.0, help="s po advertu")
|
|
ap.add_argument("--out", default=".", help="adresář na log + JSONL")
|
|
a = ap.parse_args()
|
|
if a.boards is None:
|
|
from mctools_config import by_role
|
|
|
|
a.boards = ",".join(n for n, b in by_role("bench").items() if b.get("serial") and not b.get("refuse"))
|
|
if not a.boards:
|
|
ap.error("no --boards and no role=bench board in the local config (tools/mctools_config.py)")
|
|
|
|
os.makedirs(a.out, exist_ok=True)
|
|
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
log = open(os.path.join(a.out, f"soak-{stamp}.log"), "a")
|
|
rounds_fh = open(os.path.join(a.out, f"soak-{stamp}.jsonl"), "a")
|
|
events = []
|
|
boards = [Board(n.strip(), log, events) for n in a.boards.split(",")]
|
|
if len(boards) < 2:
|
|
sys.exit("potřeba aspoň dvě desky -- svědkem musí být JINÁ deska")
|
|
|
|
stop = {"v": False}
|
|
signal.signal(signal.SIGINT, lambda *_: stop.update(v=True))
|
|
signal.signal(signal.SIGTERM, lambda *_: stop.update(v=True))
|
|
|
|
time.sleep(3)
|
|
last_up = {}
|
|
pairs = {} # (tx, rx) -> [heard, tried]
|
|
miss_streak = {} # (tx, rx) -> po sobě jdoucí neslyšení
|
|
reboots = {b.name: 0 for b in boards}
|
|
txto = {b.name: 0 for b in boards}
|
|
t_end = time.time() + a.hours * 3600
|
|
rnd = 0
|
|
while not stop["v"] and time.time() < t_end:
|
|
t0 = time.time()
|
|
tx = boards[rnd % len(boards)]
|
|
before = {b.name: b.stats() for b in boards}
|
|
ok = tx.cmd("advert.zerohop", window=1.0)
|
|
time.sleep(a.settle)
|
|
after = {b.name: b.stats() for b in boards}
|
|
|
|
rec = {"t": now(), "round": rnd, "tx": tx.name,
|
|
"tx_cmd_ok": bool(ok and "OK" in ok), "heard": {}, "boards": {}}
|
|
for b in boards:
|
|
s0, s1 = before[b.name], after[b.name]
|
|
rec["boards"][b.name] = s1
|
|
if s1 and "uptime_secs" in s1:
|
|
u = s1["uptime_secs"]
|
|
if b.name in last_up and u < last_up[b.name]:
|
|
reboots[b.name] += 1
|
|
b._event("reboot", uptime_before=last_up[b.name], uptime_now=u)
|
|
last_up[b.name] = u
|
|
if s0 and s1:
|
|
d_to = s1.get("tx_timeout", 0) - s0.get("tx_timeout", 0)
|
|
if d_to > 0:
|
|
txto[b.name] += d_to
|
|
b._event("tx_timeout", delta=d_to)
|
|
if b is tx or not (s0 and s1):
|
|
continue
|
|
heard = s1.get("recv", 0) - s0.get("recv", 0) >= 1
|
|
rec["heard"][b.name] = heard
|
|
key = (tx.name, b.name)
|
|
h = pairs.setdefault(key, [0, 0])
|
|
h[1] += 1
|
|
h[0] += int(heard)
|
|
miss_streak[key] = 0 if heard else miss_streak.get(key, 0) + 1
|
|
if miss_streak[key] == 3:
|
|
b._event("deaf?", tx=tx.name, streak=3,
|
|
note="3 kola po sobě neslyšel -- ověř svědkem")
|
|
tb, ta = before[tx.name], after[tx.name]
|
|
if tb and ta:
|
|
rec["tx_sent_delta"] = ta.get("sent", 0) - tb.get("sent", 0)
|
|
rounds_fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|
rounds_fh.flush()
|
|
rnd += 1
|
|
time.sleep(max(0.0, a.period - (time.time() - t0)))
|
|
|
|
for b in boards:
|
|
b.stop = True
|
|
summary = {
|
|
"rounds": rnd,
|
|
"pairs": {f"{k[0]}->{k[1]}": f"{v[0]}/{v[1]}" for k, v in sorted(pairs.items())},
|
|
"reboots": reboots,
|
|
"tx_timeouts": txto,
|
|
"events": len(events),
|
|
}
|
|
print(json.dumps(summary, ensure_ascii=False, indent=1))
|
|
rounds_fh.write(json.dumps({"summary": summary}, ensure_ascii=False) + "\n")
|
|
bad = any(v for v in reboots.values()) or any(v for v in txto.values()) or \
|
|
any(v[0] < v[1] for v in pairs.values())
|
|
sys.exit(1 if bad else 0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|