simple_repeater: radio silence watchdog -- re-arm RX, then reboot a deaf receiver (#6)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
tthandClaude Opus 5.5 committed 2026-11-03 20:03:01 +01:00
1 parent 10f0349207
commit 278c1b8c8a
8 files changed
+440

No files matched your search

+17
View File
@@ -718,6 +718,23 @@ so the icon and the Grafana percentage tell one story.
---
#### Radio silence watchdog (repeater only)
**Usage:**
- `get radio.silence`
- `set radio.silence <minutes>`
- `get radio.reboots`
**Parameters:**
- `minutes`: 1-1440, or `0`/`off` to disable.
**Description:** Recovers a receiver that has gone deaf while the node keeps running (the hardware watchdog only catches a frozen loop). After `minutes` without a single received frame — a receive error counts as heard, too — the receiver is re-armed (IRQs cleared, RX restarted). If nothing arrives within a further quarter of the threshold (at least 5 minutes), the node reboots.
To keep a quiet site out of a reboot loop, each such reboot doubles the threshold for the next boot (up to 16×); the first frame received afterwards resets it. `get radio.silence` shows the threshold, the current multiplier, how long the receiver has been quiet and the stage (0 healthy, 1 re-armed). `get radio.reboots` counts the reboots done by this watchdog; it survives power loss. Pick a threshold well above the longest quiet spell the site normally has.
**Default:** `0` (off)
---
#### Enable or disable Multi-Acks support
**Usage:**
- `get multi.acks`
+9
View File
@@ -1404,6 +1404,11 @@ MyMesh::MyMesh(mesh::MainBoard &board, mesh::Radio &radio, mesh::MillisecondCloc
#define FAIRNESS_ENABLED_DEFAULT 1
#endif
_prefs.fairness_enabled = FAIRNESS_ENABLED_DEFAULT;
// CUSTOM (TeTeHacko): radio silence watchdog, minutes; 0 = off (see main.cpp).
#ifndef RADIO_SILENCE_MIN_DEFAULT
#define RADIO_SILENCE_MIN_DEFAULT 0
#endif
_prefs.radio_silence_min = RADIO_SILENCE_MIN_DEFAULT;
filter_hash_dropped = 0;
filter_hash_passed = 0;
_prefs.interference_threshold = 0; // disabled
@@ -1869,6 +1874,10 @@ void MyMesh::handleCommand(uint32_t sender_timestamp, char *command, char *reply
// verdict, which `set tx` swallows. See main.cpp.
extern bool txPwrHandleCommand(const char* command, char* reply);
if (txPwrHandleCommand(command, reply)) return;
// CUSTOM (TeTeHacko): `get/set radio.silence`, `get radio.reboots` -- the
// deaf-receiver watchdog, see main.cpp. Ahead of CommonCLI's get/set.
extern bool radioSilenceHandleCommand(const char* command, char* reply);
if (radioSilenceHandleCommand(command, reply)) return;
#ifdef RADIO_DIAG
// CUSTOM (TeTeHacko): `radiostat` / `radiokick` -- see main.cpp.
extern bool radioDiagHandleCommand(const char* command, char* reply);
+113
View File
@@ -2,6 +2,7 @@
#include <Mesh.h>
#include "MyMesh.h"
#include <helpers/RadioSilenceWatchdog.h>
// ===========================================================================
// CUSTOM (TeTeHacko): `blink [n]` -- make THIS board flash its LED on demand,
@@ -750,6 +751,116 @@ bool txPwrHandleCommand(const char* command, char* reply) {
return true;
}
// CUSTOM (TeTeHacko, tth/meshcore-tth#6): radio silence watchdog -- see
// RadioSilenceWatchdog.h for why it is staged and how it avoids a reboot loop.
// `set radio.silence <min>` (0 = off), `get radio.silence`, `get radio.reboots`.
// Reboots it caused and the threshold backoff survive power loss in a small file,
// written only right before such a reboot and once when the radio proves itself.
static RadioSilenceWatchdog radio_silence;
static FILESYSTEM* radio_silence_fs = NULL;
static uint32_t radio_silence_reboots = 0;
#define RADIO_SILENCE_FILE "/rsilence"
static uint32_t radioActivity() {
return radio_driver.getPacketsRecv() + radio_driver.getPacketsRecvErrors();
}
static void radioSilenceLoad(uint8_t* backoff) {
*backoff = 0;
if (!radio_silence_fs || !radio_silence_fs->exists(RADIO_SILENCE_FILE)) return;
#if defined(RP2040_PLATFORM)
File f = radio_silence_fs->open(RADIO_SILENCE_FILE, "r");
#else
File f = radio_silence_fs->open(RADIO_SILENCE_FILE);
#endif
if (!f) return;
if (f.read((uint8_t*)&radio_silence_reboots, sizeof(radio_silence_reboots)) != sizeof(radio_silence_reboots)) {
radio_silence_reboots = 0;
}
if (f.read(backoff, 1) != 1) *backoff = 0;
f.close();
}
static void radioSilenceSave(uint8_t backoff) {
if (!radio_silence_fs) return;
#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM)
radio_silence_fs->remove(RADIO_SILENCE_FILE);
File f = radio_silence_fs->open(RADIO_SILENCE_FILE, FILE_O_WRITE);
#elif defined(RP2040_PLATFORM)
File f = radio_silence_fs->open(RADIO_SILENCE_FILE, "w");
#else
File f = radio_silence_fs->open(RADIO_SILENCE_FILE, "w", true);
#endif
if (!f) return;
f.write((const uint8_t*)&radio_silence_reboots, sizeof(radio_silence_reboots));
f.write(&backoff, 1);
f.close();
}
static void radioSilenceBegin(FILESYSTEM* fs) {
radio_silence_fs = fs;
uint8_t backoff;
radioSilenceLoad(&backoff);
radio_silence.begin(millis(), radioActivity(), backoff);
radio_silence.setMinutes(the_mesh.getNodePrefs()->radio_silence_min);
Serial.printf("boot: radio silence watchdog %u min, reboots %lu, backoff x%u\n",
(unsigned) radio_silence.minutes(), (unsigned long) radio_silence_reboots,
(unsigned) (1u << radio_silence.backoff()));
}
static void radioSilenceLoop() {
switch (radio_silence.poll(millis(), radioActivity())) {
case RadioSilenceWatchdog::KICK_RX:
Serial.printf("radio silence: nothing received for %lu min, re-arming RX\n",
(unsigned long) (radio_silence.quietMs(millis()) / 60000));
radio_driver.kickRx();
break;
case RadioSilenceWatchdog::REBOOT:
Serial.printf("radio silence: still nothing after %lu min, rebooting\n",
(unsigned long) (radio_silence.quietMs(millis()) / 60000));
radio_silence_reboots++;
radioSilenceSave(radio_silence.nextBackoff());
delay(100); // let the line out
board.reboot();
break;
default:
break;
}
if (radio_silence.takeBackoffCleared()) radioSilenceSave(0);
}
bool radioSilenceHandleCommand(const char* command, char* reply) {
if (strcmp(command, "get radio.reboots") == 0) {
sprintf(reply, "> %lu", (unsigned long) radio_silence_reboots);
return true;
}
if (strcmp(command, "get radio.silence") == 0) {
if (radio_silence.minutes() == 0) {
strcpy(reply, "> off");
} else {
sprintf(reply, "> %u min (now x%u), quiet %lu min, stage %u",
(unsigned) radio_silence.minutes(), (unsigned) (1u << radio_silence.backoff()),
(unsigned long) (radio_silence.quietMs(millis()) / 60000), (unsigned) radio_silence.stage());
}
return true;
}
if (memcmp(command, "set radio.silence ", 18) == 0) {
const char* arg = &command[18];
bool numeric = *arg != 0 && strspn(arg, "0123456789") == strlen(arg);
int m = strcmp(arg, "off") == 0 ? 0 : (numeric ? atoi(arg) : -1);
if (m < 0 || m > RadioSilenceWatchdog::MAX_MINUTES) {
sprintf(reply, "ERR - minutes 0..%u (0/off = disabled)", (unsigned) RadioSilenceWatchdog::MAX_MINUTES);
return true;
}
the_mesh.getNodePrefs()->radio_silence_min = (uint16_t) m;
the_mesh.savePrefs();
radio_silence.setMinutes((uint16_t) m);
strcpy(reply, "OK");
return true;
}
return false;
}
#ifdef RADIO_DIAG
// CUSTOM (TeTeHacko): `radiostat` / `radiokick` -- see RadioLibWrappers.cpp.
// On the masts since 2. 11. 2026: mast-three came up deaf after a BLE OTA (recv 0,
@@ -1018,6 +1129,7 @@ void setup() {
sensors.begin();
the_mesh.begin(fs);
radioSilenceBegin(fs); // CUSTOM (TeTeHacko): needs the prefs loaded by begin()
#ifdef DISPLAY_CLASS
ui_task.begin(the_mesh.getNodePrefs(), FIRMWARE_BUILD_DATE, FIRMWARE_VERSION);
@@ -1258,6 +1370,7 @@ void loop() {
#endif
the_mesh.loop();
radioSilenceLoop(); // CUSTOM (TeTeHacko): deaf-receiver watchdog
sensors.loop();
#ifdef DISPLAY_CLASS
ui_task.loop();
+4
View File
@@ -94,6 +94,9 @@ public:
uint16_t fair_group_refill_s = 0, fair_sender_refill_s = 0, fair_advert_refill_s = 0;
uint16_t fair_advert_hours = 0; // per-origin advert window, hours; 0 = off
uint8_t fair_dry_run = 0; // count FairnessLimiter verdicts, forward anyway
// Radio silence watchdog (RadioSilenceWatchdog.h, simple_repeater): minutes
// without a received frame before the receiver is re-armed, then rebooted; 0 = off.
uint16_t radio_silence_min = 0;
private:
class RadioPrefs : public CommonRadioPrefs {
@@ -117,6 +120,7 @@ private:
def("agc_int", _parent->agc_reset_interval);
def("hash_mode", _parent->path_hash_mode);
def("multi_ack", _parent->multi_acks);
def("rsilence", _parent->radio_silence_min);
}
public:
RadioPrefs(NodePrefs* parent) : _parent(parent) { }
+113
View File
@@ -0,0 +1,113 @@
#pragma once
#include <stdint.h>
// CUSTOM (TeTeHacko, tth/meshcore-tth#6): a receiver that went deaf while the
// main loop keeps running. The hardware watchdog only catches a frozen loop, and
// two of our nodes have been found deaf with the loop alive -- bench-sensor on
// 31. 10. 2026 (recv frozen for minutes, a reboot revived it) and mast-three on
// 2. 11. after a BLE OTA (recv 0, tx_timeout 4 for 17 minutes, a reboot fixed it).
//
// Silence is judged by whether the receive counter MOVES, never by comparing
// timestamps: taking the largest of several millis() stamps aliased across the
// 32-bit wrap and reset healthy radios for weeks in another fork (Dutch-MeshCore
// d8ffae32). Received packets and receive errors both count -- a CRC error still
// proves the front end hears something.
//
// Staged: after `minutes` without a received frame the receiver is re-armed
// (clear IRQ, startReceive -- covers an interrupt that never reached the MCU);
// if that does not bring anything in within a step, the node reboots. There is
// no in-between full chip re-init: it would have to restore every parameter
// begin() applies, and a radio silently left on its default frequency is worse
// than the fault (chendo 9a9a716a hit exactly that). AGC resets already run on
// their own interval where configured.
//
// A quiet site must not turn this into a reboot loop: every reboot by this
// watchdog doubles the threshold for the next boot (up to x16), and the first
// frame received after boot clears it. The caller persists `backoff()` across
// the reboot and reports `takeBackoffCleared()` so it can rewrite it.
//
// Pure logic, no Arduino: driven from the repeater loop, tested on the host.
class RadioSilenceWatchdog {
public:
enum Action : uint8_t { NONE = 0, KICK_RX, REBOOT };
static constexpr uint32_t CHECK_MS = 30000; // evaluate at most this often
static constexpr uint32_t MIN_STEP_MS = 5 * 60000; // kick -> reboot, at least
static constexpr uint16_t MAX_MINUTES = 1440; // 24 h
static constexpr uint8_t MAX_BACKOFF = 4; // threshold x16 at most
void begin(uint32_t now, uint32_t activity, uint8_t backoff) {
_last_activity = activity;
_quiet_since = _last_check = now;
_backoff = backoff > MAX_BACKOFF ? MAX_BACKOFF : backoff;
_stage = 0;
_backoff_cleared = false;
}
void setMinutes(uint16_t minutes) {
_minutes = minutes > MAX_MINUTES ? MAX_MINUTES : minutes;
_stage = 0;
}
uint16_t minutes() const { return _minutes; }
// `activity`: any counter that moves when the radio receives something.
Action poll(uint32_t now, uint32_t activity) {
if ((uint32_t)(now - _last_check) < CHECK_MS) return NONE;
_last_check = now;
if (activity != _last_activity) {
_last_activity = activity;
_quiet_since = now;
_stage = 0;
if (_backoff) {
_backoff = 0;
_backoff_cleared = true;
}
return NONE;
}
if (_minutes == 0) {
_stage = 0;
return NONE;
}
uint32_t quiet = (uint32_t)(now - _quiet_since);
uint32_t threshold = thresholdMs();
if (_stage == 0 && quiet >= threshold) {
_stage = 1;
return KICK_RX;
}
if (_stage == 1 && quiet >= threshold + stepMs()) {
_stage = 2; // stays here: the caller reboots
return REBOOT;
}
return NONE;
}
uint32_t thresholdMs() const { return ((uint32_t)_minutes * 60000UL) << _backoff; }
uint32_t stepMs() const {
uint32_t step = thresholdMs() / 4;
return step < MIN_STEP_MS ? MIN_STEP_MS : step;
}
uint32_t quietMs(uint32_t now) const { return (uint32_t)(now - _quiet_since); }
uint8_t stage() const { return _stage; }
uint8_t backoff() const { return _backoff; }
// Backoff for the boot after a reboot by this watchdog.
uint8_t nextBackoff() const { return _backoff < MAX_BACKOFF ? _backoff + 1 : MAX_BACKOFF; }
// True once after the radio proved itself and the persisted backoff went stale.
bool takeBackoffCleared() {
bool c = _backoff_cleared;
_backoff_cleared = false;
return c;
}
private:
uint32_t _last_activity = 0;
uint32_t _quiet_since = 0;
uint32_t _last_check = 0;
uint16_t _minutes = 0;
uint8_t _backoff = 0;
uint8_t _stage = 0;
bool _backoff_cleared = false;
};
@@ -89,6 +89,12 @@ void RadioLibWrapper::doResetAGC() {
_radio->sleep(); // warm sleep to reset analog frontend
}
void RadioLibWrapper::kickRx() {
if (state == STATE_TX_WAIT) return; // mid-transmit: next check will try again
_radio->clearIrqFlags(0xFFFFFFFF); // a latched IRQ keeps DIO1 high: no new edge
idle(); // standby; Dispatcher re-arms RX on its next loop
}
void RadioLibWrapper::resetAGC() {
// make sure we're not mid-receive of packet!
if ((state & STATE_INT_READY) != 0 || isReceivingPacket()) return;
+3
View File
@@ -84,6 +84,9 @@ public:
void loop() override;
uint32_t getPacketsRecv() const { return n_recv; }
// CUSTOM (TeTeHacko): first recovery step of the radio silence watchdog --
// drop latched IRQs and re-arm RX. Leaves a transmit in progress alone.
void kickRx();
uint32_t getPacketsRecvErrors() const { return n_recv_errors; }
uint32_t getPacketsSent() const { return n_sent; }
void resetStats() { n_recv = n_sent = n_recv_errors = 0; }
@@ -0,0 +1,175 @@
#include <gtest/gtest.h>
#include "helpers/RadioSilenceWatchdog.h"
using A = RadioSilenceWatchdog;
static const uint32_t MIN = 60000;
// Polls every CHECK_MS from `from` to `to` with a constant activity counter and
// returns the first non-NONE action (and where it happened).
static A::Action runQuiet(A& wd, uint32_t from, uint32_t to, uint32_t activity, uint32_t* at = nullptr) {
for (uint32_t t = from; (uint32_t)(t - from) <= (uint32_t)(to - from); t += A::CHECK_MS) {
A::Action a = wd.poll(t, activity);
if (a != A::NONE) {
if (at) *at = t;
return a;
}
}
return A::NONE;
}
TEST(RadioSilenceWatchdog, OffNeverActs) {
A wd;
wd.begin(0, 0, 0);
wd.setMinutes(0);
EXPECT_EQ(runQuiet(wd, 0, 48 * 60 * MIN, 0), A::NONE);
}
TEST(RadioSilenceWatchdog, KicksAtThresholdThenReboots) {
A wd;
wd.begin(0, 0, 0);
wd.setMinutes(60);
uint32_t at = 0;
EXPECT_EQ(runQuiet(wd, 0, 61 * MIN, 0, &at), A::KICK_RX);
EXPECT_GE(at, 60 * MIN);
EXPECT_LT(at, 60 * MIN + A::CHECK_MS + 1);
// step = max(60/4 min, 5 min) = 15 min after the threshold
EXPECT_EQ(runQuiet(wd, at + A::CHECK_MS, at + 20 * MIN, 0, &at), A::REBOOT);
EXPECT_GE(at, 75 * MIN);
EXPECT_LT(at, 75 * MIN + A::CHECK_MS + 1);
}
TEST(RadioSilenceWatchdog, ShortThresholdUsesMinimumStep) {
A wd;
wd.begin(0, 0, 0);
wd.setMinutes(4);
uint32_t at = 0;
EXPECT_EQ(runQuiet(wd, 0, 5 * MIN, 0, &at), A::KICK_RX);
EXPECT_EQ(runQuiet(wd, at + A::CHECK_MS, at + 10 * MIN, 0, &at), A::REBOOT);
EXPECT_GE(at, 9 * MIN); // 4 min + MIN_STEP_MS (5 min)
}
TEST(RadioSilenceWatchdog, TrafficKeepsItHealthy) {
A wd;
wd.begin(0, 0, 0);
wd.setMinutes(30);
uint32_t rx = 0;
// one frame every 20 minutes, for two days
for (uint32_t t = A::CHECK_MS; t < 48 * 60 * MIN; t += A::CHECK_MS) {
if (t % (20 * MIN) == 0) rx++;
EXPECT_EQ(wd.poll(t, rx), A::NONE) << "t=" << t;
}
}
TEST(RadioSilenceWatchdog, KickThatWorksCancelsTheReboot) {
A wd;
wd.begin(0, 7, 0);
wd.setMinutes(10);
uint32_t at = 0;
ASSERT_EQ(runQuiet(wd, 0, 11 * MIN, 7, &at), A::KICK_RX);
// the re-armed receiver hears something
EXPECT_EQ(wd.poll(at + A::CHECK_MS, 8), A::NONE);
EXPECT_EQ(wd.stage(), 0);
// and the next silence starts over from the threshold, not from the step
uint32_t again = 0;
EXPECT_EQ(runQuiet(wd, at + 2 * A::CHECK_MS, at + 30 * MIN, 8, &again), A::KICK_RX);
EXPECT_GE(again - (at + A::CHECK_MS), 10 * MIN);
}
TEST(RadioSilenceWatchdog, ReceiveErrorsCountAsActivity) {
// the caller feeds recv + recv_errors; only the counter moving matters
A wd;
wd.begin(0, 100, 0);
wd.setMinutes(10);
uint32_t c = 100;
for (uint32_t t = A::CHECK_MS; t < 5 * 60 * MIN; t += A::CHECK_MS) {
if (t % (5 * MIN) == 0) c++;
EXPECT_EQ(wd.poll(t, c), A::NONE);
}
}
TEST(RadioSilenceWatchdog, BackoffDoublesThresholdAndCaps) {
A wd;
wd.begin(0, 0, 2);
wd.setMinutes(60);
EXPECT_EQ(wd.thresholdMs(), 4u * 60 * MIN);
EXPECT_EQ(wd.nextBackoff(), 3);
uint32_t at = 0;
EXPECT_EQ(runQuiet(wd, 0, 4 * 60 * MIN + MIN, 0, &at), A::KICK_RX);
EXPECT_GE(at, 4u * 60 * MIN);
A capped;
capped.begin(0, 0, 9);
EXPECT_EQ(capped.backoff(), A::MAX_BACKOFF);
EXPECT_EQ(capped.nextBackoff(), A::MAX_BACKOFF);
capped.setMinutes(A::MAX_MINUTES);
EXPECT_EQ(capped.thresholdMs(), (uint32_t)A::MAX_MINUTES * MIN * 16); // fits in 32 bits
}
TEST(RadioSilenceWatchdog, FirstFrameClearsBackoffOnce) {
A wd;
wd.begin(0, 0, 3);
wd.setMinutes(60);
EXPECT_FALSE(wd.takeBackoffCleared());
EXPECT_EQ(wd.poll(A::CHECK_MS, 1), A::NONE);
EXPECT_EQ(wd.backoff(), 0);
EXPECT_TRUE(wd.takeBackoffCleared());
EXPECT_FALSE(wd.takeBackoffCleared());
EXPECT_EQ(wd.thresholdMs(), 60u * MIN);
}
TEST(RadioSilenceWatchdog, NoClearWithoutBackoff) {
A wd;
wd.begin(0, 0, 0);
wd.setMinutes(60);
wd.poll(A::CHECK_MS, 1);
EXPECT_FALSE(wd.takeBackoffCleared());
}
TEST(RadioSilenceWatchdog, SurvivesMillisWrap) {
// traffic right up to the wrap, then a healthy receiver after it: no action
A wd;
uint32_t start = 0xFFFFFFFFu - 30 * MIN;
wd.begin(start, 0, 0);
wd.setMinutes(20);
uint32_t rx = 0;
for (uint32_t i = 1; i < 240; i++) {
uint32_t t = start + i * A::CHECK_MS; // wraps around zero half-way
if (i % 10 == 0) rx++;
EXPECT_EQ(wd.poll(t, rx), A::NONE) << "i=" << i;
}
}
TEST(RadioSilenceWatchdog, GenuineSilenceAcrossWrapStillDetected) {
A wd;
uint32_t start = 0xFFFFFFFFu - 5 * MIN;
wd.begin(start, 0, 0);
wd.setMinutes(20);
uint32_t at = 0;
EXPECT_EQ(runQuiet(wd, start, start + 21 * MIN, 0, &at), A::KICK_RX);
EXPECT_GE((uint32_t)(at - start), 20 * MIN);
}
TEST(RadioSilenceWatchdog, PacedToCheckInterval) {
A wd;
wd.begin(0, 0, 0);
wd.setMinutes(1);
// polled every millisecond the logic still runs only on CHECK_MS boundaries:
// one kick, exactly at the check that first sees the 1 min threshold
int actions = 0;
uint32_t kicked_at = 0;
for (uint32_t t = 1; t < 3 * MIN; t++) {
A::Action a = wd.poll(t, 0);
if (a != A::NONE) {
actions++;
if (a == A::KICK_RX) kicked_at = t;
}
}
EXPECT_EQ(kicked_at, 2 * A::CHECK_MS);
EXPECT_EQ(actions, 1); // the reboot step (5 min) is not due yet
}
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}