Repeater: TX stall watchdog -- packets queued, nothing sent for 15 min: re-arm, then reboot
pub-release / release (push) Successful in 39m1s

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
tthandClaude Opus 5.5 committed 2026-11-14 11:02:58 +01:00
1 parent 01d66a66ce
commit d9c7b9c008
6 files changed
+167 -4

No files matched your search

+14
View File
@@ -774,6 +774,20 @@ To keep a quiet site out of a reboot loop, each such reboot doubles the threshol
---
#### TX stall watchdog (repeater)
**Usage:**
- `get radio.txstall`
- `set radio.txstall <minutes>`
**Parameters:**
- `minutes`: 1-1440, or `0`/`off` to disable. Not saved: the build default (`RADIO_TXSTALL_MIN`) is back after a reboot.
**Description:** Recovers a node that still receives but no longer transmits. If packets wait in the outbound queue and the sent counter does not move for `minutes`, the radio is re-armed (IRQs cleared, back to standby); if still nothing goes out within a further quarter of the threshold (at least 5 minutes), the node reboots. An empty queue never counts, so a quiet site cannot trigger it. `get radio.txstall` shows the threshold, the queue (in total and due now), how long nothing has been sent and the stage (0 healthy, 1 re-armed). Its reboots are counted in `get radio.reboots` and are logged with the queue numbers.
**Default:** `15`
---
#### Reboot into the bootloader (repeater, sensor, room server; nRF52, local console only)
**Usage:**
- `dfu [uf2|serial|ota]`
+3
View File
@@ -521,6 +521,9 @@ public:
// To check if there is pending work
bool hasPendingWork() const;
// CUSTOM (TeTeHacko): for the TX stall watchdog (helpers/NodeMaintenance.h).
int txQueued() const { return _mgr->getOutboundTotal(); }
int txDue() const { return _mgr->getOutboundCount(_ms->getMillis()); }
bool setRxBoostedGain(bool enable) override;
+1 -1
View File
@@ -1129,7 +1129,7 @@ void loop() {
#endif
the_mesh.loop();
maint.loop(); // CUSTOM (TeTeHacko): deferred `dfu`, deaf-receiver watchdog
maint.loop(the_mesh.txQueued(), the_mesh.txDue()); // CUSTOM (TeTeHacko): deferred `dfu`, deaf-receiver and TX stall watchdogs
sensors.loop();
#ifdef DISPLAY_CLASS
ui_task.loop();
+77 -1
View File
@@ -5,6 +5,10 @@
#define RADIO_SILENCE_FILE "/rsilence"
#ifndef RADIO_TXSTALL_MIN
#define RADIO_TXSTALL_MIN 15
#endif
#if defined(NRF52_PLATFORM)
extern "C" {
void enterUf2Dfu(void);
@@ -93,13 +97,16 @@ void NodeMaintenance::begin(FILESYSTEM* fs, NodePrefs* prefs, CommonCLICallbacks
silenceLoad(&backoff);
_silence.begin(millis(), radioActivity(), backoff);
_silence.setMinutes(_prefs->radio_silence_min);
_txstall.begin(millis(), _radio.getPacketsSent(), 0);
_txstall.setMinutes(RADIO_TXSTALL_MIN);
Serial.printf("boot: radio silence watchdog %u min, reboots %lu, backoff x%u\n",
(unsigned) _silence.minutes(), (unsigned long) _silence_reboots,
(unsigned) (1u << _silence.backoff()));
}
void NodeMaintenance::loop() {
void NodeMaintenance::loop(int tx_queued, int tx_due) {
dfuTick();
txStallTick(tx_queued, tx_due);
#ifdef FW_OTA
_fw.loop();
#endif
@@ -123,6 +130,74 @@ void NodeMaintenance::loop() {
if (_silence.takeBackoffCleared()) silenceSave(0);
}
// --- TX stall watchdog ----------------------------------------------------------
// The other half of the silence watchdog: a node that hears fine but no longer
// transmits. mast-three, 14. 11. 2026, during an rxps trial: `sent` frozen for 6+
// hours with the outbound queue full (32), RX counting, no tx_start_fail, no
// tx_timeout, no CAD timeout -- so no adverts, no replies, no admin login over
// the mesh, and nothing the silence watchdog could see. `rxps off` did not bring
// TX back, a reboot did. The cause is not known, so this keys on the symptom:
// packets waiting while the sent counter does not move. Never normal -- the
// longest legitimate holds (airtime budget, CAD retries) are seconds.
// Same staging as the silence watchdog: re-arm the radio (clears latched IRQ
// flags), then reboot. No persisted backoff: a node that cannot send is useless
// either way, and the queue-length/due numbers in the log say where it hung.
void NodeMaintenance::txStallTick(int tx_queued, int tx_due) {
_tx_queued = tx_queued;
_tx_due = tx_due;
uint32_t now = millis();
if (tx_queued <= 0) { // nothing waiting (or not reported): nothing can stall
_txstall.begin(now, _radio.getPacketsSent(), 0);
return;
}
switch (_txstall.poll(now, _radio.getPacketsSent())) {
case RadioSilenceWatchdog::KICK_RX:
Serial.printf("tx stall: %d queued (%d due), nothing sent for %lu min, re-arming radio\n",
tx_queued, tx_due, (unsigned long) (_txstall.quietMs(now) / 60000));
_radio.kickRx();
break;
case RadioSilenceWatchdog::REBOOT:
Serial.printf("tx stall: %d queued (%d due), still nothing sent after %lu min, rebooting\n",
tx_queued, tx_due, (unsigned long) (_txstall.quietMs(now) / 60000));
_silence_reboots++;
silenceSave(_silence.backoff());
delay(100); // let the line out
_board.reboot();
break;
default:
break;
}
}
bool NodeMaintenance::txStallCommand(const char* command, char* reply) {
if (strcmp(command, "get radio.txstall") == 0) {
if (_tx_queued < 0) {
strcpy(reply, "> n/a (this role does not report its queue)");
} else if (_txstall.minutes() == 0) {
strcpy(reply, "> off");
} else {
sprintf(reply, "> %u min, queued %d (due %d), no TX for %lu min, stage %u",
(unsigned) _txstall.minutes(), _tx_queued, _tx_due,
_tx_queued > 0 ? (unsigned long) (_txstall.quietMs(millis()) / 60000) : 0UL,
(unsigned) _txstall.stage());
}
return true;
}
if (memcmp(command, "set radio.txstall ", 18) == 0) {
const char* arg = &command[18];
bool numeric = *arg != 0 && strspn(arg, "0123456789") == strlen(arg);
int m = strcmp(arg, "off") == 0 ? 0 : (numeric ? atoi(arg) : -1);
if (m < 0 || m > RadioSilenceWatchdog::MAX_MINUTES) {
sprintf(reply, "ERR - minutes 0..%u (0/off = disabled)", (unsigned) RadioSilenceWatchdog::MAX_MINUTES);
return true;
}
_txstall.setMinutes((uint16_t) m);
strcpy(reply, "OK (until reboot)");
return true;
}
return false;
}
bool NodeMaintenance::silenceCommand(const char* command, char* reply) {
if (strcmp(command, "get radio.reboots") == 0) {
sprintf(reply, "> %lu", (unsigned long) _silence_reboots);
@@ -255,6 +330,7 @@ bool NodeMaintenance::handleCommand(uint32_t sender_timestamp, const char* comma
if (_fw.handleCommand(sender_timestamp, command, reply)) return true;
#endif
if (silenceCommand(command, reply)) return true;
if (txStallCommand(command, reply)) return true;
if (strcmp(command, "get boot") == 0) {
#if defined(NRF52_PLATFORM)
if (_hardfault_seen) {
+10 -2
View File
@@ -16,12 +16,15 @@
// set radio.silence <min> radio silence watchdog, 0/off = disabled (#6)
// get radio.silence threshold, backoff, quiet time, stage
// get radio.reboots reboots done by the watchdog, survives power loss
// get radio.txstall TX stall watchdog: threshold, queue, time without TX
// set radio.txstall <min> 0/off = disabled; RAM only, RADIO_TXSTALL_MIN at boot
// get boot why the board last reset, and the hardfault record
//
// Wiring (see simple_sensor/main.cpp):
// setup(): maint.printBootDiag() early, maint.begin(fs, prefs, cb) after the
// mesh has loaded its prefs
// loop(): maint.loop()
// loop(): maint.loop(), or maint.loop(queued, due) to arm the TX stall
// watchdog (outbound packets in total / due now)
// commands: if (maint.handleCommand(sender_timestamp, command, reply)) return;
#include <Arduino.h>
@@ -46,7 +49,8 @@ public:
void printBootDiag();
void begin(FILESYSTEM* fs, NodePrefs* prefs, CommonCLICallbacks* callbacks);
void loop();
// tx_queued < 0: the role does not report its queue, no TX stall watchdog.
void loop(int tx_queued = -1, int tx_due = -1);
// sender_timestamp == 0 means the local console (the same test CommonCLI uses
// for `erase`); `dfu` refuses anything else.
@@ -70,6 +74,8 @@ private:
void silenceSave(uint8_t backoff);
bool dfuCommand(uint32_t sender_timestamp, const char* command, char* reply);
bool silenceCommand(const char* command, char* reply);
void txStallTick(int tx_queued, int tx_due);
bool txStallCommand(const char* command, char* reply);
void dfuTick();
mesh::MainBoard& _board;
@@ -83,6 +89,8 @@ private:
#endif
RadioSilenceWatchdog _silence;
uint32_t _silence_reboots = 0;
RadioSilenceWatchdog _txstall; // same staging, `activity` = packets sent
int _tx_queued = -1, _tx_due = -1;
unsigned long _dfu_at = 0; // 0 = nothing armed
uint8_t _dfu_mode = 0; // 0=uf2 1=serial 2=ota
@@ -169,6 +169,68 @@ TEST(RadioSilenceWatchdog, PacedToCheckInterval) {
EXPECT_EQ(actions, 1); // the reboot step (5 min) is not due yet
}
// The TX stall watchdog (NodeMaintenance::txStallTick) drives the same class:
// begin() on every pass while the outbound queue is empty, poll() with the sent
// counter while something waits. Mirror of that glue, one pass per ms.
static A::Action txStallPass(A& wd, uint32_t now, int queued, uint32_t sent) {
if (queued <= 0) {
wd.begin(now, sent, 0);
return A::NONE;
}
return wd.poll(now, sent);
}
TEST(TxStallWatchdog, IdleQueueNeverActs) {
A wd;
wd.begin(0, 0, 0);
wd.setMinutes(15);
for (uint32_t t = 0; t < 24 * 60 * MIN; t += 1000) EXPECT_EQ(txStallPass(wd, t, 0, 0), A::NONE);
}
TEST(TxStallWatchdog, LongIdleThenQueuedStartsFromZero) {
A wd;
wd.begin(0, 0, 0);
wd.setMinutes(15);
for (uint32_t t = 0; t < 10 * 60 * MIN; t += 1000) txStallPass(wd, t, 0, 7);
// hours with an empty queue must not count: a packet queued now gets 15 min
uint32_t start = 10 * 60 * MIN, kicked = 0;
for (uint32_t t = start; t < start + 20 * MIN; t += 1000) {
if (txStallPass(wd, t, 3, 7) == A::KICK_RX) { kicked = t; break; }
}
EXPECT_GE(kicked - start, 15 * MIN - 1000); // counted from the last empty-queue pass
EXPECT_LT(kicked - start, 15 * MIN + A::CHECK_MS + 1000);
}
TEST(TxStallWatchdog, BusyButSendingNeverActs) {
A wd;
wd.begin(0, 0, 0);
wd.setMinutes(15);
// a queue that never drains (busy repeater) is fine while packets go out
uint32_t sent = 0;
for (uint32_t t = 0; t < 6 * 60 * MIN; t += 1000) {
if (t % (2 * MIN) == 0) sent++;
EXPECT_EQ(txStallPass(wd, t, 32, sent), A::NONE);
}
}
TEST(TxStallWatchdog, FrozenSentKicksThenReboots) {
A wd;
wd.begin(0, 0, 0);
wd.setMinutes(15);
// mast-three 14. 11. 2026: queue full, sent frozen
A::Action first = A::NONE, second = A::NONE;
uint32_t t1 = 0, t2 = 0;
for (uint32_t t = 0; t < 30 * MIN; t += 1000) {
A::Action a = txStallPass(wd, t, 32, 3444);
if (a != A::NONE && first == A::NONE) { first = a; t1 = t; }
else if (a != A::NONE) { second = a; t2 = t; break; }
}
EXPECT_EQ(first, A::KICK_RX);
EXPECT_EQ(second, A::REBOOT);
EXPECT_GE(t1, 15 * MIN);
EXPECT_GE(t2 - t1, 5 * MIN); // step = max(15/4, 5) min
}
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();