From d9c7b9c0080459528c901d0dafed7ff4c736e8c4 Mon Sep 17 00:00:00 2001 From: TeTeHacko Date: Sat, 14 Nov 2026 11:02:58 +0100 Subject: [PATCH] Repeater: TX stall watchdog -- packets queued, nothing sent for 15 min: re-arm, then reboot Co-Authored-By: Claude Opus 5.5 --- docs/cli_commands.md | 14 ++++ examples/simple_repeater/MyMesh.h | 3 + examples/simple_repeater/main.cpp | 2 +- src/helpers/NodeMaintenance.cpp | 78 ++++++++++++++++++- src/helpers/NodeMaintenance.h | 12 ++- .../test_radio_silence_watchdog.cpp | 62 +++++++++++++++ 6 files changed, 167 insertions(+), 4 deletions(-) diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 9048be9c..f85ba18a 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -774,6 +774,20 @@ To keep a quiet site out of a reboot loop, each such reboot doubles the threshol --- +#### TX stall watchdog (repeater) +**Usage:** +- `get radio.txstall` +- `set radio.txstall ` + +**Parameters:** +- `minutes`: 1-1440, or `0`/`off` to disable. Not saved: the build default (`RADIO_TXSTALL_MIN`) is back after a reboot. + +**Description:** Recovers a node that still receives but no longer transmits. If packets wait in the outbound queue and the sent counter does not move for `minutes`, the radio is re-armed (IRQs cleared, back to standby); if still nothing goes out within a further quarter of the threshold (at least 5 minutes), the node reboots. An empty queue never counts, so a quiet site cannot trigger it. `get radio.txstall` shows the threshold, the queue (in total and due now), how long nothing has been sent and the stage (0 healthy, 1 re-armed). Its reboots are counted in `get radio.reboots` and are logged with the queue numbers. + +**Default:** `15` + +--- + #### Reboot into the bootloader (repeater, sensor, room server; nRF52, local console only) **Usage:** - `dfu [uf2|serial|ota]` diff --git a/examples/simple_repeater/MyMesh.h b/examples/simple_repeater/MyMesh.h index 761dd5d8..e3c52dd3 100644 --- a/examples/simple_repeater/MyMesh.h +++ b/examples/simple_repeater/MyMesh.h @@ -521,6 +521,9 @@ public: // To check if there is pending work bool hasPendingWork() const; + // CUSTOM (TeTeHacko): for the TX stall watchdog (helpers/NodeMaintenance.h). + int txQueued() const { return _mgr->getOutboundTotal(); } + int txDue() const { return _mgr->getOutboundCount(_ms->getMillis()); } bool setRxBoostedGain(bool enable) override; diff --git a/examples/simple_repeater/main.cpp b/examples/simple_repeater/main.cpp index b05f5524..3055c673 100644 --- a/examples/simple_repeater/main.cpp +++ b/examples/simple_repeater/main.cpp @@ -1129,7 +1129,7 @@ void loop() { #endif the_mesh.loop(); - maint.loop(); // CUSTOM (TeTeHacko): deferred `dfu`, deaf-receiver watchdog + maint.loop(the_mesh.txQueued(), the_mesh.txDue()); // CUSTOM (TeTeHacko): deferred `dfu`, deaf-receiver and TX stall watchdogs sensors.loop(); #ifdef DISPLAY_CLASS ui_task.loop(); diff --git a/src/helpers/NodeMaintenance.cpp b/src/helpers/NodeMaintenance.cpp index 10df4f9d..50682bad 100644 --- a/src/helpers/NodeMaintenance.cpp +++ b/src/helpers/NodeMaintenance.cpp @@ -5,6 +5,10 @@ #define RADIO_SILENCE_FILE "/rsilence" +#ifndef RADIO_TXSTALL_MIN +#define RADIO_TXSTALL_MIN 15 +#endif + #if defined(NRF52_PLATFORM) extern "C" { void enterUf2Dfu(void); @@ -93,13 +97,16 @@ void NodeMaintenance::begin(FILESYSTEM* fs, NodePrefs* prefs, CommonCLICallbacks silenceLoad(&backoff); _silence.begin(millis(), radioActivity(), backoff); _silence.setMinutes(_prefs->radio_silence_min); + _txstall.begin(millis(), _radio.getPacketsSent(), 0); + _txstall.setMinutes(RADIO_TXSTALL_MIN); Serial.printf("boot: radio silence watchdog %u min, reboots %lu, backoff x%u\n", (unsigned) _silence.minutes(), (unsigned long) _silence_reboots, (unsigned) (1u << _silence.backoff())); } -void NodeMaintenance::loop() { +void NodeMaintenance::loop(int tx_queued, int tx_due) { dfuTick(); + txStallTick(tx_queued, tx_due); #ifdef FW_OTA _fw.loop(); #endif @@ -123,6 +130,74 @@ void NodeMaintenance::loop() { if (_silence.takeBackoffCleared()) silenceSave(0); } +// --- TX stall watchdog ---------------------------------------------------------- +// The other half of the silence watchdog: a node that hears fine but no longer +// transmits. mast-three, 14. 11. 2026, during an rxps trial: `sent` frozen for 6+ +// hours with the outbound queue full (32), RX counting, no tx_start_fail, no +// tx_timeout, no CAD timeout -- so no adverts, no replies, no admin login over +// the mesh, and nothing the silence watchdog could see. `rxps off` did not bring +// TX back, a reboot did. The cause is not known, so this keys on the symptom: +// packets waiting while the sent counter does not move. Never normal -- the +// longest legitimate holds (airtime budget, CAD retries) are seconds. +// Same staging as the silence watchdog: re-arm the radio (clears latched IRQ +// flags), then reboot. No persisted backoff: a node that cannot send is useless +// either way, and the queue-length/due numbers in the log say where it hung. +void NodeMaintenance::txStallTick(int tx_queued, int tx_due) { + _tx_queued = tx_queued; + _tx_due = tx_due; + uint32_t now = millis(); + if (tx_queued <= 0) { // nothing waiting (or not reported): nothing can stall + _txstall.begin(now, _radio.getPacketsSent(), 0); + return; + } + switch (_txstall.poll(now, _radio.getPacketsSent())) { + case RadioSilenceWatchdog::KICK_RX: + Serial.printf("tx stall: %d queued (%d due), nothing sent for %lu min, re-arming radio\n", + tx_queued, tx_due, (unsigned long) (_txstall.quietMs(now) / 60000)); + _radio.kickRx(); + break; + case RadioSilenceWatchdog::REBOOT: + Serial.printf("tx stall: %d queued (%d due), still nothing sent after %lu min, rebooting\n", + tx_queued, tx_due, (unsigned long) (_txstall.quietMs(now) / 60000)); + _silence_reboots++; + silenceSave(_silence.backoff()); + delay(100); // let the line out + _board.reboot(); + break; + default: + break; + } +} + +bool NodeMaintenance::txStallCommand(const char* command, char* reply) { + if (strcmp(command, "get radio.txstall") == 0) { + if (_tx_queued < 0) { + strcpy(reply, "> n/a (this role does not report its queue)"); + } else if (_txstall.minutes() == 0) { + strcpy(reply, "> off"); + } else { + sprintf(reply, "> %u min, queued %d (due %d), no TX for %lu min, stage %u", + (unsigned) _txstall.minutes(), _tx_queued, _tx_due, + _tx_queued > 0 ? (unsigned long) (_txstall.quietMs(millis()) / 60000) : 0UL, + (unsigned) _txstall.stage()); + } + return true; + } + if (memcmp(command, "set radio.txstall ", 18) == 0) { + const char* arg = &command[18]; + bool numeric = *arg != 0 && strspn(arg, "0123456789") == strlen(arg); + int m = strcmp(arg, "off") == 0 ? 0 : (numeric ? atoi(arg) : -1); + if (m < 0 || m > RadioSilenceWatchdog::MAX_MINUTES) { + sprintf(reply, "ERR - minutes 0..%u (0/off = disabled)", (unsigned) RadioSilenceWatchdog::MAX_MINUTES); + return true; + } + _txstall.setMinutes((uint16_t) m); + strcpy(reply, "OK (until reboot)"); + return true; + } + return false; +} + bool NodeMaintenance::silenceCommand(const char* command, char* reply) { if (strcmp(command, "get radio.reboots") == 0) { sprintf(reply, "> %lu", (unsigned long) _silence_reboots); @@ -255,6 +330,7 @@ bool NodeMaintenance::handleCommand(uint32_t sender_timestamp, const char* comma if (_fw.handleCommand(sender_timestamp, command, reply)) return true; #endif if (silenceCommand(command, reply)) return true; + if (txStallCommand(command, reply)) return true; if (strcmp(command, "get boot") == 0) { #if defined(NRF52_PLATFORM) if (_hardfault_seen) { diff --git a/src/helpers/NodeMaintenance.h b/src/helpers/NodeMaintenance.h index f2a98715..46c00073 100644 --- a/src/helpers/NodeMaintenance.h +++ b/src/helpers/NodeMaintenance.h @@ -16,12 +16,15 @@ // set radio.silence radio silence watchdog, 0/off = disabled (#6) // get radio.silence threshold, backoff, quiet time, stage // get radio.reboots reboots done by the watchdog, survives power loss +// get radio.txstall TX stall watchdog: threshold, queue, time without TX +// set radio.txstall 0/off = disabled; RAM only, RADIO_TXSTALL_MIN at boot // get boot why the board last reset, and the hardfault record // // Wiring (see simple_sensor/main.cpp): // setup(): maint.printBootDiag() early, maint.begin(fs, prefs, cb) after the // mesh has loaded its prefs -// loop(): maint.loop() +// loop(): maint.loop(), or maint.loop(queued, due) to arm the TX stall +// watchdog (outbound packets in total / due now) // commands: if (maint.handleCommand(sender_timestamp, command, reply)) return; #include @@ -46,7 +49,8 @@ public: void printBootDiag(); void begin(FILESYSTEM* fs, NodePrefs* prefs, CommonCLICallbacks* callbacks); - void loop(); + // tx_queued < 0: the role does not report its queue, no TX stall watchdog. + void loop(int tx_queued = -1, int tx_due = -1); // sender_timestamp == 0 means the local console (the same test CommonCLI uses // for `erase`); `dfu` refuses anything else. @@ -70,6 +74,8 @@ private: void silenceSave(uint8_t backoff); bool dfuCommand(uint32_t sender_timestamp, const char* command, char* reply); bool silenceCommand(const char* command, char* reply); + void txStallTick(int tx_queued, int tx_due); + bool txStallCommand(const char* command, char* reply); void dfuTick(); mesh::MainBoard& _board; @@ -83,6 +89,8 @@ private: #endif RadioSilenceWatchdog _silence; uint32_t _silence_reboots = 0; + RadioSilenceWatchdog _txstall; // same staging, `activity` = packets sent + int _tx_queued = -1, _tx_due = -1; unsigned long _dfu_at = 0; // 0 = nothing armed uint8_t _dfu_mode = 0; // 0=uf2 1=serial 2=ota diff --git a/test/test_radio_silence_watchdog/test_radio_silence_watchdog.cpp b/test/test_radio_silence_watchdog/test_radio_silence_watchdog.cpp index 2e9e69c1..2180dd7f 100644 --- a/test/test_radio_silence_watchdog/test_radio_silence_watchdog.cpp +++ b/test/test_radio_silence_watchdog/test_radio_silence_watchdog.cpp @@ -169,6 +169,68 @@ TEST(RadioSilenceWatchdog, PacedToCheckInterval) { EXPECT_EQ(actions, 1); // the reboot step (5 min) is not due yet } +// The TX stall watchdog (NodeMaintenance::txStallTick) drives the same class: +// begin() on every pass while the outbound queue is empty, poll() with the sent +// counter while something waits. Mirror of that glue, one pass per ms. +static A::Action txStallPass(A& wd, uint32_t now, int queued, uint32_t sent) { + if (queued <= 0) { + wd.begin(now, sent, 0); + return A::NONE; + } + return wd.poll(now, sent); +} + +TEST(TxStallWatchdog, IdleQueueNeverActs) { + A wd; + wd.begin(0, 0, 0); + wd.setMinutes(15); + for (uint32_t t = 0; t < 24 * 60 * MIN; t += 1000) EXPECT_EQ(txStallPass(wd, t, 0, 0), A::NONE); +} + +TEST(TxStallWatchdog, LongIdleThenQueuedStartsFromZero) { + A wd; + wd.begin(0, 0, 0); + wd.setMinutes(15); + for (uint32_t t = 0; t < 10 * 60 * MIN; t += 1000) txStallPass(wd, t, 0, 7); + // hours with an empty queue must not count: a packet queued now gets 15 min + uint32_t start = 10 * 60 * MIN, kicked = 0; + for (uint32_t t = start; t < start + 20 * MIN; t += 1000) { + if (txStallPass(wd, t, 3, 7) == A::KICK_RX) { kicked = t; break; } + } + EXPECT_GE(kicked - start, 15 * MIN - 1000); // counted from the last empty-queue pass + EXPECT_LT(kicked - start, 15 * MIN + A::CHECK_MS + 1000); +} + +TEST(TxStallWatchdog, BusyButSendingNeverActs) { + A wd; + wd.begin(0, 0, 0); + wd.setMinutes(15); + // a queue that never drains (busy repeater) is fine while packets go out + uint32_t sent = 0; + for (uint32_t t = 0; t < 6 * 60 * MIN; t += 1000) { + if (t % (2 * MIN) == 0) sent++; + EXPECT_EQ(txStallPass(wd, t, 32, sent), A::NONE); + } +} + +TEST(TxStallWatchdog, FrozenSentKicksThenReboots) { + A wd; + wd.begin(0, 0, 0); + wd.setMinutes(15); + // mast-three 14. 11. 2026: queue full, sent frozen + A::Action first = A::NONE, second = A::NONE; + uint32_t t1 = 0, t2 = 0; + for (uint32_t t = 0; t < 30 * MIN; t += 1000) { + A::Action a = txStallPass(wd, t, 32, 3444); + if (a != A::NONE && first == A::NONE) { first = a; t1 = t; } + else if (a != A::NONE) { second = a; t2 = t; break; } + } + EXPECT_EQ(first, A::KICK_RX); + EXPECT_EQ(second, A::REBOOT); + EXPECT_GE(t1, 15 * MIN); + EXPECT_GE(t2 - t1, 5 * MIN); // step = max(15/4, 5) min +} + int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS();