Repeater: TX stall watchdog -- packets queued, nothing sent for 15 min: re-arm, then reboot
pub-release / release (push) Successful in 39m1s
pub-release / release (push) Successful in 39m1s
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
01d66a66ce
commit
d9c7b9c008
6 files changed
+167
-4
No files matched your search
@@ -774,6 +774,20 @@ To keep a quiet site out of a reboot loop, each such reboot doubles the threshol
|
||||
|
||||
---
|
||||
|
||||
#### TX stall watchdog (repeater)
|
||||
**Usage:**
|
||||
- `get radio.txstall`
|
||||
- `set radio.txstall <minutes>`
|
||||
|
||||
**Parameters:**
|
||||
- `minutes`: 1-1440, or `0`/`off` to disable. Not saved: the build default (`RADIO_TXSTALL_MIN`) is back after a reboot.
|
||||
|
||||
**Description:** Recovers a node that still receives but no longer transmits. If packets wait in the outbound queue and the sent counter does not move for `minutes`, the radio is re-armed (IRQs cleared, back to standby); if still nothing goes out within a further quarter of the threshold (at least 5 minutes), the node reboots. An empty queue never counts, so a quiet site cannot trigger it. `get radio.txstall` shows the threshold, the queue (in total and due now), how long nothing has been sent and the stage (0 healthy, 1 re-armed). Its reboots are counted in `get radio.reboots` and are logged with the queue numbers.
|
||||
|
||||
**Default:** `15`
|
||||
|
||||
---
|
||||
|
||||
#### Reboot into the bootloader (repeater, sensor, room server; nRF52, local console only)
|
||||
**Usage:**
|
||||
- `dfu [uf2|serial|ota]`
|
||||
|
||||
@@ -521,6 +521,9 @@ public:
|
||||
|
||||
// To check if there is pending work
|
||||
bool hasPendingWork() const;
|
||||
// CUSTOM (TeTeHacko): for the TX stall watchdog (helpers/NodeMaintenance.h).
|
||||
int txQueued() const { return _mgr->getOutboundTotal(); }
|
||||
int txDue() const { return _mgr->getOutboundCount(_ms->getMillis()); }
|
||||
|
||||
bool setRxBoostedGain(bool enable) override;
|
||||
|
||||
|
||||
@@ -1129,7 +1129,7 @@ void loop() {
|
||||
#endif
|
||||
|
||||
the_mesh.loop();
|
||||
maint.loop(); // CUSTOM (TeTeHacko): deferred `dfu`, deaf-receiver watchdog
|
||||
maint.loop(the_mesh.txQueued(), the_mesh.txDue()); // CUSTOM (TeTeHacko): deferred `dfu`, deaf-receiver and TX stall watchdogs
|
||||
sensors.loop();
|
||||
#ifdef DISPLAY_CLASS
|
||||
ui_task.loop();
|
||||
|
||||
@@ -5,6 +5,10 @@
|
||||
|
||||
#define RADIO_SILENCE_FILE "/rsilence"
|
||||
|
||||
#ifndef RADIO_TXSTALL_MIN
|
||||
#define RADIO_TXSTALL_MIN 15
|
||||
#endif
|
||||
|
||||
#if defined(NRF52_PLATFORM)
|
||||
extern "C" {
|
||||
void enterUf2Dfu(void);
|
||||
@@ -93,13 +97,16 @@ void NodeMaintenance::begin(FILESYSTEM* fs, NodePrefs* prefs, CommonCLICallbacks
|
||||
silenceLoad(&backoff);
|
||||
_silence.begin(millis(), radioActivity(), backoff);
|
||||
_silence.setMinutes(_prefs->radio_silence_min);
|
||||
_txstall.begin(millis(), _radio.getPacketsSent(), 0);
|
||||
_txstall.setMinutes(RADIO_TXSTALL_MIN);
|
||||
Serial.printf("boot: radio silence watchdog %u min, reboots %lu, backoff x%u\n",
|
||||
(unsigned) _silence.minutes(), (unsigned long) _silence_reboots,
|
||||
(unsigned) (1u << _silence.backoff()));
|
||||
}
|
||||
|
||||
void NodeMaintenance::loop() {
|
||||
void NodeMaintenance::loop(int tx_queued, int tx_due) {
|
||||
dfuTick();
|
||||
txStallTick(tx_queued, tx_due);
|
||||
#ifdef FW_OTA
|
||||
_fw.loop();
|
||||
#endif
|
||||
@@ -123,6 +130,74 @@ void NodeMaintenance::loop() {
|
||||
if (_silence.takeBackoffCleared()) silenceSave(0);
|
||||
}
|
||||
|
||||
// --- TX stall watchdog ----------------------------------------------------------
|
||||
// The other half of the silence watchdog: a node that hears fine but no longer
|
||||
// transmits. mast-three, 14. 11. 2026, during an rxps trial: `sent` frozen for 6+
|
||||
// hours with the outbound queue full (32), RX counting, no tx_start_fail, no
|
||||
// tx_timeout, no CAD timeout -- so no adverts, no replies, no admin login over
|
||||
// the mesh, and nothing the silence watchdog could see. `rxps off` did not bring
|
||||
// TX back, a reboot did. The cause is not known, so this keys on the symptom:
|
||||
// packets waiting while the sent counter does not move. Never normal -- the
|
||||
// longest legitimate holds (airtime budget, CAD retries) are seconds.
|
||||
// Same staging as the silence watchdog: re-arm the radio (clears latched IRQ
|
||||
// flags), then reboot. No persisted backoff: a node that cannot send is useless
|
||||
// either way, and the queue-length/due numbers in the log say where it hung.
|
||||
void NodeMaintenance::txStallTick(int tx_queued, int tx_due) {
|
||||
_tx_queued = tx_queued;
|
||||
_tx_due = tx_due;
|
||||
uint32_t now = millis();
|
||||
if (tx_queued <= 0) { // nothing waiting (or not reported): nothing can stall
|
||||
_txstall.begin(now, _radio.getPacketsSent(), 0);
|
||||
return;
|
||||
}
|
||||
switch (_txstall.poll(now, _radio.getPacketsSent())) {
|
||||
case RadioSilenceWatchdog::KICK_RX:
|
||||
Serial.printf("tx stall: %d queued (%d due), nothing sent for %lu min, re-arming radio\n",
|
||||
tx_queued, tx_due, (unsigned long) (_txstall.quietMs(now) / 60000));
|
||||
_radio.kickRx();
|
||||
break;
|
||||
case RadioSilenceWatchdog::REBOOT:
|
||||
Serial.printf("tx stall: %d queued (%d due), still nothing sent after %lu min, rebooting\n",
|
||||
tx_queued, tx_due, (unsigned long) (_txstall.quietMs(now) / 60000));
|
||||
_silence_reboots++;
|
||||
silenceSave(_silence.backoff());
|
||||
delay(100); // let the line out
|
||||
_board.reboot();
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
bool NodeMaintenance::txStallCommand(const char* command, char* reply) {
|
||||
if (strcmp(command, "get radio.txstall") == 0) {
|
||||
if (_tx_queued < 0) {
|
||||
strcpy(reply, "> n/a (this role does not report its queue)");
|
||||
} else if (_txstall.minutes() == 0) {
|
||||
strcpy(reply, "> off");
|
||||
} else {
|
||||
sprintf(reply, "> %u min, queued %d (due %d), no TX for %lu min, stage %u",
|
||||
(unsigned) _txstall.minutes(), _tx_queued, _tx_due,
|
||||
_tx_queued > 0 ? (unsigned long) (_txstall.quietMs(millis()) / 60000) : 0UL,
|
||||
(unsigned) _txstall.stage());
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (memcmp(command, "set radio.txstall ", 18) == 0) {
|
||||
const char* arg = &command[18];
|
||||
bool numeric = *arg != 0 && strspn(arg, "0123456789") == strlen(arg);
|
||||
int m = strcmp(arg, "off") == 0 ? 0 : (numeric ? atoi(arg) : -1);
|
||||
if (m < 0 || m > RadioSilenceWatchdog::MAX_MINUTES) {
|
||||
sprintf(reply, "ERR - minutes 0..%u (0/off = disabled)", (unsigned) RadioSilenceWatchdog::MAX_MINUTES);
|
||||
return true;
|
||||
}
|
||||
_txstall.setMinutes((uint16_t) m);
|
||||
strcpy(reply, "OK (until reboot)");
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool NodeMaintenance::silenceCommand(const char* command, char* reply) {
|
||||
if (strcmp(command, "get radio.reboots") == 0) {
|
||||
sprintf(reply, "> %lu", (unsigned long) _silence_reboots);
|
||||
@@ -255,6 +330,7 @@ bool NodeMaintenance::handleCommand(uint32_t sender_timestamp, const char* comma
|
||||
if (_fw.handleCommand(sender_timestamp, command, reply)) return true;
|
||||
#endif
|
||||
if (silenceCommand(command, reply)) return true;
|
||||
if (txStallCommand(command, reply)) return true;
|
||||
if (strcmp(command, "get boot") == 0) {
|
||||
#if defined(NRF52_PLATFORM)
|
||||
if (_hardfault_seen) {
|
||||
|
||||
@@ -16,12 +16,15 @@
|
||||
// set radio.silence <min> radio silence watchdog, 0/off = disabled (#6)
|
||||
// get radio.silence threshold, backoff, quiet time, stage
|
||||
// get radio.reboots reboots done by the watchdog, survives power loss
|
||||
// get radio.txstall TX stall watchdog: threshold, queue, time without TX
|
||||
// set radio.txstall <min> 0/off = disabled; RAM only, RADIO_TXSTALL_MIN at boot
|
||||
// get boot why the board last reset, and the hardfault record
|
||||
//
|
||||
// Wiring (see simple_sensor/main.cpp):
|
||||
// setup(): maint.printBootDiag() early, maint.begin(fs, prefs, cb) after the
|
||||
// mesh has loaded its prefs
|
||||
// loop(): maint.loop()
|
||||
// loop(): maint.loop(), or maint.loop(queued, due) to arm the TX stall
|
||||
// watchdog (outbound packets in total / due now)
|
||||
// commands: if (maint.handleCommand(sender_timestamp, command, reply)) return;
|
||||
|
||||
#include <Arduino.h>
|
||||
@@ -46,7 +49,8 @@ public:
|
||||
void printBootDiag();
|
||||
|
||||
void begin(FILESYSTEM* fs, NodePrefs* prefs, CommonCLICallbacks* callbacks);
|
||||
void loop();
|
||||
// tx_queued < 0: the role does not report its queue, no TX stall watchdog.
|
||||
void loop(int tx_queued = -1, int tx_due = -1);
|
||||
|
||||
// sender_timestamp == 0 means the local console (the same test CommonCLI uses
|
||||
// for `erase`); `dfu` refuses anything else.
|
||||
@@ -70,6 +74,8 @@ private:
|
||||
void silenceSave(uint8_t backoff);
|
||||
bool dfuCommand(uint32_t sender_timestamp, const char* command, char* reply);
|
||||
bool silenceCommand(const char* command, char* reply);
|
||||
void txStallTick(int tx_queued, int tx_due);
|
||||
bool txStallCommand(const char* command, char* reply);
|
||||
void dfuTick();
|
||||
|
||||
mesh::MainBoard& _board;
|
||||
@@ -83,6 +89,8 @@ private:
|
||||
#endif
|
||||
RadioSilenceWatchdog _silence;
|
||||
uint32_t _silence_reboots = 0;
|
||||
RadioSilenceWatchdog _txstall; // same staging, `activity` = packets sent
|
||||
int _tx_queued = -1, _tx_due = -1;
|
||||
|
||||
unsigned long _dfu_at = 0; // 0 = nothing armed
|
||||
uint8_t _dfu_mode = 0; // 0=uf2 1=serial 2=ota
|
||||
|
||||
@@ -169,6 +169,68 @@ TEST(RadioSilenceWatchdog, PacedToCheckInterval) {
|
||||
EXPECT_EQ(actions, 1); // the reboot step (5 min) is not due yet
|
||||
}
|
||||
|
||||
// The TX stall watchdog (NodeMaintenance::txStallTick) drives the same class:
|
||||
// begin() on every pass while the outbound queue is empty, poll() with the sent
|
||||
// counter while something waits. Mirror of that glue, one pass per ms.
|
||||
static A::Action txStallPass(A& wd, uint32_t now, int queued, uint32_t sent) {
|
||||
if (queued <= 0) {
|
||||
wd.begin(now, sent, 0);
|
||||
return A::NONE;
|
||||
}
|
||||
return wd.poll(now, sent);
|
||||
}
|
||||
|
||||
TEST(TxStallWatchdog, IdleQueueNeverActs) {
|
||||
A wd;
|
||||
wd.begin(0, 0, 0);
|
||||
wd.setMinutes(15);
|
||||
for (uint32_t t = 0; t < 24 * 60 * MIN; t += 1000) EXPECT_EQ(txStallPass(wd, t, 0, 0), A::NONE);
|
||||
}
|
||||
|
||||
TEST(TxStallWatchdog, LongIdleThenQueuedStartsFromZero) {
|
||||
A wd;
|
||||
wd.begin(0, 0, 0);
|
||||
wd.setMinutes(15);
|
||||
for (uint32_t t = 0; t < 10 * 60 * MIN; t += 1000) txStallPass(wd, t, 0, 7);
|
||||
// hours with an empty queue must not count: a packet queued now gets 15 min
|
||||
uint32_t start = 10 * 60 * MIN, kicked = 0;
|
||||
for (uint32_t t = start; t < start + 20 * MIN; t += 1000) {
|
||||
if (txStallPass(wd, t, 3, 7) == A::KICK_RX) { kicked = t; break; }
|
||||
}
|
||||
EXPECT_GE(kicked - start, 15 * MIN - 1000); // counted from the last empty-queue pass
|
||||
EXPECT_LT(kicked - start, 15 * MIN + A::CHECK_MS + 1000);
|
||||
}
|
||||
|
||||
TEST(TxStallWatchdog, BusyButSendingNeverActs) {
|
||||
A wd;
|
||||
wd.begin(0, 0, 0);
|
||||
wd.setMinutes(15);
|
||||
// a queue that never drains (busy repeater) is fine while packets go out
|
||||
uint32_t sent = 0;
|
||||
for (uint32_t t = 0; t < 6 * 60 * MIN; t += 1000) {
|
||||
if (t % (2 * MIN) == 0) sent++;
|
||||
EXPECT_EQ(txStallPass(wd, t, 32, sent), A::NONE);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TxStallWatchdog, FrozenSentKicksThenReboots) {
|
||||
A wd;
|
||||
wd.begin(0, 0, 0);
|
||||
wd.setMinutes(15);
|
||||
// mast-three 14. 11. 2026: queue full, sent frozen
|
||||
A::Action first = A::NONE, second = A::NONE;
|
||||
uint32_t t1 = 0, t2 = 0;
|
||||
for (uint32_t t = 0; t < 30 * MIN; t += 1000) {
|
||||
A::Action a = txStallPass(wd, t, 32, 3444);
|
||||
if (a != A::NONE && first == A::NONE) { first = a; t1 = t; }
|
||||
else if (a != A::NONE) { second = a; t2 = t; break; }
|
||||
}
|
||||
EXPECT_EQ(first, A::KICK_RX);
|
||||
EXPECT_EQ(second, A::REBOOT);
|
||||
EXPECT_GE(t1, 15 * MIN);
|
||||
EXPECT_GE(t2 - t1, 5 * MIN); // step = max(15/4, 5) min
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
|
||||
Reference in new issue
Block a user