FwUpdate phase 1: only in FW_OTA images, manifest layout 2, no downgrade over the mesh

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
tthandClaude Opus 5.5 committed 2026-11-13 16:33:29 +01:00
1 parent c93f5d906d
commit 0b38eab008
7 files changed
+1107 -12

No files matched your search

+53 -9
View File
@@ -1,3 +1,7 @@
// Compiled only into the experimental -ota images (-D FW_OTA=1); a classic
// image has no firmware update at all (helpers/NodeMaintenance.h).
#ifdef FW_OTA
#include "FwUpdate.h"
#include "FwPatch.h"
@@ -36,17 +40,32 @@ extern "C" uint32_t __etext, __data_start__, __data_end__;
struct __attribute__((packed)) Manifest {
char magic[4]; // "MCFW"
uint8_t format; // 1 = deflate full image, 2 = in-place delta (FwPatch)
uint8_t reserved0[3];
uint8_t layout; // MANIFEST_LAYOUT; was a reserved 0 in the first layout
uint8_t reserved0[2];
uint32_t raw_size;
uint32_t comp_size;
uint8_t raw_sha[32];
uint8_t comp_sha[32];
char target[32];
char version[48];
char target[48]; // env + image variant, e.g. "SenseCap_Solar_repeater-observer-ota"
char version[28]; // FIRMWARE_VERSION, at most 19 chars (tools/build_version.py)
uint32_t build_epoch; // FIRMWARE_BUILD_EPOCH of the new image: no downgrade over the mesh
uint8_t base_sha[32]; // format 2: sha256 of the image the delta applies to
uint8_t signature[64]; // Ed25519 over bytes 0..191
};
#define MANIFEST_LAYOUT 2
// What a package for THIS image must carry, read by tools/ota_pack.py from the
// ELF -- the target string and the epoch come from the build, never from a
// command line that could disagree with it. `fw ver` prints it (and keeps it
// in the image: an unreferenced string would fall to --gc-sections).
#define FWU_STR2(x) #x
#define FWU_STR(x) FWU_STR2(x)
static const char FW_OTA_TAGS[] = "MCFW-TAGS target=" FW_OTA_TARGET " epoch=" FWU_STR(FIRMWARE_BUILD_EPOCH);
static_assert(sizeof(Manifest) == FW_OTA_MANIFEST_SIZE, "manifest layout");
static_assert(offsetof(Manifest, target) == 80 && offsetof(Manifest, version) == 128 &&
offsetof(Manifest, build_epoch) == 156 && offsetof(Manifest, base_sha) == 160 &&
offsetof(Manifest, signature) == 192,
"manifest offsets shared with tools/ota_pack.py");
// --- inflate ------------------------------------------------------------------
// A small RFC 1951 decoder in the style of zlib's contrib/puff. EVERYTHING it
@@ -628,6 +647,19 @@ bool FwUpdate::checkManifest() {
strcpy(_err, "bad signature");
return false;
}
if (m->layout != MANIFEST_LAYOUT) {
strcpy(_err, "old package layout, repack it (tools/ota_pack.py)");
return false;
}
// Not newer than what runs (an older build, or this very one): never over the
// mesh -- a bad release is undone by a revert and a NEW release. Locally it
// takes `fw apply downgrade`.
_older = m->build_epoch <= (uint32_t)FIRMWARE_BUILD_EPOCH;
if (_older && _remote) {
sprintf(_err, "not newer than this image (pkg %lu, node %lu)", (unsigned long)m->build_epoch,
(unsigned long)FIRMWARE_BUILD_EPOCH);
return false;
}
if (strncmp(m->target, FW_OTA_TARGET, sizeof(m->target)) != 0) {
strcpy(_err, "wrong target");
return false;
@@ -719,6 +751,7 @@ void FwUpdate::cmdBegin(uint32_t sender_timestamp, const char *arg, char *reply)
_stage = stage;
_total = total;
_remote = sender_timestamp != 0;
_older = false;
memset(_have, 0, sizeof(_have));
_state = RECEIVING;
_manifest_ok = false;
@@ -890,8 +923,8 @@ void FwUpdate::doVerify() {
}
}
_raw_size = m->raw_size;
strncpy(_version, m->version, sizeof(_version) - 1);
_version[sizeof(_version) - 1] = 0;
memcpy(_version, m->version, sizeof(m->version)); // may fill all 28 bytes, no NUL
_version[sizeof(m->version)] = 0;
_state = READY;
return;
fail:
@@ -909,9 +942,15 @@ void FwUpdate::cmdApply(uint32_t sender_timestamp, const char *arg, char *reply)
strcpy(reply, "ERR: nothing verified, fw verify first");
return;
}
bool force = strcmp(arg, "force") == 0;
if (force && sender_timestamp != 0) {
strcpy(reply, "ERR: force is local-only");
// arguments: `force` (skip the battery gate) and/or `downgrade`, both local only
bool force = strstr(arg, "force") != NULL;
bool downgrade = strstr(arg, "downgrade") != NULL;
if ((force || downgrade) && sender_timestamp != 0) {
strcpy(reply, "ERR: force / downgrade are local-only");
return;
}
if (_older && !downgrade) {
strcpy(reply, "ERR: not newer than this image: fw apply downgrade (local only)");
return;
}
uint16_t bank0 = *(const volatile uint16_t *)BL_SETTINGS;
@@ -1049,6 +1088,9 @@ bool FwUpdate::handleCommand(uint32_t sender_timestamp, const char *command, cha
strcpy(reply, "OK");
} else if (strcmp(arg, "bl") == 0)
cmdBootloader(reply);
else if (strcmp(arg, "ver") == 0)
sprintf(reply, "> %s", FW_OTA_TAGS + 10); // "target=... epoch=..."
else if (strncmp(arg, "bin ", 4) == 0) {
// Bench: feed a binary REQ_TYPE_FW_OTA payload (base64) through the same
// handler the mesh uses -- padded to the AES block exactly as the node sees
@@ -1089,7 +1131,7 @@ bool FwUpdate::handleCommand(uint32_t sender_timestamp, const char *command, cha
}
}
else
strcpy(reply, "ERR: fw status|begin|w|missing|verify|apply|abort|bl|base");
strcpy(reply, "ERR: fw status|begin|w|missing|verify|apply|abort|bl|base|ver");
return true;
}
@@ -1103,3 +1145,5 @@ bool FwUpdate::handleCommand(uint32_t, const char *command, char *reply) {
void FwUpdate::loop() {}
#endif
#endif // FW_OTA
+4 -1
View File
@@ -23,7 +23,9 @@
// fw w <offset> <base64> one chunk, offset a multiple of FW_OTA_CHUNK
// fw missing first missing chunk offsets (for resending)
// fw verify signature + hashes + dry-run inflate
// fw apply [force] battery gate, then copy and reboot
// fw apply [force] [downgrade] battery gate, then copy and reboot; both
// arguments local only (no downgrade over the mesh)
// fw ver target and build epoch a package for this image needs
// fw abort forget the staged package
// fw bl print the bootloader settings page
@@ -113,6 +115,7 @@ private:
// request). Then only a DELTA is accepted: a full image is ~300 kB, hours of
// airtime on a shared channel -- it goes over USB or the BLE console.
bool _remote = false;
bool _older = false; // the package is not newer than this image (manifest build_epoch)
bool _base_known = false;
uint8_t _base_sha[32];
// The manifest is checked from this RAM copy, not from the stage in flash: three
+4
View File
@@ -100,7 +100,9 @@ void NodeMaintenance::begin(FILESYSTEM* fs, NodePrefs* prefs, CommonCLICallbacks
void NodeMaintenance::loop() {
dfuTick();
#ifdef FW_OTA
_fw.loop();
#endif
switch (_silence.poll(millis(), radioActivity())) {
case RadioSilenceWatchdog::KICK_RX:
Serial.printf("radio silence: nothing received for %lu min, re-arming RX\n",
@@ -249,7 +251,9 @@ bool NodeMaintenance::dfuCommand(uint32_t sender_timestamp, const char* command,
bool NodeMaintenance::handleCommand(uint32_t sender_timestamp, const char* command, char* reply) {
if (dfuCommand(sender_timestamp, command, reply)) return true;
#ifdef FW_OTA
if (_fw.handleCommand(sender_timestamp, command, reply)) return true;
#endif
if (silenceCommand(command, reply)) return true;
if (strcmp(command, "get boot") == 0) {
#if defined(NRF52_PLATFORM)
+14
View File
@@ -12,6 +12,7 @@
//
// dfu [uf2|serial|ota] reboot into the bootloader (local console only)
// fw ... admin-pushed firmware update, see helpers/FwUpdate.h
// (only with -D FW_OTA=1: the experimental -ota images)
// set radio.silence <min> radio silence watchdog, 0/off = disabled (#6)
// get radio.silence threshold, backoff, quiet time, stage
// get radio.reboots reboots done by the watchdog, survives power loss
@@ -26,13 +27,19 @@
#include <Arduino.h>
#include <Mesh.h>
#include <helpers/CommonCLI.h>
#ifdef FW_OTA
#include <helpers/FwUpdate.h>
#endif
#include <helpers/RadioSilenceWatchdog.h>
#include <helpers/radiolib/RadioLibWrappers.h>
class NodeMaintenance {
public:
#ifdef FW_OTA
NodeMaintenance(mesh::MainBoard& board, RadioLibWrapper& radio) : _board(board), _radio(radio), _fw(board) { }
#else
NodeMaintenance(mesh::MainBoard& board, RadioLibWrapper& radio) : _board(board), _radio(radio) { }
#endif
// Prints (and latches for `get boot`) the reset reason and the hardfault record.
// The record is cleared once read, so call it once, as early as Serial is up.
@@ -48,8 +55,13 @@ public:
// Binary request from a logged-in client (REQ_TYPE_FW_OTA, firmware chunks).
// Returns the reply length, 0 = no reply. Admins only.
int handleRequest(bool admin, const uint8_t* req, int len, uint8_t* reply, int reply_max) {
#ifdef FW_OTA
if (!admin || len < 1 || req[0] != REQ_TYPE_FW_OTA) return 0;
return _fw.handleBinary(req, len, reply, reply_max);
#else
(void)admin; (void)req; (void)len; (void)reply; (void)reply_max;
return 0; // a classic image: no firmware update, the request is dropped
#endif
}
private:
@@ -66,7 +78,9 @@ private:
NodePrefs* _prefs = NULL;
CommonCLICallbacks* _callbacks = NULL;
#ifdef FW_OTA
FwUpdate _fw;
#endif
RadioSilenceWatchdog _silence;
uint32_t _silence_reboots = 0;
+11 -2
View File
@@ -29,6 +29,7 @@ if __name__ == "__main__": # run by hand (--help): the header above, nothing el
raise SystemExit(0)
Import("env")
import os
import re
import subprocess
from datetime import datetime
@@ -117,11 +118,19 @@ build_date = now.strftime("%y%m%d %H%M")[:DATE_LIMIT]
# the tree is already fully recompiled each time.
build_epoch = int(now.timestamp())
# src/helpers/FwUpdate refuses a package built for another target: the env,
# plus the image variant when one env is released as several
# (FW_OTA_VARIANT=observer-ota -> "SenseCap_Solar_repeater-observer-ota"), or a
# full package would silently swap a node's variant.
ota_variant = os.environ.get("FW_OTA_VARIANT", "").strip()
ota_target = env["PIOENV"] + ("-" + ota_variant if ota_variant else "")
if len(ota_target) > 47: # Manifest.target[48] in FwUpdate.cpp, NUL-terminated
raise SystemExit("build_version.py: OTA target %r longer than 47 chars" % ota_target)
env.Append(CPPDEFINES=[
("FIRMWARE_VERSION", env.StringifyMacro(version)),
("FIRMWARE_BUILD_DATE", env.StringifyMacro(build_date)),
("FIRMWARE_BUILD_EPOCH", str(build_epoch)),
# src/helpers/FwUpdate refuses a package built for another env
("FW_OTA_TARGET", env.StringifyMacro(env["PIOENV"])),
("FW_OTA_TARGET", env.StringifyMacro(ota_target)),
])
print("build_version.py: %s (Build: %s, epoch %d)" % (version, build_date, build_epoch))
+512
View File
@@ -0,0 +1,512 @@
#!/usr/bin/env python3
"""Push an OTA package (tools/ota_pack.py) to a node OVER THE MESH, as admin.
tools/ota_mesh_push.py <companion> <dest> <package> [--apply] [--duty 1]
<companion> how to reach the companion that talks for us:
tcp://127.0.0.1:7003 (serial_tcp_proxy.py + ssh -L) or a
serial port path
<dest> the target node's public key prefix (>= 8 hex), as in its
`get public.key`
<package> a DELTA package only (ota_pack.py diff / fetch --base): a
full image is refused here and by the node itself
--apply after a good `fw verify`, `fw apply` and wait for the node to
come back with the new image (ver + fw status over the mesh)
--duty P airtime budget in percent per hour (default 1: the
865-868 MHz bench band). Token bucket of P% x 3600 s. It
counts what the transfer puts on the CHANNEL, not just our
radio: request and the expected reply, each times
(hops + 1), because every repeater on the path sends both
again.
--gap S pause after every reply before the next command (default 5).
Without it a small delta goes out as fast as the round trip
(~2 s per chunk at SF8) and holds the channel for that time.
--via HEX force a direct path through these repeater hashes (2 bytes
each, e.g. c327 or c327a1b2) -- also the login goes along
it, so the node answers back the same way. For testing
multi-hop on the bench.
--text send chunks as text CLI (`fw w <off> <base64>`, 96 B, each
answered) instead of the default binary requests (128 B,
unanswered, missing ones found by a bitmap query). Falls
back to text by itself when the node does not answer the
binary query (older firmware, or the sensor build).
--wait S how long to wait for the dest's advert if the companion
does not know it yet (default 120 s); send `advert.zerohop`
on the dest meanwhile
The same `fw` commands as tools/ota_push.py (USB), carried as remote admin CLI.
Every `fw w` is idempotent on the node, so a lost reply is simply resent; the
node checks the manifest as soon as its chunks arrive and answers ERR for a
wrong board or base -- that stops the transfer.
The admin password comes from platformio.local.ini ([secrets] admin_pw) and is
never printed. CHECK `get radio` on both ends first: this transmits.
"""
import argparse
import asyncio
import base64
import configparser
import math
import random
import sys
import time
from pathlib import Path
if __name__ == "__main__" and (
any(a in ("-h", "--help") for a in sys.argv[1:]) or len(sys.argv) == 1
):
print(__doc__)
sys.exit(0)
from meshcore import EventType, MeshCore
CHUNK = 96 # FW_OTA_CHUNK (text)
BCHUNK = 128 # FW_OTA_BCHUNK (binary)
# The companion takes host frames of at most MAX_FRAME_SIZE 176 B, and a binary
# request frame is 1 (cmd) + 32 (pubkey) + payload. 160 B chunks made 199 B
# frames that the companion dropped -- every one of them (13. 11. 2026).
MAX_FRAME = 176
UNIT = 32 # FW_OTA_UNIT
MANIFEST = 256 # FW_OTA_MANIFEST_SIZE
REQ_FW = 0xF0 # REQ_TYPE_FW_OTA
INI = Path(__file__).resolve().parent.parent / "platformio.local.ini"
def crc16(data: bytes, crc: int = 0xFFFF) -> int:
"""Nordic crc16_compute (CCITT, init 0xFFFF) -- FwUpdate.cpp crc16_update."""
for b in data:
crc = ((crc >> 8) | (crc << 8)) & 0xFFFF
crc ^= b
crc ^= (crc & 0xFF) >> 4
crc ^= (crc << 12) & 0xFFFF
crc ^= ((crc & 0xFF) << 5) & 0xFFFF
return crc
def w_payload(sid: int, off: int, chunk: bytes) -> bytes:
"""REQ_TYPE_FW_OTA 'w': [F0]['w'][sid u16][off u32][len u8][crc16][data]."""
head = sid.to_bytes(2, "little") + off.to_bytes(4, "little") + bytes([len(chunk)])
# the crc binds the data to this session and offset (FwUpdate::handleBinary)
return (
bytes([0xF0, ord("w")])
+ head
+ crc16(chunk, crc16(head)).to_bytes(2, "little")
+ chunk
)
def admin_pw() -> str:
cp = configparser.ConfigParser(interpolation=None, strict=False)
cp.read(INI)
pw = cp.get("secrets", "admin_pw", fallback="").strip()
if not pw:
sys.exit(f"no [secrets] admin_pw in {INI}")
return pw
def lora_airtime(n: int, sf: int, bw_khz: float, cr: int, preamble: int = 16) -> float:
"""Seconds on air for an n-byte LoRa frame (explicit header, CRC on)."""
tsym = (2**sf) / (bw_khz * 1000)
de = 1 if tsym > 0.016 else 0
payload = 8 + max(
math.ceil((8 * n - 4 * sf + 28 + 16) / (4 * (sf - 2 * de))) * cr, 0
)
return (preamble + 4.25) * tsym + payload * tsym
def frame_len(text: str) -> int:
"""MeshCore direct text frame: header+path_len, dest/src hash, MAC, and
timestamp+flags+text padded to the AES block."""
body = 5 + len(text.encode())
return 2 + 2 + 2 + 16 * math.ceil(body / 16)
class Bucket:
def __init__(self, duty: float):
self.rate = duty / 100.0
self.cap = self.rate * 3600
self.level = self.cap
self.t = time.monotonic()
self.used = 0.0
async def spend(self, secs: float) -> None:
while True:
now = time.monotonic()
self.level = min(self.cap, self.level + (now - self.t) * self.rate)
self.t = now
if self.level >= secs:
self.level -= secs
self.used += secs
return
await asyncio.sleep((secs - self.level) / self.rate)
REPLY_TEXT = 40 # typical reply ("ok", status lines are longer but rare)
def hops_of(contact: dict) -> int:
"""out_path_len: low 6 bits = hops, top 2 = hash size - 1; -1 = flood."""
v = int(contact.get("out_path_len", -1))
return -1 if v < 0 else v & 0x3F
class Link:
def __init__(
self,
mc: MeshCore,
contact: dict,
bucket: Bucket,
radio: tuple,
gap: float = 0.0,
):
self.mc, self.c, self.bucket, self.radio, self.gap = (
mc,
contact,
bucket,
radio,
gap,
)
self.prefix = contact["public_key"][:12]
self.retries = 0
def cost(self, text: str) -> float:
hops = max(hops_of(self.c), 0)
one = lora_airtime(frame_len(text), *self.radio) + lora_airtime(
frame_len("x" * REPLY_TEXT), *self.radio
)
return one * (hops + 1)
async def cmd(self, text: str, timeout: float = 20.0, tries: int = 4) -> str:
for attempt in range(tries):
await self.bucket.spend(self.cost(text))
await self.mc.commands.send_cmd(self.c, text)
ev = await self.mc.wait_for_event(
EventType.CONTACT_MSG_RECV,
attribute_filters={"pubkey_prefix": self.prefix},
timeout=timeout,
)
if ev is not None:
if self.gap:
await asyncio.sleep(self.gap)
return str(ev.payload.get("text", "")).strip()
self.retries += 1
print(
f" ({text[:12]}...: no reply, try {attempt + 2}/{tries})", flush=True
)
raise TimeoutError(f"no reply to {text[:30]!r}")
def req_frame_len(payload_len: int) -> int:
"""Direct REQ: header+path_len, dest/src hash, MAC, timestamp+payload padded."""
return 2 + 2 + 2 + 16 * math.ceil((4 + payload_len) / 16)
class Binary:
"""Firmware chunks as binary requests (REQ_TYPE_FW_OTA)."""
def __init__(self, link: "Link"):
self.link = link
self.pub = bytes.fromhex(link.c["public_key"])
self.sid = 0
def cost(self, payload_len: int, reply_len: int = 0) -> float:
hops = max(hops_of(self.link.c), 0)
t = lora_airtime(req_frame_len(payload_len), *self.link.radio)
if reply_len:
t += lora_airtime(req_frame_len(reply_len), *self.link.radio)
return t * (hops + 1)
async def send(self, payload: bytes) -> str | None:
if 1 + 32 + len(payload) > MAX_FRAME:
raise ValueError(
f"binary request of {len(payload)} B exceeds the companion frame"
)
ev = await self.link.mc.commands.send(
b"\x32" + self.pub + payload, [EventType.MSG_SENT, EventType.ERROR]
)
if ev.type != EventType.MSG_SENT:
return None
tag = ev.payload.get("expected_ack")
return tag.hex() if isinstance(tag, bytes | bytearray) else str(tag)
async def write(self, off: int, chunk: bytes) -> None:
payload = w_payload(self.sid, off, chunk)
c = self.cost(len(payload))
await self.link.bucket.spend(c)
await self.send(payload)
# no reply comes: give the packet time to go out along the whole path
await asyncio.sleep(max(self.link.gap, c + 1.0))
async def missing(self, timeout: float = 20.0) -> tuple[int, list[int]] | None:
"""(state, missing unit indices) or None when the node does not answer."""
for _ in range(2):
await self.link.bucket.spend(self.cost(2, 160))
fut: asyncio.Future = asyncio.get_running_loop().create_future()
def got(ev, fut=fut):
if not fut.done():
fut.set_result(ev)
sub = self.link.mc.subscribe(EventType.BINARY_RESPONSE, got)
try:
tag = await self.send(bytes([REQ_FW, ord("m")]))
ev = await asyncio.wait_for(fut, timeout)
except TimeoutError:
ev = None
finally:
self.link.mc.unsubscribe(sub)
if ev is None or (tag and ev.payload.get("tag") != tag):
continue
d = bytes.fromhex(ev.payload["data"])
if len(d) < 5:
continue
cnt = int.from_bytes(d[3:5], "little")
miss = [
int.from_bytes(d[5 + 2 * i : 7 + 2 * i], "little") for i in range(cnt)
]
await asyncio.sleep(self.link.gap)
return d[0], miss
return None
async def login(mc: MeshCore, contact: dict) -> str:
"""Admin login; returns the event type name."""
fut: asyncio.Future = asyncio.get_running_loop().create_future()
def done(ev):
if not fut.done():
fut.set_result(ev.type.name)
subs = [
mc.subscribe(t, done) for t in (EventType.LOGIN_SUCCESS, EventType.LOGIN_FAILED)
]
try:
await mc.commands.send_login(contact, admin_pw())
return await asyncio.wait_for(fut, 30)
except TimeoutError:
return "timeout"
finally:
for sub in subs:
mc.unsubscribe(sub)
async def find_contact(mc: MeshCore, prefix: str, wait: float) -> dict:
added: list[dict] = []
async def on_new(ev):
added.append(ev.payload)
sub = mc.subscribe(EventType.NEW_CONTACT, on_new)
t0 = time.time()
try:
while True:
await mc.ensure_contacts(follow=True)
for c in mc.contacts.values():
if c["public_key"].lower().startswith(prefix):
return c
for p in added:
if p.get("public_key", "").lower().startswith(prefix):
await mc.commands.add_contact(p)
if time.time() - t0 > wait:
sys.exit(
f"{prefix}: not among the companion's contacts after {wait:.0f} s (advert.zerohop on the node?)"
)
await asyncio.sleep(3)
finally:
mc.unsubscribe(sub)
async def run(args: argparse.Namespace) -> int:
data = Path(args.package).read_bytes()
# Only deltas go over the mesh -- the node refuses anything else too
# (FwUpdate: `fw begin` over 64 kB, a format 1 manifest). A full image is
# ~300 kB, hours of a shared channel: USB (ota_push.py) or the BLE console.
if data[:4] != b"MCFW" or data[4] != 2:
sys.exit(
f"{args.package}: not a delta package (format {data[4] if len(data) > 4 else '?'}) -- "
"over the mesh only deltas; a full image goes over USB (tools/ota_push.py) or BLE"
)
if args.companion.startswith("tcp://"):
host, port = args.companion[6:].rsplit(":", 1)
mc = await MeshCore.create_tcp(host, int(port))
else:
mc = await MeshCore.create_serial(args.companion)
if mc is None:
sys.exit("companion does not answer")
try:
si = mc.self_info
cr = int(si["radio_cr"]) - 4
radio = (int(si["radio_sf"]), float(si["radio_bw"]), cr)
print(
f"companion {si['name']}: {si['radio_freq']} MHz, SF{radio[0]}, BW{radio[1]}, CR4/{si['radio_cr']}"
)
await mc.start_auto_message_fetching()
contact = await find_contact(mc, args.dest.lower(), args.wait)
if args.via:
path = bytes.fromhex(args.via)
if len(path) % 2:
sys.exit("--via: whole 2-byte hashes")
await mc.commands.update_contact(contact, path=path.hex(), path_hash_mode=1)
contact = await find_contact(mc, args.dest.lower(), 5)
print(
f"dest {contact['adv_name']} {contact['public_key'][:8]}, out_path_len {contact.get('out_path_len')}"
)
bucket = Bucket(args.duty)
await bucket.spend(lora_airtime(48, *radio))
r = await login(mc, contact)
if r != "LOGIN_SUCCESS":
sys.exit(f"login: {r}")
print("login: ok (admin)")
await mc.ensure_contacts(follow=True) # login may have taught it a path
contact = await find_contact(mc, args.dest.lower(), 5)
print(
f"path: {hops_of(contact)} hops (out_path_len {contact.get('out_path_len')})"
)
link = Link(mc, contact, bucket, radio, args.gap)
t0 = time.time()
print("status:", await link.cmd("fw status"))
sid = random.randint(1, 0xFFFF) # chunks of an earlier transfer are dropped
r = await link.cmd(f"fw begin {len(data)} {sid}")
print("begin:", r)
if not r.startswith("OK"):
return 1
binary = None if args.text else Binary(link)
if binary:
binary.sid = sid
text_todo: list[int] | None = None
if binary:
st = await binary.missing()
if st is None:
print("node does not answer binary requests: text chunks")
binary = None
if binary:
nb = (len(data) + BCHUNK - 1) // BCHUNK
todo = list(range(nb))
last_miss = 1 << 30
async def settled():
for _ in range(10):
st = await binary.missing()
if st is None or st[0] != 4: # 4 = busy (manifest check)
return st
await asyncio.sleep(3)
return st
manifest_checked = False
for rnd in range(6):
for k, i in enumerate(todo):
off = i * BCHUNK
await binary.write(off, data[off : off + BCHUNK])
print(
f"\r{k + 1}/{len(todo)} binary chunks (round {rnd + 1}), {time.time() - t0:.0f} s, "
f"airtime {bucket.used:.1f} s",
end="",
flush=True,
)
if not manifest_checked and off + BCHUNK >= MANIFEST and k + 1 < len(todo):
# The node checks the manifest as soon as it is in (signature,
# target, base, NOT OLDER): ask before the rest goes on air --
# a refused package otherwise costs the whole first round.
manifest_checked = True
st = await settled()
if st is not None and st[0] == 3:
print("\nnode:", await link.cmd("fw status"))
print(f"stopped after the manifest, airtime {bucket.used:.1f} s")
return 1
print()
st = await settled()
if st is None:
print("missing: no answer")
return 1
state, miss = st
print(f"missing: state {state}, {len(miss)} units")
if state == 3:
print("node:", await link.cmd("fw status"))
return 1
if not miss:
break
if rnd > 0 and len(miss) >= last_miss:
# a round that adds nothing will not be fixed by more rounds
print("binary chunks do not arrive: text chunks for the rest")
binary = None
text_todo = sorted({u * UNIT // CHUNK for u in miss})
break
last_miss = len(miss)
todo = sorted({u * UNIT // BCHUNK for u in miss})
if binary:
todo = []
elif text_todo is not None:
todo = text_todo # only what the binary rounds did not deliver
else:
todo = list(range((len(data) + CHUNK - 1) // CHUNK))
for rnd in range(4 if todo else 0):
for k, i in enumerate(todo):
off = i * CHUNK
r = await link.cmd(
f"fw w {off} {base64.b64encode(data[off : off + CHUNK]).decode()}"
)
if r != "ok":
print(f"\nchunk {off}: {r}")
return 1
print(
f"\r{k + 1}/{len(todo)} chunks (round {rnd + 1}), {time.time() - t0:.0f} s, "
f"airtime {bucket.used:.1f} s",
end="",
flush=True,
)
print()
r = await link.cmd("fw missing")
print("missing:", r)
if r.endswith("none"):
break
todo = [int(x) // CHUNK for x in r.lstrip(">").split()]
r = await link.cmd("fw verify", timeout=40)
for _ in range(20): # the node runs it in loop(), not in the handler
if "verifying" not in r:
break
await asyncio.sleep(3)
r = await link.cmd("fw status")
print("verify:", r)
if not r.startswith(">") or "ready" not in r:
return 1
print("bootloader:", await link.cmd("fw bl"))
if args.apply:
r = await link.cmd("fw apply", timeout=30, tries=1)
print("apply:", r)
if not r.startswith("OK"):
return 1
print("waiting 60 s for the node to rewrite itself and boot ...")
await asyncio.sleep(60)
print("login after apply:", await login(mc, contact))
print("ver:", await link.cmd("ver"))
print("status:", await link.cmd("fw status"))
print("bootloader:", await link.cmd("fw bl"))
print(
f"done in {time.time() - t0:.0f} s, channel airtime {bucket.used:.1f} s, retries {link.retries}"
)
return 0
finally:
await mc.disconnect()
def main() -> None:
ap = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
ap.add_argument("companion")
ap.add_argument("dest")
ap.add_argument("package")
ap.add_argument("--apply", action="store_true")
ap.add_argument("--duty", type=float, default=1.0)
ap.add_argument("--wait", type=float, default=120.0)
ap.add_argument("--gap", type=float, default=5.0)
ap.add_argument("--via")
ap.add_argument("--text", action="store_true")
sys.exit(asyncio.run(run(ap.parse_args())))
if __name__ == "__main__":
main()
+509
View File
@@ -0,0 +1,509 @@
#!/usr/bin/env python3
"""Pack a firmware image for the admin-pushed OTA (src/helpers/FwUpdate.*).
ota_pack.py keygen [--key PATH]
new Ed25519 key; prints the PUBLIC key for FW_OTA_PUBKEY
ota_pack.py pubkey [--key PATH]
ota_pack.py pack <env> [--key PATH] [--out FILE]
.pio/build/<env>/firmware.elf -> <env>.mcfw (format 1: manifest + raw deflate)
ota_pack.py diff <env> --base <file|sha-prefix> [--key PATH] [--out FILE]
format 2: detools in-place DELTA from the base image to the env's image
(src/helpers/FwPatch.cpp). The base must be byte-exact what the node
runs: `fw status` prints its size and sha256 prefix; --base takes that
prefix and looks it up in the image archive.
ota_pack.py archive <env>
store the env's image in the archive (pack and diff do it too)
ota_pack.py info <file.mcfw> [--pubkey HEX]
ota_pack.py fetch <env> --release fw-<sha8> (--base <sha-prefix> | --full) [--out FILE]
download a package that CI built (tth-fw-release.yml) and check it: the
ssh signature of SHA256SUMS (meshcore-fw@tth, tools/fw_allowed_signers),
the file's sha256, and the manifest signature against [ota] pubkey.
--base picks the delta for the image the node runs (`fw status`).
Image archive: ~/.local/share/meshcore-ota/images/<sha256>.bin -- every image
that may become a base has to land there BEFORE it is flashed by other means
(USB/BLE), or there is nothing to diff against later. Needs detools for `diff`
(not in Arch repos: the home-scripts venv has it).
The private key never goes into the repo. Default: ~/.config/meshcore-ota/test.key
(the bench key, PEM). CI signs with the fleet key FW_SIGNING_KEY (OpenSSH format,
the one that signs the mast releases); its public half is the default
[ota] pubkey in platformio.ini, so mast builds take only CI-signed packages.
--elf packs a given ELF instead of .pio/build/<env>/firmware.elf.
Target and build epoch come from the ELF (the image's MCFW-TAGS string, also
printed by `fw ver` on the node), never from the command line: only an image
built with -D FW_OTA=1 can be packed. The node refuses over the mesh a package
that is not newer than itself.
Layout 2 (256 B manifest, little-endian; must match struct Manifest in FwUpdate.cpp):
0 magic "MCFW" | 4 format=1|2 | 5 layout=2 | 8 raw_size | 12 comp_size
16 raw_sha256 | 48 comp_sha256 | 80 target[48] | 128 version[28]
156 build_epoch u32 | 160 base_sha256 (format 2) | 192 Ed25519 signature over 0..191
"""
import argparse
import hashlib
import json
import os
import re
import struct
import subprocess
import sys
import tempfile
import urllib.request
import zlib
from pathlib import Path
from typing import NamedTuple
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import (
Ed25519PrivateKey,
Ed25519PublicKey,
)
DEFAULT_KEY = Path.home() / ".config/meshcore-ota/test.key"
OBJCOPY = (
Path.home()
/ ".platformio/packages/toolchain-gccarmnoneeabi/bin/arm-none-eabi-objcopy"
)
REPO = Path(__file__).resolve().parent.parent
APP_START = 0x27000
# must match FwUpdate: FW_OTA_STAGE_END - FW_OTA_PATCH_AREA, and FwPatch.cpp
PATCH_LO = 0xED000 - 0x10000
PATCH_AREA = 0x10000
MANIFEST = 256
PAGE = 4096
HS_WINDOW, HS_LOOKAHEAD = 11, 10
ARCHIVE = Path.home() / ".local/share/meshcore-ota/images"
API = "https://git.rfa.cz/api/v1/repos/tth/meshcore-tth"
SIGNERS = REPO / "tools/fw_allowed_signers"
SIGNER, NAMESPACE = "meshcore-fw@tth", "meshcore-fw"
def load_key(path: Path) -> Ed25519PrivateKey:
data = path.read_bytes()
key: object
if b"OPENSSH PRIVATE KEY" in data: # FW_SIGNING_KEY, `ssh-keygen -t ed25519`
key = serialization.load_ssh_private_key(data, password=None)
else:
key = serialization.load_pem_private_key(data, password=None)
assert isinstance(key, Ed25519PrivateKey)
return key
def pub_hex(key: Ed25519PrivateKey) -> str:
raw = key.public_key().public_bytes(
serialization.Encoding.Raw, serialization.PublicFormat.Raw
)
return raw.hex()
def cmd_keygen(args: argparse.Namespace) -> None:
path = Path(args.key)
if path.exists():
sys.exit(f"{path} exists, not overwriting")
path.parent.mkdir(parents=True, exist_ok=True)
key = Ed25519PrivateKey.generate()
pem = key.private_bytes(
serialization.Encoding.PEM,
serialization.PrivateFormat.PKCS8,
serialization.NoEncryption(),
)
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, "wb") as f:
f.write(pem)
print(pub_hex(key))
def cmd_pubkey(args: argparse.Namespace) -> None:
print(pub_hex(load_key(Path(args.key))))
LAYOUT = 2
HEAD = "<4sBB2xII32s32s48s28sI32s" # the signed 192 bytes
class Image(NamedTuple):
raw: bytes
version: str
target: str
epoch: int
def elf_version(elf: Path) -> str:
data = elf.read_bytes()
m = re.search(rb"v\d+\.\d+\.\d+-tth[0-9a-f]+\+?", data)
return m.group(0).decode() if m else "unknown"
def elf_tags(elf: Path) -> tuple[str, int]:
"""(target, build epoch) the image itself carries (FwUpdate.cpp FW_OTA_TAGS)."""
m = re.search(rb"MCFW-TAGS target=([\x21-\x7e]+) epoch=(\d+)\x00", elf.read_bytes())
if not m:
sys.exit(f"{elf}: no MCFW-TAGS -- not an OTA image (build it with -D FW_OTA=1)")
return m.group(1).decode(), int(m.group(2))
def env_image(
env: str, dirty_ok: bool, elf_path: str | None = None
) -> tuple[bytes, str]:
elf = Path(elf_path) if elf_path else REPO / ".pio/build" / env / "firmware.elf"
if not elf.exists():
sys.exit(f"{elf} missing, build first: pio run -e {env}")
raw = subprocess.run(
[str(OBJCOPY), "-O", "binary", str(elf), "/dev/stdout"],
check=True,
capture_output=True,
).stdout
version = elf_version(elf)
if version.endswith("+") and not dirty_ok:
sys.exit(
f"{version}: built from a dirty tree, refusing (--dirty for the bench)"
)
return raw, version
def archive(raw: bytes, env: str, version: str) -> str:
sha = hashlib.sha256(raw).hexdigest()
ARCHIVE.mkdir(parents=True, exist_ok=True)
path = ARCHIVE / f"{sha}.bin"
if not path.exists():
path.write_bytes(raw)
with open(ARCHIVE / "index.tsv", "a") as f:
f.write(f"{sha}\t{env}\t{version}\t{len(raw)}\n")
return sha
def find_base(spec: str) -> bytes:
p = Path(spec)
if p.exists():
return p.read_bytes()
spec = spec.lower()
if len(spec) < 8 or any(c not in "0123456789abcdef" for c in spec):
sys.exit(f"--base {spec}: not a file and not a sha256 prefix (>= 8 hex)")
hits = sorted(ARCHIVE.glob(f"{spec}*.bin"))
if len(hits) != 1:
sys.exit(f"--base {spec}: {len(hits)} images in {ARCHIVE}, need exactly one")
return hits[0].read_bytes()
def ota_image(env: str, dirty_ok: bool, elf_path: str | None = None) -> Image:
raw, version = env_image(env, dirty_ok, elf_path)
elf = Path(elf_path) if elf_path else REPO / ".pio/build" / env / "firmware.elf"
target, epoch = elf_tags(elf)
return Image(raw, version, target, epoch)
def manifest(fmt: int, img: Image, blob: bytes, base_sha: bytes, key: str) -> bytes:
if len(img.target) > 47 or len(img.version) > 27:
sys.exit(
f"target {img.target!r} / version {img.version!r} too long for the manifest"
)
head = struct.pack(
HEAD,
b"MCFW",
fmt,
LAYOUT,
len(img.raw),
len(blob),
hashlib.sha256(img.raw).digest(),
hashlib.sha256(blob).digest(),
img.target.encode(),
img.version.encode(),
img.epoch,
base_sha,
)
assert len(head) == 192
return head + load_key(Path(key)).sign(head)
def cmd_pack(args: argparse.Namespace) -> None:
img = ota_image(args.env, args.dirty, args.elf)
archive(img.raw, args.env, img.version)
co = zlib.compressobj(9, zlib.DEFLATED, -15, 9)
comp = co.compress(img.raw) + co.flush()
out = Path(args.out or f"{img.target}.mcfw")
out.write_bytes(manifest(1, img, comp, bytes(32), args.key) + comp)
print(
f"{out}: {img.target} {img.version} epoch {img.epoch}, image {len(img.raw)} B, "
f"package {MANIFEST + len(comp)} B"
)
def make_delta(base: bytes, raw: bytes) -> bytes:
import io
import detools # type: ignore[import-untyped]
mem = PATCH_LO - APP_START
patch = io.BytesIO()
detools.create_patch(
io.BytesIO(base),
io.BytesIO(raw),
patch,
compression="heatshrink",
patch_type="in-place",
memory_size=mem,
segment_size=PAGE,
heatshrink_window_sz2=HS_WINDOW,
heatshrink_lookahead_sz2=HS_LOOKAHEAD,
)
data = patch.getvalue()
# prove it here too: apply to a model of the flash
flash = io.BytesIO(base + b"\xff" * (mem - len(base)))
detools.apply_patch_in_place(flash, io.BytesIO(data))
if flash.getvalue()[: len(raw)] != raw:
sys.exit("delta does not reproduce the image (detools bug?)")
return data
def cmd_diff(args: argparse.Namespace) -> None:
img = ota_image(args.env, args.dirty, args.elf)
raw, version = img.raw, img.version
archive(raw, args.env, version)
base = find_base(args.base)
if APP_START + len(raw) > PATCH_LO or APP_START + len(base) > PATCH_LO:
sys.exit("image reaches into the patch area")
patch = make_delta(base, raw)
if MANIFEST + len(patch) > PATCH_AREA:
sys.exit(
f"delta {len(patch)} B does not fit the {PATCH_AREA} B patch area: use `pack`"
)
base_sha = hashlib.sha256(base).digest()
out = Path(args.out or f"{img.target}.delta.mcfw")
out.write_bytes(manifest(2, img, patch, base_sha, args.key) + patch)
print(
f"{out}: {img.target} {version} epoch {img.epoch}, image {len(raw)} B, "
f"base {len(base)} B {base_sha.hex()[:8]}, "
f"delta {len(patch)} B, package {MANIFEST + len(patch)} B"
)
def cmd_archive(args: argparse.Namespace) -> None:
raw, version = env_image(args.env, args.dirty, args.elf)
sha = archive(raw, args.env, version)
print(f"{ARCHIVE / (sha + '.bin')}: {version}, {len(raw)} B, sha {sha[:8]}")
def cmd_info(args: argparse.Namespace) -> None:
data = Path(args.file).read_bytes()
if data[5] != LAYOUT:
sys.exit(
f"{args.file}: manifest layout {data[5] or 1}, this tool and the node want {LAYOUT}: repack it"
)
(
magic,
fmt,
_layout,
raw_size,
comp_size,
raw_sha,
comp_sha,
target,
version,
epoch,
base_sha,
) = struct.unpack(HEAD, data[:192])
print(
f"magic={magic!r} format={fmt} layout={LAYOUT} target={target.rstrip(b'\0').decode()} epoch={epoch}"
)
if fmt == 2:
blob = data[MANIFEST:]
ok = len(blob) == comp_size and hashlib.sha256(blob).digest() == comp_sha
print(
f"version={version.rstrip(b'\0').decode()} image={raw_size} delta={comp_size}"
)
print(f"base sha256 {base_sha.hex()}")
print(f"hashes {'ok (delta; image hash needs the base)' if ok else 'BAD'}")
if args.pubkey:
Ed25519PublicKey.from_public_bytes(bytes.fromhex(args.pubkey)).verify(
data[192:256], data[:192]
)
print("signature ok")
return
print(
f"version={version.rstrip(b'\0').decode()} image={raw_size} compressed={comp_size}"
)
blob = data[256:]
ok = len(blob) == comp_size and hashlib.sha256(blob).digest() == comp_sha
img = zlib.decompress(blob, -15) if ok else b""
ok = ok and hashlib.sha256(img).digest() == raw_sha
print(f"hashes {'ok' if ok else 'BAD'}")
if args.pubkey:
Ed25519PublicKey.from_public_bytes(bytes.fromhex(args.pubkey)).verify(
data[192:256], data[:192]
)
print("signature ok")
def gitea_token() -> str:
tok = os.environ.get("GITEA_TOKEN")
if tok:
return tok
path = Path.home() / ".config/gitea/credentials"
m = re.search(
r"^\s*(?:export\s+)?GITEA_TOKEN=['\"]?([^'\"\s]+)",
path.read_text() if path.exists() else "",
re.MULTILINE,
)
if not m:
sys.exit(f"no GITEA_TOKEN in the environment or in {path}")
return m.group(1)
def fleet_pubkey() -> str | None:
"""[ota] pubkey of the committed platformio.ini: the fleet key CI signs with.
NOT platformio.local.ini -- that one carries the bench key."""
import configparser
cp = configparser.ConfigParser(interpolation=None, strict=False)
cp.read(REPO / "platformio.ini")
return cp.get("ota", "pubkey", fallback=None)
def cmd_fetch(args: argparse.Namespace) -> None:
tag = args.release if args.release.startswith("fw-") else f"fw-{args.release[:8]}"
tok = gitea_token()
def get(url: str) -> bytes:
req = urllib.request.Request(url, headers={"Authorization": f"token {tok}"})
with urllib.request.urlopen(req, timeout=60) as r:
return r.read()
rel = json.loads(get(f"{API}/releases/tags/{tag}"))
assets = {a["name"]: a["browser_download_url"] for a in rel.get("assets", [])}
for name in ("SHA256SUMS", "SHA256SUMS.sig", "manifest.json"):
if name not in assets:
sys.exit(f"{tag}: no asset {name}")
sums = get(assets["SHA256SUMS"])
with tempfile.NamedTemporaryFile(suffix=".sig") as sig:
sig.write(get(assets["SHA256SUMS.sig"]))
sig.flush()
v = subprocess.run(
[
"ssh-keygen",
"-Y",
"verify",
"-f",
str(args.allowed_signers),
"-I",
SIGNER,
"-n",
NAMESPACE,
"-s",
sig.name,
],
input=sums,
capture_output=True,
check=False,
)
if v.returncode != 0:
sys.exit(f"{tag}: SIGNATURE INVALID: {(v.stderr or v.stdout).decode().strip()}")
listed = {}
for line in sums.decode().splitlines():
digest, name = line.split(None, 1)
listed[name.strip().lstrip("*")] = digest
def checked(name: str) -> bytes:
data = get(assets[name])
if listed.get(name) != hashlib.sha256(data).hexdigest():
sys.exit(f"{tag}: {name} does not match the signed SHA256SUMS")
return data
ota = (
json.loads(checked("manifest.json"))
.get("envs", {})
.get(args.env, {})
.get("ota")
)
if not ota:
sys.exit(
f"{tag}: no OTA packages for {args.env} (built before FwUpdate, or another env)"
)
if args.full:
name = ota["full"]
else:
spec = args.base.lower()
hits = [
d
for d in ota.get("deltas", {}).values()
if d["base_sha256"].startswith(spec)
]
if len(hits) != 1:
have = ", ".join(
f"{d['base_sha256'][:8]} ({b})"
for b, d in ota.get("deltas", {}).items()
)
sys.exit(
f"{tag}: {len(hits)} deltas for base {spec}; built for: {have or 'none'}"
" -- rerun tth-fw-release with delta_from, or --full"
)
name = hits[0]["file"]
data = checked(name)
pub = args.pubkey or fleet_pubkey()
if pub:
Ed25519PublicKey.from_public_bytes(bytes.fromhex(pub)).verify(
data[192:256], data[:192]
)
out = Path(args.out or name)
out.write_bytes(data)
print(
f"{out}: {tag} {args.env}, {len(data)} B, image sha {ota['image_sha256'][:8]}"
f"{', manifest signature ok' if pub else ''}"
)
def main() -> None:
ap = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
sub = ap.add_subparsers(dest="cmd", required=True)
for name in ("keygen", "pubkey"):
p = sub.add_parser(name)
p.add_argument("--key", default=str(DEFAULT_KEY))
p = sub.add_parser("pack")
p.add_argument("env")
p.add_argument("--elf")
p.add_argument("--key", default=str(DEFAULT_KEY))
p.add_argument("--out")
p.add_argument("--dirty", action="store_true")
p = sub.add_parser("diff")
p.add_argument("env")
p.add_argument("--base", required=True)
p.add_argument("--elf")
p.add_argument("--key", default=str(DEFAULT_KEY))
p.add_argument("--out")
p.add_argument("--dirty", action="store_true")
p = sub.add_parser("archive")
p.add_argument("env")
p.add_argument("--elf")
p.add_argument("--dirty", action="store_true")
p = sub.add_parser("info")
p.add_argument("file")
p.add_argument("--pubkey")
p = sub.add_parser("fetch")
p.add_argument("env")
p.add_argument("--release", required=True, help="fw-<sha8> or a commit sha")
g = p.add_mutually_exclusive_group(required=True)
g.add_argument("--base", help="sha256 prefix of the node's image (`fw status`)")
g.add_argument("--full", action="store_true", help="the full image package")
p.add_argument("--out")
p.add_argument(
"--pubkey", help="default: the fleet key, [ota] pubkey of platformio.ini"
)
p.add_argument("--allowed-signers", default=str(SIGNERS))
args = ap.parse_args()
{
"keygen": cmd_keygen,
"pubkey": cmd_pubkey,
"pack": cmd_pack,
"diff": cmd_diff,
"archive": cmd_archive,
"info": cmd_info,
"fetch": cmd_fetch,
}[args.cmd](args)
if __name__ == "__main__":
main()