Files
tthandClaude Opus 5.5 d22005a6fd
pub-release / release (push) Successful in 32m57s
OTA phase 5: tools and docs ready for the public export
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 18:07:39 +01:00

562 lines
19 KiB
Python
Executable File

#!/usr/bin/env python3
# SPDX-License-Identifier: MIT
"""Pack a firmware image for the admin-pushed OTA (src/helpers/FwUpdate.*).
ota_pack.py keygen [--key PATH]
new Ed25519 key; prints the PUBLIC key for FW_OTA_PUBKEY
ota_pack.py pubkey [--key PATH]
ota_pack.py pack <env> [--key PATH] [--out FILE]
.pio/build/<env>/firmware.elf -> <env>.mcfw (format 1: manifest + raw deflate)
ota_pack.py diff <env> --base <file|sha-prefix> [--key PATH] [--out FILE]
format 2: detools in-place DELTA from the base image to the env's image
(src/helpers/FwPatch.cpp). The base must be byte-exact what the node
runs: `fw status` prints its size and sha256 prefix; --base takes that
prefix and looks it up in the image archive.
ota_pack.py archive <env>
store the env's image in the archive (pack and diff do it too)
ota_pack.py info <file.mcfw> [--pubkey HEX]
ota_pack.py fetch <artifact> [--release TAG|latest] (--base <sha-prefix> | --full) [--out FILE]
download an OTA package of a public release and check it: the ssh
signature of SHA256SUMS (meshcore-ota@tth-pub, tools/ota_allowed_signers),
the file's sha256, and the manifest signature against [ota] pubkey.
<artifact> is the image's name in the release, e.g.
Xiao_nrf52_repeater-ota or SenseCap_Solar_repeater-observer-ota.
--base picks the delta for the image the node runs (`fw status`).
--private: our mast releases fw-<sha8> instead (token, fleet signer).
Image archive: ~/.local/share/meshcore-ota/images/<sha256>.bin -- every image
that may become a base has to land there BEFORE it is flashed by other means
(USB/BLE), or there is nothing to diff against later. Needs detools for `diff`
(not in Arch repos: the home-scripts venv has it).
The private key never goes into the repo. Default: ~/.config/meshcore-ota/test.key
(the bench key, PEM). CI signs with the public OTA key meshcore-ota@tth-pub
(secret OTA_SIGNING_KEY, OpenSSH format); its public half is the default
[ota] pubkey in platformio.ini, so release images take only CI-signed packages.
--elf packs a given ELF instead of .pio/build/<env>/firmware.elf.
Target and build epoch come from the ELF (the image's MCFW-TAGS string, also
printed by `fw ver` on the node), never from the command line: only an image
built with -D FW_OTA=1 can be packed. The node refuses over the mesh a package
that is not newer than itself.
Layout 2 (256 B manifest, little-endian; must match struct Manifest in FwUpdate.cpp):
0 magic "MCFW" | 4 format=1|2 | 5 layout=2 | 8 raw_size | 12 comp_size
16 raw_sha256 | 48 comp_sha256 | 80 target[48] | 128 version[28]
156 build_epoch u32 | 160 base_sha256 (format 2) | 192 Ed25519 signature over 0..191
"""
import argparse
import hashlib
import json
import os
import re
import struct
import subprocess
import sys
import tempfile
import urllib.parse
import urllib.request
import zlib
from pathlib import Path
from typing import NamedTuple
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric.ed25519 import (
Ed25519PrivateKey,
Ed25519PublicKey,
)
DEFAULT_KEY = Path.home() / ".config/meshcore-ota/test.key"
OBJCOPY = (
Path.home()
/ ".platformio/packages/toolchain-gccarmnoneeabi/bin/arm-none-eabi-objcopy"
)
REPO = Path(__file__).resolve().parent.parent
APP_START = 0x27000
# must match FwUpdate: FW_OTA_STAGE_END - FW_OTA_PATCH_AREA, and FwPatch.cpp
PATCH_LO = 0xED000 - 0x10000
PATCH_AREA = 0x10000
MANIFEST = 256
PAGE = 4096
HS_WINDOW, HS_LOOKAHEAD = 11, 10
ARCHIVE = Path.home() / ".local/share/meshcore-ota/images"
# where `fetch` looks: the public releases, or (--private) our mast releases
SOURCES = {
"public": {
"api": "https://git.rfa.cz/api/v1/repos/tth-pub/meshcore-tth",
"signers": REPO / "tools/ota_allowed_signers",
"signer": "meshcore-ota@tth-pub",
"namespace": "meshcore-ota",
},
"private": {
"api": "https://git.rfa.cz/api/v1/repos/tth/meshcore-tth",
"signers": REPO / "tools/fw_allowed_signers",
"signer": "meshcore-fw@tth",
"namespace": "meshcore-fw",
},
}
def load_key(path: Path) -> Ed25519PrivateKey:
data = path.read_bytes()
key: object
if b"OPENSSH PRIVATE KEY" in data: # FW_SIGNING_KEY, `ssh-keygen -t ed25519`
key = serialization.load_ssh_private_key(data, password=None)
else:
key = serialization.load_pem_private_key(data, password=None)
assert isinstance(key, Ed25519PrivateKey)
return key
def pub_hex(key: Ed25519PrivateKey) -> str:
raw = key.public_key().public_bytes(
serialization.Encoding.Raw, serialization.PublicFormat.Raw
)
return raw.hex()
def cmd_keygen(args: argparse.Namespace) -> None:
path = Path(args.key)
if path.exists():
sys.exit(f"{path} exists, not overwriting")
path.parent.mkdir(parents=True, exist_ok=True)
key = Ed25519PrivateKey.generate()
pem = key.private_bytes(
serialization.Encoding.PEM,
serialization.PrivateFormat.PKCS8,
serialization.NoEncryption(),
)
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(fd, "wb") as f:
f.write(pem)
print(pub_hex(key))
def cmd_pubkey(args: argparse.Namespace) -> None:
print(pub_hex(load_key(Path(args.key))))
LAYOUT = 2
HEAD = "<4sBB2xII32s32s48s28sI32s" # the signed 192 bytes
class Image(NamedTuple):
raw: bytes
version: str
target: str
epoch: int
def elf_version(elf: Path) -> str:
data = elf.read_bytes()
m = re.search(rb"v\d+\.\d+\.\d+-tth[0-9a-f]+\+?", data)
return m.group(0).decode() if m else "unknown"
def elf_tags(elf: Path) -> tuple[str, int]:
"""(target, build epoch) the image itself carries (FwUpdate.cpp FW_OTA_TAGS)."""
m = re.search(rb"MCFW-TAGS target=([\x21-\x7e]+) epoch=(\d+)\x00", elf.read_bytes())
if not m:
sys.exit(f"{elf}: no MCFW-TAGS -- not an OTA image (build it with -D FW_OTA=1)")
return m.group(1).decode(), int(m.group(2))
def env_image(
env: str, dirty_ok: bool, elf_path: str | None = None
) -> tuple[bytes, str]:
elf = Path(elf_path) if elf_path else REPO / ".pio/build" / env / "firmware.elf"
if not elf.exists():
sys.exit(f"{elf} missing, build first: pio run -e {env}")
raw = subprocess.run(
[str(OBJCOPY), "-O", "binary", str(elf), "/dev/stdout"],
check=True,
capture_output=True,
).stdout
version = elf_version(elf)
if version.endswith("+") and not dirty_ok:
sys.exit(
f"{version}: built from a dirty tree, refusing (--dirty for the bench)"
)
return raw, version
def archive(raw: bytes, env: str, version: str) -> str:
sha = hashlib.sha256(raw).hexdigest()
ARCHIVE.mkdir(parents=True, exist_ok=True)
path = ARCHIVE / f"{sha}.bin"
if not path.exists():
path.write_bytes(raw)
with open(ARCHIVE / "index.tsv", "a") as f:
f.write(f"{sha}\t{env}\t{version}\t{len(raw)}\n")
return sha
def find_base(spec: str) -> bytes:
p = Path(spec)
if p.exists():
return p.read_bytes()
spec = spec.lower()
if len(spec) < 8 or any(c not in "0123456789abcdef" for c in spec):
sys.exit(f"--base {spec}: not a file and not a sha256 prefix (>= 8 hex)")
hits = sorted(ARCHIVE.glob(f"{spec}*.bin"))
if len(hits) != 1:
sys.exit(f"--base {spec}: {len(hits)} images in {ARCHIVE}, need exactly one")
return hits[0].read_bytes()
def ota_image(env: str, dirty_ok: bool, elf_path: str | None = None) -> Image:
raw, version = env_image(env, dirty_ok, elf_path)
elf = Path(elf_path) if elf_path else REPO / ".pio/build" / env / "firmware.elf"
target, epoch = elf_tags(elf)
return Image(raw, version, target, epoch)
def manifest(fmt: int, img: Image, blob: bytes, base_sha: bytes, key: str) -> bytes:
if len(img.target) > 47 or len(img.version) > 27:
sys.exit(
f"target {img.target!r} / version {img.version!r} too long for the manifest"
)
head = struct.pack(
HEAD,
b"MCFW",
fmt,
LAYOUT,
len(img.raw),
len(blob),
hashlib.sha256(img.raw).digest(),
hashlib.sha256(blob).digest(),
img.target.encode(),
img.version.encode(),
img.epoch,
base_sha,
)
assert len(head) == 192
return head + load_key(Path(key)).sign(head)
def cmd_pack(args: argparse.Namespace) -> None:
img = ota_image(args.env, args.dirty, args.elf)
archive(img.raw, args.env, img.version)
co = zlib.compressobj(9, zlib.DEFLATED, -15, 9)
comp = co.compress(img.raw) + co.flush()
out = Path(args.out or f"{img.target}.mcfw")
out.write_bytes(manifest(1, img, comp, bytes(32), args.key) + comp)
print(
f"{out}: {img.target} {img.version} epoch {img.epoch}, image {len(img.raw)} B, "
f"package {MANIFEST + len(comp)} B"
)
def make_delta(base: bytes, raw: bytes) -> bytes:
import io
import detools # type: ignore[import-untyped]
mem = PATCH_LO - APP_START
patch = io.BytesIO()
detools.create_patch(
io.BytesIO(base),
io.BytesIO(raw),
patch,
compression="heatshrink",
patch_type="in-place",
memory_size=mem,
segment_size=PAGE,
heatshrink_window_sz2=HS_WINDOW,
heatshrink_lookahead_sz2=HS_LOOKAHEAD,
)
data = patch.getvalue()
# prove it here too: apply to a model of the flash
flash = io.BytesIO(base + b"\xff" * (mem - len(base)))
detools.apply_patch_in_place(flash, io.BytesIO(data))
if flash.getvalue()[: len(raw)] != raw:
sys.exit("delta does not reproduce the image (detools bug?)")
return data
def cmd_diff(args: argparse.Namespace) -> None:
img = ota_image(args.env, args.dirty, args.elf)
raw, version = img.raw, img.version
archive(raw, args.env, version)
base = find_base(args.base)
if APP_START + len(raw) > PATCH_LO or APP_START + len(base) > PATCH_LO:
sys.exit("image reaches into the patch area")
patch = make_delta(base, raw)
if MANIFEST + len(patch) > PATCH_AREA:
sys.exit(
f"delta {len(patch)} B does not fit the {PATCH_AREA} B patch area: use `pack`"
)
base_sha = hashlib.sha256(base).digest()
out = Path(args.out or f"{img.target}.delta.mcfw")
out.write_bytes(manifest(2, img, patch, base_sha, args.key) + patch)
print(
f"{out}: {img.target} {version} epoch {img.epoch}, image {len(raw)} B, "
f"base {len(base)} B {base_sha.hex()[:8]}, "
f"delta {len(patch)} B, package {MANIFEST + len(patch)} B"
)
def cmd_archive(args: argparse.Namespace) -> None:
raw, version = env_image(args.env, args.dirty, args.elf)
sha = archive(raw, args.env, version)
print(f"{ARCHIVE / (sha + '.bin')}: {version}, {len(raw)} B, sha {sha[:8]}")
def cmd_info(args: argparse.Namespace) -> None:
data = Path(args.file).read_bytes()
if data[5] != LAYOUT:
sys.exit(
f"{args.file}: manifest layout {data[5] or 1}, this tool and the node want {LAYOUT}: repack it"
)
(
magic,
fmt,
_layout,
raw_size,
comp_size,
raw_sha,
comp_sha,
target,
version,
epoch,
base_sha,
) = struct.unpack(HEAD, data[:192])
print(
f"magic={magic!r} format={fmt} layout={LAYOUT} target={target.rstrip(b'\0').decode()} epoch={epoch}"
)
if fmt == 2:
blob = data[MANIFEST:]
ok = len(blob) == comp_size and hashlib.sha256(blob).digest() == comp_sha
print(
f"version={version.rstrip(b'\0').decode()} image={raw_size} delta={comp_size}"
)
print(f"base sha256 {base_sha.hex()}")
print(f"hashes {'ok (delta; image hash needs the base)' if ok else 'BAD'}")
if args.pubkey:
Ed25519PublicKey.from_public_bytes(bytes.fromhex(args.pubkey)).verify(
data[192:256], data[:192]
)
print("signature ok")
return
print(
f"version={version.rstrip(b'\0').decode()} image={raw_size} compressed={comp_size}"
)
blob = data[256:]
ok = len(blob) == comp_size and hashlib.sha256(blob).digest() == comp_sha
img = zlib.decompress(blob, -15) if ok else b""
ok = ok and hashlib.sha256(img).digest() == raw_sha
print(f"hashes {'ok' if ok else 'BAD'}")
if args.pubkey:
Ed25519PublicKey.from_public_bytes(bytes.fromhex(args.pubkey)).verify(
data[192:256], data[:192]
)
print("signature ok")
def gitea_token() -> str:
tok = os.environ.get("GITEA_TOKEN")
if tok:
return tok
path = Path.home() / ".config/gitea/credentials"
m = re.search(
r"^\s*(?:export\s+)?GITEA_TOKEN=['\"]?([^'\"\s]+)",
path.read_text() if path.exists() else "",
re.MULTILINE,
)
if not m:
sys.exit(f"no GITEA_TOKEN in the environment or in {path}")
return m.group(1)
def release_pubkey() -> str | None:
"""[ota] pubkey of the committed platformio.ini: the OTA key CI signs with.
NOT platformio.local.ini -- that one carries the bench key."""
import configparser
cp = configparser.ConfigParser(interpolation=None, strict=False)
cp.read(REPO / "platformio.ini")
return cp.get("ota", "pubkey", fallback=None)
def cmd_fetch(args: argparse.Namespace) -> None:
src = SOURCES["private" if args.private else "public"]
api = str(src["api"])
# the public releases are readable without a token, and get none
headers = {"Authorization": f"token {gitea_token()}"} if args.private else {}
def get(url: str) -> bytes:
req = urllib.request.Request(url, headers=headers)
with urllib.request.urlopen(req, timeout=60) as r:
return r.read()
if args.private:
tag = (
args.release if args.release.startswith("fw-") else f"fw-{args.release[:8]}"
)
rel = json.loads(get(f"{api}/releases/tags/{tag}"))
elif args.release == "latest":
rels = [
r
for r in json.loads(get(f"{api}/releases?limit=20"))
if "+tth." in r["tag_name"]
and any(a["name"] == "manifest.json" for a in r.get("assets", []))
]
if not rels:
sys.exit("no public release with OTA packages yet")
rel = max(rels, key=lambda r: r["created_at"])
tag = rel["tag_name"]
else:
tag = args.release
rel = json.loads(get(f"{api}/releases/tags/{urllib.parse.quote(tag, safe='')}"))
assets = {a["name"]: a["browser_download_url"] for a in rel.get("assets", [])}
for name in ("SHA256SUMS", "SHA256SUMS.sig", "manifest.json"):
if name not in assets:
sys.exit(f"{tag}: no asset {name}")
sums = get(assets["SHA256SUMS"])
with tempfile.NamedTemporaryFile(suffix=".sig") as sig:
sig.write(get(assets["SHA256SUMS.sig"]))
sig.flush()
v = subprocess.run(
[
"ssh-keygen",
"-Y",
"verify",
"-f",
str(args.allowed_signers or src["signers"]),
"-I",
str(src["signer"]),
"-n",
str(src["namespace"]),
"-s",
sig.name,
],
input=sums,
capture_output=True,
check=False,
)
if v.returncode != 0:
sys.exit(f"{tag}: SIGNATURE INVALID: {(v.stderr or v.stdout).decode().strip()}")
listed = {}
for line in sums.decode().splitlines():
digest, name = line.split(None, 1)
listed[name.strip().lstrip("*")] = digest
def checked(name: str) -> bytes:
data = get(assets[name])
if listed.get(name) != hashlib.sha256(data).hexdigest():
sys.exit(f"{tag}: {name} does not match the signed SHA256SUMS")
return data
ota = (
json.loads(checked("manifest.json"))
.get("envs", {})
.get(args.env, {})
.get("ota")
)
if not ota:
sys.exit(
f"{tag}: no OTA packages for {args.env} (built before FwUpdate, or another env)"
)
if args.full:
name = ota["full"]
else:
spec = args.base.lower()
hits = [
d
for d in ota.get("deltas", {}).values()
if d["base_sha256"].startswith(spec)
]
if len(hits) != 1:
have = ", ".join(
f"{d['base_sha256'][:8]} ({b})"
for b, d in ota.get("deltas", {}).items()
)
sys.exit(
f"{tag}: {len(hits)} deltas for base {spec}; built for: {have or 'none'}"
" -- a release has deltas only from the earlier releases it was built"
" against (tth-fw-release: delta_from); otherwise --full over USB/BLE"
)
name = hits[0]["file"]
data = checked(name)
pub = args.pubkey or release_pubkey()
if pub:
Ed25519PublicKey.from_public_bytes(bytes.fromhex(pub)).verify(
data[192:256], data[:192]
)
out = Path(args.out or name)
out.write_bytes(data)
print(
f"{out}: {tag} {args.env}, {len(data)} B, image sha {ota['image_sha256'][:8]}"
f"{', manifest signature ok' if pub else ''}"
)
def main() -> None:
ap = argparse.ArgumentParser(
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
)
sub = ap.add_subparsers(dest="cmd", required=True)
for name in ("keygen", "pubkey"):
p = sub.add_parser(name)
p.add_argument("--key", default=str(DEFAULT_KEY))
p = sub.add_parser("pack")
p.add_argument("env")
p.add_argument("--elf")
p.add_argument("--key", default=str(DEFAULT_KEY))
p.add_argument("--out")
p.add_argument("--dirty", action="store_true")
p = sub.add_parser("diff")
p.add_argument("env")
p.add_argument("--base", required=True)
p.add_argument("--elf")
p.add_argument("--key", default=str(DEFAULT_KEY))
p.add_argument("--out")
p.add_argument("--dirty", action="store_true")
p = sub.add_parser("archive")
p.add_argument("env")
p.add_argument("--elf")
p.add_argument("--dirty", action="store_true")
p = sub.add_parser("info")
p.add_argument("file")
p.add_argument("--pubkey")
p = sub.add_parser("fetch")
p.add_argument(
"env", help="artifact, e.g. Xiao_nrf52_repeater-ota (--private: the env)"
)
p.add_argument(
"--release",
default="latest",
help="public tag (v1.18.0+tth.YYYYMMDD) or latest; --private: fw-<sha8>",
)
p.add_argument(
"--private",
action="store_true",
help="our mast releases (tth/meshcore-tth, token, fleet signer)",
)
g = p.add_mutually_exclusive_group(required=True)
g.add_argument("--base", help="sha256 prefix of the node's image (`fw status`)")
g.add_argument("--full", action="store_true", help="the full image package")
p.add_argument("--out")
p.add_argument(
"--pubkey", help="default: the release OTA key, [ota] pubkey of platformio.ini"
)
p.add_argument(
"--allowed-signers",
help="default: tools/ota_allowed_signers (--private: fw_allowed_signers)",
)
args = ap.parse_args()
{
"keygen": cmd_keygen,
"pubkey": cmd_pubkey,
"pack": cmd_pack,
"diff": cmd_diff,
"archive": cmd_archive,
"info": cmd_info,
"fetch": cmd_fetch,
}[args.cmd](args)
if __name__ == "__main__":
main()