diff --git a/.github/workflows/pub-release.yml b/.github/workflows/pub-release.yml index 7874dd6b..ea22fa5d 100644 --- a/.github/workflows/pub-release.yml +++ b/.github/workflows/pub-release.yml @@ -20,6 +20,19 @@ # PRs the upstream merges brought in. # nRF52: .uf2 (drag onto the bootloader drive) and .zip (DFU over BLE or # serial); ESP32: -merged.bin (fresh install at 0x0) and .bin (update). +# +# EXPERIMENTAL -ota images (docs/ota-public-plan.md): every nRF52 env with a +# text console (OTA_ENVS) is built once more with -D FW_OTA=1 (admin-pushed +# update, FwUpdate), the observer ones as -observer-ota too. Each comes with a +# full .mcfw package and a delta from the same image of every kept earlier +# release (its zip's firmware.bin, checked against that release's SHA256SUMS); +# over the mesh a node takes only deltas, and none older than itself. +# Packages and SHA256SUMS are signed with OTA_SIGNING_KEY (meshcore-ota@tth-pub, +# tools/ota_allowed_signers); manifest.json lists what is there. +# +# dry_run: the same build anywhere (the private repo too), no release -- the +# result goes up as a workflow artifact. Delta bases always come from the +# public repo's releases. name: pub-release "on": @@ -35,6 +48,10 @@ name: pub-release description: "replace an existing release of this commit" required: false default: "false" + dry_run: + description: "build and package only, no release (runs in the private repo too)" + required: false + default: "false" concurrency: group: pub-release @@ -47,6 +64,11 @@ env: # release, which builds the same observer image). RAM 37-39 %, flash 41-58 %. OBS_ENVS: >- Xiao_nrf52_repeater SenseCap_Solar_repeater RAK_4631_repeater t1000e_repeater Heltec_t114_repeater + # nRF52 envs with a text console: also released as experimental -ota images + OTA_ENVS: >- + Xiao_nrf52_repeater Xiao_nrf52_room_server Xiao_nrf52_sensor SenseCap_Solar_repeater + RAK_4631_repeater RAK_4631_room_server t1000e_repeater Heltec_t114_repeater + PUB_REPO: tth-pub/meshcore-tth # where delta bases come from, dry run or not ENVS: >- Xiao_nrf52_repeater Xiao_nrf52_companion_radio_ble Xiao_nrf52_room_server Xiao_nrf52_sensor SenseCap_Solar_repeater SenseCap_Solar_companion_radio_ble @@ -57,7 +79,7 @@ env: jobs: release: - if: github.repository == 'tth-pub/meshcore-tth' + if: github.repository == 'tth-pub/meshcore-tth' || github.event.inputs.dry_run == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -70,9 +92,15 @@ jobs: env: TOKEN: ${{ secrets.GITEA_TOKEN || github.token }} REPLACE: ${{ github.event.inputs.replace }} + DRY: ${{ github.event.inputs.dry_run }} run: | set -euo pipefail git fetch -q --tags + # the private repo carries no upstream release tags (the export brings + # them along): a dry run there fetches them, or the version base is "nover" + if [ "$DRY" = "true" ]; then + git fetch -q --no-tags https://github.com/meshcore-dev/MeshCore.git 'refs/tags/*:refs/tags/*' + fi sha=$(git rev-parse HEAD) # upstream base as tools/build_version.py finds it: "companion-v1.18.0" -> v1.18.0 desc=$(git describe --tags --abbrev=0 --match '*v[0-9]*' --exclude '*+tth*' --exclude 'backup/*' HEAD) @@ -119,6 +147,8 @@ jobs: id: build env: TAG: ${{ steps.src.outputs.tag }} + OTA_SIGNING_KEY: ${{ secrets.OTA_SIGNING_KEY }} + TOKEN: ${{ secrets.GITEA_TOKEN || github.token }} run: | set -euo pipefail source .github/fw-flags.sh @@ -126,12 +156,58 @@ jobs: dist="$GITHUB_WORKSPACE/dist" mkdir -p "$dist" version="" - # build ; PlatformIO appends - # PLATFORMIO_BUILD_FLAGS after each env's flags: last -D wins + [ -n "$OTA_SIGNING_KEY" ] || { echo "::error::OTA_SIGNING_KEY secret missing"; exit 1; } + okey="$RUNNER_TEMP/ota_signing_key" + (umask 077 && printf '%s\n' "$OTA_SIGNING_KEY" >"$okey") + trap 'rm -f "$okey"' EXIT + pip install -q detools==0.53.0 cryptography + echo '{}' >"$dist/manifest.json" + # earlier public releases (newest first): delta bases for the -ota images + api="$GITEA/api/v1/repos/$PUB_REPO" + auth="header = \"Authorization: token $TOKEN\"" + bases="$RUNNER_TEMP/bases" + mkdir -p "$bases" + curl -sSf --config - "$api/releases?limit=50" <<<"$auth" | + jq -c --arg t "$TAG" '[.[] | select(.tag_name | startswith("tth-") or test("[+]tth[.]")) | select(.tag_name != $t)] + | sort_by(.created_at) | reverse | .[]' >"$bases/releases.jsonl" || : >"$bases/releases.jsonl" + asset() { jq -r --arg n "$2" '.assets[] | select(.name == $n) | .browser_download_url' <<<"$1"; } + # ota_package : full package + deltas from earlier releases + ota_package() { + local env=$1 art=$2 b=".pio/build/$1" tags img rel ptag url dir out + tools/ota_check_ramfunc.sh "$env" + tags=$(grep -aoE 'MCFW-TAGS target=[!-~]+ epoch=[0-9]+' "$b/firmware.elf" | head -1) + img=$(sha256sum <"$b/firmware.bin" | cut -d' ' -f1) + python3 tools/ota_pack.py pack "$env" --key "$okey" --out "$dist/$art-$TAG.mcfw" + jq --arg a "$art" --arg z "$art-$TAG.zip" --arg f "$art-$TAG.mcfw" --arg i "$img" --arg t "${tags#MCFW-TAGS }" \ + '.envs[$a] = {zip: $z, ota: {image_sha256: $i, tags: $t, full: $f, deltas: {}}}' \ + "$dist/manifest.json" >"$dist/m.tmp" && mv "$dist/m.tmp" "$dist/manifest.json" + while read -r rel; do + ptag=$(jq -r .tag_name <<<"$rel") + url=$(asset "$rel" "$art-$ptag.zip") + [ -n "$url" ] || continue # that release had no such -ota image yet + dir="$bases/$ptag" + mkdir -p "$dir" + [ -f "$dir/SHA256SUMS" ] || curl -sSfL --config - -o "$dir/SHA256SUMS" "$(asset "$rel" SHA256SUMS)" <<<"$auth" + curl -sSfL --config - -o "$dir/$art-$ptag.zip" "$url" <<<"$auth" + (cd "$dir" && grep " $art-$ptag.zip\$" SHA256SUMS | sha256sum -c --quiet -) + unzip -p "$dir/$art-$ptag.zip" firmware.bin >"$dir/$art.bin" + out="$art-$TAG.from-$ptag.mcfw" + if python3 tools/ota_pack.py diff "$env" --base "$dir/$art.bin" --key "$okey" --out "$dist/$out"; then + jq --arg a "$art" --arg p "$ptag" --arg f "$out" --arg s "$(sha256sum <"$dir/$art.bin" | cut -d' ' -f1)" \ + '.envs[$a].ota.deltas[$p] = {file: $f, base_sha256: $s}' \ + "$dist/manifest.json" >"$dist/m.tmp" && mv "$dist/m.tmp" "$dist/manifest.json" + else + echo "::warning::$art: no delta from $ptag, full package only" + rm -f "$dist/$out" + fi + done <"$bases/releases.jsonl" + } + # build [ota variant]; PlatformIO + # appends PLATFORMIO_BUILD_FLAGS after each env's flags: last -D wins build() { - local env=$1 suffix=$3 b v + local env=$1 suffix=$3 variant=${4:-} b v echo "::group::$env$suffix" - PLATFORMIO_BUILD_FLAGS="$PUB_FLAGS $2" pio run -e "$env" + FW_OTA_VARIANT=$variant PLATFORMIO_BUILD_FLAGS="$PUB_FLAGS $2" pio run -e "$env" b=".pio/build/$env" if [ -f "$b/firmware.zip" ]; then # nRF52 python3 bin/uf2conv/uf2conv.py "$b/firmware.hex" -c -o "$b/firmware.uf2" -f 0xADA52840 @@ -140,6 +216,7 @@ jobs: v=$(grep -m1 -aoE 'v[0-9]+\.[0-9]+\.[0-9]+-tth[0-9a-f]+\+?' "$b/firmware.elf" || true) case "$v" in *+) echo "::error::$env is dirty ($v)"; exit 1 ;; esac [ -n "$version" ] || version="$v" + [ -z "$variant" ] || { unzip -p "$b/firmware.zip" firmware.bin >"$b/firmware.bin"; ota_package "$env" "$env$suffix"; } else # ESP32 PLATFORMIO_BUILD_FLAGS="$PUB_FLAGS $2" pio run -t mergebin -e "$env" cp "$b/firmware.bin" "$dist/$env$suffix-$TAG.bin" @@ -149,12 +226,27 @@ jobs: } for env in $ENVS; do build "$env" "" ""; done for env in $OBS_ENVS; do build "$env" "$OBS_FLAGS" "-observer"; done + for env in $OTA_ENVS; do build "$env" "-D FW_OTA=1" "-ota" ota; done + for env in $OBS_ENVS; do build "$env" "$OBS_FLAGS -D FW_OTA=1" "-observer-ota" observer-ota; done + jq --arg c "$(git rev-parse HEAD)" --arg t "$TAG" '.commit = $c | .tag = $t' \ + "$dist/manifest.json" >"$dist/m.tmp" && mv "$dist/m.tmp" "$dist/manifest.json" cd "$dist" sha256sum -- * >SHA256SUMS + ssh-keygen -q -Y sign -f "$okey" -n meshcore-ota SHA256SUMS + ssh-keygen -Y verify -f "$GITHUB_WORKSPACE/tools/ota_allowed_signers" -I meshcore-ota@tth-pub \ + -n meshcore-ota -s SHA256SUMS.sig >"$GITHUB_OUTPUT" + - name: Dry run artifact + if: github.event.inputs.dry_run == 'true' + uses: actions/upload-artifact@v4 + with: + name: pub-release-dry-run + path: dist/" + - name: Release + if: github.event.inputs.dry_run != 'true' env: TOKEN: ${{ secrets.GITEA_TOKEN || github.token }} SHA: ${{ steps.src.outputs.sha }} @@ -175,6 +267,10 @@ jobs: - **\`*-observer\`**: repeater s BLE konzolí a buffery \`rxlog\` / \`pktlog\` pro repeater observer. BLE je po nahrání vypnuté: nejdřív \`ble pin\`, pak \`ble on\`. - **RX duty cycle** (\`set radio.rxps\`, jen SX1262) je zabudovaný a vypnutý. + - **EXPERIMENTÁLNÍ \`*-ota\` / \`*-observer-ota\`**: totéž s aktualizací firmwaru od admina (\`fw ...\`, i přes + mesh). Ke každému plný balíček \`.mcfw\` (USB/BLE) a delta \`.from-.mcfw\` od předchozích vydání – + přes mesh uzel přijme jen deltu a nikdy starší verzi. Balíčky i \`SHA256SUMS\` jsou podepsané klíčem + \`meshcore-ota@tth-pub\` (\`tools/ota_allowed_signers\`). - Ověření: \`ver\` ukáže verzi z řádku *Verze v uzlu* výše. Co fork mění proti upstreamu: [README]($readme).
English @@ -187,6 +283,10 @@ jobs: - **\`*-observer\`**: repeater with a BLE console and the \`rxlog\` / \`pktlog\` rings for the repeater observer. BLE is off after flashing: \`ble pin\` first, then \`ble on\`. - **RX duty cycle** (\`set radio.rxps\`, SX1262 only) is built in and off. + - **EXPERIMENTAL \`*-ota\` / \`*-observer-ota\`**: the same with admin-pushed firmware update (\`fw ...\`, over the + mesh too). Each with a full \`.mcfw\` package (USB/BLE) and a delta \`.from-.mcfw\` from earlier + releases -- over the mesh a node takes only a delta and never an older version. Packages and \`SHA256SUMS\` + are signed by \`meshcore-ota@tth-pub\` (\`tools/ota_allowed_signers\`). - Check: \`ver\` shows the version from the *Verze v uzlu* (node version) line above. What the fork changes against upstream: [README]($readme).