From 4dca3688eab860e508b678825026f4a8864c770c Mon Sep 17 00:00:00 2001 From: agessaman Date: Sat, 19 Sep 2026 18:03:09 -0700 Subject: [PATCH] ci(observer): prune stale manifests and verify the production OTA channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps found while tracing a ThinkNode M7 report of a missing OTA manifest. The M7 itself was healthy in both channels; these are what the investigation turned up around it. Pass --prune to gen-slim-manifests.py in both workflows. The generator only ever added, so an env that stopped being built kept a manifest pointing at a release asset the KEEP_BUILDS=2 prune later deleted — a 404 for `ota update` on any node still running it. The flasher-side commit adds the flag along with an empty-input guard and --prune-limit, which fails the run rather than pruning when a build looks like it silently lost envs. Add "Verify production channel is baked in", which production lacked while beta has had it from the start. The manifest base is a compile-time -D, so a build that lost it or picked up the other channel's is invisible until a node runs `ota check` — and the failure mode is production hardware OTA-ing itself onto beta. Checked against real published binaries: passes the production M7 build, rejects the beta one. OTA_MANIFEST_BASE_URL is now stated explicitly in the production env: block, equal to build.sh's default, so the verify step asserts against the value the build was actually handed instead of a second hardcoded copy that could drift. Neither change triggers a build: .github/** is in both workflows' paths-ignore. --- .../build-observer-firmwares-beta.yml | 8 +++++ .../workflows/build-observer-firmwares.yml | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/.github/workflows/build-observer-firmwares-beta.yml b/.github/workflows/build-observer-firmwares-beta.yml index 8618da4b9..a81e34844 100644 --- a/.github/workflows/build-observer-firmwares-beta.yml +++ b/.github/workflows/build-observer-firmwares-beta.yml @@ -312,8 +312,16 @@ jobs: # Slim manifests come from the build output in out/ (the assets # actually uploaded to the release), not from config-beta.json — see # gen-slim-manifests.py's --bin-dir mode (flasher repo PR #1). + # --prune deletes manifests for envs this build no longer produces. + # Without it the dir only ever grew: the retired + # LilyGo_TLora_V2_1_1_6_*_observer_mqtt_ manifests sat here for two + # months pointing at release assets that had since been pruned, so any + # node still on that build got a 404 from `ota update`. The generator + # caps the prune (--prune-limit, default 4) and fails the run instead + # of pruning when a build looks like it silently lost envs. python3 flasher/scripts/gen-slim-manifests.py \ --bin-dir out \ + --prune \ --static-path "$STATIC_PATH" \ --out-dir "flasher/$MANIFEST_DIR" \ --base-version "$FIRMWARE_VERSION" \ diff --git a/.github/workflows/build-observer-firmwares.yml b/.github/workflows/build-observer-firmwares.yml index 13d9cda0f..81312ea2a 100644 --- a/.github/workflows/build-observer-firmwares.yml +++ b/.github/workflows/build-observer-firmwares.yml @@ -48,6 +48,12 @@ env: # repo). Baked into the slim OTA manifests' file URLs; must stay consistent # with config.json's staticPath. STATIC_PATH: https://observer-fw.gessaman.com + # Where the firmware fetches .json from — this URL *is* the release + # channel. Stated explicitly (it equals build.sh's default) rather than left + # unset, so "Verify production channel is baked in" below can assert against the + # same value the build was handed instead of a second hardcoded copy. Must match + # build.sh's OTA_MANIFEST_BASE_URL default and the Pages path serving flasher/v. + OTA_MANIFEST_BASE_URL: https://observer.gessaman.com/v jobs: @@ -137,6 +143,23 @@ jobs: FIRMWARE_BUILD_NUMBER: ${{ needs.enumerate.outputs.build_number }} run: /usr/bin/env bash build.sh build-firmware ${{ matrix.shard.envs }} + - name: Verify production channel is baked in + shell: bash + run: | + # Mirror of the beta workflow's guard, which production lacked. Fail fast + # rather than publish firmware that would OTA itself onto the wrong + # channel: the manifest base is a compile-time -D, so a build that lost + # it (or picked up beta's) is invisible until a node tries `ota check`. + BIN=$(find .pio/build -name firmware.elf | head -1) + if [ -z "$BIN" ]; then echo "no ELF found to verify" >&2; exit 1; fi + if ! strings "$BIN" | grep -qF "$OTA_MANIFEST_BASE_URL"; then + echo "ERROR: production manifest base missing from $BIN" >&2; exit 1 + fi + if strings "$BIN" | grep -qF 'https://observer.gessaman.com/beta/v'; then + echo "ERROR: beta manifest base present in a production build" >&2; exit 1 + fi + echo "OK: $BIN carries $OTA_MANIFEST_BASE_URL" + - name: Upload Shard Artifact uses: actions/upload-artifact@v4 with: @@ -253,8 +276,15 @@ jobs: # in out/ — the assets actually uploaded to the release — stamping this # build's number. Then persist the counter so the next run increments # from here. + # --prune deletes manifests for envs this build no longer produces, so + # a retired env cannot leave behind a manifest pointing at a release + # asset that later gets pruned (which is a 404 for `ota update` on any + # node still running it). The generator caps the prune (--prune-limit, + # default 4) and fails the run rather than pruning when a build looks + # like it silently lost envs. python3 flasher/scripts/gen-slim-manifests.py \ --bin-dir out \ + --prune \ --static-path "$STATIC_PATH" \ --out-dir flasher/v \ --base-version "$FIRMWARE_VERSION" \