From 94d0ba6b7877c1d04fbe488dd43f16cb4632d693 Mon Sep 17 00:00:00 2001 From: TeTeHacko Date: Tue, 3 Nov 2026 01:40:24 +0100 Subject: [PATCH] portainer-deploy: wait for the Swarm rollout before the smoke check (v12) The smoke URL answered from the old task right after the stack PUT, so CI went green before anything was deployed (birdnet-go 2026-11-03: green, then 1 min of 503) and a paused update stayed unnoticed (revaj-crm-stage 2026-11-02, new image exited 1 twice, CI green). Now the step waits until every service whose UpdateStatus.StartedAt changed is completed and fails on paused / rollback_*; untouched services are ignored. Co-Authored-By: Claude Opus 5.5 --- README.md | 11 +++++- portainer-deploy/action.yml | 74 ++++++++++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7a47feb..007588c 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,10 @@ binary is verified by SHA-256. (dispatch/cron runs scanned the whole tree as a root commit before). `v8` dtrack-sbom really never fails the job: the runner's `bash -e` turned an unreachable Dependency-Track into a failed build instead of a warning. +`v12` portainer-deploy waits for the Swarm rollout before the smoke check +and fails on a paused or rolled-back update: the smoke URL used to answer +from the old task, so a new image that never started still gave a green +deploy. Pinned tools (Trivy binary + image, gitleaks) are bumped by Renovate (tth/renovate). Its post-upgrade task `.gitea/update-checksums.sh` @@ -122,7 +126,11 @@ last step of the build → `trivy-gate` → move `latest` → deploy pipeline. T compose file in git is the single source of truth: Portainer's stored compose is replaced on every deploy. The stack's env vars (where the secrets live) are kept and never logged; only `vars` are overwritten. Images are -re-pulled (`pullImage`). +re-pulled (`pullImage`). The step then waits until every service the deploy +changed has finished its Swarm update (`wait-rollout`, `rollout-timeout`, +default 600 s incl. image pulls) and fails when the update is paused or +rolled back; only then is `smoke-url` checked. Services the deploy did not +touch are ignored, even when an older update of theirs is still paused. ```yaml - uses: https://git.rfa.cz/tth-pub/ci-actions/portainer-deploy@v5 @@ -136,6 +144,7 @@ re-pulled (`pullImage`). smoke-url: https://app.example.org/health # optional smoke-status: "200" smoke-timeout: "180" + rollout-timeout: "600" # optional; wait-rollout: "false" = old behaviour ``` Config files from the repo can ride along as Swarm configs. Each becomes a diff --git a/portainer-deploy/action.yml b/portainer-deploy/action.yml index 739b0a6..e376f6c 100644 --- a/portainer-deploy/action.yml +++ b/portainer-deploy/action.yml @@ -5,7 +5,9 @@ description: > replaced on every deploy, the stack's env vars (where the secrets live) are kept and only the given variables are overwritten. Images are re-pulled. Config files from the repo can be shipped as content-addressed Swarm - configs. Optionally waits until a URL answers with the expected status. + configs. Waits until Swarm has finished rolling out the services the + deploy changed (a paused or rolled-back update fails the step), then + optionally until a URL answers with the expected status. inputs: url: @@ -46,8 +48,19 @@ inputs: prune: description: Remove services that are no longer in the compose file. default: "false" + wait-rollout: + description: > + Wait until every service this deploy changed has finished its Swarm + update; fail when the update is paused or rolled back. "false" = return + right after the stack update (the old behaviour). + default: "true" + rollout-timeout: + description: Seconds to wait for the rollout (image pulls included). + default: "600" smoke-url: - description: URL polled after the deploy until it answers smoke-status. Empty = no check. + description: > + URL polled after the rollout until it answers smoke-status. Empty = no + check. default: "" smoke-status: description: Expected HTTP status of smoke-url. @@ -70,6 +83,8 @@ runs: CONFIGS: ${{ inputs.configs }} CONFIG_PREFIX: ${{ inputs.config-prefix }} PRUNE: ${{ inputs.prune }} + WAIT_ROLLOUT: ${{ inputs.wait-rollout }} + ROLLOUT_TIMEOUT: ${{ inputs.rollout-timeout }} SMOKE_URL: ${{ inputs.smoke-url }} SMOKE_STATUS: ${{ inputs.smoke-status }} SMOKE_TIMEOUT: ${{ inputs.smoke-timeout }} @@ -145,6 +160,24 @@ runs: sets[var] = name _, stack = api("GET", f"/api/stacks/{stack_id}") + + # Rollout: remember each service's last update start before the + # PUT; afterwards only the services whose update start changed are + # this deploy's. Comparing with the previous value, not with the + # clock, keeps runner/Swarm clock skew out of it. Without this the + # smoke check below answered from the OLD task (birdnet-go + # 2026-11-03: green CI, then 1 min of 503) and a paused update + # went unnoticed (revaj-crm-stage 2026-11-02). + wait = E.get("WAIT_ROLLOUT", "true").lower() == "true" + sflt = urllib.parse.quote(json.dumps( + {"label": [f"com.docker.stack.namespace={stack.get('Name', '')}"]})) + + def services(): + _, out = api("GET", f"{docker}/services?filters={sflt}") + return {s["ID"]: s for s in out or []} + + before = {i: (s.get("UpdateStatus") or {}).get("StartedAt") + for i, s in services().items()} if wait else {} env = [e for e in (stack.get("Env") or []) if e.get("name") not in sets] kept = len(env) env += [{"name": k, "value": v} for k, v in sets.items()] @@ -159,6 +192,43 @@ runs: shown = ", ".join(f"{k}={v}" for k, v in sets.items()) or "nothing" print(f"stack {stack.get('Name', stack_id)} updated from {E['COMPOSE_FILE']}: {shown}; {kept} other variable(s) kept") + if wait: + rollout_timeout = float(E.get("ROLLOUT_TIMEOUT") or 600) + start = time.time() + deadline, grace, shown_at = start + rollout_timeout, start + 20, 0.0 + while True: + pending, failed, done = [], [], [] + for i, s in services().items(): + u = s.get("UpdateStatus") or {} + if not u or (i in before and u.get("StartedAt") == before[i]): + continue # untouched by this deploy, or newly created + name, state = s["Spec"]["Name"], u.get("State", "") + if state == "completed": + done.append(name) + elif state == "updating": + pending.append(name) + else: # paused, rollback_started / _paused / _completed + failed.append(f"{name}: {state} ({u.get('Message', '')})") + if failed: + raise RuntimeError("rollout failed - " + "; ".join(failed)) + if pending: + if time.time() >= deadline: + raise RuntimeError(f"rollout not finished after {rollout_timeout:.0f} s: {', '.join(pending)}") + if time.time() - shown_at >= 30: + print(f"rollout: waiting for {', '.join(pending)}", flush=True) + shown_at = time.time() + elif done: + print(f"rollout: completed in {time.time() - start:.0f} s: {', '.join(done)}") + break + elif time.time() >= grace: + # The PUT can return before Swarm registers the update + # (birdnet-go: PUT logged 00:29:25.908, update started + # 25.929), hence the grace. Nothing after it = same + # image digest and configs, Swarm had nothing to update. + print("rollout: no service was updated by this deploy") + break + time.sleep(5) + # Older versions go once nothing mounts them; Docker refuses to # delete a config in use (the previous task may still be draining), # the next deploy retries.