diff --git a/README.md b/README.md index 7a47feb..007588c 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,10 @@ binary is verified by SHA-256. (dispatch/cron runs scanned the whole tree as a root commit before). `v8` dtrack-sbom really never fails the job: the runner's `bash -e` turned an unreachable Dependency-Track into a failed build instead of a warning. +`v12` portainer-deploy waits for the Swarm rollout before the smoke check +and fails on a paused or rolled-back update: the smoke URL used to answer +from the old task, so a new image that never started still gave a green +deploy. Pinned tools (Trivy binary + image, gitleaks) are bumped by Renovate (tth/renovate). Its post-upgrade task `.gitea/update-checksums.sh` @@ -122,7 +126,11 @@ last step of the build → `trivy-gate` → move `latest` → deploy pipeline. T compose file in git is the single source of truth: Portainer's stored compose is replaced on every deploy. The stack's env vars (where the secrets live) are kept and never logged; only `vars` are overwritten. Images are -re-pulled (`pullImage`). +re-pulled (`pullImage`). The step then waits until every service the deploy +changed has finished its Swarm update (`wait-rollout`, `rollout-timeout`, +default 600 s incl. image pulls) and fails when the update is paused or +rolled back; only then is `smoke-url` checked. Services the deploy did not +touch are ignored, even when an older update of theirs is still paused. ```yaml - uses: https://git.rfa.cz/tth-pub/ci-actions/portainer-deploy@v5 @@ -136,6 +144,7 @@ re-pulled (`pullImage`). smoke-url: https://app.example.org/health # optional smoke-status: "200" smoke-timeout: "180" + rollout-timeout: "600" # optional; wait-rollout: "false" = old behaviour ``` Config files from the repo can ride along as Swarm configs. Each becomes a diff --git a/portainer-deploy/action.yml b/portainer-deploy/action.yml index 739b0a6..e376f6c 100644 --- a/portainer-deploy/action.yml +++ b/portainer-deploy/action.yml @@ -5,7 +5,9 @@ description: > replaced on every deploy, the stack's env vars (where the secrets live) are kept and only the given variables are overwritten. Images are re-pulled. Config files from the repo can be shipped as content-addressed Swarm - configs. Optionally waits until a URL answers with the expected status. + configs. Waits until Swarm has finished rolling out the services the + deploy changed (a paused or rolled-back update fails the step), then + optionally until a URL answers with the expected status. inputs: url: @@ -46,8 +48,19 @@ inputs: prune: description: Remove services that are no longer in the compose file. default: "false" + wait-rollout: + description: > + Wait until every service this deploy changed has finished its Swarm + update; fail when the update is paused or rolled back. "false" = return + right after the stack update (the old behaviour). + default: "true" + rollout-timeout: + description: Seconds to wait for the rollout (image pulls included). + default: "600" smoke-url: - description: URL polled after the deploy until it answers smoke-status. Empty = no check. + description: > + URL polled after the rollout until it answers smoke-status. Empty = no + check. default: "" smoke-status: description: Expected HTTP status of smoke-url. @@ -70,6 +83,8 @@ runs: CONFIGS: ${{ inputs.configs }} CONFIG_PREFIX: ${{ inputs.config-prefix }} PRUNE: ${{ inputs.prune }} + WAIT_ROLLOUT: ${{ inputs.wait-rollout }} + ROLLOUT_TIMEOUT: ${{ inputs.rollout-timeout }} SMOKE_URL: ${{ inputs.smoke-url }} SMOKE_STATUS: ${{ inputs.smoke-status }} SMOKE_TIMEOUT: ${{ inputs.smoke-timeout }} @@ -145,6 +160,24 @@ runs: sets[var] = name _, stack = api("GET", f"/api/stacks/{stack_id}") + + # Rollout: remember each service's last update start before the + # PUT; afterwards only the services whose update start changed are + # this deploy's. Comparing with the previous value, not with the + # clock, keeps runner/Swarm clock skew out of it. Without this the + # smoke check below answered from the OLD task (birdnet-go + # 2026-11-03: green CI, then 1 min of 503) and a paused update + # went unnoticed (revaj-crm-stage 2026-11-02). + wait = E.get("WAIT_ROLLOUT", "true").lower() == "true" + sflt = urllib.parse.quote(json.dumps( + {"label": [f"com.docker.stack.namespace={stack.get('Name', '')}"]})) + + def services(): + _, out = api("GET", f"{docker}/services?filters={sflt}") + return {s["ID"]: s for s in out or []} + + before = {i: (s.get("UpdateStatus") or {}).get("StartedAt") + for i, s in services().items()} if wait else {} env = [e for e in (stack.get("Env") or []) if e.get("name") not in sets] kept = len(env) env += [{"name": k, "value": v} for k, v in sets.items()] @@ -159,6 +192,43 @@ runs: shown = ", ".join(f"{k}={v}" for k, v in sets.items()) or "nothing" print(f"stack {stack.get('Name', stack_id)} updated from {E['COMPOSE_FILE']}: {shown}; {kept} other variable(s) kept") + if wait: + rollout_timeout = float(E.get("ROLLOUT_TIMEOUT") or 600) + start = time.time() + deadline, grace, shown_at = start + rollout_timeout, start + 20, 0.0 + while True: + pending, failed, done = [], [], [] + for i, s in services().items(): + u = s.get("UpdateStatus") or {} + if not u or (i in before and u.get("StartedAt") == before[i]): + continue # untouched by this deploy, or newly created + name, state = s["Spec"]["Name"], u.get("State", "") + if state == "completed": + done.append(name) + elif state == "updating": + pending.append(name) + else: # paused, rollback_started / _paused / _completed + failed.append(f"{name}: {state} ({u.get('Message', '')})") + if failed: + raise RuntimeError("rollout failed - " + "; ".join(failed)) + if pending: + if time.time() >= deadline: + raise RuntimeError(f"rollout not finished after {rollout_timeout:.0f} s: {', '.join(pending)}") + if time.time() - shown_at >= 30: + print(f"rollout: waiting for {', '.join(pending)}", flush=True) + shown_at = time.time() + elif done: + print(f"rollout: completed in {time.time() - start:.0f} s: {', '.join(done)}") + break + elif time.time() >= grace: + # The PUT can return before Swarm registers the update + # (birdnet-go: PUT logged 00:29:25.908, update started + # 25.929), hence the grace. Nothing after it = same + # image digest and configs, Swarm had nothing to update. + print("rollout: no service was updated by this deploy") + break + time.sleep(5) + # Older versions go once nothing mounts them; Docker refuses to # delete a config in use (the previous task may still be draining), # the next deploy retries.