From 2d676cc279e18b5b3387e8759c2ef6cf3972e3c6 Mon Sep 17 00:00:00 2001 From: Julien Gautier Date: Sun, 10 May 2026 06:37:07 +0200 Subject: [PATCH] feat(infra): add ci-runners.sh wrapper for the runner stack (#81) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Mirrors the convenience-script pattern we adopted for `infra/local/dev.sh`: typing `docker compose -f infra/ci-runners.compose.yml ...` for routine ops gets old fast, the pre-pull of the catthehacker job images is documented but easy to forget, and the "rotation of one runner at a time" tip in `infra/README.md` is a sequence the contributor was supposed to hand-roll every time. `infra/ci-runners.sh` exposes the everyday verbs and automates the rolling-restart pattern. ## What lands | Command | Effect | | --------------------------------------------- | --------------------------------------------------------------------------------- | | `./ci-runners.sh up` | Bring the three runner containers up | | `./ci-runners.sh up --prepull` | Pre-pull the job images (`act-22.04` + `:full-22.04`) on the host first | | `./ci-runners.sh down` | Stop and remove the containers (**preserves** `data/runner-N/.runner` credentials) | | `./ci-runners.sh restart ` | Restart one runner | | `./ci-runners.sh rotate` | Rolling restart of every runner with a 15 s pause between each — keeps at least N-1 runners online through a config refresh | | `./ci-runners.sh status` | `docker compose ps` for the runner services | | `./ci-runners.sh logs [runner]` | Follow logs (one runner or all of them) | | `./ci-runners.sh pull-images` | Pre-pull / refresh the job images (idempotent) | | `./ci-runners.sh ` | Pass-through to `docker compose -f ci-runners.compose.yml ...` | The destructive `down -v` (wipes `data/`, forces re-registration with a fresh Gitea token) is intentionally **not** exposed as a verb — invoke `docker compose -f ci-runners.compose.yml down -v` directly so the path is explicit at the typing level. ## Doc updates (`infra/README.md`) - Inventory table at the top picks up the script. - "First-time registration" walkthrough swaps the explicit `docker pull` / `docker compose up` steps for `./ci-runners.sh up --prepull`. - New "Convenience script — `ci-runners.sh`" subsection with the cheat-sheet table. - "Operational tips" rephrased to point at the script's `rotate` / `restart` / `logs` verbs as the canonical commands; the raw-docker-compose form is kept in parentheses as the underlying mechanism. - "Adding a fourth runner" tip now reminds to update the `RUNNERS=()` array at the top of the script. ## Trade-off The 15 s pause in `rotate` is a conservative approximation — `act_runner` doesn't expose a Compose healthcheck, so we can't poll for ready. Adjust the constant at the top of the script if reality argues for a different value. ## Test plan - [ ] CI green on this PR. - [ ] On the runner host: `./infra/ci-runners.sh status` shows the three runners running. - [ ] `./infra/ci-runners.sh logs runner-1` tails runner-1's stdout. - [ ] `./infra/ci-runners.sh rotate` cycles through runner-1 → runner-2 → runner-3 with the 15 s pauses; `status` between rotations shows N-1 runners online at any moment (with a brief gap for the one currently restarting). - [ ] `./infra/ci-runners.sh restart runner-99` errors out with the "unknown runner" message. --------- Co-authored-by: Julien Gautier Reviewed-on: https://git.unespace.com/julien/apf_portal/pulls/81 --- infra/README.md | 46 ++++++++---- infra/ci-runners.sh | 169 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+), 13 deletions(-) create mode 100755 infra/ci-runners.sh diff --git a/infra/README.md b/infra/README.md index 5ff0b4d..b1d1d1f 100644 --- a/infra/README.md +++ b/infra/README.md @@ -6,6 +6,7 @@ Infrastructure-as-code artefacts for the project. Separate from application code | -------------------------------------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Self-hosted CI runners (Gitea Actions) | [`ci-runners.compose.yml`](ci-runners.compose.yml) | [ADR-0015 §"Runners"](../docs/decisions/0015-cicd-gitea-actions.md) | | Shared `act_runner` configuration | [`runner-config.yaml`](runner-config.yaml) | [ADR-0015 §"Runners"](../docs/decisions/0015-cicd-gitea-actions.md) | +| CI runners convenience script | [`ci-runners.sh`](ci-runners.sh) | See "Convenience script" below | | Runtime state of the runners | `data/` (git-ignored after `.gitignore`) | — | | Env-vars template for the runners | `.env.example` (`.env` is git-ignored) | — | | Local-dev runtime stack | [`local/`](local/) | [ADR-0006](../docs/decisions/0006-persistence-postgresql-prisma.md), [ADR-0010](../docs/decisions/0010-session-management-redis.md), [ADR-0012](../docs/decisions/0012-observability-pino-opentelemetry.md), [ADR-0013](../docs/decisions/0013-audit-trail-separated-postgres-append-only.md) | @@ -36,28 +37,47 @@ $EDITOR .env # Set GITEA_INSTANCE_URL (https, no trailing slash) and # GITEA_RUNNER_REGISTRATION_TOKEN. -# 3. Pre-pull the job images so the runner doesn't have to (see -# "Job image pinning and pre-pull" below for the rationale). -docker pull catthehacker/ubuntu:act-22.04 -docker pull catthehacker/ubuntu:full-22.04 +# 3. Pre-pull the job images and bring the runners up. The script +# chains the two — see "Job image pinning and pre-pull" below +# for the rationale. +./ci-runners.sh up --prepull -# 4. Bring the runners up. -docker compose -f ci-runners.compose.yml up -d - -# 5. Verify in Gitea: the three runners appear as online with the +# 4. Verify in Gitea: the three runners appear as online with the # self-hosted, on-prem labels. If a runner doesn't come online, # inspect its logs: -docker compose -f ci-runners.compose.yml logs runner-1 +./ci-runners.sh logs runner-1 ``` -After the first successful boot, each runner stores its credentials under `data/runner-N/.runner`. The registration token is no longer needed and **should be removed** from `.env`. Subsequent restarts (`docker compose restart`) authenticate from the persisted credential. +After the first successful boot, each runner stores its credentials under `data/runner-N/.runner`. The registration token is no longer needed and **should be removed** from `.env`. Subsequent restarts (`./ci-runners.sh restart …` or direct `docker compose restart …`) authenticate from the persisted credential. + +### Convenience script — `ci-runners.sh` + +[`ci-runners.sh`](ci-runners.sh) is a thin wrapper around `docker compose -f ci-runners.compose.yml ...` for the everyday verbs. Two reasons to use it: + +1. **Hides the compose-file path** on every command. `./ci-runners.sh up` instead of `docker compose -f ci-runners.compose.yml up -d`. +2. **`rotate` automates the rolling restart** the "Operational tips" below recommend: runner-1 → wait → runner-2 → wait → runner-3, so the CI pipeline always has at least N-1 runners online while you push a config change. + +| Command | Effect | +| ---------------------------------- | ------------------------------------------------------------------------------ | +| `./ci-runners.sh up` | Bring the three runner containers up | +| `./ci-runners.sh up --prepull` | Pre-pull the job images (`act-22.04` + `:full-22.04`) on the host first | +| `./ci-runners.sh down` | Stop and remove the containers (preserves `data/runner-N/.runner` credentials) | +| `./ci-runners.sh restart ` | Restart one runner | +| `./ci-runners.sh rotate` | Rolling restart of every runner with a 15 s pause between each | +| `./ci-runners.sh status` | `docker compose ps` for the runner services | +| `./ci-runners.sh logs [runner]` | Follow logs (one runner or all of them) | +| `./ci-runners.sh pull-images` | Pre-pull / refresh the job images (idempotent) | + +Anything not matching one of the named verbs is passed through to `docker compose -f ci-runners.compose.yml ...`. Run `./ci-runners.sh help` for the full reference. + +For the destructive `down -v` (wipes `data/`, forces re-registration with a fresh Gitea token), the script intentionally **doesn't** offer a verb — invoke `docker compose -f ci-runners.compose.yml down -v` directly so the path is explicit at the typing level. ### Operational tips -- **Rotation of one runner at a time** — to upgrade the image or change config, restart runners one by one (`docker compose restart runner-1`, wait, runner-2, …) so the CI pipeline is never starved. -- **Logs** — `docker compose logs -f --tail=100 runner-N` for a single runner; jobs being executed appear here. +- **Rotation of one runner at a time** — to upgrade the image or change config, run `./ci-runners.sh rotate` (or restart manually one by one — `./ci-runners.sh restart runner-1`, wait, …) so the CI pipeline is never starved. +- **Logs** — `./ci-runners.sh logs runner-N` (or `docker compose logs -f --tail=100 runner-N`) for a single runner; jobs being executed appear here. - **Disk pressure** — the runner caches each job's container image in `/var/lib/docker` on the host. On a small host, prune periodically (`docker system prune -af` while no job is running). -- **Adding a fourth runner** — copy any `runner-N` block in the compose file, increment the suffix in `container_name`, `GITEA_RUNNER_NAME`, and the `data/` mount path. Then `docker compose up -d`. The runner registers using the same `GITEA_RUNNER_REGISTRATION_TOKEN` (which must be regenerated if it has expired). +- **Adding a fourth runner** — copy any `runner-N` block in the compose file, increment the suffix in `container_name`, `GITEA_RUNNER_NAME`, and the `data/` mount path. Add the new name to the `RUNNERS=(…)` array at the top of `ci-runners.sh` so `rotate` and `restart` learn about it. Then `./ci-runners.sh up` (or `docker compose up -d`). The runner registers using the same `GITEA_RUNNER_REGISTRATION_TOKEN` (which must be regenerated if it has expired). ### Security — Docker socket exposure diff --git a/infra/ci-runners.sh b/infra/ci-runners.sh new file mode 100755 index 0000000..5de7645 --- /dev/null +++ b/infra/ci-runners.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Convenience wrapper around infra/ci-runners.compose.yml. Documented +# in infra/README.md → "CI runners" → "Convenience script". +# +# Hides the compose-file path on every command, automates the +# pre-pull of the job images that ADR-0015 §"Job image pinning and +# pre-pull" otherwise leaves to a manual step, and exposes the +# "one-runner-at-a-time" rotation pattern as a single verb so the CI +# pipeline is never starved during a config refresh. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_FILE="${SCRIPT_DIR}/ci-runners.compose.yml" + +# Runner service names defined in the compose file. Update when a +# fourth runner is added (rare; documented in infra/README.md). +RUNNERS=(runner-1 runner-2 runner-3) + +# Pre-pulled job images. The labels in ci-runners.compose.yml point at +# these tags; act_runner with `force_pull: false` (per +# infra/runner-config.yaml) refuses to start a job if the image is not +# already cached locally. +JOB_IMAGES=( + "catthehacker/ubuntu:act-22.04" + "catthehacker/ubuntu:full-22.04" +) + +# Seconds to wait between rotation steps. The runners do not expose a +# Compose healthcheck; this is a conservative pause to let act_runner +# finish its registration handshake before we touch the next one. +ROTATE_WAIT_SECONDS=15 + +dc() { + docker compose -f "$COMPOSE_FILE" "$@" +} + +usage() { + cat <<'USAGE' +Usage: ./infra/ci-runners.sh [args] + +Commands: + up [--prepull] Bring up the three runner containers. + --prepull also fetches the act_runner job images + (`catthehacker/ubuntu:act-22.04` and + `:full-22.04`) on the host first — required + because `runner-config.yaml` sets + `container.force_pull: false`. + + down Stop and remove the runner containers. + Persistent state under `data/runner-N/.runner` + survives — the runners re-authenticate from + their stored credentials on the next `up`. + For a clean wipe (forces re-registration with + a fresh Gitea token), invoke + `docker compose -f ci-runners.compose.yml down -v` + directly. + + restart Restart one runner. + + rotate Rolling restart of every runner (runner-1, + wait, runner-2, wait, …). The pause between + steps lets the previous runner finish its + registration handshake before the next one + goes down, so the CI pipeline always has at + least N-1 runners online. + + status docker compose ps for the runner services. + + logs [runner] Follow logs (one runner or all of them). + + pull-images Pre-pull / refresh the act_runner job images + on the host. Idempotent (Docker reports + "Image is up to date" when nothing changed). + + help Show this help. + + [args...] Anything else is passed through to + `docker compose -f ci-runners.compose.yml ...`. + +Examples: + ./infra/ci-runners.sh up --prepull + ./infra/ci-runners.sh rotate + ./infra/ci-runners.sh logs runner-2 + ./infra/ci-runners.sh restart runner-1 +USAGE +} + +require_args() { + local verb="$1" + local n="$2" + local hint="$3" + shift 3 + if [[ $# -lt $n ]]; then + echo "Error: '$verb' requires $hint." >&2 + echo "Try: ./infra/ci-runners.sh help" >&2 + exit 64 + fi +} + +assert_known_runner() { + local name="$1" + for r in "${RUNNERS[@]}"; do + if [[ "$r" == "$name" ]]; then + return 0 + fi + done + echo "Error: unknown runner '$name'. Known: ${RUNNERS[*]}." >&2 + exit 64 +} + +pull_images() { + echo "Pre-pulling job images on the runner host:" + for img in "${JOB_IMAGES[@]}"; do + echo " • $img" + docker pull "$img" + done +} + +cmd="${1:-help}" +[[ $# -gt 0 ]] && shift + +case "$cmd" in + up) + if [[ "${1:-}" == "--prepull" ]]; then + pull_images + shift + fi + dc up -d "$@" + ;; + down) + dc down "$@" + ;; + restart) + require_args restart 1 "a runner name (e.g. runner-1)" "$@" + assert_known_runner "$1" + dc restart "$1" + ;; + rotate) + echo "Rolling restart of ${#RUNNERS[@]} runners — ${ROTATE_WAIT_SECONDS}s pause between each." + for r in "${RUNNERS[@]}"; do + echo + echo "→ Restarting $r…" + dc restart "$r" + if [[ "$r" != "${RUNNERS[-1]}" ]]; then + echo " Waiting ${ROTATE_WAIT_SECONDS}s before the next runner." + sleep "$ROTATE_WAIT_SECONDS" + fi + done + echo + echo "✓ Rotation complete." + ;; + status | ps) + dc ps "$@" + ;; + logs) + dc logs -f "$@" + ;; + pull-images) + pull_images + ;; + help | -h | --help | "") + usage + ;; + *) + # Pass-through. + dc "$cmd" "$@" + ;; +esac