diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c44edeb..f917566 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,17 +61,70 @@ jobs: test: runs-on: ubuntu-latest + # ~16-19 min of actual work (runs 117-124); the extra budget covers time + # spent queued in "Wait for any earlier test job" below. Also a runaway + # ceiling — Mocha has no `--exit` (`apps/api/.mocharc.json`), so one + # leaked handle would otherwise hang the job until the runner's global + # timeout (a `test` job zombied ~1h once). If a hung job hits this + # ceiling the runner removes its container, which unblocks the queue. + timeout-minutes: 45 steps: - name: Install curl and the docker CLI run: | apt-get update && apt-get install -y curl docker.io + # Serialize `test` across every concurrent run. Two `test` jobs must + # never overlap: they share one `--name postgres` container on the + # runner's shared docker network (next step), so their `prisma migrate + # deploy` + `seedReferenceData` (root Mocha `beforeAll`) race — + # `Unique constraint failed on (key)` in `seedReferenceData`, "No + # pending migrations" when there should be some, `tech_step`/`diet` id + # shifts breaking unrelated assertions; and even past that they starve + # each other's RAM/CPU (each also trains a spaCy model) until + # `resetDatabase()` overruns Mocha's 10s hook timeout and cascades. + # Hit repeatedly (PRs #47, #53, #54), each needing an empty-commit + # re-trigger once the runner was idle. + # + # GitHub's `concurrency:` is the native fix but this Forgejo (9.0.3) + # predates Actions `concurrency` — it arrived in Forgejo v14 / Gitea + # 1.26 and is silently ignored here. A lock object (network, container, + # file) would leak: this runner (forgejo-runner v3.5.0) does NOT run a + # trailing `if: always()` step after an earlier step fails, so there is + # no reliable in-job release. Instead, key off the one thing the runner + # *does* always clean up — the job container itself (removed on pass, + # fail, and cancel alike): wait while any lower-numbered `test` job's + # container is still running. Lower task id goes first, so two jobs + # that start together can't deadlock; nothing to release. + - name: Wait for any earlier test job + run: | + my_id="$(printf '%s\n' "${JOB_CONTAINER_NAME:-}" | sed -n 's/.*TASK-\([0-9]\{1,\}\).*/\1/p')" + echo "this test job is task ${my_id:-}" + # ~25 min cap (75 * 20s): comfortably covers one earlier run + # (the realistic overlap is two runs); a rarer 3-way pile-up whose + # wait exceeds this just fails and gets an empty-commit re-run. + for attempt in $(seq 1 75); do + earlier="$(docker ps --format '{{.Names}}' 2>/dev/null \ + | sed -n 's/.*TASK-\([0-9]\{1,\}\)_.*_JOB-test$/\1/p' \ + | awk -v me="${my_id:-0}" 'NF && ($1 + 0) < (me + 0)')" + if [ -z "$earlier" ]; then + [ "$attempt" -gt 1 ] && echo "earlier test job(s) finished — proceeding" + exit 0 + fi + echo "waiting for earlier test job(s): $(printf '%s ' $earlier)(attempt ${attempt})" + sleep 20 + done + echo "still blocked by an earlier test job after ~25 min — giving up; a re-run should get through" >&2 + exit 1 + # Postgres as a plain background container instead of a `services:` # block — see this workflow's top-level `env.DATABASE_URL` comment for # why. Reachable as `postgres` from this job (the runner mounts the # docker socket and runs jobs on a shared user-defined network, so a # `--name postgres` sibling resolves by name); the network is - # discovered from the job container itself rather than hardcoded. + # discovered from the job container itself rather than hardcoded. The + # `docker rm -f postgres` below also clears a container left behind by + # the previous run (this runner has no reliable post-job cleanup), and + # the serialization step above means no other run is using it now. - name: Start Postgres in the background run: | net="$(docker inspect "$(hostname)" \ @@ -79,6 +132,15 @@ jobs: | awk '{print $1}')" echo "job network: ${net:?could not determine the job container network}" docker rm -f postgres >/dev/null 2>&1 || true + # Also drop any `postgres-` container. A since-reverted + # attempt at per-run Postgres naming on this branch left some + # running on this persistent runner (their `if: always()` teardown + # never fired — this runner skips trailing steps after a failure), + # and nothing else removes them. A second idle Postgres here starves + # the runner's RAM and slows every query ~5x → `resetDatabase()` + # blows Mocha's 10s hook timeout. Nothing creates `postgres-*` any + # more, so this only ever clears that debris. + docker ps -aq --filter 'name=^postgres-' | xargs -r docker rm -f >/dev/null 2>&1 || true docker run -d --name postgres --network "$net" \ -e POSTGRES_USER=ci -e POSTGRES_PASSWORD=ci -e POSTGRES_DB=batchcooking_ci \ postgres:16-alpine