.github/workflows/ci.yml

@@ -61,17 +61,70 @@ jobs:
 
   test:
     runs-on: ubuntu-latest
+    # ~16-19 min of actual work (runs 117-124); the extra budget covers time
+    # spent queued in "Wait for any earlier test job" below. Also a runaway
+    # ceiling — Mocha has no `--exit` (`apps/api/.mocharc.json`), so one
+    # leaked handle would otherwise hang the job until the runner's global
+    # timeout (a `test` job zombied ~1h once). If a hung job hits this
+    # ceiling the runner removes its container, which unblocks the queue.
+    timeout-minutes: 45
     steps:
       - name: Install curl and the docker CLI
         run: |
           apt-get update && apt-get install -y curl docker.io
 
+      # Serialize `test` across every concurrent run. Two `test` jobs must
+      # never overlap: they share one `--name postgres` container on the
+      # runner's shared docker network (next step), so their `prisma migrate
+      # deploy` + `seedReferenceData` (root Mocha `beforeAll`) race —
+      # `Unique constraint failed on (key)` in `seedReferenceData`, "No
+      # pending migrations" when there should be some, `tech_step`/`diet` id
+      # shifts breaking unrelated assertions; and even past that they starve
+      # each other's RAM/CPU (each also trains a spaCy model) until
+      # `resetDatabase()` overruns Mocha's 10s hook timeout and cascades.
+      # Hit repeatedly (PRs #47, #53, #54), each needing an empty-commit
+      # re-trigger once the runner was idle.
+      #
+      # GitHub's `concurrency:` is the native fix but this Forgejo (9.0.3)
+      # predates Actions `concurrency` — it arrived in Forgejo v14 / Gitea
+      # 1.26 and is silently ignored here. A lock object (network, container,
+      # file) would leak: this runner (forgejo-runner v3.5.0) does NOT run a
+      # trailing `if: always()` step after an earlier step fails, so there is
+      # no reliable in-job release. Instead, key off the one thing the runner
+      # *does* always clean up — the job container itself (removed on pass,
+      # fail, and cancel alike): wait while any lower-numbered `test` job's
+      # container is still running. Lower task id goes first, so two jobs
+      # that start together can't deadlock; nothing to release.
+      - name: Wait for any earlier test job
+        run: |
+          my_id="$(printf '%s\n' "${JOB_CONTAINER_NAME:-}" | sed -n 's/.*TASK-\([0-9]\{1,\}\).*/\1/p')"
+          echo "this test job is task ${my_id:-<unknown, not serializing>}"
+          # ~25 min cap (75 * 20s): comfortably covers one earlier run
+          # (the realistic overlap is two runs); a rarer 3-way pile-up whose
+          # wait exceeds this just fails and gets an empty-commit re-run.
+          for attempt in $(seq 1 75); do
+            earlier="$(docker ps --format '{{.Names}}' 2>/dev/null \
+              | sed -n 's/.*TASK-\([0-9]\{1,\}\)_.*_JOB-test$/\1/p' \
+              | awk -v me="${my_id:-0}" 'NF && ($1 + 0) < (me + 0)')"
+            if [ -z "$earlier" ]; then
+              [ "$attempt" -gt 1 ] && echo "earlier test job(s) finished — proceeding"
+              exit 0
+            fi
+            echo "waiting for earlier test job(s): $(printf '%s ' $earlier)(attempt ${attempt})"
+            sleep 20
+          done
+          echo "still blocked by an earlier test job after ~25 min — giving up; a re-run should get through" >&2
+          exit 1
+
       # Postgres as a plain background container instead of a `services:`
       # block — see this workflow's top-level `env.DATABASE_URL` comment for
       # why. Reachable as `postgres` from this job (the runner mounts the
       # docker socket and runs jobs on a shared user-defined network, so a
       # `--name postgres` sibling resolves by name); the network is
-      # discovered from the job container itself rather than hardcoded.
+      # discovered from the job container itself rather than hardcoded. The
+      # `docker rm -f postgres` below also clears a container left behind by
+      # the previous run (this runner has no reliable post-job cleanup), and
+      # the serialization step above means no other run is using it now.
       - name: Start Postgres in the background
         run: |
           net="$(docker inspect "$(hostname)" \
@@ -79,6 +132,15 @@ jobs:
             | awk '{print $1}')"
           echo "job network: ${net:?could not determine the job container network}"
           docker rm -f postgres >/dev/null 2>&1 || true
+          # Also drop any `postgres-<suffix>` container. A since-reverted
+          # attempt at per-run Postgres naming on this branch left some
+          # running on this persistent runner (their `if: always()` teardown
+          # never fired — this runner skips trailing steps after a failure),
+          # and nothing else removes them. A second idle Postgres here starves
+          # the runner's RAM and slows every query ~5x → `resetDatabase()`
+          # blows Mocha's 10s hook timeout. Nothing creates `postgres-*` any
+          # more, so this only ever clears that debris.
+          docker ps -aq --filter 'name=^postgres-' | xargs -r docker rm -f >/dev/null 2>&1 || true
           docker run -d --name postgres --network "$net" \
             -e POSTGRES_USER=ci -e POSTGRES_PASSWORD=ci -e POSTGRES_DB=batchcooking_ci \
             postgres:16-alpine