From 57cb2b7b3d16e7dea6dcda20254bc208799c3cd3 Mon Sep 17 00:00:00 2001 From: Nicolas Date: Fri, 28 Aug 2026 12:55:47 +0200 Subject: [PATCH] feat(admin): monitoring des microservices + heartbeat du worker LLM PR 4 du chantier admin. Board de sante temps reel des dependances. - POST /internal/tech-steps/heartbeat (requireInternalWorker) -> recordWorkerHeartbeat : upsert WorkerHeartbeat (cle fixe "tech-step-llm-worker"), lastRunAt/lastResult pour un ping "job". Schema workerHeartbeatSchema dans packages/shared. - services/tech-step-llm-worker : api-client.postHeartbeat (best-effort, ne throw jamais) appele au boot (index.ts), a chaque tick et apres chaque job (scheduler.ts, avec job/ok/counts). - admin-monitoring.service.ts + GET /admin/monitoring (requireAdmin) : sonde active bornee (~2 s) de Postgres (SELECT 1), l'API (uptime/RSS), tech-step-intent-service (/health), et le worker via son heartbeat. Statut up/degraded/down/unknown ; une sonde down ne casse ni les autres ni l'endpoint. Seuils worker : > 8 j degraded, > 21 j down. - MonitoringView / ServiceHealthView dans packages/shared. - Front : MonitoringPage (grille de cartes coloree par statut, re-poll 15 s), logique pure monitoring.ts, i18n admin.monitoring.*, AdminApiClient.getMonitoring. - Tests : Mocha admin-monitoring.test.ts (heartbeat 401/400/upsert job+boot ; GET /admin/monitoring 401, board 4 cibles, worker unknown sans heartbeat puis up apres) ; Cypress monitoring.cy.ts (2 verts). Worker mocha : 6/6 toujours verts. - specs/backend-architecture.md : section monitoring. .gitignore : apps/admin-web/cypress/{screenshots,videos,downloads}. Co-Authored-By: Claude Sonnet 5 --- .gitignore | 3 + apps/admin-web/cypress/e2e/monitoring.cy.ts | 83 +++++++++ apps/admin-web/src/api/client.ts | 6 + .../admin-web/src/locales/fr/translation.json | 20 +- .../src/pages/monitoring/MonitoringPage.tsx | 100 +++++++++- .../src/pages/monitoring/monitoring-page.scss | 98 ++++++++++ .../src/pages/monitoring/monitoring.ts | 19 ++ .../modules/admin/admin-monitoring.routes.ts | 20 ++ .../modules/admin/admin-monitoring.service.ts | 174 ++++++++++++++++++ apps/api/src/modules/admin/admin.routes.ts | 2 + .../internal/tech-step-worker.routes.ts | 18 ++ .../internal/tech-step-worker.service.ts | 43 +++++ apps/api/test/admin-monitoring.test.ts | 160 ++++++++++++++++ packages/shared/src/schemas/admin.ts | 19 ++ packages/shared/src/types/admin.ts | 31 ++++ .../tech-step-llm-worker/src/api-client.ts | 29 +++ services/tech-step-llm-worker/src/index.ts | 6 + .../tech-step-llm-worker/src/scheduler.ts | 16 ++ specs/backend-architecture.md | 18 +- 19 files changed, 858 insertions(+), 7 deletions(-) create mode 100644 apps/admin-web/cypress/e2e/monitoring.cy.ts create mode 100644 apps/admin-web/src/pages/monitoring/monitoring-page.scss create mode 100644 apps/admin-web/src/pages/monitoring/monitoring.ts create mode 100644 apps/api/src/modules/admin/admin-monitoring.routes.ts create mode 100644 apps/api/src/modules/admin/admin-monitoring.service.ts create mode 100644 apps/api/test/admin-monitoring.test.ts diff --git a/.gitignore b/.gitignore index 0df7f12..bdcfe13 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,6 @@ tmp-mockups/ apps/web/cypress/screenshots/ apps/web/cypress/videos/ apps/web/cypress/downloads/ +apps/admin-web/cypress/screenshots/ +apps/admin-web/cypress/videos/ +apps/admin-web/cypress/downloads/ diff --git a/apps/admin-web/cypress/e2e/monitoring.cy.ts b/apps/admin-web/cypress/e2e/monitoring.cy.ts new file mode 100644 index 0000000..4b14571 --- /dev/null +++ b/apps/admin-web/cypress/e2e/monitoring.cy.ts @@ -0,0 +1,83 @@ +// Mocks the admin API via cy.intercept — no live backend. + +const adminBody = { + id: 1, + email: "ops@example.com", + name: "Ops", + createdAt: "2026-08-01T00:00:00.000Z", + lastLoginAt: "2026-08-28T09:00:00.000Z", +}; + +function monitoringFixture() { + return { + generatedAt: "2026-08-28T09:15:00.000Z", + services: [ + { + key: "postgres", + status: "up", + latencyMs: 3.2, + detail: null, + checkedAt: "2026-08-28T09:15:00.000Z", + }, + { + key: "api", + status: "up", + latencyMs: 0, + detail: "uptime 3 h 12 min · RSS 120 Mo", + checkedAt: "2026-08-28T09:15:00.000Z", + }, + { + key: "intent-service", + status: "down", + latencyMs: null, + detail: "fetch failed", + checkedAt: "2026-08-28T09:15:00.000Z", + }, + { + key: "tech-step-llm-worker", + status: "degraded", + latencyMs: null, + detail: "dernier battement il y a 9 j", + checkedAt: "2026-08-28T09:15:00.000Z", + lastRunAt: "2026-08-19T03:00:00.000Z", + lastResult: { job: "audit-low-confidence", ok: true, counts: { suggestions: 2 } }, + }, + ], + }; +} + +describe("Admin monitoring", () => { + beforeEach(() => { + cy.viewport(1400, 900); + cy.intercept("GET", "**/admin/auth/me", { statusCode: 200, body: adminBody }); + }); + + it("renders one card per service with its status and details", () => { + cy.intercept("GET", "**/admin/monitoring", { statusCode: 200, body: monitoringFixture() }).as( + "getMonitoring", + ); + cy.visit("/monitoring"); + cy.wait("@getMonitoring"); + + cy.get(".monitoring-card").should("have.length", 4); + + cy.contains(".monitoring-card", "Base de données") + .should("have.class", "monitoring-card--up") + .and("contain.text", "3.2 ms"); + cy.contains(".monitoring-card", "Service NLP (spaCy)") + .should("have.class", "monitoring-card--down") + .and("contain.text", "Hors service"); + cy.contains(".monitoring-card", "Worker LLM") + .should("have.class", "monitoring-card--degraded") + .and("contain.text", "audit-low-confidence"); + }); + + it("shows an error state when the request fails", () => { + cy.intercept("GET", "**/admin/monitoring", { + statusCode: 500, + body: { code: 5000, message: "x" }, + }); + cy.visit("/monitoring"); + cy.contains("Impossible de charger").should("be.visible"); + }); +}); diff --git a/apps/admin-web/src/api/client.ts b/apps/admin-web/src/api/client.ts index ed7fb21..288fc68 100644 --- a/apps/admin-web/src/api/client.ts +++ b/apps/admin-web/src/api/client.ts @@ -4,6 +4,7 @@ import { type ApiErrorResponse, ErrorCode, type MetricsView, + type MonitoringView, } from "@batch-cooking/shared"; /** @@ -96,6 +97,11 @@ export class AdminApiClient { public getMetrics(days: number): Promise { return this._request(`/admin/metrics?days=${days}`); } + + /** Live health of Postgres, the API, the intent-service and the LLM worker — polled by the monitoring board. */ + public getMonitoring(): Promise { + return this._request("/admin/monitoring"); + } } /** Single shared instance — stateless, same reasoning as apps/web's `apiClient`. */ diff --git a/apps/admin-web/src/locales/fr/translation.json b/apps/admin-web/src/locales/fr/translation.json index 297aebf..11bc033 100644 --- a/apps/admin-web/src/locales/fr/translation.json +++ b/apps/admin-web/src/locales/fr/translation.json @@ -59,7 +59,25 @@ }, "monitoring": { "title": "Monitoring", - "lead": "Santé des microservices et de la base de données." + "lead": "Santé des microservices et de la base de données.", + "lastChecked": "Dernière vérification à {{time}}", + "latency": "Latence", + "detail": "Détail", + "lastRun": "Dernier job", + "never": "jamais", + "jobFailed": "échec", + "status": { + "up": "OK", + "degraded": "Dégradé", + "down": "Hors service", + "unknown": "Inconnu" + }, + "service": { + "postgres": "Base de données", + "api": "API", + "intent-service": "Service NLP (spaCy)", + "tech-step-llm-worker": "Worker LLM" + } }, "corrections": { "title": "Corrections", diff --git a/apps/admin-web/src/pages/monitoring/MonitoringPage.tsx b/apps/admin-web/src/pages/monitoring/MonitoringPage.tsx index f531842..302c93f 100644 --- a/apps/admin-web/src/pages/monitoring/MonitoringPage.tsx +++ b/apps/admin-web/src/pages/monitoring/MonitoringPage.tsx @@ -1,18 +1,110 @@ +import type { MonitoringView, ServiceHealthView } from "@batch-cooking/shared"; +import { useEffect, useState } from "react"; import { useTranslation } from "react-i18next"; +import { adminApiClient } from "../../api/client"; import "../admin-page.scss"; +import "./monitoring-page.scss"; +import { clockTime, POLL_INTERVAL_MS, statusModifier } from "./monitoring"; + +type MonitoringState = + | { status: "loading" } + | { status: "loaded"; data: MonitoringView } + | { status: "error" }; /** - * Microservice health board — active probes of Postgres, the API, the - * intent-service and the LLM worker's heartbeat, fed by - * `GET /admin/monitoring`. Placeholder until PR 4 (monitoring). + * Microservice health board. Fetches `GET /admin/monitoring` on mount and + * re-polls every {@link POLL_INTERVAL_MS} ms. One card per probed target + * (Postgres, API, intent-service, LLM worker), coloured by status. */ export function MonitoringPage() { const { t } = useTranslation(); + const [state, setState] = useState({ status: "loading" }); + + useEffect(() => { + let cancelled = false; + + function load() { + adminApiClient + .getMonitoring() + .then((data) => { + if (!cancelled) setState({ status: "loaded", data }); + }) + .catch(() => { + if (!cancelled) + setState((prev) => (prev.status === "loaded" ? prev : { status: "error" })); + }); + } + + load(); + const timer = setInterval(load, POLL_INTERVAL_MS); + return () => { + cancelled = true; + clearInterval(timer); + }; + }, []); + return (

{t("admin.monitoring.title")}

{t("admin.monitoring.lead")}

-

{t("admin.common.comingSoon")}

+ + {state.status === "loading" && ( +

{t("admin.common.loading")}

+ )} + {state.status === "error" && ( +

{t("admin.common.loadError")}

+ )} + {state.status === "loaded" && ( + <> +

+ {t("admin.monitoring.lastChecked", { time: clockTime(state.data.generatedAt) })} +

+
+ {state.data.services.map((service) => ( + + ))} +
+ + )}
); } + +function ServiceCard({ service }: { service: ServiceHealthView }) { + const { t } = useTranslation(); + return ( +
+
+
+
+ {service.latencyMs !== null && ( +
+
{t("admin.monitoring.latency")}
+
{service.latencyMs} ms
+
+ )} + {service.detail && ( +
+
{t("admin.monitoring.detail")}
+
{service.detail}
+
+ )} + {service.lastRunAt !== undefined && ( +
+
{t("admin.monitoring.lastRun")}
+
+ {service.lastRunAt ? clockTime(service.lastRunAt) : t("admin.monitoring.never")} + {service.lastResult?.job ? ` · ${service.lastResult.job}` : ""} + {service.lastResult?.ok === false ? ` · ${t("admin.monitoring.jobFailed")}` : ""} +
+
+ )} +
+
+ ); +} diff --git a/apps/admin-web/src/pages/monitoring/monitoring-page.scss b/apps/admin-web/src/pages/monitoring/monitoring-page.scss new file mode 100644 index 0000000..d9fe572 --- /dev/null +++ b/apps/admin-web/src/pages/monitoring/monitoring-page.scss @@ -0,0 +1,98 @@ +// ============================================================================= +// MonitoringPage — a grid of service health cards, one per probed target. +// Status drives a coloured left border + dot. +// ============================================================================= + +.monitoring-checked { + margin: 0 0 var(--space-md); + font-size: var(--font-size-sm); + color: var(--color-text-muted); +} + +.monitoring-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(18rem, 1fr)); + gap: var(--space-md); +} + +.monitoring-card { + padding: var(--space-md); + background: var(--color-surface); + border: 1px solid var(--color-border); + border-left: 4px solid var(--color-border); + border-radius: var(--radius-md); + + --status-color: var(--color-text-muted); + &--up { + --status-color: var(--color-success); + } + &--degraded { + --status-color: var(--color-warning); + } + &--down { + --status-color: var(--color-error); + } + &--unknown { + --status-color: var(--color-text-muted); + } + + border-left-color: var(--status-color); + + &__head { + display: flex; + align-items: center; + gap: var(--space-sm); + margin-bottom: var(--space-sm); + + h2 { + flex: 1; + min-width: 0; + font-size: var(--font-size-md); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + } + + &__dot { + flex-shrink: 0; + width: 0.6rem; + height: 0.6rem; + border-radius: 50%; + background: var(--status-color); + } + + &__status { + flex-shrink: 0; + font-size: var(--font-size-xs); + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--status-color); + } + + &__meta { + margin: 0; + display: flex; + flex-direction: column; + gap: var(--space-xs); + + div { + display: flex; + gap: var(--space-sm); + font-size: var(--font-size-sm); + } + + dt { + flex-shrink: 0; + color: var(--color-text-muted); + min-width: 5rem; + } + + dd { + margin: 0; + color: var(--color-text); + word-break: break-word; + } + } +} diff --git a/apps/admin-web/src/pages/monitoring/monitoring.ts b/apps/admin-web/src/pages/monitoring/monitoring.ts new file mode 100644 index 0000000..215bf8b --- /dev/null +++ b/apps/admin-web/src/pages/monitoring/monitoring.ts @@ -0,0 +1,19 @@ +import type { ServiceStatus } from "@batch-cooking/shared"; + +/** + * Pure helpers for `MonitoringPage` — kept out of the `.tsx` per repo + * convention. + */ + +/** CSS modifier suffix for a status pill (`monitoring-card--up`, etc.). */ +export function statusModifier(status: ServiceStatus): string { + return status; +} + +/** How often the board re-polls `GET /admin/monitoring`, in ms. */ +export const POLL_INTERVAL_MS = 15_000; + +/** `"2026-08-28T09:00:00.000Z"` → `"09:00:00"` (local time) for the "last checked" line. */ +export function clockTime(iso: string): string { + return new Date(iso).toLocaleTimeString("fr-FR"); +} diff --git a/apps/api/src/modules/admin/admin-monitoring.routes.ts b/apps/api/src/modules/admin/admin-monitoring.routes.ts new file mode 100644 index 0000000..62e6be3 --- /dev/null +++ b/apps/api/src/modules/admin/admin-monitoring.routes.ts @@ -0,0 +1,20 @@ +import { wrapAsyncHandler } from "@batch-cooking/express-tools"; +import { Router } from "express"; +import { requireAdmin } from "../../middlewares/require-admin.js"; +import { getMonitoring } from "./admin-monitoring.service.js"; + +/** Router mounted at `/admin/monitoring` (via `admin.routes.ts`) — behind {@link requireAdmin}. */ +export const adminMonitoringRouter = Router(); + +/** + * Actively probes Postgres, the API, `tech-step-intent-service` and the LLM + * worker's heartbeat, returning a {@link MonitoringView} status board (see + * {@link getMonitoring}). No params; the admin UI polls it on an interval. + */ +adminMonitoringRouter.get( + "/", + requireAdmin, + wrapAsyncHandler(async (_req, res) => { + res.status(200).json(await getMonitoring()); + }), +); diff --git a/apps/api/src/modules/admin/admin-monitoring.service.ts b/apps/api/src/modules/admin/admin-monitoring.service.ts new file mode 100644 index 0000000..db0ffd8 --- /dev/null +++ b/apps/api/src/modules/admin/admin-monitoring.service.ts @@ -0,0 +1,174 @@ +import type { MonitoringView, ServiceHealthView, ServiceStatus } from "@batch-cooking/shared"; +import { env } from "../../config/env.js"; +import { prisma } from "../../db/prisma.js"; + +/** How long each outbound probe (Postgres query, intent-service HTTP) is allowed to take before it counts as `down`. */ +const PROBE_TIMEOUT_MS = 2000; + +/** + * Heartbeat-age thresholds for the LLM worker. Its default cron is weekly + * (`TECH_STEP_WORKER_CRON`, `0 3 * * 0`), and it also pings on boot/tick — + * so no ping for **8 days** means it likely missed its last scheduled fire + * (`degraded`), and none for **3 weeks** means it's almost certainly not + * running at all (`down`). + */ +const WORKER_STALE_AFTER_MS = 8 * 24 * 60 * 60 * 1000; +const WORKER_DOWN_AFTER_MS = 21 * 24 * 60 * 60 * 1000; + +const WORKER_KEY = "tech-step-llm-worker"; + +function nowIso(): string { + return new Date().toISOString(); +} + +function roundMs(value: number): number { + return Math.round(value * 10) / 10; +} + +function errMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + +/** `12500` → `"il y a 12 s"`, `90000` → `"il y a 1 min"`, `172800000` → `"il y a 2 j"`. */ +function formatAgo(ms: number): string { + const s = Math.round(ms / 1000); + if (s < 60) return `il y a ${s} s`; + const m = Math.round(s / 60); + if (m < 60) return `il y a ${m} min`; + const h = Math.round(m / 60); + if (h < 48) return `il y a ${h} h`; + return `il y a ${Math.round(h / 24)} j`; +} + +/** `process.uptime()` seconds → `"3 h 12 min"` / `"5 min"` / `"42 s"`. */ +function formatUptime(seconds: number): string { + const s = Math.floor(seconds); + if (s < 60) return `${s} s`; + const m = Math.floor(s / 60); + if (m < 60) return `${m} min`; + const h = Math.floor(m / 60); + return `${h} h ${m % 60} min`; +} + +/** `prisma.$queryRaw\`SELECT 1\`` with a bounded timeout — the DB connectivity probe. */ +async function probePostgres(): Promise { + const start = performance.now(); + try { + // `$queryRaw` doesn't take an AbortSignal — bound it with a race instead. + await Promise.race([ + prisma.$queryRaw`SELECT 1`, + new Promise((_resolve, reject) => + setTimeout(() => reject(new Error("timeout")), PROBE_TIMEOUT_MS), + ), + ]); + return { + key: "postgres", + status: "up", + latencyMs: roundMs(performance.now() - start), + detail: null, + checkedAt: nowIso(), + }; + } catch (err) { + return { + key: "postgres", + status: "down", + latencyMs: null, + detail: errMessage(err), + checkedAt: nowIso(), + }; + } +} + +/** The API itself — trivially "up" (it's answering), reported with its process uptime/memory. */ +function probeApi(): ServiceHealthView { + const mem = process.memoryUsage(); + return { + key: "api", + status: "up", + latencyMs: 0, + detail: `uptime ${formatUptime(process.uptime())} · RSS ${Math.round(mem.rss / 1_000_000)} Mo`, + checkedAt: nowIso(), + }; +} + +/** `GET {INTENT_SERVICE_BASE_URL}/health` — no secret needed on that route (see the service's `routes/health.py`). */ +async function probeIntentService(): Promise { + const start = performance.now(); + try { + const res = await fetch(`${env.INTENT_SERVICE_BASE_URL}/health`, { + signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }); + const latencyMs = roundMs(performance.now() - start); + return { + key: "intent-service", + status: res.ok ? "up" : "degraded", + latencyMs, + detail: `HTTP ${res.status}`, + checkedAt: nowIso(), + }; + } catch (err) { + return { + key: "intent-service", + status: "down", + latencyMs: null, + detail: errMessage(err), + checkedAt: nowIso(), + }; + } +} + +/** Reads the LLM worker's stored `WorkerHeartbeat` (it has no HTTP surface to probe directly) and grades it by age + last job outcome. */ +async function probeWorker(): Promise { + const heartbeat = await prisma.workerHeartbeat.findUnique({ where: { workerKey: WORKER_KEY } }); + if (!heartbeat) { + return { + key: WORKER_KEY, + status: "unknown", + latencyMs: null, + detail: "aucun battement reçu", + checkedAt: nowIso(), + lastRunAt: null, + lastResult: null, + }; + } + + const ageMs = Date.now() - heartbeat.lastSeenAt.getTime(); + const lastResult = (heartbeat.lastResult ?? null) as ServiceHealthView["lastResult"]; + + let status: ServiceStatus = "up"; + if (ageMs > WORKER_DOWN_AFTER_MS) status = "down"; + else if (ageMs > WORKER_STALE_AFTER_MS || lastResult?.ok === false) status = "degraded"; + + return { + key: WORKER_KEY, + status, + latencyMs: null, + detail: `dernier battement ${formatAgo(ageMs)}`, + checkedAt: nowIso(), + lastRunAt: heartbeat.lastRunAt?.toISOString() ?? null, + lastResult, + }; +} + +/** + * Actively probes every dependency the admin monitoring board watches — + * Postgres, the API itself, `tech-step-intent-service` (`/health`), and the + * LLM worker (via its stored heartbeat). Each probe is independent and + * bounded ({@link PROBE_TIMEOUT_MS}); one being `down` never fails the + * others or the endpoint. + */ +export async function getMonitoring(): Promise { + try { + const [postgres, intentService, worker] = await Promise.all([ + probePostgres(), + probeIntentService(), + probeWorker(), + ]); + return { + generatedAt: nowIso(), + services: [postgres, probeApi(), intentService, worker], + }; + } catch (err) { + throw err; // see recipe.service.ts's equivalent catch comment + } +} diff --git a/apps/api/src/modules/admin/admin.routes.ts b/apps/api/src/modules/admin/admin.routes.ts index 75ffe8e..3d69e58 100644 --- a/apps/api/src/modules/admin/admin.routes.ts +++ b/apps/api/src/modules/admin/admin.routes.ts @@ -1,6 +1,7 @@ import { Router } from "express"; import { adminAuthRouter } from "./admin-auth.routes.js"; import { adminMetricsRouter } from "./admin-metrics.routes.js"; +import { adminMonitoringRouter } from "./admin-monitoring.routes.js"; /** * Aggregator for the admin application's API surface, mounted at `/admin` @@ -13,3 +14,4 @@ export const adminRouter = Router(); adminRouter.use("/auth", adminAuthRouter); adminRouter.use("/metrics", adminMetricsRouter); +adminRouter.use("/monitoring", adminMonitoringRouter); diff --git a/apps/api/src/modules/internal/tech-step-worker.routes.ts b/apps/api/src/modules/internal/tech-step-worker.routes.ts index 748cd9a..f309096 100644 --- a/apps/api/src/modules/internal/tech-step-worker.routes.ts +++ b/apps/api/src/modules/internal/tech-step-worker.routes.ts @@ -3,12 +3,14 @@ import { auditBatchQuerySchema, submitTrainingSuggestionsSchema, workerBatchQuerySchema, + workerHeartbeatSchema, } from "@batch-cooking/shared"; import { Router } from "express"; import { requireInternalWorker } from "../../middlewares/require-internal-worker.js"; import { getAuditBatch, getPendingCorrections, + recordWorkerHeartbeat, submitTrainingSuggestions, } from "./tech-step-worker.service.js"; @@ -47,3 +49,19 @@ techStepWorkerRouter.post( res.status(201).json(await submitTrainingSuggestions(input)); }), ); + +/** + * Liveness ping from the worker (which has no inbound HTTP surface of its + * own) — upserts its `WorkerHeartbeat` row so the admin monitoring board + * can show it as up / stale / down and surface its last job result. Sent + * on boot, on every scheduler tick, and after each job. + */ +techStepWorkerRouter.post( + "/heartbeat", + requireInternalWorker, + wrapAsyncHandler(async (req, res) => { + const input = workerHeartbeatSchema.parse(req.body); + await recordWorkerHeartbeat(input); + res.status(200).json({ ok: true }); + }), +); diff --git a/apps/api/src/modules/internal/tech-step-worker.service.ts b/apps/api/src/modules/internal/tech-step-worker.service.ts index 607c0f1..7b58541 100644 --- a/apps/api/src/modules/internal/tech-step-worker.service.ts +++ b/apps/api/src/modules/internal/tech-step-worker.service.ts @@ -4,7 +4,9 @@ import { type PendingTechStepCorrectionView, type SubmitTrainingSuggestionsInput, type TechStepAuditClauseView, + type WorkerHeartbeatInput, } from "@batch-cooking/shared"; +import type { Prisma } from "@prisma/client"; import { prisma } from "../../db/prisma.js"; import { CONFIDENCE_THRESHOLD, @@ -199,3 +201,44 @@ export async function submitTrainingSuggestions( throw err; // see recipe.service.ts's equivalent catch comment } } + +/** + * The one worker with a `WorkerHeartbeat` row today — a fixed key, not + * something the caller supplies (only one worker exists, and letting it + * name itself would just be a spoofing surface behind the same shared + * secret). + */ +const WORKER_KEY = "tech-step-llm-worker"; + +/** + * Upserts `services/tech-step-llm-worker`'s heartbeat row (see + * `POST /internal/tech-steps/heartbeat`). Every ping bumps `lastSeenAt`; a + * `"job"` ping also records `lastRunAt` + a small `lastResult` summary so + * the admin monitoring board can show what the worker last did and whether + * it worked. + */ +export async function recordWorkerHeartbeat(input: WorkerHeartbeatInput): Promise { + try { + const now = new Date(); + const jobResult: Prisma.InputJsonValue | undefined = + input.event === "job" + ? { job: input.job ?? null, ok: input.ok ?? null, counts: input.counts ?? {} } + : undefined; + + await prisma.workerHeartbeat.upsert({ + where: { workerKey: WORKER_KEY }, + create: { + workerKey: WORKER_KEY, + lastSeenAt: now, + lastRunAt: input.event === "job" ? now : null, + lastResult: jobResult, + }, + update: { + lastSeenAt: now, + ...(input.event === "job" ? { lastRunAt: now, lastResult: jobResult } : {}), + }, + }); + } catch (err) { + throw err; // see recipe.service.ts's equivalent catch comment + } +} diff --git a/apps/api/test/admin-monitoring.test.ts b/apps/api/test/admin-monitoring.test.ts new file mode 100644 index 0000000..fcc8e8c --- /dev/null +++ b/apps/api/test/admin-monitoring.test.ts @@ -0,0 +1,160 @@ +import { faker } from "@faker-js/faker"; +import { expect } from "chai"; +import request from "supertest"; +import { createApp } from "../src/app.js"; +import { env } from "../src/config/env.js"; +import { prisma } from "../src/db/prisma.js"; +import { hashAdminPassword } from "../src/modules/admin/admin-auth.service.js"; +import { resetDatabase } from "../test-support/reset-db.js"; + +const SECRET_HEADER = "X-Internal-Worker-Secret"; +const VALID_STATUSES = ["up", "degraded", "down", "unknown"]; + +async function seedAdmin(): Promise<{ email: string; password: string }> { + const email = faker.internet.email().toLowerCase(); + const password = faker.internet.password({ length: 16 }); + await prisma.adminUser.create({ + data: { email, name: faker.person.fullName(), passwordHash: await hashAdminPassword(password) }, + }); + return { email, password }; +} + +const adminSecretConfigured = env.ADMIN_JWT_SECRET !== undefined; +const workerSecretConfigured = env.INTERNAL_WORKER_SECRET !== undefined; + +describe("Admin monitoring", () => { + const app = createApp(); + + beforeEach(async () => { + await resetDatabase(); + }); + + after(async () => { + await prisma.$disconnect(); + }); + + describe("POST /internal/tech-steps/heartbeat", () => { + it("rejects a request with no worker secret with 401", async () => { + const res = await request(app).post("/internal/tech-steps/heartbeat").send({ event: "boot" }); + expect(res.status).to.equal(401); + }); + + it("rejects a malformed body with 400", async function () { + if (!workerSecretConfigured) { + // biome-ignore lint/suspicious/noExplicitAny: mocha's `this.skip()` isn't typed here. + (this as any).skip(); + return; + } + const res = await request(app) + .post("/internal/tech-steps/heartbeat") + .set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string) + .send({ event: "not-a-real-event" }); + expect(res.status).to.equal(400); + }); + + it("upserts the worker heartbeat, recording lastRunAt/lastResult for a job ping", async function () { + if (!workerSecretConfigured) { + // biome-ignore lint/suspicious/noExplicitAny: see above. + (this as any).skip(); + return; + } + const res = await request(app) + .post("/internal/tech-steps/heartbeat") + .set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string) + .send({ + event: "job", + job: "audit-low-confidence", + ok: true, + counts: { suggestions: 3 }, + }); + expect(res.status).to.equal(200); + expect(res.body).to.deep.equal({ ok: true }); + + const stored = await prisma.workerHeartbeat.findUniqueOrThrow({ + where: { workerKey: "tech-step-llm-worker" }, + }); + expect(stored.lastRunAt).to.not.equal(null); + expect(stored.lastResult).to.deep.equal({ + job: "audit-low-confidence", + ok: true, + counts: { suggestions: 3 }, + }); + }); + + it("leaves lastRunAt null for a boot/tick ping", async function () { + if (!workerSecretConfigured) { + // biome-ignore lint/suspicious/noExplicitAny: see above. + (this as any).skip(); + return; + } + await request(app) + .post("/internal/tech-steps/heartbeat") + .set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string) + .send({ event: "boot" }); + + const stored = await prisma.workerHeartbeat.findUniqueOrThrow({ + where: { workerKey: "tech-step-llm-worker" }, + }); + expect(stored.lastSeenAt).to.be.instanceOf(Date); + expect(stored.lastRunAt).to.equal(null); + }); + }); + + describe("GET /admin/monitoring", () => { + it("rejects a request with no admin session with 401", async () => { + const res = await request(app).get("/admin/monitoring"); + expect(res.status).to.equal(401); + }); + + it("returns a status board covering all four targets, never crashing on an unreachable probe", async function () { + if (!adminSecretConfigured) { + // biome-ignore lint/suspicious/noExplicitAny: see above. + (this as any).skip(); + return; + } + const { email, password } = await seedAdmin(); + const agent = request.agent(app); + await agent.post("/admin/auth/login").send({ email, password }); + + const res = await agent.get("/admin/monitoring"); + expect(res.status).to.equal(200); + + const keys = res.body.services.map((s: { key: string }) => s.key); + expect(keys).to.have.members(["postgres", "api", "intent-service", "tech-step-llm-worker"]); + for (const service of res.body.services) { + expect(VALID_STATUSES).to.include(service.status); + } + + const byKey = Object.fromEntries(res.body.services.map((s: { key: string }) => [s.key, s])); + // The DB is up during the test run, and the API is answering us. + expect(byKey.postgres.status).to.equal("up"); + expect(byKey.api.status).to.equal("up"); + // No heartbeat has ever been recorded (resetDatabase truncated it). + expect(byKey["tech-step-llm-worker"].status).to.equal("unknown"); + }); + + it("reports the worker as up once it has sent a recent heartbeat", async function () { + if (!adminSecretConfigured || !workerSecretConfigured) { + // biome-ignore lint/suspicious/noExplicitAny: see above. + (this as any).skip(); + return; + } + await request(app) + .post("/internal/tech-steps/heartbeat") + .set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string) + .send({ event: "job", job: "transform-corrections", ok: true, counts: { suggestions: 0 } }); + + const { email, password } = await seedAdmin(); + const agent = request.agent(app); + await agent.post("/admin/auth/login").send({ email, password }); + + const res = await agent.get("/admin/monitoring"); + const worker = res.body.services.find( + (s: { key: string }) => s.key === "tech-step-llm-worker", + ); + expect(worker.status).to.equal("up"); + expect(worker.lastResult.job).to.equal("transform-corrections"); + expect(worker.lastRunAt).to.be.a("string"); + }); + }); +}); diff --git a/packages/shared/src/schemas/admin.ts b/packages/shared/src/schemas/admin.ts index 2b7fb28..6086d13 100644 --- a/packages/shared/src/schemas/admin.ts +++ b/packages/shared/src/schemas/admin.ts @@ -28,3 +28,22 @@ export const getMetricsSchema = z.object({ }); /** Inferred TS type for {@link getMetricsSchema}'s validated output. */ export type GetMetricsInput = z.infer; + +/** + * Payload of `POST /internal/tech-steps/heartbeat` — `services/tech-step-llm-worker` + * (which has no inbound HTTP of its own) reporting that it's alive. Sent on + * `boot`, on every scheduler `tick`, and after each `job` (with that job's + * name, outcome and counts). The worker key is fixed server-side (only one + * worker exists), so it isn't in the payload. + */ +export const workerHeartbeatSchema = z.object({ + event: z.enum(["boot", "tick", "job"]), + /** The job that just ran — present only when `event === "job"`. */ + job: z.string().max(100).optional(), + /** Whether that job succeeded — present only when `event === "job"`. */ + ok: z.boolean().optional(), + /** Small `{ label: number }` summary of that job (e.g. `{ suggestions: 3 }`). */ + counts: z.record(z.string(), z.number()).optional(), +}); +/** Inferred TS type for {@link workerHeartbeatSchema}'s validated output. */ +export type WorkerHeartbeatInput = z.infer; diff --git a/packages/shared/src/types/admin.ts b/packages/shared/src/types/admin.ts index 364139e..886ce1e 100644 --- a/packages/shared/src/types/admin.ts +++ b/packages/shared/src/types/admin.ts @@ -85,3 +85,34 @@ export interface MetricsView { }; events: MetricsEventSeries[]; } + +/** + * Health of one thing the admin monitoring board watches: + * - `"up"` — reachable and healthy. + * - `"degraded"` — reachable but not fully OK (e.g. the worker's last heartbeat + * is old, or a job it ran failed). + * - `"down"` — unreachable / erroring. + * - `"unknown"` — never observed (e.g. the worker has never sent a heartbeat). + */ +export type ServiceStatus = "up" | "degraded" | "down" | "unknown"; + +/** One row of `GET /admin/monitoring` — a single probed target. */ +export interface ServiceHealthView { + /** Stable id — `"postgres"`, `"api"`, `"intent-service"`, `"tech-step-llm-worker"`. Resolved to a label client-side via `admin.monitoring.service.`. */ + key: string; + status: ServiceStatus; + /** Round-trip of the probe in ms, or `null` when there was nothing to time (the worker, read from a stored heartbeat). */ + latencyMs: number | null; + /** Short human-readable extra (`"HTTP 200"`, `"uptime 3h 12m"`, `"dernier battement il y a 9 j"`). */ + detail: string | null; + checkedAt: string; + /** Worker only — when it last actually ran a job, and that job's summary. */ + lastRunAt?: string | null; + lastResult?: { job?: string; ok?: boolean; counts?: Record } | null; +} + +/** Response of `GET /admin/monitoring` — an actively-probed status board. */ +export interface MonitoringView { + generatedAt: string; + services: ServiceHealthView[]; +} diff --git a/services/tech-step-llm-worker/src/api-client.ts b/services/tech-step-llm-worker/src/api-client.ts index 2512fa5..80106f3 100644 --- a/services/tech-step-llm-worker/src/api-client.ts +++ b/services/tech-step-llm-worker/src/api-client.ts @@ -94,6 +94,35 @@ export function getPendingCorrections(limit: number): Promise; +} + +/** + * Best-effort liveness ping to `apps/api` so the admin monitoring board can + * see this worker (which has no inbound HTTP surface of its own). Sent on + * boot, on every scheduler tick, and after each job. **Never throws** — a + * failed heartbeat must never break or abort a run; it's logged and + * swallowed here. + */ +export async function postHeartbeat(payload: HeartbeatPayload): Promise { + try { + await request("/internal/tech-steps/heartbeat", { + method: "POST", + body: JSON.stringify(payload), + }); + } catch (err) { + console.warn( + "[tech-step-llm-worker] heartbeat failed:", + err instanceof Error ? err.message : err, + ); + } +} + /** Submits a batch of suggestions — a no-op (resolves immediately) if `suggestions` is empty, so a job with nothing to report doesn't need its own guard at every call site. */ export function postTrainingSuggestions( suggestions: TrainingSuggestionInput[], diff --git a/services/tech-step-llm-worker/src/index.ts b/services/tech-step-llm-worker/src/index.ts index 873f992..0eda857 100644 --- a/services/tech-step-llm-worker/src/index.ts +++ b/services/tech-step-llm-worker/src/index.ts @@ -1,6 +1,12 @@ +import { postHeartbeat } from "./api-client.js"; import { env } from "./config.js"; import { runOnce, startScheduler } from "./scheduler.js"; +// Tell `apps/api` we're alive as early as possible — before either the +// one-shot run or the cron loop — so the admin monitoring board reflects a +// fresh deploy immediately, not only after the first scheduled fire. +await postHeartbeat({ event: "boot" }); + /** * Entrypoint — `RUN_ONCE=true` runs both jobs a single time and exits * (manual/CI-triggered invocation, `pnpm start`), otherwise starts the diff --git a/services/tech-step-llm-worker/src/scheduler.ts b/services/tech-step-llm-worker/src/scheduler.ts index 6540601..3e882e1 100644 --- a/services/tech-step-llm-worker/src/scheduler.ts +++ b/services/tech-step-llm-worker/src/scheduler.ts @@ -1,4 +1,5 @@ import cron from "node-cron"; +import { postHeartbeat } from "./api-client.js"; import { env } from "./config.js"; import { runAuditLowConfidenceJob } from "./jobs/audit-low-confidence.js"; import { runTransformCorrectionsJob } from "./jobs/transform-corrections.js"; @@ -27,11 +28,23 @@ export async function runOnce(): Promise { limit: env.TECH_STEP_WORKER_BATCH_LIMIT, }); console.info(`[tech-step-llm-worker] audit-low-confidence: ${auditCount} suggestion(s)`); + await postHeartbeat({ + event: "job", + job: "audit-low-confidence", + ok: true, + counts: { suggestions: auditCount }, + }); const correctionCount = await runTransformCorrectionsJob(llm, { limit: env.TECH_STEP_WORKER_BATCH_LIMIT, }); console.info(`[tech-step-llm-worker] transform-corrections: ${correctionCount} suggestion(s)`); + await postHeartbeat({ + event: "job", + job: "transform-corrections", + ok: true, + counts: { suggestions: correctionCount }, + }); } finally { await llm.dispose(); } @@ -49,8 +62,11 @@ export async function runOnce(): Promise { export function startScheduler(): void { console.info(`[tech-step-llm-worker] scheduling runs on "${env.TECH_STEP_WORKER_CRON}"`); cron.schedule(env.TECH_STEP_WORKER_CRON, () => { + // Prove liveness even for a fire that then fails inside `runOnce`. + void postHeartbeat({ event: "tick" }); runOnce().catch((err: unknown) => { console.error("[tech-step-llm-worker] scheduled run failed:", err); + void postHeartbeat({ event: "job", ok: false }); }); }); } diff --git a/specs/backend-architecture.md b/specs/backend-architecture.md index c19462a..1580db8 100644 --- a/specs/backend-architecture.md +++ b/specs/backend-architecture.md @@ -336,8 +336,22 @@ est un `String` libre (`"user.signup"`, `"recipe.imported"`, `"recipe.created"`, ligne, **sans migration**. `AnalyticsEvent.actorId` n'a **pas** de FK (un évènement est un fait historique qui survit au compte qu'il décrit). -*(La table `WorkerHeartbeat` est créée par la même migration mais n'est -câblée que par le monitoring — section à venir.)* +**Monitoring** (`admin-monitoring.service.ts`, `GET /admin/monitoring`, +`requireAdmin`) — sonde active, chaque cible bornée à ~2 s, latence mesurée : +Postgres (`SELECT 1`), l'API elle-même (uptime/RSS), `tech-step-intent-service` +(`GET /health`, sans secret), et le **worker LLM** via sa ligne +`WorkerHeartbeat`. `MonitoringView { services: ServiceHealthView[] }`, statut +`up | degraded | down | unknown` ; une sonde `down` ne fait jamais échouer les +autres ni l'endpoint. Le front (`MonitoringPage`) re-poll toutes les 15 s. + +Le worker (`services/tech-step-llm-worker`) n'a **aucune** surface HTTP +entrante — il devient observable via `POST /internal/tech-steps/heartbeat` +(`requireInternalWorker`, `tech-step-worker.service.ts`'s +`recordWorkerHeartbeat` → upsert `WorkerHeartbeat`, clé fixe +`"tech-step-llm-worker"`). Le worker l'appelle au boot, à chaque tick du +scheduler, et après chaque job (avec `job`/`ok`/`counts`) — best-effort, un +heartbeat en échec ne casse jamais un run. Seuils d'âge : > 8 j ⇒ `degraded`, +> 21 j ⇒ `down` (cron par défaut hebdomadaire). ---