feat(admin): monitoring des microservices + heartbeat du worker LLM #14

Closed
kyuno wants to merge 1 commit from feat/admin-monitoring into feat/admin-metrics
19 changed files with 858 additions and 7 deletions

3
.gitignore vendored
View file

@ -163,3 +163,6 @@ tmp-mockups/
apps/web/cypress/screenshots/
apps/web/cypress/videos/
apps/web/cypress/downloads/
apps/admin-web/cypress/screenshots/
apps/admin-web/cypress/videos/
apps/admin-web/cypress/downloads/

View file

@ -0,0 +1,83 @@
// Mocks the admin API via cy.intercept — no live backend.
const adminBody = {
id: 1,
email: "ops@example.com",
name: "Ops",
createdAt: "2026-08-01T00:00:00.000Z",
lastLoginAt: "2026-08-28T09:00:00.000Z",
};
function monitoringFixture() {
return {
generatedAt: "2026-08-28T09:15:00.000Z",
services: [
{
key: "postgres",
status: "up",
latencyMs: 3.2,
detail: null,
checkedAt: "2026-08-28T09:15:00.000Z",
},
{
key: "api",
status: "up",
latencyMs: 0,
detail: "uptime 3 h 12 min · RSS 120 Mo",
checkedAt: "2026-08-28T09:15:00.000Z",
},
{
key: "intent-service",
status: "down",
latencyMs: null,
detail: "fetch failed",
checkedAt: "2026-08-28T09:15:00.000Z",
},
{
key: "tech-step-llm-worker",
status: "degraded",
latencyMs: null,
detail: "dernier battement il y a 9 j",
checkedAt: "2026-08-28T09:15:00.000Z",
lastRunAt: "2026-08-19T03:00:00.000Z",
lastResult: { job: "audit-low-confidence", ok: true, counts: { suggestions: 2 } },
},
],
};
}
describe("Admin monitoring", () => {
beforeEach(() => {
cy.viewport(1400, 900);
cy.intercept("GET", "**/admin/auth/me", { statusCode: 200, body: adminBody });
});
it("renders one card per service with its status and details", () => {
cy.intercept("GET", "**/admin/monitoring", { statusCode: 200, body: monitoringFixture() }).as(
"getMonitoring",
);
cy.visit("/monitoring");
cy.wait("@getMonitoring");
cy.get(".monitoring-card").should("have.length", 4);
cy.contains(".monitoring-card", "Base de données")
.should("have.class", "monitoring-card--up")
.and("contain.text", "3.2 ms");
cy.contains(".monitoring-card", "Service NLP (spaCy)")
.should("have.class", "monitoring-card--down")
.and("contain.text", "Hors service");
cy.contains(".monitoring-card", "Worker LLM")
.should("have.class", "monitoring-card--degraded")
.and("contain.text", "audit-low-confidence");
});
it("shows an error state when the request fails", () => {
cy.intercept("GET", "**/admin/monitoring", {
statusCode: 500,
body: { code: 5000, message: "x" },
});
cy.visit("/monitoring");
cy.contains("Impossible de charger").should("be.visible");
});
});

View file

@ -4,6 +4,7 @@ import {
type ApiErrorResponse,
ErrorCode,
type MetricsView,
type MonitoringView,
} from "@batch-cooking/shared";
/**
@ -96,6 +97,11 @@ export class AdminApiClient {
public getMetrics(days: number): Promise<MetricsView> {
return this._request(`/admin/metrics?days=${days}`);
}
/** Live health of Postgres, the API, the intent-service and the LLM worker — polled by the monitoring board. */
public getMonitoring(): Promise<MonitoringView> {
return this._request("/admin/monitoring");
}
}
/** Single shared instance — stateless, same reasoning as apps/web's `apiClient`. */

View file

@ -59,7 +59,25 @@
},
"monitoring": {
"title": "Monitoring",
"lead": "Santé des microservices et de la base de données."
"lead": "Santé des microservices et de la base de données.",
"lastChecked": "Dernière vérification à {{time}}",
"latency": "Latence",
"detail": "Détail",
"lastRun": "Dernier job",
"never": "jamais",
"jobFailed": "échec",
"status": {
"up": "OK",
"degraded": "Dégradé",
"down": "Hors service",
"unknown": "Inconnu"
},
"service": {
"postgres": "Base de données",
"api": "API",
"intent-service": "Service NLP (spaCy)",
"tech-step-llm-worker": "Worker LLM"
}
},
"corrections": {
"title": "Corrections",

View file

@ -1,18 +1,110 @@
import type { MonitoringView, ServiceHealthView } from "@batch-cooking/shared";
import { useEffect, useState } from "react";
import { useTranslation } from "react-i18next";
import { adminApiClient } from "../../api/client";
import "../admin-page.scss";
import "./monitoring-page.scss";
import { clockTime, POLL_INTERVAL_MS, statusModifier } from "./monitoring";
type MonitoringState =
| { status: "loading" }
| { status: "loaded"; data: MonitoringView }
| { status: "error" };
/**
* Microservice health board active probes of Postgres, the API, the
* intent-service and the LLM worker's heartbeat, fed by
* `GET /admin/monitoring`. Placeholder until PR 4 (monitoring).
* Microservice health board. Fetches `GET /admin/monitoring` on mount and
* re-polls every {@link POLL_INTERVAL_MS} ms. One card per probed target
* (Postgres, API, intent-service, LLM worker), coloured by status.
*/
export function MonitoringPage() {
const { t } = useTranslation();
const [state, setState] = useState<MonitoringState>({ status: "loading" });
useEffect(() => {
let cancelled = false;
function load() {
adminApiClient
.getMonitoring()
.then((data) => {
if (!cancelled) setState({ status: "loaded", data });
})
.catch(() => {
if (!cancelled)
setState((prev) => (prev.status === "loaded" ? prev : { status: "error" }));
});
}
load();
const timer = setInterval(load, POLL_INTERVAL_MS);
return () => {
cancelled = true;
clearInterval(timer);
};
}, []);
return (
<div className="admin-page">
<h1 className="admin-page__title">{t("admin.monitoring.title")}</h1>
<p className="admin-page__lead">{t("admin.monitoring.lead")}</p>
<p className="admin-page__placeholder">{t("admin.common.comingSoon")}</p>
{state.status === "loading" && (
<p className="admin-page__placeholder">{t("admin.common.loading")}</p>
)}
{state.status === "error" && (
<p className="admin-page__placeholder">{t("admin.common.loadError")}</p>
)}
{state.status === "loaded" && (
<>
<p className="monitoring-checked">
{t("admin.monitoring.lastChecked", { time: clockTime(state.data.generatedAt) })}
</p>
<div className="monitoring-grid">
{state.data.services.map((service) => (
<ServiceCard key={service.key} service={service} />
))}
</div>
</>
)}
</div>
);
}
function ServiceCard({ service }: { service: ServiceHealthView }) {
const { t } = useTranslation();
return (
<article className={`monitoring-card monitoring-card--${statusModifier(service.status)}`}>
<header className="monitoring-card__head">
<span className="monitoring-card__dot" aria-hidden="true" />
<h2>{t(`admin.monitoring.service.${service.key}`, { defaultValue: service.key })}</h2>
<span className="monitoring-card__status">
{t(`admin.monitoring.status.${service.status}`)}
</span>
</header>
<dl className="monitoring-card__meta">
{service.latencyMs !== null && (
<div>
<dt>{t("admin.monitoring.latency")}</dt>
<dd>{service.latencyMs} ms</dd>
</div>
)}
{service.detail && (
<div>
<dt>{t("admin.monitoring.detail")}</dt>
<dd>{service.detail}</dd>
</div>
)}
{service.lastRunAt !== undefined && (
<div>
<dt>{t("admin.monitoring.lastRun")}</dt>
<dd>
{service.lastRunAt ? clockTime(service.lastRunAt) : t("admin.monitoring.never")}
{service.lastResult?.job ? ` · ${service.lastResult.job}` : ""}
{service.lastResult?.ok === false ? ` · ${t("admin.monitoring.jobFailed")}` : ""}
</dd>
</div>
)}
</dl>
</article>
);
}

View file

@ -0,0 +1,98 @@
// =============================================================================
// MonitoringPage a grid of service health cards, one per probed target.
// Status drives a coloured left border + dot.
// =============================================================================
.monitoring-checked {
margin: 0 0 var(--space-md);
font-size: var(--font-size-sm);
color: var(--color-text-muted);
}
.monitoring-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(18rem, 1fr));
gap: var(--space-md);
}
.monitoring-card {
padding: var(--space-md);
background: var(--color-surface);
border: 1px solid var(--color-border);
border-left: 4px solid var(--color-border);
border-radius: var(--radius-md);
--status-color: var(--color-text-muted);
&--up {
--status-color: var(--color-success);
}
&--degraded {
--status-color: var(--color-warning);
}
&--down {
--status-color: var(--color-error);
}
&--unknown {
--status-color: var(--color-text-muted);
}
border-left-color: var(--status-color);
&__head {
display: flex;
align-items: center;
gap: var(--space-sm);
margin-bottom: var(--space-sm);
h2 {
flex: 1;
min-width: 0;
font-size: var(--font-size-md);
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
}
&__dot {
flex-shrink: 0;
width: 0.6rem;
height: 0.6rem;
border-radius: 50%;
background: var(--status-color);
}
&__status {
flex-shrink: 0;
font-size: var(--font-size-xs);
font-weight: 700;
text-transform: uppercase;
letter-spacing: 0.04em;
color: var(--status-color);
}
&__meta {
margin: 0;
display: flex;
flex-direction: column;
gap: var(--space-xs);
div {
display: flex;
gap: var(--space-sm);
font-size: var(--font-size-sm);
}
dt {
flex-shrink: 0;
color: var(--color-text-muted);
min-width: 5rem;
}
dd {
margin: 0;
color: var(--color-text);
word-break: break-word;
}
}
}

View file

@ -0,0 +1,19 @@
import type { ServiceStatus } from "@batch-cooking/shared";
/**
* Pure helpers for `MonitoringPage` kept out of the `.tsx` per repo
* convention.
*/
/** CSS modifier suffix for a status pill (`monitoring-card--up`, etc.). */
export function statusModifier(status: ServiceStatus): string {
return status;
}
/** How often the board re-polls `GET /admin/monitoring`, in ms. */
export const POLL_INTERVAL_MS = 15_000;
/** `"2026-08-28T09:00:00.000Z"` → `"09:00:00"` (local time) for the "last checked" line. */
export function clockTime(iso: string): string {
return new Date(iso).toLocaleTimeString("fr-FR");
}

View file

@ -0,0 +1,20 @@
import { wrapAsyncHandler } from "@batch-cooking/express-tools";
import { Router } from "express";
import { requireAdmin } from "../../middlewares/require-admin.js";
import { getMonitoring } from "./admin-monitoring.service.js";
/** Router mounted at `/admin/monitoring` (via `admin.routes.ts`) — behind {@link requireAdmin}. */
export const adminMonitoringRouter = Router();
/**
* Actively probes Postgres, the API, `tech-step-intent-service` and the LLM
* worker's heartbeat, returning a {@link MonitoringView} status board (see
* {@link getMonitoring}). No params; the admin UI polls it on an interval.
*/
adminMonitoringRouter.get(
"/",
requireAdmin,
wrapAsyncHandler(async (_req, res) => {
res.status(200).json(await getMonitoring());
}),
);

View file

@ -0,0 +1,174 @@
import type { MonitoringView, ServiceHealthView, ServiceStatus } from "@batch-cooking/shared";
import { env } from "../../config/env.js";
import { prisma } from "../../db/prisma.js";
/** How long each outbound probe (Postgres query, intent-service HTTP) is allowed to take before it counts as `down`. */
const PROBE_TIMEOUT_MS = 2000;
/**
* Heartbeat-age thresholds for the LLM worker. Its default cron is weekly
* (`TECH_STEP_WORKER_CRON`, `0 3 * * 0`), and it also pings on boot/tick
* so no ping for **8 days** means it likely missed its last scheduled fire
* (`degraded`), and none for **3 weeks** means it's almost certainly not
* running at all (`down`).
*/
const WORKER_STALE_AFTER_MS = 8 * 24 * 60 * 60 * 1000;
const WORKER_DOWN_AFTER_MS = 21 * 24 * 60 * 60 * 1000;
const WORKER_KEY = "tech-step-llm-worker";
function nowIso(): string {
return new Date().toISOString();
}
function roundMs(value: number): number {
return Math.round(value * 10) / 10;
}
function errMessage(err: unknown): string {
return err instanceof Error ? err.message : String(err);
}
/** `12500` → `"il y a 12 s"`, `90000` → `"il y a 1 min"`, `172800000` → `"il y a 2 j"`. */
function formatAgo(ms: number): string {
const s = Math.round(ms / 1000);
if (s < 60) return `il y a ${s} s`;
const m = Math.round(s / 60);
if (m < 60) return `il y a ${m} min`;
const h = Math.round(m / 60);
if (h < 48) return `il y a ${h} h`;
return `il y a ${Math.round(h / 24)} j`;
}
/** `process.uptime()` seconds → `"3 h 12 min"` / `"5 min"` / `"42 s"`. */
function formatUptime(seconds: number): string {
const s = Math.floor(seconds);
if (s < 60) return `${s} s`;
const m = Math.floor(s / 60);
if (m < 60) return `${m} min`;
const h = Math.floor(m / 60);
return `${h} h ${m % 60} min`;
}
/** `prisma.$queryRaw\`SELECT 1\`` with a bounded timeout — the DB connectivity probe. */
async function probePostgres(): Promise<ServiceHealthView> {
const start = performance.now();
try {
// `$queryRaw` doesn't take an AbortSignal — bound it with a race instead.
await Promise.race([
prisma.$queryRaw`SELECT 1`,
new Promise((_resolve, reject) =>
setTimeout(() => reject(new Error("timeout")), PROBE_TIMEOUT_MS),
),
]);
return {
key: "postgres",
status: "up",
latencyMs: roundMs(performance.now() - start),
detail: null,
checkedAt: nowIso(),
};
} catch (err) {
return {
key: "postgres",
status: "down",
latencyMs: null,
detail: errMessage(err),
checkedAt: nowIso(),
};
}
}
/** The API itself — trivially "up" (it's answering), reported with its process uptime/memory. */
function probeApi(): ServiceHealthView {
const mem = process.memoryUsage();
return {
key: "api",
status: "up",
latencyMs: 0,
detail: `uptime ${formatUptime(process.uptime())} · RSS ${Math.round(mem.rss / 1_000_000)} Mo`,
checkedAt: nowIso(),
};
}
/** `GET {INTENT_SERVICE_BASE_URL}/health` — no secret needed on that route (see the service's `routes/health.py`). */
async function probeIntentService(): Promise<ServiceHealthView> {
const start = performance.now();
try {
const res = await fetch(`${env.INTENT_SERVICE_BASE_URL}/health`, {
signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
});
const latencyMs = roundMs(performance.now() - start);
return {
key: "intent-service",
status: res.ok ? "up" : "degraded",
latencyMs,
detail: `HTTP ${res.status}`,
checkedAt: nowIso(),
};
} catch (err) {
return {
key: "intent-service",
status: "down",
latencyMs: null,
detail: errMessage(err),
checkedAt: nowIso(),
};
}
}
/** Reads the LLM worker's stored `WorkerHeartbeat` (it has no HTTP surface to probe directly) and grades it by age + last job outcome. */
async function probeWorker(): Promise<ServiceHealthView> {
const heartbeat = await prisma.workerHeartbeat.findUnique({ where: { workerKey: WORKER_KEY } });
if (!heartbeat) {
return {
key: WORKER_KEY,
status: "unknown",
latencyMs: null,
detail: "aucun battement reçu",
checkedAt: nowIso(),
lastRunAt: null,
lastResult: null,
};
}
const ageMs = Date.now() - heartbeat.lastSeenAt.getTime();
const lastResult = (heartbeat.lastResult ?? null) as ServiceHealthView["lastResult"];
let status: ServiceStatus = "up";
if (ageMs > WORKER_DOWN_AFTER_MS) status = "down";
else if (ageMs > WORKER_STALE_AFTER_MS || lastResult?.ok === false) status = "degraded";
return {
key: WORKER_KEY,
status,
latencyMs: null,
detail: `dernier battement ${formatAgo(ageMs)}`,
checkedAt: nowIso(),
lastRunAt: heartbeat.lastRunAt?.toISOString() ?? null,
lastResult,
};
}
/**
* Actively probes every dependency the admin monitoring board watches
* Postgres, the API itself, `tech-step-intent-service` (`/health`), and the
* LLM worker (via its stored heartbeat). Each probe is independent and
* bounded ({@link PROBE_TIMEOUT_MS}); one being `down` never fails the
* others or the endpoint.
*/
export async function getMonitoring(): Promise<MonitoringView> {
try {
const [postgres, intentService, worker] = await Promise.all([
probePostgres(),
probeIntentService(),
probeWorker(),
]);
return {
generatedAt: nowIso(),
services: [postgres, probeApi(), intentService, worker],
};
} catch (err) {
throw err; // see recipe.service.ts's equivalent catch comment
}
}

View file

@ -1,6 +1,7 @@
import { Router } from "express";
import { adminAuthRouter } from "./admin-auth.routes.js";
import { adminMetricsRouter } from "./admin-metrics.routes.js";
import { adminMonitoringRouter } from "./admin-monitoring.routes.js";
/**
* Aggregator for the admin application's API surface, mounted at `/admin`
@ -13,3 +14,4 @@ export const adminRouter = Router();
adminRouter.use("/auth", adminAuthRouter);
adminRouter.use("/metrics", adminMetricsRouter);
adminRouter.use("/monitoring", adminMonitoringRouter);

View file

@ -3,12 +3,14 @@ import {
auditBatchQuerySchema,
submitTrainingSuggestionsSchema,
workerBatchQuerySchema,
workerHeartbeatSchema,
} from "@batch-cooking/shared";
import { Router } from "express";
import { requireInternalWorker } from "../../middlewares/require-internal-worker.js";
import {
getAuditBatch,
getPendingCorrections,
recordWorkerHeartbeat,
submitTrainingSuggestions,
} from "./tech-step-worker.service.js";
@ -47,3 +49,19 @@ techStepWorkerRouter.post(
res.status(201).json(await submitTrainingSuggestions(input));
}),
);
/**
* Liveness ping from the worker (which has no inbound HTTP surface of its
* own) upserts its `WorkerHeartbeat` row so the admin monitoring board
* can show it as up / stale / down and surface its last job result. Sent
* on boot, on every scheduler tick, and after each job.
*/
techStepWorkerRouter.post(
"/heartbeat",
requireInternalWorker,
wrapAsyncHandler(async (req, res) => {
const input = workerHeartbeatSchema.parse(req.body);
await recordWorkerHeartbeat(input);
res.status(200).json({ ok: true });
}),
);

View file

@ -4,7 +4,9 @@ import {
type PendingTechStepCorrectionView,
type SubmitTrainingSuggestionsInput,
type TechStepAuditClauseView,
type WorkerHeartbeatInput,
} from "@batch-cooking/shared";
import type { Prisma } from "@prisma/client";
import { prisma } from "../../db/prisma.js";
import {
CONFIDENCE_THRESHOLD,
@ -199,3 +201,44 @@ export async function submitTrainingSuggestions(
throw err; // see recipe.service.ts's equivalent catch comment
}
}
/**
* The one worker with a `WorkerHeartbeat` row today a fixed key, not
* something the caller supplies (only one worker exists, and letting it
* name itself would just be a spoofing surface behind the same shared
* secret).
*/
const WORKER_KEY = "tech-step-llm-worker";
/**
* Upserts `services/tech-step-llm-worker`'s heartbeat row (see
* `POST /internal/tech-steps/heartbeat`). Every ping bumps `lastSeenAt`; a
* `"job"` ping also records `lastRunAt` + a small `lastResult` summary so
* the admin monitoring board can show what the worker last did and whether
* it worked.
*/
export async function recordWorkerHeartbeat(input: WorkerHeartbeatInput): Promise<void> {
try {
const now = new Date();
const jobResult: Prisma.InputJsonValue | undefined =
input.event === "job"
? { job: input.job ?? null, ok: input.ok ?? null, counts: input.counts ?? {} }
: undefined;
await prisma.workerHeartbeat.upsert({
where: { workerKey: WORKER_KEY },
create: {
workerKey: WORKER_KEY,
lastSeenAt: now,
lastRunAt: input.event === "job" ? now : null,
lastResult: jobResult,
},
update: {
lastSeenAt: now,
...(input.event === "job" ? { lastRunAt: now, lastResult: jobResult } : {}),
},
});
} catch (err) {
throw err; // see recipe.service.ts's equivalent catch comment
}
}

View file

@ -0,0 +1,160 @@
import { faker } from "@faker-js/faker";
import { expect } from "chai";
import request from "supertest";
import { createApp } from "../src/app.js";
import { env } from "../src/config/env.js";
import { prisma } from "../src/db/prisma.js";
import { hashAdminPassword } from "../src/modules/admin/admin-auth.service.js";
import { resetDatabase } from "../test-support/reset-db.js";
const SECRET_HEADER = "X-Internal-Worker-Secret";
const VALID_STATUSES = ["up", "degraded", "down", "unknown"];
async function seedAdmin(): Promise<{ email: string; password: string }> {
const email = faker.internet.email().toLowerCase();
const password = faker.internet.password({ length: 16 });
await prisma.adminUser.create({
data: { email, name: faker.person.fullName(), passwordHash: await hashAdminPassword(password) },
});
return { email, password };
}
const adminSecretConfigured = env.ADMIN_JWT_SECRET !== undefined;
const workerSecretConfigured = env.INTERNAL_WORKER_SECRET !== undefined;
describe("Admin monitoring", () => {
const app = createApp();
beforeEach(async () => {
await resetDatabase();
});
after(async () => {
await prisma.$disconnect();
});
describe("POST /internal/tech-steps/heartbeat", () => {
it("rejects a request with no worker secret with 401", async () => {
const res = await request(app).post("/internal/tech-steps/heartbeat").send({ event: "boot" });
expect(res.status).to.equal(401);
});
it("rejects a malformed body with 400", async function () {
if (!workerSecretConfigured) {
// biome-ignore lint/suspicious/noExplicitAny: mocha's `this.skip()` isn't typed here.
(this as any).skip();
return;
}
const res = await request(app)
.post("/internal/tech-steps/heartbeat")
.set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string)
.send({ event: "not-a-real-event" });
expect(res.status).to.equal(400);
});
it("upserts the worker heartbeat, recording lastRunAt/lastResult for a job ping", async function () {
if (!workerSecretConfigured) {
// biome-ignore lint/suspicious/noExplicitAny: see above.
(this as any).skip();
return;
}
const res = await request(app)
.post("/internal/tech-steps/heartbeat")
.set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string)
.send({
event: "job",
job: "audit-low-confidence",
ok: true,
counts: { suggestions: 3 },
});
expect(res.status).to.equal(200);
expect(res.body).to.deep.equal({ ok: true });
const stored = await prisma.workerHeartbeat.findUniqueOrThrow({
where: { workerKey: "tech-step-llm-worker" },
});
expect(stored.lastRunAt).to.not.equal(null);
expect(stored.lastResult).to.deep.equal({
job: "audit-low-confidence",
ok: true,
counts: { suggestions: 3 },
});
});
it("leaves lastRunAt null for a boot/tick ping", async function () {
if (!workerSecretConfigured) {
// biome-ignore lint/suspicious/noExplicitAny: see above.
(this as any).skip();
return;
}
await request(app)
.post("/internal/tech-steps/heartbeat")
.set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string)
.send({ event: "boot" });
const stored = await prisma.workerHeartbeat.findUniqueOrThrow({
where: { workerKey: "tech-step-llm-worker" },
});
expect(stored.lastSeenAt).to.be.instanceOf(Date);
expect(stored.lastRunAt).to.equal(null);
});
});
describe("GET /admin/monitoring", () => {
it("rejects a request with no admin session with 401", async () => {
const res = await request(app).get("/admin/monitoring");
expect(res.status).to.equal(401);
});
it("returns a status board covering all four targets, never crashing on an unreachable probe", async function () {
if (!adminSecretConfigured) {
// biome-ignore lint/suspicious/noExplicitAny: see above.
(this as any).skip();
return;
}
const { email, password } = await seedAdmin();
const agent = request.agent(app);
await agent.post("/admin/auth/login").send({ email, password });
const res = await agent.get("/admin/monitoring");
expect(res.status).to.equal(200);
const keys = res.body.services.map((s: { key: string }) => s.key);
expect(keys).to.have.members(["postgres", "api", "intent-service", "tech-step-llm-worker"]);
for (const service of res.body.services) {
expect(VALID_STATUSES).to.include(service.status);
}
const byKey = Object.fromEntries(res.body.services.map((s: { key: string }) => [s.key, s]));
// The DB is up during the test run, and the API is answering us.
expect(byKey.postgres.status).to.equal("up");
expect(byKey.api.status).to.equal("up");
// No heartbeat has ever been recorded (resetDatabase truncated it).
expect(byKey["tech-step-llm-worker"].status).to.equal("unknown");
});
it("reports the worker as up once it has sent a recent heartbeat", async function () {
if (!adminSecretConfigured || !workerSecretConfigured) {
// biome-ignore lint/suspicious/noExplicitAny: see above.
(this as any).skip();
return;
}
await request(app)
.post("/internal/tech-steps/heartbeat")
.set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string)
.send({ event: "job", job: "transform-corrections", ok: true, counts: { suggestions: 0 } });
const { email, password } = await seedAdmin();
const agent = request.agent(app);
await agent.post("/admin/auth/login").send({ email, password });
const res = await agent.get("/admin/monitoring");
const worker = res.body.services.find(
(s: { key: string }) => s.key === "tech-step-llm-worker",
);
expect(worker.status).to.equal("up");
expect(worker.lastResult.job).to.equal("transform-corrections");
expect(worker.lastRunAt).to.be.a("string");
});
});
});

View file

@ -28,3 +28,22 @@ export const getMetricsSchema = z.object({
});
/** Inferred TS type for {@link getMetricsSchema}'s validated output. */
export type GetMetricsInput = z.infer<typeof getMetricsSchema>;
/**
* Payload of `POST /internal/tech-steps/heartbeat` `services/tech-step-llm-worker`
* (which has no inbound HTTP of its own) reporting that it's alive. Sent on
* `boot`, on every scheduler `tick`, and after each `job` (with that job's
* name, outcome and counts). The worker key is fixed server-side (only one
* worker exists), so it isn't in the payload.
*/
export const workerHeartbeatSchema = z.object({
event: z.enum(["boot", "tick", "job"]),
/** The job that just ran — present only when `event === "job"`. */
job: z.string().max(100).optional(),
/** Whether that job succeeded — present only when `event === "job"`. */
ok: z.boolean().optional(),
/** Small `{ label: number }` summary of that job (e.g. `{ suggestions: 3 }`). */
counts: z.record(z.string(), z.number()).optional(),
});
/** Inferred TS type for {@link workerHeartbeatSchema}'s validated output. */
export type WorkerHeartbeatInput = z.infer<typeof workerHeartbeatSchema>;

View file

@ -85,3 +85,34 @@ export interface MetricsView {
};
events: MetricsEventSeries[];
}
/**
* Health of one thing the admin monitoring board watches:
* - `"up"` reachable and healthy.
* - `"degraded"` reachable but not fully OK (e.g. the worker's last heartbeat
* is old, or a job it ran failed).
* - `"down"` unreachable / erroring.
* - `"unknown"` never observed (e.g. the worker has never sent a heartbeat).
*/
export type ServiceStatus = "up" | "degraded" | "down" | "unknown";
/** One row of `GET /admin/monitoring` — a single probed target. */
export interface ServiceHealthView {
/** Stable id — `"postgres"`, `"api"`, `"intent-service"`, `"tech-step-llm-worker"`. Resolved to a label client-side via `admin.monitoring.service.<key>`. */
key: string;
status: ServiceStatus;
/** Round-trip of the probe in ms, or `null` when there was nothing to time (the worker, read from a stored heartbeat). */
latencyMs: number | null;
/** Short human-readable extra (`"HTTP 200"`, `"uptime 3h 12m"`, `"dernier battement il y a 9 j"`). */
detail: string | null;
checkedAt: string;
/** Worker only — when it last actually ran a job, and that job's summary. */
lastRunAt?: string | null;
lastResult?: { job?: string; ok?: boolean; counts?: Record<string, number> } | null;
}
/** Response of `GET /admin/monitoring` — an actively-probed status board. */
export interface MonitoringView {
generatedAt: string;
services: ServiceHealthView[];
}

View file

@ -94,6 +94,35 @@ export function getPendingCorrections(limit: number): Promise<PendingCorrection[
return request(`/internal/tech-steps/pending-corrections?${params}`);
}
/** Payload of `POST /internal/tech-steps/heartbeat` — mirrors `workerHeartbeatSchema` (`packages/shared`), duplicated here for the same "outside the workspace" reason as {@link AuditClause}. */
export interface HeartbeatPayload {
event: "boot" | "tick" | "job";
job?: string;
ok?: boolean;
counts?: Record<string, number>;
}
/**
* Best-effort liveness ping to `apps/api` so the admin monitoring board can
* see this worker (which has no inbound HTTP surface of its own). Sent on
* boot, on every scheduler tick, and after each job. **Never throws** a
* failed heartbeat must never break or abort a run; it's logged and
* swallowed here.
*/
export async function postHeartbeat(payload: HeartbeatPayload): Promise<void> {
try {
await request("/internal/tech-steps/heartbeat", {
method: "POST",
body: JSON.stringify(payload),
});
} catch (err) {
console.warn(
"[tech-step-llm-worker] heartbeat failed:",
err instanceof Error ? err.message : err,
);
}
}
/** Submits a batch of suggestions — a no-op (resolves immediately) if `suggestions` is empty, so a job with nothing to report doesn't need its own guard at every call site. */
export function postTrainingSuggestions(
suggestions: TrainingSuggestionInput[],

View file

@ -1,6 +1,12 @@
import { postHeartbeat } from "./api-client.js";
import { env } from "./config.js";
import { runOnce, startScheduler } from "./scheduler.js";
// Tell `apps/api` we're alive as early as possible — before either the
// one-shot run or the cron loop — so the admin monitoring board reflects a
// fresh deploy immediately, not only after the first scheduled fire.
await postHeartbeat({ event: "boot" });
/**
* Entrypoint `RUN_ONCE=true` runs both jobs a single time and exits
* (manual/CI-triggered invocation, `pnpm start`), otherwise starts the

View file

@ -1,4 +1,5 @@
import cron from "node-cron";
import { postHeartbeat } from "./api-client.js";
import { env } from "./config.js";
import { runAuditLowConfidenceJob } from "./jobs/audit-low-confidence.js";
import { runTransformCorrectionsJob } from "./jobs/transform-corrections.js";
@ -27,11 +28,23 @@ export async function runOnce(): Promise<void> {
limit: env.TECH_STEP_WORKER_BATCH_LIMIT,
});
console.info(`[tech-step-llm-worker] audit-low-confidence: ${auditCount} suggestion(s)`);
await postHeartbeat({
event: "job",
job: "audit-low-confidence",
ok: true,
counts: { suggestions: auditCount },
});
const correctionCount = await runTransformCorrectionsJob(llm, {
limit: env.TECH_STEP_WORKER_BATCH_LIMIT,
});
console.info(`[tech-step-llm-worker] transform-corrections: ${correctionCount} suggestion(s)`);
await postHeartbeat({
event: "job",
job: "transform-corrections",
ok: true,
counts: { suggestions: correctionCount },
});
} finally {
await llm.dispose();
}
@ -49,8 +62,11 @@ export async function runOnce(): Promise<void> {
export function startScheduler(): void {
console.info(`[tech-step-llm-worker] scheduling runs on "${env.TECH_STEP_WORKER_CRON}"`);
cron.schedule(env.TECH_STEP_WORKER_CRON, () => {
// Prove liveness even for a fire that then fails inside `runOnce`.
void postHeartbeat({ event: "tick" });
runOnce().catch((err: unknown) => {
console.error("[tech-step-llm-worker] scheduled run failed:", err);
void postHeartbeat({ event: "job", ok: false });
});
});
}

View file

@ -336,8 +336,22 @@ est un `String` libre (`"user.signup"`, `"recipe.imported"`, `"recipe.created"`,
ligne, **sans migration**. `AnalyticsEvent.actorId` n'a **pas** de FK (un
évènement est un fait historique qui survit au compte qu'il décrit).
*(La table `WorkerHeartbeat` est créée par la même migration mais n'est
câblée que par le monitoring — section à venir.)*
**Monitoring** (`admin-monitoring.service.ts`, `GET /admin/monitoring`,
`requireAdmin`) — sonde active, chaque cible bornée à ~2 s, latence mesurée :
Postgres (`SELECT 1`), l'API elle-même (uptime/RSS), `tech-step-intent-service`
(`GET /health`, sans secret), et le **worker LLM** via sa ligne
`WorkerHeartbeat`. `MonitoringView { services: ServiceHealthView[] }`, statut
`up | degraded | down | unknown` ; une sonde `down` ne fait jamais échouer les
autres ni l'endpoint. Le front (`MonitoringPage`) re-poll toutes les 15 s.
Le worker (`services/tech-step-llm-worker`) n'a **aucune** surface HTTP
entrante — il devient observable via `POST /internal/tech-steps/heartbeat`
(`requireInternalWorker`, `tech-step-worker.service.ts`'s
`recordWorkerHeartbeat` → upsert `WorkerHeartbeat`, clé fixe
`"tech-step-llm-worker"`). Le worker l'appelle au boot, à chaque tick du
scheduler, et après chaque job (avec `job`/`ok`/`counts`) — best-effort, un
heartbeat en échec ne casse jamais un run. Seuils d'âge : > 8 j ⇒ `degraded`,
> 21 j ⇒ `down` (cron par défaut hebdomadaire).
---