feat(admin): monitoring des microservices + heartbeat du worker LLM
PR 4 du chantier admin. Board de sante temps reel des dependances.
- POST /internal/tech-steps/heartbeat (requireInternalWorker) ->
recordWorkerHeartbeat : upsert WorkerHeartbeat (cle fixe
"tech-step-llm-worker"), lastRunAt/lastResult pour un ping "job".
Schema workerHeartbeatSchema dans packages/shared.
- services/tech-step-llm-worker : api-client.postHeartbeat (best-effort,
ne throw jamais) appele au boot (index.ts), a chaque tick et apres
chaque job (scheduler.ts, avec job/ok/counts).
- admin-monitoring.service.ts + GET /admin/monitoring (requireAdmin) :
sonde active bornee (~2 s) de Postgres (SELECT 1), l'API (uptime/RSS),
tech-step-intent-service (/health), et le worker via son heartbeat.
Statut up/degraded/down/unknown ; une sonde down ne casse ni les autres
ni l'endpoint. Seuils worker : > 8 j degraded, > 21 j down.
- MonitoringView / ServiceHealthView dans packages/shared.
- Front : MonitoringPage (grille de cartes coloree par statut, re-poll
15 s), logique pure monitoring.ts, i18n admin.monitoring.*,
AdminApiClient.getMonitoring.
- Tests : Mocha admin-monitoring.test.ts (heartbeat 401/400/upsert
job+boot ; GET /admin/monitoring 401, board 4 cibles, worker unknown
sans heartbeat puis up apres) ; Cypress monitoring.cy.ts (2 verts).
Worker mocha : 6/6 toujours verts.
- specs/backend-architecture.md : section monitoring. .gitignore :
apps/admin-web/cypress/{screenshots,videos,downloads}.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
afe47e161c
commit
ba7218347f
19 changed files with 858 additions and 7 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -163,3 +163,6 @@ tmp-mockups/
|
||||||
apps/web/cypress/screenshots/
|
apps/web/cypress/screenshots/
|
||||||
apps/web/cypress/videos/
|
apps/web/cypress/videos/
|
||||||
apps/web/cypress/downloads/
|
apps/web/cypress/downloads/
|
||||||
|
apps/admin-web/cypress/screenshots/
|
||||||
|
apps/admin-web/cypress/videos/
|
||||||
|
apps/admin-web/cypress/downloads/
|
||||||
|
|
|
||||||
83
apps/admin-web/cypress/e2e/monitoring.cy.ts
Normal file
83
apps/admin-web/cypress/e2e/monitoring.cy.ts
Normal file
|
|
@ -0,0 +1,83 @@
|
||||||
|
// Mocks the admin API via cy.intercept — no live backend.
|
||||||
|
|
||||||
|
const adminBody = {
|
||||||
|
id: 1,
|
||||||
|
email: "ops@example.com",
|
||||||
|
name: "Ops",
|
||||||
|
createdAt: "2026-08-01T00:00:00.000Z",
|
||||||
|
lastLoginAt: "2026-08-28T09:00:00.000Z",
|
||||||
|
};
|
||||||
|
|
||||||
|
function monitoringFixture() {
|
||||||
|
return {
|
||||||
|
generatedAt: "2026-08-28T09:15:00.000Z",
|
||||||
|
services: [
|
||||||
|
{
|
||||||
|
key: "postgres",
|
||||||
|
status: "up",
|
||||||
|
latencyMs: 3.2,
|
||||||
|
detail: null,
|
||||||
|
checkedAt: "2026-08-28T09:15:00.000Z",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "api",
|
||||||
|
status: "up",
|
||||||
|
latencyMs: 0,
|
||||||
|
detail: "uptime 3 h 12 min · RSS 120 Mo",
|
||||||
|
checkedAt: "2026-08-28T09:15:00.000Z",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "intent-service",
|
||||||
|
status: "down",
|
||||||
|
latencyMs: null,
|
||||||
|
detail: "fetch failed",
|
||||||
|
checkedAt: "2026-08-28T09:15:00.000Z",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
key: "tech-step-llm-worker",
|
||||||
|
status: "degraded",
|
||||||
|
latencyMs: null,
|
||||||
|
detail: "dernier battement il y a 9 j",
|
||||||
|
checkedAt: "2026-08-28T09:15:00.000Z",
|
||||||
|
lastRunAt: "2026-08-19T03:00:00.000Z",
|
||||||
|
lastResult: { job: "audit-low-confidence", ok: true, counts: { suggestions: 2 } },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("Admin monitoring", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
cy.viewport(1400, 900);
|
||||||
|
cy.intercept("GET", "**/admin/auth/me", { statusCode: 200, body: adminBody });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("renders one card per service with its status and details", () => {
|
||||||
|
cy.intercept("GET", "**/admin/monitoring", { statusCode: 200, body: monitoringFixture() }).as(
|
||||||
|
"getMonitoring",
|
||||||
|
);
|
||||||
|
cy.visit("/monitoring");
|
||||||
|
cy.wait("@getMonitoring");
|
||||||
|
|
||||||
|
cy.get(".monitoring-card").should("have.length", 4);
|
||||||
|
|
||||||
|
cy.contains(".monitoring-card", "Base de données")
|
||||||
|
.should("have.class", "monitoring-card--up")
|
||||||
|
.and("contain.text", "3.2 ms");
|
||||||
|
cy.contains(".monitoring-card", "Service NLP (spaCy)")
|
||||||
|
.should("have.class", "monitoring-card--down")
|
||||||
|
.and("contain.text", "Hors service");
|
||||||
|
cy.contains(".monitoring-card", "Worker LLM")
|
||||||
|
.should("have.class", "monitoring-card--degraded")
|
||||||
|
.and("contain.text", "audit-low-confidence");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("shows an error state when the request fails", () => {
|
||||||
|
cy.intercept("GET", "**/admin/monitoring", {
|
||||||
|
statusCode: 500,
|
||||||
|
body: { code: 5000, message: "x" },
|
||||||
|
});
|
||||||
|
cy.visit("/monitoring");
|
||||||
|
cy.contains("Impossible de charger").should("be.visible");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -4,6 +4,7 @@ import {
|
||||||
type ApiErrorResponse,
|
type ApiErrorResponse,
|
||||||
ErrorCode,
|
ErrorCode,
|
||||||
type MetricsView,
|
type MetricsView,
|
||||||
|
type MonitoringView,
|
||||||
} from "@batch-cooking/shared";
|
} from "@batch-cooking/shared";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -96,6 +97,11 @@ export class AdminApiClient {
|
||||||
public getMetrics(days: number): Promise<MetricsView> {
|
public getMetrics(days: number): Promise<MetricsView> {
|
||||||
return this._request(`/admin/metrics?days=${days}`);
|
return this._request(`/admin/metrics?days=${days}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Live health of Postgres, the API, the intent-service and the LLM worker — polled by the monitoring board. */
|
||||||
|
public getMonitoring(): Promise<MonitoringView> {
|
||||||
|
return this._request("/admin/monitoring");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Single shared instance — stateless, same reasoning as apps/web's `apiClient`. */
|
/** Single shared instance — stateless, same reasoning as apps/web's `apiClient`. */
|
||||||
|
|
|
||||||
|
|
@ -59,7 +59,25 @@
|
||||||
},
|
},
|
||||||
"monitoring": {
|
"monitoring": {
|
||||||
"title": "Monitoring",
|
"title": "Monitoring",
|
||||||
"lead": "Santé des microservices et de la base de données."
|
"lead": "Santé des microservices et de la base de données.",
|
||||||
|
"lastChecked": "Dernière vérification à {{time}}",
|
||||||
|
"latency": "Latence",
|
||||||
|
"detail": "Détail",
|
||||||
|
"lastRun": "Dernier job",
|
||||||
|
"never": "jamais",
|
||||||
|
"jobFailed": "échec",
|
||||||
|
"status": {
|
||||||
|
"up": "OK",
|
||||||
|
"degraded": "Dégradé",
|
||||||
|
"down": "Hors service",
|
||||||
|
"unknown": "Inconnu"
|
||||||
|
},
|
||||||
|
"service": {
|
||||||
|
"postgres": "Base de données",
|
||||||
|
"api": "API",
|
||||||
|
"intent-service": "Service NLP (spaCy)",
|
||||||
|
"tech-step-llm-worker": "Worker LLM"
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"corrections": {
|
"corrections": {
|
||||||
"title": "Corrections",
|
"title": "Corrections",
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,110 @@
|
||||||
|
import type { MonitoringView, ServiceHealthView } from "@batch-cooking/shared";
|
||||||
|
import { useEffect, useState } from "react";
|
||||||
import { useTranslation } from "react-i18next";
|
import { useTranslation } from "react-i18next";
|
||||||
|
import { adminApiClient } from "../../api/client";
|
||||||
import "../admin-page.scss";
|
import "../admin-page.scss";
|
||||||
|
import "./monitoring-page.scss";
|
||||||
|
import { clockTime, POLL_INTERVAL_MS, statusModifier } from "./monitoring";
|
||||||
|
|
||||||
|
type MonitoringState =
|
||||||
|
| { status: "loading" }
|
||||||
|
| { status: "loaded"; data: MonitoringView }
|
||||||
|
| { status: "error" };
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Microservice health board — active probes of Postgres, the API, the
|
* Microservice health board. Fetches `GET /admin/monitoring` on mount and
|
||||||
* intent-service and the LLM worker's heartbeat, fed by
|
* re-polls every {@link POLL_INTERVAL_MS} ms. One card per probed target
|
||||||
* `GET /admin/monitoring`. Placeholder until PR 4 (monitoring).
|
* (Postgres, API, intent-service, LLM worker), coloured by status.
|
||||||
*/
|
*/
|
||||||
export function MonitoringPage() {
|
export function MonitoringPage() {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
|
const [state, setState] = useState<MonitoringState>({ status: "loading" });
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false;
|
||||||
|
|
||||||
|
function load() {
|
||||||
|
adminApiClient
|
||||||
|
.getMonitoring()
|
||||||
|
.then((data) => {
|
||||||
|
if (!cancelled) setState({ status: "loaded", data });
|
||||||
|
})
|
||||||
|
.catch(() => {
|
||||||
|
if (!cancelled)
|
||||||
|
setState((prev) => (prev.status === "loaded" ? prev : { status: "error" }));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
load();
|
||||||
|
const timer = setInterval(load, POLL_INTERVAL_MS);
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
clearInterval(timer);
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="admin-page">
|
<div className="admin-page">
|
||||||
<h1 className="admin-page__title">{t("admin.monitoring.title")}</h1>
|
<h1 className="admin-page__title">{t("admin.monitoring.title")}</h1>
|
||||||
<p className="admin-page__lead">{t("admin.monitoring.lead")}</p>
|
<p className="admin-page__lead">{t("admin.monitoring.lead")}</p>
|
||||||
<p className="admin-page__placeholder">{t("admin.common.comingSoon")}</p>
|
|
||||||
|
{state.status === "loading" && (
|
||||||
|
<p className="admin-page__placeholder">{t("admin.common.loading")}</p>
|
||||||
|
)}
|
||||||
|
{state.status === "error" && (
|
||||||
|
<p className="admin-page__placeholder">{t("admin.common.loadError")}</p>
|
||||||
|
)}
|
||||||
|
{state.status === "loaded" && (
|
||||||
|
<>
|
||||||
|
<p className="monitoring-checked">
|
||||||
|
{t("admin.monitoring.lastChecked", { time: clockTime(state.data.generatedAt) })}
|
||||||
|
</p>
|
||||||
|
<div className="monitoring-grid">
|
||||||
|
{state.data.services.map((service) => (
|
||||||
|
<ServiceCard key={service.key} service={service} />
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function ServiceCard({ service }: { service: ServiceHealthView }) {
|
||||||
|
const { t } = useTranslation();
|
||||||
|
return (
|
||||||
|
<article className={`monitoring-card monitoring-card--${statusModifier(service.status)}`}>
|
||||||
|
<header className="monitoring-card__head">
|
||||||
|
<span className="monitoring-card__dot" aria-hidden="true" />
|
||||||
|
<h2>{t(`admin.monitoring.service.${service.key}`, { defaultValue: service.key })}</h2>
|
||||||
|
<span className="monitoring-card__status">
|
||||||
|
{t(`admin.monitoring.status.${service.status}`)}
|
||||||
|
</span>
|
||||||
|
</header>
|
||||||
|
<dl className="monitoring-card__meta">
|
||||||
|
{service.latencyMs !== null && (
|
||||||
|
<div>
|
||||||
|
<dt>{t("admin.monitoring.latency")}</dt>
|
||||||
|
<dd>{service.latencyMs} ms</dd>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{service.detail && (
|
||||||
|
<div>
|
||||||
|
<dt>{t("admin.monitoring.detail")}</dt>
|
||||||
|
<dd>{service.detail}</dd>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{service.lastRunAt !== undefined && (
|
||||||
|
<div>
|
||||||
|
<dt>{t("admin.monitoring.lastRun")}</dt>
|
||||||
|
<dd>
|
||||||
|
{service.lastRunAt ? clockTime(service.lastRunAt) : t("admin.monitoring.never")}
|
||||||
|
{service.lastResult?.job ? ` · ${service.lastResult.job}` : ""}
|
||||||
|
{service.lastResult?.ok === false ? ` · ${t("admin.monitoring.jobFailed")}` : ""}
|
||||||
|
</dd>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</dl>
|
||||||
|
</article>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
|
||||||
98
apps/admin-web/src/pages/monitoring/monitoring-page.scss
Normal file
98
apps/admin-web/src/pages/monitoring/monitoring-page.scss
Normal file
|
|
@ -0,0 +1,98 @@
|
||||||
|
// =============================================================================
|
||||||
|
// MonitoringPage — a grid of service health cards, one per probed target.
|
||||||
|
// Status drives a coloured left border + dot.
|
||||||
|
// =============================================================================
|
||||||
|
|
||||||
|
.monitoring-checked {
|
||||||
|
margin: 0 0 var(--space-md);
|
||||||
|
font-size: var(--font-size-sm);
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
}
|
||||||
|
|
||||||
|
.monitoring-grid {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: repeat(auto-fill, minmax(18rem, 1fr));
|
||||||
|
gap: var(--space-md);
|
||||||
|
}
|
||||||
|
|
||||||
|
.monitoring-card {
|
||||||
|
padding: var(--space-md);
|
||||||
|
background: var(--color-surface);
|
||||||
|
border: 1px solid var(--color-border);
|
||||||
|
border-left: 4px solid var(--color-border);
|
||||||
|
border-radius: var(--radius-md);
|
||||||
|
|
||||||
|
--status-color: var(--color-text-muted);
|
||||||
|
&--up {
|
||||||
|
--status-color: var(--color-success);
|
||||||
|
}
|
||||||
|
&--degraded {
|
||||||
|
--status-color: var(--color-warning);
|
||||||
|
}
|
||||||
|
&--down {
|
||||||
|
--status-color: var(--color-error);
|
||||||
|
}
|
||||||
|
&--unknown {
|
||||||
|
--status-color: var(--color-text-muted);
|
||||||
|
}
|
||||||
|
|
||||||
|
border-left-color: var(--status-color);
|
||||||
|
|
||||||
|
&__head {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: var(--space-sm);
|
||||||
|
margin-bottom: var(--space-sm);
|
||||||
|
|
||||||
|
h2 {
|
||||||
|
flex: 1;
|
||||||
|
min-width: 0;
|
||||||
|
font-size: var(--font-size-md);
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
&__dot {
|
||||||
|
flex-shrink: 0;
|
||||||
|
width: 0.6rem;
|
||||||
|
height: 0.6rem;
|
||||||
|
border-radius: 50%;
|
||||||
|
background: var(--status-color);
|
||||||
|
}
|
||||||
|
|
||||||
|
&__status {
|
||||||
|
flex-shrink: 0;
|
||||||
|
font-size: var(--font-size-xs);
|
||||||
|
font-weight: 700;
|
||||||
|
text-transform: uppercase;
|
||||||
|
letter-spacing: 0.04em;
|
||||||
|
color: var(--status-color);
|
||||||
|
}
|
||||||
|
|
||||||
|
&__meta {
|
||||||
|
margin: 0;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: var(--space-xs);
|
||||||
|
|
||||||
|
div {
|
||||||
|
display: flex;
|
||||||
|
gap: var(--space-sm);
|
||||||
|
font-size: var(--font-size-sm);
|
||||||
|
}
|
||||||
|
|
||||||
|
dt {
|
||||||
|
flex-shrink: 0;
|
||||||
|
color: var(--color-text-muted);
|
||||||
|
min-width: 5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
dd {
|
||||||
|
margin: 0;
|
||||||
|
color: var(--color-text);
|
||||||
|
word-break: break-word;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
19
apps/admin-web/src/pages/monitoring/monitoring.ts
Normal file
19
apps/admin-web/src/pages/monitoring/monitoring.ts
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
import type { ServiceStatus } from "@batch-cooking/shared";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pure helpers for `MonitoringPage` — kept out of the `.tsx` per repo
|
||||||
|
* convention.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** CSS modifier suffix for a status pill (`monitoring-card--up`, etc.). */
|
||||||
|
export function statusModifier(status: ServiceStatus): string {
|
||||||
|
return status;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** How often the board re-polls `GET /admin/monitoring`, in ms. */
|
||||||
|
export const POLL_INTERVAL_MS = 15_000;
|
||||||
|
|
||||||
|
/** `"2026-08-28T09:00:00.000Z"` → `"09:00:00"` (local time) for the "last checked" line. */
|
||||||
|
export function clockTime(iso: string): string {
|
||||||
|
return new Date(iso).toLocaleTimeString("fr-FR");
|
||||||
|
}
|
||||||
20
apps/api/src/modules/admin/admin-monitoring.routes.ts
Normal file
20
apps/api/src/modules/admin/admin-monitoring.routes.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
||||||
|
import { wrapAsyncHandler } from "@batch-cooking/express-tools";
|
||||||
|
import { Router } from "express";
|
||||||
|
import { requireAdmin } from "../../middlewares/require-admin.js";
|
||||||
|
import { getMonitoring } from "./admin-monitoring.service.js";
|
||||||
|
|
||||||
|
/** Router mounted at `/admin/monitoring` (via `admin.routes.ts`) — behind {@link requireAdmin}. */
|
||||||
|
export const adminMonitoringRouter = Router();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Actively probes Postgres, the API, `tech-step-intent-service` and the LLM
|
||||||
|
* worker's heartbeat, returning a {@link MonitoringView} status board (see
|
||||||
|
* {@link getMonitoring}). No params; the admin UI polls it on an interval.
|
||||||
|
*/
|
||||||
|
adminMonitoringRouter.get(
|
||||||
|
"/",
|
||||||
|
requireAdmin,
|
||||||
|
wrapAsyncHandler(async (_req, res) => {
|
||||||
|
res.status(200).json(await getMonitoring());
|
||||||
|
}),
|
||||||
|
);
|
||||||
174
apps/api/src/modules/admin/admin-monitoring.service.ts
Normal file
174
apps/api/src/modules/admin/admin-monitoring.service.ts
Normal file
|
|
@ -0,0 +1,174 @@
|
||||||
|
import type { MonitoringView, ServiceHealthView, ServiceStatus } from "@batch-cooking/shared";
|
||||||
|
import { env } from "../../config/env.js";
|
||||||
|
import { prisma } from "../../db/prisma.js";
|
||||||
|
|
||||||
|
/** How long each outbound probe (Postgres query, intent-service HTTP) is allowed to take before it counts as `down`. */
|
||||||
|
const PROBE_TIMEOUT_MS = 2000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Heartbeat-age thresholds for the LLM worker. Its default cron is weekly
|
||||||
|
* (`TECH_STEP_WORKER_CRON`, `0 3 * * 0`), and it also pings on boot/tick —
|
||||||
|
* so no ping for **8 days** means it likely missed its last scheduled fire
|
||||||
|
* (`degraded`), and none for **3 weeks** means it's almost certainly not
|
||||||
|
* running at all (`down`).
|
||||||
|
*/
|
||||||
|
const WORKER_STALE_AFTER_MS = 8 * 24 * 60 * 60 * 1000;
|
||||||
|
const WORKER_DOWN_AFTER_MS = 21 * 24 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
const WORKER_KEY = "tech-step-llm-worker";
|
||||||
|
|
||||||
|
function nowIso(): string {
|
||||||
|
return new Date().toISOString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function roundMs(value: number): number {
|
||||||
|
return Math.round(value * 10) / 10;
|
||||||
|
}
|
||||||
|
|
||||||
|
function errMessage(err: unknown): string {
|
||||||
|
return err instanceof Error ? err.message : String(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `12500` → `"il y a 12 s"`, `90000` → `"il y a 1 min"`, `172800000` → `"il y a 2 j"`. */
|
||||||
|
function formatAgo(ms: number): string {
|
||||||
|
const s = Math.round(ms / 1000);
|
||||||
|
if (s < 60) return `il y a ${s} s`;
|
||||||
|
const m = Math.round(s / 60);
|
||||||
|
if (m < 60) return `il y a ${m} min`;
|
||||||
|
const h = Math.round(m / 60);
|
||||||
|
if (h < 48) return `il y a ${h} h`;
|
||||||
|
return `il y a ${Math.round(h / 24)} j`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `process.uptime()` seconds → `"3 h 12 min"` / `"5 min"` / `"42 s"`. */
|
||||||
|
function formatUptime(seconds: number): string {
|
||||||
|
const s = Math.floor(seconds);
|
||||||
|
if (s < 60) return `${s} s`;
|
||||||
|
const m = Math.floor(s / 60);
|
||||||
|
if (m < 60) return `${m} min`;
|
||||||
|
const h = Math.floor(m / 60);
|
||||||
|
return `${h} h ${m % 60} min`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `prisma.$queryRaw\`SELECT 1\`` with a bounded timeout — the DB connectivity probe. */
|
||||||
|
async function probePostgres(): Promise<ServiceHealthView> {
|
||||||
|
const start = performance.now();
|
||||||
|
try {
|
||||||
|
// `$queryRaw` doesn't take an AbortSignal — bound it with a race instead.
|
||||||
|
await Promise.race([
|
||||||
|
prisma.$queryRaw`SELECT 1`,
|
||||||
|
new Promise((_resolve, reject) =>
|
||||||
|
setTimeout(() => reject(new Error("timeout")), PROBE_TIMEOUT_MS),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
return {
|
||||||
|
key: "postgres",
|
||||||
|
status: "up",
|
||||||
|
latencyMs: roundMs(performance.now() - start),
|
||||||
|
detail: null,
|
||||||
|
checkedAt: nowIso(),
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
return {
|
||||||
|
key: "postgres",
|
||||||
|
status: "down",
|
||||||
|
latencyMs: null,
|
||||||
|
detail: errMessage(err),
|
||||||
|
checkedAt: nowIso(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The API itself — trivially "up" (it's answering), reported with its process uptime/memory. */
|
||||||
|
function probeApi(): ServiceHealthView {
|
||||||
|
const mem = process.memoryUsage();
|
||||||
|
return {
|
||||||
|
key: "api",
|
||||||
|
status: "up",
|
||||||
|
latencyMs: 0,
|
||||||
|
detail: `uptime ${formatUptime(process.uptime())} · RSS ${Math.round(mem.rss / 1_000_000)} Mo`,
|
||||||
|
checkedAt: nowIso(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `GET {INTENT_SERVICE_BASE_URL}/health` — no secret needed on that route (see the service's `routes/health.py`). */
|
||||||
|
async function probeIntentService(): Promise<ServiceHealthView> {
|
||||||
|
const start = performance.now();
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${env.INTENT_SERVICE_BASE_URL}/health`, {
|
||||||
|
signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
|
||||||
|
});
|
||||||
|
const latencyMs = roundMs(performance.now() - start);
|
||||||
|
return {
|
||||||
|
key: "intent-service",
|
||||||
|
status: res.ok ? "up" : "degraded",
|
||||||
|
latencyMs,
|
||||||
|
detail: `HTTP ${res.status}`,
|
||||||
|
checkedAt: nowIso(),
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
return {
|
||||||
|
key: "intent-service",
|
||||||
|
status: "down",
|
||||||
|
latencyMs: null,
|
||||||
|
detail: errMessage(err),
|
||||||
|
checkedAt: nowIso(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Reads the LLM worker's stored `WorkerHeartbeat` (it has no HTTP surface to probe directly) and grades it by age + last job outcome. */
|
||||||
|
async function probeWorker(): Promise<ServiceHealthView> {
|
||||||
|
const heartbeat = await prisma.workerHeartbeat.findUnique({ where: { workerKey: WORKER_KEY } });
|
||||||
|
if (!heartbeat) {
|
||||||
|
return {
|
||||||
|
key: WORKER_KEY,
|
||||||
|
status: "unknown",
|
||||||
|
latencyMs: null,
|
||||||
|
detail: "aucun battement reçu",
|
||||||
|
checkedAt: nowIso(),
|
||||||
|
lastRunAt: null,
|
||||||
|
lastResult: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const ageMs = Date.now() - heartbeat.lastSeenAt.getTime();
|
||||||
|
const lastResult = (heartbeat.lastResult ?? null) as ServiceHealthView["lastResult"];
|
||||||
|
|
||||||
|
let status: ServiceStatus = "up";
|
||||||
|
if (ageMs > WORKER_DOWN_AFTER_MS) status = "down";
|
||||||
|
else if (ageMs > WORKER_STALE_AFTER_MS || lastResult?.ok === false) status = "degraded";
|
||||||
|
|
||||||
|
return {
|
||||||
|
key: WORKER_KEY,
|
||||||
|
status,
|
||||||
|
latencyMs: null,
|
||||||
|
detail: `dernier battement ${formatAgo(ageMs)}`,
|
||||||
|
checkedAt: nowIso(),
|
||||||
|
lastRunAt: heartbeat.lastRunAt?.toISOString() ?? null,
|
||||||
|
lastResult,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Actively probes every dependency the admin monitoring board watches —
|
||||||
|
* Postgres, the API itself, `tech-step-intent-service` (`/health`), and the
|
||||||
|
* LLM worker (via its stored heartbeat). Each probe is independent and
|
||||||
|
* bounded ({@link PROBE_TIMEOUT_MS}); one being `down` never fails the
|
||||||
|
* others or the endpoint.
|
||||||
|
*/
|
||||||
|
export async function getMonitoring(): Promise<MonitoringView> {
|
||||||
|
try {
|
||||||
|
const [postgres, intentService, worker] = await Promise.all([
|
||||||
|
probePostgres(),
|
||||||
|
probeIntentService(),
|
||||||
|
probeWorker(),
|
||||||
|
]);
|
||||||
|
return {
|
||||||
|
generatedAt: nowIso(),
|
||||||
|
services: [postgres, probeApi(), intentService, worker],
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
throw err; // see recipe.service.ts's equivalent catch comment
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
import { Router } from "express";
|
import { Router } from "express";
|
||||||
import { adminAuthRouter } from "./admin-auth.routes.js";
|
import { adminAuthRouter } from "./admin-auth.routes.js";
|
||||||
import { adminMetricsRouter } from "./admin-metrics.routes.js";
|
import { adminMetricsRouter } from "./admin-metrics.routes.js";
|
||||||
|
import { adminMonitoringRouter } from "./admin-monitoring.routes.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Aggregator for the admin application's API surface, mounted at `/admin`
|
* Aggregator for the admin application's API surface, mounted at `/admin`
|
||||||
|
|
@ -13,3 +14,4 @@ export const adminRouter = Router();
|
||||||
|
|
||||||
adminRouter.use("/auth", adminAuthRouter);
|
adminRouter.use("/auth", adminAuthRouter);
|
||||||
adminRouter.use("/metrics", adminMetricsRouter);
|
adminRouter.use("/metrics", adminMetricsRouter);
|
||||||
|
adminRouter.use("/monitoring", adminMonitoringRouter);
|
||||||
|
|
|
||||||
|
|
@ -3,12 +3,14 @@ import {
|
||||||
auditBatchQuerySchema,
|
auditBatchQuerySchema,
|
||||||
submitTrainingSuggestionsSchema,
|
submitTrainingSuggestionsSchema,
|
||||||
workerBatchQuerySchema,
|
workerBatchQuerySchema,
|
||||||
|
workerHeartbeatSchema,
|
||||||
} from "@batch-cooking/shared";
|
} from "@batch-cooking/shared";
|
||||||
import { Router } from "express";
|
import { Router } from "express";
|
||||||
import { requireInternalWorker } from "../../middlewares/require-internal-worker.js";
|
import { requireInternalWorker } from "../../middlewares/require-internal-worker.js";
|
||||||
import {
|
import {
|
||||||
getAuditBatch,
|
getAuditBatch,
|
||||||
getPendingCorrections,
|
getPendingCorrections,
|
||||||
|
recordWorkerHeartbeat,
|
||||||
submitTrainingSuggestions,
|
submitTrainingSuggestions,
|
||||||
} from "./tech-step-worker.service.js";
|
} from "./tech-step-worker.service.js";
|
||||||
|
|
||||||
|
|
@ -47,3 +49,19 @@ techStepWorkerRouter.post(
|
||||||
res.status(201).json(await submitTrainingSuggestions(input));
|
res.status(201).json(await submitTrainingSuggestions(input));
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Liveness ping from the worker (which has no inbound HTTP surface of its
|
||||||
|
* own) — upserts its `WorkerHeartbeat` row so the admin monitoring board
|
||||||
|
* can show it as up / stale / down and surface its last job result. Sent
|
||||||
|
* on boot, on every scheduler tick, and after each job.
|
||||||
|
*/
|
||||||
|
techStepWorkerRouter.post(
|
||||||
|
"/heartbeat",
|
||||||
|
requireInternalWorker,
|
||||||
|
wrapAsyncHandler(async (req, res) => {
|
||||||
|
const input = workerHeartbeatSchema.parse(req.body);
|
||||||
|
await recordWorkerHeartbeat(input);
|
||||||
|
res.status(200).json({ ok: true });
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,9 @@ import {
|
||||||
type PendingTechStepCorrectionView,
|
type PendingTechStepCorrectionView,
|
||||||
type SubmitTrainingSuggestionsInput,
|
type SubmitTrainingSuggestionsInput,
|
||||||
type TechStepAuditClauseView,
|
type TechStepAuditClauseView,
|
||||||
|
type WorkerHeartbeatInput,
|
||||||
} from "@batch-cooking/shared";
|
} from "@batch-cooking/shared";
|
||||||
|
import type { Prisma } from "@prisma/client";
|
||||||
import { prisma } from "../../db/prisma.js";
|
import { prisma } from "../../db/prisma.js";
|
||||||
import {
|
import {
|
||||||
CONFIDENCE_THRESHOLD,
|
CONFIDENCE_THRESHOLD,
|
||||||
|
|
@ -199,3 +201,44 @@ export async function submitTrainingSuggestions(
|
||||||
throw err; // see recipe.service.ts's equivalent catch comment
|
throw err; // see recipe.service.ts's equivalent catch comment
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The one worker with a `WorkerHeartbeat` row today — a fixed key, not
|
||||||
|
* something the caller supplies (only one worker exists, and letting it
|
||||||
|
* name itself would just be a spoofing surface behind the same shared
|
||||||
|
* secret).
|
||||||
|
*/
|
||||||
|
const WORKER_KEY = "tech-step-llm-worker";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Upserts `services/tech-step-llm-worker`'s heartbeat row (see
|
||||||
|
* `POST /internal/tech-steps/heartbeat`). Every ping bumps `lastSeenAt`; a
|
||||||
|
* `"job"` ping also records `lastRunAt` + a small `lastResult` summary so
|
||||||
|
* the admin monitoring board can show what the worker last did and whether
|
||||||
|
* it worked.
|
||||||
|
*/
|
||||||
|
export async function recordWorkerHeartbeat(input: WorkerHeartbeatInput): Promise<void> {
|
||||||
|
try {
|
||||||
|
const now = new Date();
|
||||||
|
const jobResult: Prisma.InputJsonValue | undefined =
|
||||||
|
input.event === "job"
|
||||||
|
? { job: input.job ?? null, ok: input.ok ?? null, counts: input.counts ?? {} }
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
await prisma.workerHeartbeat.upsert({
|
||||||
|
where: { workerKey: WORKER_KEY },
|
||||||
|
create: {
|
||||||
|
workerKey: WORKER_KEY,
|
||||||
|
lastSeenAt: now,
|
||||||
|
lastRunAt: input.event === "job" ? now : null,
|
||||||
|
lastResult: jobResult,
|
||||||
|
},
|
||||||
|
update: {
|
||||||
|
lastSeenAt: now,
|
||||||
|
...(input.event === "job" ? { lastRunAt: now, lastResult: jobResult } : {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
throw err; // see recipe.service.ts's equivalent catch comment
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
160
apps/api/test/admin-monitoring.test.ts
Normal file
160
apps/api/test/admin-monitoring.test.ts
Normal file
|
|
@ -0,0 +1,160 @@
|
||||||
|
import { faker } from "@faker-js/faker";
|
||||||
|
import { expect } from "chai";
|
||||||
|
import request from "supertest";
|
||||||
|
import { createApp } from "../src/app.js";
|
||||||
|
import { env } from "../src/config/env.js";
|
||||||
|
import { prisma } from "../src/db/prisma.js";
|
||||||
|
import { hashAdminPassword } from "../src/modules/admin/admin-auth.service.js";
|
||||||
|
import { resetDatabase } from "../test-support/reset-db.js";
|
||||||
|
|
||||||
|
const SECRET_HEADER = "X-Internal-Worker-Secret";
|
||||||
|
const VALID_STATUSES = ["up", "degraded", "down", "unknown"];
|
||||||
|
|
||||||
|
async function seedAdmin(): Promise<{ email: string; password: string }> {
|
||||||
|
const email = faker.internet.email().toLowerCase();
|
||||||
|
const password = faker.internet.password({ length: 16 });
|
||||||
|
await prisma.adminUser.create({
|
||||||
|
data: { email, name: faker.person.fullName(), passwordHash: await hashAdminPassword(password) },
|
||||||
|
});
|
||||||
|
return { email, password };
|
||||||
|
}
|
||||||
|
|
||||||
|
const adminSecretConfigured = env.ADMIN_JWT_SECRET !== undefined;
|
||||||
|
const workerSecretConfigured = env.INTERNAL_WORKER_SECRET !== undefined;
|
||||||
|
|
||||||
|
describe("Admin monitoring", () => {
|
||||||
|
const app = createApp();
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
await resetDatabase();
|
||||||
|
});
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
await prisma.$disconnect();
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("POST /internal/tech-steps/heartbeat", () => {
|
||||||
|
it("rejects a request with no worker secret with 401", async () => {
|
||||||
|
const res = await request(app).post("/internal/tech-steps/heartbeat").send({ event: "boot" });
|
||||||
|
expect(res.status).to.equal(401);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects a malformed body with 400", async function () {
|
||||||
|
if (!workerSecretConfigured) {
|
||||||
|
// biome-ignore lint/suspicious/noExplicitAny: mocha's `this.skip()` isn't typed here.
|
||||||
|
(this as any).skip();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const res = await request(app)
|
||||||
|
.post("/internal/tech-steps/heartbeat")
|
||||||
|
.set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string)
|
||||||
|
.send({ event: "not-a-real-event" });
|
||||||
|
expect(res.status).to.equal(400);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("upserts the worker heartbeat, recording lastRunAt/lastResult for a job ping", async function () {
|
||||||
|
if (!workerSecretConfigured) {
|
||||||
|
// biome-ignore lint/suspicious/noExplicitAny: see above.
|
||||||
|
(this as any).skip();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const res = await request(app)
|
||||||
|
.post("/internal/tech-steps/heartbeat")
|
||||||
|
.set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string)
|
||||||
|
.send({
|
||||||
|
event: "job",
|
||||||
|
job: "audit-low-confidence",
|
||||||
|
ok: true,
|
||||||
|
counts: { suggestions: 3 },
|
||||||
|
});
|
||||||
|
expect(res.status).to.equal(200);
|
||||||
|
expect(res.body).to.deep.equal({ ok: true });
|
||||||
|
|
||||||
|
const stored = await prisma.workerHeartbeat.findUniqueOrThrow({
|
||||||
|
where: { workerKey: "tech-step-llm-worker" },
|
||||||
|
});
|
||||||
|
expect(stored.lastRunAt).to.not.equal(null);
|
||||||
|
expect(stored.lastResult).to.deep.equal({
|
||||||
|
job: "audit-low-confidence",
|
||||||
|
ok: true,
|
||||||
|
counts: { suggestions: 3 },
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("leaves lastRunAt null for a boot/tick ping", async function () {
|
||||||
|
if (!workerSecretConfigured) {
|
||||||
|
// biome-ignore lint/suspicious/noExplicitAny: see above.
|
||||||
|
(this as any).skip();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await request(app)
|
||||||
|
.post("/internal/tech-steps/heartbeat")
|
||||||
|
.set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string)
|
||||||
|
.send({ event: "boot" });
|
||||||
|
|
||||||
|
const stored = await prisma.workerHeartbeat.findUniqueOrThrow({
|
||||||
|
where: { workerKey: "tech-step-llm-worker" },
|
||||||
|
});
|
||||||
|
expect(stored.lastSeenAt).to.be.instanceOf(Date);
|
||||||
|
expect(stored.lastRunAt).to.equal(null);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("GET /admin/monitoring", () => {
|
||||||
|
it("rejects a request with no admin session with 401", async () => {
|
||||||
|
const res = await request(app).get("/admin/monitoring");
|
||||||
|
expect(res.status).to.equal(401);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns a status board covering all four targets, never crashing on an unreachable probe", async function () {
|
||||||
|
if (!adminSecretConfigured) {
|
||||||
|
// biome-ignore lint/suspicious/noExplicitAny: see above.
|
||||||
|
(this as any).skip();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const { email, password } = await seedAdmin();
|
||||||
|
const agent = request.agent(app);
|
||||||
|
await agent.post("/admin/auth/login").send({ email, password });
|
||||||
|
|
||||||
|
const res = await agent.get("/admin/monitoring");
|
||||||
|
expect(res.status).to.equal(200);
|
||||||
|
|
||||||
|
const keys = res.body.services.map((s: { key: string }) => s.key);
|
||||||
|
expect(keys).to.have.members(["postgres", "api", "intent-service", "tech-step-llm-worker"]);
|
||||||
|
for (const service of res.body.services) {
|
||||||
|
expect(VALID_STATUSES).to.include(service.status);
|
||||||
|
}
|
||||||
|
|
||||||
|
const byKey = Object.fromEntries(res.body.services.map((s: { key: string }) => [s.key, s]));
|
||||||
|
// The DB is up during the test run, and the API is answering us.
|
||||||
|
expect(byKey.postgres.status).to.equal("up");
|
||||||
|
expect(byKey.api.status).to.equal("up");
|
||||||
|
// No heartbeat has ever been recorded (resetDatabase truncated it).
|
||||||
|
expect(byKey["tech-step-llm-worker"].status).to.equal("unknown");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports the worker as up once it has sent a recent heartbeat", async function () {
|
||||||
|
if (!adminSecretConfigured || !workerSecretConfigured) {
|
||||||
|
// biome-ignore lint/suspicious/noExplicitAny: see above.
|
||||||
|
(this as any).skip();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await request(app)
|
||||||
|
.post("/internal/tech-steps/heartbeat")
|
||||||
|
.set(SECRET_HEADER, env.INTERNAL_WORKER_SECRET as string)
|
||||||
|
.send({ event: "job", job: "transform-corrections", ok: true, counts: { suggestions: 0 } });
|
||||||
|
|
||||||
|
const { email, password } = await seedAdmin();
|
||||||
|
const agent = request.agent(app);
|
||||||
|
await agent.post("/admin/auth/login").send({ email, password });
|
||||||
|
|
||||||
|
const res = await agent.get("/admin/monitoring");
|
||||||
|
const worker = res.body.services.find(
|
||||||
|
(s: { key: string }) => s.key === "tech-step-llm-worker",
|
||||||
|
);
|
||||||
|
expect(worker.status).to.equal("up");
|
||||||
|
expect(worker.lastResult.job).to.equal("transform-corrections");
|
||||||
|
expect(worker.lastRunAt).to.be.a("string");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
@ -28,3 +28,22 @@ export const getMetricsSchema = z.object({
|
||||||
});
|
});
|
||||||
/** Inferred TS type for {@link getMetricsSchema}'s validated output. */
|
/** Inferred TS type for {@link getMetricsSchema}'s validated output. */
|
||||||
export type GetMetricsInput = z.infer<typeof getMetricsSchema>;
|
export type GetMetricsInput = z.infer<typeof getMetricsSchema>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Payload of `POST /internal/tech-steps/heartbeat` — `services/tech-step-llm-worker`
|
||||||
|
* (which has no inbound HTTP of its own) reporting that it's alive. Sent on
|
||||||
|
* `boot`, on every scheduler `tick`, and after each `job` (with that job's
|
||||||
|
* name, outcome and counts). The worker key is fixed server-side (only one
|
||||||
|
* worker exists), so it isn't in the payload.
|
||||||
|
*/
|
||||||
|
export const workerHeartbeatSchema = z.object({
|
||||||
|
event: z.enum(["boot", "tick", "job"]),
|
||||||
|
/** The job that just ran — present only when `event === "job"`. */
|
||||||
|
job: z.string().max(100).optional(),
|
||||||
|
/** Whether that job succeeded — present only when `event === "job"`. */
|
||||||
|
ok: z.boolean().optional(),
|
||||||
|
/** Small `{ label: number }` summary of that job (e.g. `{ suggestions: 3 }`). */
|
||||||
|
counts: z.record(z.string(), z.number()).optional(),
|
||||||
|
});
|
||||||
|
/** Inferred TS type for {@link workerHeartbeatSchema}'s validated output. */
|
||||||
|
export type WorkerHeartbeatInput = z.infer<typeof workerHeartbeatSchema>;
|
||||||
|
|
|
||||||
|
|
@ -85,3 +85,34 @@ export interface MetricsView {
|
||||||
};
|
};
|
||||||
events: MetricsEventSeries[];
|
events: MetricsEventSeries[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Health of one thing the admin monitoring board watches:
|
||||||
|
* - `"up"` — reachable and healthy.
|
||||||
|
* - `"degraded"` — reachable but not fully OK (e.g. the worker's last heartbeat
|
||||||
|
* is old, or a job it ran failed).
|
||||||
|
* - `"down"` — unreachable / erroring.
|
||||||
|
* - `"unknown"` — never observed (e.g. the worker has never sent a heartbeat).
|
||||||
|
*/
|
||||||
|
export type ServiceStatus = "up" | "degraded" | "down" | "unknown";
|
||||||
|
|
||||||
|
/** One row of `GET /admin/monitoring` — a single probed target. */
|
||||||
|
export interface ServiceHealthView {
|
||||||
|
/** Stable id — `"postgres"`, `"api"`, `"intent-service"`, `"tech-step-llm-worker"`. Resolved to a label client-side via `admin.monitoring.service.<key>`. */
|
||||||
|
key: string;
|
||||||
|
status: ServiceStatus;
|
||||||
|
/** Round-trip of the probe in ms, or `null` when there was nothing to time (the worker, read from a stored heartbeat). */
|
||||||
|
latencyMs: number | null;
|
||||||
|
/** Short human-readable extra (`"HTTP 200"`, `"uptime 3h 12m"`, `"dernier battement il y a 9 j"`). */
|
||||||
|
detail: string | null;
|
||||||
|
checkedAt: string;
|
||||||
|
/** Worker only — when it last actually ran a job, and that job's summary. */
|
||||||
|
lastRunAt?: string | null;
|
||||||
|
lastResult?: { job?: string; ok?: boolean; counts?: Record<string, number> } | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Response of `GET /admin/monitoring` — an actively-probed status board. */
|
||||||
|
export interface MonitoringView {
|
||||||
|
generatedAt: string;
|
||||||
|
services: ServiceHealthView[];
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -94,6 +94,35 @@ export function getPendingCorrections(limit: number): Promise<PendingCorrection[
|
||||||
return request(`/internal/tech-steps/pending-corrections?${params}`);
|
return request(`/internal/tech-steps/pending-corrections?${params}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Payload of `POST /internal/tech-steps/heartbeat` — mirrors `workerHeartbeatSchema` (`packages/shared`), duplicated here for the same "outside the workspace" reason as {@link AuditClause}. */
|
||||||
|
export interface HeartbeatPayload {
|
||||||
|
event: "boot" | "tick" | "job";
|
||||||
|
job?: string;
|
||||||
|
ok?: boolean;
|
||||||
|
counts?: Record<string, number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Best-effort liveness ping to `apps/api` so the admin monitoring board can
|
||||||
|
* see this worker (which has no inbound HTTP surface of its own). Sent on
|
||||||
|
* boot, on every scheduler tick, and after each job. **Never throws** — a
|
||||||
|
* failed heartbeat must never break or abort a run; it's logged and
|
||||||
|
* swallowed here.
|
||||||
|
*/
|
||||||
|
export async function postHeartbeat(payload: HeartbeatPayload): Promise<void> {
|
||||||
|
try {
|
||||||
|
await request("/internal/tech-steps/heartbeat", {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(
|
||||||
|
"[tech-step-llm-worker] heartbeat failed:",
|
||||||
|
err instanceof Error ? err.message : err,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Submits a batch of suggestions — a no-op (resolves immediately) if `suggestions` is empty, so a job with nothing to report doesn't need its own guard at every call site. */
|
/** Submits a batch of suggestions — a no-op (resolves immediately) if `suggestions` is empty, so a job with nothing to report doesn't need its own guard at every call site. */
|
||||||
export function postTrainingSuggestions(
|
export function postTrainingSuggestions(
|
||||||
suggestions: TrainingSuggestionInput[],
|
suggestions: TrainingSuggestionInput[],
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,12 @@
|
||||||
|
import { postHeartbeat } from "./api-client.js";
|
||||||
import { env } from "./config.js";
|
import { env } from "./config.js";
|
||||||
import { runOnce, startScheduler } from "./scheduler.js";
|
import { runOnce, startScheduler } from "./scheduler.js";
|
||||||
|
|
||||||
|
// Tell `apps/api` we're alive as early as possible — before either the
|
||||||
|
// one-shot run or the cron loop — so the admin monitoring board reflects a
|
||||||
|
// fresh deploy immediately, not only after the first scheduled fire.
|
||||||
|
await postHeartbeat({ event: "boot" });
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Entrypoint — `RUN_ONCE=true` runs both jobs a single time and exits
|
* Entrypoint — `RUN_ONCE=true` runs both jobs a single time and exits
|
||||||
* (manual/CI-triggered invocation, `pnpm start`), otherwise starts the
|
* (manual/CI-triggered invocation, `pnpm start`), otherwise starts the
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import cron from "node-cron";
|
import cron from "node-cron";
|
||||||
|
import { postHeartbeat } from "./api-client.js";
|
||||||
import { env } from "./config.js";
|
import { env } from "./config.js";
|
||||||
import { runAuditLowConfidenceJob } from "./jobs/audit-low-confidence.js";
|
import { runAuditLowConfidenceJob } from "./jobs/audit-low-confidence.js";
|
||||||
import { runTransformCorrectionsJob } from "./jobs/transform-corrections.js";
|
import { runTransformCorrectionsJob } from "./jobs/transform-corrections.js";
|
||||||
|
|
@ -27,11 +28,23 @@ export async function runOnce(): Promise<void> {
|
||||||
limit: env.TECH_STEP_WORKER_BATCH_LIMIT,
|
limit: env.TECH_STEP_WORKER_BATCH_LIMIT,
|
||||||
});
|
});
|
||||||
console.info(`[tech-step-llm-worker] audit-low-confidence: ${auditCount} suggestion(s)`);
|
console.info(`[tech-step-llm-worker] audit-low-confidence: ${auditCount} suggestion(s)`);
|
||||||
|
await postHeartbeat({
|
||||||
|
event: "job",
|
||||||
|
job: "audit-low-confidence",
|
||||||
|
ok: true,
|
||||||
|
counts: { suggestions: auditCount },
|
||||||
|
});
|
||||||
|
|
||||||
const correctionCount = await runTransformCorrectionsJob(llm, {
|
const correctionCount = await runTransformCorrectionsJob(llm, {
|
||||||
limit: env.TECH_STEP_WORKER_BATCH_LIMIT,
|
limit: env.TECH_STEP_WORKER_BATCH_LIMIT,
|
||||||
});
|
});
|
||||||
console.info(`[tech-step-llm-worker] transform-corrections: ${correctionCount} suggestion(s)`);
|
console.info(`[tech-step-llm-worker] transform-corrections: ${correctionCount} suggestion(s)`);
|
||||||
|
await postHeartbeat({
|
||||||
|
event: "job",
|
||||||
|
job: "transform-corrections",
|
||||||
|
ok: true,
|
||||||
|
counts: { suggestions: correctionCount },
|
||||||
|
});
|
||||||
} finally {
|
} finally {
|
||||||
await llm.dispose();
|
await llm.dispose();
|
||||||
}
|
}
|
||||||
|
|
@ -49,8 +62,11 @@ export async function runOnce(): Promise<void> {
|
||||||
export function startScheduler(): void {
|
export function startScheduler(): void {
|
||||||
console.info(`[tech-step-llm-worker] scheduling runs on "${env.TECH_STEP_WORKER_CRON}"`);
|
console.info(`[tech-step-llm-worker] scheduling runs on "${env.TECH_STEP_WORKER_CRON}"`);
|
||||||
cron.schedule(env.TECH_STEP_WORKER_CRON, () => {
|
cron.schedule(env.TECH_STEP_WORKER_CRON, () => {
|
||||||
|
// Prove liveness even for a fire that then fails inside `runOnce`.
|
||||||
|
void postHeartbeat({ event: "tick" });
|
||||||
runOnce().catch((err: unknown) => {
|
runOnce().catch((err: unknown) => {
|
||||||
console.error("[tech-step-llm-worker] scheduled run failed:", err);
|
console.error("[tech-step-llm-worker] scheduled run failed:", err);
|
||||||
|
void postHeartbeat({ event: "job", ok: false });
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -387,8 +387,22 @@ est un `String` libre (`"user.signup"`, `"recipe.imported"`, `"recipe.created"`,
|
||||||
ligne, **sans migration**. `AnalyticsEvent.actorId` n'a **pas** de FK (un
|
ligne, **sans migration**. `AnalyticsEvent.actorId` n'a **pas** de FK (un
|
||||||
évènement est un fait historique qui survit au compte qu'il décrit).
|
évènement est un fait historique qui survit au compte qu'il décrit).
|
||||||
|
|
||||||
*(La table `WorkerHeartbeat` est créée par la même migration mais n'est
|
**Monitoring** (`admin-monitoring.service.ts`, `GET /admin/monitoring`,
|
||||||
câblée que par le monitoring — section à venir.)*
|
`requireAdmin`) — sonde active, chaque cible bornée à ~2 s, latence mesurée :
|
||||||
|
Postgres (`SELECT 1`), l'API elle-même (uptime/RSS), `tech-step-intent-service`
|
||||||
|
(`GET /health`, sans secret), et le **worker LLM** via sa ligne
|
||||||
|
`WorkerHeartbeat`. `MonitoringView { services: ServiceHealthView[] }`, statut
|
||||||
|
`up | degraded | down | unknown` ; une sonde `down` ne fait jamais échouer les
|
||||||
|
autres ni l'endpoint. Le front (`MonitoringPage`) re-poll toutes les 15 s.
|
||||||
|
|
||||||
|
Le worker (`services/tech-step-llm-worker`) n'a **aucune** surface HTTP
|
||||||
|
entrante — il devient observable via `POST /internal/tech-steps/heartbeat`
|
||||||
|
(`requireInternalWorker`, `tech-step-worker.service.ts`'s
|
||||||
|
`recordWorkerHeartbeat` → upsert `WorkerHeartbeat`, clé fixe
|
||||||
|
`"tech-step-llm-worker"`). Le worker l'appelle au boot, à chaque tick du
|
||||||
|
scheduler, et après chaque job (avec `job`/`ok`/`counts`) — best-effort, un
|
||||||
|
heartbeat en échec ne casse jamais un run. Seuils d'âge : > 8 j ⇒ `degraded`,
|
||||||
|
> 21 j ⇒ `down` (cron par défaut hebdomadaire).
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue