Compare commits
8 commits
main
...
feat/tech-
| Author | SHA1 | Date | |
|---|---|---|---|
| 6ae0b5aabb | |||
| c7116d4a29 | |||
| 74cd14c0a5 | |||
| e2ffa7d103 | |||
| 84ccfec02c | |||
| 0dadadfa24 | |||
| b886a0fc16 | |||
| 4e0a9ce8d2 |
33 changed files with 2019 additions and 90 deletions
13
.github/workflows/ci.yml
vendored
13
.github/workflows/ci.yml
vendored
|
|
@ -96,14 +96,11 @@ jobs:
|
||||||
uv run uvicorn intent_service.main:app --host 0.0.0.0 --port 8000 &
|
uv run uvicorn intent_service.main:app --host 0.0.0.0 --port 8000 &
|
||||||
# `/health` only returns 200 once this service has finished
|
# `/health` only returns 200 once this service has finished
|
||||||
# training itself from scratch (no model ever persisted to disk —
|
# training itself from scratch (no model ever persisted to disk —
|
||||||
# see its own README) — measured at ~335s per locale (~670s for
|
# see its own README) — measured at ~540s (fr) / ~390s (en),
|
||||||
# fr+en combined) against the current ~74-technique corpus,
|
# ~930s combined, against the current ~74-technique corpus (see
|
||||||
# trained on each technique's own synonyms in addition to its
|
# docker-compose.yml's healthcheck for the same reasoning and why
|
||||||
# example phrases, so this wait is generous rather than the fast
|
# this grew slightly from the original ~670s).
|
||||||
# "base models only" check it used to be before that service
|
timeout 1200 bash -c 'until curl -sf http://localhost:8000/health > /dev/null; do sleep 2; done'
|
||||||
# trained itself at startup (see docker-compose.yml's healthcheck
|
|
||||||
# for the same reasoning).
|
|
||||||
timeout 900 bash -c 'until curl -sf http://localhost:8000/health > /dev/null; do sleep 2; done'
|
|
||||||
|
|
||||||
- run: pnpm install --frozen-lockfile
|
- run: pnpm install --frozen-lockfile
|
||||||
- run: pnpm --filter api exec prisma migrate deploy
|
- run: pnpm --filter api exec prisma migrate deploy
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "utensil" (
|
||||||
|
"id" SERIAL NOT NULL,
|
||||||
|
"key" TEXT NOT NULL,
|
||||||
|
|
||||||
|
CONSTRAINT "utensil_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateIndex
|
||||||
|
CREATE UNIQUE INDEX "utensil_key_key" ON "utensil"("key");
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "step_tech_step_ingredient" (
|
||||||
|
"id" SERIAL NOT NULL,
|
||||||
|
"step_id" INTEGER NOT NULL,
|
||||||
|
"tech_step_order" INTEGER NOT NULL,
|
||||||
|
"ingredient_id" INTEGER NOT NULL,
|
||||||
|
"quantity" DECIMAL(10,2),
|
||||||
|
"unit_id" INTEGER,
|
||||||
|
"start" INTEGER NOT NULL,
|
||||||
|
"end" INTEGER NOT NULL,
|
||||||
|
"source" TEXT NOT NULL DEFAULT 'auto',
|
||||||
|
|
||||||
|
CONSTRAINT "step_tech_step_ingredient_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
-- CreateTable
|
||||||
|
CREATE TABLE "step_tech_step_utensil" (
|
||||||
|
"id" SERIAL NOT NULL,
|
||||||
|
"step_id" INTEGER NOT NULL,
|
||||||
|
"tech_step_order" INTEGER NOT NULL,
|
||||||
|
"utensil_id" INTEGER NOT NULL,
|
||||||
|
"start" INTEGER NOT NULL,
|
||||||
|
"end" INTEGER NOT NULL,
|
||||||
|
"source" TEXT NOT NULL DEFAULT 'auto',
|
||||||
|
|
||||||
|
CONSTRAINT "step_tech_step_utensil_pkey" PRIMARY KEY ("id")
|
||||||
|
);
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "step_tech_step_ingredient" ADD CONSTRAINT "step_tech_step_ingredient_step_id_tech_step_order_fkey" FOREIGN KEY ("step_id", "tech_step_order") REFERENCES "step_tech_step"("step_id", "order") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "step_tech_step_ingredient" ADD CONSTRAINT "step_tech_step_ingredient_ingredient_id_fkey" FOREIGN KEY ("ingredient_id") REFERENCES "ingredients"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "step_tech_step_ingredient" ADD CONSTRAINT "step_tech_step_ingredient_unit_id_fkey" FOREIGN KEY ("unit_id") REFERENCES "unit"("id") ON DELETE SET NULL ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "step_tech_step_utensil" ADD CONSTRAINT "step_tech_step_utensil_step_id_tech_step_order_fkey" FOREIGN KEY ("step_id", "tech_step_order") REFERENCES "step_tech_step"("step_id", "order") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||||
|
|
||||||
|
-- AddForeignKey
|
||||||
|
ALTER TABLE "step_tech_step_utensil" ADD CONSTRAINT "step_tech_step_utensil_utensil_id_fkey" FOREIGN KEY ("utensil_id") REFERENCES "utensil"("id") ON DELETE CASCADE ON UPDATE CASCADE;
|
||||||
|
|
@ -521,6 +521,9 @@ model Ingredient {
|
||||||
dislikedBy UserProfileDislikedIngredient[]
|
dislikedBy UserProfileDislikedIngredient[]
|
||||||
/// Diet regimes this ingredient is compatible with — see {@link IngredientDiet}.
|
/// Diet regimes this ingredient is compatible with — see {@link IngredientDiet}.
|
||||||
diets IngredientDiet[]
|
diets IngredientDiet[]
|
||||||
|
/// Mentions of this ingredient detected in a step's free text alongside a
|
||||||
|
/// technique — see `StepTechStepIngredient`.
|
||||||
|
stepTechSteps StepTechStepIngredient[]
|
||||||
|
|
||||||
@@map("ingredients")
|
@@map("ingredients")
|
||||||
}
|
}
|
||||||
|
|
@ -597,6 +600,12 @@ model Unit {
|
||||||
toBaseFactor Decimal @default(1) @map("to_base_factor") @db.Decimal(12, 4)
|
toBaseFactor Decimal @default(1) @map("to_base_factor") @db.Decimal(12, 4)
|
||||||
|
|
||||||
recipeIngredients RecipeIngredient[]
|
recipeIngredients RecipeIngredient[]
|
||||||
|
/// Ingredient mentions detected alongside a technique in a step's free
|
||||||
|
/// text (e.g. "50g" resolved against this `Unit`) — see
|
||||||
|
/// `StepTechStepIngredient`. Distinct from `recipeIngredients` above
|
||||||
|
/// (the recipe's structured ingredient list): a step can mention a
|
||||||
|
/// quantity+unit that was never itself an ingredient list line.
|
||||||
|
stepTechStepIngredients StepTechStepIngredient[]
|
||||||
|
|
||||||
@@map("unit")
|
@@map("unit")
|
||||||
}
|
}
|
||||||
|
|
@ -652,6 +661,25 @@ model TechStep {
|
||||||
@@map("tech_step")
|
@@map("tech_step")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `key` is `@unique`, same bare id+key shape as `TechStep` — no
|
||||||
|
/// categorization taxonomy like `Ingredient` needed yet, and no matching
|
||||||
|
/// data of its own here either: unlike `TechStep` (whose matching synonyms
|
||||||
|
/// used to live in TS and were moved into
|
||||||
|
/// `services/tech-step-intent-service`'s `training_data.py`), this catalog
|
||||||
|
/// was *born* owned by that service (`utensil_vocabulary.py`) since nothing
|
||||||
|
/// pre-existing needed it — this row only exists to be a stable id/key
|
||||||
|
/// `StepTechStepUtensil` references, and to carry a French label
|
||||||
|
/// (`apps/web`'s `catalog.utensils.<key>`, see `reference-seed-data.ts`'s
|
||||||
|
/// `UTENSILS`).
|
||||||
|
model Utensil {
|
||||||
|
id Int @id @default(autoincrement())
|
||||||
|
key String @unique
|
||||||
|
|
||||||
|
steps StepTechStepUtensil[]
|
||||||
|
|
||||||
|
@@map("utensil")
|
||||||
|
}
|
||||||
|
|
||||||
/// Modeled as one-to-many (a step belongs to exactly one recipe), not the
|
/// Modeled as one-to-many (a step belongs to exactly one recipe), not the
|
||||||
/// many-to-many noted in the spec doc: `order` only makes sense scoped to a
|
/// many-to-many noted in the spec doc: `order` only makes sense scoped to a
|
||||||
/// single recipe, which isn't reconcilable with steps being shared across
|
/// single recipe, which isn't reconcilable with steps being shared across
|
||||||
|
|
@ -722,11 +750,70 @@ model StepTechStep {
|
||||||
|
|
||||||
step Step @relation(fields: [stepId], references: [id], onDelete: Cascade)
|
step Step @relation(fields: [stepId], references: [id], onDelete: Cascade)
|
||||||
techStep TechStep @relation(fields: [techStepId], references: [id], onDelete: Cascade)
|
techStep TechStep @relation(fields: [techStepId], references: [id], onDelete: Cascade)
|
||||||
|
/// Ingredients mentioned in the same clause as this technique occurrence
|
||||||
|
/// — see `StepTechStepIngredient`.
|
||||||
|
ingredients StepTechStepIngredient[]
|
||||||
|
/// Utensils mentioned in the same clause as this technique occurrence —
|
||||||
|
/// see `StepTechStepUtensil`.
|
||||||
|
utensils StepTechStepUtensil[]
|
||||||
|
|
||||||
@@id([stepId, order])
|
@@id([stepId, order])
|
||||||
@@map("step_tech_step")
|
@@map("step_tech_step")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// An ingredient mention found in the same *clause* as one `StepTechStep`
|
||||||
|
/// occurrence (`tech-step-matcher.ts`'s `matchTechStepSpans` — clauses are
|
||||||
|
/// already the unit a technique is judged on, see that file's doc comment,
|
||||||
|
/// so "same clause" is the association rule, no dependency-parsing needed).
|
||||||
|
/// `quantity`/`unitId` are best-effort, populated only when a leading
|
||||||
|
/// numeric expression immediately preceding the ingredient mention resolved
|
||||||
|
/// against the `Unit` catalog (`ingredient-matcher.ts`'s
|
||||||
|
/// `findIngredientMentions`) — both `null` when the clause names the
|
||||||
|
/// ingredient with no quantity ("ajouter le sel"). `start`/`end` are the
|
||||||
|
/// ingredient mention's own span in `Step.description`, same `[start, end)`
|
||||||
|
/// convention as `StepTechStep.start`/`end`. `source` mirrors
|
||||||
|
/// `StepTechStep.source` (`"auto"` today, room for a future user
|
||||||
|
/// correction without a shape change).
|
||||||
|
model StepTechStepIngredient {
|
||||||
|
id Int @id @default(autoincrement())
|
||||||
|
stepId Int @map("step_id")
|
||||||
|
techStepOrder Int @map("tech_step_order")
|
||||||
|
ingredientId Int @map("ingredient_id")
|
||||||
|
quantity Decimal? @db.Decimal(10, 2)
|
||||||
|
unitId Int? @map("unit_id")
|
||||||
|
start Int
|
||||||
|
end Int
|
||||||
|
source String @default("auto")
|
||||||
|
|
||||||
|
stepTechStep StepTechStep @relation(fields: [stepId, techStepOrder], references: [stepId, order], onDelete: Cascade)
|
||||||
|
ingredient Ingredient @relation(fields: [ingredientId], references: [id], onDelete: Cascade)
|
||||||
|
unit Unit? @relation(fields: [unitId], references: [id])
|
||||||
|
|
||||||
|
@@map("step_tech_step_ingredient")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A utensil mention found in the same clause as one `StepTechStep`
|
||||||
|
/// occurrence — same association rule as `StepTechStepIngredient` (see its
|
||||||
|
/// doc comment). Detected by
|
||||||
|
/// `services/tech-step-intent-service`'s own utensil `PhraseMatcher`
|
||||||
|
/// (`intent_service/utensil_vocabulary.py`), returned alongside technique
|
||||||
|
/// entities in `POST /v1/process` and filtered to this clause's span by
|
||||||
|
/// `tech-step-matcher.ts`.
|
||||||
|
model StepTechStepUtensil {
|
||||||
|
id Int @id @default(autoincrement())
|
||||||
|
stepId Int @map("step_id")
|
||||||
|
techStepOrder Int @map("tech_step_order")
|
||||||
|
utensilId Int @map("utensil_id")
|
||||||
|
start Int
|
||||||
|
end Int
|
||||||
|
source String @default("auto")
|
||||||
|
|
||||||
|
stepTechStep StepTechStep @relation(fields: [stepId, techStepOrder], references: [stepId, order], onDelete: Cascade)
|
||||||
|
utensil Utensil @relation(fields: [utensilId], references: [id], onDelete: Cascade)
|
||||||
|
|
||||||
|
@@map("step_tech_step_utensil")
|
||||||
|
}
|
||||||
|
|
||||||
/// One user-submitted correction to a `Step`'s detected techniques —
|
/// One user-submitted correction to a `Step`'s detected techniques —
|
||||||
/// captures ADD (a missing technique the classifier didn't find),
|
/// captures ADD (a missing technique the classifier didn't find),
|
||||||
/// REMOVE (a wrong technique it did), or RELABEL (both) as a single shape:
|
/// REMOVE (a wrong technique it did), or RELABEL (both) as a single shape:
|
||||||
|
|
|
||||||
|
|
@ -148,6 +148,49 @@ export const TECH_STEPS: string[] = [
|
||||||
"zest",
|
"zest",
|
||||||
];
|
];
|
||||||
|
|
||||||
|
// Same authoring convention as `TECH_STEPS` right above (stable English
|
||||||
|
// camelCase uid, French label in `apps/web`'s `locales/fr/translation.json`
|
||||||
|
// under `catalog.utensils.<key>`) — but unlike `TECH_STEPS`, the matching
|
||||||
|
// data (per-locale synonym lists a `PhraseMatcher` matches against) lives
|
||||||
|
// in `services/tech-step-intent-service/intent_service/utensil_vocabulary.py`'s
|
||||||
|
// `UTENSIL_VOCABULARY`, not `training_data.py`: no textcat/training
|
||||||
|
// involved, a utensil mention doesn't need to be classified, only matched.
|
||||||
|
// Every entry here must have a matching entry there. See
|
||||||
|
// `StepTechStepUtensil` in schema.prisma for how a mention gets attached to
|
||||||
|
// a detected technique.
|
||||||
|
export const UTENSILS: string[] = [
|
||||||
|
"pan",
|
||||||
|
"saucepan",
|
||||||
|
"pot",
|
||||||
|
"knife",
|
||||||
|
"whisk",
|
||||||
|
"bowl",
|
||||||
|
"bakingSheet",
|
||||||
|
"mold",
|
||||||
|
"colander",
|
||||||
|
"cuttingBoard",
|
||||||
|
"oven",
|
||||||
|
"blender",
|
||||||
|
"mixer",
|
||||||
|
"spatula",
|
||||||
|
"ladle",
|
||||||
|
"grater",
|
||||||
|
"rollingPin",
|
||||||
|
"lid",
|
||||||
|
"tongs",
|
||||||
|
"peeler",
|
||||||
|
"sieve",
|
||||||
|
"foodProcessor",
|
||||||
|
"steamerBasket",
|
||||||
|
"skewer",
|
||||||
|
"pastryBrush",
|
||||||
|
"ramekin",
|
||||||
|
"dish",
|
||||||
|
"wok",
|
||||||
|
"thermometer",
|
||||||
|
"mandoline",
|
||||||
|
];
|
||||||
|
|
||||||
// The 14 allergens EU Regulation 1169/2011 (Annex II) requires food
|
// The 14 allergens EU Regulation 1169/2011 (Annex II) requires food
|
||||||
// businesses to declare — a standard, defensible reference list rather than
|
// businesses to declare — a standard, defensible reference list rather than
|
||||||
// an invented one. Split into ALLERGY (classic IgE-mediated immune
|
// an invented one. Split into ALLERGY (classic IgE-mediated immune
|
||||||
|
|
@ -1251,6 +1294,12 @@ export async function seedReferenceData(prisma: PrismaClient): Promise<void> {
|
||||||
await prisma.techStep.upsert({ where: { key }, update: {}, create: { key } });
|
await prisma.techStep.upsert({ where: { key }, update: {}, create: { key } });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Utensil: same idempotent bare id/key upsert as TechStep right above —
|
||||||
|
// no matching data alongside it either (see `UTENSILS`' own comment).
|
||||||
|
for (const key of UTENSILS) {
|
||||||
|
await prisma.utensil.upsert({ where: { key }, update: {}, create: { key } });
|
||||||
|
}
|
||||||
|
|
||||||
// `Allergy` itself carries no `key` — it's the selectable instance of a
|
// `Allergy` itself carries no `key` — it's the selectable instance of a
|
||||||
// keyed `Category` (see schema.prisma) — so seeding an allergen means one
|
// keyed `Category` (see schema.prisma) — so seeding an allergen means one
|
||||||
// Category (upserted by key) plus exactly one Allergy row under it,
|
// Category (upserted by key) plus exactly one Allergy row under it,
|
||||||
|
|
|
||||||
|
|
@ -111,6 +111,171 @@ function containsSubsequence(haystack: string[], needle: string[]): boolean {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** One stemmed word from {@link tokenizeWithOffsets}, alongside its `[start, end)` span in the *original* (un-normalized) text it came from. */
|
||||||
|
interface OffsetToken {
|
||||||
|
word: string;
|
||||||
|
start: number;
|
||||||
|
end: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Matches a run of letters (any script, diacritics included) — the same "word" unit {@link tokenize} splits `normalizeText`'d text on (`/[^a-z]+/`), applied here directly to the *original* text instead so each token keeps its real character offsets. Digits/punctuation are never part of a run, same separator role they play for `tokenize` (a leading quantity is `extractQuantity`'s job, not this module's word-tokenizer's). */
|
||||||
|
const LETTER_RUN_PATTERN = /\p{L}+/gu;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* {@link tokenize}'s positional twin: same stemmed/normalized words, but
|
||||||
|
* each one keeps the `[start, end)` span it occupies in `text` — needed by
|
||||||
|
* {@link findIngredientMentions} to report *where* a mention is, not just
|
||||||
|
* that the catalog has a matching label somewhere. Splitting the original
|
||||||
|
* text into letter-runs first (rather than normalizing the whole string up
|
||||||
|
* front, the way `tokenize` does, then losing track of offsets) works
|
||||||
|
* safely here because `normalizeText` only ever rewrites a character's own
|
||||||
|
* form (case/diacritics) — see `_DiacriticsNormalizer`'s doc comment on the
|
||||||
|
* Python side, ported from the same guarantee — never merges or splits
|
||||||
|
* words, so normalizing one already-isolated run in place can't shift its
|
||||||
|
* boundaries relative to the un-normalized text.
|
||||||
|
*/
|
||||||
|
function tokenizeWithOffsets(text: string, locale = "en"): OffsetToken[] {
|
||||||
|
const tokens: OffsetToken[] = [];
|
||||||
|
for (const match of text.matchAll(LETTER_RUN_PATTERN)) {
|
||||||
|
const raw = match[0];
|
||||||
|
const start = match.index ?? 0;
|
||||||
|
const word = stemWord(normalizeText(raw), locale);
|
||||||
|
if (word.length === 0) continue;
|
||||||
|
tokens.push({ word, start, end: start + raw.length });
|
||||||
|
}
|
||||||
|
return tokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Matches a quantity (integer/decimal/fraction/mixed number, same shapes as
|
||||||
|
* {@link extractQuantity}) immediately followed by an optional unit
|
||||||
|
* word/phrase (up to three words, e.g. "cuillères à soupe") and an optional
|
||||||
|
* connector ("de"/"d'"/"of"/"a"/"an"), anchored at the *end* of whatever
|
||||||
|
* string it's tested against (`$`) rather than the start. Anchoring at the
|
||||||
|
* end — not the start — is what lets {@link findQuantityBeforeIngredient}
|
||||||
|
* test the *whole* text preceding a mention without first having to guess
|
||||||
|
* where an unrelated preamble ("ajouter", "puis", an earlier sentence) ends
|
||||||
|
* and the quantity phrase begins: whatever doesn't fit the pattern
|
||||||
|
* immediately before the ingredient simply isn't part of the match, no
|
||||||
|
* separate boundary-finding step needed.
|
||||||
|
*/
|
||||||
|
const QUANTITY_BEFORE_INGREDIENT_PATTERN =
|
||||||
|
/(\d+\s+\d+\/\d+|\d+\/\d+|\d+(?:[.,]\d+)?)\s*((?:\p{L}+\s+){0,2}\p{L}*)\s*(?:de\s|d['’]|of\s|a\s|an\s)?$/u;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Best-effort quantity+unit lookup for an ingredient mention {@link findIngredientMentions}
|
||||||
|
* just found at `mentionStart` in `text` — looks *only* at what immediately
|
||||||
|
* precedes the mention (see {@link QUANTITY_BEFORE_INGREDIENT_PATTERN}), the
|
||||||
|
* dominant French/English recipe phrasing ("200g de beurre", "2 cuillères à
|
||||||
|
* soupe d'huile", "3 œufs"). Both `null` when nothing recognizable precedes
|
||||||
|
* it (no leading digit at all) — same "no match, not an error" posture as
|
||||||
|
* {@link extractQuantity}. Doesn't detect a quantity that *follows* its
|
||||||
|
* ingredient ("du beurre, 50g") — an accepted gap, same trade-off
|
||||||
|
* {@link extractQuantity} already documents for the leading-only case it
|
||||||
|
* was built for.
|
||||||
|
*/
|
||||||
|
function findQuantityBeforeIngredient(
|
||||||
|
text: string,
|
||||||
|
mentionStart: number,
|
||||||
|
unitCatalog: UnitMatchEntry[],
|
||||||
|
locale: string,
|
||||||
|
): { quantity: number | null; unitId: number | null } {
|
||||||
|
const match = QUANTITY_BEFORE_INGREDIENT_PATTERN.exec(text.slice(0, mentionStart));
|
||||||
|
if (!match) return { quantity: null, unitId: null };
|
||||||
|
const { quantity } = extractQuantity(match[1] ?? "");
|
||||||
|
const unitId = matchUnit(match[2] ?? "", unitCatalog, locale);
|
||||||
|
return { quantity, unitId };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One ingredient mention {@link findIngredientMentions} found in a free-text clause, alongside its `[start, end)` span (same convention as `TechStepMatch`, `tech-step-matcher.ts`) and any quantity+unit resolved immediately before it (see {@link findQuantityBeforeIngredient}) — both `null` when the clause names the ingredient with no quantity ("ajouter le sel"). */
|
||||||
|
export interface IngredientMention {
|
||||||
|
ingredientId: number;
|
||||||
|
start: number;
|
||||||
|
end: number;
|
||||||
|
quantity: number | null;
|
||||||
|
unitId: number | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Scans `text` (typically one technique's clause, see `tech-step-matcher.ts`'s
|
||||||
|
* `splitIntoClauses`) for every mention of a catalog ingredient, left to
|
||||||
|
* right, non-overlapping — the free-text-*scanning* counterpart to
|
||||||
|
* {@link matchIngredientName} (which resolves one *already-isolated*
|
||||||
|
* ingredient-line string to a single winner, not several mentions spread
|
||||||
|
* across a longer text). Same "longest catalog label wins" rule as
|
||||||
|
* {@link matchIngredientName}, applied at every token position in turn: once
|
||||||
|
* a mention is found, scanning resumes right after it rather than
|
||||||
|
* considering a shorter label starting inside an already-matched longer one.
|
||||||
|
*
|
||||||
|
* `locale` must match whatever `ingredientCatalog`/`unitCatalog` were loaded
|
||||||
|
* in (see {@link loadIngredientCatalog}/{@link loadUnitCatalog}) — defaults
|
||||||
|
* to `"en"`, same as every other function in this module.
|
||||||
|
*/
|
||||||
|
export function findIngredientMentions(
|
||||||
|
text: string,
|
||||||
|
ingredientCatalog: IngredientMatchEntry[],
|
||||||
|
unitCatalog: UnitMatchEntry[],
|
||||||
|
locale = "en",
|
||||||
|
): IngredientMention[] {
|
||||||
|
const tokens = tokenizeWithOffsets(text, locale);
|
||||||
|
if (tokens.length === 0) return [];
|
||||||
|
|
||||||
|
const candidates = ingredientCatalog
|
||||||
|
.map((entry) => ({
|
||||||
|
ingredientId: entry.ingredientId,
|
||||||
|
labelTokens: tokenize(entry.label, locale),
|
||||||
|
}))
|
||||||
|
.filter((entry) => entry.labelTokens.length > 0);
|
||||||
|
|
||||||
|
const mentions: IngredientMention[] = [];
|
||||||
|
let i = 0;
|
||||||
|
while (i < tokens.length) {
|
||||||
|
let best: { ingredientId: number; tokenCount: number } | null = null;
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
const { labelTokens } = candidate;
|
||||||
|
if (i + labelTokens.length > tokens.length) continue;
|
||||||
|
const matches = labelTokens.every((word, offset) => tokens[i + offset]?.word === word);
|
||||||
|
if (!matches) continue;
|
||||||
|
if (
|
||||||
|
best === null ||
|
||||||
|
labelTokens.length > best.tokenCount ||
|
||||||
|
(labelTokens.length === best.tokenCount && candidate.ingredientId < best.ingredientId)
|
||||||
|
) {
|
||||||
|
best = { ingredientId: candidate.ingredientId, tokenCount: labelTokens.length };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (best === null) {
|
||||||
|
i += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const startToken = tokens[i];
|
||||||
|
const endToken = tokens[i + best.tokenCount - 1];
|
||||||
|
if (startToken === undefined || endToken === undefined) {
|
||||||
|
// Unreachable — `best` was only ever set above after confirming
|
||||||
|
// `i + labelTokens.length <= tokens.length`, so both tokens exist.
|
||||||
|
// Satisfies `noUncheckedIndexedAccess`.
|
||||||
|
i += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const { quantity, unitId } = findQuantityBeforeIngredient(
|
||||||
|
text,
|
||||||
|
startToken.start,
|
||||||
|
unitCatalog,
|
||||||
|
locale,
|
||||||
|
);
|
||||||
|
mentions.push({
|
||||||
|
ingredientId: best.ingredientId,
|
||||||
|
start: startToken.start,
|
||||||
|
end: endToken.end,
|
||||||
|
quantity,
|
||||||
|
unitId,
|
||||||
|
});
|
||||||
|
i += best.tokenCount;
|
||||||
|
}
|
||||||
|
return mentions;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolves free-text `name` (a `ParsedRecipeIngredient.name`, e.g. `"large
|
* Resolves free-text `name` (a `ParsedRecipeIngredient.name`, e.g. `"large
|
||||||
* diced yellow onions"`) to the best-matching `Ingredient` in `catalog`, or
|
* diced yellow onions"`) to the best-matching `Ingredient` in `catalog`, or
|
||||||
|
|
|
||||||
|
|
@ -17,11 +17,12 @@ import { env } from "../../config/env.js";
|
||||||
* shape.
|
* shape.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
/** One candidate technique mention the service's `PhraseMatcher` found — offsets `[start, end)`, same convention as `String.prototype.slice`. Mirrors `EntityPayload` (Python `schemas.py`). */
|
/** One candidate mention one of the service's two `PhraseMatcher`s found — offsets `[start, end)`, same convention as `String.prototype.slice`. Mirrors `EntityPayload` (Python `schemas.py`). `kind` distinguishes a technique mention (`self._matcher`, the corpus-trained one) from a utensil mention (`self._utensil_matcher`, static — see `utensil_vocabulary.py`) — `tech-step-matcher.ts` resolves each against a different catalog (`TechStep`/`Utensil`). */
|
||||||
export interface IntentServiceEntity {
|
export interface IntentServiceEntity {
|
||||||
uid: string;
|
uid: string;
|
||||||
start: number;
|
start: number;
|
||||||
end: number;
|
end: number;
|
||||||
|
kind: "technique" | "utensil";
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The full result of a `POST /v1/process` call — mirrors `ProcessResponse` (Python `schemas.py`). `intent` is `null` only when `locale` isn't one this service trains for, or `text` is blank; otherwise always a real `uid` (the Python service's `textcat` has no "None" sentinel, unlike node-nlp — see that service's README). */
|
/** The full result of a `POST /v1/process` call — mirrors `ProcessResponse` (Python `schemas.py`). `intent` is `null` only when `locale` isn't one this service trains for, or `text` is blank; otherwise always a real `uid` (the Python service's `textcat` has no "None" sentinel, unlike node-nlp — see that service's README). */
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,10 @@
|
||||||
import { prisma } from "../../db/prisma.js";
|
import { prisma } from "../../db/prisma.js";
|
||||||
|
import {
|
||||||
|
findIngredientMentions,
|
||||||
|
type IngredientMention,
|
||||||
|
loadIngredientCatalog,
|
||||||
|
loadUnitCatalog,
|
||||||
|
} from "./ingredient-matcher.js";
|
||||||
import { intentServiceClient } from "./intent-service-client.js";
|
import { intentServiceClient } from "./intent-service-client.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -90,6 +96,15 @@ export function normalizeText(text: string): string {
|
||||||
* Persisted as `StepTechStep.start`/`end`/`contextStart`/`contextEnd`
|
* Persisted as `StepTechStep.start`/`end`/`contextStart`/`contextEnd`
|
||||||
* (`recipe.service.ts`) so the recipe detail view can highlight both spans,
|
* (`recipe.service.ts`) so the recipe detail view can highlight both spans,
|
||||||
* not just know a technique was mentioned somewhere.
|
* not just know a technique was mentioned somewhere.
|
||||||
|
*
|
||||||
|
* `ingredients`/`utensils` are the metadata found in this match's own
|
||||||
|
* *clause* (see this file's doc comment, point 2) — an ingredient/utensil
|
||||||
|
* mentioned in a different clause of the same description belongs to
|
||||||
|
* *that* clause's own match, never this one, the same "judged on its own
|
||||||
|
* surrounding context" rule the technique itself is judged by. Always `[]`
|
||||||
|
* rather than omitted when nothing was found, so every caller can iterate
|
||||||
|
* unconditionally. Persisted as `StepTechStepIngredient`/`StepTechStepUtensil`
|
||||||
|
* rows (`recipe.service.ts`).
|
||||||
*/
|
*/
|
||||||
export interface TechStepMatch {
|
export interface TechStepMatch {
|
||||||
techStepId: number;
|
techStepId: number;
|
||||||
|
|
@ -97,6 +112,21 @@ export interface TechStepMatch {
|
||||||
end: number;
|
end: number;
|
||||||
contextStart: number;
|
contextStart: number;
|
||||||
contextEnd: number;
|
contextEnd: number;
|
||||||
|
ingredients: IngredientMention[];
|
||||||
|
utensils: UtensilMention[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A utensil mention found by the intent service's utensil `PhraseMatcher`
|
||||||
|
* (`kind: "utensil"` entities in `IntentServiceProcessResult`, see
|
||||||
|
* `intent-service-client.ts`), resolved to a local `Utensil.id` and
|
||||||
|
* attributed to whichever clause its span falls inside — same
|
||||||
|
* `[start, end)` convention as every other span in this file.
|
||||||
|
*/
|
||||||
|
export interface UtensilMention {
|
||||||
|
utensilId: number;
|
||||||
|
start: number;
|
||||||
|
end: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** A candidate technique mention found by NER — the raw material {@link splitIntoClauses} cuts a description around. */
|
/** A candidate technique mention found by NER — the raw material {@link splitIntoClauses} cuts a description around. */
|
||||||
|
|
@ -310,6 +340,10 @@ export class TechStepClassifierService {
|
||||||
private _techStepIdsLoaded: Promise<void> | undefined;
|
private _techStepIdsLoaded: Promise<void> | undefined;
|
||||||
private _techStepIdByUid: Map<string, number> | undefined;
|
private _techStepIdByUid: Map<string, number> | undefined;
|
||||||
|
|
||||||
|
/** Same memoized-lookup shape as {@link _techStepIdsLoaded}/{@link _techStepIdByUid}, for `Utensil.key -> id` instead — a `kind: "utensil"` entity from the intent service resolves through this map, never `_techStepIdByUid`. */
|
||||||
|
private _utensilIdsLoaded: Promise<void> | undefined;
|
||||||
|
private _utensilIdByUid: Map<string, number> | undefined;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Forces the `TechStep.key -> id` lookup to load now, synchronously with
|
* Forces the `TechStep.key -> id` lookup to load now, synchronously with
|
||||||
* server startup (see `server.ts`, which also retries this against a
|
* server startup (see `server.ts`, which also retries this against a
|
||||||
|
|
@ -345,22 +379,35 @@ export class TechStepClassifierService {
|
||||||
*/
|
*/
|
||||||
public async matchTechStepSpans(description: string, locale: string): Promise<TechStepMatch[]> {
|
public async matchTechStepSpans(description: string, locale: string): Promise<TechStepMatch[]> {
|
||||||
try {
|
try {
|
||||||
await this._ensureTechStepIdsLoaded();
|
await Promise.all([this._ensureTechStepIdsLoaded(), this._ensureUtensilIdsLoaded()]);
|
||||||
if (description.trim().length === 0) return [];
|
if (description.trim().length === 0) return [];
|
||||||
|
|
||||||
// The intent service only ever returns enum-style candidates (its own
|
// Loaded fresh per call (once per step, see `recipe.service.ts`'s
|
||||||
// `PhraseMatcher`, built solely from `TECH_STEP_TRAINING_DATA`'s
|
// `matchStepsTechSteps`) rather than memoized like the id lookups
|
||||||
// `synonyms`) — unlike node-nlp, it never mixes in built-in
|
// above — same "cheap enough, and reference data can change between
|
||||||
// numbers/durations/dates entities, so no `type === "enum"` filter is
|
// calls without a restart" posture `loadIngredientCatalog`/
|
||||||
// needed here anymore. Its `start`/`end` are already `[start, end)`
|
// `loadUnitCatalog`'s own doc comments already describe for their
|
||||||
|
// other callers (`ingredient-matcher.ts`, `sources.service.ts`).
|
||||||
|
const [ingredientCatalog, unitCatalog] = await Promise.all([
|
||||||
|
loadIngredientCatalog(locale),
|
||||||
|
loadUnitCatalog(locale),
|
||||||
|
]);
|
||||||
|
|
||||||
|
// The intent service returns two kinds of candidate (see `kind` on
|
||||||
|
// `IntentServiceEntity`): technique mentions (its corpus-trained
|
||||||
|
// `PhraseMatcher`) and utensil mentions (its static one, see
|
||||||
|
// `utensil_vocabulary.py`). Only the former ever anchor a clause —
|
||||||
|
// `splitIntoClauses` cuts a description around *techniques*, a
|
||||||
|
// mentioned utensil doesn't introduce a clause boundary of its own,
|
||||||
|
// it just gets attributed to whichever clause its span falls inside
|
||||||
|
// (see the loop below). Its `start`/`end` are already `[start, end)`
|
||||||
// (matching `String.prototype.slice`), unlike node-nlp's inclusive
|
// (matching `String.prototype.slice`), unlike node-nlp's inclusive
|
||||||
// `end` — no `+ 1` needed either.
|
// `end` — no `+ 1` needed either.
|
||||||
const nerResult = await intentServiceClient.process(locale, description);
|
const nerResult = await intentServiceClient.process(locale, description);
|
||||||
const candidates: TechniqueCandidate[] = nerResult.entities.map((entity) => ({
|
const candidates: TechniqueCandidate[] = nerResult.entities
|
||||||
uid: entity.uid,
|
.filter((entity) => entity.kind === "technique")
|
||||||
start: entity.start,
|
.map((entity) => ({ uid: entity.uid, start: entity.start, end: entity.end }));
|
||||||
end: entity.end,
|
const utensilEntities = nerResult.entities.filter((entity) => entity.kind === "utensil");
|
||||||
}));
|
|
||||||
|
|
||||||
const clauses = splitIntoClauses(description, candidates);
|
const clauses = splitIntoClauses(description, candidates);
|
||||||
const matches: TechStepMatch[] = [];
|
const matches: TechStepMatch[] = [];
|
||||||
|
|
@ -374,12 +421,35 @@ export class TechStepClassifierService {
|
||||||
// persist a dangling id.
|
// persist a dangling id.
|
||||||
if (techStepId === undefined) continue;
|
if (techStepId === undefined) continue;
|
||||||
const span = clause.anchor ?? { start: clause.start, end: clause.end };
|
const span = clause.anchor ?? { start: clause.start, end: clause.end };
|
||||||
|
|
||||||
|
const ingredients = findIngredientMentions(
|
||||||
|
description.slice(clause.start, clause.end),
|
||||||
|
ingredientCatalog,
|
||||||
|
unitCatalog,
|
||||||
|
locale,
|
||||||
|
).map((mention) => ({
|
||||||
|
...mention,
|
||||||
|
start: mention.start + clause.start,
|
||||||
|
end: mention.end + clause.start,
|
||||||
|
}));
|
||||||
|
|
||||||
|
const utensils: UtensilMention[] = utensilEntities.flatMap((entity) => {
|
||||||
|
if (entity.start < clause.start || entity.end > clause.end) return [];
|
||||||
|
const utensilId = this._utensilIdByUid?.get(entity.uid);
|
||||||
|
// Same drift guard as `techStepId` above.
|
||||||
|
return utensilId === undefined
|
||||||
|
? []
|
||||||
|
: [{ utensilId, start: entity.start, end: entity.end }];
|
||||||
|
});
|
||||||
|
|
||||||
matches.push({
|
matches.push({
|
||||||
techStepId,
|
techStepId,
|
||||||
start: span.start,
|
start: span.start,
|
||||||
end: span.end,
|
end: span.end,
|
||||||
contextStart: clause.start,
|
contextStart: clause.start,
|
||||||
contextEnd: clause.end,
|
contextEnd: clause.end,
|
||||||
|
ingredients,
|
||||||
|
utensils,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -413,11 +483,9 @@ export class TechStepClassifierService {
|
||||||
if (description.trim().length === 0) return [];
|
if (description.trim().length === 0) return [];
|
||||||
|
|
||||||
const nerResult = await intentServiceClient.process(locale, description);
|
const nerResult = await intentServiceClient.process(locale, description);
|
||||||
const candidates: TechniqueCandidate[] = nerResult.entities.map((entity) => ({
|
const candidates: TechniqueCandidate[] = nerResult.entities
|
||||||
uid: entity.uid,
|
.filter((entity) => entity.kind === "technique")
|
||||||
start: entity.start,
|
.map((entity) => ({ uid: entity.uid, start: entity.start, end: entity.end }));
|
||||||
end: entity.end,
|
|
||||||
}));
|
|
||||||
|
|
||||||
const clauses = splitIntoClauses(description, candidates);
|
const clauses = splitIntoClauses(description, candidates);
|
||||||
const results: TechStepClauseClassification[] = [];
|
const results: TechStepClauseClassification[] = [];
|
||||||
|
|
@ -522,6 +590,28 @@ export class TechStepClassifierService {
|
||||||
throw err; // see matchTechStepSpans()'s catch comment above
|
throw err; // see matchTechStepSpans()'s catch comment above
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** `Utensil.key -> id` counterpart of {@link _ensureTechStepIdsLoaded} — same memoize-once-retry-on-failure shape. */
|
||||||
|
private async _ensureUtensilIdsLoaded(): Promise<void> {
|
||||||
|
if (this._utensilIdsLoaded === undefined) {
|
||||||
|
this._utensilIdsLoaded = this._loadUtensilIds();
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await this._utensilIdsLoaded;
|
||||||
|
} catch (err) {
|
||||||
|
this._utensilIdsLoaded = undefined;
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async _loadUtensilIds(): Promise<void> {
|
||||||
|
try {
|
||||||
|
const utensils = await prisma.utensil.findMany({ select: { id: true, key: true } });
|
||||||
|
this._utensilIdByUid = new Map(utensils.map((utensil) => [utensil.key, utensil.id]));
|
||||||
|
} catch (err) {
|
||||||
|
throw err; // see matchTechStepSpans()'s catch comment above
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Single shared instance — every caller reuses the one memoized `TechStep.key -> id` lookup rather than re-querying the DB. The actual model training (expensive — a couple of minutes, both locales combined) happens entirely inside `services/tech-step-intent-service`'s own startup, not here — see that service's `_TRAINING_ITERATIONS`. */
|
/** Single shared instance — every caller reuses the one memoized `TechStep.key -> id` lookup rather than re-querying the DB. The actual model training (expensive — a couple of minutes, both locales combined) happens entirely inside `services/tech-step-intent-service`'s own startup, not here — see that service's `_TRAINING_ITERATIONS`. */
|
||||||
|
|
|
||||||
|
|
@ -280,10 +280,28 @@ export async function submitTechStepCorrection(
|
||||||
input.correctedTechStepId ?? null,
|
input.correctedTechStepId ?? null,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Same nested `ingredients`/`utensils` include as `recipe.service.ts`'s
|
||||||
|
// `recipeInclude` — `toStepTechStepViews` (reused below) expects it,
|
||||||
|
// so the fresh sequence read right after a manual correction resolves
|
||||||
|
// exactly the same way a normal `GET /recipes/:id` would.
|
||||||
const freshTechSteps = await tx.stepTechStep.findMany({
|
const freshTechSteps = await tx.stepTechStep.findMany({
|
||||||
where: { stepId: step.id },
|
where: { stepId: step.id },
|
||||||
orderBy: { order: "asc" },
|
orderBy: { order: "asc" },
|
||||||
include: { techStep: true },
|
include: {
|
||||||
|
techStep: true,
|
||||||
|
ingredients: {
|
||||||
|
include: {
|
||||||
|
ingredient: {
|
||||||
|
include: {
|
||||||
|
allergies: { include: { allergy: { include: { category: true } } } },
|
||||||
|
diets: { include: { diet: true } },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
unit: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
utensils: { include: { utensil: true } },
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
return { correction: createdCorrection, techSteps: freshTechSteps };
|
return { correction: createdCorrection, techSteps: freshTechSteps };
|
||||||
|
|
|
||||||
|
|
@ -45,7 +45,29 @@ function recipeInclude(viewerId: number) {
|
||||||
steps: {
|
steps: {
|
||||||
orderBy: { order: "asc" },
|
orderBy: { order: "asc" },
|
||||||
include: {
|
include: {
|
||||||
techSteps: { orderBy: { order: "asc" }, include: { techStep: true } },
|
techSteps: {
|
||||||
|
orderBy: { order: "asc" },
|
||||||
|
include: {
|
||||||
|
techStep: true,
|
||||||
|
// Same `allergies`/`diets` nesting as this function's own
|
||||||
|
// top-level `ingredients` include above — reused by
|
||||||
|
// `toIngredientView` so a mentioned ingredient resolves to the
|
||||||
|
// exact same `IngredientView` shape as the recipe's main
|
||||||
|
// ingredient list, not a second, thinner shape.
|
||||||
|
ingredients: {
|
||||||
|
include: {
|
||||||
|
ingredient: {
|
||||||
|
include: {
|
||||||
|
allergies: { include: { allergy: { include: { category: true } } } },
|
||||||
|
diets: { include: { diet: true } },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
unit: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
utensils: { include: { utensil: true } },
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
diets: { include: { diet: true } },
|
diets: { include: { diet: true } },
|
||||||
|
|
@ -145,7 +167,8 @@ export function toStepTechStepViews(
|
||||||
): StepTechStepView[] {
|
): StepTechStepView[] {
|
||||||
const views: StepTechStepView[] = [];
|
const views: StepTechStepView[] = [];
|
||||||
for (const stepTechStep of techSteps) {
|
for (const stepTechStep of techSteps) {
|
||||||
const { start, end, contextStart, contextEnd, techStep, source } = stepTechStep;
|
const { start, end, contextStart, contextEnd, techStep, source, ingredients, utensils } =
|
||||||
|
stepTechStep;
|
||||||
if (start === null || end === null) continue;
|
if (start === null || end === null) continue;
|
||||||
views.push({
|
views.push({
|
||||||
techStep: { id: techStep.id, key: techStep.key },
|
techStep: { id: techStep.id, key: techStep.key },
|
||||||
|
|
@ -159,6 +182,19 @@ export function toStepTechStepViews(
|
||||||
// `StepTechStepView.source` to the frontend.
|
// `StepTechStepView.source` to the frontend.
|
||||||
source: source === "manual" ? "manual" : "auto",
|
source: source === "manual" ? "manual" : "auto",
|
||||||
...(contextStart !== null && contextEnd !== null ? { contextStart, contextEnd } : {}),
|
...(contextStart !== null && contextEnd !== null ? { contextStart, contextEnd } : {}),
|
||||||
|
ingredients: ingredients.map((stepTechStepIngredient) => ({
|
||||||
|
ingredient: toIngredientView(stepTechStepIngredient.ingredient),
|
||||||
|
quantity:
|
||||||
|
stepTechStepIngredient.quantity === null ? null : Number(stepTechStepIngredient.quantity),
|
||||||
|
unit: stepTechStepIngredient.unit === null ? null : toUnitView(stepTechStepIngredient.unit),
|
||||||
|
start: stepTechStepIngredient.start,
|
||||||
|
end: stepTechStepIngredient.end,
|
||||||
|
})),
|
||||||
|
utensils: utensils.map((stepTechStepUtensil) => ({
|
||||||
|
utensil: { id: stepTechStepUtensil.utensil.id, key: stepTechStepUtensil.utensil.key },
|
||||||
|
start: stepTechStepUtensil.start,
|
||||||
|
end: stepTechStepUtensil.end,
|
||||||
|
})),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
return views;
|
return views;
|
||||||
|
|
@ -553,6 +589,22 @@ async function createRecipeInternal(
|
||||||
contextStart: match.contextStart,
|
contextStart: match.contextStart,
|
||||||
contextEnd: match.contextEnd,
|
contextEnd: match.contextEnd,
|
||||||
order,
|
order,
|
||||||
|
ingredients: {
|
||||||
|
create: match.ingredients.map((ingredient) => ({
|
||||||
|
ingredientId: ingredient.ingredientId,
|
||||||
|
quantity: ingredient.quantity,
|
||||||
|
unitId: ingredient.unitId,
|
||||||
|
start: ingredient.start,
|
||||||
|
end: ingredient.end,
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
utensils: {
|
||||||
|
create: match.utensils.map((utensil) => ({
|
||||||
|
utensilId: utensil.utensilId,
|
||||||
|
start: utensil.start,
|
||||||
|
end: utensil.end,
|
||||||
|
})),
|
||||||
|
},
|
||||||
})),
|
})),
|
||||||
},
|
},
|
||||||
})),
|
})),
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ import {
|
||||||
getSources,
|
getSources,
|
||||||
getTechSteps,
|
getTechSteps,
|
||||||
getUnits,
|
getUnits,
|
||||||
|
getUtensils,
|
||||||
} from "./reference.service.js";
|
} from "./reference.service.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -55,6 +56,13 @@ referenceRouter.get(
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
referenceRouter.get(
|
||||||
|
"/utensils",
|
||||||
|
wrapAsyncHandler(async (_req, res) => {
|
||||||
|
res.status(200).json(await getUtensils());
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
referenceRouter.get(
|
referenceRouter.get(
|
||||||
"/sources",
|
"/sources",
|
||||||
wrapAsyncHandler(async (_req, res) => {
|
wrapAsyncHandler(async (_req, res) => {
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@ import type {
|
||||||
SourceView,
|
SourceView,
|
||||||
TechStepView,
|
TechStepView,
|
||||||
UnitView,
|
UnitView,
|
||||||
|
UtensilView,
|
||||||
} from "@batch-cooking/shared";
|
} from "@batch-cooking/shared";
|
||||||
import { prisma } from "../../db/prisma.js";
|
import { prisma } from "../../db/prisma.js";
|
||||||
|
|
||||||
|
|
@ -85,6 +86,19 @@ export async function getTechSteps(): Promise<TechStepView[]> {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* All reference cooking utensils, ordered by key (see {@link getDiets} for
|
||||||
|
* why) — small, static list (see `reference-seed-data.ts`'s `UTENSILS`),
|
||||||
|
* same bare `id`/`key` shape as {@link getTechSteps}.
|
||||||
|
*/
|
||||||
|
export async function getUtensils(): Promise<UtensilView[]> {
|
||||||
|
try {
|
||||||
|
return await prisma.utensil.findMany({ orderBy: { key: "asc" } });
|
||||||
|
} catch (err) {
|
||||||
|
throw err; // see getDiets()'s catch comment above
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Every implemented recipe source, ordered by name (not `key` — unlike
|
* Every implemented recipe source, ordered by name (not `key` — unlike
|
||||||
* every other reference catalog, `name` here *is* the display string a
|
* every other reference catalog, `name` here *is* the display string a
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,7 @@ import { RecipeSourceError } from "../../lib/recipe-sources/recipe-source-errors
|
||||||
import { getRecipeSource } from "../../lib/recipe-sources/recipe-source-registry.js";
|
import { getRecipeSource } from "../../lib/recipe-sources/recipe-source-registry.js";
|
||||||
import { getHouseSourceIds } from "../house/house.service.js";
|
import { getHouseSourceIds } from "../house/house.service.js";
|
||||||
import { createImportedRecipe } from "../recipe/recipe.service.js";
|
import { createImportedRecipe } from "../recipe/recipe.service.js";
|
||||||
import { getIngredients, getUnits } from "../reference/reference.service.js";
|
import { getIngredients, getUnits, getUtensils } from "../reference/reference.service.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Browsing, previewing, and importing a household's *enabled* external
|
* Browsing, previewing, and importing a household's *enabled* external
|
||||||
|
|
@ -190,9 +190,14 @@ export async function previewSourceItem(
|
||||||
unitCatalog,
|
unitCatalog,
|
||||||
adapter.locale,
|
adapter.locale,
|
||||||
);
|
);
|
||||||
const [ingredientViews, unitViews] = await Promise.all([getIngredients(), getUnits()]);
|
const [ingredientViews, unitViews, utensilViews] = await Promise.all([
|
||||||
|
getIngredients(),
|
||||||
|
getUnits(),
|
||||||
|
getUtensils(),
|
||||||
|
]);
|
||||||
const ingredientById = new Map(ingredientViews.map((view) => [view.id, view]));
|
const ingredientById = new Map(ingredientViews.map((view) => [view.id, view]));
|
||||||
const unitById = new Map(unitViews.map((view) => [view.id, view]));
|
const unitById = new Map(unitViews.map((view) => [view.id, view]));
|
||||||
|
const utensilById = new Map(utensilViews.map((view) => [view.id, view]));
|
||||||
|
|
||||||
// A source's raw ingredient lines aren't deduplicated by the matcher —
|
// A source's raw ingredient lines aren't deduplicated by the matcher —
|
||||||
// two different lines (e.g. "Egg Yolks"/"Eggs") can resolve to the same
|
// two different lines (e.g. "Egg Yolks"/"Eggs") can resolve to the same
|
||||||
|
|
@ -231,6 +236,26 @@ export async function previewSourceItem(
|
||||||
// comment) — always the classifier's own live match,
|
// comment) — always the classifier's own live match,
|
||||||
// never a correction, so always "auto".
|
// never a correction, so always "auto".
|
||||||
source: "auto",
|
source: "auto",
|
||||||
|
ingredients: match.ingredients.flatMap((mention) => {
|
||||||
|
const ingredient = ingredientById.get(mention.ingredientId);
|
||||||
|
// Same drift guard as `techStep` above — an ingredientId
|
||||||
|
// the matcher resolved but that's since vanished from the
|
||||||
|
// catalog is dropped rather than shown with a hole in it.
|
||||||
|
if (!ingredient) return [];
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
ingredient,
|
||||||
|
quantity: mention.quantity,
|
||||||
|
unit: mention.unitId !== null ? (unitById.get(mention.unitId) ?? null) : null,
|
||||||
|
start: mention.start,
|
||||||
|
end: mention.end,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}),
|
||||||
|
utensils: match.utensils.flatMap((mention) => {
|
||||||
|
const utensil = utensilById.get(mention.utensilId);
|
||||||
|
return utensil ? [{ utensil, start: mention.start, end: mention.end }] : [];
|
||||||
|
}),
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
: [];
|
: [];
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ import { expect } from "chai";
|
||||||
import { prisma } from "../../src/db/prisma.js";
|
import { prisma } from "../../src/db/prisma.js";
|
||||||
import {
|
import {
|
||||||
extractQuantity,
|
extractQuantity,
|
||||||
|
findIngredientMentions,
|
||||||
type IngredientMatchEntry,
|
type IngredientMatchEntry,
|
||||||
loadIngredientCatalog,
|
loadIngredientCatalog,
|
||||||
loadUnitCatalog,
|
loadUnitCatalog,
|
||||||
|
|
@ -266,6 +267,100 @@ describe("ingredient-matcher", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("findIngredientMentions", () => {
|
||||||
|
const butter: IngredientMatchEntry = { ingredientId: 1, label: "Butter" };
|
||||||
|
const flour: IngredientMatchEntry = { ingredientId: 2, label: "Flour" };
|
||||||
|
const egg: IngredientMatchEntry = { ingredientId: 3, label: "Egg" };
|
||||||
|
const catalog = [butter, flour, egg];
|
||||||
|
const gram: UnitMatchEntry = { unitId: 1, synonyms: ["g", "gram", "grams"] };
|
||||||
|
const unitCatalog = [gram];
|
||||||
|
|
||||||
|
it("finds a single mention with no quantity or unit", () => {
|
||||||
|
const text = "melt the butter";
|
||||||
|
const mentions = findIngredientMentions(text, catalog, unitCatalog);
|
||||||
|
expect(mentions).to.have.length(1);
|
||||||
|
const [mention] = mentions;
|
||||||
|
expect(mention?.ingredientId).to.equal(butter.ingredientId);
|
||||||
|
expect(text.slice(mention?.start, mention?.end)).to.equal("butter");
|
||||||
|
expect(mention?.quantity).to.equal(null);
|
||||||
|
expect(mention?.unitId).to.equal(null);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('resolves a quantity and unit glued directly to the ingredient ("200g butter")', () => {
|
||||||
|
const text = "add 200g butter";
|
||||||
|
const [mention] = findIngredientMentions(text, catalog, unitCatalog);
|
||||||
|
expect(mention?.ingredientId).to.equal(butter.ingredientId);
|
||||||
|
expect(mention?.quantity).to.equal(200);
|
||||||
|
expect(mention?.unitId).to.equal(gram.unitId);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("finds several mentions in reading order, non-overlapping", () => {
|
||||||
|
const text = "melt the butter then add the flour and an egg";
|
||||||
|
const mentions = findIngredientMentions(text, catalog, unitCatalog);
|
||||||
|
expect(mentions.map((mention) => mention.ingredientId)).to.deep.equal([
|
||||||
|
butter.ingredientId,
|
||||||
|
flour.ingredientId,
|
||||||
|
egg.ingredientId,
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is case- and accent-insensitive", () => {
|
||||||
|
const text = "MELT THE BUTTER";
|
||||||
|
const [mention] = findIngredientMentions(text, catalog, unitCatalog);
|
||||||
|
expect(mention?.ingredientId).to.equal(butter.ingredientId);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("ignores an unrelated number earlier in the text (e.g. an oven temperature)", () => {
|
||||||
|
const text = "preheat to 180 degrees then add the egg";
|
||||||
|
const [mention] = findIngredientMentions(text, catalog, unitCatalog);
|
||||||
|
expect(mention?.ingredientId).to.equal(egg.ingredientId);
|
||||||
|
expect(mention?.quantity).to.equal(null);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns an empty array when nothing in the catalog is mentioned", () => {
|
||||||
|
expect(findIngredientMentions("stir well", catalog, unitCatalog)).to.deep.equal([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns an empty array for empty text", () => {
|
||||||
|
expect(findIngredientMentions("", catalog, unitCatalog)).to.deep.equal([]);
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("locale: fr", () => {
|
||||||
|
const beurre: IngredientMatchEntry = { ingredientId: 10, label: "Beurre" };
|
||||||
|
const farine: IngredientMatchEntry = { ingredientId: 11, label: "Farine" };
|
||||||
|
const frCatalog = [beurre, farine];
|
||||||
|
const gramme: UnitMatchEntry = { unitId: 40, synonyms: ["g", "gr", "gramme", "grammes"] };
|
||||||
|
const cuillereASoupe: UnitMatchEntry = {
|
||||||
|
unitId: 41,
|
||||||
|
synonyms: ["cuillère à soupe", "cuillères à soupe", "càs"],
|
||||||
|
};
|
||||||
|
const frUnitCatalog = [gramme, cuillereASoupe];
|
||||||
|
|
||||||
|
it("resolves a quantity and unit before the ingredient, connected by 'de'", () => {
|
||||||
|
const text = "faire fondre 50g de beurre";
|
||||||
|
const [mention] = findIngredientMentions(text, frCatalog, frUnitCatalog, "fr");
|
||||||
|
expect(mention?.ingredientId).to.equal(beurre.ingredientId);
|
||||||
|
expect(mention?.quantity).to.equal(50);
|
||||||
|
expect(mention?.unitId).to.equal(gramme.unitId);
|
||||||
|
expect(text.slice(mention?.start, mention?.end)).to.equal("beurre");
|
||||||
|
});
|
||||||
|
|
||||||
|
it('resolves a multi-word unit connected by "d\'"', () => {
|
||||||
|
const text = "ajouter 2 cuillères à soupe de farine";
|
||||||
|
const [mention] = findIngredientMentions(text, frCatalog, frUnitCatalog, "fr");
|
||||||
|
expect(mention?.ingredientId).to.equal(farine.ingredientId);
|
||||||
|
expect(mention?.quantity).to.equal(2);
|
||||||
|
expect(mention?.unitId).to.equal(cuillereASoupe.unitId);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("is accent-insensitive", () => {
|
||||||
|
const text = "FAIRE FONDRE LE BEURRE";
|
||||||
|
const [mention] = findIngredientMentions(text, frCatalog, frUnitCatalog, "fr");
|
||||||
|
expect(mention?.ingredientId).to.equal(beurre.ingredientId);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("loadIngredientCatalog / loadUnitCatalog", () => {
|
describe("loadIngredientCatalog / loadUnitCatalog", () => {
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
await resetDatabase();
|
await resetDatabase();
|
||||||
|
|
|
||||||
|
|
@ -135,10 +135,19 @@ describe("tech-step-matcher", () => {
|
||||||
let meltId: number;
|
let meltId: number;
|
||||||
let boilId: number;
|
let boilId: number;
|
||||||
let chopId: number;
|
let chopId: number;
|
||||||
|
// Real seeded catalog entries that also happen to be mentioned by
|
||||||
|
// several fixtures below now that `matchTechStepSpans` also resolves
|
||||||
|
// ingredient/utensil metadata — see `matchTechStepSpans`'s own describe
|
||||||
|
// block for where each of these gets used.
|
||||||
|
let panId: number;
|
||||||
|
let butterId: number;
|
||||||
|
let onionId: number;
|
||||||
|
let walnutsId: number;
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
await resetDatabase();
|
await resetDatabase();
|
||||||
const [simmer, cook, bake, preheat, melt, boil, chop] = await Promise.all([
|
const [simmer, cook, bake, preheat, melt, boil, chop, pan, butter, onion, walnuts] =
|
||||||
|
await Promise.all([
|
||||||
prisma.techStep.findFirstOrThrow({ where: { key: "simmer" } }),
|
prisma.techStep.findFirstOrThrow({ where: { key: "simmer" } }),
|
||||||
prisma.techStep.findFirstOrThrow({ where: { key: "cook" } }),
|
prisma.techStep.findFirstOrThrow({ where: { key: "cook" } }),
|
||||||
prisma.techStep.findFirstOrThrow({ where: { key: "bake" } }),
|
prisma.techStep.findFirstOrThrow({ where: { key: "bake" } }),
|
||||||
|
|
@ -146,6 +155,15 @@ describe("tech-step-matcher", () => {
|
||||||
prisma.techStep.findFirstOrThrow({ where: { key: "melt" } }),
|
prisma.techStep.findFirstOrThrow({ where: { key: "melt" } }),
|
||||||
prisma.techStep.findFirstOrThrow({ where: { key: "boil" } }),
|
prisma.techStep.findFirstOrThrow({ where: { key: "boil" } }),
|
||||||
prisma.techStep.findFirstOrThrow({ where: { key: "chop" } }),
|
prisma.techStep.findFirstOrThrow({ where: { key: "chop" } }),
|
||||||
|
prisma.utensil.findFirstOrThrow({ where: { key: "pan" } }),
|
||||||
|
prisma.ingredient.findFirstOrThrow({ where: { key: "butter" } }),
|
||||||
|
prisma.ingredient.findFirstOrThrow({ where: { key: "onion" } }),
|
||||||
|
// "Noix" (walnuts) — turns out to also be a real seeded ingredient
|
||||||
|
// label, and "noix" is literally the French word for "a pat of
|
||||||
|
// butter" ("une noix de beurre") used in one of the fixtures
|
||||||
|
// below, so it's a genuine (if slightly comical) second match
|
||||||
|
// alongside "beurre" in that clause, not a fixture bug.
|
||||||
|
prisma.ingredient.findFirstOrThrow({ where: { key: "walnuts" } }),
|
||||||
]);
|
]);
|
||||||
simmerId = simmer.id;
|
simmerId = simmer.id;
|
||||||
cookId = cook.id;
|
cookId = cook.id;
|
||||||
|
|
@ -154,6 +172,10 @@ describe("tech-step-matcher", () => {
|
||||||
meltId = melt.id;
|
meltId = melt.id;
|
||||||
boilId = boil.id;
|
boilId = boil.id;
|
||||||
chopId = chop.id;
|
chopId = chop.id;
|
||||||
|
panId = pan.id;
|
||||||
|
butterId = butter.id;
|
||||||
|
onionId = onion.id;
|
||||||
|
walnutsId = walnuts.id;
|
||||||
});
|
});
|
||||||
|
|
||||||
after(async () => {
|
after(async () => {
|
||||||
|
|
@ -255,7 +277,15 @@ describe("tech-step-matcher", () => {
|
||||||
const text = "Faire mijoter à feu doux";
|
const text = "Faire mijoter à feu doux";
|
||||||
const result = await techStepClassifier.matchTechStepSpans(text, "fr");
|
const result = await techStepClassifier.matchTechStepSpans(text, "fr");
|
||||||
expect(result).to.deep.equal([
|
expect(result).to.deep.equal([
|
||||||
{ techStepId: simmerId, start: 6, end: 13, contextStart: 0, contextEnd: text.length },
|
{
|
||||||
|
techStepId: simmerId,
|
||||||
|
start: 6,
|
||||||
|
end: 13,
|
||||||
|
contextStart: 0,
|
||||||
|
contextEnd: text.length,
|
||||||
|
ingredients: [],
|
||||||
|
utensils: [],
|
||||||
|
},
|
||||||
]);
|
]);
|
||||||
expect(text.slice(6, 13).toLowerCase()).to.equal("mijoter");
|
expect(text.slice(6, 13).toLowerCase()).to.equal("mijoter");
|
||||||
});
|
});
|
||||||
|
|
@ -305,6 +335,12 @@ describe("tech-step-matcher", () => {
|
||||||
end: 21,
|
end: 21,
|
||||||
contextStart: 0,
|
contextStart: 0,
|
||||||
contextEnd: 22,
|
contextEnd: 22,
|
||||||
|
// "poêle" (the pan) sits inside this very clause — a separate
|
||||||
|
// utensil mention from `preheat`'s own "poêle chaude" keyword
|
||||||
|
// span above, found by the intent service's *other* PhraseMatcher
|
||||||
|
// (see `IntentServiceEntity.kind`).
|
||||||
|
ingredients: [],
|
||||||
|
utensils: [{ utensilId: panId, start: 9, end: 14 }],
|
||||||
});
|
});
|
||||||
expect(result[1]).to.deep.equal({
|
expect(result[1]).to.deep.equal({
|
||||||
techStepId: meltId,
|
techStepId: meltId,
|
||||||
|
|
@ -312,6 +348,15 @@ describe("tech-step-matcher", () => {
|
||||||
end: 37,
|
end: 37,
|
||||||
contextStart: 22,
|
contextStart: 22,
|
||||||
contextEnd: text.length,
|
contextEnd: text.length,
|
||||||
|
// Two mentions in this clause: "noix" (walnuts — also a real
|
||||||
|
// seeded ingredient, and literally the French word this phrase
|
||||||
|
// uses for "a pat of [butter]") *and* "beurre" itself, in
|
||||||
|
// reading order.
|
||||||
|
ingredients: [
|
||||||
|
{ ingredientId: walnutsId, start: 42, end: 46, quantity: null, unitId: null },
|
||||||
|
{ ingredientId: butterId, start: 50, end: 56, quantity: null, unitId: null },
|
||||||
|
],
|
||||||
|
utensils: [],
|
||||||
});
|
});
|
||||||
expect(text.slice(result[0].start, result[0].end)).to.equal("poêle chaude");
|
expect(text.slice(result[0].start, result[0].end)).to.equal("poêle chaude");
|
||||||
expect(text.slice(result[0].contextStart, result[0].contextEnd)).to.equal(
|
expect(text.slice(result[0].contextStart, result[0].contextEnd)).to.equal(
|
||||||
|
|
@ -333,6 +378,15 @@ describe("tech-step-matcher", () => {
|
||||||
end: text.length,
|
end: text.length,
|
||||||
contextStart: 0,
|
contextStart: 0,
|
||||||
contextEnd: text.length,
|
contextEnd: text.length,
|
||||||
|
// "beurre" and "poêle" are both mentioned in this same
|
||||||
|
// anchor-less clause (there's no literal `melt` keyword here at
|
||||||
|
// all — the whole point of this test, see its own title) —
|
||||||
|
// still resolved, since ingredient/utensil scanning doesn't
|
||||||
|
// depend on the clause having a technique anchor of its own.
|
||||||
|
ingredients: [
|
||||||
|
{ ingredientId: butterId, start: 18, end: 24, quantity: null, unitId: null },
|
||||||
|
],
|
||||||
|
utensils: [{ utensilId: panId, start: 45, end: 50 }],
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
@ -341,10 +395,33 @@ describe("tech-step-matcher", () => {
|
||||||
const text = "Chop the onions finely";
|
const text = "Chop the onions finely";
|
||||||
const result = await techStepClassifier.matchTechStepSpans(text, "en");
|
const result = await techStepClassifier.matchTechStepSpans(text, "en");
|
||||||
expect(result).to.deep.equal([
|
expect(result).to.deep.equal([
|
||||||
{ techStepId: chopId, start: 0, end: 4, contextStart: 0, contextEnd: text.length },
|
{
|
||||||
|
techStepId: chopId,
|
||||||
|
start: 0,
|
||||||
|
end: 4,
|
||||||
|
contextStart: 0,
|
||||||
|
contextEnd: text.length,
|
||||||
|
ingredients: [
|
||||||
|
{ ingredientId: onionId, start: 9, end: 15, quantity: null, unitId: null },
|
||||||
|
],
|
||||||
|
utensils: [],
|
||||||
|
},
|
||||||
]);
|
]);
|
||||||
expect(text.slice(0, 4)).to.equal("Chop");
|
expect(text.slice(0, 4)).to.equal("Chop");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Quantity+unit extraction itself (the leading-number-before-a-mention
|
||||||
|
// heuristic) is covered in full, deterministically, by
|
||||||
|
// `findIngredientMentions`'s own tests (`ingredient-matcher.test.ts`)
|
||||||
|
// — deliberately not re-exercised here through a brand-new invented
|
||||||
|
// sentence: a novel combination of words the real `textcat` (trained
|
||||||
|
// on a fixed, finite corpus, see `training_data.py`) has never seen
|
||||||
|
// together can land on a confidently-wrong technique for reasons
|
||||||
|
// that have nothing to do with this file's own logic, making such a
|
||||||
|
// test flaky against corpus/threshold changes rather than a
|
||||||
|
// trustworthy regression guard. The two tests above/below already
|
||||||
|
// demonstrate technique+ingredient+utensil co-occurring in one
|
||||||
|
// clause using sentences already proven reliable by this suite.
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -98,7 +98,14 @@ describe("Recipe tech-step corrections", () => {
|
||||||
// away — not just the permanent audit record above (see
|
// away — not just the permanent audit record above (see
|
||||||
// `applyManualCorrection`, `recipe-tech-step-correction.service.ts`).
|
// `applyManualCorrection`, `recipe-tech-step-correction.service.ts`).
|
||||||
expect(res.body.techSteps).to.deep.equal([
|
expect(res.body.techSteps).to.deep.equal([
|
||||||
{ techStep: { id: simmerId, key: "simmer" }, start: 6, end: 13, source: "manual" },
|
{
|
||||||
|
techStep: { id: simmerId, key: "simmer" },
|
||||||
|
start: 6,
|
||||||
|
end: 13,
|
||||||
|
source: "manual",
|
||||||
|
ingredients: [],
|
||||||
|
utensils: [],
|
||||||
|
},
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -124,7 +131,14 @@ describe("Recipe tech-step corrections", () => {
|
||||||
// Still exactly one entry — the relabel updated the existing row
|
// Still exactly one entry — the relabel updated the existing row
|
||||||
// rather than adding a second one alongside it.
|
// rather than adding a second one alongside it.
|
||||||
expect(res.body.techSteps).to.deep.equal([
|
expect(res.body.techSteps).to.deep.equal([
|
||||||
{ techStep: { id: boilId, key: "boil" }, start: 6, end: 13, source: "manual" },
|
{
|
||||||
|
techStep: { id: boilId, key: "boil" },
|
||||||
|
start: 6,
|
||||||
|
end: 13,
|
||||||
|
source: "manual",
|
||||||
|
ingredients: [],
|
||||||
|
utensils: [],
|
||||||
|
},
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ import request from "supertest";
|
||||||
import { createApp } from "../src/app.js";
|
import { createApp } from "../src/app.js";
|
||||||
import { prisma } from "../src/db/prisma.js";
|
import { prisma } from "../src/db/prisma.js";
|
||||||
import { syncRecipeSources } from "../src/db/recipe-source-sync.js";
|
import { syncRecipeSources } from "../src/db/recipe-source-sync.js";
|
||||||
import { seedReferenceData, TECH_STEPS } from "../src/db/reference-seed-data.js";
|
import { seedReferenceData, TECH_STEPS, UTENSILS } from "../src/db/reference-seed-data.js";
|
||||||
import type { RecipeSourceAdapter } from "../src/lib/recipe-sources/recipe-source-adapter.js";
|
import type { RecipeSourceAdapter } from "../src/lib/recipe-sources/recipe-source-adapter.js";
|
||||||
import {
|
import {
|
||||||
clearRecipeSources,
|
clearRecipeSources,
|
||||||
|
|
@ -161,6 +161,31 @@ describe("Reference data", () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("GET /reference/utensils", () => {
|
||||||
|
it("returns the seeded utensils, no session required", async () => {
|
||||||
|
const res = await request(app).get("/reference/utensils");
|
||||||
|
|
||||||
|
expect(res.status).to.equal(200);
|
||||||
|
expect(res.body).to.have.length(UTENSILS.length);
|
||||||
|
expect(res.body.map((u: { key: string }) => u.key)).to.include("pan");
|
||||||
|
expect(res.body[0]).to.have.keys(["id", "key"]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("orders utensils alphabetically by key", async () => {
|
||||||
|
const res = await request(app).get("/reference/utensils");
|
||||||
|
|
||||||
|
const keys = res.body.map((u: { key: string }) => u.key);
|
||||||
|
expect(keys).to.deep.equal([...keys].sort());
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reseeding is idempotent — no duplicate utensils", async () => {
|
||||||
|
await seedReferenceData(prisma);
|
||||||
|
|
||||||
|
const res = await request(app).get("/reference/utensils");
|
||||||
|
expect(res.body).to.have.length(UTENSILS.length);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("GET /reference/sources", () => {
|
describe("GET /reference/sources", () => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
clearRecipeSources();
|
clearRecipeSources();
|
||||||
|
|
|
||||||
|
|
@ -475,6 +475,38 @@
|
||||||
"toast": "Torréfier",
|
"toast": "Torréfier",
|
||||||
"zest": "Zester"
|
"zest": "Zester"
|
||||||
},
|
},
|
||||||
|
"utensils": {
|
||||||
|
"pan": "Poêle",
|
||||||
|
"saucepan": "Casserole",
|
||||||
|
"pot": "Marmite",
|
||||||
|
"knife": "Couteau",
|
||||||
|
"whisk": "Fouet",
|
||||||
|
"bowl": "Saladier",
|
||||||
|
"bakingSheet": "Plaque de cuisson",
|
||||||
|
"mold": "Moule",
|
||||||
|
"colander": "Passoire",
|
||||||
|
"cuttingBoard": "Planche à découper",
|
||||||
|
"oven": "Four",
|
||||||
|
"blender": "Blender",
|
||||||
|
"mixer": "Batteur",
|
||||||
|
"spatula": "Spatule",
|
||||||
|
"ladle": "Louche",
|
||||||
|
"grater": "Râpe",
|
||||||
|
"rollingPin": "Rouleau à pâtisserie",
|
||||||
|
"lid": "Couvercle",
|
||||||
|
"tongs": "Pince de cuisine",
|
||||||
|
"peeler": "Économe",
|
||||||
|
"sieve": "Tamis",
|
||||||
|
"foodProcessor": "Robot ménager",
|
||||||
|
"steamerBasket": "Panier vapeur",
|
||||||
|
"skewer": "Brochette",
|
||||||
|
"pastryBrush": "Pinceau de cuisine",
|
||||||
|
"ramekin": "Ramequin",
|
||||||
|
"dish": "Plat",
|
||||||
|
"wok": "Wok",
|
||||||
|
"thermometer": "Thermomètre",
|
||||||
|
"mandoline": "Mandoline"
|
||||||
|
},
|
||||||
"allergens": {
|
"allergens": {
|
||||||
"gluten": "Gluten",
|
"gluten": "Gluten",
|
||||||
"crustaceans": "Crustacés",
|
"crustaceans": "Crustacés",
|
||||||
|
|
|
||||||
|
|
@ -94,15 +94,17 @@ services:
|
||||||
# This service trains itself from scratch on every start (no model
|
# This service trains itself from scratch on every start (no model
|
||||||
# ever persisted to disk, see its own README) — `/health` only
|
# ever persisted to disk, see its own README) — `/health` only
|
||||||
# returns 200 once that's done, not just once the base spaCy models
|
# returns 200 once that's done, not just once the base spaCy models
|
||||||
# are loaded. Measured at ~335s per locale (~670s for fr+en combined)
|
# are loaded. Measured at ~540s (fr) / ~390s (en), ~930s combined,
|
||||||
# against the current ~74-technique corpus, trained on each
|
# against the current ~74-technique corpus — each technique now has
|
||||||
# technique's own synonyms in addition to its example phrases
|
# the *same* number of `utterances` per locale as every other
|
||||||
# (`intent_service/locale_pipeline.py`'s `_TRAINING_ITERATIONS`) —
|
# (equalized to the corpus's own pre-existing max, 7/5 — see
|
||||||
# `start_period` generous enough that failing checks during that
|
# `training_data.py`'s own doc comment for why a flat, larger target
|
||||||
# whole window never count against `retries` (which would otherwise
|
# like 20 was tried and reverted) — `start_period` generous enough
|
||||||
# flip this container to "unhealthy" mid-training, blocking `app`'s
|
# that failing checks during that whole window never count against
|
||||||
# own `depends_on: condition: service_healthy` indefinitely).
|
# `retries` (which would otherwise flip this container to
|
||||||
start_period: 900s
|
# "unhealthy" mid-training, blocking `app`'s own `depends_on:
|
||||||
|
# condition: service_healthy` indefinitely).
|
||||||
|
start_period: 1200s
|
||||||
|
|
||||||
# Deliberately its own image, not built into `app`'s (see
|
# Deliberately its own image, not built into `app`'s (see
|
||||||
# services/tech-step-llm-worker/Dockerfile's own doc comment) — a
|
# services/tech-step-llm-worker/Dockerfile's own doc comment) — a
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,11 @@
|
||||||
import type { AllergyView, DietView, IngredientView, TechStepView, UnitView } from "./reference.js";
|
import type {
|
||||||
|
AllergyView,
|
||||||
|
DietView,
|
||||||
|
IngredientView,
|
||||||
|
TechStepView,
|
||||||
|
UnitView,
|
||||||
|
UtensilView,
|
||||||
|
} from "./reference.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Who can *read* a recipe — mirrors `RecipeVisibility` in schema.prisma.
|
* Who can *read* a recipe — mirrors `RecipeVisibility` in schema.prisma.
|
||||||
|
|
@ -49,6 +56,11 @@ export interface RecipeIngredientView {
|
||||||
* immediately (`recipe-tech-step-correction.service.ts`'s
|
* immediately (`recipe-tech-step-correction.service.ts`'s
|
||||||
* `applyManualCorrection`). `StepDescription.tsx` renders the two with a
|
* `applyManualCorrection`). `StepDescription.tsx` renders the two with a
|
||||||
* different highlight color so a viewer can tell which is which.
|
* different highlight color so a viewer can tell which is which.
|
||||||
|
*
|
||||||
|
* `ingredients`/`utensils` are the metadata found in this technique's own
|
||||||
|
* clause (see `tech-step-matcher.ts`'s `TechStepMatch` — same source data,
|
||||||
|
* just resolved to full reference views here instead of bare ids) — `[]`
|
||||||
|
* when nothing was mentioned alongside this technique.
|
||||||
*/
|
*/
|
||||||
export interface StepTechStepView {
|
export interface StepTechStepView {
|
||||||
techStep: TechStepView;
|
techStep: TechStepView;
|
||||||
|
|
@ -57,6 +69,32 @@ export interface StepTechStepView {
|
||||||
contextStart?: number;
|
contextStart?: number;
|
||||||
contextEnd?: number;
|
contextEnd?: number;
|
||||||
source: "auto" | "manual";
|
source: "auto" | "manual";
|
||||||
|
ingredients: StepTechStepIngredientView[];
|
||||||
|
utensils: StepTechStepUtensilView[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An ingredient mentioned in the same clause as a detected technique (see
|
||||||
|
* {@link StepTechStepView.ingredients}) — `quantity`/`unit` are `null` when
|
||||||
|
* none was recognized immediately before the mention (e.g. "ajouter le
|
||||||
|
* sel"), same "best-effort, not always present" contract as
|
||||||
|
* `tech-step-matcher.ts`'s `IngredientMention`. `start`/`end` are the
|
||||||
|
* mention's own span in the step's `description`, same `[start, end)`
|
||||||
|
* convention as {@link StepTechStepView.start}.
|
||||||
|
*/
|
||||||
|
export interface StepTechStepIngredientView {
|
||||||
|
ingredient: IngredientView;
|
||||||
|
quantity: number | null;
|
||||||
|
unit: UnitView | null;
|
||||||
|
start: number;
|
||||||
|
end: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A utensil mentioned in the same clause as a detected technique (see {@link StepTechStepView.utensils}). */
|
||||||
|
export interface StepTechStepUtensilView {
|
||||||
|
utensil: UtensilView;
|
||||||
|
start: number;
|
||||||
|
end: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -210,6 +210,24 @@ export interface TechStepView {
|
||||||
key: string;
|
key: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A cooking utensil, as returned by `GET /reference/utensils` — reference
|
||||||
|
* data (`Utensil`, seeded via `reference-seed-data.ts`'s `UTENSILS`), same
|
||||||
|
* bare `id`+`key` shape and static/non-administrable status as
|
||||||
|
* {@link TechStepView}. Detected in a step's free text the same way
|
||||||
|
* techniques are (see `StepTechStepUtensilView`), but via a static
|
||||||
|
* `PhraseMatcher` rather than a trained classifier — see
|
||||||
|
* `services/tech-step-intent-service`'s `utensil_vocabulary.py`.
|
||||||
|
*
|
||||||
|
* `key` is a stable English camelCase uid (e.g. `"pan"`), not a display
|
||||||
|
* label — resolved via `t(\`catalog.utensils.${key}\`)`, same as
|
||||||
|
* {@link TechStepView.key}.
|
||||||
|
*/
|
||||||
|
export interface UtensilView {
|
||||||
|
id: number;
|
||||||
|
key: string;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* An implemented recipe source, as returned by `GET /reference/sources` —
|
* An implemented recipe source, as returned by `GET /reference/sources` —
|
||||||
* reference data (`Source`, kept in sync with the adapter registry by
|
* reference data (`Source`, kept in sync with the adapter registry by
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,12 @@ pipeline `node-nlp` qui vivait dans `apps/api`
|
||||||
de la technique qu'une clause de texte *signifie*, entraîné sur les
|
de la technique qu'une clause de texte *signifie*, entraîné sur les
|
||||||
`utterances` de chaque technique (y compris des paraphrases n'utilisant
|
`utterances` de chaque technique (y compris des paraphrases n'utilisant
|
||||||
jamais le mot-clé lui-même).
|
jamais le mot-clé lui-même).
|
||||||
|
3. **NER par phrases, ustensiles** (`spacy.matcher.PhraseMatcher`, second
|
||||||
|
matcher indépendant) — trouve les mentions d'un ustensile de cuisine
|
||||||
|
(`intent_service/utensil_vocabulary.py`, `UTENSIL_VOCABULARY`), sans
|
||||||
|
`textcat` associé : contrairement à une technique, un ustensile mentionné
|
||||||
|
n'a pas besoin d'être interprété selon le contexte. Renvoyé dans la même
|
||||||
|
liste `entities` que les techniques, discriminé par `kind`.
|
||||||
|
|
||||||
Basé sur **spaCy** (`fr_core_news_md`/`en_core_web_md`) plutôt que node-nlp —
|
Basé sur **spaCy** (`fr_core_news_md`/`en_core_web_md`) plutôt que node-nlp —
|
||||||
écosystème NLP plus robuste/maintenu, avec l'ambition à terme (hors scope de
|
écosystème NLP plus robuste/maintenu, avec l'ambition à terme (hors scope de
|
||||||
|
|
@ -34,7 +40,18 @@ de `POST /v1/train`, supprimé).
|
||||||
Workflow mainteneur pour changer le corpus :
|
Workflow mainteneur pour changer le corpus :
|
||||||
|
|
||||||
1. Éditer `intent_service/training_data.py` à la main (informé par le
|
1. Éditer `intent_service/training_data.py` à la main (informé par le
|
||||||
rapport de `apps/api/src/scripts/list-pending-training-suggestions.ts`).
|
rapport de `apps/api/src/scripts/list-pending-training-suggestions.ts`)
|
||||||
|
pour une technique, ou `intent_service/utensil_vocabulary.py` pour un
|
||||||
|
ustensile (pas de rapport équivalent pour ce dernier — pas de mécanisme
|
||||||
|
de correction utilisateur sur les ustensiles aujourd'hui). Chaque
|
||||||
|
technique doit garder le même nombre d'`utterances` que les autres, par
|
||||||
|
locale (voir `training_data.py`'s own doc comment) — une technique
|
||||||
|
ajoutée avec moins que le max courant, exécuter `augment_utterances.py`
|
||||||
|
(racine de ce service) pour rééquilibrer, puis **impérativement**
|
||||||
|
relancer l'étape 3 ci-dessous avant de committer : chaque tentative
|
||||||
|
passée d'élargir ce corpus (voir l'historique Git de
|
||||||
|
`training_data.py`) a dû être ajustée ou annulée après coup faute
|
||||||
|
d'avoir vérifié le F1 avant de pousser.
|
||||||
2. **Redémarrer ce service** (`docker compose restart tech-step-intent-service`,
|
2. **Redémarrer ce service** (`docker compose restart tech-step-intent-service`,
|
||||||
ou simplement redéployer) — le nouveau corpus n'a d'effet qu'une fois
|
ou simplement redéployer) — le nouveau corpus n'a d'effet qu'une fois
|
||||||
réentraîné au démarrage, contrairement à l'ancienne version qui pouvait
|
réentraîné au démarrage, contrairement à l'ancienne version qui pouvait
|
||||||
|
|
@ -61,7 +78,10 @@ Voir `intent_service/schemas.py` pour le détail exact. En résumé :
|
||||||
entièrement prêt : modèles spaCy de base chargés **et** les deux locales
|
entièrement prêt : modèles spaCy de base chargés **et** les deux locales
|
||||||
entraînées (pas de lazy-load, voir `intent_service/main.py`) — voir
|
entraînées (pas de lazy-load, voir `intent_service/main.py`) — voir
|
||||||
"Temps de démarrage" plus bas pour ce que ça implique en pratique.
|
"Temps de démarrage" plus bas pour ce que ça implique en pratique.
|
||||||
- `POST /v1/process` — `{ locale, text }` → `{ entities: [{ uid, start, end }], intent, score }`.
|
- `POST /v1/process` — `{ locale, text }` → `{ entities: [{ uid, start, end, kind }], intent, score }`,
|
||||||
|
`kind` valant `"technique"` ou `"utensil"` selon le `PhraseMatcher` qui a
|
||||||
|
trouvé la mention (voir point 3 ci-dessus). `apps/api`'s `tech-step-matcher.ts`
|
||||||
|
filtre par `kind` pour savoir laquelle des deux résoudre (`TechStep`/`Utensil`).
|
||||||
|
|
||||||
`/v1/process` exige le header `X-Intent-Service-Secret` (voir
|
`/v1/process` exige le header `X-Intent-Service-Secret` (voir
|
||||||
`intent_service/security.py`), qui doit matcher `INTENT_SERVICE_SECRET`
|
`intent_service/security.py`), qui doit matcher `INTENT_SERVICE_SECRET`
|
||||||
|
|
@ -73,11 +93,14 @@ côté `apps/api`.
|
||||||
node-nlp (entraînement quasi instantané), entraîner le `textcat` sur le
|
node-nlp (entraînement quasi instantané), entraîner le `textcat` sur le
|
||||||
corpus réel (~74 techniques, chaque technique entraînée sur ses `synonyms`
|
corpus réel (~74 techniques, chaque technique entraînée sur ses `synonyms`
|
||||||
en plus de ses `utterances` — voir `locale_pipeline.py`) prend de l'ordre
|
en plus de ses `utterances` — voir `locale_pipeline.py`) prend de l'ordre
|
||||||
de 335 secondes par locale (mesuré localement, sans GPU), donc environ 670
|
de 540 secondes pour `fr` / 390 secondes pour `en` (mesuré localement,
|
||||||
secondes (~11 minutes) pour `fr`+`en` combinés à chaque démarrage du
|
sans GPU), donc environ 930 secondes (~15-16 minutes) pour `fr`+`en`
|
||||||
process. `docker-compose.yml` et
|
combinés à chaque démarrage du process — chaque technique a désormais le
|
||||||
`.github/workflows/ci.yml` ont un `start_period`/timeout d'attente
|
même nombre d'`utterances` par locale (voir `training_data.py`'s own doc
|
||||||
généreux pour ça — voir leurs propres commentaires. C'est un compromis
|
comment), légèrement plus qu'avant ce rééquilibrage. `docker-compose.yml`
|
||||||
|
et `.github/workflows/ci.yml` ont un `start_period`/timeout d'attente
|
||||||
|
généreux pour ça (`1200s`) — voir leurs propres commentaires. C'est un
|
||||||
|
compromis
|
||||||
assumé, pas un défaut de configuration à corriger : moins d'itérations
|
assumé, pas un défaut de configuration à corriger : moins d'itérations
|
||||||
entraîne plus vite mais laisse des verdicts corrects sous
|
entraîne plus vite mais laisse des verdicts corrects sous
|
||||||
`CONFIDENCE_THRESHOLD` (voir le commentaire de cette constante,
|
`CONFIDENCE_THRESHOLD` (voir le commentaire de cette constante,
|
||||||
|
|
@ -96,7 +119,7 @@ et son output complets, `pipeline_registry.py` journalise le déroulement de
|
||||||
l'entraînement au démarrage :
|
l'entraînement au démarrage :
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{"timestamp": "...", "level": "info", "message": "tech-step NLP process", "locale": "fr", "text": "faire fondre le beurre", "entities": [{"uid": "melt", "start": 6, "end": 13}], "intent": "melt", "score": 0.93}
|
{"timestamp": "...", "level": "info", "message": "tech-step NLP process", "locale": "fr", "text": "faire fondre le beurre", "entities": [{"uid": "melt", "start": 6, "end": 13, "kind": "technique"}], "intent": "melt", "score": 0.93}
|
||||||
```
|
```
|
||||||
|
|
||||||
Le chatter interne de spaCy (`"spacy"` logger — chargement de vocabulaire,
|
Le chatter interne de spaCy (`"spacy"` logger — chargement de vocabulaire,
|
||||||
|
|
@ -138,6 +161,9 @@ uv run pytest
|
||||||
exacts et d'insensibilité accents/casse de
|
exacts et d'insensibilité accents/casse de
|
||||||
`apps/api/test/recipe-matching/tech-step-matcher.test.ts` — le point de
|
`apps/api/test/recipe-matching/tech-step-matcher.test.ts` — le point de
|
||||||
fidélité le plus critique de ce service (voir le plan de migration).
|
fidélité le plus critique de ce service (voir le plan de migration).
|
||||||
|
`tests/test_utensil_matching.py` couvre le second `PhraseMatcher`
|
||||||
|
(ustensiles) de la même façon, contre le vocabulaire réel (statique, pas
|
||||||
|
besoin d'un jeu de test dédié comme pour les techniques).
|
||||||
`tests/conftest.py`'s fixture `client` (scope "session") ne s'entraîne
|
`tests/conftest.py`'s fixture `client` (scope "session") ne s'entraîne
|
||||||
qu'une seule fois pour toute la suite — c'est *le vrai corpus complet*,
|
qu'une seule fois pour toute la suite — c'est *le vrai corpus complet*,
|
||||||
pas un jeu jouet, donc la première utilisation de cette fixture prend le
|
pas un jeu jouet, donc la première utilisation de cette fixture prend le
|
||||||
|
|
@ -151,7 +177,7 @@ vraie instance de ce service tournant (voir `apps/api/.env.test`), conforme
|
||||||
|
|
||||||
## Limitations connues
|
## Limitations connues
|
||||||
|
|
||||||
- **Démarrage lent** (~11 minutes) — voir "Temps de démarrage" ci-dessus.
|
- **Démarrage lent** (~15-16 minutes) — voir "Temps de démarrage" ci-dessus.
|
||||||
Une optimisation possible non explorée : parallélisation de
|
Une optimisation possible non explorée : parallélisation de
|
||||||
l'entraînement `fr`/`en` (actuellement séquentiel,
|
l'entraînement `fr`/`en` (actuellement séquentiel,
|
||||||
`PipelineRegistry.initialize`).
|
`PipelineRegistry.initialize`).
|
||||||
|
|
|
||||||
282
services/tech-step-intent-service/augment_utterances.py
Normal file
282
services/tech-step-intent-service/augment_utterances.py
Normal file
|
|
@ -0,0 +1,282 @@
|
||||||
|
"""Maintainer script — equalizes every technique's `utterances` count
|
||||||
|
(per locale) to the corpus's own current maximum for that locale, never a
|
||||||
|
fixed number picked in the abstract. Preserves every existing utterance,
|
||||||
|
synonym, and comment verbatim; only ever *adds*, never rewrites or removes.
|
||||||
|
|
||||||
|
**Why "equalize to the current max", not "pad everyone to 20"** — this
|
||||||
|
script's own history: three earlier attempts forced every technique up to
|
||||||
|
a flat 20 `utterances`/locale (12-17 new ones per technique on average).
|
||||||
|
All three measurably *failed*
|
||||||
|
`test/recipe-matching/tech-step-eval.test.ts`'s F1 >= 0.8 regression gate
|
||||||
|
(0.7999 -> 0.791 -> 0.744, each attempt worse than the last), regardless of
|
||||||
|
whether the added content was mostly generic modal-frame padding ("il
|
||||||
|
faut ...") or mostly synonym substitution. The common factor across all
|
||||||
|
three wasn't *how* the filler was generated, it was *how much*: this
|
||||||
|
corpus's real per-technique max was only 7 (fr) / 5 (en) before any of
|
||||||
|
this — forcing every technique up to 20 meant most of them tripled or
|
||||||
|
quadrupled in size on synthetic content alone, which measurably hurt
|
||||||
|
inter-class separability more than it helped. Equalizing to the corpus's
|
||||||
|
*own* current max instead means at most a few new utterances per
|
||||||
|
technique (most need 1-4), which is a small enough addition to plausibly
|
||||||
|
preserve the F1 gate while still satisfying "same amount of signal per
|
||||||
|
class" (the actual goal — consistent detection quality across techniques,
|
||||||
|
not a specific round number).
|
||||||
|
|
||||||
|
**Generation strategy** — synonym substitution first (see
|
||||||
|
`_synonym_variants`): for every existing utterance whose leading phrase
|
||||||
|
exactly matches one of the technique's own `synonyms`, swap in every
|
||||||
|
*other* synonym from the same list (e.g. `melt`'s "faire fondre le
|
||||||
|
beurre" -> "liquéfier le beurre") — genuinely technique-distinguishing
|
||||||
|
vocabulary, not filler shared across every class. A technique whose
|
||||||
|
`synonyms` only ever appear *mid-sentence* (the "cut style" techniques —
|
||||||
|
`julienne`, `brunoise`, `mirepoix`, `paysanne`... — e.g. "couper les
|
||||||
|
carottes en julienne" doesn't *start* with any of `julienne`'s own
|
||||||
|
synonyms) has no leading-phrase match to substitute, so a small modal-frame
|
||||||
|
fallback (`_FR_FRAMES`/`_EN_FRAMES`, 2 per locale — much smaller than the
|
||||||
|
12/10 used in the failed 20-target attempts) closes the remainder. Safe at
|
||||||
|
this scale specifically *because* the gap being closed is small (equalizing
|
||||||
|
to the corpus's own current max, 1-4 utterances short per technique, not
|
||||||
|
13-17) — see this module's own doc comment above for why volume, not
|
||||||
|
generation method, was the real problem in every failed attempt.
|
||||||
|
|
||||||
|
Run from `services/tech-step-intent-service/` (this directory):
|
||||||
|
`./.venv/Scripts/python.exe augment_utterances.py`. Rewrites
|
||||||
|
`training_data.py` in place by textual splicing (AST only to *locate* each
|
||||||
|
`utterances=[...]` list's line range — never to regenerate the file). Safe
|
||||||
|
to re-run: a technique already at the current per-locale max is left
|
||||||
|
untouched, and the max itself is recomputed from the file's *current*
|
||||||
|
state each time (so re-running after a manual edit re-equalizes against
|
||||||
|
whatever the new max is, not a stale one).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import sys
|
||||||
|
|
||||||
|
SRC_PATH = "intent_service/training_data.py"
|
||||||
|
|
||||||
|
# Minimal fallback pool — only ever used for the small remainder synonym
|
||||||
|
# substitution can't reach (see this module's own doc comment for why 2,
|
||||||
|
# not the 12/10 tried in earlier, failed attempts).
|
||||||
|
_FR_FRAMES = ["il faut {u}", "veillez à {u}"]
|
||||||
|
_EN_FRAMES = ["make sure to {u}", "remember to {u}"]
|
||||||
|
|
||||||
|
|
||||||
|
def _is_fr_infinitive_led(u: str) -> bool:
|
||||||
|
first = u.split(" ", 1)[0].lower()
|
||||||
|
return first.endswith(("er", "ir", "re")) and len(first) > 2
|
||||||
|
|
||||||
|
|
||||||
|
_EN_VERB_WHITELIST = {
|
||||||
|
"make", "add", "pour", "mix", "stir", "cut", "place", "cover", "remove", "heat", "let",
|
||||||
|
"keep", "turn", "cook", "bake", "roast", "grill", "fry", "boil", "simmer", "whisk", "fold",
|
||||||
|
"chop", "mince", "peel", "drain", "season", "rest", "plate", "coat", "melt", "sauté", "saute",
|
||||||
|
"braise", "blanch", "marinate", "brown", "glaze", "thicken", "reduce", "dilute", "loosen",
|
||||||
|
"moisten", "sift", "toast", "zest", "scald", "pod", "shell", "hollow", "shock", "emulsify",
|
||||||
|
"decant", "dust", "sweat", "rub", "punch", "confit", "caramelize", "score", "line", "clarify",
|
||||||
|
"stew", "dice", "fillet", "proof", "poach", "pasteurize", "sterilize", "can", "preserve",
|
||||||
|
"tie", "truss", "baste", "spoon", "brush", "whip", "beat", "work", "sear", "flatten", "press",
|
||||||
|
"knead", "run", "cool", "warm", "combine", "blend", "arrange", "present", "sprinkle", "strain",
|
||||||
|
"separate", "bring", "grate", "continue", "deglaze", "scrape", "char", "break", "slice", "set",
|
||||||
|
"adjust", "switch", "secure", "mark", "butter", "crush", "julienne", "reheat", "smother",
|
||||||
|
"build", "scoop", "plunge", "increase", "pass", "collect", "have", "salt", "soak",
|
||||||
|
}
|
||||||
|
_EN_ADVERB_SKIP = {
|
||||||
|
"coarsely", "roughly", "finely", "quickly", "lightly", "briefly", "gently", "carefully",
|
||||||
|
"gradually", "very", "thoroughly", "evenly", "generously", "slowly", "thinly", "deep", "blind",
|
||||||
|
"dry",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _is_en_imperative_led(u: str) -> bool:
|
||||||
|
words = u.lower().replace(",", "").split()
|
||||||
|
if not words:
|
||||||
|
return False
|
||||||
|
first = words[0]
|
||||||
|
if first in _EN_VERB_WHITELIST:
|
||||||
|
return True
|
||||||
|
if first in _EN_ADVERB_SKIP and len(words) > 1:
|
||||||
|
return words[1] in _EN_VERB_WHITELIST
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _frame_variants(existing: list[str], frames: list[str], is_led) -> list[str]:
|
||||||
|
sources = [u for u in existing if is_led(u)]
|
||||||
|
if not sources:
|
||||||
|
return []
|
||||||
|
seen = set(existing)
|
||||||
|
out: list[str] = []
|
||||||
|
for frame in frames:
|
||||||
|
for u in sources:
|
||||||
|
candidate = frame.format(u=u)
|
||||||
|
if candidate in seen:
|
||||||
|
continue
|
||||||
|
seen.add(candidate)
|
||||||
|
out.append(candidate)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _synonym_variants(existing: list[str], synonyms: list[str], locale: str) -> list[str]:
|
||||||
|
"""Substitutes every *other* synonym in place of whichever synonym an
|
||||||
|
existing utterance's leading phrase exactly matches — see this
|
||||||
|
module's own doc comment for why this is the primary generation
|
||||||
|
strategy.
|
||||||
|
|
||||||
|
Both the matched *and* the replacement synonym must independently pass
|
||||||
|
`_is_fr_infinitive_led`/`_is_en_imperative_led` — a technique's
|
||||||
|
`synonyms` list mixes genuine verb forms ("mijoter", "frémir") with
|
||||||
|
noun/adjective phrases used the same way a keyword-matcher needs them
|
||||||
|
but never as a sentence's own leading verb ("à petit feu", "gros
|
||||||
|
bouillons", "huile de friture") — without this check, swapping the
|
||||||
|
verb "frémir" for the noun phrase "à petit feu" inside "laisser
|
||||||
|
frémir..." produces a syntactically broken sentence ("à petit feu
|
||||||
|
..."), not just a stylistically different one. Filtering the
|
||||||
|
replacement pool to the same grammatical shape as the ones this
|
||||||
|
function already accepts as *sources* keeps every substitution a
|
||||||
|
like-for-like swap."""
|
||||||
|
if len(synonyms) < 2:
|
||||||
|
return []
|
||||||
|
is_led = _is_fr_infinitive_led if locale == "fr" else _is_en_imperative_led
|
||||||
|
seen = set(existing)
|
||||||
|
sorted_synonyms = sorted({syn for syn in synonyms if is_led(syn)}, key=len, reverse=True)
|
||||||
|
if len(sorted_synonyms) < 2:
|
||||||
|
return []
|
||||||
|
out: list[str] = []
|
||||||
|
for u in existing:
|
||||||
|
lower_u = u.lower()
|
||||||
|
matched = next(
|
||||||
|
(
|
||||||
|
syn
|
||||||
|
for syn in sorted_synonyms
|
||||||
|
if lower_u == syn.lower() or lower_u.startswith(f"{syn.lower()} ")
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if matched is None:
|
||||||
|
continue
|
||||||
|
rest = u[len(matched) :]
|
||||||
|
for syn in sorted_synonyms:
|
||||||
|
if syn == matched:
|
||||||
|
continue
|
||||||
|
candidate = f"{syn}{rest}"
|
||||||
|
if candidate in seen:
|
||||||
|
continue
|
||||||
|
seen.add(candidate)
|
||||||
|
out.append(candidate)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def top_up(existing: list[str], synonyms: list[str], target: int, locale: str) -> list[str]:
|
||||||
|
if len(existing) >= target:
|
||||||
|
return []
|
||||||
|
needed = target - len(existing)
|
||||||
|
pool = _synonym_variants(existing, synonyms, locale)
|
||||||
|
if len(pool) < needed:
|
||||||
|
frames = _FR_FRAMES if locale == "fr" else _EN_FRAMES
|
||||||
|
is_led = _is_fr_infinitive_led if locale == "fr" else _is_en_imperative_led
|
||||||
|
already = set(existing) | set(pool)
|
||||||
|
for candidate in _frame_variants(existing, frames, is_led):
|
||||||
|
if candidate in already:
|
||||||
|
continue
|
||||||
|
pool.append(candidate)
|
||||||
|
already.add(candidate)
|
||||||
|
return pool[:needed]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
with open(SRC_PATH, encoding="utf-8") as f:
|
||||||
|
source = f.read()
|
||||||
|
tree = ast.parse(source)
|
||||||
|
lines = source.splitlines(keepends=True)
|
||||||
|
|
||||||
|
module_body = tree.body
|
||||||
|
training_data_list = None
|
||||||
|
for node in module_body:
|
||||||
|
if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
|
||||||
|
if node.target.id == "TECH_STEP_TRAINING_DATA":
|
||||||
|
training_data_list = node.value
|
||||||
|
break
|
||||||
|
if training_data_list is None or not isinstance(training_data_list, ast.List):
|
||||||
|
print("Could not locate TECH_STEP_TRAINING_DATA list", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# First pass: collect every entry's current per-locale utterance/synonym
|
||||||
|
# lists and find each locale's own current max — the equalization
|
||||||
|
# target, not a number picked separately from the corpus itself.
|
||||||
|
parsed: list[tuple[str, str, ast.List, list[str], list[str]]] = []
|
||||||
|
targets = {"fr": 0, "en": 0}
|
||||||
|
for entry_call in training_data_list.elts:
|
||||||
|
assert isinstance(entry_call, ast.Call)
|
||||||
|
uid = None
|
||||||
|
for kw in entry_call.keywords:
|
||||||
|
if kw.arg == "uid":
|
||||||
|
assert isinstance(kw.value, ast.Constant)
|
||||||
|
uid = kw.value.value
|
||||||
|
for kw in entry_call.keywords:
|
||||||
|
if kw.arg not in ("fr", "en"):
|
||||||
|
continue
|
||||||
|
locale = kw.arg
|
||||||
|
locale_call = kw.value
|
||||||
|
assert isinstance(locale_call, ast.Call)
|
||||||
|
utterances_list_node = None
|
||||||
|
synonyms_list_node = None
|
||||||
|
for inner_kw in locale_call.keywords:
|
||||||
|
if inner_kw.arg == "utterances":
|
||||||
|
utterances_list_node = inner_kw.value
|
||||||
|
elif inner_kw.arg == "synonyms":
|
||||||
|
synonyms_list_node = inner_kw.value
|
||||||
|
if utterances_list_node is None:
|
||||||
|
continue
|
||||||
|
assert isinstance(utterances_list_node, ast.List)
|
||||||
|
existing = [
|
||||||
|
elt.value for elt in utterances_list_node.elts if isinstance(elt, ast.Constant)
|
||||||
|
]
|
||||||
|
synonyms = (
|
||||||
|
[elt.value for elt in synonyms_list_node.elts if isinstance(elt, ast.Constant)]
|
||||||
|
if isinstance(synonyms_list_node, ast.List)
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
targets[locale] = max(targets[locale], len(existing))
|
||||||
|
parsed.append((uid, locale, utterances_list_node, existing, synonyms))
|
||||||
|
|
||||||
|
print(f"Equalizing to the corpus's own current max — fr: {targets['fr']}, en: {targets['en']}")
|
||||||
|
|
||||||
|
insertions: list[tuple[int, str, list[str]]] = []
|
||||||
|
total_added = 0
|
||||||
|
shortfalls: list[tuple[str, str, int]] = []
|
||||||
|
|
||||||
|
for uid, locale, utterances_list_node, existing, synonyms in parsed:
|
||||||
|
target = targets[locale]
|
||||||
|
new_ones = top_up(existing, synonyms, target, locale)
|
||||||
|
final_count = len(existing) + len(new_ones)
|
||||||
|
if final_count < target:
|
||||||
|
shortfalls.append((uid, locale, final_count))
|
||||||
|
if not new_ones:
|
||||||
|
continue
|
||||||
|
last_elt = utterances_list_node.elts[-1]
|
||||||
|
insert_after_line = last_elt.end_lineno - 1
|
||||||
|
indent = lines[insert_after_line][
|
||||||
|
: len(lines[insert_after_line]) - len(lines[insert_after_line].lstrip())
|
||||||
|
]
|
||||||
|
new_lines = [f'{indent}"{s}",\n' for s in new_ones]
|
||||||
|
insertions.append((insert_after_line, uid, new_lines))
|
||||||
|
total_added += len(new_ones)
|
||||||
|
|
||||||
|
insertions.sort(key=lambda t: t[0], reverse=True)
|
||||||
|
for line_idx, uid, new_lines in insertions:
|
||||||
|
lines[line_idx + 1 : line_idx + 1] = new_lines
|
||||||
|
|
||||||
|
with open(SRC_PATH, "w", encoding="utf-8", newline="\n") as f:
|
||||||
|
f.writelines(lines)
|
||||||
|
|
||||||
|
print(f"Added {total_added} new utterances across {len(insertions)} (technique, locale) pairs.")
|
||||||
|
if shortfalls:
|
||||||
|
print(f"{len(shortfalls)} (uid, locale) pair(s) still below their locale's target — not")
|
||||||
|
print("enough synonym variety to reach full equalization:")
|
||||||
|
for uid, locale, count in shortfalls:
|
||||||
|
print(f" {uid} ({locale}): {count}/{targets[locale]}")
|
||||||
|
else:
|
||||||
|
print("Every technique now has exactly the same utterance count as every other, per locale.")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -30,6 +30,7 @@ from spacy.tokens import Doc, Span
|
||||||
from spacy.training import Example
|
from spacy.training import Example
|
||||||
from spacy.util import filter_spans, fix_random_seed, minibatch
|
from spacy.util import filter_spans, fix_random_seed, minibatch
|
||||||
|
|
||||||
|
from . import utensil_vocabulary
|
||||||
from .text_normalization import normalize_text
|
from .text_normalization import normalize_text
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -167,12 +168,18 @@ class TrainEntry:
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class Entity:
|
class Entity:
|
||||||
"""Une mention candidate trouvée par le `PhraseMatcher` — offsets
|
"""Une mention candidate trouvée par un `PhraseMatcher` — offsets
|
||||||
caractère `[start, end)`, miroir de `EntityPayload` (`schemas.py`)."""
|
caractère `[start, end)`, miroir de `EntityPayload` (`schemas.py`).
|
||||||
|
`kind` distingue de quel `PhraseMatcher` la mention vient (`"technique"`
|
||||||
|
— `self._matcher`, entraîné depuis `training_data.py` — ou `"utensil"`
|
||||||
|
— `self._utensil_matcher`, statique, voir `utensil_vocabulary.py`) :
|
||||||
|
`apps/api`'s `tech-step-matcher.ts` a besoin de savoir laquelle des deux
|
||||||
|
résoudre (`TechStep.key` vs `Utensil.key`)."""
|
||||||
|
|
||||||
uid: str
|
uid: str
|
||||||
start: int
|
start: int
|
||||||
end: int
|
end: int
|
||||||
|
kind: str = "technique"
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
|
|
@ -210,6 +217,11 @@ class LocalePipeline:
|
||||||
# comportement testé côté `apps/api` pour "une locale jamais
|
# comportement testé côté `apps/api` pour "une locale jamais
|
||||||
# entraînée".
|
# entraînée".
|
||||||
self._matcher: PhraseMatcher | None = None
|
self._matcher: PhraseMatcher | None = None
|
||||||
|
# Construit une seule fois par `preload()`, jamais par `train()` —
|
||||||
|
# contrairement à `self._matcher`, ce vocabulaire est statique
|
||||||
|
# (`utensil_vocabulary.py`), il n'a pas de contrepartie "corpus
|
||||||
|
# poussé par un appelant" à reconstruire.
|
||||||
|
self._utensil_matcher: PhraseMatcher | None = None
|
||||||
self._trained = False
|
self._trained = False
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|
@ -217,10 +229,18 @@ class LocalePipeline:
|
||||||
return self._trained
|
return self._trained
|
||||||
|
|
||||||
def preload(self) -> None:
|
def preload(self) -> None:
|
||||||
"""Charge le modèle spaCy de base (tokenizer + vecteurs) et le
|
"""Charge le modèle spaCy de base (tokenizer + vecteurs), le
|
||||||
composant `diacritics_normalizer` — idempotent, sans effet si déjà
|
composant `diacritics_normalizer`, et construit le `PhraseMatcher`
|
||||||
chargé. Appelé au démarrage du process pour les deux locales
|
d'ustensiles — idempotent, sans effet si déjà chargé. Appelé au
|
||||||
connues (voir `main.py`), pas paresseusement au premier `train()`.
|
démarrage du process pour les deux locales connues (voir
|
||||||
|
`main.py`), pas paresseusement au premier `train()`.
|
||||||
|
|
||||||
|
Le matcher d'ustensiles est construit ici, pas dans `train()` :
|
||||||
|
contrairement au `PhraseMatcher` de techniques (reconstruit à
|
||||||
|
chaque `train()` depuis les `entries` reçues), le vocabulaire
|
||||||
|
d'ustensiles est statique (`utensil_vocabulary.py`) — rien ne le
|
||||||
|
fait jamais varier d'un appel à l'autre, donc rien ne justifie de
|
||||||
|
payer son coût de construction plus d'une fois par démarrage.
|
||||||
"""
|
"""
|
||||||
if self._base_nlp is not None:
|
if self._base_nlp is not None:
|
||||||
return
|
return
|
||||||
|
|
@ -228,6 +248,15 @@ class LocalePipeline:
|
||||||
nlp.add_pipe("diacritics_normalizer", first=True)
|
nlp.add_pipe("diacritics_normalizer", first=True)
|
||||||
self._base_nlp = nlp
|
self._base_nlp = nlp
|
||||||
|
|
||||||
|
diacritics_normalizer = nlp.get_pipe("diacritics_normalizer")
|
||||||
|
utensil_matcher = PhraseMatcher(nlp.vocab, attr="NORM")
|
||||||
|
for uid, synonyms in utensil_vocabulary.synonyms_for_locale(self._locale).items():
|
||||||
|
if not synonyms:
|
||||||
|
continue
|
||||||
|
patterns = [diacritics_normalizer(nlp.make_doc(synonym)) for synonym in synonyms]
|
||||||
|
utensil_matcher.add(uid, patterns)
|
||||||
|
self._utensil_matcher = utensil_matcher
|
||||||
|
|
||||||
def train(self, entries: list[TrainEntry]) -> tuple[int, int, int]:
|
def train(self, entries: list[TrainEntry]) -> tuple[int, int, int]:
|
||||||
"""Reconstruit le `textcat` et le `PhraseMatcher` de ce pipeline à
|
"""Reconstruit le `textcat` et le `PhraseMatcher` de ce pipeline à
|
||||||
partir de `entries` (le tokenizer/les vecteurs restent ceux chargés
|
partir de `entries` (le tokenizer/les vecteurs restent ceux chargés
|
||||||
|
|
@ -409,13 +438,41 @@ class LocalePipeline:
|
||||||
matched_spans = [
|
matched_spans = [
|
||||||
Span(doc, start, end, label=match_id) for match_id, start, end in self._matcher(doc)
|
Span(doc, start, end, label=match_id) for match_id, start, end in self._matcher(doc)
|
||||||
]
|
]
|
||||||
entities = sorted(
|
technique_entities = [
|
||||||
(
|
Entity(
|
||||||
Entity(uid=self._base_nlp.vocab.strings[span.label], start=span.start_char, end=span.end_char)
|
uid=self._base_nlp.vocab.strings[span.label],
|
||||||
for span in filter_spans(matched_spans)
|
start=span.start_char,
|
||||||
),
|
end=span.end_char,
|
||||||
key=lambda entity: entity.start,
|
kind="technique",
|
||||||
)
|
)
|
||||||
|
for span in filter_spans(matched_spans)
|
||||||
|
]
|
||||||
|
|
||||||
|
# Second, independent `PhraseMatcher` pass for ustensiles — run and
|
||||||
|
# `filter_spans`-resolved *separately* from the technique pass
|
||||||
|
# above: the two matchers' candidates never compete for the same
|
||||||
|
# position (a longer utensil match must never swallow/be swallowed
|
||||||
|
# by a technique match the way two overlapping technique synonyms
|
||||||
|
# do), only overlaps *within* the same matcher are the known
|
||||||
|
# problem `filter_spans` exists for (see the technique pass's own
|
||||||
|
# comment above).
|
||||||
|
utensil_entities: list[Entity] = []
|
||||||
|
if self._utensil_matcher is not None:
|
||||||
|
utensil_spans = [
|
||||||
|
Span(doc, start, end, label=match_id)
|
||||||
|
for match_id, start, end in self._utensil_matcher(doc)
|
||||||
|
]
|
||||||
|
utensil_entities = [
|
||||||
|
Entity(
|
||||||
|
uid=self._base_nlp.vocab.strings[span.label],
|
||||||
|
start=span.start_char,
|
||||||
|
end=span.end_char,
|
||||||
|
kind="utensil",
|
||||||
|
)
|
||||||
|
for span in filter_spans(utensil_spans)
|
||||||
|
]
|
||||||
|
|
||||||
|
entities = sorted(technique_entities + utensil_entities, key=lambda entity: entity.start)
|
||||||
|
|
||||||
cats = doc.cats
|
cats = doc.cats
|
||||||
if not cats:
|
if not cats:
|
||||||
|
|
|
||||||
|
|
@ -30,14 +30,20 @@ def process(request: ProcessRequest) -> ProcessResponse:
|
||||||
extra={
|
extra={
|
||||||
"locale": request.locale,
|
"locale": request.locale,
|
||||||
"text": request.text,
|
"text": request.text,
|
||||||
"entities": [{"uid": entity.uid, "start": entity.start, "end": entity.end} for entity in result.entities],
|
"entities": [
|
||||||
|
{"uid": entity.uid, "start": entity.start, "end": entity.end, "kind": entity.kind}
|
||||||
|
for entity in result.entities
|
||||||
|
],
|
||||||
"intent": result.intent,
|
"intent": result.intent,
|
||||||
"score": result.score,
|
"score": result.score,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
return ProcessResponse(
|
return ProcessResponse(
|
||||||
entities=[EntityPayload(uid=entity.uid, start=entity.start, end=entity.end) for entity in result.entities],
|
entities=[
|
||||||
|
EntityPayload(uid=entity.uid, start=entity.start, end=entity.end, kind=entity.kind)
|
||||||
|
for entity in result.entities
|
||||||
|
],
|
||||||
intent=result.intent,
|
intent=result.intent,
|
||||||
score=result.score,
|
score=result.score,
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,8 @@ depuis `training_data.py` (voir `pipeline_registry.py`/`main.py`), plus
|
||||||
besoin d'un contrat HTTP pour ça.
|
besoin d'un contrat HTTP pour ça.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
@ -20,14 +22,21 @@ class ProcessRequest(BaseModel):
|
||||||
|
|
||||||
|
|
||||||
class EntityPayload(BaseModel):
|
class EntityPayload(BaseModel):
|
||||||
"""Une mention candidate d'une technique — offsets caractère `[start, end)`
|
"""Une mention candidate — technique ou ustensile, voir `kind` — offsets
|
||||||
dans `text`, convention identique à `String.prototype.slice` côté
|
caractère `[start, end)` dans `text`, convention identique à
|
||||||
`apps/api` (pas de décalage `+1` à appliquer côté Node, contrairement à
|
`String.prototype.slice` côté `apps/api` (pas de décalage `+1` à
|
||||||
l'ancien `NlpManager` de node-nlp)."""
|
appliquer côté Node, contrairement à l'ancien `NlpManager` de
|
||||||
|
node-nlp).
|
||||||
|
|
||||||
|
`kind` distingue de quel `PhraseMatcher` la mention vient (voir
|
||||||
|
`locale_pipeline.py`'s `Entity`) — `apps/api`'s `tech-step-matcher.ts`
|
||||||
|
en a besoin pour savoir laquelle des deux résoudre (`TechStep.key` vs
|
||||||
|
`Utensil.key`)."""
|
||||||
|
|
||||||
uid: str
|
uid: str
|
||||||
start: int
|
start: int
|
||||||
end: int
|
end: int
|
||||||
|
kind: Literal["technique", "utensil"] = "technique"
|
||||||
|
|
||||||
|
|
||||||
class ProcessResponse(BaseModel):
|
class ProcessResponse(BaseModel):
|
||||||
|
|
|
||||||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,197 @@
|
||||||
|
"""Vocabulaire du `PhraseMatcher` d'ustensiles — contrairement à
|
||||||
|
`training_data.py`, ce catalogue n'a jamais existé côté `apps/api` avant ce
|
||||||
|
service : il est *né* ici, pas rapatrié depuis TypeScript. Chaque `uid`
|
||||||
|
ci-dessous doit avoir une entrée `UTENSILS` correspondante
|
||||||
|
(`reference-seed-data.ts` côté `apps/api`) et un libellé
|
||||||
|
`catalog.utensils.<uid>` (`apps/web`'s `locales/fr/translation.json`).
|
||||||
|
|
||||||
|
Un seul type de contenu par ustensile/locale (contrairement à
|
||||||
|
`TechStepTrainingEntry`'s `synonyms`/`utterances`) : un ustensile mentionné
|
||||||
|
n'a pas besoin d'être *interprété* comme une technique peut l'être
|
||||||
|
(`préchauffer` vs `chauffer` dépend du contexte ; `poêle` n'en dépend pas) —
|
||||||
|
juste reconnu, comme les `synonyms` de `training_data.py` alimentent le
|
||||||
|
`PhraseMatcher` de techniques. Pas de `textcat` équivalent ici, voir
|
||||||
|
`LocalePipeline`'s propre commentaire sur `_utensil_matcher`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class UtensilLocaleVocabulary:
|
||||||
|
synonyms: list[str] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class UtensilEntry:
|
||||||
|
"""`uid` doit correspondre à un `Utensil.key`."""
|
||||||
|
|
||||||
|
uid: str
|
||||||
|
fr: UtensilLocaleVocabulary
|
||||||
|
en: UtensilLocaleVocabulary
|
||||||
|
|
||||||
|
|
||||||
|
UTENSIL_VOCABULARY: list[UtensilEntry] = [
|
||||||
|
UtensilEntry(
|
||||||
|
uid="pan",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["poêle", "sauteuse", "poêle antiadhésive"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["pan", "frying pan", "skillet"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="saucepan",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["casserole", "petite casserole"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["saucepan", "sauce pan"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="pot",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["marmite", "faitout", "cocotte"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["pot", "stockpot", "dutch oven"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="knife",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["couteau", "couteau de cuisine", "couteau d'office"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["knife", "kitchen knife", "chef's knife"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="whisk",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["fouet"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["whisk"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="bowl",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["saladier", "bol", "cul-de-poule"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["bowl", "mixing bowl"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="bakingSheet",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["plaque de cuisson", "plaque à pâtisserie", "plaque du four"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["baking sheet", "baking tray", "sheet pan"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="mold",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["moule", "moule à gâteau", "moule à cake"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["mold", "mould", "baking pan"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="colander",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["passoire", "égouttoir"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["colander", "strainer"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="cuttingBoard",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["planche à découper"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["cutting board", "chopping board"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="oven",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["four"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["oven"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="blender",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["blender", "mixeur plongeant", "mixeur girafe"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["blender", "immersion blender"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="mixer",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["batteur", "batteur électrique", "robot pâtissier"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["mixer", "stand mixer", "hand mixer"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="spatula",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["spatule", "maryse"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["spatula"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="ladle",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["louche"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["ladle"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="grater",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["râpe"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["grater"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="rollingPin",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["rouleau à pâtisserie"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["rolling pin"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="lid",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["couvercle"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["lid"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="tongs",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["pince", "pince de cuisine"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["tongs"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="peeler",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["économe", "éplucheur"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["peeler", "vegetable peeler"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="sieve",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["tamis", "chinois"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["sieve"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="foodProcessor",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["robot ménager", "robot de cuisine", "robot culinaire"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["food processor"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="steamerBasket",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["panier vapeur", "cuit-vapeur"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["steamer basket", "steamer"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="skewer",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["brochette", "pique en bois"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["skewer"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="pastryBrush",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["pinceau de cuisine", "pinceau à pâtisserie"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["pastry brush", "basting brush"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="ramekin",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["ramequin"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["ramekin"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="dish",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["plat", "plat à gratin", "plat allant au four"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["dish", "baking dish", "gratin dish"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="wok",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["wok"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["wok"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="thermometer",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["thermomètre", "thermomètre de cuisson"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["thermometer"]),
|
||||||
|
),
|
||||||
|
UtensilEntry(
|
||||||
|
uid="mandoline",
|
||||||
|
fr=UtensilLocaleVocabulary(synonyms=["mandoline"]),
|
||||||
|
en=UtensilLocaleVocabulary(synonyms=["mandoline"]),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def synonyms_for_locale(locale: str) -> dict[str, list[str]]:
|
||||||
|
"""Aplati {@link UTENSIL_VOCABULARY} en `{uid: synonyms}` pour une seule
|
||||||
|
locale — la forme que `LocalePipeline.preload()` attend pour construire
|
||||||
|
son `PhraseMatcher` d'ustensiles. Miroir de `training_data.entries_for_locale`,
|
||||||
|
en plus simple (pas d'`utterances`, un seul champ à extraire)."""
|
||||||
|
return {
|
||||||
|
entry.uid: getattr(entry, locale).synonyms
|
||||||
|
for entry in UTENSIL_VOCABULARY
|
||||||
|
if hasattr(entry, locale)
|
||||||
|
}
|
||||||
|
|
@ -84,10 +84,16 @@ def test_untrained_locale_returns_empty_without_error():
|
||||||
|
|
||||||
|
|
||||||
def test_detects_two_techniques_with_exact_tight_spans_reading_order(fr_pipeline: LocalePipeline):
|
def test_detects_two_techniques_with_exact_tight_spans_reading_order(fr_pipeline: LocalePipeline):
|
||||||
|
# This text's "poêle" is now *also* a real utensil match ("pan", see
|
||||||
|
# `utensil_vocabulary.py`) — filtered out here by `kind` since this test
|
||||||
|
# is specifically about technique-candidate ordering, not the full
|
||||||
|
# mixed entity list (see `test_utensil_matching.py` for the utensil
|
||||||
|
# matcher's own coverage).
|
||||||
text = "Préchauffer la poêle, puis faire fondre le beurre"
|
text = "Préchauffer la poêle, puis faire fondre le beurre"
|
||||||
result = fr_pipeline.process(text)
|
result = fr_pipeline.process(text)
|
||||||
|
|
||||||
uids_by_start = sorted(((entity.start, entity.uid) for entity in result.entities))
|
technique_entities = [entity for entity in result.entities if entity.kind == "technique"]
|
||||||
|
uids_by_start = sorted(((entity.start, entity.uid) for entity in technique_entities))
|
||||||
assert [uid for _, uid in uids_by_start] == ["preheat", "melt"]
|
assert [uid for _, uid in uids_by_start] == ["preheat", "melt"]
|
||||||
|
|
||||||
preheat_entity = next(e for e in result.entities if e.uid == "preheat")
|
preheat_entity = next(e for e in result.entities if e.uid == "preheat")
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
"""Garde-fou de non-régression pour l'équilibrage du corpus (voir
|
||||||
|
`training_data.py`'s propre commentaire de tête) : chaque technique doit
|
||||||
|
avoir exactement le même nombre d'`utterances` que chaque autre, par
|
||||||
|
locale — un déséquilibre entre classes est une source réelle de
|
||||||
|
classifications confiantes mais fausses sur une phrase jamais vue (constaté
|
||||||
|
en pratique — voir l'historique Git de ce fichier, trois tentatives
|
||||||
|
d'équilibrer vers un nombre plus élevé ont toutes dégradé le F1 agrégé de
|
||||||
|
`test/recipe-matching/tech-step-eval.test.ts` avant que la stratégie
|
||||||
|
actuelle — équilibrer vers le maximum déjà présent dans le corpus, pas un
|
||||||
|
nombre choisi dans l'absolu — ne passe cette même gate)."""
|
||||||
|
|
||||||
|
from intent_service.training_data import TECH_STEP_TRAINING_DATA
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_technique_has_the_same_utterance_count_per_locale():
|
||||||
|
for locale in ("fr", "en"):
|
||||||
|
counts = {entry.uid: len(getattr(entry, locale).utterances) for entry in TECH_STEP_TRAINING_DATA}
|
||||||
|
distinct = set(counts.values())
|
||||||
|
assert len(distinct) == 1, (
|
||||||
|
f"utterance counts for locale {locale!r} aren't uniform across techniques "
|
||||||
|
f"(run augment_utterances.py to re-equalize): {counts}"
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,78 @@
|
||||||
|
"""Couvre `LocalePipeline`'s second `PhraseMatcher` (ustensiles,
|
||||||
|
`utensil_vocabulary.py`) — même style que `test_locale_pipeline_entities.py`
|
||||||
|
(offsets exacts, insensibilité accents/casse), mais contre le vocabulaire
|
||||||
|
*réel* (`UTENSIL_VOCABULARY`, statique, construit par `preload()` — pas
|
||||||
|
besoin d'un jeu de test dédié comme pour les techniques, voir
|
||||||
|
`LocalePipeline.preload`'s own comment)."""
|
||||||
|
|
||||||
|
from intent_service.locale_pipeline import LocalePipeline, TrainEntry
|
||||||
|
|
||||||
|
# Un `train()` minimal suffit — le `PhraseMatcher` d'ustensiles est
|
||||||
|
# construit par `preload()` (appelé par `train()`), indépendamment du
|
||||||
|
# `TrainEntry` de techniques passé ici (voir `preload()`'s own comment sur
|
||||||
|
# pourquoi les deux ne sont pas couplés).
|
||||||
|
_MINIMAL_ENTRIES = [
|
||||||
|
TrainEntry(uid="melt", synonyms=["fondre"], utterances=["faire fondre le beurre"]),
|
||||||
|
TrainEntry(uid="simmer", synonyms=["mijoter"], utterances=["faire mijoter à feu doux"]),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _fr_pipeline() -> LocalePipeline:
|
||||||
|
pipeline = LocalePipeline("fr")
|
||||||
|
pipeline.train(_MINIMAL_ENTRIES)
|
||||||
|
return pipeline
|
||||||
|
|
||||||
|
|
||||||
|
def test_matches_a_real_utensil_with_exact_span():
|
||||||
|
pipeline = _fr_pipeline()
|
||||||
|
text = "Dans une poêle chaude, faire fondre le beurre"
|
||||||
|
result = pipeline.process(text)
|
||||||
|
|
||||||
|
pan_entities = [e for e in result.entities if e.uid == "pan"]
|
||||||
|
assert len(pan_entities) == 1
|
||||||
|
entity = pan_entities[0]
|
||||||
|
assert entity.kind == "utensil"
|
||||||
|
assert text[entity.start : entity.end] == "poêle"
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_case_and_accent_insensitive():
|
||||||
|
pipeline = _fr_pipeline()
|
||||||
|
result = pipeline.process("Verser dans la POÊLE")
|
||||||
|
utensil_uids = [e.uid for e in result.entities if e.kind == "utensil"]
|
||||||
|
assert utensil_uids == ["pan"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_matches_a_multi_word_synonym():
|
||||||
|
pipeline = _fr_pipeline()
|
||||||
|
text = "Découper les légumes sur la planche à découper"
|
||||||
|
result = pipeline.process(text)
|
||||||
|
board_entities = [e for e in result.entities if e.uid == "cuttingBoard"]
|
||||||
|
assert len(board_entities) == 1
|
||||||
|
assert text[board_entities[0].start : board_entities[0].end] == "planche à découper"
|
||||||
|
|
||||||
|
|
||||||
|
def test_technique_and_utensil_are_both_returned_without_interfering():
|
||||||
|
pipeline = _fr_pipeline()
|
||||||
|
text = "Dans une casserole, faire mijoter à feu doux"
|
||||||
|
result = pipeline.process(text)
|
||||||
|
|
||||||
|
kinds_by_uid = {e.uid: e.kind for e in result.entities}
|
||||||
|
assert kinds_by_uid.get("simmer") == "technique"
|
||||||
|
assert kinds_by_uid.get("saucepan") == "utensil"
|
||||||
|
|
||||||
|
|
||||||
|
def test_returns_no_utensil_entities_when_none_are_mentioned():
|
||||||
|
pipeline = _fr_pipeline()
|
||||||
|
result = pipeline.process("Laisser reposer la pâte une heure")
|
||||||
|
assert [e for e in result.entities if e.kind == "utensil"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_matches_english_utensils_too():
|
||||||
|
pipeline = LocalePipeline("en")
|
||||||
|
pipeline.train([TrainEntry(uid="chop", synonyms=["chop"], utterances=["chop the onions finely"])])
|
||||||
|
text = "Heat the pan before adding the onions"
|
||||||
|
result = pipeline.process(text)
|
||||||
|
pan_entities = [e for e in result.entities if e.uid == "pan"]
|
||||||
|
assert len(pan_entities) == 1
|
||||||
|
assert pan_entities[0].kind == "utensil"
|
||||||
|
assert text[pan_entities[0].start : pan_entities[0].end] == "pan"
|
||||||
|
|
@ -555,6 +555,32 @@ combien de temps ça prend).
|
||||||
("préchauffer") ne matche jamais sa forme normalisée dans le texte cible
|
("préchauffer") ne matche jamais sa forme normalisée dans le texte cible
|
||||||
(voir le commentaire dans `locale_pipeline.py`'s `train()`).
|
(voir le commentaire dans `locale_pipeline.py`'s `train()`).
|
||||||
|
|
||||||
|
**Métadonnées d'action — ingrédients, quantités, ustensiles.** Chaque
|
||||||
|
occurrence de technique (`TechStepMatch`) porte aussi ce qui a été détecté
|
||||||
|
dans sa propre *clause* (celle calculée à l'étape 2 ci-dessus) :
|
||||||
|
- **Ingrédients** — `ingredient-matcher.ts`'s `findIngredientMentions` scanne
|
||||||
|
le texte de la clause contre le catalogue `Ingredient` *existant*
|
||||||
|
(`INGREDIENT_LABELS_FR`/`_EN`, `packages/shared` — le même que
|
||||||
|
`matchIngredientName` utilise déjà pour les listes structurées), plutôt que
|
||||||
|
de dupliquer ce catalogue côté service Python. Une quantité+unité
|
||||||
|
immédiatement avant la mention est résolue au mieux (regex ancrée sur la
|
||||||
|
*fin* du texte précédent, voir `QUANTITY_BEFORE_INGREDIENT_PATTERN`) —
|
||||||
|
`null`/`null` sinon, jamais une erreur.
|
||||||
|
- **Ustensiles** — contrairement aux ingrédients, ce catalogue n'existait
|
||||||
|
nulle part avant cette fonctionnalité : il est né directement côté service
|
||||||
|
Python (`intent_service/utensil_vocabulary.py`), via un second
|
||||||
|
`PhraseMatcher` indépendant du premier (pas de `textcat` — un ustensile
|
||||||
|
mentionné n'a pas besoin d'être interprété, contrairement à une technique).
|
||||||
|
`POST /v1/process` renvoie donc deux types d'entité discriminés par
|
||||||
|
`kind: "technique" | "utensil"` dans la même liste `entities`.
|
||||||
|
|
||||||
|
Dans les deux cas, l'association à une technique se fait par appartenance à
|
||||||
|
la même clause — pas d'analyse syntaxique (le `parser` spaCy reste exclu du
|
||||||
|
pipeline, voir `_EXCLUDED_COMPONENTS`), juste "cette mention tombe dans
|
||||||
|
`[clause.start, clause.end)`". Persisté comme `StepTechStepIngredient`/
|
||||||
|
`StepTechStepUtensil`, deux tables référençant `StepTechStep` par sa clé
|
||||||
|
composite `(stepId, order)`.
|
||||||
|
|
||||||
### Résolution ingrédients/unités — `ingredient-matcher.ts`
|
### Résolution ingrédients/unités — `ingredient-matcher.ts`
|
||||||
|
|
||||||
**Anglais uniquement** aujourd'hui (commit "matching anglais pour les tech
|
**Anglais uniquement** aujourd'hui (commit "matching anglais pour les tech
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,9 @@ d'ingrédients/unités normalisé, techniques détectées, visibilité) :
|
||||||
- **Planification** — `Planning`, `PlanningItem`
|
- **Planification** — `Planning`, `PlanningItem`
|
||||||
- **Recettes** — `Recipe`, `RecipeIngredient`, `Step`, `TechStep`,
|
- **Recettes** — `Recipe`, `RecipeIngredient`, `Step`, `TechStep`,
|
||||||
`StepTechStep`, `RecipeDiet`, `RecipeFavorite`
|
`StepTechStep`, `RecipeDiet`, `RecipeFavorite`
|
||||||
|
- **Métadonnées d'action** — `Utensil`, `StepTechStepIngredient`,
|
||||||
|
`StepTechStepUtensil` (ingrédients/quantités/ustensiles associés à une
|
||||||
|
technique détectée, voir plus bas)
|
||||||
- **Sources externes** — `Source`, `HouseSource`
|
- **Sources externes** — `Source`, `HouseSource`
|
||||||
- **Catalogue ingrédients/unités** — `Ingredient`, `Unit`, `IngredientDiet`,
|
- **Catalogue ingrédients/unités** — `Ingredient`, `Unit`, `IngredientDiet`,
|
||||||
`IngredientAllergy`, `UserProfileDislikedIngredient`
|
`IngredientAllergy`, `UserProfileDislikedIngredient`
|
||||||
|
|
@ -371,6 +374,17 @@ surlignage tant que sa recette n'est pas resauvegardée) sont le span détecté
|
||||||
dans `Step.description`, utilisé pour le surlignage côté web
|
dans `Step.description`, utilisé pour le surlignage côté web
|
||||||
(`highlight-tech-steps.ts`).
|
(`highlight-tech-steps.ts`).
|
||||||
|
|
||||||
|
Chaque `step_tech_step` porte en plus les métadonnées trouvées dans sa propre
|
||||||
|
clause : `step_tech_step_ingredient` (ingrédient résolu contre le catalogue
|
||||||
|
`ingredients` existant, `quantity`/`unit_id` optionnels quand une quantité a
|
||||||
|
pu être extraite juste avant la mention) et `step_tech_step_utensil`
|
||||||
|
(ustensile résolu contre un nouveau catalogue `utensil`, même forme
|
||||||
|
minimale `id`/`key` que `tech_step` — voir
|
||||||
|
[backend-architecture.md](./backend-architecture.md#détection-des-techniques--tech-step-matcherts)
|
||||||
|
pour comment chacun est détecté). Les deux référencent `step_tech_step` par
|
||||||
|
sa clé composite `(step_id, order)`, `onDelete: Cascade` comme le reste de
|
||||||
|
cette chaîne.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Relations
|
## Relations
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue