/** * Hand-labeled evaluation set for {@link techStepClassifier} — what * `tech-step-eval.test.ts` runs the real classifier against to compute * precision/recall/F1 (`tech-step-evaluator.ts`), the objective gate any * future change to `tech-step-training-data.ts` must clear (see that * module's own doc comment). * * Deliberately *not* reusing `TECH_STEP_TRAINING_DATA`'s own `utterances` * verbatim — scoring the classifier against the exact sentences it was * trained on would measure memorization, not generalization. Every * description below is original phrasing; where a case still needs to name * a technique's own verb to be labeled with confidence (most of them, see * this file's own limits below), it's at least a different sentence shape * than anything in the training corpus. * * `expectedKeys` is a multiset in reading order (see * `TechStepEvalOutcome`'s doc comment in `tech-step-evaluator.ts` for why * order isn't scored but repetition is) of `TechStep.key`s — resolved to * real DB ids and back by `tech-step-eval-runner.ts`, this file only ever * deals in stable keys so it doesn't need DB access to author or read. * * Known limit of this dataset, confirmed against a real run (see * `MIN_OVERALL_F1`'s own doc comment, `tech-step-eval-runner.ts`): most * cases anchor on a technique's own registered synonym, but `_classifyClause` * only falls back to that anchor when the intent classifier's own score is * *below* `CONFIDENCE_THRESHOLD` — a confidently *wrong* whole-clause * classification (e.g. "Blanchissez les haricots verts..." scoring * confidently as `peel` despite the correct `blanch` anchor) overrides the * anchor just as readily as a confidently *right* one does, so this * dataset genuinely does measure real classifier failures, not just a * synthetic floor. A handful of such real mismatches are expected and * intentionally left uncorrected here (see `MIN_OVERALL_F1`'s doc comment) * — fixing the classifier's actual behavior on them is corpus work for a * future change, not something to hide by loosening this dataset's own * expectations to match whatever it currently outputs. */ export interface TechStepEvalCase { description: string; locale: "fr" | "en"; expectedKeys: string[]; } export const TECH_STEP_EVAL_DATASET: TechStepEvalCase[] = [ // --- One straightforward case per technique (fr), covering all 26 --- { description: "Faites cuire les pâtes al dente dans une grande casserole d'eau bien salée.", locale: "fr", expectedKeys: ["cook"], }, { description: "Faites bouillir l'eau dans une grande casserole avant d'y plonger les pâtes.", locale: "fr", expectedKeys: ["boil"], }, { description: "Plongez les beignets dans l'huile très chaude pour les faire frire.", locale: "fr", expectedKeys: ["fry"], }, { description: "Faites fondre le chocolat noir au bain-marie en remuant.", locale: "fr", expectedKeys: ["melt"], }, { description: "Déglacez la casserole avec un trait de vinaigre balsamique.", locale: "fr", expectedKeys: ["deglaze"], }, { description: "Laissez frémir la sauce tomate vingt minutes à couvert.", locale: "fr", expectedKeys: ["simmer"], }, { description: "Faites rôtir la volaille entière sur la broche du four.", locale: "fr", expectedKeys: ["roast"], }, { description: "Faites griller les brochettes de poulet quelques minutes de chaque côté.", locale: "fr", expectedKeys: ["grill"], }, { description: "Faites sauter les champignons à feu vif dans une poêle très chaude.", locale: "fr", expectedKeys: ["panFry"], }, { description: "Blanchissez les haricots verts trois minutes avant de les refroidir.", locale: "fr", expectedKeys: ["blanch"], }, { description: "Laissez mariner les brochettes de poulet deux heures au frais.", locale: "fr", expectedKeys: ["marinate"], }, { description: "Hachez grossièrement le persil frais avant de le parsemer.", locale: "fr", expectedKeys: ["chop"], }, { description: "Épluchez les carottes avant de les couper en rondelles.", locale: "fr", expectedKeys: ["peel"], }, { description: "Émincez finement l'échalote pour la vinaigrette.", locale: "fr", expectedKeys: ["mince"], }, { description: "Mélangez la farine, le sucre et les œufs dans un grand saladier.", locale: "fr", expectedKeys: ["mix"], }, { description: "Fouettez énergiquement la crème jusqu'à ce qu'elle épaississe.", locale: "fr", expectedKeys: ["whisk"], }, { description: "Incorporez délicatement la farine tamisée à la préparation.", locale: "fr", expectedKeys: ["foldIn"], }, { description: "Réservez la pâte au réfrigérateur pendant que vous préparez la garniture.", locale: "fr", expectedKeys: ["setAside"], }, { description: "Assaisonnez le poisson avec du sel, du poivre et un filet de citron.", locale: "fr", expectedKeys: ["season"], }, { description: "Égouttez soigneusement le riz dans une passoire fine.", locale: "fr", expectedKeys: ["drain"], }, { description: "Faites dorer les morceaux de veau sur toutes leurs faces avant de mouiller avec le bouillon.", locale: "fr", expectedKeys: ["brown"], }, { description: "Laissez reposer la viande dix minutes avant de la trancher.", locale: "fr", expectedKeys: ["rest"], }, { description: "Préchauffez le four à 200 degrés avant d'y glisser le gratin.", locale: "fr", expectedKeys: ["preheat"], }, { description: "Enfournez la tarte pendant trente-cinq minutes jusqu'à ce qu'elle soit dorée.", locale: "fr", expectedKeys: ["bake"], }, { description: "Dressez harmonieusement les légumes autour de la pièce de viande.", locale: "fr", expectedKeys: ["plate"], }, { description: "Nappez le fond du moule d'une fine couche de caramel.", locale: "fr", expectedKeys: ["coat"], }, // --- English coverage (same technique verbs, distinct sentences) --- { description: "Simmer the stock gently for forty minutes, skimming occasionally.", locale: "en", expectedKeys: ["simmer"], }, { description: "Peel the potatoes and rinse them under cold water.", locale: "en", expectedKeys: ["peel"], }, { description: "Whisk the eggs with a pinch of salt until frothy.", locale: "en", expectedKeys: ["whisk"], }, { description: "Season the soup generously with black pepper before serving.", locale: "en", expectedKeys: ["season"], }, { description: "Make sure the chicken is cooked through before serving.", locale: "en", expectedKeys: ["cook"], }, // --- Multi-technique sentences, in reading order --- { description: "Préchauffez le four, puis faites rôtir le poulet pendant une heure.", locale: "fr", expectedKeys: ["preheat", "roast"], }, { description: "Faites revenir les oignons, puis déglacez la poêle avec du vin blanc.", locale: "fr", expectedKeys: ["brown", "deglaze"], }, { description: "Faites cuire les légumes à la vapeur, puis assaisonnez-les avec des herbes fraîches.", locale: "fr", expectedKeys: ["cook", "season"], }, { description: "Émincez l'oignon, faites-le suer, puis mouillez avec le bouillon et laissez mijoter.", locale: "fr", expectedKeys: ["mince", "simmer"], }, // --- No technique mentioned at all --- { description: "Répartissez les convives autour de la table avant de commencer le repas.", locale: "fr", expectedKeys: [], }, { description: "Rangez les couverts propres dans le tiroir de la cuisine.", locale: "fr", expectedKeys: [], }, { description: "Take the dishes and glasses out of the cupboard.", locale: "en", expectedKeys: [], }, // --- Documented false-positive traps, re-verified with fresh wording --- // `brown`'s EN synonyms are verb forms only ("browned"/"browning"), not // bare "brown" — precisely so this doesn't false-positive (see that // entry's own comment in tech-step-training-data.ts). { description: "This recipe calls for two tablespoons of brown sugar.", locale: "en", expectedKeys: [], }, // `rest`'s EN synonyms are anchored phrases ("let it rest"/"resting // for"...), not bare "rest" — so a sentence using the word in its // "remainder" sense must not anchor `rest` at all. { description: "There is no time to rest before the guests arrive.", locale: "en", expectedKeys: [], }, ];