test(playground): require the demo project description to match its reports
New gate tests/kb-update/test-demo-project-description.test.mjs. The demo project card must name the domain of the classify fixture's system description (bostøtte), must not name byggesak, and the inlined description must match scripts/build-demo-state.mjs. Red on 70731af: 1 of 3 fails. Chose a pinned domain term shared with the classify fixture over a generic word-overlap score, because the old description already shared generic words (kommune, chatbot, innbyggere) with the reports and would have passed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
2ed24add12
commit
820d62ddb4
1 changed files with 72 additions and 0 deletions
72
tests/kb-update/test-demo-project-description.test.mjs
Normal file
72
tests/kb-update/test-demo-project-description.test.mjs
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
// tests/kb-update/test-demo-project-description.test.mjs
|
||||
// Preventive gate: the demo project card describes the same system as the 17
|
||||
// demo reports it carries.
|
||||
//
|
||||
// The demo project (`scripts/build-demo-state.mjs`, inlined into the playground
|
||||
// HTML as `<script id="demo-state-v1">`) is shown on the project card and in the
|
||||
// screenshots. The reports come from `playground/test-fixtures/*.md`. The
|
||||
// classify fixture's `Beskrivelse:` line is the canonical system description:
|
||||
// a citizen chatbot that pre-assesses applications for housing allowance
|
||||
// (bostøtte). The gate fails when the project description does not name that
|
||||
// domain, or names the building-permit domain (byggesak) the reports never
|
||||
// mention.
|
||||
//
|
||||
// Chose a pinned domain term shared with the classify fixture over a generic
|
||||
// word-overlap score, because the old description already shared generic words
|
||||
// (kommune, chatbot, innbyggere) with the reports and would have passed.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const pluginRoot = fileURLToPath(new URL('../../', import.meta.url));
|
||||
const PLAYGROUND_HTML = 'playground/ms-ai-architect-playground.html';
|
||||
const BUILD_SCRIPT = 'scripts/build-demo-state.mjs';
|
||||
const CLASSIFY_FIXTURE = 'playground/test-fixtures/classify.md';
|
||||
const DOMAIN_TERM = 'bostøtte';
|
||||
const FOREIGN_TERM = 'byggesak';
|
||||
|
||||
function demoProject() {
|
||||
const html = readFileSync(pluginRoot + PLAYGROUND_HTML, 'utf8');
|
||||
const m = html.match(/<script type="application\/json" id="demo-state-v1">([\s\S]*?)<\/script>/);
|
||||
assert.ok(m, `demo-state-v1 block not found in ${PLAYGROUND_HTML}`);
|
||||
const state = JSON.parse(m[1]);
|
||||
const project = state.projects.find((p) => p.id === state.activeProjectId);
|
||||
assert.ok(project, `active demo project ${state.activeProjectId} not found`);
|
||||
return project;
|
||||
}
|
||||
|
||||
function classifySystemDescription() {
|
||||
const md = readFileSync(pluginRoot + CLASSIFY_FIXTURE, 'utf8');
|
||||
const line = md.split('\n').find((l) => /^\s*(\*\*)?Beskrivelse:/.test(l));
|
||||
assert.ok(line, `no "Beskrivelse:" line in ${CLASSIFY_FIXTURE}`);
|
||||
return line;
|
||||
}
|
||||
|
||||
test('the classify fixture describes the system with the pinned domain term', () => {
|
||||
assert.match(classifySystemDescription().toLowerCase(), new RegExp(DOMAIN_TERM));
|
||||
});
|
||||
|
||||
test('the demo project description names the domain its reports are about', () => {
|
||||
const { description } = demoProject();
|
||||
assert.match(
|
||||
description.toLowerCase(),
|
||||
new RegExp(DOMAIN_TERM),
|
||||
`demo project description does not mention "${DOMAIN_TERM}": ${description}`,
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
description.toLowerCase(),
|
||||
new RegExp(FOREIGN_TERM),
|
||||
`demo project description mentions "${FOREIGN_TERM}", which no report is about: ${description}`,
|
||||
);
|
||||
});
|
||||
|
||||
test('the inlined demo project description matches the build script', () => {
|
||||
const { description } = demoProject();
|
||||
const source = readFileSync(pluginRoot + BUILD_SCRIPT, 'utf8');
|
||||
assert.ok(
|
||||
source.includes(description),
|
||||
`${BUILD_SCRIPT} does not contain the inlined description; rerun node ${BUILD_SCRIPT}`,
|
||||
);
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue