diff --git a/docs/cosmo-removal-brief-2026-06.md b/docs/cosmo-removal-brief-2026-06.md index 7d5b255..aa8ac53 100644 --- a/docs/cosmo-removal-brief-2026-06.md +++ b/docs/cosmo-removal-brief-2026-06.md @@ -34,6 +34,15 @@ Fase 1a A+B (sitemap-prefiks + skjemaløs URL-ekstraksjon) er levert (`e74646d`) - **3 omdøpte microsoftsearch-citater** (`connectors-overview`, `federated-connectors-overview`, `licensing`) — samme redirect-fiks. - Etterpå: `node scripts/kb-update/build-registry.mjs --merge` + poll fanger dem som tracked. Gjør dette som ren citat-fiks (root-cause), ikke en ny redirect-map-mekanisme. +## Sammenfall: store-fil-TOC / ref-KB Fase 1b (foldet inn hit — operatør 2026-06-26) + +Fase 1b (innholdsfortegnelse i de **20 filene >800 linjer**) ble opprinnelig skopet som «trygt håndverk, uavhengig av Cosmo». Ground truth motbeviste premisset: **11 av 12 ikke-advisor-storfiler** (og alle advisor-storfilene) har en `## …Cosmo…`-seksjon på nivå 2. En TOC bygget fra `##`-overskriftene ville da emittere en **ny Cosmo-anker** (`- [For arkitekten (Cosmo)](#…)`) → bryter «aldri ny Cosmo-innhold», og å nøytralisere overskriften er nettopp denne persona-fjerningen (ikke en sidehandling). Store-fil-TOC folder derfor inn her. + +- **Når du nøytraliserer `## …Cosmo…`-overskriften i en storfil:** kjør `node scripts/kb-update/backfill-toc.mjs --write ` i samme diff. Den setter `## Innhold` rett før første ikke-fence `## `-seksjon (samme slot som `composeKbFile`), er idempotent og fence-aware, og lister da de **nøytraliserte** overskriftene. Dry-run uten `--write` for å se diff først. +- **Mekanismen** er `transform.insertToc` (testet, `tests/kb-update/test-transform.test.mjs`). Nye filer fødes allerede med TOC via `composeKbFile` (Fase 1c, `2240f1e`). +- **Verifisering:** `node scripts/kb-eval/eval.mjs` → `checkN4 hasToc` = true for de berørte filene; ingen diff-churn på små filer (<100 linjer røres ikke). +- Eneste storfil uten `##`-Cosmo-overskrift: `zero-trust-ai-services.md` (eneste Cosmo-token er inline `**For Cosmo:**`) — kan TOC-es uten å skape Cosmo-anker, men tas naturlig i samme pass. + ## Risiko / merknader - **Ikke en bugfix-sidehandling.** Persona er en dokumentert feature — full plan + godkjenning før start (scope-guard). diff --git a/scripts/kb-update/backfill-toc.mjs b/scripts/kb-update/backfill-toc.mjs new file mode 100644 index 0000000..45fdf6b --- /dev/null +++ b/scripts/kb-update/backfill-toc.mjs @@ -0,0 +1,51 @@ +#!/usr/bin/env node +// backfill-toc.mjs — Fase 1b applier: insert a `## Innhold` TOC into EXISTING large +// reference files via transform.insertToc. Dry-run by default; --write applies in place. +// +// transform.mjs stays pure (it never writes — architecture invariant); THIS script is +// the operator-gate-side applier, the same seam commands/kb-update.md occupies when it +// writes a composed file after the gate clears. All the logic lives in the tested +// insertToc (no-op on small/already-TOC'd files, body byte-identical) — this is glue. +// +// node scripts/kb-update/backfill-toc.mjs # dry-run (report) +// node scripts/kb-update/backfill-toc.mjs --write # apply in place +// +// Per file: would/write (+N lines, M sections) | skip (small or already has TOC). +// A file whose content insertToc leaves unchanged is never rewritten (no churn). + +import { readFileSync, writeFileSync } from 'node:fs'; +import { insertToc, buildToc } from './lib/transform.mjs'; + +const argv = process.argv.slice(2); +const write = argv.includes('--write'); +const files = argv.filter((a) => a !== '--write'); + +if (files.length === 0) { + console.error('usage: backfill-toc.mjs [--write] '); + process.exit(2); +} + +let changed = 0; +let skipped = 0; +for (const path of files) { + const before = readFileSync(path, 'utf8'); + const after = insertToc(before); + if (after === before) { + skipped++; + console.log(`skip ${path} (small or already has TOC)`); + continue; + } + const addedLines = after.split('\n').length - before.split('\n').length; + const sections = (buildToc(before).match(/^- /gm) || []).length; + changed++; + if (write) { + writeFileSync(path, after); + console.log(`write ${path} (+${addedLines} lines, ${sections} sections)`); + } else { + console.log(`would ${path} (+${addedLines} lines, ${sections} sections)`); + } +} + +const verb = write ? 'wrote' : 'would change'; +const hint = write ? '' : ' — re-run with --write to apply'; +console.log(`\n${verb} ${changed}, skipped ${skipped}${hint}`); diff --git a/scripts/kb-update/lib/transform.mjs b/scripts/kb-update/lib/transform.mjs index 8ddfba3..38f1401 100644 --- a/scripts/kb-update/lib/transform.mjs +++ b/scripts/kb-update/lib/transform.mjs @@ -139,6 +139,41 @@ export function composeKbFile(meta, body) { return `${header}\n${toc}\n${b}`; } +/** + * Backfill a `## Innhold` TOC into an EXISTING large reference file that lacks one, + * placing it immediately before the first non-fenced `## ` section heading — the same + * relative slot composeKbFile gives a freshly-generated file (TOC between the header + * preamble and the first content section), but robust to the varied legacy headers on + * disk: some carry a `---` rule after the metadata, some don't. This is the Fase 1b + * retrofit for the ~20 largest hand-written files; composeKbFile covers NEW files. + * + * No-op (returns content unchanged) when the file is small (≤ TOC_MIN_LINES) or already + * carries a TOC — so it is idempotent and never churns a small file. The body below the + * insertion point is left byte-identical: only the `## Innhold` block is added. + * + * @param {string} content — full existing file content + * @returns {string} content with a `## Innhold` block inserted, or unchanged + */ +export function insertToc(content) { + const s = String(content ?? ''); + if (!isLargeContent(s) || hasToc(s)) return s; + const toc = buildToc(s); + if (!toc) return s; // no ## sections to list — nothing to anchor a TOC to + const lines = s.split('\n'); + let inFence = false; + let idx = -1; + for (let i = 0; i < lines.length; i++) { + if (/^\s*```/.test(lines[i])) { inFence = !inFence; continue; } + if (inFence) continue; + if (/^##\s+/.test(lines[i])) { idx = i; break; } + } + if (idx === -1) return s; // defensive: buildToc found a section we didn't — leave as-is + const before = lines.slice(0, idx); + while (before.length && before[before.length - 1].trim() === '') before.pop(); + const block = ['', ...toc.replace(/\n$/, '').split('\n'), '']; + return [...before, ...block, ...lines.slice(idx)].join('\n'); +} + const RE_TITLE = /^#\s+\S/m; const RE_LAST_UPDATED = /\*\*Last updated:\*\*\s*[\d-]+/i; const RE_STATUS = /\*\*Status:\*\*\s*\S/i; diff --git a/tests/kb-update/test-transform.test.mjs b/tests/kb-update/test-transform.test.mjs index 3e0c166..6ee3bdc 100644 --- a/tests/kb-update/test-transform.test.mjs +++ b/tests/kb-update/test-transform.test.mjs @@ -23,6 +23,7 @@ import { resolveTargetPath, buildToc, composeKbFile, + insertToc, } from '../../scripts/kb-update/lib/transform.mjs'; import { parseSourceHeader } from '../../scripts/kb-update/lib/kb-headers.mjs'; import { classifyChange } from '../../scripts/kb-update/lib/verify-out.mjs'; @@ -217,6 +218,89 @@ test('composeKbFile is idempotent: a body that already carries a TOC is not doub assert.equal(validateKbFile(twice).valid, true); }); +// --- insertToc: backfill a TOC into an EXISTING large file (Fase 1b) ----------- +// composeKbFile builds a NEW file from meta+body; insertToc retrofits the ~20 largest +// hand-written files on disk, whose legacy headers vary (some carry a `---` rule, some +// don't), without touching their bodies. It places `## Innhold` immediately before the +// first non-fenced `## ` section — the same relative slot composeKbFile uses. + +// Large legacy file WITHOUT a `---` header rule (the secure-model-deployment shape). +const LEGACY_NO_RULE = + '# Secure Model Deployment\n\n' + + '**Kategori:** AI Security Engineering\n' + + '**Dato:** 2026-02-05\n\n' + + '## Introduksjon\n\n' + 'Brødtekst.\n'.repeat(110) + + '\n## Container Scanning\n\nMer tekst.\n' + + '\n## Avslutning\n\nSlutt.\n'; + +// Large legacy file WITH a `---` header rule (the ros-ai-threat-library shape). +const LEGACY_WITH_RULE = + '# AI-trusselbibliotek\n\n' + + '**Sist oppdatert:** 2026-06-18\n' + + '**Kategori:** Norwegian Public Sector AI Governance\n\n' + + '---\n\n' + + '## Oversikt\n\n' + 'Brødtekst.\n'.repeat(110) + + '\n## Metode\n\nMer tekst.\n'; + +test('insertToc adds a ## Innhold before the first ## section, body untouched (no `---` header)', () => { + const out = insertToc(LEGACY_NO_RULE); + assert.equal((out.match(/^##\s+Innhold/gm) || []).length, 1); + // TOC sits before the first content section … + assert.ok(out.indexOf('## Innhold') < out.indexOf('## Introduksjon')); + // … and lists every ## section in order + assert.match(out, /- \[Introduksjon\]\(#introduksjon\)/); + assert.match(out, /- \[Container Scanning\]\(#container-scanning\)/); + assert.match(out, /- \[Avslutning\]\(#avslutning\)/); + // body from the first section onward is byte-identical (zero churn) + const tail = (s) => s.slice(s.indexOf('## Introduksjon')); + assert.equal(tail(out), tail(LEGACY_NO_RULE)); +}); + +test('insertToc places the TOC after a `---` header rule, before the first section', () => { + const out = insertToc(LEGACY_WITH_RULE); + assert.ok(out.indexOf('---') < out.indexOf('## Innhold')); + assert.ok(out.indexOf('## Innhold') < out.indexOf('## Oversikt')); + const tail = (s) => s.slice(s.indexOf('## Oversikt')); + assert.equal(tail(out), tail(LEGACY_WITH_RULE)); // body preserved +}); + +test('insertToc is a no-op on a small file (TOC is for large files only)', () => { + const small = '# T\n\n## A\n\ntekst\n\n## B\n\nmer\n'; + assert.equal(insertToc(small), small); +}); + +test('insertToc is idempotent: a file that already carries a TOC is unchanged', () => { + const once = insertToc(LEGACY_NO_RULE); + const twice = insertToc(once); + assert.equal(twice, once); + assert.equal((once.match(/^##\s+Innhold/gm) || []).length, 1); +}); + +test('insertToc turns an N4-failing large file into an N4-passing one (real-check oracle)', () => { + const before = checkN4([{ path: 'skills/x/references/y/z.md', content: LEGACY_NO_RULE }]); + assert.equal(before.largeFiles, 1); + assert.equal(before.withToc, 0); + assert.equal(before.pass, false); + const after = checkN4([{ path: 'skills/x/references/y/z.md', content: insertToc(LEGACY_NO_RULE) }]); + assert.equal(after.withToc, 1); + assert.equal(after.pass, true); +}); + +test('insertToc is fence-aware: a ## inside a code fence is neither anchor nor listed', () => { + const fenced = + '# T\n\n**Kategori:** X\n\n' + + '```md\n## Not A Real Section\n```\n\n' + + '## Faktisk Seksjon\n\n' + 'Brødtekst.\n'.repeat(110) + + '\n## Andre Seksjon\n\nx.\n'; + const out = insertToc(fenced); + assert.doesNotMatch(out, /\[Not A Real Section\]\(#/); // fenced heading not listed + assert.match(out, /- \[Faktisk Seksjon\]\(#faktisk-seksjon\)/); + assert.match(out, /- \[Andre Seksjon\]\(#andre-seksjon\)/); + // the fenced block stays above the TOC; TOC sits before the first real section + assert.ok(out.indexOf('## Not A Real Section') < out.indexOf('## Innhold')); + assert.ok(out.indexOf('## Innhold') < out.indexOf('## Faktisk Seksjon')); +}); + // --- validateKbFile: large files must carry a TOC (the write-path regression gate) test('validateKbFile flags a large file missing a TOC', () => {