139 lines
7.1 KiB
JavaScript
139 lines
7.1 KiB
JavaScript
#!/usr/bin/env node
|
|
// build-gold-set.mjs — Fase 0, steg 2/3 glue: assemble the gold correctness set.
|
|
//
|
|
// Consolidates the per-batch verification returns (scripts/kb-eval/data/
|
|
// fase0-returns/batch-*.json — produced by Opus 4.8 subagents that checked each
|
|
// volatile claim against live MS Learn) into a single, reusable gold set, and
|
|
// joins the DETERMINISTIC lastmod_changed signal in code (never an LLM call):
|
|
// for each file, did any cited source change on the MS Learn sitemap after the
|
|
// file's own "last updated" date? That is exactly what the existing KB staleness
|
|
// loop sees, so it tells us which genuine errors a correctness judge would catch
|
|
// that the staleness loop would miss.
|
|
//
|
|
// The non-trivial logic (date parse, lastmod comparison) lives in tested
|
|
// lib/base-rate.mjs; this CLI is thin wiring over it.
|
|
//
|
|
// Usage: node scripts/kb-eval/build-gold-set.mjs [--write]
|
|
// (default: print summary; --write: persist gold-correctness-set.json)
|
|
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { parseLastUpdated, fileLastmodChanged } from './lib/base-rate.mjs';
|
|
import { lintGoldConsistency } from './lib/gold-consistency.mjs';
|
|
|
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
const ROOT = path.resolve(__dirname, '..', '..');
|
|
const DATA = path.join(__dirname, 'data');
|
|
const RETURNS = path.join(DATA, 'fase0-returns');
|
|
|
|
const frame = JSON.parse(fs.readFileSync(path.join(DATA, 'fase0-sample-frame.json'), 'utf8'));
|
|
const registry = JSON.parse(
|
|
fs.readFileSync(path.join(ROOT, 'scripts/kb-update/data/url-registry.json'), 'utf8'),
|
|
);
|
|
|
|
// file -> citedUrls (from the deterministic sample frame)
|
|
const citedByFile = {};
|
|
for (const e of [...frame.volatile, ...frame.control]) citedByFile[e.file] = e.citedUrls;
|
|
|
|
// file -> { date, lastmod_changed } (read each file once)
|
|
const fileMeta = {};
|
|
function metaFor(file) {
|
|
if (fileMeta[file]) return fileMeta[file];
|
|
let date = null;
|
|
try {
|
|
date = parseLastUpdated(fs.readFileSync(path.join(ROOT, file), 'utf8'));
|
|
} catch {
|
|
date = null;
|
|
}
|
|
const changed = fileLastmodChanged(citedByFile[file] || [], registry, date);
|
|
return (fileMeta[file] = { date, lastmod_changed: changed });
|
|
}
|
|
|
|
// relpath after skills/<skill>/references/ for a compact, stable claim id
|
|
function relOf(file, skill) {
|
|
const prefix = `skills/${skill}/references/`;
|
|
return file.startsWith(prefix) ? file.slice(prefix.length) : file;
|
|
}
|
|
|
|
const claims = [];
|
|
const batchFiles = fs.readdirSync(RETURNS).filter((f) => /^batch-\d+\.json$/.test(f)).sort();
|
|
for (const bf of batchFiles) {
|
|
const batch = JSON.parse(fs.readFileSync(path.join(RETURNS, bf), 'utf8'));
|
|
for (const c of batch.claims) {
|
|
const meta = metaFor(c.file);
|
|
claims.push({
|
|
id: `${c.skill}/${relOf(c.file, c.skill)}#${c.n}`,
|
|
file: c.file,
|
|
skill: c.skill,
|
|
stratum: c.stratum,
|
|
claim: c.claim,
|
|
claim_type: c.claim_type,
|
|
verdict: c.verdict,
|
|
evidence_url: c.evidence_url ?? null,
|
|
lastmod_changed: meta.lastmod_changed,
|
|
file_last_updated: meta.date,
|
|
notes: c.notes ?? '',
|
|
});
|
|
}
|
|
}
|
|
|
|
const goldSet = {
|
|
_meta: {
|
|
created: '2026-06-26',
|
|
method:
|
|
'Per-file volatile-claim verification against live MS Learn by Opus 4.8 subagents (strict v2 evidence rule: correct/outdated/wrong require a fetched learn.microsoft.com quote; otherwise unsourced). lastmod_changed joined deterministically in code from url-registry sitemap_lastmod vs the file last-updated date.',
|
|
sample_frame: 'fase0-sample-frame.json',
|
|
raw_returns: 'fase0-returns/batch-*.json',
|
|
note_evidence_quote:
|
|
'Per-claim evidence_quote omitted here to bound size; full verbatim quotes live in the session transcript. notes + evidence_url capture each verdict basis.',
|
|
// G4 (programdok §8 / §3-kjennelse 2026-06-30): nedre-grense-policy for
|
|
// subagenter som bygger fremtidig gull. En nedre-grense-påstand («100+»,
|
|
// «200k+») teller som FEIL (outdated/wrong) hvis den GROVT understater
|
|
// (>~2x, beslutnings-endrende) den sanne verdien; en STRAM nedre grense (sann
|
|
// verdi i samme størrelsesorden) blir stående correct. Anvendt på FN2 (Spor 2b).
|
|
lower_bound_policy:
|
|
'Lower-bound claims (e.g. "100+", "200k+"): verdict=outdated/wrong if the bound grossly understates (>~2x, decision-changing) the true value; a tight bound (true value within the same order of magnitude) stays correct.',
|
|
// G3 (programdok §8): gull-intern-konsistens. Et verdict=correct-claim hvis
|
|
// egen `notes` innrømmer at verdien ikke er bekreftet mot kilden («uverifisert
|
|
// / illustrativ / ikke i kilden») MÅ enten relabel-es ELLER bære et
|
|
// `consistency_waiver` med begrunnelse. Håndhevet av lint-gold-consistency.mjs.
|
|
consistency_policy:
|
|
'verdict=correct claims whose notes admit the value is unverified/illustrative/not-in-source must be relabeled OR carry a consistency_waiver. Enforced by lint-gold-consistency.mjs.',
|
|
claim_count: claims.length,
|
|
},
|
|
claims,
|
|
};
|
|
|
|
// G3 (programdok §8): gull fødes konsistent. Hard gate kun på --write — et
|
|
// nybygget gull med uwaived selvmotsigende correct-claims (verdict=correct mens
|
|
// noten innrømmer ikke-bekreftet: «uverifisert/illustrativ/ikke i kilden») nektes
|
|
// SKREVET; relabel eller waiver først. Dry-run forhåndsviser (advarer, blokkerer
|
|
// ikke). Escape-hatch --allow-inconsistent (samme idiom som
|
|
// run-judge-bakeoff --allow-incomplete) for bevisst override.
|
|
const consistency = lintGoldConsistency(goldSet);
|
|
|
|
if (process.argv.includes('--write')) {
|
|
if (!consistency.ok && !process.argv.includes('--allow-inconsistent')) {
|
|
console.error(`error: ${consistency.flagged.length} uwaived selvmotsigende correct-claim(s) — gull nektes skrevet (G3):`);
|
|
for (const f of consistency.flagged) console.error(` - ${f.id} [${f.markers.join(', ')}]: ${f.notes}`);
|
|
console.error('Resolver hver (relabel ELLER consistency_waiver), eller --allow-inconsistent for å overstyre.');
|
|
process.exit(2);
|
|
}
|
|
const out = path.join(DATA, 'gold-correctness-set.json');
|
|
fs.writeFileSync(out, JSON.stringify(goldSet, null, 2) + '\n');
|
|
console.log(`wrote ${out} (${claims.length} claims)`);
|
|
} else {
|
|
const filesWithChange = Object.entries(fileMeta).filter(([, m]) => m.lastmod_changed === true).length;
|
|
const filesNoChange = Object.entries(fileMeta).filter(([, m]) => m.lastmod_changed === false).length;
|
|
const filesUnknown = Object.entries(fileMeta).filter(([, m]) => m.lastmod_changed === null).length;
|
|
console.log(`claims: ${claims.length} | files: ${Object.keys(fileMeta).length}`);
|
|
console.log(`file lastmod_changed: true=${filesWithChange} false=${filesNoChange} unknown(null date)=${filesUnknown}`);
|
|
if (consistency.ok) {
|
|
console.log('G3 gull-konsistens: OK (0 uwaived selvmotsigende correct-claims)');
|
|
} else {
|
|
console.log(`G3 gull-konsistens: ⚠ ${consistency.flagged.length} uwaived selvmotsigende correct-claim(s) — --write vil nektes:`);
|
|
for (const f of consistency.flagged) console.log(` - ${f.id} [${f.markers.join(', ')}]`);
|
|
}
|
|
console.log('(dry run — pass --write to persist gold-correctness-set.json)');
|
|
}
|