open/ms-ai-architect
1
0
Fork 0
Code Issues Releases 1 Activity

feat(ms-ai-architect): G3 gull-intern-konsistens-lint + G4 build-instruks-policy — §8 gap-lukk [skip-docs]

Browse source
This commit is contained in:
Kjell Tore Guttormsen 2026-06-30 09:55:15 +02:00
commit 9b147b470f
6 changed files with 256 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

34
scripts/kb-eval/build-gold-set.mjs
View file

@ -20,6 +20,7 @@ import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { parseLastUpdated, fileLastmodChanged } from './lib/base-rate.mjs';
import { lintGoldConsistency } from './lib/gold-consistency.mjs';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(__dirname, '..', '..');
@ -86,12 +87,39 @@ const goldSet = {
raw_returns: 'fase0-returns/batch-*.json',
note_evidence_quote:
'Per-claim evidence_quote omitted here to bound size; full verbatim quotes live in the session transcript. notes + evidence_url capture each verdict basis.',
// G4 (programdok §8 / §3-kjennelse 2026-06-30): nedre-grense-policy for
// subagenter som bygger fremtidig gull. En nedre-grense-påstand («100+»,
// «200k+») teller som FEIL (outdated/wrong) hvis den GROVT understater
// (>~2x, beslutnings-endrende) den sanne verdien; en STRAM nedre grense (sann
// verdi i samme størrelsesorden) blir stående correct. Anvendt på FN2 (Spor 2b).
lower_bound_policy:
'Lower-bound claims (e.g. "100+", "200k+"): verdict=outdated/wrong if the bound grossly understates (>~2x, decision-changing) the true value; a tight bound (true value within the same order of magnitude) stays correct.',
// G3 (programdok §8): gull-intern-konsistens. Et verdict=correct-claim hvis
// egen `notes` innrømmer at verdien ikke er bekreftet mot kilden («uverifisert
// / illustrativ / ikke i kilden») MÅ enten relabel-es ELLER bære et
// `consistency_waiver` med begrunnelse. Håndhevet av lint-gold-consistency.mjs.
consistency_policy:
'verdict=correct claims whose notes admit the value is unverified/illustrative/not-in-source must be relabeled OR carry a consistency_waiver. Enforced by lint-gold-consistency.mjs.',
claim_count: claims.length,
},
claims,
};
// G3 (programdok §8): gull fødes konsistent. Hard gate kun på --write — et
// nybygget gull med uwaived selvmotsigende correct-claims (verdict=correct mens
// noten innrømmer ikke-bekreftet: «uverifisert/illustrativ/ikke i kilden») nektes
// SKREVET; relabel eller waiver først. Dry-run forhåndsviser (advarer, blokkerer
// ikke). Escape-hatch --allow-inconsistent (samme idiom som
// run-judge-bakeoff --allow-incomplete) for bevisst override.
const consistency = lintGoldConsistency(goldSet);
if (process.argv.includes('--write')) {
if (!consistency.ok && !process.argv.includes('--allow-inconsistent')) {
console.error(`error: ${consistency.flagged.length} uwaived selvmotsigende correct-claim(s) — gull nektes skrevet (G3):`);
for (const f of consistency.flagged) console.error(` - ${f.id} [${f.markers.join(', ')}]: ${f.notes}`);
console.error('Resolver hver (relabel ELLER consistency_waiver), eller --allow-inconsistent for å overstyre.');
process.exit(2);
}
const out = path.join(DATA, 'gold-correctness-set.json');
fs.writeFileSync(out, JSON.stringify(goldSet, null, 2) + '\n');
console.log(`wrote ${out} (${claims.length} claims)`);
@ -101,5 +129,11 @@ if (process.argv.includes('--write')) {
const filesUnknown = Object.entries(fileMeta).filter(([, m]) => m.lastmod_changed === null).length;
console.log(`claims: ${claims.length} | files: ${Object.keys(fileMeta).length}`);
console.log(`file lastmod_changed: true=${filesWithChange} false=${filesNoChange} unknown(null date)=${filesUnknown}`);
if (consistency.ok) {
console.log('G3 gull-konsistens: OK (0 uwaived selvmotsigende correct-claims)');
} else {
console.log(`G3 gull-konsistens: ⚠ ${consistency.flagged.length} uwaived selvmotsigende correct-claim(s) — --write vil nektes:`);
for (const f of consistency.flagged) console.log(` - ${f.id} [${f.markers.join(', ')}]`);
}
console.log('(dry run — pass --write to persist gold-correctness-set.json)');
}