Kjell Tore Guttormsen
2fe29152b3
The `coverage` block never looked at the 61 hook invocations. It probed every
payload STRING against every injection regex in-process, so `47/83` meant
"if you threw all 61 strings at all 83 patterns, 47 would match" - not "the
reference run exercised 47". The pre-bash-destructive payloads never reach
injection-patterns at all, yet their strings were in the probe set and could
mark a pattern exercised.
That number was asserted as reference-run coverage in three places that
instruct a future session: tests/golden/README.md, the STATE golden-gate
section, and the generator's summary line. In a repo whose v7.8.2 lesson was
"the check reported success without running", a figure that measures one
thing while labelled another is the same defect wearing a different hat.
Relabelled rather than re-measured - the probe still honestly bounds the gate
(an unreachable pattern is one the corpus cannot protect under ANY
attribution), it just has to say what it is:
coverage.kind = 'static-reachability', with the caveat in a `note` field.
patternsExercised -> patternsReachable
uncoveredPatterns -> unreachablePatterns
tablesExercised -> corpusContains (a payload CONTAINS a homoglyph; it
does not say the run folded one)
The gate now pins both the `kind` and the note, so the honest label cannot be
dropped quietly by a later edit.
Also surfaced the gap the old wording hid: the four OWASP maps have NO
behavioural coverage - they are scanner-side and no hook in this corpus
reaches them. They are precisely the tables the dump was widened to cover, so
the table digest is their only protection. Stated in the README next to the
47/83 line, where a reader was previously left to infer the reference run
backed them.
Flakiness check for the new gate (61 sequential spawns, ~12s, the most
process-heavy file in the suite, added to a suite npm test runs concurrently):
three consecutive full runs, 2053/2053, 0 fail. The three known
timing-sensitive files did not destabilise.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01BB4vXvwvtW4dxbPRd6vsez
88 lines
3.1 KiB
JavaScript
88 lines
3.1 KiB
JavaScript
#!/usr/bin/env node
|
|
// golden-baseline.mjs — regenerate the v8 Phase 5 golden baseline artifacts.
|
|
//
|
|
// node scripts/golden-baseline.mjs # print a summary, write nothing
|
|
// node scripts/golden-baseline.mjs --write # (re-)bless the artifacts
|
|
//
|
|
// Re-blessing is a deliberate act. During Phase 5, a diff here means the
|
|
// table swap under test changed observable behaviour — the correct response
|
|
// is to roll the swap back, not to re-run this with --write.
|
|
|
|
import { mkdirSync, writeFileSync, existsSync, readFileSync, readdirSync } from 'node:fs';
|
|
import { resolve, dirname } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import {
|
|
buildPatternDump,
|
|
buildReferenceRun,
|
|
buildSuiteCounts,
|
|
serialize,
|
|
} from './lib/golden-dump.mjs';
|
|
|
|
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
const GOLDEN_DIR = resolve(ROOT, 'tests/golden');
|
|
const write = process.argv.includes('--write');
|
|
// Opt-in: one node process per test file, sequentially. Minutes, not seconds.
|
|
const withSuite = process.argv.includes('--suite');
|
|
|
|
const artifacts = [
|
|
['patterns.json', await buildPatternDump(ROOT)],
|
|
['reference-run.json', await buildReferenceRun(ROOT)],
|
|
];
|
|
|
|
/** `fs.globSync` is Node 22+; package.json promises >=18, so walk by hand. */
|
|
function findTestFiles(dir, acc = []) {
|
|
for (const e of readdirSync(dir, { withFileTypes: true }).sort((a, b) =>
|
|
a.name.localeCompare(b.name)
|
|
)) {
|
|
const full = resolve(dir, e.name);
|
|
if (e.isDirectory()) findTestFiles(full, acc);
|
|
else if (e.name.endsWith('.test.mjs')) acc.push(full);
|
|
}
|
|
return acc;
|
|
}
|
|
|
|
if (withSuite) {
|
|
artifacts.push([
|
|
'suite-counts.json',
|
|
buildSuiteCounts(ROOT, findTestFiles(resolve(ROOT, 'tests'))),
|
|
]);
|
|
}
|
|
|
|
mkdirSync(GOLDEN_DIR, { recursive: true });
|
|
|
|
let drift = false;
|
|
for (const [name, data] of artifacts) {
|
|
const path = resolve(GOLDEN_DIR, name);
|
|
const next = serialize(data);
|
|
const prev = existsSync(path) ? readFileSync(path, 'utf8') : null;
|
|
const status = prev === null ? 'new' : prev === next ? 'unchanged' : 'CHANGED';
|
|
if (status === 'CHANGED') drift = true;
|
|
|
|
if (write) writeFileSync(path, next);
|
|
console.log(`${write ? 'wrote' : 'checked'} tests/golden/${name} [${status}]`);
|
|
}
|
|
|
|
const [, dump] = artifacts[0];
|
|
const [, run] = artifacts[1];
|
|
console.log(
|
|
`\n regex records : ${dump.counts.regex}` +
|
|
`\n table records : ${dump.counts.table}` +
|
|
`\n file digests : ${dump.counts.file}` +
|
|
`\n reference run : ${run.summary.total} cases, ` +
|
|
`${run.summary.matchingExpectation} match expectation` +
|
|
`\n reachability : ${run.coverage.patternsReachable}/${run.coverage.patternsTotal} patterns` +
|
|
` (STATIC probe, not observed coverage — see tests/golden/README.md)`
|
|
);
|
|
|
|
const suite = artifacts.find(([n]) => n === 'suite-counts.json');
|
|
if (suite) {
|
|
const [, s] = suite;
|
|
console.log(
|
|
` suite : ${s.totals.pass} pass / ${s.totals.fail} fail across ${s.totals.files} files`
|
|
);
|
|
}
|
|
|
|
if (!write && drift) {
|
|
console.error('\nBaseline drift detected. Investigate before re-blessing with --write.');
|
|
process.exit(1);
|
|
}
|