fix(stats): no command promises a stats key its record does not write
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
074feef475
commit
95eaa8dc57
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
11 changed files with 160 additions and 41 deletions
|
|
@ -400,3 +400,97 @@ test('keep: /trekexecute\'s controller and its model note stay pinned to opus',
|
|||
assert.match(text, /`subagent_type: "general-purpose"`, `model: "opus"`, no `name`/);
|
||||
assert.match(flat(text), /controller's `model: "opus"` — so `phase_signal_result\.model` still has no spawn site here/);
|
||||
});
|
||||
|
||||
// --- #46 ---------------------------------------------------------------------------
|
||||
// Six commands said "Stats records emit `profile`, `profile_source` ..." under
|
||||
// a stats heredoc that writes none of those keys. A paragraph outside a code
|
||||
// fence that mentions stats may name a profile key only when the command's own
|
||||
// `stats-append.mjs <kind> <<'JSON'` record writes it.
|
||||
|
||||
const STATS_PROFILE_KEYS = ['profile', 'phase_models', 'parallel_agents', 'external_research_enabled',
|
||||
'profile_source', 'model_used', 'phase_models_resolved'];
|
||||
|
||||
/** The keys of every `stats-append.mjs <kind> <<'JSON'` … `JSON` heredoc in `text`. */
|
||||
function statsHeredocs(text) {
|
||||
const out = [];
|
||||
for (const m of text.matchAll(/stats-append\.mjs (\S+) <<'JSON'\n([\s\S]*?)\nJSON\s*$/gm)) {
|
||||
out.push({ kind: m[1], keys: new Set([...m[2].matchAll(/"([A-Za-z_][A-Za-z0-9_]*)"\s*:/g)].map((k) => k[1])) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every backticked profile key named in a paragraph (outside code fences,
|
||||
* split on blank lines) that mentions stats, and not in `written`; sorted.
|
||||
*/
|
||||
function unwrittenStatsKeys(text, written) {
|
||||
const paras = [];
|
||||
let cur = [];
|
||||
let fenced = false;
|
||||
for (const l of text.split('\n')) {
|
||||
if (/^\s{0,3}(`{3,}|~{3,})/.test(l)) { fenced = !fenced; paras.push(cur); cur = []; continue; }
|
||||
if (fenced) continue;
|
||||
if (l.trim() === '') { paras.push(cur); cur = []; } else cur.push(l);
|
||||
}
|
||||
paras.push(cur);
|
||||
const bad = new Set();
|
||||
for (const p of paras.map((x) => x.join('\n')).filter((p) => /stats/i.test(p))) {
|
||||
for (const m of p.matchAll(/`([^`\n]+)`/g)) {
|
||||
if (STATS_PROFILE_KEYS.includes(m[1]) && !written.has(m[1])) bad.add(m[1]);
|
||||
}
|
||||
}
|
||||
return [...bad].sort();
|
||||
}
|
||||
|
||||
for (const cmd of ['trekbrief', 'trekresearch', 'trekplan', 'trekexecute', 'trekreview', 'trekcontinue']) {
|
||||
test(`[#46] commands/${cmd}.md: every profile key its stats prose names is one its stats record writes`, () => {
|
||||
const text = read(`commands/${cmd}.md`);
|
||||
const docs = statsHeredocs(text);
|
||||
assert.equal(docs.length, 1, `${cmd}: expected exactly one stats-append.mjs heredoc; found ${docs.length}`);
|
||||
assert.equal(docs[0].kind, cmd, `${cmd}: the stats heredoc writes kind ${docs[0].kind}`);
|
||||
assert.ok(docs[0].keys.has('ts'), `${cmd}: the stats heredoc's keys were not read`);
|
||||
assert.deepEqual(unwrittenStatsKeys(text, docs[0].keys), [],
|
||||
`${cmd}: stats prose promises keys its stats record does not write`);
|
||||
});
|
||||
}
|
||||
|
||||
test('control: the stats-key checker reports a planted paragraph that promises `profile`', () => {
|
||||
const written = statsHeredocs(read('commands/trekplan.md'))[0].keys;
|
||||
assert.deepEqual(unwrittenStatsKeys('Intro.\n\nStats rows carry `profile`.\n', written), ['profile']);
|
||||
assert.deepEqual(unwrittenStatsKeys('Intro.\n\n```\nStats rows carry `profile`.\n```\n', written), []);
|
||||
assert.deepEqual(unwrittenStatsKeys('Rows carry `profile`.\n', written), []);
|
||||
});
|
||||
|
||||
test('[#46] profile_source appears in no file under commands/', () => {
|
||||
const hits = readdirSync(join(ROOT, 'commands'))
|
||||
.filter((f) => read(`commands/${f}`).includes('profile_source'));
|
||||
assert.deepEqual(hits, [], 'a command still names profile_source, a key no stats record writes');
|
||||
});
|
||||
|
||||
test('[#46] sibling: docs/profiles.md claims no profile field in the stats records', () => {
|
||||
const text = read('docs/profiles.md');
|
||||
const f = flat(text);
|
||||
assert.doesNotMatch(f, /recorded in two places/);
|
||||
assert.doesNotMatch(f, /recorded in one place/);
|
||||
assert.ok(text.split('\n').includes('The plan-file fields for the resolved value:'),
|
||||
'missing line: The plan-file fields for the resolved value:');
|
||||
assert.doesNotMatch(f, /Stats stream/);
|
||||
assert.doesNotMatch(f, /`profile_source` distinguishes/);
|
||||
assert.doesNotMatch(f, /Per-profile actuals are emitted/);
|
||||
assert.match(f, /The stats records carry no profile field, so they cannot split cost by profile\./);
|
||||
});
|
||||
|
||||
/** The one blank-line paragraph of `rel` that opens with `opening`, flattened. */
|
||||
function paraOpening(rel, opening) {
|
||||
const hits = read(rel).split(/\n[ \t]*\n/).filter((p) => p.trimStart().startsWith(opening));
|
||||
assert.equal(hits.length, 1, `${rel}: expected exactly one paragraph opening "${opening}"; found ${hits.length}`);
|
||||
return flat(hits[0]);
|
||||
}
|
||||
|
||||
for (const rel of ['README.md', 'docs/operations.md']) {
|
||||
test(`[#46] sibling: ${rel} profile paragraph says nothing is emitted to stats`, () => {
|
||||
const p = paraOpening(rel, 'Four built-in model profiles');
|
||||
assert.doesNotMatch(p, /emitted to/, `${rel}: the profile paragraph still says the profile is emitted`);
|
||||
assert.doesNotMatch(p, /stats/i, `${rel}: the profile paragraph still names stats`);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
// tests/lib/profile-stats-fields.test.mjs
|
||||
// SC #11 contract-test per brief design — kombinasjonen av:
|
||||
// (a) fixture-records valideres som JSONL-contracts AND
|
||||
// (b) command-prose contains field-names
|
||||
// (b) command prose promises no profile key its stats record does not write
|
||||
// er den brief-designede gating-mekanismen. Faktisk runtime-emission av
|
||||
// feltene er LLM-prose-driven og ikke testbart i node:test alene.
|
||||
// (b) was turned around in fix round 4-5a (#46): it used to require the prose
|
||||
// to name profile_source and friends, which no stats heredoc writes.
|
||||
|
||||
import { test } from 'node:test';
|
||||
import { strict as assert } from 'node:assert';
|
||||
|
|
@ -20,6 +22,8 @@ const PROFILE_FIELDS = [
|
|||
'parallel_agents',
|
||||
'external_research_enabled',
|
||||
'profile_source',
|
||||
'model_used',
|
||||
'phase_models_resolved',
|
||||
];
|
||||
|
||||
const VALID_PROFILE_SOURCES = new Set(['flag', 'env', 'default', 'inheritance']);
|
||||
|
|
@ -74,24 +78,68 @@ test('SC #11(a): fixture coverage — all 4 profile_source values represented',
|
|||
}
|
||||
});
|
||||
|
||||
// (b) Command prose contains field-names (false-confidence kompensasjon per plan-critic Major 4)
|
||||
// (b) Command prose promises no profile key its stats record does not write
|
||||
// (false-confidence kompensasjon per plan-critic Major 4). A paragraph outside
|
||||
// a code fence that mentions stats may name a PROFILE_FIELDS key in backticks
|
||||
// only when the command's own `stats-append.mjs <kind> <<'JSON'` heredoc
|
||||
// writes it.
|
||||
|
||||
/** The keys of every `stats-append.mjs <kind> <<'JSON'` … `JSON` heredoc in `text`. */
|
||||
function statsHeredocs(text) {
|
||||
const out = [];
|
||||
for (const m of text.matchAll(/stats-append\.mjs (\S+) <<'JSON'\n([\s\S]*?)\nJSON\s*$/gm)) {
|
||||
out.push({ kind: m[1], keys: new Set([...m[2].matchAll(/"([A-Za-z_][A-Za-z0-9_]*)"\s*:/g)].map((k) => k[1])) });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Backticked PROFILE_FIELDS keys in stats paragraphs (outside fences) that `written` lacks; sorted. */
|
||||
function unwrittenStatsKeys(text, written) {
|
||||
const paras = [];
|
||||
let cur = [];
|
||||
let fenced = false;
|
||||
for (const l of text.split('\n')) {
|
||||
if (/^\s{0,3}(`{3,}|~{3,})/.test(l)) { fenced = !fenced; paras.push(cur); cur = []; continue; }
|
||||
if (fenced) continue;
|
||||
if (l.trim() === '') { paras.push(cur); cur = []; } else cur.push(l);
|
||||
}
|
||||
paras.push(cur);
|
||||
const bad = new Set();
|
||||
for (const p of paras.map((x) => x.join('\n')).filter((p) => /stats/i.test(p))) {
|
||||
for (const m of p.matchAll(/`([^`\n]+)`/g)) {
|
||||
if (PROFILE_FIELDS.includes(m[1]) && !written.has(m[1])) bad.add(m[1]);
|
||||
}
|
||||
}
|
||||
return [...bad].sort();
|
||||
}
|
||||
|
||||
for (const filename of COMMAND_FILES) {
|
||||
test(`SC #11(b): commands/${filename} prose mentions profile + profile_source`, () => {
|
||||
test(`SC #11(b): commands/${filename} stats prose promises no profile key its stats record does not write`, () => {
|
||||
const content = readFileSync(join(REPO_ROOT, 'commands', filename), 'utf-8');
|
||||
assert.match(content, /profile_source/,
|
||||
`${filename} prose missing profile_source — Step 8 stats schema additive must be documented`);
|
||||
assert.match(content, /profile/,
|
||||
`${filename} prose missing profile — Step 8 stats schema additive must be documented`);
|
||||
const docs = statsHeredocs(content);
|
||||
assert.equal(docs.length, 1, `${filename}: expected exactly one stats-append.mjs heredoc; found ${docs.length}`);
|
||||
assert.ok(docs[0].keys.has('ts'), `${filename}: the stats heredoc's keys were not read`);
|
||||
assert.deepEqual(unwrittenStatsKeys(content, docs[0].keys), [],
|
||||
`${filename} stats prose promises keys its stats record does not write`);
|
||||
assert.doesNotMatch(content, /profile_source/,
|
||||
`${filename} names profile_source, a key no stats record writes`);
|
||||
});
|
||||
}
|
||||
|
||||
test('SC #11(b): commands/trekplan.md prose mentions phase_models + parallel_agents', () => {
|
||||
test('SC #11(b): commands/trekplan.md stats prose names neither phase_models nor parallel_agents, which its record does not write', () => {
|
||||
const content = readFileSync(join(REPO_ROOT, 'commands', 'trekplan.md'), 'utf-8');
|
||||
assert.match(content, /phase_models/,
|
||||
'trekplan.md prose must mention phase_models (additive stats field)');
|
||||
assert.match(content, /parallel_agents/,
|
||||
'trekplan.md prose must mention parallel_agents (additive stats field)');
|
||||
const docs = statsHeredocs(content);
|
||||
assert.equal(docs.length, 1, `trekplan.md: expected exactly one stats-append.mjs heredoc; found ${docs.length}`);
|
||||
assert.equal(docs[0].keys.has('phase_models'), false, 'the trekplan stats record now writes phase_models: update this test');
|
||||
assert.equal(docs[0].keys.has('parallel_agents'), false, 'the trekplan stats record now writes parallel_agents: update this test');
|
||||
const bad = unwrittenStatsKeys(content, docs[0].keys);
|
||||
assert.ok(!bad.includes('phase_models'), 'trekplan.md stats prose promises phase_models');
|
||||
assert.ok(!bad.includes('parallel_agents'), 'trekplan.md stats prose promises parallel_agents');
|
||||
});
|
||||
|
||||
test('SC #11(b): the stats-key checker reports a planted paragraph that promises `profile`', () => {
|
||||
const written = statsHeredocs(readFileSync(join(REPO_ROOT, 'commands', 'trekplan.md'), 'utf-8'))[0].keys;
|
||||
assert.deepEqual(unwrittenStatsKeys('Stats rows carry `profile`.', written), ['profile']);
|
||||
});
|
||||
|
||||
// --- Step 9: the five STORM measurement fields -----------------------------
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue