// tests/lib/profile-stats-fields.test.mjs // SC #11 contract-test per brief design — kombinasjonen av: // (a) fixture-records valideres som JSONL-contracts AND // (b) command prose promises no profile key its stats record does not write // er den brief-designede gating-mekanismen. Faktisk runtime-emission av // feltene er LLM-prose-driven og ikke testbart i node:test alene. // (b) was turned around in fix round 4-5a (#46): it used to require the prose // to name profile_source and friends, which no stats heredoc writes. import { test } from 'node:test'; import { strict as assert } from 'node:assert'; import { readFileSync } from 'node:fs'; import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; const __dirname = dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = join(__dirname, '..', '..'); const PROFILE_FIELDS = [ 'profile', 'phase_models', 'parallel_agents', 'external_research_enabled', 'profile_source', 'model_used', 'phase_models_resolved', ]; const VALID_PROFILE_SOURCES = new Set(['flag', 'env', 'default', 'inheritance']); const COMMAND_FILES = [ 'trekbrief.md', 'trekresearch.md', 'trekplan.md', 'trekexecute.md', 'trekreview.md', 'trekcontinue.md', ]; // (a) Fixture validates as JSONL contracts test('SC #11(a): tests/fixtures/stats-with-profile.jsonl parses as JSONL', () => { const text = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'stats-with-profile.jsonl'), 'utf-8'); const lines = text.trim().split('\n').filter(Boolean); assert.equal(lines.length, 5, `expected 5 simulated stats records, got ${lines.length}`); for (const line of lines) { const record = JSON.parse(line); // throws if malformed assert.equal(typeof record, 'object'); assert.ok(record.ts, 'record missing ts'); } }); test('SC #11(a): every fixture record contains profile + profile_source', () => { const text = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'stats-with-profile.jsonl'), 'utf-8'); const records = text.trim().split('\n').filter(Boolean).map(l => JSON.parse(l)); for (const r of records) { assert.ok('profile' in r, `record missing profile: ${JSON.stringify(r)}`); assert.ok('profile_source' in r, `record missing profile_source: ${JSON.stringify(r)}`); } }); test('SC #11(a): profile_source values are in {flag, env, default, inheritance}', () => { const text = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'stats-with-profile.jsonl'), 'utf-8'); const records = text.trim().split('\n').filter(Boolean).map(l => JSON.parse(l)); for (const r of records) { assert.ok(VALID_PROFILE_SOURCES.has(r.profile_source), `profile_source "${r.profile_source}" not in valid set`); } }); test('SC #11(a): fixture coverage — all 4 profile_source values represented', () => { const text = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'stats-with-profile.jsonl'), 'utf-8'); const records = text.trim().split('\n').filter(Boolean).map(l => JSON.parse(l)); const seen = new Set(records.map(r => r.profile_source)); for (const expected of VALID_PROFILE_SOURCES) { assert.ok(seen.has(expected), `fixture missing profile_source value: ${expected}; seen: ${[...seen].join(', ')}`); } }); // (b) Command prose promises no profile key its stats record does not write // (false-confidence kompensasjon per plan-critic Major 4). A paragraph outside // a code fence that mentions stats may name a PROFILE_FIELDS key in backticks // only when the command's own `stats-append.mjs <<'JSON'` heredoc // writes it. /** The keys of every `stats-append.mjs <<'JSON'` … `JSON` heredoc in `text`. */ function statsHeredocs(text) { const out = []; for (const m of text.matchAll(/stats-append\.mjs (\S+) <<'JSON'\n([\s\S]*?)\nJSON\s*$/gm)) { out.push({ kind: m[1], keys: new Set([...m[2].matchAll(/"([A-Za-z_][A-Za-z0-9_]*)"\s*:/g)].map((k) => k[1])) }); } return out; } /** Backticked PROFILE_FIELDS keys in stats paragraphs (outside fences) that `written` lacks; sorted. */ function unwrittenStatsKeys(text, written) { const paras = []; let cur = []; let fenced = false; for (const l of text.split('\n')) { if (/^\s{0,3}(`{3,}|~{3,})/.test(l)) { fenced = !fenced; paras.push(cur); cur = []; continue; } if (fenced) continue; if (l.trim() === '') { paras.push(cur); cur = []; } else cur.push(l); } paras.push(cur); const bad = new Set(); for (const p of paras.map((x) => x.join('\n')).filter((p) => /stats/i.test(p))) { for (const m of p.matchAll(/`([^`\n]+)`/g)) { if (PROFILE_FIELDS.includes(m[1]) && !written.has(m[1])) bad.add(m[1]); } } return [...bad].sort(); } for (const filename of COMMAND_FILES) { test(`SC #11(b): commands/${filename} stats prose promises no profile key its stats record does not write`, () => { const content = readFileSync(join(REPO_ROOT, 'commands', filename), 'utf-8'); const docs = statsHeredocs(content); assert.equal(docs.length, 1, `${filename}: expected exactly one stats-append.mjs heredoc; found ${docs.length}`); assert.ok(docs[0].keys.has('ts'), `${filename}: the stats heredoc's keys were not read`); assert.deepEqual(unwrittenStatsKeys(content, docs[0].keys), [], `${filename} stats prose promises keys its stats record does not write`); assert.doesNotMatch(content, /profile_source/, `${filename} names profile_source, a key no stats record writes`); }); } test('SC #11(b): commands/trekplan.md stats prose names neither phase_models nor parallel_agents, which its record does not write', () => { const content = readFileSync(join(REPO_ROOT, 'commands', 'trekplan.md'), 'utf-8'); const docs = statsHeredocs(content); assert.equal(docs.length, 1, `trekplan.md: expected exactly one stats-append.mjs heredoc; found ${docs.length}`); assert.equal(docs[0].keys.has('phase_models'), false, 'the trekplan stats record now writes phase_models: update this test'); assert.equal(docs[0].keys.has('parallel_agents'), false, 'the trekplan stats record now writes parallel_agents: update this test'); const bad = unwrittenStatsKeys(content, docs[0].keys); assert.ok(!bad.includes('phase_models'), 'trekplan.md stats prose promises phase_models'); assert.ok(!bad.includes('parallel_agents'), 'trekplan.md stats prose promises parallel_agents'); }); test('SC #11(b): the stats-key checker reports a planted paragraph that promises `profile`', () => { const written = statsHeredocs(readFileSync(join(REPO_ROOT, 'commands', 'trekplan.md'), 'utf-8'))[0].keys; assert.deepEqual(unwrittenStatsKeys('Stats rows carry `profile`.', written), ['profile']); }); // --- Step 9: the five STORM measurement fields ----------------------------- // Four numerics + one low-cardinality label. Without `effort` there is no axis // to group high-vs-standard runs on, and the measurement gate cannot be // computed at all. const STORM_FIELDS = [ 'effort', 'unique_sources', 'dimensions_baseline', 'conv_turns', 'empty_turns', ]; test('Step 9: a standard-run trekresearch record parses and survives applyFieldAllowlist', async () => { const { applyFieldAllowlist } = await import('../../lib/exporters/field-allowlist.mjs'); // A standard run: loop never armed, so no discovered dimensions and no turns. const raw = JSON.parse(JSON.stringify({ _schema_id: 'trekresearch', ts: '2026-08-09T12:00:00.000Z', slug: 'add-auth', mode: 'default', scope: 'both', engine: 'swarm', question: 'free prose that must never reach the exporter', project_dir: '/Users/somebody/repos/x', brief_path: '/Users/somebody/repos/x/brief.md', dimensions: 4, dimensions_baseline: 4, conv_turns: 0, empty_turns: 0, unique_sources: 11, effort: 'standard', agents_local: 5, agents_external: 4, gemini_used: false, confidence: 0.8, contradictions: 1, open_questions: 2, profile: 'premium', profile_source: 'default', })); assert.equal(raw.dimensions_baseline, raw.dimensions, 'a standard run discovers no dimensions — baseline must equal the final count'); const out = applyFieldAllowlist(raw, 'trekresearch'); for (const field of STORM_FIELDS) { assert.ok(field in out, `${field} must survive the trekresearch allowlist`); } assert.equal(out.conv_turns, 0); assert.equal(out.empty_turns, 0); assert.equal(out.effort, 'standard'); // Deny-by-omission must still hold for the PII-ish fields. for (const denied of ['question', 'project_dir', 'brief_path']) { assert.equal(denied in out, false, `${denied} must NOT reach the exporter`); } }); test('Step 9: commands/trekresearch.md prose names all five measurement fields', () => { const content = readFileSync(join(REPO_ROOT, 'commands', 'trekresearch.md'), 'utf-8'); for (const field of STORM_FIELDS) { assert.ok(content.includes(field), `trekresearch.md prose must name ${field} — an emitted-but-undocumented field is unauditable`); } }); test('Step 9: tests/fixtures/jsonl-schemas.md trekresearch row lists the five fields', () => { const doc = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'jsonl-schemas.md'), 'utf-8'); const row = doc.split('\n').find(l => l.startsWith('| trekresearch-stats ')); assert.ok(row, 'jsonl-schemas.md is missing the trekresearch-stats row'); for (const field of STORM_FIELDS) { assert.ok(row.includes(field), `authoring reference must list ${field}`); } }); test('SC #11(b): commands/trekresearch.md prose mentions external_research_enabled', () => { const content = readFileSync(join(REPO_ROOT, 'commands', 'trekresearch.md'), 'utf-8'); assert.match(content, /external_research_enabled/, 'trekresearch.md prose must mention external_research_enabled (additive stats field)'); });