voyage/tests/lib/profile-stats-fields.test.mjs

// tests/lib/profile-stats-fields.test.mjs
// SC #11 contract-test per brief design — kombinasjonen av:
//   (a) fixture-records valideres som JSONL-contracts AND
//   (b) command prose promises no profile key its stats record does not write
// er den brief-designede gating-mekanismen. Faktisk runtime-emission av
// feltene er LLM-prose-driven og ikke testbart i node:test alene.
// (b) was turned around in fix round 4-5a (#46): it used to require the prose
// to name profile_source and friends, which no stats heredoc writes.

import { test } from 'node:test';
import { strict as assert } from 'node:assert';
import { readFileSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';

const __dirname = dirname(fileURLToPath(import.meta.url));
const REPO_ROOT = join(__dirname, '..', '..');

const PROFILE_FIELDS = [
  'profile',
  'phase_models',
  'parallel_agents',
  'external_research_enabled',
  'profile_source',
  'model_used',
  'phase_models_resolved',
];

const VALID_PROFILE_SOURCES = new Set(['flag', 'env', 'default', 'inheritance']);

const COMMAND_FILES = [
  'trekbrief.md',
  'trekresearch.md',
  'trekplan.md',
  'trekexecute.md',
  'trekreview.md',
  'trekcontinue.md',
];

// (a) Fixture validates as JSONL contracts

test('SC #11(a): tests/fixtures/stats-with-profile.jsonl parses as JSONL', () => {
  const text = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'stats-with-profile.jsonl'), 'utf-8');
  const lines = text.trim().split('\n').filter(Boolean);
  assert.equal(lines.length, 5, `expected 5 simulated stats records, got ${lines.length}`);
  for (const line of lines) {
    const record = JSON.parse(line); // throws if malformed
    assert.equal(typeof record, 'object');
    assert.ok(record.ts, 'record missing ts');
  }
});

test('SC #11(a): every fixture record contains profile + profile_source', () => {
  const text = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'stats-with-profile.jsonl'), 'utf-8');
  const records = text.trim().split('\n').filter(Boolean).map(l => JSON.parse(l));
  for (const r of records) {
    assert.ok('profile' in r, `record missing profile: ${JSON.stringify(r)}`);
    assert.ok('profile_source' in r, `record missing profile_source: ${JSON.stringify(r)}`);
  }
});

test('SC #11(a): profile_source values are in {flag, env, default, inheritance}', () => {
  const text = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'stats-with-profile.jsonl'), 'utf-8');
  const records = text.trim().split('\n').filter(Boolean).map(l => JSON.parse(l));
  for (const r of records) {
    assert.ok(VALID_PROFILE_SOURCES.has(r.profile_source),
      `profile_source "${r.profile_source}" not in valid set`);
  }
});

test('SC #11(a): fixture coverage — all 4 profile_source values represented', () => {
  const text = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'stats-with-profile.jsonl'), 'utf-8');
  const records = text.trim().split('\n').filter(Boolean).map(l => JSON.parse(l));
  const seen = new Set(records.map(r => r.profile_source));
  for (const expected of VALID_PROFILE_SOURCES) {
    assert.ok(seen.has(expected),
      `fixture missing profile_source value: ${expected}; seen: ${[...seen].join(', ')}`);
  }
});

// (b) Command prose promises no profile key its stats record does not write
// (false-confidence kompensasjon per plan-critic Major 4). A paragraph outside
// a code fence that mentions stats may name a PROFILE_FIELDS key in backticks
// only when the command's own `stats-append.mjs <kind> <<'JSON'` heredoc
// writes it.

/** The keys of every `stats-append.mjs <kind> <<'JSON'` … `JSON` heredoc in `text`. */
function statsHeredocs(text) {
  const out = [];
  for (const m of text.matchAll(/stats-append\.mjs (\S+) <<'JSON'\n([\s\S]*?)\nJSON\s*$/gm)) {
    out.push({ kind: m[1], keys: new Set([...m[2].matchAll(/"([A-Za-z_][A-Za-z0-9_]*)"\s*:/g)].map((k) => k[1])) });
  }
  return out;
}

/** Backticked PROFILE_FIELDS keys in stats paragraphs (outside fences) that `written` lacks; sorted. */
function unwrittenStatsKeys(text, written) {
  const paras = [];
  let cur = [];
  let fenced = false;
  for (const l of text.split('\n')) {
    if (/^\s{0,3}(`{3,}|~{3,})/.test(l)) { fenced = !fenced; paras.push(cur); cur = []; continue; }
    if (fenced) continue;
    if (l.trim() === '') { paras.push(cur); cur = []; } else cur.push(l);
  }
  paras.push(cur);
  const bad = new Set();
  for (const p of paras.map((x) => x.join('\n')).filter((p) => /stats/i.test(p))) {
    for (const m of p.matchAll(/`([^`\n]+)`/g)) {
      if (PROFILE_FIELDS.includes(m[1]) && !written.has(m[1])) bad.add(m[1]);
    }
  }
  return [...bad].sort();
}

for (const filename of COMMAND_FILES) {
  test(`SC #11(b): commands/${filename} stats prose promises no profile key its stats record does not write`, () => {
    const content = readFileSync(join(REPO_ROOT, 'commands', filename), 'utf-8');
    const docs = statsHeredocs(content);
    assert.equal(docs.length, 1, `${filename}: expected exactly one stats-append.mjs heredoc; found ${docs.length}`);
    assert.ok(docs[0].keys.has('ts'), `${filename}: the stats heredoc's keys were not read`);
    assert.deepEqual(unwrittenStatsKeys(content, docs[0].keys), [],
      `${filename} stats prose promises keys its stats record does not write`);
    assert.doesNotMatch(content, /profile_source/,
      `${filename} names profile_source, a key no stats record writes`);
  });
}

test('SC #11(b): commands/trekplan.md stats prose names neither phase_models nor parallel_agents, which its record does not write', () => {
  const content = readFileSync(join(REPO_ROOT, 'commands', 'trekplan.md'), 'utf-8');
  const docs = statsHeredocs(content);
  assert.equal(docs.length, 1, `trekplan.md: expected exactly one stats-append.mjs heredoc; found ${docs.length}`);
  assert.equal(docs[0].keys.has('phase_models'), false, 'the trekplan stats record now writes phase_models: update this test');
  assert.equal(docs[0].keys.has('parallel_agents'), false, 'the trekplan stats record now writes parallel_agents: update this test');
  const bad = unwrittenStatsKeys(content, docs[0].keys);
  assert.ok(!bad.includes('phase_models'), 'trekplan.md stats prose promises phase_models');
  assert.ok(!bad.includes('parallel_agents'), 'trekplan.md stats prose promises parallel_agents');
});

test('SC #11(b): the stats-key checker reports a planted paragraph that promises `profile`', () => {
  const written = statsHeredocs(readFileSync(join(REPO_ROOT, 'commands', 'trekplan.md'), 'utf-8'))[0].keys;
  assert.deepEqual(unwrittenStatsKeys('Stats rows carry `profile`.', written), ['profile']);
});

// --- Step 9: the five STORM measurement fields -----------------------------
// Four numerics + one low-cardinality label. Without `effort` there is no axis
// to group high-vs-standard runs on, and the measurement gate cannot be
// computed at all.

const STORM_FIELDS = [
  'effort',
  'unique_sources',
  'dimensions_baseline',
  'conv_turns',
  'empty_turns',
];

test('Step 9: a standard-run trekresearch record parses and survives applyFieldAllowlist', async () => {
  const { applyFieldAllowlist } = await import('../../lib/exporters/field-allowlist.mjs');
  // A standard run: loop never armed, so no discovered dimensions and no turns.
  const raw = JSON.parse(JSON.stringify({
    _schema_id: 'trekresearch',
    ts: '2026-08-09T12:00:00.000Z',
    slug: 'add-auth',
    mode: 'default',
    scope: 'both',
    engine: 'swarm',
    question: 'free prose that must never reach the exporter',
    project_dir: '/Users/somebody/repos/x',
    brief_path: '/Users/somebody/repos/x/brief.md',
    dimensions: 4,
    dimensions_baseline: 4,
    conv_turns: 0,
    empty_turns: 0,
    unique_sources: 11,
    effort: 'standard',
    agents_local: 5,
    agents_external: 4,
    gemini_used: false,
    confidence: 0.8,
    contradictions: 1,
    open_questions: 2,
    profile: 'premium',
    profile_source: 'default',
  }));

  assert.equal(raw.dimensions_baseline, raw.dimensions,
    'a standard run discovers no dimensions — baseline must equal the final count');

  const out = applyFieldAllowlist(raw, 'trekresearch');
  for (const field of STORM_FIELDS) {
    assert.ok(field in out, `${field} must survive the trekresearch allowlist`);
  }
  assert.equal(out.conv_turns, 0);
  assert.equal(out.empty_turns, 0);
  assert.equal(out.effort, 'standard');
  // Deny-by-omission must still hold for the PII-ish fields.
  for (const denied of ['question', 'project_dir', 'brief_path']) {
    assert.equal(denied in out, false, `${denied} must NOT reach the exporter`);
  }
});

test('Step 9: commands/trekresearch.md prose names all five measurement fields', () => {
  const content = readFileSync(join(REPO_ROOT, 'commands', 'trekresearch.md'), 'utf-8');
  for (const field of STORM_FIELDS) {
    assert.ok(content.includes(field),
      `trekresearch.md prose must name ${field} — an emitted-but-undocumented field is unauditable`);
  }
});

test('Step 9: tests/fixtures/jsonl-schemas.md trekresearch row lists the five fields', () => {
  const doc = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'jsonl-schemas.md'), 'utf-8');
  const row = doc.split('\n').find(l => l.startsWith('| trekresearch-stats '));
  assert.ok(row, 'jsonl-schemas.md is missing the trekresearch-stats row');
  for (const field of STORM_FIELDS) {
    assert.ok(row.includes(field), `authoring reference must list ${field}`);
  }
});

test('SC #11(b): commands/trekresearch.md prose mentions external_research_enabled', () => {
  const content = readFileSync(join(REPO_ROOT, 'commands', 'trekresearch.md'), 'utf-8');
  assert.match(content, /external_research_enabled/,
    'trekresearch.md prose must mention external_research_enabled (additive stats field)');
});