176 lines
7 KiB
JavaScript
176 lines
7 KiB
JavaScript
// tests/lib/profile-stats-fields.test.mjs
|
|
// SC #11 contract-test per brief design — kombinasjonen av:
|
|
// (a) fixture-records valideres som JSONL-contracts AND
|
|
// (b) command-prose contains field-names
|
|
// er den brief-designede gating-mekanismen. Faktisk runtime-emission av
|
|
// feltene er LLM-prose-driven og ikke testbart i node:test alene.
|
|
|
|
import { test } from 'node:test';
|
|
import { strict as assert } from 'node:assert';
|
|
import { readFileSync } from 'node:fs';
|
|
import { join, dirname } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
|
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
const REPO_ROOT = join(__dirname, '..', '..');
|
|
|
|
const PROFILE_FIELDS = [
|
|
'profile',
|
|
'phase_models',
|
|
'parallel_agents',
|
|
'external_research_enabled',
|
|
'profile_source',
|
|
];
|
|
|
|
const VALID_PROFILE_SOURCES = new Set(['flag', 'env', 'default', 'inheritance']);
|
|
|
|
const COMMAND_FILES = [
|
|
'trekbrief.md',
|
|
'trekresearch.md',
|
|
'trekplan.md',
|
|
'trekexecute.md',
|
|
'trekreview.md',
|
|
'trekcontinue.md',
|
|
];
|
|
|
|
// (a) Fixture validates as JSONL contracts
|
|
|
|
test('SC #11(a): tests/fixtures/stats-with-profile.jsonl parses as JSONL', () => {
|
|
const text = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'stats-with-profile.jsonl'), 'utf-8');
|
|
const lines = text.trim().split('\n').filter(Boolean);
|
|
assert.equal(lines.length, 5, `expected 5 simulated stats records, got ${lines.length}`);
|
|
for (const line of lines) {
|
|
const record = JSON.parse(line); // throws if malformed
|
|
assert.equal(typeof record, 'object');
|
|
assert.ok(record.ts, 'record missing ts');
|
|
}
|
|
});
|
|
|
|
test('SC #11(a): every fixture record contains profile + profile_source', () => {
|
|
const text = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'stats-with-profile.jsonl'), 'utf-8');
|
|
const records = text.trim().split('\n').filter(Boolean).map(l => JSON.parse(l));
|
|
for (const r of records) {
|
|
assert.ok('profile' in r, `record missing profile: ${JSON.stringify(r)}`);
|
|
assert.ok('profile_source' in r, `record missing profile_source: ${JSON.stringify(r)}`);
|
|
}
|
|
});
|
|
|
|
test('SC #11(a): profile_source values are in {flag, env, default, inheritance}', () => {
|
|
const text = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'stats-with-profile.jsonl'), 'utf-8');
|
|
const records = text.trim().split('\n').filter(Boolean).map(l => JSON.parse(l));
|
|
for (const r of records) {
|
|
assert.ok(VALID_PROFILE_SOURCES.has(r.profile_source),
|
|
`profile_source "${r.profile_source}" not in valid set`);
|
|
}
|
|
});
|
|
|
|
test('SC #11(a): fixture coverage — all 4 profile_source values represented', () => {
|
|
const text = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'stats-with-profile.jsonl'), 'utf-8');
|
|
const records = text.trim().split('\n').filter(Boolean).map(l => JSON.parse(l));
|
|
const seen = new Set(records.map(r => r.profile_source));
|
|
for (const expected of VALID_PROFILE_SOURCES) {
|
|
assert.ok(seen.has(expected),
|
|
`fixture missing profile_source value: ${expected}; seen: ${[...seen].join(', ')}`);
|
|
}
|
|
});
|
|
|
|
// (b) Command prose contains field-names (false-confidence kompensasjon per plan-critic Major 4)
|
|
|
|
for (const filename of COMMAND_FILES) {
|
|
test(`SC #11(b): commands/${filename} prose mentions profile + profile_source`, () => {
|
|
const content = readFileSync(join(REPO_ROOT, 'commands', filename), 'utf-8');
|
|
assert.match(content, /profile_source/,
|
|
`${filename} prose missing profile_source — Step 8 stats schema additive must be documented`);
|
|
assert.match(content, /profile/,
|
|
`${filename} prose missing profile — Step 8 stats schema additive must be documented`);
|
|
});
|
|
}
|
|
|
|
test('SC #11(b): commands/trekplan.md prose mentions phase_models + parallel_agents', () => {
|
|
const content = readFileSync(join(REPO_ROOT, 'commands', 'trekplan.md'), 'utf-8');
|
|
assert.match(content, /phase_models/,
|
|
'trekplan.md prose must mention phase_models (additive stats field)');
|
|
assert.match(content, /parallel_agents/,
|
|
'trekplan.md prose must mention parallel_agents (additive stats field)');
|
|
});
|
|
|
|
// --- Step 9: the five STORM measurement fields -----------------------------
|
|
// Four numerics + one low-cardinality label. Without `effort` there is no axis
|
|
// to group high-vs-standard runs on, and the measurement gate cannot be
|
|
// computed at all.
|
|
|
|
const STORM_FIELDS = [
|
|
'effort',
|
|
'unique_sources',
|
|
'dimensions_baseline',
|
|
'conv_turns',
|
|
'empty_turns',
|
|
];
|
|
|
|
test('Step 9: a standard-run trekresearch record parses and survives applyFieldAllowlist', async () => {
|
|
const { applyFieldAllowlist } = await import('../../lib/exporters/field-allowlist.mjs');
|
|
// A standard run: loop never armed, so no discovered dimensions and no turns.
|
|
const raw = JSON.parse(JSON.stringify({
|
|
_schema_id: 'trekresearch',
|
|
ts: '2026-08-09T12:00:00.000Z',
|
|
slug: 'add-auth',
|
|
mode: 'default',
|
|
scope: 'both',
|
|
engine: 'swarm',
|
|
question: 'free prose that must never reach the exporter',
|
|
project_dir: '/Users/somebody/repos/x',
|
|
brief_path: '/Users/somebody/repos/x/brief.md',
|
|
dimensions: 4,
|
|
dimensions_baseline: 4,
|
|
conv_turns: 0,
|
|
empty_turns: 0,
|
|
unique_sources: 11,
|
|
effort: 'standard',
|
|
agents_local: 5,
|
|
agents_external: 4,
|
|
gemini_used: false,
|
|
confidence: 0.8,
|
|
contradictions: 1,
|
|
open_questions: 2,
|
|
profile: 'premium',
|
|
profile_source: 'default',
|
|
}));
|
|
|
|
assert.equal(raw.dimensions_baseline, raw.dimensions,
|
|
'a standard run discovers no dimensions — baseline must equal the final count');
|
|
|
|
const out = applyFieldAllowlist(raw, 'trekresearch');
|
|
for (const field of STORM_FIELDS) {
|
|
assert.ok(field in out, `${field} must survive the trekresearch allowlist`);
|
|
}
|
|
assert.equal(out.conv_turns, 0);
|
|
assert.equal(out.empty_turns, 0);
|
|
assert.equal(out.effort, 'standard');
|
|
// Deny-by-omission must still hold for the PII-ish fields.
|
|
for (const denied of ['question', 'project_dir', 'brief_path']) {
|
|
assert.equal(denied in out, false, `${denied} must NOT reach the exporter`);
|
|
}
|
|
});
|
|
|
|
test('Step 9: commands/trekresearch.md prose names all five measurement fields', () => {
|
|
const content = readFileSync(join(REPO_ROOT, 'commands', 'trekresearch.md'), 'utf-8');
|
|
for (const field of STORM_FIELDS) {
|
|
assert.ok(content.includes(field),
|
|
`trekresearch.md prose must name ${field} — an emitted-but-undocumented field is unauditable`);
|
|
}
|
|
});
|
|
|
|
test('Step 9: tests/fixtures/jsonl-schemas.md trekresearch row lists the five fields', () => {
|
|
const doc = readFileSync(join(REPO_ROOT, 'tests', 'fixtures', 'jsonl-schemas.md'), 'utf-8');
|
|
const row = doc.split('\n').find(l => l.startsWith('| trekresearch-stats '));
|
|
assert.ok(row, 'jsonl-schemas.md is missing the trekresearch-stats row');
|
|
for (const field of STORM_FIELDS) {
|
|
assert.ok(row.includes(field), `authoring reference must list ${field}`);
|
|
}
|
|
});
|
|
|
|
test('SC #11(b): commands/trekresearch.md prose mentions external_research_enabled', () => {
|
|
const content = readFileSync(join(REPO_ROOT, 'commands', 'trekresearch.md'), 'utf-8');
|
|
assert.match(content, /external_research_enabled/,
|
|
'trekresearch.md prose must mention external_research_enabled (additive stats field)');
|
|
});
|