#!/usr/bin/env node /** * SC-4 scenario read-test runner. * * Loads each scenario in tests/scenarios/0[1-9]-*.json, feeds the * `scannerInput` into `humanizeFinding`, and asserts that humanized * `title` / `description` / `recommendation` match the regex patterns * declared in `expectedHumanized`. The patterns encode the * brief-owner-approved ground-truth answers ("what / why / what next") * so that passing the deterministic regex match is equivalent to the * humanized output answering the three questions a reader would ask. * * Per brief-owner decision (1a) the gate is deterministic regex * matching — no human-in-the-loop step at runtime. * * Exit 0 = PASS (all scenarios match), exit 1 = FAIL. * * Usage: * node tests/scenario-read-test.mjs */ import { readdir, readFile } from 'node:fs/promises'; import { resolve, dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { humanizeFinding } from '../scanners/lib/humanizer.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const SCENARIOS_DIR = resolve(__dirname, 'scenarios'); async function loadScenarios() { const entries = await readdir(SCENARIOS_DIR); const files = entries .filter((f) => /^\d{2}-[a-z0-9-]+\.json$/.test(f)) .sort(); const scenarios = []; for (const f of files) { const raw = await readFile(join(SCENARIOS_DIR, f), 'utf8'); scenarios.push({ file: f, body: JSON.parse(raw) }); } return scenarios; } function checkPattern(field, value, pattern) { if (typeof value !== 'string') { return { ok: false, reason: `${field} is not a string (got ${typeof value})` }; } let re; try { re = new RegExp(pattern, 'i'); } catch (err) { return { ok: false, reason: `${field} pattern is not a valid regex: ${err.message}` }; } if (!re.test(value)) { return { ok: false, reason: `${field} did not match /${pattern}/i\n actual: ${JSON.stringify(value)}`, }; } return { ok: true }; } /** * Run one scenario through humanizeFinding and return per-scenario result. */ export function runOne(scenario) { const { findingId, scannerInput, expectedHumanized } = scenario.body; const humanized = humanizeFinding(scannerInput); const failures = []; for (const [field, key] of [ ['title', 'titlePattern'], ['description', 'descriptionPattern'], ['recommendation', 'recommendationPattern'], ]) { const pattern = expectedHumanized?.[key]; if (typeof pattern !== 'string' || pattern.length === 0) { failures.push({ field, reason: `missing or empty pattern key "${key}"` }); continue; } const r = checkPattern(field, humanized?.[field], pattern); if (!r.ok) failures.push({ field, reason: r.reason }); } // Sanity: humanizer-added structural fields must be present for (const sysField of ['userImpactCategory', 'userActionLanguage', 'relevanceContext']) { if (typeof humanized?.[sysField] !== 'string' || humanized[sysField].length === 0) { failures.push({ field: sysField, reason: `expected non-empty string from humanizer; got ${JSON.stringify(humanized?.[sysField])}`, }); } } return { file: scenario.file, findingId, humanized, failures }; } /** * Run every scenario, returning aggregate results. */ export async function runAll() { const scenarios = await loadScenarios(); const results = scenarios.map(runOne); const failed = results.filter((r) => r.failures.length > 0); return { scenarios: results, failed, passed: results.length - failed.length, total: results.length }; } async function main() { const { scenarios, failed, passed, total } = await runAll(); if (total === 0) { process.stderr.write('SC-4 FAIL: no scenarios found in tests/scenarios/\n'); process.exit(1); } if (failed.length === 0) { process.stderr.write( `SC-4 PASS: ${passed}/${total} scenarios match humanizer output\n`, ); for (const r of scenarios) { process.stderr.write(` ${r.file} (${r.findingId}) - OK\n`); } process.exit(0); } process.stderr.write(`SC-4 FAIL: ${failed.length}/${total} scenarios did not match\n`); for (const r of failed) { process.stderr.write(`\n ${r.file} (${r.findingId})\n`); for (const f of r.failures) { process.stderr.write(` [${f.field}] ${f.reason}\n`); } } process.exit(1); } const isDirectRun = process.argv[1] && resolve(process.argv[1]) === resolve(new URL(import.meta.url).pathname); if (isDirectRun) { main().catch((err) => { process.stderr.write(`Scenario runner error: ${err.message}\n`); process.exit(2); }); }