ktg-plugin-marketplace/plugins/config-audit/tests/scenario-read-test.mjs

#!/usr/bin/env node
/**
 * SC-4 scenario read-test runner.
 *
 * Loads each scenario in tests/scenarios/0[1-9]-*.json, feeds the
 * `scannerInput` into `humanizeFinding`, and asserts that humanized
 * `title` / `description` / `recommendation` match the regex patterns
 * declared in `expectedHumanized`. The patterns encode the
 * brief-owner-approved ground-truth answers ("what / why / what next")
 * so that passing the deterministic regex match is equivalent to the
 * humanized output answering the three questions a reader would ask.
 *
 * Per brief-owner decision (1a) the gate is deterministic regex
 * matching — no human-in-the-loop step at runtime.
 *
 * Exit 0 = PASS (all scenarios match), exit 1 = FAIL.
 *
 * Usage:
 *   node tests/scenario-read-test.mjs
 */
import { readdir, readFile } from 'node:fs/promises';
import { resolve, dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { humanizeFinding } from '../scanners/lib/humanizer.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const SCENARIOS_DIR = resolve(__dirname, 'scenarios');

async function loadScenarios() {
  const entries = await readdir(SCENARIOS_DIR);
  const files = entries
    .filter((f) => /^\d{2}-[a-z0-9-]+\.json$/.test(f))
    .sort();
  const scenarios = [];
  for (const f of files) {
    const raw = await readFile(join(SCENARIOS_DIR, f), 'utf8');
    scenarios.push({ file: f, body: JSON.parse(raw) });
  }
  return scenarios;
}

function checkPattern(field, value, pattern) {
  if (typeof value !== 'string') {
    return { ok: false, reason: `${field} is not a string (got ${typeof value})` };
  }
  let re;
  try {
    re = new RegExp(pattern, 'i');
  } catch (err) {
    return { ok: false, reason: `${field} pattern is not a valid regex: ${err.message}` };
  }
  if (!re.test(value)) {
    return {
      ok: false,
      reason: `${field} did not match /${pattern}/i\n      actual: ${JSON.stringify(value)}`,
    };
  }
  return { ok: true };
}

/**
 * Run one scenario through humanizeFinding and return per-scenario result.
 */
export function runOne(scenario) {
  const { findingId, scannerInput, expectedHumanized } = scenario.body;
  const humanized = humanizeFinding(scannerInput);

  const failures = [];
  for (const [field, key] of [
    ['title', 'titlePattern'],
    ['description', 'descriptionPattern'],
    ['recommendation', 'recommendationPattern'],
  ]) {
    const pattern = expectedHumanized?.[key];
    if (typeof pattern !== 'string' || pattern.length === 0) {
      failures.push({ field, reason: `missing or empty pattern key "${key}"` });
      continue;
    }
    const r = checkPattern(field, humanized?.[field], pattern);
    if (!r.ok) failures.push({ field, reason: r.reason });
  }

  // Sanity: humanizer-added structural fields must be present
  for (const sysField of ['userImpactCategory', 'userActionLanguage', 'relevanceContext']) {
    if (typeof humanized?.[sysField] !== 'string' || humanized[sysField].length === 0) {
      failures.push({
        field: sysField,
        reason: `expected non-empty string from humanizer; got ${JSON.stringify(humanized?.[sysField])}`,
      });
    }
  }

  return { file: scenario.file, findingId, humanized, failures };
}

/**
 * Run every scenario, returning aggregate results.
 */
export async function runAll() {
  const scenarios = await loadScenarios();
  const results = scenarios.map(runOne);
  const failed = results.filter((r) => r.failures.length > 0);
  return { scenarios: results, failed, passed: results.length - failed.length, total: results.length };
}

async function main() {
  const { scenarios, failed, passed, total } = await runAll();

  if (total === 0) {
    process.stderr.write('SC-4 FAIL: no scenarios found in tests/scenarios/\n');
    process.exit(1);
  }

  if (failed.length === 0) {
    process.stderr.write(
      `SC-4 PASS: ${passed}/${total} scenarios match humanizer output\n`,
    );
    for (const r of scenarios) {
      process.stderr.write(`  ${r.file} (${r.findingId}) - OK\n`);
    }
    process.exit(0);
  }

  process.stderr.write(`SC-4 FAIL: ${failed.length}/${total} scenarios did not match\n`);
  for (const r of failed) {
    process.stderr.write(`\n  ${r.file} (${r.findingId})\n`);
    for (const f of r.failures) {
      process.stderr.write(`    [${f.field}] ${f.reason}\n`);
    }
  }
  process.exit(1);
}

const isDirectRun =
  process.argv[1] && resolve(process.argv[1]) === resolve(new URL(import.meta.url).pathname);
if (isDirectRun) {
  main().catch((err) => {
    process.stderr.write(`Scenario runner error: ${err.message}\n`);
    process.exit(2);
  });
}