ktg-plugin-marketplace/plugins/llm-security/tests/lib/doc-consistency.test.mjs

// doc-consistency.test.mjs — Static asserts that prose documentation
// stays aligned with the v2 risk-scoring model in scanners/lib/severity.mjs.
//
// Background: v7.0.0 introduced the severity-dominated v2 risk-score model
// (BLOCK ≥65, WARNING ≥15) but several prose surfaces (commands/, agents/)
// continued to emit the v1 formula (`critical*25 + ...`, BLOCK ≥61,
// WARNING ≥21). v7.1.1 fixed two of them (agents/skill-scanner-agent.md,
// templates/unified-report.md). Batch B → v7.2.0 closes the trifecta:
// commands/scan.md, commands/audit.md, agents/mcp-scanner-agent.md.
//
// This test pins the closure. If any future edit re-introduces v1 formula
// tokens in commands/ or agents/, this test fails fast.

import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { readdirSync, readFileSync, statSync } from 'node:fs';
import { join, dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';

const __filename = fileURLToPath(import.meta.url);
const PLUGIN_ROOT = resolve(dirname(__filename), '..', '..');

// v1 formula tokens that must NOT appear in commands/ or agents/.
// These are the patterns the brief's verification step 4 grep checks.
const V1_TOKENS = [
  /\bscore\s*[><]?=\s*61\b/,                        // verdict cutoff
  /\bscore\s*[><]?=\s*21\b/,                        // verdict cutoff
  /score\s*≥\s*61/,                                 // unicode variant
  /score\s*≥\s*21/,                                 // unicode variant
  /critical\s*\*\s*25/,                             // formula multiplier
  /Critical\s*[×x]\s*25/,                           // formula multiplier (table form)
  /min\(\s*100\s*,\s*critical\s*\*\s*25/i,          // full v1 formula prefix
];

function* walkMarkdown(dir) {
  for (const entry of readdirSync(dir)) {
    const full = join(dir, entry);
    const stat = statSync(full);
    if (stat.isDirectory()) {
      yield* walkMarkdown(full);
    } else if (entry.endsWith('.md')) {
      yield full;
    }
  }
}

describe('doc-consistency — v1 risk-formula tokens are absent from prose', () => {
  const COMMANDS_DIR = join(PLUGIN_ROOT, 'commands');
  const AGENTS_DIR = join(PLUGIN_ROOT, 'agents');

  for (const dir of [COMMANDS_DIR, AGENTS_DIR]) {
    for (const file of walkMarkdown(dir)) {
      const rel = file.replace(PLUGIN_ROOT + '/', '');
      it(`${rel} contains no v1 formula tokens`, () => {
        const content = readFileSync(file, 'utf-8');
        for (const token of V1_TOKENS) {
          assert.equal(
            token.test(content),
            false,
            `${rel} still contains v1 formula token matching ${token}. ` +
              `v7.2.0 unified all command/agent prose to v2 (BLOCK ≥65, WARNING ≥15). ` +
              `If a new file legitimately needs to reference v1 (e.g. CHANGELOG history), ` +
              `move that file out of commands/ or agents/.`,
          );
        }
      });
    }
  }
});

describe('doc-consistency — v2 cutoffs are documented in unified prose', () => {
  it('commands/scan.md mentions the v2 BLOCK cutoff (≥ 65)', () => {
    const content = readFileSync(join(PLUGIN_ROOT, 'commands', 'scan.md'), 'utf-8');
    assert.match(content, /score\s*[≥>=]+\s*65/);
  });

  it('commands/audit.md references riskScore() (v2 helper)', () => {
    const content = readFileSync(join(PLUGIN_ROOT, 'commands', 'audit.md'), 'utf-8');
    assert.match(content, /riskScore/);
  });

  it('agents/mcp-scanner-agent.md mentions the v2 BLOCK cutoff (≥ 65)', () => {
    const content = readFileSync(join(PLUGIN_ROOT, 'agents', 'mcp-scanner-agent.md'), 'utf-8');
    assert.match(content, /score\s*[≥>=]+\s*65/);
  });
});

// ---------------------------------------------------------------------------
// D4 (Batch C, Wave D): Hooks count must stay synchronized across three
// surfaces — the CLAUDE.md `## Hooks (N)` header, the markdown table directly
// underneath that header, and the canonical hooks/hooks.json definition.
// Drift previously masked a missing `pre-compact-scan.mjs` row in CLAUDE.md.
// This block fails fast if any of the three surfaces drift.
// ---------------------------------------------------------------------------

describe('doc-consistency — Hooks count consistency (D4)', () => {
  const CLAUDE_MD = join(PLUGIN_ROOT, 'CLAUDE.md');
  const HOOKS_JSON = join(PLUGIN_ROOT, 'hooks', 'hooks.json');

  function readHeaderNumber(text) {
    const match = text.match(/^##\s+Hooks\s*\((\d+)\)\s*$/m);
    if (!match) throw new Error('No `## Hooks (N)` header found in CLAUDE.md');
    return parseInt(match[1], 10);
  }

  function readTableRowCount(text) {
    // Section spans from `## Hooks (N)` to next `^## ` heading.
    const startIdx = text.search(/^##\s+Hooks\s*\(\d+\)\s*$/m);
    if (startIdx < 0) throw new Error('Hooks header not found');
    const tail = text.slice(startIdx);
    const nextHeader = tail.search(/\n##\s+\S/);
    const section = nextHeader > 0 ? tail.slice(0, nextHeader) : tail;
    // Count rows that look like `| \`<name>.mjs\` | ...`.
    // Excludes the header row (which uses bare `Script` not a backtick).
    const rows = section.match(/^\|\s*`[^`|]+\.mjs`\s*\|/gm) || [];
    return rows.length;
  }

  function readJsonHookCount(jsonText) {
    const parsed = JSON.parse(jsonText);
    const seen = new Set();
    for (const eventArr of Object.values(parsed.hooks || {})) {
      for (const entry of eventArr) {
        for (const h of entry.hooks || []) {
          // Dedupe by command path — a hook registered to multiple events
          // counts as one script.
          if (h.command) seen.add(h.command);
        }
      }
    }
    return seen.size;
  }

  it('header count, table row count, and hooks.json count agree', () => {
    const claudeText = readFileSync(CLAUDE_MD, 'utf-8');
    const hooksJsonText = readFileSync(HOOKS_JSON, 'utf-8');

    const headerNumber = readHeaderNumber(claudeText);
    const tableRowCount = readTableRowCount(claudeText);
    const jsonHookCount = readJsonHookCount(hooksJsonText);

    assert.equal(
      headerNumber,
      jsonHookCount,
      `CLAUDE.md '## Hooks (${headerNumber})' header disagrees with hooks/hooks.json (${jsonHookCount} hooks). ` +
        `Update the header to match.`,
    );
    assert.equal(
      tableRowCount,
      jsonHookCount,
      `CLAUDE.md hooks table has ${tableRowCount} rows but hooks/hooks.json defines ${jsonHookCount} hooks. ` +
        `Add/remove rows in the table to match.`,
    );
    assert.equal(
      headerNumber,
      tableRowCount,
      `CLAUDE.md header (${headerNumber}) and table row count (${tableRowCount}) disagree. ` +
        `These two surfaces must stay in sync.`,
    );
  });
});