voyage/lib/verification/test-integrity.mjs

// lib/verification/test-integrity.mjs
// The floor guard (P9 (2)): read a step's diff and report every change that
// makes the tests weaker instead of the code better.
//
//   removed-assert      a test file loses more assertions than it gains
//   deleted-test        a test file loses more test declarations than it gains
//   skipped-test        a test is skipped, marked todo/xfail, or focused (.only)
//   muted-check         a check is silenced: `|| true`, continue-on-error,
//                       --passWithNoTests, an inline lint/type suppression ...
//   stub                non-test code gains a "not implemented" body
//   loosened-threshold  a numeric bound in a test or a check config moves in the
//                       permissive direction (`ms < 200` -> `ms < 2000`)
//
// Exit 0 = clean, 1 = findings, 2 = could not run (no input, unreadable input,
// an unresolvable --base). 2 is never green: an absent measurement is reported
// as absent, not as a pass.
//
// A finding is not a verdict. /trekexecute hands every finding to its
// controller, which either quotes the plan line that asked for the change or
// fails the step; nothing is dropped silently.
//
// Borrowed: the floor guard of addyosmani/agent-skills@9d0c60d (MIT),
// constraint-driven development - mechanized here as a diff reader so a
// command, not the model's word, says whether a test was weakened.

import { readFileSync, realpathSync } from 'node:fs';
import { spawnSync } from 'node:child_process';
import { resolve } from 'node:path';
import { fileURLToPath } from 'node:url';

// --- what counts as a test / a check config ----------------------------------

const TEST_PATH_RE = [
  /(^|\/)(tests?|__tests__|spec|specs)\//,
  /\.(test|spec)\.[a-z0-9]+$/i,
  /_(test|spec)\.[a-z0-9]+$/i,
  /(^|\/)test_[^/]*\.py$/,
];

const CHECK_CONFIG_RE =
  /(^|\/)(\.github|\.forgejo|\.gitea|\.woodpecker)\/|(^|\/)(\.gitlab-ci\.yml|Makefile|package\.json|pyproject\.toml|setup\.cfg|tox\.ini|\.pre-commit-config\.yaml|tsconfig[^/]*\.json|\.eslintrc[^/]*|eslint\.config\.[a-z]+|(jest|vitest|playwright)\.config\.[a-z]+|\.c8rc[^/]*|\.nycrc[^/]*|codecov\.ya?ml)$/;

export function isTestPath(p) {
  return TEST_PATH_RE.some((re) => re.test(p));
}

const isCheckPath = (p) => isTestPath(p) || CHECK_CONFIG_RE.test(p);

// --- the patterns -------------------------------------------------------------

const ASSERT_RE =
  /\bassert\w*\b|\bexpect\s*\(|\.should\b|\bt\.(equal|is|ok|true|false|deepEqual|throws|notThrows|match)\b|\bself\.assert\w+|\brequire\.\w+\(|\bt\.(Error|Fatal)f?\(/g;

const TEST_DECL_RE =
  /^\s*(test|it|describe|suite)(\.\w+)?\s*\(|^\s*(async\s+)?def\s+test_\w*|^\s*func\s+Test\w*|^\s*#\[test\]|@Test\b/;

const SKIP_RE =
  /\b(test|it|describe|suite)\.(skip|todo|only)\s*\(|\bx(it|describe|test)\s*\(|\bf(it|describe)\s*\(|\{\s*(skip|todo)\s*:\s*(true|['"`])|@pytest\.mark\.(skip|skipif|xfail)\b|\bpytest\.skip\s*\(|@unittest\.skip|\bt\.Skip(Now|f)?\s*\(|#\[ignore\]|@(Disabled|Ignore)\b/;

// Inline suppressions silence a check wherever they appear.
const SUPPRESS_RE =
  /@ts-ignore|@ts-nocheck|eslint-disable|#\s*type:\s*ignore|#\s*noqa\b|\/\/\s*nolint\b|#\s*pragma:\s*no\s*cover|istanbul\s+ignore|c8\s+ignore/;

// Exit-code mutes count only where a check runs: tests, scripts under tests/, CI
// and check configs. In ordinary code `|| true` is a normal shell idiom.
const MUTE_RE =
  /\|\|\s*true\b|\|\|\s*:\s*($|[;)])|\|\|\s*exit\s+0\b|continue-on-error\s*:\s*true|--passWithNoTests|--pass-with-no-tests|\bset\s+\+e\b|--no-verify\b|allow_failure\s*:\s*true/;

const STUB_RE =
  /throw\s+new\s+Error\s*\(\s*['"`]\s*(not\s+implemented|unimplemented|todo)\b|\braise\s+NotImplementedError\b|\b(unimplemented|todo)!\s*\(|panic\s*\(\s*"(not\s+implemented|todo)|(\/\/|#)\s*stub\b|TODO:?\s*implement/i;

// Number after a "smaller is stricter" bound: bigger = looser.
const UPPER_BOUND_RE = /(<=?|toBeLessThan(OrEqual)?|lessThan|\bmax\w*|timeout\w*|tolerance|epsilon|delta|within|retries)\W*$/i;
// Number after a "bigger is stricter" bound: smaller = looser.
const LOWER_BOUND_RE = /(>=?|toBeGreaterThan(OrEqual)?|greaterThan|\bmin\w*|threshold|coverage|branches|lines|functions|statements)\W*$/i;

const NUM_RE = /\d+(?:\.\d+)?/g;

// --- diff parsing ---------------------------------------------------------------

/**
 * Parse a unified diff into files with hunks of {type: ' '|'-'|'+', text, line}.
 * Returns null when non-empty input carries no diff header at all.
 */
export function parseDiff(text) {
  const src = String(text ?? '');
  if (src.trim() === '') return [];
  if (!/^(diff --git |--- |\+\+\+ )/m.test(src)) return null;
  const files = [];
  let file = null;
  let hunk = null;
  let oldLine = 0;
  let newLine = 0;
  // Lines still owed to the current hunk, from its @@ header. While a hunk is
  // owed lines, every line is content - even one that reads "--- x" or
  // "+++ y" (a removed "-- x" / an added "++ y"). Headers only between hunks.
  let oldLeft = 0;
  let newLeft = 0;
  const lines = src.split('\n');
  for (const l of lines) {
    if (hunk && (oldLeft > 0 || newLeft > 0)) {
      if (l.startsWith('\\')) continue; // "\ No newline at end of file"
      if (l.startsWith('-')) { hunk.push({ type: '-', text: l.slice(1), line: oldLine++ }); oldLeft--; continue; }
      if (l.startsWith('+')) { hunk.push({ type: '+', text: l.slice(1), line: newLine++ }); newLeft--; continue; }
      if (l.startsWith(' ') || l === '') {
        hunk.push({ type: ' ', text: l.slice(1), line: newLine });
        oldLine++; newLine++; oldLeft--; newLeft--;
        continue;
      }
      // Anything else ends a malformed hunk early; fall through to headers.
      oldLeft = 0; newLeft = 0;
    }
    if (l.startsWith('diff --git ')) {
      const m = l.match(/^diff --git a\/(.+) b\/(.+)$/);
      file = { path: m ? m[2] : l.slice(11), deleted: false, hunks: [] };
      files.push(file);
      hunk = null;
      continue;
    }
    if (l.startsWith('--- ')) {
      if (!file || file.hunks.length > 0) { file = { path: '', deleted: false, hunks: [] }; files.push(file); }
      const oldPath = l.slice(4).replace(/^a\//, '');
      if (!file.path && oldPath !== '/dev/null') file.path = oldPath;
      hunk = null;
      continue;
    }
    if (l.startsWith('+++ ') && file) {
      const newPath = l.slice(4).replace(/^b\//, '');
      if (newPath === '/dev/null') file.deleted = true;
      else file.path = newPath;
      continue;
    }
    const h = l.match(/^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/);
    if (h && file) {
      oldLine = Number(h[1]);
      newLine = Number(h[3]);
      oldLeft = h[2] === undefined ? 1 : Number(h[2]);
      newLeft = h[4] === undefined ? 1 : Number(h[4]);
      hunk = [];
      file.hunks.push(hunk);
    }
  }
  return files;
}

// --- detectors ------------------------------------------------------------------

const count = (lines, re) => lines.reduce((n, l) => n + (l.text.match(new RegExp(re.source, 'g')) || []).length, 0);
const norm = (s) => s.trim().replace(/\s+/g, ' ');

function addedNotRemoved(removed, added, re) {
  const before = new Set(removed.filter((l) => re.test(l.text)).map((l) => norm(l.text)));
  return added.filter((l) => re.test(l.text) && !before.has(norm(l.text)));
}

/** A paired -/+ line whose only change is numbers; returns the looser numbers. */
function loosened(oldText, newText) {
  const shape = (s) => norm(s).replace(NUM_RE, '#');
  if (shape(oldText) !== shape(newText)) return null;
  const a = [...norm(oldText).matchAll(NUM_RE)];
  const b = [...norm(newText).matchAll(NUM_RE)];
  for (let i = 0; i < a.length; i++) {
    const before = Number(a[i][0]);
    const after = Number(b[i][0]);
    if (before === after) continue;
    const lead = norm(newText).slice(Math.max(0, b[i].index - 30), b[i].index);
    if (UPPER_BOUND_RE.test(lead) && after > before) return `${before} -> ${after}`;
    if (LOWER_BOUND_RE.test(lead) && after < before) return `${before} -> ${after}`;
  }
  return null;
}

function finding(kind, file, l, detail) {
  return { kind, file, line: l ? l.line : null, text: l ? l.text.trim() : '', ...(detail ? { detail } : {}) };
}

export function analyzeDiff(text) {
  const files = parseDiff(text);
  if (files === null) return { status: 'unreadable', findings: [], reason: 'the input is not a unified diff' };
  const findings = [];
  for (const f of files) {
    const all = f.hunks.flat();
    const removed = all.filter((l) => l.type === '-');
    const added = all.filter((l) => l.type === '+');
    const test = isTestPath(f.path);

    if (test) {
      const lostAsserts = count(removed, ASSERT_RE) - count(added, ASSERT_RE);
      if (lostAsserts > 0) {
        findings.push(finding('removed-assert', f.path, removed.find((l) => new RegExp(ASSERT_RE.source).test(l.text)), `${lostAsserts} fewer`));
      }
      const lostTests = removed.filter((l) => TEST_DECL_RE.test(l.text)).length - added.filter((l) => TEST_DECL_RE.test(l.text)).length;
      if (lostTests > 0 || (f.deleted && removed.some((l) => TEST_DECL_RE.test(l.text)))) {
        findings.push(finding('deleted-test', f.path, removed.find((l) => TEST_DECL_RE.test(l.text)), f.deleted ? 'test file deleted' : `${lostTests} fewer`));
      }
      for (const l of addedNotRemoved(removed, added, SKIP_RE)) findings.push(finding('skipped-test', f.path, l));
    }

    for (const l of addedNotRemoved(removed, added, SUPPRESS_RE)) findings.push(finding('muted-check', f.path, l));
    if (isCheckPath(f.path)) {
      for (const l of addedNotRemoved(removed, added, MUTE_RE)) findings.push(finding('muted-check', f.path, l));
      for (const h of f.hunks) {
        const minus = h.filter((l) => l.type === '-');
        const plus = h.filter((l) => l.type === '+');
        for (const m of minus) {
          for (const p of plus) {
            const d = loosened(m.text, p.text);
            if (d) { findings.push(finding('loosened-threshold', f.path, p, d)); break; }
          }
        }
      }
    }

    if (!test) for (const l of addedNotRemoved(removed, added, STUB_RE)) findings.push(finding('stub', f.path, l));
  }
  return { status: findings.length ? 'findings' : 'clean', findings };
}

// --- CLI ------------------------------------------------------------------------

const USAGE = 'usage: test-integrity.mjs (--diff <file|-> | --base <ref> [--head <ref>]) [--cwd <dir>] [--json]';

function diffFromGit(cwd, base, head) {
  const ok = spawnSync('git', ['rev-parse', '--verify', '--quiet', `${base}^{commit}`], { cwd, encoding: 'utf8' });
  if (ok.status !== 0) return { error: `cannot resolve --base ${base} in ${cwd}` };
  if (head) {
    const okHead = spawnSync('git', ['rev-parse', '--verify', '--quiet', `${head}^{commit}`], { cwd, encoding: 'utf8' });
    if (okHead.status !== 0) return { error: `cannot resolve --head ${head} in ${cwd}` };
  }
  const args = ['diff', '--no-color', '--no-ext-diff', '--find-renames', base, ...(head ? [head] : [])];
  const r = spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 });
  if (r.status !== 0) return { error: `git diff failed: ${String(r.stderr).trim()}` };
  return { text: r.stdout };
}

export function main(argv) {
  let diffPath = null;
  let base = null;
  let head = null;
  let cwd = process.cwd();
  let json = false;
  for (let i = 0; i < argv.length; i++) {
    const a = argv[i];
    if (a === '--diff' && argv[i + 1] !== undefined) diffPath = argv[++i];
    else if (a === '--base' && argv[i + 1]) base = argv[++i];
    else if (a === '--head' && argv[i + 1]) head = argv[++i];
    else if (a === '--cwd' && argv[i + 1]) cwd = resolve(argv[++i]);
    else if (a === '--json') json = true;
    else { process.stderr.write(`test-integrity: unknown argument ${a}\n${USAGE}\n`); return 2; }
  }
  if ((diffPath === null) === (base === null)) {
    process.stderr.write(`test-integrity: give exactly one of --diff or --base\n${USAGE}\n`);
    return 2;
  }
  let text;
  if (diffPath !== null) {
    try { text = readFileSync(diffPath === '-' ? 0 : diffPath, 'utf8'); }
    catch (e) { return out({ status: 'unreadable', findings: [], reason: `cannot read ${diffPath}: ${e.message}` }, json); }
  } else {
    const g = diffFromGit(cwd, base, head);
    if (g.error) return out({ status: 'unreadable', findings: [], reason: g.error }, json);
    text = g.text;
  }
  return out(analyzeDiff(text), json);
}

function out(r, json) {
  if (json) process.stdout.write(JSON.stringify(r, null, 2) + '\n');
  else if (r.status === 'unreadable') process.stdout.write(`floor guard: NOT RUN - ${r.reason} (exit 2, never green)\n`);
  else if (r.status === 'clean') process.stdout.write('floor guard: clean\n');
  else {
    process.stdout.write(`floor guard: ${r.findings.length} finding(s)\n`);
    for (const f of r.findings) process.stdout.write(`  ${f.kind}  ${f.file}:${f.line ?? '-'}  ${f.text}${f.detail ? `  (${f.detail})` : ''}\n`);
  }
  return r.status === 'clean' ? 0 : r.status === 'findings' ? 1 : 2;
}

function isMain() {
  try { return realpathSync(process.argv[1]) === realpathSync(fileURLToPath(import.meta.url)); }
  catch { return false; }
}

if (process.argv[1] && isMain()) process.exitCode = main(process.argv.slice(2));