voyage/lib/review/pre-existing.mjs

// lib/review/pre-existing.mjs
// Pre-existing findings (review_version 1.2, fix round 4.5b2 part B): a
// finding on a line the review range did not add or change. The mark is read
// from git and never trusted as written: `lib/review/review-finalize.mjs`
// writes it (`markFindings`), and the verdict gate re-derives it
// (`preExistingCheck`) before a MAJOR leaves the answer set. A BLOCKER or a
// security finding counts in full whatever the mark (lib/review/review-answers.mjs
// `countingMajors`); every uncertain case counts as new.
//
// The line sets come from one `git diff -U0` of `scope_sha_start..scope_sha_end`
// with flags that no repo config changes (`--no-color`, `--no-ext-diff`,
// `--no-textconv`, `--no-relative`, `--find-renames`, explicit prefixes,
// `--ignore-submodules=none`, `core.quotePath=false`):
//   - added or changed lines are `c..c+d-1` of each `@@ -a,b +c,d @@`;
//   - a pure deletion (`d = 0`) marks lines `c` and `c+1` (both sides of it);
//   - a new or binary file is `'all'`;
//   - a renamed or copied file goes into `renamed` under both names.
// Any git failure, ENOBUFS, a range whose start is not a proper ancestor of its
// end, or a line the reader cannot parse (a quoted path included) is
// `{ok:false}`: then nothing is pre-existing.
//
// A finding's path and line must also be in the tree at `scope_sha_end`: the
// path, byte for byte, names a regular file there (mode 100644 or 100755, read
// once with `git ls-tree -r -z --full-tree`), and the line lies inside that
// file's line count. A path the diff does not name is otherwise taken as
// untouched; this check keeps a case variant (`A.mjs` on a case-insensitive
// disk), a subdirectory-relative path, a missing file or a line past the end of
// the file from passing as pre-existing. Any git failure here also means
// nothing is pre-existing.
//
// Known limit: `scope_sha_start` is read from review.md and is not bound to
// anything the gate checks (`scope_sha_end` is, by `checkReviewedCommit` in
// lib/review/reviewed-commit.mjs). A forged narrower start, for example a
// later commit of the same run, shrinks the range, so a line the run added can
// look pre-existing for a MAJOR finding. BLOCKER and security findings are
// never affected: they count in full whatever the mark.
//
// No CLI of its own.

import { spawnSync } from 'node:child_process';
import { FULL_SHA_RE, reviewedSha } from './reviewed-commit.mjs';

const MAX_BUFFER = 64 * 1024 * 1024;
const HUNK_RE = /^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(?: .*)?$/;
const REGULAR_MODES = new Set(['100644', '100755']);
const TREE_ENTRY_RE = /^(\d{6}) ([a-z]+) ([0-9a-f]{40}|[0-9a-f]{64})$/;
const HEADER_LINE_RE = /^(?:old mode \d+|new mode \d+|deleted file mode \d+|new file mode \d+|index [0-9a-f]+\.\.[0-9a-f]+(?: \d+)?|similarity index \d+%|dissimilarity index \d+%)$/;

// The argv git helper of lib/review/reviewed-commit.mjs:64-66 (copied, not exported there).
function git(repo, args) {
  return spawnSync('git', args, { cwd: repo, encoding: 'utf8', maxBuffer: MAX_BUFFER });
}

// The `notMeasured` shape of lib/review/reviewed-commit.mjs:72-75 (copied).
function notMeasured(reason) {
  return { ok: false, reason: `pre-existing not measured: ${reason}` };
}

function gitFailed(what, r) {
  const note = r.error?.code === 'ENOBUFS' ? 'output past the 64 MiB buffer'
    : (r.error?.message ?? String(r.stderr ?? '').trim().split('\n')[0]);
  return notMeasured(`git ${what} failed (${note || `exit ${r.status}`})`);
}

/** A frontmatter sha field as written (the `reviewedSha` reading, for any key): a full sha, else ''. */
function frontmatterSha(text, key) {
  const fm = /^---\r?\n([\s\S]*?)\r?\n---\r?\n/.exec(String(text ?? ''));
  const re = new RegExp(`^${key}:[ \\t]*(["']?)([^"'\\s#]*)\\1[ \\t]*(?:#.*)?$`, 'm');
  const m = fm && re.exec(fm[1]);
  return m && FULL_SHA_RE.test(m[2]) ? m[2] : '';
}

/**
 * The review's range, read raw from the frontmatter text as `reviewedSha` reads
 * `scope_sha_end`: a YAML number, `null` or a ref name is '' (no range, no mark).
 * @param {string} text - review.md
 * @returns {{ start: string, end: string }}
 */
export function scopeShas(text) {
  return { start: frontmatterSha(text, 'scope_sha_start'), end: reviewedSha(text) };
}

/** A path as the diff prints it after its prefix; null when it is quoted (not read). */
function unprefixed(path, prefix) {
  const p = path.replace(/\t$/, '');
  if (p.startsWith('"') || !p.startsWith(prefix) || p.length === prefix.length) return null;
  return p.slice(prefix.length);
}

/** The one path of a `diff --git a/P b/P` header (both halves equal), else null. */
function headerPath(rest) {
  if ((rest.length - 5) % 2 !== 0) return null;
  const n = (rest.length - 5) / 2;
  const p = rest.slice(2, 2 + n);
  return rest === `a/${p} b/${p}` && !p.startsWith('"') ? p : null;
}

/**
 * Read a `git diff -U0` text into line sets.
 * @param {string} diff
 * @returns {{ok: true, files: Map<string, Set<number>|'all'>, renamed: Set<string>} | {ok: false, reason: string}}
 */
export function parseDiff(diff) {
  const files = new Map();
  const renamed = new Set();
  let cur = null;
  let minus = 0;
  let plus = 0;
  const bad = (line) => notMeasured(`cannot read diff line ${JSON.stringify(line.slice(0, 120))}`);
  const finish = () => {
    if (!cur) return null;
    const path = cur.to ?? cur.plusPath ?? cur.header;
    if (cur.from) renamed.add(cur.from);
    if (cur.to) renamed.add(cur.to);
    if (cur.deleted) return null;
    if (!path) return 'no path';
    if (cur.isNew || cur.binary) { files.set(path, 'all'); return null; }
    const prev = files.get(path);
    if (prev === 'all') return null;
    const set = prev ?? new Set();
    for (const n of cur.lines) set.add(n);
    files.set(path, set);
    return null;
  };
  const lines = String(diff).split('\n');
  if (lines.at(-1) === '') lines.pop();
  for (const line of lines) {
    if (minus > 0 || plus > 0) {
      if (line.startsWith('-') && minus > 0) { minus--; continue; }
      if (line.startsWith('+') && plus > 0) { plus--; continue; }
      if (line.startsWith('\\')) continue;
      return bad(line);
    }
    if (line.startsWith('\\')) continue;
    if (line.startsWith('diff --git ')) {
      if (finish()) return bad(line);
      cur = { header: headerPath(line.slice('diff --git '.length)), lines: [] };
      continue;
    }
    if (!cur) return bad(line);
    let m;
    if (HEADER_LINE_RE.test(line)) {
      if (line.startsWith('new file mode')) cur.isNew = true;
      if (line.startsWith('deleted file mode')) cur.deleted = true;
    } else if ((m = /^(rename|copy) (from|to) (.+)$/.exec(line))) {
      if (m[3].startsWith('"')) return bad(line);
      cur[m[2]] = m[3];
    } else if (line.startsWith('--- ')) {
      const p = line.slice(4);
      if (p !== '/dev/null' && unprefixed(p, 'a/') === null) return bad(line);
    } else if (line.startsWith('+++ ')) {
      const p = line.slice(4);
      if (p === '/dev/null') cur.deleted = true;
      else {
        const q = unprefixed(p, 'b/');
        if (q === null) return bad(line);
        cur.plusPath = q;
      }
    } else if (/^Binary files .* differ$/.test(line)) {
      cur.binary = true;
    } else if ((m = HUNK_RE.exec(line))) {
      const b = m[2] === undefined ? 1 : Number(m[2]);
      const c = Number(m[3]);
      const d = m[4] === undefined ? 1 : Number(m[4]);
      if (d === 0) cur.lines.push(c, c + 1);
      else for (let n = c; n < c + d; n++) cur.lines.push(n);
      minus = b;
      plus = d;
    } else {
      return bad(line);
    }
  }
  if (minus > 0 || plus > 0) return notMeasured('the diff ends inside a hunk');
  if (finish()) return notMeasured('a file in the diff has no path the reader can name');
  return { ok: true, files, renamed };
}

/**
 * The regular files of the tree at `end`, by exact repo-root path: path -> blob id.
 * @returns {{ok: true, blobs: Map<string, string>} | {ok: false, reason: string}}
 */
function regularFilesAt(repo, end) {
  const r = git(repo, ['ls-tree', '-r', '-z', '--full-tree', end]);
  if (r.error || r.status !== 0) return gitFailed('ls-tree', r);
  const blobs = new Map();
  for (const entry of r.stdout.split('\0')) {
    if (entry === '') continue;
    const tab = entry.indexOf('\t');
    const m = tab === -1 ? null : TREE_ENTRY_RE.exec(entry.slice(0, tab));
    if (!m) return notMeasured(`cannot read ls-tree entry ${JSON.stringify(entry.slice(0, 120))}`);
    if (m[2] === 'blob' && REGULAR_MODES.has(m[1])) blobs.set(entry.slice(tab + 1), m[3]);
  }
  return { ok: true, blobs };
}

/** The line count of one blob (a last line without a newline counts); null when git fails. */
function blobLineCount(repo, blob) {
  const r = spawnSync('git', ['cat-file', 'blob', blob], { cwd: repo, maxBuffer: MAX_BUFFER });
  if (r.error || r.status !== 0 || !Buffer.isBuffer(r.stdout)) return null;
  const b = r.stdout;
  let n = 0;
  for (let i = b.indexOf(10); i !== -1; i = b.indexOf(10, i + 1)) n++;
  if (b.length > 0 && b[b.length - 1] !== 10) n++;
  return n;
}

/**
 * The lines `start..end` added or changed, per file (repo-root paths), plus
 * `lineCount(path)`: the line count of a regular file at `end` under exactly
 * that path, or null when there is none (or git fails). Full commit ids only;
 * `start` must be a proper ancestor of `end`.
 * @param {string} repo
 * @param {string} start
 * @param {string} end
 * @returns {{ok: true, files: Map<string, Set<number>|'all'>, renamed: Set<string>, lineCount: (path: string) => number|null} | {ok: false, reason: string}}
 */
export function changedLineSets(repo, start, end) {
  if (!repo) return notMeasured('no repo');
  if (!FULL_SHA_RE.test(String(start ?? '')) || !FULL_SHA_RE.test(String(end ?? ''))) {
    return notMeasured('scope_sha_start and scope_sha_end must be full commit ids');
  }
  if (start.toLowerCase() === end.toLowerCase()) return notMeasured('an empty range (start = end) proves nothing');
  const anc = git(repo, ['merge-base', '--is-ancestor', start, end]);
  if (anc.error || (anc.status !== 0 && anc.status !== 1)) return gitFailed('merge-base --is-ancestor', anc);
  if (anc.status === 1) return notMeasured(`${start.slice(0, 12)} is not an ancestor of ${end.slice(0, 12)}`);
  const r = git(repo, ['-c', 'core.quotePath=false', 'diff', '-U0', '--no-color', '--no-ext-diff', '--no-textconv',
    '--no-relative', '--find-renames', '--src-prefix=a/', '--dst-prefix=b/', '--ignore-submodules=none', start, end]);
  if (r.error || r.status !== 0) return gitFailed('diff', r);
  const sets = parseDiff(r.stdout);
  if (!sets.ok) return sets;
  const tree = regularFilesAt(repo, end);
  if (!tree.ok) return tree;
  const counts = new Map();
  sets.lineCount = (path) => {
    if (typeof path !== 'string' || !tree.blobs.has(path)) return null;
    if (!counts.has(path)) counts.set(path, blobLineCount(repo, tree.blobs.get(path)));
    return counts.get(path);
  };
  return sets;
}

/** A repo-relative posix path as the diff names it: no `./`, `..`, empty segment, leading `/` or backslash. */
function plainPath(p) {
  return typeof p === 'string' && p !== '' && !p.includes('\\')
    && p.split('/').every((s) => s !== '' && s !== '.' && s !== '..');
}

/**
 * True only when the sets are ok, `f.line >= 1`, the file is not renamed, its
 * entry is not `'all'`, `f.line` is not in its set, and `f.file` is a regular
 * file at `end` under exactly that path with at least `f.line` lines
 * (`sets.lineCount`; sets without it are never pre-existing).
 * @param {{file?: string, line?: number}} f
 * @param {ReturnType<typeof changedLineSets>} sets
 * @returns {boolean}
 */
export function isPreExisting(f, sets) {
  if (!sets || sets.ok !== true) return false;
  if (!f || !Number.isInteger(f.line) || f.line < 1 || !plainPath(f.file)) return false;
  if (sets.renamed.has(f.file)) return false;
  const entry = sets.files.get(f.file);
  if (entry === 'all') return false;
  if (entry && entry.has(f.line)) return false;
  if (typeof sets.lineCount !== 'function') return false;
  const n = sets.lineCount(f.file);
  return Number.isInteger(n) && f.line <= n;
}

/** True when `file` has uncommitted changes, or git cannot say (the stricter side). */
function uncommitted(repo, file) {
  const r = git(repo, ['status', '--porcelain=v1', '-z', '--', `:(top,literal)${file}`]);
  return Boolean(r.error) || r.status !== 0 || r.stdout !== '';
}

/**
 * Set `pre_existing` (a boolean) on every JSON finding of `payload`, in place.
 * With no repo or no full `start`/`end`, every mark is false; a file with
 * uncommitted changes is false.
 * @param {{findings?: object[]}} payload
 * @param {{ repo?: string, start?: string, end?: string }} range
 * @returns {object} payload
 */
export function markFindings(payload, { repo, start, end } = {}) {
  const findings = Array.isArray(payload?.findings) ? payload.findings : [];
  const sets = repo && start && end ? changedLineSets(repo, start, end) : notMeasured('no range');
  const dirty = new Map();
  for (const f of findings) {
    if (!f || typeof f !== 'object') continue;
    let pre = isPreExisting(f, sets);
    if (pre) {
      if (!dirty.has(f.file)) dirty.set(f.file, uncommitted(repo, f.file));
      pre = !dirty.get(f.file);
    }
    f.pre_existing = pre;
  }
  return payload;
}

/**
 * The gate's predicate for a review: true only when the finding says
 * `pre_existing: true` and `changedLineSets(repo, scope_sha_start,
 * scope_sha_end)` agrees through `isPreExisting`. With no repo, or no full
 * `scope_sha_start`, it is always false. The sets are read once, when a marked
 * finding is first asked about.
 * @param {string} text - review.md
 * @param {string|undefined} repo
 * @returns {(f: object) => boolean}
 */
export function preExistingCheck(text, repo) {
  const { start, end } = scopeShas(text);
  if (!repo || !start || !end) return () => false;
  let sets = null;
  return (f) => {
    if (f?.pre_existing !== true) return false;
    if (sets === null) sets = changedLineSets(repo, start, end);
    return isPreExisting(f, sets);
  };
}