llm-security/tests/av-surface.test.mjs

// av-surface.test.mjs — the antivirus surface of the tracked tree.
//
// Outcome this gates (operator, 2026-09-22): a Windows user can clone the
// repository and install the plugin without Defender or a similar product
// quarantining anything, and the plugin still finds what it finds today.
//
// What a documented AV quarantine in a security repository reacts to is a
// payload sitting on disk in the file type that would run it (a reverse shell
// in `.sh`, a webshell in `.php`, a SKILL.md clustering attack techniques) —
// not a regex table. So this test walks `git ls-files` — the same set a clone
// and the plugin cache put on a user's disk — and fails on four surfaces:
//
//   (a) a file whose non-comment content matches one of our OWN SIG rules in
//       the webshell / reverse_shell / cryptominer families, through the same
//       decode variants the SIG scanner tests (raw, decoded, homoglyph-folded,
//       rot13). The rules are reused from scanners/lib/malware-signatures.mjs;
//       no new regexes. Excluded from the denominator:
//         - code/data hosts (.mjs .js .cjs .json), where a payload is a quoted
//           string or a detection-table entry, not a runnable file. This keeps
//           the untouchable signature tables (commons JSON, golden
//           patterns.json, the supply-chain blocklist) out of a gate that must
//           reach zero. Literal payloads inside test .mjs files are therefore
//           NOT gated here; S1 moves them to test-time construction anyway.
//         - scanners/commons/**, the vendored pull-only subtree this
//           repository may not edit.
//   (b) a base64 blob of 24+ characters that decodes to printable text with
//       a COMMAND SHAPE: it starts with curl, wget, bash, sh, python, eval or
//       nc as a word, or it contains a pipe into bash/sh or `$(`. Word
//       boundaries are deliberate: a bare substring `sh` matches "should".
//       Narrowed in S2 (2026-09-22, PM decision): the S0 rule fired on `http`
//       ANYWHERE in the decoded text, which caught an SVG data URI in
//       tests/fixtures/skill-scan/hyperframes-like (xmlns="http://www.w3.org/
//       2000/svg") — markup, not a command, and not what an AV reacts to.
//       Measured before the real blobs were removed: 8 blobs in 7 files under
//       the S0 rule, 8 in 6 under this one — the SVG dropped out, and `$(`
//       caught the second half of a payload split across two blobs in
//       examples/malicious-skill-demo/security-assessment.md, which the S0
//       rule missed. All 7 command blobs the S0 rule saw still hit.
//   (b2) a line that holds a base64 decode piped into a shell AND a base64
//       blob of ANY length (8+ characters) decoding to a command shape or a
//       recursive `rm`: the whole runnable line, comments included. (b) alone
//       misses it below 24 characters: a home-directory deletion encodes to 16.
//       Added in v8.1.3 (2026-09-23) after the README fact-check found one
//       in a comment.
//   (c) a Unicode Tag (U+E0000-U+E007F), zero-width (U+200B-U+200D, U+2060,
//       U+FEFF) or bidi-control (U+202A-U+202E, U+2066-U+2069) codepoint in a
//       TEXT file outside scanners/commons/conformance/**. Binary files (a NUL
//       byte in the first 8 KiB, the rule readTextFile uses) are outside the
//       denominator: decoding compressed PNG/WOFF2 bytes as UTF-8 yields these
//       codepoints by chance, and no reader ever sees them as characters.
//   (d) any tracked file under a known payload tree (six since S3, which added
//       tests/fixtures/ast-scan/).
//   (a2) a test source (tests/**/*.mjs|.js, tests/golden/** excluded) whose RAW
//       text holds a contiguous literal matching one of the same SIG rules.
//       Raw bytes only, comments included: this is what sits on disk, and a
//       payload built at test time from fragments ('ev' + 'al') or rot13 does
//       not match here — which is the point. Added in S1 (2026-09-22) because
//       (a) keeps .mjs out of its denominator.
//   (e) a file under agents/**, commands/** or hooks/** — what a Claude Code
//       session loads at the user's end, so a quarantine there breaks the
//       installed plugin, not just a clone — whose RAW text holds a contiguous
//       literal matching one of the same SIG rules, a pipe into a shell
//       (curl/wget ... | sh|bash) or a base64 decode piped into a shell. Raw
//       text, comments included, one line at a time, every file type: a hook
//       script's comment sits on disk like any other byte. knowledge/** is
//       measured by the same rule and REPORTED as its own number in the
//       diagnostic, not gated (S3, 2026-09-22): part of it is a scanner's or
//       the attack simulator's input, and S3 records which literal stays and
//       why. Added in S3 (2026-09-22).
//
// This test was written RED on purpose (order S0, 2026-09-22): it is the
// failing test for sessions S1-S3 of the v8.1.0 plan. It is expected to fail
// until those sessions land.

import { describe, it } from 'node:test';
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { readFileSync } from 'node:fs';
import { resolve, extname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { SIGNATURE_RULES } from '../scanners/lib/malware-signatures.mjs';
import { normalizeForScan, foldHomoglyphs, rot13 } from '../scanners/lib/string-utils.mjs';

const ROOT = resolve(fileURLToPath(new URL('.', import.meta.url)), '..');

const PAYLOAD_FAMILIES = new Set(['webshell', 'reverse_shell', 'cryptominer']);
const QUOTING_HOSTS = new Set(['.mjs', '.js', '.cjs', '.json']);
const VENDORED = 'scanners/commons/';
const CONFORMANCE = 'scanners/commons/conformance/';
const TEST_SOURCE = /^tests\/(?!golden\/).*\.(?:mjs|js)$/;

// A line counts as a comment when it opens with a comment marker. `#!` is a
// shebang, not a comment, and stays in.
const COMMENT_LINE = /^\s*(?:\/\/|#(?!!)|\/\*|\*|<!--)/;

const BASE64_BLOB = /[A-Za-z0-9+/]{24,}={0,2}/g;
const PRINTABLE = /^[\x20-\x7e\t\r\n]+$/;
const COMMAND_SHAPE = /^\s*(?:curl|wget|bash|sh|python[0-9.]*|eval|nc)\b|\|\s*(?:bash|sh)\b|\$\(/;

const SHORT_BASE64_BLOB = /[A-Za-z0-9+/]{8,}={0,2}/g;
const DELETION_SHAPE = /\brm\s+-[a-zA-Z]*[rR]/;

const CARRIER = /[\u{E0000}-\u{E007F}\u200B-\u200D\u2060\uFEFF\u202A-\u202E\u2066-\u2069]/u;

const SESSION_LOADED = /^(?:agents|commands|hooks)\//;
const KNOWLEDGE = /^knowledge\//;
const PIPE_TO_SHELL = /\b(?:curl|wget)\b[^\n|]*\|\s*(?:sudo\s+)?(?:ba|z)?sh\b/i;
const BASE64_PIPE_TO_SHELL = /\bbase64\s+(?:-d|-D|--decode)\b[^\n|]*\|\s*(?:sudo\s+)?(?:ba|z)?sh\b/i;

const PAYLOAD_TREES = [
  'tests/fixtures/signature-scan/poisoned/',
  'tests/fixtures/memory-scan/poisoned-project/',
  'tests/fixtures/trigger-scan/poisoned/',
  'examples/malicious-skill-demo/evil-project-health/',
  'examples/poisoned-claude-md/fixture/',
  // S3 (2026-09-22, PM decision): Python taint fixtures. Not SIG payloads, but
  // creds-net.py is os.environ -> requests.post, the exfiltration shape AV
  // classifiers are trained on; "no payload-shaped runnable file on disk" is
  // one rule, easier to defend than an exception.
  'tests/fixtures/ast-scan/',
];

function trackedFiles() {
  return execFileSync('git', ['ls-files', '-z'], { cwd: ROOT, encoding: 'utf8' })
    .split('\0')
    .filter(Boolean);
}

function isBinary(buf) {
  return buf.subarray(0, 8192).includes(0);
}

/** Measure all four surfaces over the tracked tree. */
function measureAvSurface() {
  const files = trackedFiles();
  const rules = SIGNATURE_RULES.filter(r => PAYLOAD_FAMILIES.has(r.family));
  const a = [];
  const b = [];
  const b2 = [];
  const c = [];
  const a2 = [];
  const e = [];
  const eKnowledge = [];
  let testSources = 0;
  let textFiles = 0;
  let sessionLoaded = 0;
  let knowledgeFiles = 0;

  // (e): every hit is `rel:line [rule]`, so the report names the exact literal.
  const loadedSurfaceHits = (rel, text) => {
    const hits = [];
    text.split('\n').forEach((line, i) => {
      const ids = rules.filter(r => r.re.test(line)).map(r => r.id);
      if (PIPE_TO_SHELL.test(line)) ids.push('pipe-to-shell');
      if (BASE64_PIPE_TO_SHELL.test(line)) ids.push('base64-pipe-to-shell');
      if (ids.length > 0) hits.push(`${rel}:${i + 1} [${ids.join(', ')}]`);
    });
    return hits;
  };

  for (const rel of files) {
    if (SESSION_LOADED.test(rel)) {
      sessionLoaded++;
      e.push(...loadedSurfaceHits(rel, readFileSync(resolve(ROOT, rel), 'utf8')));
    } else if (KNOWLEDGE.test(rel)) {
      knowledgeFiles++;
      eKnowledge.push(...loadedSurfaceHits(rel, readFileSync(resolve(ROOT, rel), 'utf8')));
    }
  }

  for (const rel of files) {
    const buf = readFileSync(resolve(ROOT, rel));
    if (isBinary(buf)) continue;
    textFiles++;
    const text = buf.toString('utf8');

    if (!QUOTING_HOSTS.has(extname(rel).toLowerCase()) && !rel.startsWith(VENDORED)) {
      const body = text.split('\n').filter(line => !COMMENT_LINE.test(line)).join('\n');
      const variants = [
        body,
        normalizeForScan(body, { decodeEmbedded: true }),
        normalizeForScan(body.trim(), { decodeEmbedded: true }),
        foldHomoglyphs(body),
        rot13(body),
      ];
      const hits = rules.filter(r => variants.some(v => r.re.test(v))).map(r => r.id);
      if (hits.length > 0) a.push(`${rel} [${hits.join(', ')}]`);
    }

    for (const m of text.matchAll(BASE64_BLOB)) {
      const decoded = Buffer.from(m[0], 'base64').toString('latin1');
      if (PRINTABLE.test(decoded) && COMMAND_SHAPE.test(decoded)) {
        b.push(`${rel} :: ${decoded.slice(0, 60).replace(/\s+/g, ' ')}`);
      }
    }

    text.split('\n').forEach((line, i) => {
      if (!BASE64_PIPE_TO_SHELL.test(line)) return;
      for (const m of line.matchAll(SHORT_BASE64_BLOB)) {
        const decoded = Buffer.from(m[0], 'base64').toString('latin1');
        if (PRINTABLE.test(decoded) && (COMMAND_SHAPE.test(decoded) || DELETION_SHAPE.test(decoded))) {
          b2.push(`${rel}:${i + 1} :: ${decoded.slice(0, 60).replace(/\s+/g, ' ')}`);
        }
      }
    });

    if (!rel.startsWith(CONFORMANCE) && CARRIER.test(text)) c.push(rel);

    if (TEST_SOURCE.test(rel)) {
      testSources++;
      const hits = rules.filter(r => r.re.test(text)).map(r => r.id);
      if (hits.length > 0) a2.push(`${rel} [${hits.join(', ')}]`);
    }
  }

  const d = PAYLOAD_TREES
    .map(tree => ({ tree, files: files.filter(f => f.startsWith(tree)) }))
    .filter(t => t.files.length > 0);

  return {
    tracked: files.length,
    textFiles,
    a,
    a2,
    testSources,
    b,
    bFiles: new Set(b.map(x => x.split(' :: ')[0])).size,
    b2,
    c,
    d,
    e,
    eKnowledge,
    sessionLoaded,
    knowledgeFiles,
  };
}

const report = (label, items) => `${label}: ${items.length}\n  ${items.join('\n  ')}`;

describe('av-surface: tracked tree carries no AV-triggering payloads', () => {
  const m = measureAvSurface();

  it(`(a) no SIG payload in a runnable file type (of ${m.textFiles} text files)`, (t) => {
    t.diagnostic(`a=${m.a.length}`);
    assert.deepEqual(m.a, [], report('files with a SIG payload', m.a));
  });

  it(`(a2) no contiguous SIG payload literal in a test source (of ${m.testSources} test sources)`, (t) => {
    t.diagnostic(`a2=${m.a2.length}`);
    assert.deepEqual(m.a2, [], report('test sources with a SIG payload literal', m.a2));
  });

  it(`(b) no base64 blob decoding to a shell command (of ${m.textFiles} text files)`, (t) => {
    t.diagnostic(`b=${m.b.length} blobs in ${m.bFiles} files`);
    assert.deepEqual(m.b, [], report('base64 blobs', m.b));
  });

  it(`(b2) no runnable base64-to-shell line with a short command blob (of ${m.textFiles} text files)`, (t) => {
    t.diagnostic(`b2=${m.b2.length}`);
    assert.deepEqual(m.b2, [], report('runnable base64-to-shell lines', m.b2));
  });

  it(`(c) no Tag/zero-width/bidi carrier outside the conformance corpus (of ${m.textFiles} text files)`, (t) => {
    t.diagnostic(`c=${m.c.length}`);
    assert.deepEqual(m.c, [], report('files with a carrier codepoint', m.c));
  });

  it(`(d) no known payload tree on disk (of ${PAYLOAD_TREES.length} trees)`, (t) => {
    const lines = m.d.map(x => `${x.tree} (${x.files.length} files)`);
    t.diagnostic(`d=${m.d.length}`);
    assert.deepEqual(lines, [], report('payload trees', lines));
  });

  it(`(e) no payload literal in what a session loads (of ${m.sessionLoaded} files in agents/commands/hooks)`, (t) => {
    t.diagnostic(`e=${m.e.length}`);
    // Reported, not gated: see the header.
    t.diagnostic(report(`knowledge (of ${m.knowledgeFiles} files, not gated)`, m.eKnowledge));
    assert.deepEqual(m.e, [], report('payload literals in session-loaded files', m.e));
  });
});