test(v8.1.3): red tests for plan points 1, 2, 3, 5 and addenda a-c

Each test fails on fd7de23:
- own-working-tree: venv site-packages, vendor/, config-dir skills/
  (punkt 1); ~/.claude/plugins next to $CLAUDE_CONFIG_DIR and a leading
  ~ in the variable (punkt 2); NODE_MODULES case variant and a
  case-mismatched parent segment (punkt 3, closes v8.1.2 punkt 4).
- watch-cron-scope: a watched project's own ignore file is honored
  (punkt 5).
- av-surface (b2): no runnable base64-to-shell line with a short
  command blob (addendum a; 3 hits today).
- doc-consistency: scanner-reference Knowledge Files matches knowledge/
  (addendum b); ci-cd-guide makes no offline claim (addendum c).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kjell Tore Guttormsen 2026-09-23 11:29:22 +02:00
commit b62c3e60f5
Signed by: ktg
SSH key fingerprint: SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
4 changed files with 290 additions and 0 deletions

View file

@ -36,6 +36,12 @@
// caught the second half of a payload split across two blobs in // caught the second half of a payload split across two blobs in
// examples/malicious-skill-demo/security-assessment.md, which the S0 // examples/malicious-skill-demo/security-assessment.md, which the S0
// rule missed. All 7 command blobs the S0 rule saw still hit. // rule missed. All 7 command blobs the S0 rule saw still hit.
// (b2) a line that holds a base64 decode piped into a shell AND a base64
// blob of ANY length (8+ characters) decoding to a command shape or a
// recursive `rm`: the whole runnable line, comments included. (b) alone
// misses it below 24 characters: a home-directory deletion encodes to 16.
// Added in v8.1.3 (2026-09-23, order 20260923T092223Z, tillegg a) after
// the README fact-check found one in a comment.
// (c) a Unicode Tag (U+E0000-U+E007F), zero-width (U+200B-U+200D, U+2060, // (c) a Unicode Tag (U+E0000-U+E007F), zero-width (U+200B-U+200D, U+2060,
// U+FEFF) or bidi-control (U+202A-U+202E, U+2066-U+2069) codepoint in a // U+FEFF) or bidi-control (U+202A-U+202E, U+2066-U+2069) codepoint in a
// TEXT file outside scanners/commons/conformance/**. Binary files (a NUL // TEXT file outside scanners/commons/conformance/**. Binary files (a NUL
@ -91,6 +97,9 @@ const BASE64_BLOB = /[A-Za-z0-9+/]{24,}={0,2}/g;
const PRINTABLE = /^[\x20-\x7e\t\r\n]+$/; const PRINTABLE = /^[\x20-\x7e\t\r\n]+$/;
const COMMAND_SHAPE = /^\s*(?:curl|wget|bash|sh|python[0-9.]*|eval|nc)\b|\|\s*(?:bash|sh)\b|\$\(/; const COMMAND_SHAPE = /^\s*(?:curl|wget|bash|sh|python[0-9.]*|eval|nc)\b|\|\s*(?:bash|sh)\b|\$\(/;
const SHORT_BASE64_BLOB = /[A-Za-z0-9+/]{8,}={0,2}/g;
const DELETION_SHAPE = /\brm\s+-[a-zA-Z]*[rR]/;
const CARRIER = /[\u{E0000}-\u{E007F}\u200B-\u200D\u2060\uFEFF\u202A-\u202E\u2066-\u2069]/u; const CARRIER = /[\u{E0000}-\u{E007F}\u200B-\u200D\u2060\uFEFF\u202A-\u202E\u2066-\u2069]/u;
const SESSION_LOADED = /^(?:agents|commands|hooks)\//; const SESSION_LOADED = /^(?:agents|commands|hooks)\//;
@ -127,6 +136,7 @@ function measureAvSurface() {
const rules = SIGNATURE_RULES.filter(r => PAYLOAD_FAMILIES.has(r.family)); const rules = SIGNATURE_RULES.filter(r => PAYLOAD_FAMILIES.has(r.family));
const a = []; const a = [];
const b = []; const b = [];
const b2 = [];
const c = []; const c = [];
const a2 = []; const a2 = [];
const e = []; const e = [];
@ -184,6 +194,16 @@ function measureAvSurface() {
} }
} }
text.split('\n').forEach((line, i) => {
if (!BASE64_PIPE_TO_SHELL.test(line)) return;
for (const m of line.matchAll(SHORT_BASE64_BLOB)) {
const decoded = Buffer.from(m[0], 'base64').toString('latin1');
if (PRINTABLE.test(decoded) && (COMMAND_SHAPE.test(decoded) || DELETION_SHAPE.test(decoded))) {
b2.push(`${rel}:${i + 1} :: ${decoded.slice(0, 60).replace(/\s+/g, ' ')}`);
}
}
});
if (!rel.startsWith(CONFORMANCE) && CARRIER.test(text)) c.push(rel); if (!rel.startsWith(CONFORMANCE) && CARRIER.test(text)) c.push(rel);
if (TEST_SOURCE.test(rel)) { if (TEST_SOURCE.test(rel)) {
@ -205,6 +225,7 @@ function measureAvSurface() {
testSources, testSources,
b, b,
bFiles: new Set(b.map(x => x.split(' :: ')[0])).size, bFiles: new Set(b.map(x => x.split(' :: ')[0])).size,
b2,
c, c,
d, d,
e, e,
@ -234,6 +255,11 @@ describe('av-surface: tracked tree carries no AV-triggering payloads', () => {
assert.deepEqual(m.b, [], report('base64 blobs', m.b)); assert.deepEqual(m.b, [], report('base64 blobs', m.b));
}); });
it(`(b2) no runnable base64-to-shell line with a short command blob (of ${m.textFiles} text files)`, (t) => {
t.diagnostic(`b2=${m.b2.length}`);
assert.deepEqual(m.b2, [], report('runnable base64-to-shell lines', m.b2));
});
it(`(c) no Tag/zero-width/bidi carrier outside the conformance corpus (of ${m.textFiles} text files)`, (t) => { it(`(c) no Tag/zero-width/bidi carrier outside the conformance corpus (of ${m.textFiles} text files)`, (t) => {
t.diagnostic(`c=${m.c.length}`); t.diagnostic(`c=${m.c.length}`);
assert.deepEqual(m.c, [], report('files with a carrier codepoint', m.c)); assert.deepEqual(m.c, [], report('files with a carrier codepoint', m.c));

View file

@ -349,6 +349,20 @@ describe('doc-consistency — inventory counts are derived from source (B10)', (
} }
}); });
// v8.1.3 (tillegg b): scanner-reference.md said "Knowledge Files (20)" while
// knowledge/ held 22 — the header and the table must both follow the dir.
it('docs/scanner-reference.md Knowledge Files header and table match knowledge/', () => {
const files = readdirSync(KNOWLEDGE_DIR).sort();
const content = readFileSync(join(PLUGIN_ROOT, 'docs', 'scanner-reference.md'), 'utf-8');
const header = content.match(/^## Knowledge Files \((\d+)\)$/m);
assert.notEqual(header, null, 'scanner-reference.md has no `## Knowledge Files (N)` header');
assert.equal(Number(header[1]), files.length,
`scanner-reference.md says Knowledge Files (${header[1]}) but knowledge/ holds ${files.length}.`);
const rows = [...content.slice(header.index).split('\n## ')[0].matchAll(/^\| `([^`]+)` \|/gm)]
.map(m => m[1]).sort();
assert.deepEqual(rows, files, 'the Knowledge Files table must list exactly the files in knowledge/');
});
// -- output.mjs finding() prefix list -------------------------------------- // -- output.mjs finding() prefix list --------------------------------------
it('output.mjs JSDoc lists every prefix passed to finding()', () => { it('output.mjs JSDoc lists every prefix passed to finding()', () => {
const actual = findingPrefixes(); const actual = findingPrefixes();
@ -411,3 +425,24 @@ describe('doc-consistency — inventory counts are derived from source (B10)', (
); );
}); });
}); });
// v8.1.3 (tillegg c): docs/ci-cd-guide.md said the standalone CLI makes zero
// network calls and that OSV is opt-in. The orchestrated scanners run
// `npm audit` and `pip-audit` (dep), resolve domains over DNS (network) and
// query OSV.dev (supply-chain), with no switch to turn any of them off —
// README says so since the v8.1.2 README rewrite.
describe('doc-consistency — ci-cd-guide tells the truth about network calls (v8.1.3)', () => {
const content = readFileSync(join(PLUGIN_ROOT, 'docs', 'ci-cd-guide.md'), 'utf-8');
it('makes no offline claim', () => {
for (const claim of [/zero network calls/i, /no external API calls/i, /OSV\.dev queries \(opt-in\)/i, /no cross-border data transfer/i]) {
assert.equal(claim.test(content), false, `ci-cd-guide.md still claims ${claim}`);
}
});
it('names every network path the CLI takes', () => {
for (const path of [/npm audit/, /pip-audit/, /DNS/, /OSV\.dev/]) {
assert.match(content, path, `ci-cd-guide.md does not name ${path}`);
}
});
});

View file

@ -194,3 +194,132 @@ describe('isOwnWorkingTree(): node_modules and the plugin dir are foreign (v8.1.
} }
}); });
}); });
// v8.1.3 (order 20260923T092223Z, PM decisions):
// punkt 1 — other install locations are foreign like `node_modules`: a
// `site-packages` segment (Python venv) or a `vendor` segment
// (composer/bundler/Go) on the path from cwd to the target, and anything
// under a Claude Code config dir's `skills/` (third-party skills copied
// into a git-tracked ~/.claude share its git root).
// punkt 2 — BOTH `~/.claude` and `$CLAUDE_CONFIG_DIR` count as config dirs;
// a leading `~` in the variable is expanded to the home dir.
// punkt 3 — case is canonicalized (realpathSync.native), so `NODE_MODULES/x`
// on a case-insensitive volume is the `node_modules` it names, and a
// case-mismatched path to an own subdir is own (§ v8.1.2 punkt 4).
describe('isOwnWorkingTree(): v8.1.3 install locations, config dirs, case', () => {
let root;
let repo;
let savedConfigDir;
let savedHome;
before(() => {
root = mkOwnTreeDir('owt-813-');
repo = join(root, 'repo');
gitInit(repo);
mkdirSync(join(repo, '.venv', 'lib', 'python3.12', 'site-packages', 'evilpkg'), { recursive: true });
mkdirSync(join(repo, 'vendor', 'acme', 'lib'), { recursive: true });
mkdirSync(join(repo, 'my-vendor-notes'), { recursive: true });
mkdirSync(join(repo, 'node_modules', 'evil'), { recursive: true });
mkdirSync(join(repo, 'Sub'), { recursive: true });
// A git-tracked config dir (cwd = it) with third-party skills and its own
// non-skill content, plus a $HOME-like dir with a default ~/.claude.
gitInit(join(root, 'cfg'));
mkdirSync(join(root, 'cfg', 'skills', 'third-party'), { recursive: true });
mkdirSync(join(root, 'cfg', 'agents'), { recursive: true });
mkdirSync(join(root, 'fakehome', '.claude', 'plugins', 'cache', 'mkt', 'p', '1.0.0'), { recursive: true });
mkdirSync(join(root, 'fakehome', '.claude', 'skills', 'x'), { recursive: true });
mkdirSync(join(root, 'fakehome', 'alt', 'plugins', 'cache', 'mkt', 'q'), { recursive: true });
mkdirSync(join(root, 'fakehome', 'work'), { recursive: true });
savedConfigDir = process.env.CLAUDE_CONFIG_DIR;
savedHome = process.env.HOME;
});
after(() => {
if (savedConfigDir === undefined) delete process.env.CLAUDE_CONFIG_DIR;
else process.env.CLAUDE_CONFIG_DIR = savedConfigDir;
process.env.HOME = savedHome;
rmSync(root, { recursive: true, force: true });
});
it('punkt 1: a package in a Python venv site-packages is foreign', () => {
assert.equal(ownFrom(repo, join(repo, '.venv', 'lib', 'python3.12', 'site-packages', 'evilpkg')), false);
});
it('punkt 1: a composer/bundler vendor/ package is foreign', () => {
assert.equal(ownFrom(repo, join(repo, 'vendor', 'acme')), false);
assert.equal(ownFrom(repo, join(repo, 'vendor', 'acme', 'lib')), false);
});
it('punkt 1: the venv root and a look-alike name stay own (known-positive)', () => {
assert.equal(ownFrom(repo, join(repo, '.venv')), true);
assert.equal(ownFrom(repo, join(repo, 'my-vendor-notes')), true);
});
it('punkt 1: skills in a git-tracked config dir are foreign, the rest stays own', () => {
process.env.CLAUDE_CONFIG_DIR = join(root, 'cfg');
try {
assert.equal(ownFrom(join(root, 'cfg'), join(root, 'cfg', 'skills', 'third-party')), false);
assert.equal(ownFrom(join(root, 'cfg'), join(root, 'cfg', 'agents')), true);
} finally {
if (savedConfigDir === undefined) delete process.env.CLAUDE_CONFIG_DIR;
else process.env.CLAUDE_CONFIG_DIR = savedConfigDir;
}
});
it('punkt 1: skills under the default ~/.claude are foreign', () => {
delete process.env.CLAUDE_CONFIG_DIR;
process.env.HOME = join(root, 'fakehome');
try {
const home = join(root, 'fakehome');
assert.equal(ownFrom(home, join(home, '.claude', 'skills', 'x')), false);
} finally {
process.env.HOME = savedHome;
}
});
it('punkt 2: ~/.claude/plugins stays foreign when CLAUDE_CONFIG_DIR points at another profile', () => {
process.env.HOME = join(root, 'fakehome');
process.env.CLAUDE_CONFIG_DIR = join(root, 'fakehome', 'alt');
try {
const home = join(root, 'fakehome');
assert.equal(ownFrom(home, join(home, '.claude', 'plugins', 'cache', 'mkt', 'p', '1.0.0')), false);
assert.equal(ownFrom(home, join(home, 'alt', 'plugins', 'cache', 'mkt', 'q')), false);
assert.equal(ownFrom(home, join(home, 'work')), true);
} finally {
process.env.HOME = savedHome;
if (savedConfigDir === undefined) delete process.env.CLAUDE_CONFIG_DIR;
else process.env.CLAUDE_CONFIG_DIR = savedConfigDir;
}
});
it('punkt 2: a leading ~ in CLAUDE_CONFIG_DIR is expanded to the home dir', () => {
process.env.HOME = join(root, 'fakehome');
process.env.CLAUDE_CONFIG_DIR = '~/alt';
try {
const home = join(root, 'fakehome');
assert.equal(ownFrom(home, join(home, 'alt', 'plugins', 'cache', 'mkt', 'q')), false);
} finally {
process.env.HOME = savedHome;
if (savedConfigDir === undefined) delete process.env.CLAUDE_CONFIG_DIR;
else process.env.CLAUDE_CONFIG_DIR = savedConfigDir;
}
});
it('punkt 3: a case variant of node_modules is foreign (case-insensitive volume)', (t) => {
if (!existsSync(join(repo, 'NODE_MODULES'))) {
t.skip('volume is case-sensitive: NODE_MODULES is a different directory');
return;
}
assert.equal(ownFrom(repo, join(repo, 'NODE_MODULES', 'evil')), false);
});
it('punkt 3: a case-mismatched path to an own subdir is own (§ v8.1.2 punkt 4)', (t) => {
if (!existsSync(join(root, 'REPO'))) {
t.skip('volume is case-sensitive: REPO is not repo');
return;
}
// The mismatch sits in a parent segment: realpathSync keeps `REPO`, so the
// target did not start with cwd and was called foreign.
assert.equal(ownFrom(repo, join(root, 'REPO', 'Sub')), true);
});
});

View file

@ -0,0 +1,100 @@
// watch-cron-scope.test.mjs — `watch-cron.mjs` must scan each watched project
// as that project's own working tree (v8.1.3, PLAN § v8.1.3 punkt 5).
//
// Since v8.1.0 a project's .llm-security-ignore and .llm-security/policy.json
// are honored only when the target is the caller's own working tree (at or
// below cwd, same git root). watch-cron.mjs started the orchestrator with
// `cwd: PLUGIN_ROOT`, so every watched project was outside cwd and its own
// suppressions were dropped: the user was shown findings they had already
// suppressed. v8.1.3 runs the orchestrator with the project dir as cwd.
//
// (a) a watched project WITH a `**` ignore file reports the same counts as
// the orchestrator run from inside that project (its config honored);
// (b) a sibling WITHOUT the ignore file reports findings (known-positive:
// keeps (a) from passing on a fixture that produces nothing).
// Fixtures under $HOME: outside os.tmpdir() (always foreign) and this repo.
// watch-cron writes PLUGIN_ROOT/reports/watch/latest.json; the test restores
// whatever was there before.
import { describe, it, before, after } from 'node:test';
import assert from 'node:assert/strict';
import { resolve, dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { spawnSync } from 'node:child_process';
import { mkdirSync, writeFileSync, readFileSync, rmSync, existsSync } from 'node:fs';
import crypto from 'node:crypto';
import { mkOwnTreeDir } from '../helpers/own-tree.mjs';
const __dirname = dirname(fileURLToPath(import.meta.url));
const PLUGIN_ROOT = resolve(__dirname, '../..');
const WATCH_CRON = join(PLUGIN_ROOT, 'scanners', 'watch-cron.mjs');
const ORCHESTRATOR = join(PLUGIN_ROOT, 'scanners', 'scan-orchestrator.mjs');
const LATEST = join(PLUGIN_ROOT, 'reports', 'watch', 'latest.json');
// Not a real credential — a known-positive blob for the entropy scanner only.
const HIGH_ENTROPY_BLOB = crypto.randomBytes(72).toString('base64');
function writeProject(dir, { withIgnore }) {
mkdirSync(dir, { recursive: true });
const r = spawnSync('git', ['init', '-q', dir], { encoding: 'utf8' });
assert.equal(r.status, 0, `git init failed: ${r.stderr}`);
writeFileSync(join(dir, 'config.js'), `const payload = "${HIGH_ENTROPY_BLOB}";\nmodule.exports = { payload };\n`);
if (withIgnore) writeFileSync(join(dir, '.llm-security-ignore'), '**\n');
}
const total = (counts) => Object.values(counts || {}).reduce((a, b) => a + b, 0);
describe('watch-cron: a watched project is scanned as its own working tree', () => {
let root;
let savedLatest = null;
let watched;
let direct;
before(() => {
root = mkOwnTreeDir('watch-cron-');
writeProject(join(root, 'suppressed'), { withIgnore: true });
writeProject(join(root, 'plain'), { withIgnore: false });
const config = join(root, 'watch-config.json');
writeFileSync(config, JSON.stringify({
targets: [
{ path: join(root, 'suppressed'), label: 'suppressed' },
{ path: join(root, 'plain'), label: 'plain' },
],
options: { baseline: false, saveBaseline: false },
}));
if (existsSync(LATEST)) savedLatest = readFileSync(LATEST);
const run = spawnSync(process.execPath, [WATCH_CRON, '--config', config], {
cwd: root, encoding: 'utf8', timeout: 600_000,
});
assert.ok(run.status !== null, `watch-cron did not finish: ${run.error?.message}`);
watched = JSON.parse(readFileSync(LATEST, 'utf8'));
// Reference: the orchestrator run from inside the project (own tree).
const ref = spawnSync(process.execPath, [ORCHESTRATOR, '.'], {
cwd: join(root, 'suppressed'), encoding: 'utf8', timeout: 300_000, maxBuffer: 64 * 1024 * 1024,
});
direct = JSON.parse(ref.stdout).aggregate;
});
after(() => {
if (savedLatest !== null) writeFileSync(LATEST, savedLatest);
else rmSync(LATEST, { force: true });
rmSync(root, { recursive: true, force: true });
});
it('(b) the project without an ignore file reports findings (known-positive)', () => {
const plain = watched.targets.find(t => t.label === 'plain');
assert.equal(plain.error, null, `plain target errored: ${plain.error}`);
assert.ok(total(plain.counts) > 0, `expected findings, got ${JSON.stringify(plain.counts)}`);
});
it('(a) the project\'s own ignore file is honored (same counts as a run from inside it)', () => {
const suppressed = watched.targets.find(t => t.label === 'suppressed');
assert.equal(suppressed.error, null, `suppressed target errored: ${suppressed.error}`);
assert.deepEqual(suppressed.counts, direct.counts);
const plain = watched.targets.find(t => t.label === 'plain');
assert.ok(total(suppressed.counts) < total(plain.counts),
`ignore file had no effect: ${JSON.stringify(suppressed.counts)} vs ${JSON.stringify(plain.counts)}`);
});
});