test(v8.1.3): red tests for plan points 1, 2, 3, 5 and addenda a-c
Each test fails on fd7de23:
- own-working-tree: venv site-packages, vendor/, config-dir skills/
(punkt 1); ~/.claude/plugins next to $CLAUDE_CONFIG_DIR and a leading
~ in the variable (punkt 2); NODE_MODULES case variant and a
case-mismatched parent segment (punkt 3, closes v8.1.2 punkt 4).
- watch-cron-scope: a watched project's own ignore file is honored
(punkt 5).
- av-surface (b2): no runnable base64-to-shell line with a short
command blob (addendum a; 3 hits today).
- doc-consistency: scanner-reference Knowledge Files matches knowledge/
(addendum b); ci-cd-guide makes no offline claim (addendum c).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
fd7de230e4
commit
b62c3e60f5
SSH key fingerprint:
SHA256:JakMjO6FTBBzN0Bhfj9saOoEjaFxlSdYuZQQpM/lF9Q
4 changed files with 290 additions and 0 deletions
|
|
@ -36,6 +36,12 @@
|
|||
// caught the second half of a payload split across two blobs in
|
||||
// examples/malicious-skill-demo/security-assessment.md, which the S0
|
||||
// rule missed. All 7 command blobs the S0 rule saw still hit.
|
||||
// (b2) a line that holds a base64 decode piped into a shell AND a base64
|
||||
// blob of ANY length (8+ characters) decoding to a command shape or a
|
||||
// recursive `rm`: the whole runnable line, comments included. (b) alone
|
||||
// misses it below 24 characters: a home-directory deletion encodes to 16.
|
||||
// Added in v8.1.3 (2026-09-23, order 20260923T092223Z, tillegg a) after
|
||||
// the README fact-check found one in a comment.
|
||||
// (c) a Unicode Tag (U+E0000-U+E007F), zero-width (U+200B-U+200D, U+2060,
|
||||
// U+FEFF) or bidi-control (U+202A-U+202E, U+2066-U+2069) codepoint in a
|
||||
// TEXT file outside scanners/commons/conformance/**. Binary files (a NUL
|
||||
|
|
@ -91,6 +97,9 @@ const BASE64_BLOB = /[A-Za-z0-9+/]{24,}={0,2}/g;
|
|||
const PRINTABLE = /^[\x20-\x7e\t\r\n]+$/;
|
||||
const COMMAND_SHAPE = /^\s*(?:curl|wget|bash|sh|python[0-9.]*|eval|nc)\b|\|\s*(?:bash|sh)\b|\$\(/;
|
||||
|
||||
const SHORT_BASE64_BLOB = /[A-Za-z0-9+/]{8,}={0,2}/g;
|
||||
const DELETION_SHAPE = /\brm\s+-[a-zA-Z]*[rR]/;
|
||||
|
||||
const CARRIER = /[\u{E0000}-\u{E007F}\u200B-\u200D\u2060\uFEFF\u202A-\u202E\u2066-\u2069]/u;
|
||||
|
||||
const SESSION_LOADED = /^(?:agents|commands|hooks)\//;
|
||||
|
|
@ -127,6 +136,7 @@ function measureAvSurface() {
|
|||
const rules = SIGNATURE_RULES.filter(r => PAYLOAD_FAMILIES.has(r.family));
|
||||
const a = [];
|
||||
const b = [];
|
||||
const b2 = [];
|
||||
const c = [];
|
||||
const a2 = [];
|
||||
const e = [];
|
||||
|
|
@ -184,6 +194,16 @@ function measureAvSurface() {
|
|||
}
|
||||
}
|
||||
|
||||
text.split('\n').forEach((line, i) => {
|
||||
if (!BASE64_PIPE_TO_SHELL.test(line)) return;
|
||||
for (const m of line.matchAll(SHORT_BASE64_BLOB)) {
|
||||
const decoded = Buffer.from(m[0], 'base64').toString('latin1');
|
||||
if (PRINTABLE.test(decoded) && (COMMAND_SHAPE.test(decoded) || DELETION_SHAPE.test(decoded))) {
|
||||
b2.push(`${rel}:${i + 1} :: ${decoded.slice(0, 60).replace(/\s+/g, ' ')}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (!rel.startsWith(CONFORMANCE) && CARRIER.test(text)) c.push(rel);
|
||||
|
||||
if (TEST_SOURCE.test(rel)) {
|
||||
|
|
@ -205,6 +225,7 @@ function measureAvSurface() {
|
|||
testSources,
|
||||
b,
|
||||
bFiles: new Set(b.map(x => x.split(' :: ')[0])).size,
|
||||
b2,
|
||||
c,
|
||||
d,
|
||||
e,
|
||||
|
|
@ -234,6 +255,11 @@ describe('av-surface: tracked tree carries no AV-triggering payloads', () => {
|
|||
assert.deepEqual(m.b, [], report('base64 blobs', m.b));
|
||||
});
|
||||
|
||||
it(`(b2) no runnable base64-to-shell line with a short command blob (of ${m.textFiles} text files)`, (t) => {
|
||||
t.diagnostic(`b2=${m.b2.length}`);
|
||||
assert.deepEqual(m.b2, [], report('runnable base64-to-shell lines', m.b2));
|
||||
});
|
||||
|
||||
it(`(c) no Tag/zero-width/bidi carrier outside the conformance corpus (of ${m.textFiles} text files)`, (t) => {
|
||||
t.diagnostic(`c=${m.c.length}`);
|
||||
assert.deepEqual(m.c, [], report('files with a carrier codepoint', m.c));
|
||||
|
|
|
|||
|
|
@ -349,6 +349,20 @@ describe('doc-consistency — inventory counts are derived from source (B10)', (
|
|||
}
|
||||
});
|
||||
|
||||
// v8.1.3 (tillegg b): scanner-reference.md said "Knowledge Files (20)" while
|
||||
// knowledge/ held 22 — the header and the table must both follow the dir.
|
||||
it('docs/scanner-reference.md Knowledge Files header and table match knowledge/', () => {
|
||||
const files = readdirSync(KNOWLEDGE_DIR).sort();
|
||||
const content = readFileSync(join(PLUGIN_ROOT, 'docs', 'scanner-reference.md'), 'utf-8');
|
||||
const header = content.match(/^## Knowledge Files \((\d+)\)$/m);
|
||||
assert.notEqual(header, null, 'scanner-reference.md has no `## Knowledge Files (N)` header');
|
||||
assert.equal(Number(header[1]), files.length,
|
||||
`scanner-reference.md says Knowledge Files (${header[1]}) but knowledge/ holds ${files.length}.`);
|
||||
const rows = [...content.slice(header.index).split('\n## ')[0].matchAll(/^\| `([^`]+)` \|/gm)]
|
||||
.map(m => m[1]).sort();
|
||||
assert.deepEqual(rows, files, 'the Knowledge Files table must list exactly the files in knowledge/');
|
||||
});
|
||||
|
||||
// -- output.mjs finding() prefix list --------------------------------------
|
||||
it('output.mjs JSDoc lists every prefix passed to finding()', () => {
|
||||
const actual = findingPrefixes();
|
||||
|
|
@ -411,3 +425,24 @@ describe('doc-consistency — inventory counts are derived from source (B10)', (
|
|||
);
|
||||
});
|
||||
});
|
||||
|
||||
// v8.1.3 (tillegg c): docs/ci-cd-guide.md said the standalone CLI makes zero
|
||||
// network calls and that OSV is opt-in. The orchestrated scanners run
|
||||
// `npm audit` and `pip-audit` (dep), resolve domains over DNS (network) and
|
||||
// query OSV.dev (supply-chain), with no switch to turn any of them off —
|
||||
// README says so since the v8.1.2 README rewrite.
|
||||
describe('doc-consistency — ci-cd-guide tells the truth about network calls (v8.1.3)', () => {
|
||||
const content = readFileSync(join(PLUGIN_ROOT, 'docs', 'ci-cd-guide.md'), 'utf-8');
|
||||
|
||||
it('makes no offline claim', () => {
|
||||
for (const claim of [/zero network calls/i, /no external API calls/i, /OSV\.dev queries \(opt-in\)/i, /no cross-border data transfer/i]) {
|
||||
assert.equal(claim.test(content), false, `ci-cd-guide.md still claims ${claim}`);
|
||||
}
|
||||
});
|
||||
|
||||
it('names every network path the CLI takes', () => {
|
||||
for (const path of [/npm audit/, /pip-audit/, /DNS/, /OSV\.dev/]) {
|
||||
assert.match(content, path, `ci-cd-guide.md does not name ${path}`);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -194,3 +194,132 @@ describe('isOwnWorkingTree(): node_modules and the plugin dir are foreign (v8.1.
|
|||
}
|
||||
});
|
||||
});
|
||||
|
||||
// v8.1.3 (order 20260923T092223Z, PM decisions):
|
||||
// punkt 1 — other install locations are foreign like `node_modules`: a
|
||||
// `site-packages` segment (Python venv) or a `vendor` segment
|
||||
// (composer/bundler/Go) on the path from cwd to the target, and anything
|
||||
// under a Claude Code config dir's `skills/` (third-party skills copied
|
||||
// into a git-tracked ~/.claude share its git root).
|
||||
// punkt 2 — BOTH `~/.claude` and `$CLAUDE_CONFIG_DIR` count as config dirs;
|
||||
// a leading `~` in the variable is expanded to the home dir.
|
||||
// punkt 3 — case is canonicalized (realpathSync.native), so `NODE_MODULES/x`
|
||||
// on a case-insensitive volume is the `node_modules` it names, and a
|
||||
// case-mismatched path to an own subdir is own (§ v8.1.2 punkt 4).
|
||||
describe('isOwnWorkingTree(): v8.1.3 install locations, config dirs, case', () => {
|
||||
let root;
|
||||
let repo;
|
||||
let savedConfigDir;
|
||||
let savedHome;
|
||||
|
||||
before(() => {
|
||||
root = mkOwnTreeDir('owt-813-');
|
||||
repo = join(root, 'repo');
|
||||
gitInit(repo);
|
||||
mkdirSync(join(repo, '.venv', 'lib', 'python3.12', 'site-packages', 'evilpkg'), { recursive: true });
|
||||
mkdirSync(join(repo, 'vendor', 'acme', 'lib'), { recursive: true });
|
||||
mkdirSync(join(repo, 'my-vendor-notes'), { recursive: true });
|
||||
mkdirSync(join(repo, 'node_modules', 'evil'), { recursive: true });
|
||||
mkdirSync(join(repo, 'Sub'), { recursive: true });
|
||||
// A git-tracked config dir (cwd = it) with third-party skills and its own
|
||||
// non-skill content, plus a $HOME-like dir with a default ~/.claude.
|
||||
gitInit(join(root, 'cfg'));
|
||||
mkdirSync(join(root, 'cfg', 'skills', 'third-party'), { recursive: true });
|
||||
mkdirSync(join(root, 'cfg', 'agents'), { recursive: true });
|
||||
mkdirSync(join(root, 'fakehome', '.claude', 'plugins', 'cache', 'mkt', 'p', '1.0.0'), { recursive: true });
|
||||
mkdirSync(join(root, 'fakehome', '.claude', 'skills', 'x'), { recursive: true });
|
||||
mkdirSync(join(root, 'fakehome', 'alt', 'plugins', 'cache', 'mkt', 'q'), { recursive: true });
|
||||
mkdirSync(join(root, 'fakehome', 'work'), { recursive: true });
|
||||
savedConfigDir = process.env.CLAUDE_CONFIG_DIR;
|
||||
savedHome = process.env.HOME;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
if (savedConfigDir === undefined) delete process.env.CLAUDE_CONFIG_DIR;
|
||||
else process.env.CLAUDE_CONFIG_DIR = savedConfigDir;
|
||||
process.env.HOME = savedHome;
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('punkt 1: a package in a Python venv site-packages is foreign', () => {
|
||||
assert.equal(ownFrom(repo, join(repo, '.venv', 'lib', 'python3.12', 'site-packages', 'evilpkg')), false);
|
||||
});
|
||||
|
||||
it('punkt 1: a composer/bundler vendor/ package is foreign', () => {
|
||||
assert.equal(ownFrom(repo, join(repo, 'vendor', 'acme')), false);
|
||||
assert.equal(ownFrom(repo, join(repo, 'vendor', 'acme', 'lib')), false);
|
||||
});
|
||||
|
||||
it('punkt 1: the venv root and a look-alike name stay own (known-positive)', () => {
|
||||
assert.equal(ownFrom(repo, join(repo, '.venv')), true);
|
||||
assert.equal(ownFrom(repo, join(repo, 'my-vendor-notes')), true);
|
||||
});
|
||||
|
||||
it('punkt 1: skills in a git-tracked config dir are foreign, the rest stays own', () => {
|
||||
process.env.CLAUDE_CONFIG_DIR = join(root, 'cfg');
|
||||
try {
|
||||
assert.equal(ownFrom(join(root, 'cfg'), join(root, 'cfg', 'skills', 'third-party')), false);
|
||||
assert.equal(ownFrom(join(root, 'cfg'), join(root, 'cfg', 'agents')), true);
|
||||
} finally {
|
||||
if (savedConfigDir === undefined) delete process.env.CLAUDE_CONFIG_DIR;
|
||||
else process.env.CLAUDE_CONFIG_DIR = savedConfigDir;
|
||||
}
|
||||
});
|
||||
|
||||
it('punkt 1: skills under the default ~/.claude are foreign', () => {
|
||||
delete process.env.CLAUDE_CONFIG_DIR;
|
||||
process.env.HOME = join(root, 'fakehome');
|
||||
try {
|
||||
const home = join(root, 'fakehome');
|
||||
assert.equal(ownFrom(home, join(home, '.claude', 'skills', 'x')), false);
|
||||
} finally {
|
||||
process.env.HOME = savedHome;
|
||||
}
|
||||
});
|
||||
|
||||
it('punkt 2: ~/.claude/plugins stays foreign when CLAUDE_CONFIG_DIR points at another profile', () => {
|
||||
process.env.HOME = join(root, 'fakehome');
|
||||
process.env.CLAUDE_CONFIG_DIR = join(root, 'fakehome', 'alt');
|
||||
try {
|
||||
const home = join(root, 'fakehome');
|
||||
assert.equal(ownFrom(home, join(home, '.claude', 'plugins', 'cache', 'mkt', 'p', '1.0.0')), false);
|
||||
assert.equal(ownFrom(home, join(home, 'alt', 'plugins', 'cache', 'mkt', 'q')), false);
|
||||
assert.equal(ownFrom(home, join(home, 'work')), true);
|
||||
} finally {
|
||||
process.env.HOME = savedHome;
|
||||
if (savedConfigDir === undefined) delete process.env.CLAUDE_CONFIG_DIR;
|
||||
else process.env.CLAUDE_CONFIG_DIR = savedConfigDir;
|
||||
}
|
||||
});
|
||||
|
||||
it('punkt 2: a leading ~ in CLAUDE_CONFIG_DIR is expanded to the home dir', () => {
|
||||
process.env.HOME = join(root, 'fakehome');
|
||||
process.env.CLAUDE_CONFIG_DIR = '~/alt';
|
||||
try {
|
||||
const home = join(root, 'fakehome');
|
||||
assert.equal(ownFrom(home, join(home, 'alt', 'plugins', 'cache', 'mkt', 'q')), false);
|
||||
} finally {
|
||||
process.env.HOME = savedHome;
|
||||
if (savedConfigDir === undefined) delete process.env.CLAUDE_CONFIG_DIR;
|
||||
else process.env.CLAUDE_CONFIG_DIR = savedConfigDir;
|
||||
}
|
||||
});
|
||||
|
||||
it('punkt 3: a case variant of node_modules is foreign (case-insensitive volume)', (t) => {
|
||||
if (!existsSync(join(repo, 'NODE_MODULES'))) {
|
||||
t.skip('volume is case-sensitive: NODE_MODULES is a different directory');
|
||||
return;
|
||||
}
|
||||
assert.equal(ownFrom(repo, join(repo, 'NODE_MODULES', 'evil')), false);
|
||||
});
|
||||
|
||||
it('punkt 3: a case-mismatched path to an own subdir is own (§ v8.1.2 punkt 4)', (t) => {
|
||||
if (!existsSync(join(root, 'REPO'))) {
|
||||
t.skip('volume is case-sensitive: REPO is not repo');
|
||||
return;
|
||||
}
|
||||
// The mismatch sits in a parent segment: realpathSync keeps `REPO`, so the
|
||||
// target did not start with cwd and was called foreign.
|
||||
assert.equal(ownFrom(repo, join(root, 'REPO', 'Sub')), true);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
100
tests/scanners/watch-cron-scope.test.mjs
Normal file
100
tests/scanners/watch-cron-scope.test.mjs
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
// watch-cron-scope.test.mjs — `watch-cron.mjs` must scan each watched project
|
||||
// as that project's own working tree (v8.1.3, PLAN § v8.1.3 punkt 5).
|
||||
//
|
||||
// Since v8.1.0 a project's .llm-security-ignore and .llm-security/policy.json
|
||||
// are honored only when the target is the caller's own working tree (at or
|
||||
// below cwd, same git root). watch-cron.mjs started the orchestrator with
|
||||
// `cwd: PLUGIN_ROOT`, so every watched project was outside cwd and its own
|
||||
// suppressions were dropped: the user was shown findings they had already
|
||||
// suppressed. v8.1.3 runs the orchestrator with the project dir as cwd.
|
||||
//
|
||||
// (a) a watched project WITH a `**` ignore file reports the same counts as
|
||||
// the orchestrator run from inside that project (its config honored);
|
||||
// (b) a sibling WITHOUT the ignore file reports findings (known-positive:
|
||||
// keeps (a) from passing on a fixture that produces nothing).
|
||||
// Fixtures under $HOME: outside os.tmpdir() (always foreign) and this repo.
|
||||
// watch-cron writes PLUGIN_ROOT/reports/watch/latest.json; the test restores
|
||||
// whatever was there before.
|
||||
|
||||
import { describe, it, before, after } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { resolve, dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdirSync, writeFileSync, readFileSync, rmSync, existsSync } from 'node:fs';
|
||||
import crypto from 'node:crypto';
|
||||
import { mkOwnTreeDir } from '../helpers/own-tree.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const PLUGIN_ROOT = resolve(__dirname, '../..');
|
||||
const WATCH_CRON = join(PLUGIN_ROOT, 'scanners', 'watch-cron.mjs');
|
||||
const ORCHESTRATOR = join(PLUGIN_ROOT, 'scanners', 'scan-orchestrator.mjs');
|
||||
const LATEST = join(PLUGIN_ROOT, 'reports', 'watch', 'latest.json');
|
||||
|
||||
// Not a real credential — a known-positive blob for the entropy scanner only.
|
||||
const HIGH_ENTROPY_BLOB = crypto.randomBytes(72).toString('base64');
|
||||
|
||||
function writeProject(dir, { withIgnore }) {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const r = spawnSync('git', ['init', '-q', dir], { encoding: 'utf8' });
|
||||
assert.equal(r.status, 0, `git init failed: ${r.stderr}`);
|
||||
writeFileSync(join(dir, 'config.js'), `const payload = "${HIGH_ENTROPY_BLOB}";\nmodule.exports = { payload };\n`);
|
||||
if (withIgnore) writeFileSync(join(dir, '.llm-security-ignore'), '**\n');
|
||||
}
|
||||
|
||||
const total = (counts) => Object.values(counts || {}).reduce((a, b) => a + b, 0);
|
||||
|
||||
describe('watch-cron: a watched project is scanned as its own working tree', () => {
|
||||
let root;
|
||||
let savedLatest = null;
|
||||
let watched;
|
||||
let direct;
|
||||
|
||||
before(() => {
|
||||
root = mkOwnTreeDir('watch-cron-');
|
||||
writeProject(join(root, 'suppressed'), { withIgnore: true });
|
||||
writeProject(join(root, 'plain'), { withIgnore: false });
|
||||
const config = join(root, 'watch-config.json');
|
||||
writeFileSync(config, JSON.stringify({
|
||||
targets: [
|
||||
{ path: join(root, 'suppressed'), label: 'suppressed' },
|
||||
{ path: join(root, 'plain'), label: 'plain' },
|
||||
],
|
||||
options: { baseline: false, saveBaseline: false },
|
||||
}));
|
||||
if (existsSync(LATEST)) savedLatest = readFileSync(LATEST);
|
||||
|
||||
const run = spawnSync(process.execPath, [WATCH_CRON, '--config', config], {
|
||||
cwd: root, encoding: 'utf8', timeout: 600_000,
|
||||
});
|
||||
assert.ok(run.status !== null, `watch-cron did not finish: ${run.error?.message}`);
|
||||
watched = JSON.parse(readFileSync(LATEST, 'utf8'));
|
||||
|
||||
// Reference: the orchestrator run from inside the project (own tree).
|
||||
const ref = spawnSync(process.execPath, [ORCHESTRATOR, '.'], {
|
||||
cwd: join(root, 'suppressed'), encoding: 'utf8', timeout: 300_000, maxBuffer: 64 * 1024 * 1024,
|
||||
});
|
||||
direct = JSON.parse(ref.stdout).aggregate;
|
||||
});
|
||||
|
||||
after(() => {
|
||||
if (savedLatest !== null) writeFileSync(LATEST, savedLatest);
|
||||
else rmSync(LATEST, { force: true });
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('(b) the project without an ignore file reports findings (known-positive)', () => {
|
||||
const plain = watched.targets.find(t => t.label === 'plain');
|
||||
assert.equal(plain.error, null, `plain target errored: ${plain.error}`);
|
||||
assert.ok(total(plain.counts) > 0, `expected findings, got ${JSON.stringify(plain.counts)}`);
|
||||
});
|
||||
|
||||
it('(a) the project\'s own ignore file is honored (same counts as a run from inside it)', () => {
|
||||
const suppressed = watched.targets.find(t => t.label === 'suppressed');
|
||||
assert.equal(suppressed.error, null, `suppressed target errored: ${suppressed.error}`);
|
||||
assert.deepEqual(suppressed.counts, direct.counts);
|
||||
const plain = watched.targets.find(t => t.label === 'plain');
|
||||
assert.ok(total(suppressed.counts) < total(plain.counts),
|
||||
`ignore file had no effect: ${JSON.stringify(suppressed.counts)} vs ${JSON.stringify(plain.counts)}`);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue