216 lines
8.7 KiB
JavaScript
216 lines
8.7 KiB
JavaScript
// tests/kb-update/test-layerb-allowlist.test.mjs
|
|
// Enhet A2 — the Layer B ALLOWLIST: adjudicated, per-file/line exceptions to the
|
|
// adversarial-content gate (flag → human ONCE, then deterministically green).
|
|
//
|
|
// Design under test (docs/ingestion-security-brief-2026-07.md §8 / STATE Enhet A2):
|
|
// - An allowlist ENTRY pins {path, class, evidence, tier, match} where `match` is the
|
|
// exact TRIMMED content of the adjudicated line. Matching is CONTENT-based, not
|
|
// line-number-based: a line that merely moves stays green; a line whose content
|
|
// changes RESURFACES as WARN/BLOCK (fail-safe — re-adjudication required).
|
|
// - The scanner is NOT weakened: only findings that match an adjudicated entry on
|
|
// all four axes (class, evidence, tier, trimmed content) are downgraded to 'allow'.
|
|
// - A missing or malformed allowlist file yields an EMPTY allowlist — the gate stays
|
|
// fully strict (an allowlist can only be relaxed by a well-formed, committed file).
|
|
//
|
|
// Pure/DI throughout: no disk, no llm-security.
|
|
|
|
import { test } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { classifyFindings } from '../../scripts/kb-update/lib/adversarial-scan.mjs';
|
|
import { scanPaths, loadAllowlist } from '../../scripts/kb-update/scan-adversarial-content.mjs';
|
|
import { runGate } from '../../scripts/kb-update/pre-commit-scan.mjs';
|
|
|
|
// A KB-style doc. Line map (1-indexed):
|
|
// 1 # Title
|
|
// 2 (blank)
|
|
// 3 Prose documenting the attack: "ignore previous instructions" as an example.
|
|
// 4 (blank)
|
|
// 5 - [AI-systemer og styring](#ai-systemer-og-styring)
|
|
const DOC =
|
|
'# Title\n\n' +
|
|
'Prose documenting the attack: "ignore previous instructions" as an example.\n\n' +
|
|
'- [AI-systemer og styring](#ai-systemer-og-styring)\n';
|
|
|
|
const INJ_CRITICAL = {
|
|
class: 'injection',
|
|
severity: 'critical',
|
|
line: 3,
|
|
evidence: 'override: ignore previous instructions',
|
|
};
|
|
const INJ_ANCHOR = {
|
|
class: 'injection',
|
|
severity: 'medium',
|
|
line: 5,
|
|
evidence: 'markdown: injection payload in link anchor text',
|
|
};
|
|
|
|
const ALLOW_CRITICAL = {
|
|
path: 'skills/x/references/y/doc.md',
|
|
class: 'injection',
|
|
evidence: 'override: ignore previous instructions',
|
|
tier: 'authored-doc',
|
|
match: 'Prose documenting the attack: "ignore previous instructions" as an example.',
|
|
line: 3,
|
|
reason: 'security teaching: documents the attack pattern',
|
|
adjudicated: '2026-07-18',
|
|
};
|
|
const ALLOW_ANCHOR = {
|
|
path: 'skills/x/references/y/doc.md',
|
|
class: 'injection',
|
|
evidence: 'markdown: injection payload in link anchor text',
|
|
tier: 'authored-doc',
|
|
match: '- [AI-systemer og styring](#ai-systemer-og-styring)',
|
|
line: 5,
|
|
reason: 'benign TOC link; anchor text contains "system"',
|
|
adjudicated: '2026-07-18',
|
|
};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// classifyFindings + opts.allow — the pure matching core
|
|
// ---------------------------------------------------------------------------
|
|
|
|
test('classifyFindings — a fully matching allow entry downgrades the finding to allow, file goes clean', () => {
|
|
const { disposition, findings } = classifyFindings(DOC, [INJ_CRITICAL], {
|
|
allow: [ALLOW_CRITICAL],
|
|
});
|
|
assert.equal(findings[0].disposition, 'allow');
|
|
assert.equal(disposition, 'clean');
|
|
});
|
|
|
|
test('classifyFindings — allow requires exact trimmed-content match (changed line resurfaces)', () => {
|
|
const { disposition, findings } = classifyFindings(DOC, [INJ_CRITICAL], {
|
|
allow: [{ ...ALLOW_CRITICAL, match: 'Some OTHER line content entirely.' }],
|
|
});
|
|
assert.equal(findings[0].disposition, 'block');
|
|
assert.equal(disposition, 'block');
|
|
});
|
|
|
|
test('classifyFindings — allow matches on content, not line number (moved line stays green)', () => {
|
|
// Same flagged content, but the line moved: prepend two lines.
|
|
const moved = '<!-- new preamble -->\n\n' + DOC;
|
|
const shifted = { ...INJ_CRITICAL, line: 5 };
|
|
const { disposition, findings } = classifyFindings(moved, [shifted], {
|
|
allow: [ALLOW_CRITICAL], // entry still says line 3 — must not matter
|
|
});
|
|
assert.equal(findings[0].disposition, 'allow');
|
|
assert.equal(disposition, 'clean');
|
|
});
|
|
|
|
test('classifyFindings — allow requires matching evidence', () => {
|
|
const { findings } = classifyFindings(DOC, [INJ_CRITICAL], {
|
|
allow: [{ ...ALLOW_CRITICAL, evidence: 'identity: pretend you are' }],
|
|
});
|
|
assert.equal(findings[0].disposition, 'block');
|
|
});
|
|
|
|
test('classifyFindings — allow requires matching tier (code-sample entry does not cover authored prose)', () => {
|
|
const { findings } = classifyFindings(DOC, [INJ_CRITICAL], {
|
|
allow: [{ ...ALLOW_CRITICAL, tier: 'code-sample' }],
|
|
});
|
|
assert.equal(findings[0].disposition, 'block');
|
|
});
|
|
|
|
test('classifyFindings — allow requires matching class', () => {
|
|
const { findings } = classifyFindings(DOC, [INJ_CRITICAL], {
|
|
allow: [{ ...ALLOW_CRITICAL, class: 'encoded' }],
|
|
});
|
|
assert.equal(findings[0].disposition, 'block');
|
|
});
|
|
|
|
test('classifyFindings — non-allowlisted sibling finding in the same file still flags', () => {
|
|
const { disposition, findings } = classifyFindings(DOC, [INJ_CRITICAL, INJ_ANCHOR], {
|
|
allow: [ALLOW_CRITICAL], // only the critical one is adjudicated
|
|
});
|
|
assert.equal(findings[0].disposition, 'allow');
|
|
assert.equal(findings[1].disposition, 'warn');
|
|
assert.equal(disposition, 'warn');
|
|
});
|
|
|
|
test('classifyFindings — empty/absent allow behaves exactly as before (gate unweakened)', () => {
|
|
const a = classifyFindings(DOC, [INJ_CRITICAL], {});
|
|
const b = classifyFindings(DOC, [INJ_CRITICAL], { allow: [] });
|
|
assert.equal(a.disposition, 'block');
|
|
assert.equal(b.disposition, 'block');
|
|
assert.equal(a.findings[0].disposition, 'block');
|
|
assert.equal(b.findings[0].disposition, 'block');
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// loadAllowlist — fail-safe loading (missing/malformed → empty = fully strict)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
test('loadAllowlist — well-formed file yields its entries', () => {
|
|
const entries = loadAllowlist({
|
|
readFile: () => JSON.stringify({ entries: [ALLOW_CRITICAL] }),
|
|
});
|
|
assert.equal(entries.length, 1);
|
|
assert.equal(entries[0].evidence, 'override: ignore previous instructions');
|
|
});
|
|
|
|
test('loadAllowlist — missing file yields empty (gate stays strict)', () => {
|
|
const entries = loadAllowlist({
|
|
readFile: () => {
|
|
throw new Error('ENOENT');
|
|
},
|
|
});
|
|
assert.deepEqual(entries, []);
|
|
});
|
|
|
|
test('loadAllowlist — malformed JSON yields empty (never a crash, never a weaker gate)', () => {
|
|
const entries = loadAllowlist({ readFile: () => '{ not json' });
|
|
assert.deepEqual(entries, []);
|
|
});
|
|
|
|
test('loadAllowlist — non-array entries yields empty', () => {
|
|
const entries = loadAllowlist({ readFile: () => JSON.stringify({ entries: 'nope' }) });
|
|
assert.deepEqual(entries, []);
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// scanPaths — allowlist injection + per-path scoping
|
|
// ---------------------------------------------------------------------------
|
|
|
|
test('scanPaths — injected allowlist turns an adjudicated file clean (ok=true)', async () => {
|
|
const res = await scanPaths(['skills/x/references/y/doc.md'], {
|
|
readFile: () => DOC,
|
|
detect: async () => [INJ_CRITICAL],
|
|
allowlist: [ALLOW_CRITICAL],
|
|
});
|
|
assert.equal(res.ok, true);
|
|
assert.equal(res.blocked, false);
|
|
assert.equal(res.results[0].disposition, 'clean');
|
|
assert.equal(res.results[0].findings[0].disposition, 'allow');
|
|
});
|
|
|
|
test('scanPaths — allowlist entries are scoped to their path (no cross-file bleed)', async () => {
|
|
const res = await scanPaths(['skills/OTHER/references/z/other.md'], {
|
|
readFile: () => DOC,
|
|
detect: async () => [INJ_CRITICAL],
|
|
allowlist: [ALLOW_CRITICAL], // pinned to skills/x/references/y/doc.md
|
|
});
|
|
assert.equal(res.blocked, true);
|
|
assert.equal(res.results[0].disposition, 'block');
|
|
});
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// runGate (pre-commit) — the commit boundary honours the same allowlist
|
|
// ---------------------------------------------------------------------------
|
|
|
|
test('runGate — adjudicated staged file passes the commit gate (exit 0)', async () => {
|
|
const res = await runGate(['skills/x/references/y/doc.md'], {
|
|
readFile: () => DOC,
|
|
detect: async () => [INJ_CRITICAL],
|
|
allowlist: [ALLOW_CRITICAL],
|
|
});
|
|
assert.equal(res.exitCode, 0);
|
|
assert.equal(res.blocked, false);
|
|
});
|
|
|
|
test('runGate — non-adjudicated finding still blocks the commit', async () => {
|
|
const res = await runGate(['skills/x/references/y/doc.md'], {
|
|
readFile: () => DOC,
|
|
detect: async () => [INJ_CRITICAL, INJ_ANCHOR],
|
|
allowlist: [ALLOW_CRITICAL],
|
|
});
|
|
assert.equal(res.exitCode, 2); // the anchor WARN is not adjudicated → flag → human
|
|
});
|