feat(llm-security): v8 Phase 5 step 2 - commons-loader.mjs [skip-docs]

Thin, sync-read JSON artifact loader for the future vendored
llm-security-commons subtree, modeled on signature-scanner.mjs's
loadRules()/loadCustomRules() pair: process-cached, graceful-empty
fallback on any read/parse error, and policy-extensible via a
`commons.root` policy value (mirrors sig.custom_rules_path).

Unit-tested now against a local fixture — Phase 4 (commons repo
creation, gated on the operator creating the Forgejo remote) hasn't
run yet, so the default `shared/` vendor path doesn't exist in this
checkout. That "not vendored yet" case is itself asserted: the loader
must degrade to the caller's fallback, not crash.

Not wired to any consumer yet (that's Phase 5 step 4, table-by-table
behind the golden gate). No CLI/hook/scanner-visible behaviour exists
to document. Golden baseline unchanged; full suite 2063/2063 (2053 +
10 new).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QAYkRaBXT6tmWXTQAi1ZBg

scanners/lib/commons-loader.mjs

@@ -0,0 +1,92 @@
+// commons-loader.mjs — Reads vendored llm-security-commons JSON artifacts.
+//
+// v8 Phase 5 step 2: this loader is built and unit-tested now, against a
+// local fixture, ahead of Phase 4 (llm-security-commons repo creation +
+// vendoring — see docs/commons-extraction-plan.local.md). Consumers
+// (injection-patterns.mjs, severity.mjs, string-utils.mjs, ...) switch from
+// hardcoded tables to this loader in Phase 5 step 4, table-by-table, behind
+// the golden gate.
+//
+// Modeled on signature-scanner.mjs's loadRules()/loadCustomRules() pair:
+// hooks run per-tool-call in fresh zero-dep processes, so resolution must be
+// a fast synchronous read of the vendored copy, never network. Cached once
+// per process. A missing/unvendored commons dir degrades to the caller's
+// fallback rather than crashing a hook — the same graceful-empty contract
+// loadRules() uses for a missing knowledge/signatures.json.
+//
+// Zero external dependencies — Node.js builtins only.
+
+import { readFileSync } from 'node:fs';
+import { join, dirname, isAbsolute, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { getPolicyValue } from './policy-loader.mjs';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// Default vendored location: llm-security-commons is pulled in as a
+// pull-only subtree at the repo root, mirroring the portfolio-optimiser
+// family's `shared/` convention (docs/commons-extraction-plan.local.md).
+const DEFAULT_COMMONS_ROOT = join(__dirname, '..', '..', 'shared');
+
+// Cached, parsed artifacts, keyed by resolved absolute file path.
+const _cache = new Map();
+
+/**
+ * Resolve the commons root directory.
+ * Precedence: explicit `commonsRoot` option > `commons.root` policy value
+ * (relative paths resolve against `targetPath`) > the default vendored path.
+ * @param {string|undefined} targetPath
+ * @param {string|undefined} commonsRoot
+ * @returns {string}
+ */
+function resolveCommonsRoot(targetPath, commonsRoot) {
+  if (commonsRoot) return commonsRoot;
+  const policyRoot = getPolicyValue('commons', 'root', null, targetPath);
+  if (policyRoot && typeof policyRoot === 'string') {
+    return isAbsolute(policyRoot) ? policyRoot : resolve(targetPath || process.cwd(), policyRoot);
+  }
+  return DEFAULT_COMMONS_ROOT;
+}
+
+/**
+ * Load and parse one commons JSON artifact.
+ * Graceful fallback on any read/parse error — an unvendored, missing, or
+ * invalid commons artifact must never crash a hook or scanner.
+ *
+ * @param {string} artifactPath - relative path under the commons root,
+ *   without the `.json` extension (e.g. `'lexicon/injection-lexicon'`).
+ * @param {object} [opts]
+ * @param {*} [opts.fallback] - value returned on any load/parse failure
+ *   (default: `null`). Callers pick the shape-appropriate empty value
+ *   (`[]`, `{}`, ...) the same way loadRules() falls back to `[]`.
+ * @param {string} [opts.commonsRoot] - explicit commons root, overriding
+ *   policy and the default (for tests and one-off callers).
+ * @param {string} [opts.targetPath] - scan root used to resolve a
+ *   policy-relative `commons.root` and to locate `.llm-security/policy.json`.
+ * @returns {*} Parsed JSON, or `opts.fallback` on failure.
+ */
+export function loadArtifact(artifactPath, opts = {}) {
+  const { fallback = null, commonsRoot, targetPath } = opts;
+  const root = resolveCommonsRoot(targetPath, commonsRoot);
+  const filePath = join(root, `${artifactPath}.json`);
+
+  if (_cache.has(filePath)) return _cache.get(filePath);
+
+  let result;
+  try {
+    const raw = readFileSync(filePath, 'utf8');
+    result = JSON.parse(raw);
+  } catch {
+    result = fallback; // graceful: unvendored/missing/invalid -> caller's empty shape
+  }
+
+  _cache.set(filePath, result);
+  return result;
+}
+
+/**
+ * Reset the artifact cache (for testing only).
+ */
+export function _resetCacheForTest() {
+  _cache.clear();
+}

tests/lib/commons-loader.test.mjs

@@ -0,0 +1,116 @@
+// commons-loader.test.mjs — Tests for the vendored-commons JSON artifact loader
+//
+// v8 Phase 5 step 2: llm-security-commons doesn't exist yet (Phase 4 is
+// blocked on the operator creating the Forgejo repo), so these tests drive
+// the loader against a local fixture directory instead of a real vendored
+// subtree — including the "commons isn't vendored yet" case, which must
+// degrade gracefully rather than crash.
+
+import { describe, it, beforeEach, afterEach } from 'node:test';
+import assert from 'node:assert/strict';
+import { writeFileSync, mkdirSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { loadArtifact, _resetCacheForTest } from '../../scanners/lib/commons-loader.mjs';
+import { _resetCacheForTest as _resetPolicyCacheForTest } from '../../scanners/lib/policy-loader.mjs';
+
+const TEST_ROOT = join(tmpdir(), `llm-security-commons-loader-test-${Date.now()}`);
+const FIXTURE_COMMONS_ROOT = join(TEST_ROOT, 'fixture-commons');
+const POLICY_DIR = join(TEST_ROOT, '.llm-security');
+const POLICY_FILE = join(POLICY_DIR, 'policy.json');
+
+function writeArtifact(relPath, data) {
+  const filePath = join(FIXTURE_COMMONS_ROOT, `${relPath}.json`);
+  mkdirSync(join(filePath, '..'), { recursive: true });
+  writeFileSync(filePath, JSON.stringify(data));
+}
+
+describe('commons-loader', () => {
+  beforeEach(() => {
+    _resetCacheForTest();
+    _resetPolicyCacheForTest();
+    mkdirSync(FIXTURE_COMMONS_ROOT, { recursive: true });
+  });
+
+  afterEach(() => {
+    _resetCacheForTest();
+    _resetPolicyCacheForTest();
+    try { rmSync(TEST_ROOT, { recursive: true }); } catch {}
+  });
+
+  it('loads and parses a valid artifact from an explicit commonsRoot', () => {
+    writeArtifact('lexicon/injection-lexicon', { version: '0.1.0', terms: ['ignore previous instructions'] });
+    const data = loadArtifact('lexicon/injection-lexicon', { commonsRoot: FIXTURE_COMMONS_ROOT, fallback: null });
+    assert.deepEqual(data, { version: '0.1.0', terms: ['ignore previous instructions'] });
+  });
+
+  it('returns the caller fallback when the artifact file is missing', () => {
+    const data = loadArtifact('lexicon/injection-lexicon', { commonsRoot: FIXTURE_COMMONS_ROOT, fallback: [] });
+    assert.deepEqual(data, []);
+  });
+
+  it('returns the caller fallback when the artifact JSON is invalid', () => {
+    const filePath = join(FIXTURE_COMMONS_ROOT, 'mapping', 'owasp-map.json');
+    mkdirSync(join(filePath, '..'), { recursive: true });
+    writeFileSync(filePath, 'not valid json!!!');
+    const data = loadArtifact('mapping/owasp-map', { commonsRoot: FIXTURE_COMMONS_ROOT, fallback: {} });
+    assert.deepEqual(data, {});
+  });
+
+  it('degrades gracefully when commons has not been vendored yet (no override, no policy)', () => {
+    // Phase 4 hasn't run in this repo checkout: the default `shared/` vendor
+    // path does not exist. The loader must not throw.
+    const data = loadArtifact('lexicon/injection-lexicon', { fallback: 'EMPTY' });
+    assert.equal(data, 'EMPTY');
+  });
+
+  it('caches a loaded artifact per resolved path', () => {
+    writeArtifact('calibration/calibration', { entropyFloor: 4.5 });
+    const first = loadArtifact('calibration/calibration', { commonsRoot: FIXTURE_COMMONS_ROOT });
+    writeArtifact('calibration/calibration', { entropyFloor: 9.9 }); // mutate after first read
+    const second = loadArtifact('calibration/calibration', { commonsRoot: FIXTURE_COMMONS_ROOT });
+    assert.equal(first, second); // same reference (cached)
+    assert.equal(second.entropyFloor, 4.5); // original value, not the mutation
+  });
+
+  it('caches a failed load too, so a fixed-but-unread file still returns the cached fallback', () => {
+    const first = loadArtifact('signatures/malware-signatures', { commonsRoot: FIXTURE_COMMONS_ROOT, fallback: [] });
+    writeArtifact('signatures/malware-signatures', { rules: [{ id: 'late-arrival' }] });
+    const second = loadArtifact('signatures/malware-signatures', { commonsRoot: FIXTURE_COMMONS_ROOT, fallback: [] });
+    assert.deepEqual(first, []);
+    assert.deepEqual(second, []);
+  });
+
+  it('resolves commonsRoot from the commons.root policy value when no explicit override is given', () => {
+    mkdirSync(POLICY_DIR, { recursive: true });
+    writeFileSync(POLICY_FILE, JSON.stringify({ commons: { root: FIXTURE_COMMONS_ROOT } }));
+    writeArtifact('codepoints/carriers', { zeroWidth: ['​'] });
+    const data = loadArtifact('codepoints/carriers', { targetPath: TEST_ROOT, fallback: null });
+    assert.deepEqual(data, { zeroWidth: ['​'] });
+  });
+
+  it('resolves a relative commons.root policy value against targetPath', () => {
+    mkdirSync(POLICY_DIR, { recursive: true });
+    writeFileSync(POLICY_FILE, JSON.stringify({ commons: { root: './fixture-commons' } }));
+    writeArtifact('lexicon/injection-lexicon', { version: '0.1.0', terms: [] });
+    const data = loadArtifact('lexicon/injection-lexicon', { targetPath: TEST_ROOT, fallback: null });
+    assert.deepEqual(data, { version: '0.1.0', terms: [] });
+  });
+
+  it('an explicit commonsRoot option takes precedence over the commons.root policy value', () => {
+    mkdirSync(POLICY_DIR, { recursive: true });
+    writeFileSync(POLICY_FILE, JSON.stringify({ commons: { root: join(TEST_ROOT, 'does-not-exist') } }));
+    writeArtifact('lexicon/injection-lexicon', { version: '0.1.0', terms: ['override-wins'] });
+    const data = loadArtifact('lexicon/injection-lexicon', {
+      targetPath: TEST_ROOT,
+      commonsRoot: FIXTURE_COMMONS_ROOT,
+      fallback: null,
+    });
+    assert.deepEqual(data, { version: '0.1.0', terms: ['override-wins'] });
+  });
+
+  it('defaults the fallback to null when the caller passes none', () => {
+    const data = loadArtifact('lexicon/injection-lexicon', { commonsRoot: FIXTURE_COMMONS_ROOT });
+    assert.equal(data, null);
+  });
+});