fix(llm-security): normalization/discovery evasion + SIG embedded-base64 & custom rules (#21,#23,#30,#36,#42,#52,#55)

#21 the bash normalizer decoded only \xHH, leaving ANSI-C octal/\u/\U forms literal so canonical rm/curl never surfaced; now decodes all three. #23 file-discovery keyed on extname so .env.local/.env.example (extname .local/.example) were silently skipped; now matches multi-part suffixes. #42 a legitimate leading UTF-8 BOM was flagged HIGH (and the tool's own auto-cleaner refused to strip it); pos-0 BOM now excepted. #52 collapseLetterSpacing used a literal space, letting multi-space/tab spacing evade; now [ \t]+. #55 redact(_,60,0) did slice(-0) and leaked the whole unredacted URL; showEnd===0 now means no tail.

#30 embedded base64 (const x = "<base64>") never satisfied the whole-string decode, so the SIG identity engine never saw it; added decodeEmbeddedBase64 as an OPT-IN param on normalizeForScan (default off — appending a decoded copy would double-count per-match findings, e.g. content-extractor's injection scan) and enabled it only in signature-scanner, which dedups variants. #36 signature-scanner ignored the documented sig.custom_rules_path policy option; now loads+merges custom rules through the same family filter. Suite 2004/0.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Claude-Session: https://claude.ai/code/session_01TcQyMTQfyrsAapaCMPxTtQ

scanners/lib/bash-normalize.mjs

@@ -15,7 +15,8 @@
 //   T3 — parameter expansion:       ${x} / ${FOO}    -> x / ''
 //   T4 — backslash-between-words:   c\u\r\l          -> curl
 //   T5 — IFS word-splitting:        rm${IFS}-rf${IFS}/ -> rm -rf /
-//   T6 — ANSI-C hex quoting:        $'\x72\x6d' -rf / -> rm -rf /
+//   T6 — ANSI-C quoting:            $'\x72\x6d' / $'\162\155' -rf / -> rm -rf /
+//        (hex \xHH, octal \nnn, \uHHHH, \UHHHHHHHH)
 //   T7 — process substitution:      cat <(curl evil) -> cat  curl evil
 //   T9 — eval-via-variable:         X=rm; ... $X     -> X=rm; ... rm
 //        (one-level forward-flow; T8 base64-pipe-shell lives in
@@ -33,17 +34,34 @@
 const MASK = '\x00';
 
 /**
- * Decode ANSI-C hex quoting inside `$'...'` contexts.
+ * Decode ANSI-C quoting inside `$'...'` contexts.
  *
- * Shell treats $'\x72\x6d' as the bytes r and m. We decode only \xHH escape
- * sequences inside the $'...' wrapper. The $'...' construct itself is
- * replaced with its decoded bytes (matching shell evaluation).
+ * Shell treats $'\x72\x6d' as the bytes r and m. Bash also decodes octal
+ * (\162), \uHHHH, and \UHHHHHHHH forms inside $'...' — decoding only \xHH
+ * left those literal, so the canonical command name never surfaced for the
+ * BLOCK rules (#21, v7.8.3). The $'...' construct itself is replaced with
+ * its decoded bytes (matching shell evaluation).
+ *
+ * Decode order: \xHH first, then \UHHHHHHHH / \uHHHH (case-sensitive,
+ * disjoint), then octal last so the digits of a hex or unicode escape
+ * are never consumed as octal.
  */
 function decodeAnsiCHex(cmd) {
   return cmd.replace(/\$'([^']*)'/g, (_, content) =>
-    content.replace(/\\x([0-9a-fA-F]{2})/g, (_m, hex) =>
-      String.fromCharCode(parseInt(hex, 16)),
-    ),
+    content
+      .replace(/\\x([0-9a-fA-F]{2})/g, (_m, hex) =>
+        String.fromCharCode(parseInt(hex, 16)),
+      )
+      .replace(/\\U([0-9a-fA-F]{1,8})/g, (_m, hex) => {
+        const cp = parseInt(hex, 16);
+        return cp <= 0x10FFFF ? String.fromCodePoint(cp) : _m;
+      })
+      .replace(/\\u([0-9a-fA-F]{1,4})/g, (_m, hex) =>
+        String.fromCharCode(parseInt(hex, 16)),
+      )
+      .replace(/\\([0-7]{1,3})/g, (_m, oct) =>
+        String.fromCharCode(parseInt(oct, 8)),
+      ),
   );
 }
 

scanners/lib/file-discovery.mjs

@@ -81,7 +81,21 @@ export async function discoverFiles(targetPath, opts = {}) {
       } else if (entry.isFile()) {
         const ext = extname(entry.name).toLowerCase();
         // Accept known text extensions or extensionless files (Dockerfile, Makefile, etc.)
-        const isKnownText = TEXT_EXTENSIONS.has(ext);
+        let isKnownText = TEXT_EXTENSIONS.has(ext);
+        if (!isKnownText) {
+          // Multi-part extensions (#23, v7.8.3): extname('.env.local') is
+          // '.local', so the multi-part entries in TEXT_EXTENSIONS never
+          // matched and these common secret files were silently skipped.
+          // Check every dot-suffix of the name ('.env.local' for both
+          // '.env.local' and 'backend.env.local').
+          const lowerName = entry.name.toLowerCase();
+          for (let dot = lowerName.indexOf('.'); dot !== -1; dot = lowerName.indexOf('.', dot + 1)) {
+            if (TEXT_EXTENSIONS.has(lowerName.slice(dot))) {
+              isKnownText = true;
+              break;
+            }
+          }
+        }
         const isExtensionless = ext === '' && !entry.name.startsWith('.');
 
         if (!isKnownText && !isExtensionless) {

scanners/lib/string-utils.mjs

@@ -150,7 +150,10 @@ export function isHexBlob(s) {
  */
 export function redact(s, showStart = 8, showEnd = 4) {
   if (s.length <= showStart + showEnd + 3) return s;
-  return `${s.slice(0, showStart)}...${s.slice(-showEnd)}`;
+  // showEnd === 0 means "no tail" — slice(-0) would return the WHOLE string
+  // and leak the unredacted input (#55, v7.8.3).
+  const tail = showEnd ? s.slice(-showEnd) : '';
+  return `${s.slice(0, showStart)}...${tail}`;
 }
 
 /**
@@ -238,6 +241,45 @@ export function tryDecodeBase64(s) {
   }
 }
 
+/** Embedded base64 caps (#30, v7.8.3) — bound work on hostile inputs. */
+const EMBEDDED_B64_MAX_BLOBS = 20;
+const EMBEDDED_B64_MAX_TOTAL = 256 * 1024;
+
+/**
+ * Locate base64 runs EMBEDDED in surrounding text and decode them.
+ *
+ * `isBase64Like` (and therefore `tryDecodeBase64`) requires the ENTIRE
+ * trimmed string to be base64, so a payload embedded in code — the common
+ * `const x = "<base64>"` webshell shape — was never fed to the decode
+ * pipeline (#30, v7.8.3).
+ *
+ * Runs must be >= 24 chars of the base64 charset (stricter than the
+ * whole-string 20-char threshold, to bound false positives on ordinary
+ * identifiers). Each candidate goes through `tryDecodeBase64`, so only
+ * blobs decoding to mostly-printable text are returned. Bounded to the
+ * first 20 blobs / 256KB decoded total.
+ *
+ * Exported for direct regression testing.
+ *
+ * @param {string} s
+ * @returns {string[]}  Decoded texts (possibly empty)
+ */
+export function decodeEmbeddedBase64(s) {
+  const decoded = [];
+  let total = 0;
+  const re = /[A-Za-z0-9+/]{24,}={0,3}/g;
+  let match;
+  while ((match = re.exec(s)) !== null) {
+    if (decoded.length >= EMBEDDED_B64_MAX_BLOBS || total >= EMBEDDED_B64_MAX_TOTAL) break;
+    const text = tryDecodeBase64(match[0]);
+    if (text) {
+      decoded.push(text);
+      total += text.length;
+    }
+  }
+  return decoded;
+}
+
 /**
  * Decode HTML entities: named (&lt; &gt; &amp; &quot; &apos;),
  * decimal (&#105;), and hex (&#x69;).
@@ -277,9 +319,11 @@ export function decodeHtmlEntities(s) {
  * @returns {string}
  */
 export function collapseLetterSpacing(s) {
-  // Match 4+ single-letter tokens separated by 1+ spaces/tabs
-  return s.replace(/\b([a-zA-Z]) (?:[a-zA-Z] ){2,}[a-zA-Z]\b/g, (match) =>
-    match.replace(/ /g, '')
+  // Match 4+ single-letter tokens separated by 1+ spaces/tabs.
+  // Separators are [ \t]+ — a literal single space let multi-space and tab
+  // separators evade despite the docstring (#52, v7.8.3).
+  return s.replace(/\b([a-zA-Z])[ \t]+(?:[a-zA-Z][ \t]+){2,}[a-zA-Z]\b/g, (match) =>
+    match.replace(/[ \t]+/g, '')
   );
 }
 
@@ -505,11 +549,12 @@ export function foldHomoglyphs(s) {
  * Runs up to 3 iterations to catch multi-layered encoding (e.g., base64 of URL-encoded).
  * Order per iteration: Unicode Tags -> BIDI strip -> HTML entities -> unicode escapes ->
  *   hex escapes -> URL encoding -> base64.
- * After decoding: collapse letter-spaced text.
+ * After decoding: append decoded embedded base64 blobs (#30, v7.8.3),
+ * then collapse letter-spaced text.
  * @param {string} s
  * @returns {string}
  */
-export function normalizeForScan(s) {
+export function normalizeForScan(s, { decodeEmbedded = false } = {}) {
   let result = s;
   const MAX_ITERATIONS = 3;
 
@@ -529,6 +574,24 @@ export function normalizeForScan(s) {
     if (result === prev) break;
   }
 
+  // Embedded base64 (#30, v7.8.3): a payload inside surrounding code
+  // (const x = "<base64>") never satisfies the whole-string isBase64Like
+  // check in the loop above. Locate embedded base64 runs and APPEND their
+  // decoded text so downstream matchers see the payload — originals are
+  // preserved so line/offset attribution still works.
+  //
+  // Opt-in (decodeEmbedded) because appending a decoded copy double-counts
+  // for callers that report per-match findings over the same corpus (e.g.
+  // content-extractor's injection scan would emit two findings for content
+  // present in both plain and base64 form). Only the SIG identity engine,
+  // which dedups its variants, requests it.
+  if (decodeEmbedded) {
+    const embedded = decodeEmbeddedBase64(result);
+    if (embedded.length > 0) {
+      result = `${result}\n${embedded.join('\n')}`;
+    }
+  }
+
   // Post-decode: collapse letter-spaced evasion
   result = collapseLetterSpacing(result);
 

scanners/signature-scanner.mjs

@@ -13,7 +13,7 @@
 // Zero external dependencies — Node.js builtins only.
 
 import { readFile } from 'node:fs/promises';
-import { join, dirname } from 'node:path';
+import { join, dirname, isAbsolute, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { finding, scannerResult } from './lib/output.mjs';
 import { readTextFile } from './lib/file-discovery.mjs';
@@ -33,8 +33,36 @@ const DEFAULT_FAMILIES = ['webshell', 'reverse_shell', 'cryptominer', 'hacktool'
 let _rules = null;
 
 /**
- * Load and compile signatures.json. Each rule's `pattern` is compiled to a
- * case-insensitive RegExp; rules whose pattern fails to compile are dropped.
+ * Compile a parsed ruleset object ({ rules: [...] }) into executable rules.
+ * Each rule's `pattern` is compiled to a case-insensitive RegExp; rules whose
+ * pattern fails to compile (or that lack id/pattern) are dropped.
+ * @param {object} parsed
+ * @returns {Array<{id,family,severity,re,description,provenance}>}
+ */
+function compileRules(parsed) {
+  const compiled = [];
+  for (const rule of parsed.rules || []) {
+    if (!rule || !rule.id || !rule.pattern) continue;
+    let re;
+    try {
+      re = new RegExp(rule.pattern, 'i');
+    } catch {
+      continue; // skip uncompilable patterns
+    }
+    compiled.push({
+      id: rule.id,
+      family: rule.family || 'unknown',
+      severity: rule.severity || 'high',
+      re,
+      description: rule.description || rule.id,
+      provenance: rule.provenance || null,
+    });
+  }
+  return compiled;
+}
+
+/**
+ * Load and compile signatures.json.
  * Graceful fallback to an empty ruleset on any load/parse error.
  * @returns {Promise<Array<{id,family,severity,re,description,provenance}>>}
  */
@@ -43,32 +71,33 @@ async function loadRules() {
   const rulesetPath = join(__dirname, '..', 'knowledge', 'signatures.json');
   try {
     const raw = await readFile(rulesetPath, 'utf8');
-    const parsed = JSON.parse(raw);
-    const compiled = [];
-    for (const rule of parsed.rules || []) {
-      if (!rule || !rule.id || !rule.pattern) continue;
-      let re;
-      try {
-        re = new RegExp(rule.pattern, 'i');
-      } catch {
-        continue; // skip uncompilable patterns
-      }
-      compiled.push({
-        id: rule.id,
-        family: rule.family || 'unknown',
-        severity: rule.severity || 'high',
-        re,
-        description: rule.description || rule.id,
-        provenance: rule.provenance || null,
-      });
-    }
-    _rules = compiled;
+    _rules = compileRules(JSON.parse(raw));
   } catch {
     _rules = []; // graceful: no ruleset -> no findings
   }
   return _rules;
 }
 
+/**
+ * v7.8.3 (#36): load operator-supplied rules from the documented
+ * `sig.custom_rules_path` policy option. Relative paths resolve against the
+ * scan root (where .llm-security/policy.json lives). Never cached — policy is
+ * per-target. Fails gracefully (empty array) on a missing/invalid file.
+ * @param {string} targetPath
+ * @returns {Promise<Array<{id,family,severity,re,description,provenance}>>}
+ */
+async function loadCustomRules(targetPath) {
+  const customPath = getPolicyValue('sig', 'custom_rules_path', null, targetPath);
+  if (!customPath || typeof customPath !== 'string') return [];
+  const rulesetPath = isAbsolute(customPath) ? customPath : resolve(targetPath, customPath);
+  try {
+    const raw = await readFile(rulesetPath, 'utf8');
+    return compileRules(JSON.parse(raw));
+  } catch {
+    return []; // graceful: unreadable/invalid custom ruleset -> built-ins only
+  }
+}
+
 /** Build the decode-variant set for one file's content (de-duplicated). */
 function variantsOf(content) {
   const variants = [{ label: 'raw', text: content }];
@@ -78,8 +107,8 @@ function variantsOf(content) {
   };
   // Trimming before decode lets a base64/hex blob with surrounding whitespace
   // (a trailing newline, indentation) still decode — common in real payloads.
-  add('decoded', normalizeForScan(content));
-  add('decoded-trimmed', normalizeForScan(content.trim()));
+  add('decoded', normalizeForScan(content, { decodeEmbedded: true }));
+  add('decoded-trimmed', normalizeForScan(content.trim(), { decodeEmbedded: true }));
   add('homoglyph-folded', foldHomoglyphs(content));
   add('rot13', rot13(content));
   return variants;
@@ -98,7 +127,7 @@ export async function scan(targetPath, discovery) {
   let filesScanned = 0;
 
   try {
-    const rules = await loadRules();
+    const rules = [...await loadRules(), ...await loadCustomRules(targetPath)];
     const enabledFamilies = new Set(
       (getPolicyValue('sig', 'enabled_families', DEFAULT_FAMILIES, targetPath) || []).map(f => String(f)),
     );

scanners/unicode-scanner.mjs

@@ -92,7 +92,11 @@ function scanLineForZeroWidth(line, lineNumber, relPath) {
   for (const char of line) {
     const cp = char.codePointAt(0);
     if (ZERO_WIDTH_CHARS.has(cp)) {
-      hits.push({ cp, pos });
+      // Preserve a legitimate UTF-8 BOM at file position 0 (line 1, col 0) —
+      // mirrors auto-cleaner's stripZeroWidth exception (#42, v7.8.3).
+      if (!(cp === 0xFEFF && lineNumber === 1 && pos === 0)) {
+        hits.push({ cp, pos });
+      }
     }
     pos += char.length; // codePointAt handles surrogates; advance by JS char count
   }