fix(llm-security): normalization/discovery evasion + SIG embedded-base64 & custom rules (#21,#23,#30,#36,#42,#52,#55)

#21 the bash normalizer decoded only \xHH, leaving ANSI-C octal/\u/\U forms literal so canonical rm/curl never surfaced; now decodes all three. #23 file-discovery keyed on extname so .env.local/.env.example (extname .local/.example) were silently skipped; now matches multi-part suffixes. #42 a legitimate leading UTF-8 BOM was flagged HIGH (and the tool's own auto-cleaner refused to strip it); pos-0 BOM now excepted. #52 collapseLetterSpacing used a literal space, letting multi-space/tab spacing evade; now [ \t]+. #55 redact(_,60,0) did slice(-0) and leaked the whole unredacted URL; showEnd===0 now means no tail.

#30 embedded base64 (const x = "<base64>") never satisfied the whole-string decode, so the SIG identity engine never saw it; added decodeEmbeddedBase64 as an OPT-IN param on normalizeForScan (default off — appending a decoded copy would double-count per-match findings, e.g. content-extractor's injection scan) and enabled it only in signature-scanner, which dedups variants. #36 signature-scanner ignored the documented sig.custom_rules_path policy option; now loads+merges custom rules through the same family filter. Suite 2004/0.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Claude-Session: https://claude.ai/code/session_01TcQyMTQfyrsAapaCMPxTtQ

scanners/lib/bash-normalize.mjs

@@ -15,7 +15,8 @@
 //   T3 — parameter expansion:       ${x} / ${FOO}    -> x / ''
 //   T4 — backslash-between-words:   c\u\r\l          -> curl
 //   T5 — IFS word-splitting:        rm${IFS}-rf${IFS}/ -> rm -rf /
-//   T6 — ANSI-C hex quoting:        $'\x72\x6d' -rf / -> rm -rf /
+//   T6 — ANSI-C quoting:            $'\x72\x6d' / $'\162\155' -rf / -> rm -rf /
+//        (hex \xHH, octal \nnn, \uHHHH, \UHHHHHHHH)
 //   T7 — process substitution:      cat <(curl evil) -> cat  curl evil
 //   T9 — eval-via-variable:         X=rm; ... $X     -> X=rm; ... rm
 //        (one-level forward-flow; T8 base64-pipe-shell lives in
@@ -33,17 +34,34 @@
 const MASK = '\x00';
 
 /**
- * Decode ANSI-C hex quoting inside `$'...'` contexts.
+ * Decode ANSI-C quoting inside `$'...'` contexts.
  *
- * Shell treats $'\x72\x6d' as the bytes r and m. We decode only \xHH escape
- * sequences inside the $'...' wrapper. The $'...' construct itself is
- * replaced with its decoded bytes (matching shell evaluation).
+ * Shell treats $'\x72\x6d' as the bytes r and m. Bash also decodes octal
+ * (\162), \uHHHH, and \UHHHHHHHH forms inside $'...' — decoding only \xHH
+ * left those literal, so the canonical command name never surfaced for the
+ * BLOCK rules (#21, v7.8.3). The $'...' construct itself is replaced with
+ * its decoded bytes (matching shell evaluation).
+ *
+ * Decode order: \xHH first, then \UHHHHHHHH / \uHHHH (case-sensitive,
+ * disjoint), then octal last so the digits of a hex or unicode escape
+ * are never consumed as octal.
  */
 function decodeAnsiCHex(cmd) {
   return cmd.replace(/\$'([^']*)'/g, (_, content) =>
-    content.replace(/\\x([0-9a-fA-F]{2})/g, (_m, hex) =>
-      String.fromCharCode(parseInt(hex, 16)),
-    ),
+    content
+      .replace(/\\x([0-9a-fA-F]{2})/g, (_m, hex) =>
+        String.fromCharCode(parseInt(hex, 16)),
+      )
+      .replace(/\\U([0-9a-fA-F]{1,8})/g, (_m, hex) => {
+        const cp = parseInt(hex, 16);
+        return cp <= 0x10FFFF ? String.fromCodePoint(cp) : _m;
+      })
+      .replace(/\\u([0-9a-fA-F]{1,4})/g, (_m, hex) =>
+        String.fromCharCode(parseInt(hex, 16)),
+      )
+      .replace(/\\([0-7]{1,3})/g, (_m, oct) =>
+        String.fromCharCode(parseInt(oct, 8)),
+      ),
   );
 }
 

scanners/lib/file-discovery.mjs

@@ -81,7 +81,21 @@ export async function discoverFiles(targetPath, opts = {}) {
       } else if (entry.isFile()) {
         const ext = extname(entry.name).toLowerCase();
         // Accept known text extensions or extensionless files (Dockerfile, Makefile, etc.)
-        const isKnownText = TEXT_EXTENSIONS.has(ext);
+        let isKnownText = TEXT_EXTENSIONS.has(ext);
+        if (!isKnownText) {
+          // Multi-part extensions (#23, v7.8.3): extname('.env.local') is
+          // '.local', so the multi-part entries in TEXT_EXTENSIONS never
+          // matched and these common secret files were silently skipped.
+          // Check every dot-suffix of the name ('.env.local' for both
+          // '.env.local' and 'backend.env.local').
+          const lowerName = entry.name.toLowerCase();
+          for (let dot = lowerName.indexOf('.'); dot !== -1; dot = lowerName.indexOf('.', dot + 1)) {
+            if (TEXT_EXTENSIONS.has(lowerName.slice(dot))) {
+              isKnownText = true;
+              break;
+            }
+          }
+        }
         const isExtensionless = ext === '' && !entry.name.startsWith('.');
 
         if (!isKnownText && !isExtensionless) {

scanners/lib/string-utils.mjs

@@ -150,7 +150,10 @@ export function isHexBlob(s) {
  */
 export function redact(s, showStart = 8, showEnd = 4) {
   if (s.length <= showStart + showEnd + 3) return s;
-  return `${s.slice(0, showStart)}...${s.slice(-showEnd)}`;
+  // showEnd === 0 means "no tail" — slice(-0) would return the WHOLE string
+  // and leak the unredacted input (#55, v7.8.3).
+  const tail = showEnd ? s.slice(-showEnd) : '';
+  return `${s.slice(0, showStart)}...${tail}`;
 }
 
 /**
@@ -238,6 +241,45 @@ export function tryDecodeBase64(s) {
   }
 }
 
+/** Embedded base64 caps (#30, v7.8.3) — bound work on hostile inputs. */
+const EMBEDDED_B64_MAX_BLOBS = 20;
+const EMBEDDED_B64_MAX_TOTAL = 256 * 1024;
+
+/**
+ * Locate base64 runs EMBEDDED in surrounding text and decode them.
+ *
+ * `isBase64Like` (and therefore `tryDecodeBase64`) requires the ENTIRE
+ * trimmed string to be base64, so a payload embedded in code — the common
+ * `const x = "<base64>"` webshell shape — was never fed to the decode
+ * pipeline (#30, v7.8.3).
+ *
+ * Runs must be >= 24 chars of the base64 charset (stricter than the
+ * whole-string 20-char threshold, to bound false positives on ordinary
+ * identifiers). Each candidate goes through `tryDecodeBase64`, so only
+ * blobs decoding to mostly-printable text are returned. Bounded to the
+ * first 20 blobs / 256KB decoded total.
+ *
+ * Exported for direct regression testing.
+ *
+ * @param {string} s
+ * @returns {string[]}  Decoded texts (possibly empty)
+ */
+export function decodeEmbeddedBase64(s) {
+  const decoded = [];
+  let total = 0;
+  const re = /[A-Za-z0-9+/]{24,}={0,3}/g;
+  let match;
+  while ((match = re.exec(s)) !== null) {
+    if (decoded.length >= EMBEDDED_B64_MAX_BLOBS || total >= EMBEDDED_B64_MAX_TOTAL) break;
+    const text = tryDecodeBase64(match[0]);
+    if (text) {
+      decoded.push(text);
+      total += text.length;
+    }
+  }
+  return decoded;
+}
+
 /**
  * Decode HTML entities: named (&lt; &gt; &amp; &quot; &apos;),
  * decimal (&#105;), and hex (&#x69;).
@@ -277,9 +319,11 @@ export function decodeHtmlEntities(s) {
  * @returns {string}
  */
 export function collapseLetterSpacing(s) {
-  // Match 4+ single-letter tokens separated by 1+ spaces/tabs
-  return s.replace(/\b([a-zA-Z]) (?:[a-zA-Z] ){2,}[a-zA-Z]\b/g, (match) =>
-    match.replace(/ /g, '')
+  // Match 4+ single-letter tokens separated by 1+ spaces/tabs.
+  // Separators are [ \t]+ — a literal single space let multi-space and tab
+  // separators evade despite the docstring (#52, v7.8.3).
+  return s.replace(/\b([a-zA-Z])[ \t]+(?:[a-zA-Z][ \t]+){2,}[a-zA-Z]\b/g, (match) =>
+    match.replace(/[ \t]+/g, '')
   );
 }
 
@@ -505,11 +549,12 @@ export function foldHomoglyphs(s) {
  * Runs up to 3 iterations to catch multi-layered encoding (e.g., base64 of URL-encoded).
  * Order per iteration: Unicode Tags -> BIDI strip -> HTML entities -> unicode escapes ->
  *   hex escapes -> URL encoding -> base64.
- * After decoding: collapse letter-spaced text.
+ * After decoding: append decoded embedded base64 blobs (#30, v7.8.3),
+ * then collapse letter-spaced text.
  * @param {string} s
  * @returns {string}
  */
-export function normalizeForScan(s) {
+export function normalizeForScan(s, { decodeEmbedded = false } = {}) {
   let result = s;
   const MAX_ITERATIONS = 3;
 
@@ -529,6 +574,24 @@ export function normalizeForScan(s) {
     if (result === prev) break;
   }
 
+  // Embedded base64 (#30, v7.8.3): a payload inside surrounding code
+  // (const x = "<base64>") never satisfies the whole-string isBase64Like
+  // check in the loop above. Locate embedded base64 runs and APPEND their
+  // decoded text so downstream matchers see the payload — originals are
+  // preserved so line/offset attribution still works.
+  //
+  // Opt-in (decodeEmbedded) because appending a decoded copy double-counts
+  // for callers that report per-match findings over the same corpus (e.g.
+  // content-extractor's injection scan would emit two findings for content
+  // present in both plain and base64 form). Only the SIG identity engine,
+  // which dedups its variants, requests it.
+  if (decodeEmbedded) {
+    const embedded = decodeEmbeddedBase64(result);
+    if (embedded.length > 0) {
+      result = `${result}\n${embedded.join('\n')}`;
+    }
+  }
+
   // Post-decode: collapse letter-spaced evasion
   result = collapseLetterSpacing(result);