// codepoints.test.mjs — Tests for the commons-backed code-point carrier tables. // // v8 Phase 5 step 4, first consumer swap: ZERO_WIDTH_CHARS, the Unicode Tag // range, BIDI_CHARS and HOMOGLYPH_MAP stop being hardcoded constants in // unicode-scanner.mjs / string-utils.mjs and are built from the vendored // commons artifact `codepoints/carriers.json` instead. // // Two things are asserted here that no other gate covers: // // 1. A POSITIVE load through the real DEFAULT_COMMONS_ROOT. The loader's // graceful-empty contract means a lost or emptied commons is // indistinguishable from a legitimately empty table — the scanner would // simply stop finding anything and every other test would stay green. // Exact per-table counts make that failure loud. (`commons-loader.test.mjs` // does the same for the injection lexicon; this is the carriers half.) // 2. The graceful direction itself: an unresolvable commons root must yield // empty tables and a never-matching tag range, not a throw. string-utils // is imported by hooks, which run per-tool-call in fresh processes; a // module-load throw there would break the tool call, not just the scan. // // CYRILLIC_CONFUSABLES is deliberately NOT here. commons carries it, but // unicode-scanner.mjs's homoglyph-mixing detector tests `isCyrillic(cp)` — // the whole U+0400–U+04FF block — and never reads the set. Porting a table no // runtime consumes would move dead data into the load path. import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { buildCarrierTables, ZERO_WIDTH_CHARS, BIDI_CHARS, UNICODE_TAG_START, UNICODE_TAG_END, HOMOGLYPH_MAP } from '../../scanners/lib/codepoints.mjs'; import { HOMOGLYPH_MAP as STRING_UTILS_HOMOGLYPH_MAP } from '../../scanners/lib/string-utils.mjs'; describe('codepoints (commons carriers)', () => { describe('positive load through the real default commons root', () => { it('builds all four live tables at their recorded sizes', () => { assert.equal(ZERO_WIDTH_CHARS.size, 5, 'zero-width table lost entries — is scanners/commons vendored?'); assert.equal(BIDI_CHARS.size, 9, 'BIDI table lost entries — is scanners/commons vendored?'); assert.equal(Object.keys(HOMOGLYPH_MAP).length, 28, 'homoglyph map lost entries — is scanners/commons vendored?'); assert.equal(UNICODE_TAG_START, 0xE0001); assert.equal(UNICODE_TAG_END, 0xE007F); }); it('carries the entries that distinguish this table from its neighbours', () => { // U+00AD SOFT HYPHEN is in this table and NOT in the injection lexicon's // zero-width character class — the one recorded divergence between them. assert.ok(ZERO_WIDTH_CHARS.has(0x00AD), 'SOFT HYPHEN missing: the zero-width sets have converged by accident'); assert.ok(ZERO_WIDTH_CHARS.has(0xFEFF)); assert.ok(BIDI_CHARS.has(0x2069), 'POP DIRECTIONAL ISOLATE missing: the BIDI isolates were truncated'); assert.ok(BIDI_CHARS.has(0x202E)); // U+04CF CYRILLIC PALOCHKA -> l is the entry a naive "letters that look // alike" table always drops. assert.equal(HOMOGLYPH_MAP['ӏ'], 'l'); assert.equal(HOMOGLYPH_MAP['а'], 'a'); assert.equal(HOMOGLYPH_MAP['Α'], 'A'); }); it('keeps HOMOGLYPH_MAP frozen, as the pre-swap constant was', () => { assert.ok(Object.isFrozen(HOMOGLYPH_MAP)); assert.throws(() => { HOMOGLYPH_MAP['а'] = 'z'; }, TypeError); }); }); describe('string-utils re-export', () => { it('exports the same table object the carriers module built', () => { // string-utils.mjs's HOMOGLYPH_MAP is published surface (the golden gate // walks it). The swap must not fork it into a second copy. assert.equal(STRING_UTILS_HOMOGLYPH_MAP, HOMOGLYPH_MAP); }); }); describe('graceful degradation', () => { it('yields empty tables and a never-matching tag range when commons is unresolvable', () => { const tables = buildCarrierTables({ commonsRoot: '/nonexistent/commons-root' }); assert.equal(tables.ZERO_WIDTH_CHARS.size, 0); assert.equal(tables.BIDI_CHARS.size, 0); assert.deepEqual(tables.HOMOGLYPH_MAP, {}); // The range is a comparison, not a set: emptiness has to be expressed as // a range no codepoint can fall inside. assert.ok(tables.UNICODE_TAG_START > tables.UNICODE_TAG_END, 'an unresolvable commons left a tag range that still matches codepoints'); }); it('tolerates an artifact whose tables key is missing entirely', () => { // A truncated-but-valid JSON artifact must degrade the same way a missing // file does, rather than throwing on a property of undefined. const tables = buildCarrierTables({ commonsRoot: new URL('../fixtures/commons-empty/', import.meta.url).pathname }); assert.equal(tables.ZERO_WIDTH_CHARS.size, 0); assert.deepEqual(tables.HOMOGLYPH_MAP, {}); }); }); });