open/config-audit
1
0
Fork 0
Code Issues Releases 1 Activity

test(snapshots): make byte/snapshot tests hermetic + re-seed baseline

Browse source 
The COL collision-scanner and the CLAUDE.md cascade resolve ~/.claude from
process.env.HOME (active-config-reader). Snapshot/byte CLIs were spawned with
the developer's real HOME, so they picked up installed plugins/skills and the
user CLAUDE.md — making the v5.0.0 + default-output snapshots machine- and
time-dependent. They were seeded 2026-05-01 with COL=1 (a real ~/.claude skill
collision) and drifted to COL=0 after the polyrepo split: 26 pre-existing
failures unrelated to Batch 1.

Fix (test-only, no production change):
- tests/helpers/hermetic-home.mjs — empty temp HOME, mirroring the pattern
  collision.test.mjs already uses for the COL unit test.
- 7 harnesses spawn CLIs (or call lint()) under the hermetic HOME, so output
  depends only on committed fixtures. Determinism verified across runs.
- Re-seeded all snapshots under hermetic HOME via SEED_SNAPSHOT/UPDATE_SNAPSHOT
  (added a SEED guard to the frozen v5.0.0 byte tests). Snapshots now reflect
  the fixture alone (COL=0, fixture-only activeConfig counts).
- Also re-seeded the unused env-aware snapshots (manifest/whats-active/
  plugin-health), which had baked dozens of real ~/.claude skill/plugin names
  into the committed repo — privacy cleanup.

Full suite: 812/812 green, stable across 3 runs.
This commit is contained in:
Kjell Tore Guttormsen 2026-06-18 12:26:00 +02:00
commit 8216fb4175
20 changed files with 326 additions and 3386 deletions
Download patch file Download diff file Expand all files Collapse all files

144
tests/snapshots/v5.0.0/scan-orchestrator.json
View file

@ -1,7 +1,7 @@
{
"meta": {
"target": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium",
"timestamp": "2026-05-01T14:44:16.877Z",
"target": "/Users/ktg/repos/ktg-plugin-marketplace/config-audit/tests/fixtures/marketplace-medium",
"timestamp": "2026-06-18T10:18:35.439Z",
"version": "2.2.0",
"tool": "config-audit"
},
@ -11,12 +11,26 @@
"status": "ok",
"files_scanned": 1,
"duration_ms": 2,
"findings": [],
"findings": [
{
"id": "CA-CML-001",
"scanner": "CML",
"severity": "low",
"title": "Missing recommended sections",
"description": "CLAUDE.md is missing: Project overview, Architecture",
"file": "/Users/ktg/repos/ktg-plugin-marketplace/config-audit/tests/fixtures/marketplace-medium/CLAUDE.md",
"line": null,
"evidence": "Present sections: Marketplace Medium, Plugins, Commands, Conventions",
"category": null,
"recommendation": "Add sections for: Project overview, Architecture",
"autoFixable": false
}
],
"counts": {
"critical": 0,
"high": 0,
"medium": 0,
"low": 0,
"low": 1,
"info": 0
}
},
@ -24,7 +38,7 @@
"scanner": "SET",
"status": "ok",
"files_scanned": 1,
"duration_ms": 1,
"duration_ms": 0,
"findings": [],
"counts": {
"critical": 0,
@ -80,7 +94,7 @@
"scanner": "IMP",
"status": "ok",
"files_scanned": 1,
"duration_ms": 0,
"duration_ms": 2,
"findings": [],
"counts": {
"critical": 0,
@ -344,7 +358,7 @@
"scanner": "TOK",
"status": "ok",
"files_scanned": 2,
"duration_ms": 120,
"duration_ms": 8,
"findings": [
{
"id": "CA-TOK-001",
@ -376,76 +390,60 @@
"Review whether this source needs to load on every turn."
]
},
{
"source": "mcp:sadhguru-wisdom (plugin:sadhguru-wisdom)",
"estimated_tokens": 500,
"rank": 2,
"recommendations": [
"Review whether this source needs to load on every turn."
]
},
{
"source": "mcp:vegnorm-rag (plugin:vegnormalene)",
"estimated_tokens": 500,
"rank": 3,
"recommendations": [
"Review whether this source needs to load on every turn."
]
},
{
"source": "CLAUDE.md",
"estimated_tokens": 116,
"rank": 4,
"rank": 2,
"recommendations": [
"Move volatile top-of-file content to the bottom or extract to an @import-ed file.",
"Split overlong CLAUDE.md into focused @imports (≤200 lines each)."
],
"path": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium/CLAUDE.md"
"path": "/Users/ktg/repos/ktg-plugin-marketplace/config-audit/tests/fixtures/marketplace-medium/CLAUDE.md"
},
{
"source": "hooks/hooks.json",
"estimated_tokens": 81,
"rank": 3,
"recommendations": [
"Deduplicate overlapping entries — each duplicate inflates the per-turn schema payload.",
"Move rarely-used permissions to a project-local override."
],
"path": "/Users/ktg/repos/ktg-plugin-marketplace/config-audit/tests/fixtures/marketplace-medium/hooks/hooks.json"
},
{
"source": ".claude/settings.json",
"estimated_tokens": 59,
"rank": 4,
"recommendations": [
"Deduplicate overlapping entries — each duplicate inflates the per-turn schema payload.",
"Move rarely-used permissions to a project-local override."
],
"path": "/Users/ktg/repos/ktg-plugin-marketplace/config-audit/tests/fixtures/marketplace-medium/.claude/settings.json"
},
{
"source": ".mcp.json",
"estimated_tokens": 53,
"rank": 5,
"recommendations": [
"Deduplicate overlapping entries — each duplicate inflates the per-turn schema payload.",
"Move rarely-used permissions to a project-local override."
],
"path": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium/hooks/hooks.json"
},
{
"source": ".claude/settings.json",
"estimated_tokens": 59,
"rank": 6,
"recommendations": [
"Deduplicate overlapping entries — each duplicate inflates the per-turn schema payload.",
"Move rarely-used permissions to a project-local override."
],
"path": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium/.claude/settings.json"
},
{
"source": ".mcp.json",
"estimated_tokens": 53,
"rank": 7,
"recommendations": [
"Deduplicate overlapping entries — each duplicate inflates the per-turn schema payload.",
"Move rarely-used permissions to a project-local override."
],
"path": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium/.mcp.json"
"path": "/Users/ktg/repos/ktg-plugin-marketplace/config-audit/tests/fixtures/marketplace-medium/.mcp.json"
}
],
"total_estimated_tokens": 1809,
"total_estimated_tokens": 809,
"activeConfig": {
"claudeMdEstimatedTokens": 5716,
"mcpServerCount": 3,
"pluginCount": 41,
"skillCount": 65
"claudeMdEstimatedTokens": 1639,
"mcpServerCount": 1,
"pluginCount": 0,
"skillCount": 0
}
},
{
"scanner": "CPS",
"status": "ok",
"files_scanned": 1,
"duration_ms": 0,
"duration_ms": 1,
"findings": [],
"counts": {
"critical": 0,
@ -459,7 +457,7 @@
"scanner": "DIS",
"status": "ok",
"files_scanned": 1,
"duration_ms": 0,
"duration_ms": 1,
"findings": [
{
"id": "CA-DIS-001",
@ -467,7 +465,7 @@
"severity": "low",
"title": "Tool listed in both permissions.deny and permissions.allow",
"description": ".claude/settings.json contains 1 tool present in both deny and allow lists. The deny list wins — the allow entries are dead config but still load on every turn and may confuse future readers about intent.",
"file": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/config-audit/tests/fixtures/marketplace-medium/.claude/settings.json",
"file": "/Users/ktg/repos/ktg-plugin-marketplace/config-audit/tests/fixtures/marketplace-medium/.claude/settings.json",
"line": null,
"evidence": "Read: allow=\"Read(src/**)\" + deny=\"Read(./.env)\"",
"category": "permissions-hygiene",
@ -486,42 +484,14 @@
{
"scanner": "COL",
"status": "ok",
"files_scanned": 65,
"duration_ms": 91,
"findings": [
{
"id": "CA-COL-001",
"scanner": "COL",
"severity": "low",
"title": "Skill name \"okr-offentlig-sektor\" used by multiple plugins",
"description": "2 plugins (okr, okr) expose a skill named \"okr-offentlig-sektor\". Even when invocation is namespaced via /plugin:skill, shared names create ambiguity in error messages, search results, and the plugin-skills enumeration.",
"file": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/okr/skills/okr-offentlig-sektor/SKILL.md",
"line": null,
"evidence": "name=\"okr-offentlig-sektor\"; plugins=okr,okr",
"category": "plugin-hygiene",
"recommendation": "Coordinate naming across plugins, or rename one to clarify intent. The shared name forces every reader to disambiguate by source.",
"autoFixable": false,
"details": {
"namespaces": [
{
"source": "plugin:okr",
"name": "okr-offentlig-sektor",
"path": "/Users/ktg/.claude/plugins/marketplaces/ktg-plugin-marketplace/plugins/okr/skills/okr-offentlig-sektor/SKILL.md"
},
{
"source": "plugin:okr",
"name": "okr-offentlig-sektor",
"path": "/Users/ktg/.claude/plugins/marketplaces/ktg-privat/plugins/okr/skills/okr-offentlig-sektor/SKILL.md"
}
]
}
}
],
"files_scanned": 0,
"duration_ms": 0,
"findings": [],
"counts": {
"critical": 0,
"high": 0,
"medium": 0,
"low": 1,
"low": 0,
"info": 0
}
}
@ -542,4 +512,4 @@
"scanners_error": 0,
"scanners_skipped": 1
}
}
}