fix(knowledge): freshness can no longer be green on outdated evidence
assessFreshness aged only the entry's own source.verified stamp, so an entry re-verified against an old source stayed green while a newer source sat unnoticed (BP-SUB-001 was stamped 2026-07-31, a week after the superseding-grade article of 2026-07-24 was published). The stamp certifies the old source; it says nothing about the evidence. - entries may carry corroborating sources[] with published dates - new evidence-age rule: stale when the NEWEST published date across all sources exceeds evidenceStaleAfterDays (default 365); re-verifying the old source never clears it, only newer evidence does - source.supersededBy marks a replaced source: stale regardless of stamp - stale items now carry reasons[] (verified-age / no-verified-date / superseded / evidence-age) - BP-SUB-001 gains the 2026-07-24 context-engineering article as a verified corroborating source (near-verbatim coverage). NOT added to BP-MECH-*/BP-SIZE-001: own verification found no mechanism-choice or size-limit content in the article, contrary to the brief's assumption. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019NeaMRXVGzh9oSwigJDjE9
This commit is contained in:
parent
caea8aca23
commit
d66035ed86
3 changed files with 213 additions and 23 deletions
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"version": 1,
|
||||
"note": "Machine-readable best-practices register. SOURCE OF TRUTH for the optimization lens (v5.7 CA-OPT). Human-readable mirror lives in knowledge/*.md. Every entry is provenance-stamped (source.url + source.verified) and carries a confidence; only CONFIRMED claims are consumed user-facing (Verifiseringsplikt). Curated manually + by /config-audit knowledge-refresh (human-approved). Seeded from docs/v5.5-steering-model-plan.md V-rows + the Anthropic 'Steering Claude Code' blog.",
|
||||
"note": "Machine-readable best-practices register. SOURCE OF TRUTH for the optimization lens (v5.7 CA-OPT). Human-readable mirror lives in knowledge/*.md. Every entry is provenance-stamped (source.url + source.verified; optional corroborating sources[] with published dates feed the evidence-age freshness rule; source.supersededBy marks a source replaced by a newer one) and carries a confidence; only CONFIRMED claims are consumed user-facing (Verifiseringsplikt). Curated manually + by /config-audit knowledge-refresh (human-approved). Seeded from docs/v5.5-steering-model-plan.md V-rows + the Anthropic 'Steering Claude Code' blog.",
|
||||
"entries": [
|
||||
{
|
||||
"id": "BP-MECH-001",
|
||||
|
|
@ -12,7 +12,11 @@
|
|||
"severity": "low",
|
||||
"category": "mechanism-fit",
|
||||
"lensCheck": "claude-md-lifecycle-phrasing",
|
||||
"source": { "url": "https://claude.com/blog/steering-claude-code-skills-hooks-rules-subagents-and-more", "title": "Steering Claude Code: skills, hooks, rules, subagents and more", "verified": "2026-06-20" }
|
||||
"source": {
|
||||
"url": "https://claude.com/blog/steering-claude-code-skills-hooks-rules-subagents-and-more",
|
||||
"title": "Steering Claude Code: skills, hooks, rules, subagents and more",
|
||||
"verified": "2026-06-20"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "BP-MECH-002",
|
||||
|
|
@ -24,7 +28,11 @@
|
|||
"severity": "low",
|
||||
"category": "mechanism-fit",
|
||||
"lensCheck": "unscoped-path-specific-instruction",
|
||||
"source": { "url": "https://claude.com/blog/steering-claude-code-skills-hooks-rules-subagents-and-more", "title": "Steering Claude Code: skills, hooks, rules, subagents and more", "verified": "2026-06-20" }
|
||||
"source": {
|
||||
"url": "https://claude.com/blog/steering-claude-code-skills-hooks-rules-subagents-and-more",
|
||||
"title": "Steering Claude Code: skills, hooks, rules, subagents and more",
|
||||
"verified": "2026-06-20"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "BP-MECH-003",
|
||||
|
|
@ -36,7 +44,11 @@
|
|||
"severity": "low",
|
||||
"category": "mechanism-fit",
|
||||
"lensCheck": "procedure-in-claude-md",
|
||||
"source": { "url": "https://claude.com/blog/steering-claude-code-skills-hooks-rules-subagents-and-more", "title": "Steering Claude Code: skills, hooks, rules, subagents and more", "verified": "2026-06-20" }
|
||||
"source": {
|
||||
"url": "https://claude.com/blog/steering-claude-code-skills-hooks-rules-subagents-and-more",
|
||||
"title": "Steering Claude Code: skills, hooks, rules, subagents and more",
|
||||
"verified": "2026-06-20"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "BP-MECH-004",
|
||||
|
|
@ -48,7 +60,11 @@
|
|||
"severity": "low",
|
||||
"category": "mechanism-fit",
|
||||
"lensCheck": "never-instruction",
|
||||
"source": { "url": "https://claude.com/blog/steering-claude-code-skills-hooks-rules-subagents-and-more", "title": "Steering Claude Code: skills, hooks, rules, subagents and more", "verified": "2026-06-20" }
|
||||
"source": {
|
||||
"url": "https://claude.com/blog/steering-claude-code-skills-hooks-rules-subagents-and-more",
|
||||
"title": "Steering Claude Code: skills, hooks, rules, subagents and more",
|
||||
"verified": "2026-06-20"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "BP-MECH-005",
|
||||
|
|
@ -60,7 +76,11 @@
|
|||
"severity": "medium",
|
||||
"category": "mechanism-fit",
|
||||
"lensCheck": "CA-OST-001",
|
||||
"source": { "url": "https://code.claude.com/docs/en/output-styles", "title": "Output styles", "verified": "2026-06-20" }
|
||||
"source": {
|
||||
"url": "https://code.claude.com/docs/en/output-styles",
|
||||
"title": "Output styles",
|
||||
"verified": "2026-06-20"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "BP-LOAD-001",
|
||||
|
|
@ -69,7 +89,11 @@
|
|||
"confidence": "confirmed",
|
||||
"category": "loading-model",
|
||||
"lensCheck": null,
|
||||
"source": { "url": "https://code.claude.com/docs/en/context-window", "title": "Context window — what survives compaction", "verified": "2026-06-20" }
|
||||
"source": {
|
||||
"url": "https://code.claude.com/docs/en/context-window",
|
||||
"title": "Context window — what survives compaction",
|
||||
"verified": "2026-06-20"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "BP-LOAD-002",
|
||||
|
|
@ -78,7 +102,11 @@
|
|||
"confidence": "confirmed",
|
||||
"category": "loading-model",
|
||||
"lensCheck": null,
|
||||
"source": { "url": "https://code.claude.com/docs/en/memory", "title": "Memory — path-specific rules", "verified": "2026-06-20" }
|
||||
"source": {
|
||||
"url": "https://code.claude.com/docs/en/memory",
|
||||
"title": "Memory — path-specific rules",
|
||||
"verified": "2026-06-20"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "BP-LOAD-003",
|
||||
|
|
@ -87,7 +115,11 @@
|
|||
"confidence": "confirmed",
|
||||
"category": "loading-model",
|
||||
"lensCheck": null,
|
||||
"source": { "url": "https://code.claude.com/docs/en/context-window", "title": "Context window — what survives compaction", "verified": "2026-06-20" }
|
||||
"source": {
|
||||
"url": "https://code.claude.com/docs/en/context-window",
|
||||
"title": "Context window — what survives compaction",
|
||||
"verified": "2026-06-20"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "BP-LOAD-004",
|
||||
|
|
@ -96,7 +128,11 @@
|
|||
"confidence": "confirmed",
|
||||
"category": "loading-model",
|
||||
"lensCheck": null,
|
||||
"source": { "url": "https://code.claude.com/docs/en/skills", "title": "Skills", "verified": "2026-06-20" }
|
||||
"source": {
|
||||
"url": "https://code.claude.com/docs/en/skills",
|
||||
"title": "Skills",
|
||||
"verified": "2026-06-20"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "BP-LOAD-005",
|
||||
|
|
@ -105,7 +141,11 @@
|
|||
"confidence": "confirmed",
|
||||
"category": "loading-model",
|
||||
"lensCheck": null,
|
||||
"source": { "url": "https://code.claude.com/docs/en/hooks", "title": "Hooks", "verified": "2026-06-20" }
|
||||
"source": {
|
||||
"url": "https://code.claude.com/docs/en/hooks",
|
||||
"title": "Hooks",
|
||||
"verified": "2026-06-20"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "BP-LOAD-006",
|
||||
|
|
@ -114,7 +154,11 @@
|
|||
"confidence": "confirmed",
|
||||
"category": "loading-model",
|
||||
"lensCheck": null,
|
||||
"source": { "url": "https://code.claude.com/docs/en/sub-agents", "title": "Subagents", "verified": "2026-06-20" }
|
||||
"source": {
|
||||
"url": "https://code.claude.com/docs/en/sub-agents",
|
||||
"title": "Subagents",
|
||||
"verified": "2026-06-20"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "BP-SIZE-001",
|
||||
|
|
@ -125,7 +169,11 @@
|
|||
"severity": "medium",
|
||||
"category": "size-budget",
|
||||
"lensCheck": "CA-CML-001",
|
||||
"source": { "url": "https://claude.com/blog/steering-claude-code-skills-hooks-rules-subagents-and-more", "title": "Steering Claude Code: skills, hooks, rules, subagents and more", "verified": "2026-06-20" }
|
||||
"source": {
|
||||
"url": "https://claude.com/blog/steering-claude-code-skills-hooks-rules-subagents-and-more",
|
||||
"title": "Steering Claude Code: skills, hooks, rules, subagents and more",
|
||||
"verified": "2026-06-20"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "BP-SIZE-002",
|
||||
|
|
@ -135,7 +183,11 @@
|
|||
"severity": "low",
|
||||
"category": "size-budget",
|
||||
"lensCheck": "CA-SKL-002",
|
||||
"source": { "url": "https://code.claude.com/docs/en/skills", "title": "Skills", "verified": "2026-06-20" }
|
||||
"source": {
|
||||
"url": "https://code.claude.com/docs/en/skills",
|
||||
"title": "Skills",
|
||||
"verified": "2026-06-20"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "BP-SUB-001",
|
||||
|
|
@ -147,7 +199,20 @@
|
|||
"severity": "low",
|
||||
"category": "subtraction",
|
||||
"lensCheck": "compensatory-instruction",
|
||||
"source": { "url": "https://claude.com/blog/steering-claude-code-skills-hooks-rules-subagents-and-more", "title": "Steering Claude Code: skills, hooks, rules, subagents and more", "verified": "2026-07-31" }
|
||||
"source": {
|
||||
"url": "https://claude.com/blog/steering-claude-code-skills-hooks-rules-subagents-and-more",
|
||||
"title": "Steering Claude Code: skills, hooks, rules, subagents and more",
|
||||
"verified": "2026-07-31"
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
"url": "https://claude.com/blog/the-new-rules-of-context-engineering-for-claude-5-generation-models",
|
||||
"title": "The new rules of context engineering for Claude 5 generation models",
|
||||
"published": "2026-07-24",
|
||||
"verified": "2026-08-03",
|
||||
"note": "Near-verbatim coverage: 'briefly describe what your repo is for, but spend most of the tokens on gotchas inside of the codebase'; 'Avoid stating the obvious things Claude should know by looking at your file system or your repo.'"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,14 @@
|
|||
/** Default re-verify cadence: a confirmed best-practice older than this needs a re-check. */
|
||||
export const STALE_AFTER_DAYS_DEFAULT = 90;
|
||||
|
||||
/**
|
||||
* Default evidence cadence: when the NEWEST `published` date across an entry's
|
||||
* sources is older than this, the entry is flagged even if recently re-verified.
|
||||
* Re-verifying the old source does not clear it — only a newer source does
|
||||
* (the BP-SUB-001 defect class: green stamp, substantially outdated evidence).
|
||||
*/
|
||||
export const EVIDENCE_STALE_AFTER_DAYS_DEFAULT = 365;
|
||||
|
||||
const DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
|
||||
const DAY_MS = 86_400_000;
|
||||
|
||||
|
|
@ -46,15 +54,40 @@ function verifiedMs(entry) {
|
|||
return Number.isNaN(ms) ? null : ms;
|
||||
}
|
||||
|
||||
/** Parse a YYYY-MM-DD string to UTC-midnight ms, or null. */
|
||||
function dateMs(v) {
|
||||
if (typeof v !== 'string' || !DATE_RE.test(v)) return null;
|
||||
const ms = Date.parse(`${v}T00:00:00Z`);
|
||||
return Number.isNaN(ms) ? null : ms;
|
||||
}
|
||||
|
||||
/**
|
||||
* Newest `published` across the primary `source` and any corroborating `sources[]`.
|
||||
* Null when no source carries a parseable published date (the evidence-age rule
|
||||
* is then silent for that entry — it cannot judge evidence it cannot date).
|
||||
*/
|
||||
function newestEvidenceMs(entry) {
|
||||
const candidates = [];
|
||||
if (entry && entry.source) candidates.push(entry.source);
|
||||
if (entry && Array.isArray(entry.sources)) candidates.push(...entry.sources);
|
||||
let newest = null;
|
||||
for (const s of candidates) {
|
||||
const ms = dateMs(s && s.published);
|
||||
if (ms !== null && (newest === null || ms > newest)) newest = ms;
|
||||
}
|
||||
return newest;
|
||||
}
|
||||
|
||||
/**
|
||||
* Classify every register entry as fresh or stale by the age of its source.verified stamp.
|
||||
*
|
||||
* @param {{entries:object[]}} register
|
||||
* @param {{ referenceDate: string|Date, staleAfterDays?: number }} opts
|
||||
* @param {{ referenceDate: string|Date, staleAfterDays?: number, evidenceStaleAfterDays?: number }} opts
|
||||
* @returns {{
|
||||
* referenceDate: string,
|
||||
* staleAfterDays: number,
|
||||
* stale: Array<{id:string, verified:string|undefined, ageDays:number|null, url:string|undefined, claim:string|undefined}>,
|
||||
* evidenceStaleAfterDays: number,
|
||||
* stale: Array<{id:string, verified:string|undefined, ageDays:number|null, url:string|undefined, claim:string|undefined, reasons:string[]}>,
|
||||
* fresh: Array<{id:string, verified:string|undefined, ageDays:number}>,
|
||||
* counts: { total:number, stale:number, fresh:number }
|
||||
* }}
|
||||
|
|
@ -63,6 +96,10 @@ export function assessFreshness(register, opts = {}) {
|
|||
const ref = normalizeReferenceDate(opts.referenceDate);
|
||||
const staleAfterDays =
|
||||
typeof opts.staleAfterDays === 'number' ? opts.staleAfterDays : STALE_AFTER_DAYS_DEFAULT;
|
||||
const evidenceStaleAfterDays =
|
||||
typeof opts.evidenceStaleAfterDays === 'number'
|
||||
? opts.evidenceStaleAfterDays
|
||||
: EVIDENCE_STALE_AFTER_DAYS_DEFAULT;
|
||||
|
||||
const entries = (register && Array.isArray(register.entries)) ? register.entries : [];
|
||||
const stale = [];
|
||||
|
|
@ -71,14 +108,30 @@ export function assessFreshness(register, opts = {}) {
|
|||
for (const e of entries) {
|
||||
const verified = e && e.source ? e.source.verified : undefined;
|
||||
const vms = verifiedMs(e);
|
||||
const reasons = [];
|
||||
let ageDays = null;
|
||||
|
||||
if (vms === null) {
|
||||
// No re-checkable date → needs attention. Stale with ageDays null.
|
||||
stale.push({ id: e && e.id, verified, ageDays: null, url: e && e.source && e.source.url, claim: e && e.claim });
|
||||
continue;
|
||||
// No re-checkable date → needs attention.
|
||||
reasons.push('no-verified-date');
|
||||
} else {
|
||||
ageDays = Math.floor((ref.ms - vms) / DAY_MS);
|
||||
if (ageDays > staleAfterDays) reasons.push('verified-age');
|
||||
}
|
||||
const ageDays = Math.floor((ref.ms - vms) / DAY_MS);
|
||||
if (ageDays > staleAfterDays) {
|
||||
stale.push({ id: e.id, verified, ageDays, url: e.source && e.source.url, claim: e.claim });
|
||||
|
||||
// A source explicitly marked as superseded is stale no matter how fresh the
|
||||
// verified stamp is — the stamp certifies the OLD source.
|
||||
if (e && e.source && e.source.supersededBy) reasons.push('superseded');
|
||||
|
||||
// Evidence age: keyed on the newest published date across all sources, so a
|
||||
// re-read of the old source never clears it — only newer evidence does.
|
||||
const evMs = newestEvidenceMs(e);
|
||||
if (evMs !== null && Math.floor((ref.ms - evMs) / DAY_MS) > evidenceStaleAfterDays) {
|
||||
reasons.push('evidence-age');
|
||||
}
|
||||
|
||||
if (reasons.length > 0) {
|
||||
stale.push({ id: e && e.id, verified, ageDays, url: e && e.source && e.source.url, claim: e && e.claim, reasons });
|
||||
} else {
|
||||
fresh.push({ id: e.id, verified, ageDays });
|
||||
}
|
||||
|
|
@ -87,6 +140,7 @@ export function assessFreshness(register, opts = {}) {
|
|||
return {
|
||||
referenceDate: ref.iso,
|
||||
staleAfterDays,
|
||||
evidenceStaleAfterDays,
|
||||
stale,
|
||||
fresh,
|
||||
counts: { total: entries.length, stale: stale.length, fresh: fresh.length },
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import assert from 'node:assert/strict';
|
|||
import {
|
||||
assessFreshness,
|
||||
STALE_AFTER_DAYS_DEFAULT,
|
||||
EVIDENCE_STALE_AFTER_DAYS_DEFAULT,
|
||||
} from '../../scanners/lib/knowledge-refresh.mjs';
|
||||
|
||||
// Build a register with one entry verified on `verified`.
|
||||
|
|
@ -118,6 +119,76 @@ describe('assessFreshness — referenceDate input forms', () => {
|
|||
});
|
||||
});
|
||||
|
||||
// The BP-SUB-001 defect class (2026-08-03): an entry re-verified against an OLD
|
||||
// source stayed green while a newer source (published 2026-07-24) sat unnoticed.
|
||||
// Freshness must therefore also look at the EVIDENCE (source publication age and
|
||||
// explicit supersession), not just the entry's verified stamp. The cure for old
|
||||
// evidence is a newer source in `sources[]`, not another re-read of the old one.
|
||||
describe('assessFreshness — evidence age (green-but-outdated defect)', () => {
|
||||
it('flags a recently-verified entry whose only evidence is older than evidenceStaleAfterDays', () => {
|
||||
const e = entry('BP-A-001', '2026-06-01'); // verified 20 days before REF: green by the old rule
|
||||
e.source.published = '2025-01-01';
|
||||
const r = assessFreshness(reg([e]), { referenceDate: REF, evidenceStaleAfterDays: 365 });
|
||||
assert.equal(r.counts.stale, 1);
|
||||
assert.deepStrictEqual(r.stale[0].reasons, ['evidence-age']);
|
||||
});
|
||||
|
||||
it('a newer corroborating source in sources[] clears evidence-age (re-verifying does not)', () => {
|
||||
const e = entry('BP-A-001', '2026-06-01');
|
||||
e.source.published = '2025-01-01';
|
||||
e.sources = [{ url: 'https://example.com/newer', published: '2026-05-30' }];
|
||||
const r = assessFreshness(reg([e]), { referenceDate: REF, evidenceStaleAfterDays: 365 });
|
||||
assert.equal(r.counts.stale, 0);
|
||||
assert.equal(r.counts.fresh, 1);
|
||||
});
|
||||
|
||||
it('stays silent when no source carries a published date (rule is opt-in per entry)', () => {
|
||||
const r = assessFreshness(reg([entry('BP-A-001', '2026-06-01')]), {
|
||||
referenceDate: REF,
|
||||
evidenceStaleAfterDays: 1,
|
||||
});
|
||||
assert.equal(r.counts.stale, 0);
|
||||
});
|
||||
|
||||
it('exports a default evidence cadence of 365 days and echoes it in the result', () => {
|
||||
assert.equal(EVIDENCE_STALE_AFTER_DAYS_DEFAULT, 365);
|
||||
const r = assessFreshness(reg([]), { referenceDate: REF });
|
||||
assert.equal(r.evidenceStaleAfterDays, 365);
|
||||
});
|
||||
});
|
||||
|
||||
describe('assessFreshness — supersededBy', () => {
|
||||
it('flags a superseded source regardless of a fresh verified stamp', () => {
|
||||
const e = entry('BP-A-001', '2026-06-20'); // verified 1 day before REF
|
||||
e.source.supersededBy = { url: 'https://example.com/successor', published: '2026-05-01' };
|
||||
const r = assessFreshness(reg([e]), { referenceDate: REF });
|
||||
assert.equal(r.counts.stale, 1);
|
||||
assert.ok(r.stale[0].reasons.includes('superseded'));
|
||||
});
|
||||
|
||||
it('an entry can be stale for both verified-age and superseded at once', () => {
|
||||
const e = entry('BP-A-001', '2026-01-01'); // 171 days > 90
|
||||
e.source.supersededBy = { url: 'https://example.com/successor' };
|
||||
const r = assessFreshness(reg([e]), { referenceDate: REF });
|
||||
assert.equal(r.counts.stale, 1);
|
||||
assert.deepStrictEqual([...r.stale[0].reasons].sort(), ['superseded', 'verified-age']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('assessFreshness — reasons on existing rules (back-compat shape)', () => {
|
||||
it('verified-age staleness carries reasons ["verified-age"]', () => {
|
||||
const r = assessFreshness(reg([entry('BP-A-001', '2026-01-01')]), { referenceDate: REF });
|
||||
assert.deepStrictEqual(r.stale[0].reasons, ['verified-age']);
|
||||
});
|
||||
|
||||
it('missing/unparseable verified carries reasons ["no-verified-date"]', () => {
|
||||
const bad = entry('BP-A-001', '2026-06-01');
|
||||
bad.source.verified = 'nope';
|
||||
const r = assessFreshness(reg([bad]), { referenceDate: REF });
|
||||
assert.deepStrictEqual(r.stale[0].reasons, ['no-verified-date']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('assessFreshness — defensive cases', () => {
|
||||
it('returns all-zero counts for an empty register', () => {
|
||||
const r = assessFreshness(reg([]), { referenceDate: REF });
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue