feat(linkedin-studio): N11 — serie-destillat + skjelett-sjekk + specifics-bank bruks-logging [skip-docs]
Every long-form gate agent sees ONE edition, so «the reader has heard this
before» was structurally invisible — and the specifics-bank dedupe actively
encouraged re-surfacing the same material. At series cadence that is the
fastest-growing defect class. N11 makes it visible BEFORE the skeleton is
approved, at two grains:
- Series grain — new scripts/editions package: each locked edition's spent
anecdotes/arguments/hooks are written to <serie>/linkedin/series-distillate.json
at Step 8 lock (distil-append), and the next skeleton is checked against it at
Step 2.5 (distil-check) with the finding folded into the annotation gate the
operator already reads. Advisory, never blocking: a deliberate callback is a
legitimate move, an unnoticed retread is not.
- Material grain — specifics-bank usedIn log: record-usage stamps «used in
edition NN» on the specifics an edition actually consumed (read from the bound
slot-map, so abstrakt/ekstern stamp nothing). At lock, so it means published;
idempotent under a pivot re-lock. Additive-optional, schema stays v1.
Placement deviates from the plan text (${DATA}) after premise-verification:
per-series state belongs in the series root beside edition-state.json, where
Step 0 already resolves the path and no slug→path map is needed. Operator
approved. The distillate module also lands in scripts/editions now rather than
at N12, since the AC required a testable roundtrip and N12 planned that package
anyway — N12 extends it instead of creating it.
Similarity is character-trigram Jaccard, not word overlap: the plugin is
language-general and inflection (migrere/migreringen) breaks word tokens.
Calibrated on real paraphrase pairs — retellings 0.44-0.55, unrelated 0.04-0.06,
same-topic-different-story 0.24 — so the default threshold sits in the gap at
0.40. Word-Jaccard scored a shortened hook paraphrase at 0.11.
TDD (Iron Law): tests written first and verified red before implementation, for
both the new package and the bank logging.
Suites: editions 27/0 (new suite line + guard, floor 27) · specifics-bank
28 -> 45 · test-runner 163 -> 173 (Section 16r: 9 unconditional greps +
non-vacuity self-test; anti-erosion floor 146 -> 155) · trends 300/0 ·
brain 134/0 · hooks 140/0 · tests 35/0 · render 60/0. tsc --noEmit clean.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QxvWAjte7vPcF79QeSRvRJ
This commit is contained in:
parent
08106317db
commit
b54e450c3e
20 changed files with 2152 additions and 19 deletions
152
scripts/editions/tests/cli.test.ts
Normal file
152
scripts/editions/tests/cli.test.ts
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
import { describe, test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { mkdtempSync, rmSync, writeFileSync, readFileSync, existsSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
// Resolve the package root (scripts/editions) so the subprocess `src/cli.ts` path +
|
||||
// the `tsx` loader resolve regardless of the runner's cwd.
|
||||
const editionsDir = fileURLToPath(new URL("..", import.meta.url));
|
||||
|
||||
function run(args: string[]): { status: number | null; stdout: string; stderr: string } {
|
||||
const res = spawnSync("node", ["--import", "tsx", "src/cli.ts", ...args], {
|
||||
encoding: "utf8",
|
||||
cwd: editionsDir,
|
||||
});
|
||||
return { status: res.status, stdout: res.stdout, stderr: res.stderr };
|
||||
}
|
||||
|
||||
const tmp = () => mkdtempSync(join(tmpdir(), "editions-cli-"));
|
||||
|
||||
const EXTRACT = {
|
||||
editionId: "05",
|
||||
title: "Da pipelinen brøt sammen",
|
||||
lockedAt: "2026-07-20",
|
||||
anecdotes: ["Jeg brukte tre uker på å migrere pipeline-en, og halvparten av tiden gikk til testdata."],
|
||||
arguments: ["Modellen var ikke problemet — det var dataene ingen hadde ryddet på fem år."],
|
||||
hooks: ["De fleste AI-prosjekter dør ikke av dårlig modell. De dør av dårlig data."],
|
||||
};
|
||||
|
||||
describe("editions CLI — distil-append / distil-check round-trip (N11 AC)", () => {
|
||||
test("append -> check round-trips through a real file and flags the reuse", () => {
|
||||
const dir = tmp();
|
||||
try {
|
||||
const distillate = join(dir, "linkedin", "series-distillate.json");
|
||||
const extract = join(dir, "extract.json");
|
||||
const skeleton = join(dir, "skeleton.json");
|
||||
writeFileSync(extract, JSON.stringify(EXTRACT), "utf8");
|
||||
|
||||
// 1. lock writes the distillate (file did not exist)
|
||||
const appended = run(["distil-append", "--distillate", distillate, "--series", "seres", "--extract", extract]);
|
||||
assert.equal(appended.status, 0, appended.stderr);
|
||||
assert.ok(existsSync(distillate), "distil-append must create the distillate file");
|
||||
const onDisk = JSON.parse(readFileSync(distillate, "utf8"));
|
||||
assert.equal(onDisk.series, "seres");
|
||||
assert.equal(onDisk.editions.length, 1);
|
||||
assert.equal(onDisk.editions[0].editionId, "05");
|
||||
|
||||
// 2. a later skeleton reusing the same hook is flagged, with attribution
|
||||
writeFileSync(skeleton, JSON.stringify({ hooks: [EXTRACT.hooks[0]] }), "utf8");
|
||||
const checked = run(["distil-check", "--distillate", distillate, "--skeleton", skeleton, "--json"]);
|
||||
assert.equal(checked.status, 0, checked.stderr);
|
||||
const report = JSON.parse(checked.stdout);
|
||||
assert.equal(report.verdict, "REUSE");
|
||||
assert.equal(report.hits[0].editionId, "05");
|
||||
assert.equal(report.hits[0].kind, "hook");
|
||||
|
||||
// 3. fresh material on the same distillate is CLEAR
|
||||
writeFileSync(
|
||||
skeleton,
|
||||
JSON.stringify({ hooks: ["Innkjøpsavdelingen krevde en garanti vi ikke kunne gi uten revisjon."] }),
|
||||
"utf8",
|
||||
);
|
||||
const clear = run(["distil-check", "--distillate", distillate, "--skeleton", skeleton, "--json"]);
|
||||
assert.equal(clear.status, 0, clear.stderr);
|
||||
assert.equal(JSON.parse(clear.stdout).verdict, "CLEAR");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("re-appending the same edition replaces it — a pivot re-lock does not duplicate", () => {
|
||||
const dir = tmp();
|
||||
try {
|
||||
const distillate = join(dir, "series-distillate.json");
|
||||
const extract = join(dir, "extract.json");
|
||||
writeFileSync(extract, JSON.stringify(EXTRACT), "utf8");
|
||||
run(["distil-append", "--distillate", distillate, "--series", "seres", "--extract", extract]);
|
||||
|
||||
writeFileSync(extract, JSON.stringify({ ...EXTRACT, title: "Revidert" }), "utf8");
|
||||
const again = run(["distil-append", "--distillate", distillate, "--series", "seres", "--extract", extract]);
|
||||
assert.equal(again.status, 0, again.stderr);
|
||||
|
||||
const onDisk = JSON.parse(readFileSync(distillate, "utf8"));
|
||||
assert.equal(onDisk.editions.length, 1);
|
||||
assert.equal(onDisk.editions[0].title, "Revidert");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("distil-check against a distillate that does not exist yet is CLEAR, not an error", () => {
|
||||
const dir = tmp();
|
||||
try {
|
||||
const skeleton = join(dir, "skeleton.json");
|
||||
writeFileSync(skeleton, JSON.stringify({ hooks: ["første utgave i serien"] }), "utf8");
|
||||
const res = run([
|
||||
"distil-check",
|
||||
"--distillate",
|
||||
join(dir, "absent.json"),
|
||||
"--skeleton",
|
||||
skeleton,
|
||||
"--json",
|
||||
]);
|
||||
assert.equal(res.status, 0, res.stderr);
|
||||
const report = JSON.parse(res.stdout);
|
||||
assert.equal(report.verdict, "CLEAR");
|
||||
assert.equal(report.comparedAgainst, 0);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("human-readable output names the reused unit and its edition", () => {
|
||||
const dir = tmp();
|
||||
try {
|
||||
const distillate = join(dir, "d.json");
|
||||
const extract = join(dir, "extract.json");
|
||||
const skeleton = join(dir, "skeleton.json");
|
||||
writeFileSync(extract, JSON.stringify(EXTRACT), "utf8");
|
||||
run(["distil-append", "--distillate", distillate, "--series", "seres", "--extract", extract]);
|
||||
writeFileSync(skeleton, JSON.stringify({ hooks: [EXTRACT.hooks[0]] }), "utf8");
|
||||
|
||||
const res = run(["distil-check", "--distillate", distillate, "--skeleton", skeleton]);
|
||||
assert.equal(res.status, 0, res.stderr);
|
||||
assert.match(res.stdout, /REUSE/);
|
||||
assert.match(res.stdout, /05/);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("bad invocations exit 2 with usage on stderr", () => {
|
||||
const dir = tmp();
|
||||
try {
|
||||
assert.equal(run(["distil-append", "--series", "seres"]).status, 2); // no --extract/--distillate
|
||||
assert.equal(run(["distil-check"]).status, 2); // no --skeleton
|
||||
assert.equal(run(["nonsense"]).status, 2);
|
||||
assert.equal(run([]).status, 2);
|
||||
|
||||
// an extract missing required fields is a usage error, not a silent empty entry
|
||||
const bad = join(dir, "bad.json");
|
||||
writeFileSync(bad, JSON.stringify({ title: "no id" }), "utf8");
|
||||
const res = run(["distil-append", "--distillate", join(dir, "d.json"), "--series", "s", "--extract", bad]);
|
||||
assert.equal(res.status, 2);
|
||||
assert.match(res.stderr, /editionId/);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
260
scripts/editions/tests/distillate.test.ts
Normal file
260
scripts/editions/tests/distillate.test.ts
Normal file
|
|
@ -0,0 +1,260 @@
|
|||
import { describe, test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtempSync, rmSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
import {
|
||||
DEFAULT_THRESHOLD,
|
||||
appendEntry,
|
||||
checkSkeleton,
|
||||
defaultDistillatePath,
|
||||
emptyDistillate,
|
||||
loadDistillate,
|
||||
normalizeUnit,
|
||||
saveDistillate,
|
||||
similarity,
|
||||
} from "../src/distillate.js";
|
||||
import { SCHEMA_VERSION } from "../src/types.js";
|
||||
import type { DistillateEntry } from "../src/types.js";
|
||||
|
||||
const tmp = () => mkdtempSync(join(tmpdir(), "editions-"));
|
||||
|
||||
const entry = (over: Partial<DistillateEntry> = {}): DistillateEntry => ({
|
||||
editionId: "05",
|
||||
title: "Da pipelinen brøt sammen",
|
||||
lockedAt: "2026-07-20",
|
||||
anecdotes: ["Jeg brukte tre uker på å migrere pipeline-en, og halvparten av tiden gikk til testdata."],
|
||||
arguments: ["Modellen var ikke problemet — det var dataene ingen hadde ryddet på fem år."],
|
||||
hooks: ["De fleste AI-prosjekter dør ikke av dårlig modell. De dør av dårlig data."],
|
||||
...over,
|
||||
});
|
||||
|
||||
describe("series distillate — store", () => {
|
||||
test("emptyDistillate carries the schema version, the series and no editions", () => {
|
||||
const d = emptyDistillate("seres");
|
||||
assert.equal(d.schemaVersion, SCHEMA_VERSION);
|
||||
assert.equal(d.series, "seres");
|
||||
assert.deepEqual(d.editions, []);
|
||||
});
|
||||
|
||||
test("loadDistillate on a missing file returns an empty distillate (never throws)", () => {
|
||||
const dir = tmp();
|
||||
try {
|
||||
const d = loadDistillate(join(dir, "nope.json"), "seres");
|
||||
assert.equal(d.series, "seres");
|
||||
assert.deepEqual(d.editions, []);
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("save -> load round-trips an appended edition byte-faithfully", () => {
|
||||
const dir = tmp();
|
||||
try {
|
||||
const path = join(dir, "linkedin", "series-distillate.json");
|
||||
const { distillate } = appendEntry(emptyDistillate("seres"), entry());
|
||||
saveDistillate(path, distillate);
|
||||
|
||||
const back = loadDistillate(path, "seres");
|
||||
assert.equal(back.editions.length, 1);
|
||||
assert.deepEqual(back.editions[0], entry());
|
||||
assert.equal(back.schemaVersion, SCHEMA_VERSION);
|
||||
// trailing newline, like every other store in the plugin
|
||||
assert.ok(readFileSync(path, "utf8").endsWith("}\n"));
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("appendEntry appends a new edition and reports it", () => {
|
||||
const res = appendEntry(emptyDistillate("seres"), entry());
|
||||
assert.equal(res.appended, true);
|
||||
assert.equal(res.replaced, false);
|
||||
assert.equal(res.distillate.editions.length, 1);
|
||||
});
|
||||
|
||||
test("re-locking the same edition REPLACES it, never duplicates (pivot re-lock)", () => {
|
||||
const first = appendEntry(emptyDistillate("seres"), entry());
|
||||
const revised = entry({ title: "Da pipelinen brøt sammen (revidert)" });
|
||||
const second = appendEntry(first.distillate, revised);
|
||||
|
||||
assert.equal(second.appended, false);
|
||||
assert.equal(second.replaced, true);
|
||||
assert.equal(second.distillate.editions.length, 1);
|
||||
assert.equal(second.distillate.editions[0].title, "Da pipelinen brøt sammen (revidert)");
|
||||
});
|
||||
|
||||
test("editions keep insertion order", () => {
|
||||
let d = emptyDistillate("seres");
|
||||
d = appendEntry(d, entry({ editionId: "03" })).distillate;
|
||||
d = appendEntry(d, entry({ editionId: "05" })).distillate;
|
||||
d = appendEntry(d, entry({ editionId: "04" })).distillate;
|
||||
assert.deepEqual(
|
||||
d.editions.map((e) => e.editionId),
|
||||
["03", "05", "04"],
|
||||
);
|
||||
});
|
||||
|
||||
test("defaultDistillatePath sits beside edition-state in the series root", () => {
|
||||
assert.equal(
|
||||
defaultDistillatePath("/series/seres"),
|
||||
join("/series/seres", "linkedin", "series-distillate.json"),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("series distillate — similarity (language-agnostic character trigrams)", () => {
|
||||
test("normalizeUnit lowercases, strips punctuation and collapses whitespace", () => {
|
||||
assert.equal(normalizeUnit(" Hei, DU — da! "), "hei du da");
|
||||
});
|
||||
|
||||
test("identical text scores 1", () => {
|
||||
const s = "Vi målte 40 prosent kortere ledetid etter omleggingen.";
|
||||
assert.equal(similarity(s, s), 1);
|
||||
});
|
||||
|
||||
test("empty input scores 0 instead of dividing by zero", () => {
|
||||
assert.equal(similarity("", "noe tekst"), 0);
|
||||
assert.equal(similarity("noe tekst", " "), 0);
|
||||
});
|
||||
|
||||
test("similarity is symmetric", () => {
|
||||
const a = "Modellen var ikke problemet — det var dataene ingen hadde ryddet på fem år.";
|
||||
const b = "Problemet var aldri modellen. Det var fem år med data som ingen hadde ryddet.";
|
||||
assert.equal(similarity(a, b), similarity(b, a));
|
||||
});
|
||||
|
||||
// The calibration set the threshold is derived from. Paraphrase (the same story
|
||||
// retold) must land ABOVE the default; unrelated material and same-topic-different-
|
||||
// story must land BELOW it — inflection-heavy Norwegian included, which is why this
|
||||
// is character trigrams and not word tokens (a shortened hook paraphrase scores
|
||||
// 0.11 on word-Jaccard and would slip through).
|
||||
test("paraphrase of the same story clears the default threshold", () => {
|
||||
const pairs: Array<[string, string]> = [
|
||||
[
|
||||
"Jeg brukte tre uker på å migrere pipeline-en, og halvparten av tiden gikk til testdata.",
|
||||
"Migreringen av pipelinen tok tre uker, og halve tiden gikk med til testdata.",
|
||||
],
|
||||
[
|
||||
"Modellen var ikke problemet — det var dataene ingen hadde ryddet på fem år.",
|
||||
"Problemet var aldri modellen. Det var fem år med data som ingen hadde ryddet.",
|
||||
],
|
||||
[
|
||||
"De fleste AI-prosjekter dør ikke av dårlig modell. De dør av dårlig data.",
|
||||
"AI-prosjekter dør sjelden av modellen. De dør av dataene.",
|
||||
],
|
||||
];
|
||||
for (const [a, b] of pairs) {
|
||||
assert.ok(
|
||||
similarity(a, b) >= DEFAULT_THRESHOLD,
|
||||
`expected paraphrase >= ${DEFAULT_THRESHOLD}, got ${similarity(a, b).toFixed(3)}: ${a}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("unrelated material and same-topic-different-story stay below the threshold", () => {
|
||||
const pairs: Array<[string, string]> = [
|
||||
[
|
||||
"Jeg brukte tre uker på å migrere pipeline-en, og halvparten av tiden gikk til testdata.",
|
||||
"Innkjøpsavdelingen krevde en leverandørgaranti vi ikke kunne gi uten en revisjon.",
|
||||
],
|
||||
[
|
||||
"De fleste AI-prosjekter dør ikke av dårlig modell. De dør av dårlig data.",
|
||||
"Vi har hatt en fast ukentlig gjennomgang av porteføljen siden januar.",
|
||||
],
|
||||
[
|
||||
// same topic, different measured claim — a NEW edition, not a retread
|
||||
"Vi målte 40 prosent kortere ledetid etter omleggingen.",
|
||||
"Vi målte 12 prosent høyere treffrate etter at vi byttet embedding-modell.",
|
||||
],
|
||||
];
|
||||
for (const [a, b] of pairs) {
|
||||
assert.ok(
|
||||
similarity(a, b) < DEFAULT_THRESHOLD,
|
||||
`expected non-reuse < ${DEFAULT_THRESHOLD}, got ${similarity(a, b).toFixed(3)}: ${a}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("series distillate — skeleton check (Step 2.5)", () => {
|
||||
const distillate = appendEntry(emptyDistillate("seres"), entry()).distillate;
|
||||
|
||||
test("an empty distillate is always CLEAR (first edition in a series)", () => {
|
||||
const r = checkSkeleton(emptyDistillate("seres"), { hooks: ["hva som helst"] });
|
||||
assert.equal(r.verdict, "CLEAR");
|
||||
assert.deepEqual(r.hits, []);
|
||||
assert.equal(r.comparedAgainst, 0);
|
||||
});
|
||||
|
||||
test("fresh material is CLEAR", () => {
|
||||
const r = checkSkeleton(distillate, {
|
||||
anecdotes: ["Innkjøpsavdelingen krevde en leverandørgaranti vi ikke kunne gi uten en revisjon."],
|
||||
});
|
||||
assert.equal(r.verdict, "CLEAR");
|
||||
assert.deepEqual(r.hits, []);
|
||||
});
|
||||
|
||||
test("verbatim reuse is REUSE and names the edition it came from", () => {
|
||||
const r = checkSkeleton(distillate, { hooks: [entry().hooks[0]] });
|
||||
assert.equal(r.verdict, "REUSE");
|
||||
assert.equal(r.hits.length, 1);
|
||||
assert.equal(r.hits[0].kind, "hook");
|
||||
assert.equal(r.hits[0].editionId, "05");
|
||||
assert.equal(r.hits[0].score, 1);
|
||||
assert.equal(r.hits[0].matched, entry().hooks[0]);
|
||||
});
|
||||
|
||||
test("paraphrased reuse is caught, not just verbatim", () => {
|
||||
const r = checkSkeleton(distillate, {
|
||||
anecdotes: ["Migreringen av pipelinen tok tre uker, og halve tiden gikk med til testdata."],
|
||||
});
|
||||
assert.equal(r.verdict, "REUSE");
|
||||
assert.equal(r.hits[0].kind, "anecdote");
|
||||
assert.ok(r.hits[0].score >= DEFAULT_THRESHOLD);
|
||||
});
|
||||
|
||||
test("kinds are compared within kind — a hook is not matched against an anecdote", () => {
|
||||
// the anecdote text, offered as a hook: the anecdote bucket must not match it
|
||||
const r = checkSkeleton(distillate, { hooks: [entry().anecdotes[0]] });
|
||||
assert.equal(r.verdict, "CLEAR");
|
||||
});
|
||||
|
||||
test("hits are sorted by score descending", () => {
|
||||
let d = emptyDistillate("seres");
|
||||
d = appendEntry(d, entry({ editionId: "03", hooks: ["De fleste AI-prosjekter dør ikke av dårlig modell. De dør av dårlig data."] })).distillate;
|
||||
d = appendEntry(d, entry({ editionId: "04", hooks: ["AI-prosjekter dør sjelden av modellen. De dør av dataene."], anecdotes: [], arguments: [] })).distillate;
|
||||
|
||||
const r = checkSkeleton(d, { hooks: ["De fleste AI-prosjekter dør ikke av dårlig modell. De dør av dårlig data."] });
|
||||
assert.equal(r.verdict, "REUSE");
|
||||
assert.ok(r.hits.length >= 2);
|
||||
for (let i = 1; i < r.hits.length; i++) {
|
||||
assert.ok(r.hits[i - 1].score >= r.hits[i].score, "hits must be sorted by score desc");
|
||||
}
|
||||
assert.equal(r.hits[0].score, 1);
|
||||
});
|
||||
|
||||
test("the threshold is configurable and reported back", () => {
|
||||
const strict = checkSkeleton(
|
||||
distillate,
|
||||
{ anecdotes: ["Migreringen av pipelinen tok tre uker, og halve tiden gikk med til testdata."] },
|
||||
{ threshold: 0.95 },
|
||||
);
|
||||
assert.equal(strict.verdict, "CLEAR");
|
||||
assert.equal(strict.threshold, 0.95);
|
||||
});
|
||||
|
||||
test("comparedAgainst reports how many editions the skeleton was checked against", () => {
|
||||
let d = emptyDistillate("seres");
|
||||
d = appendEntry(d, entry({ editionId: "03" })).distillate;
|
||||
d = appendEntry(d, entry({ editionId: "04" })).distillate;
|
||||
assert.equal(checkSkeleton(d, { hooks: ["noe helt annet om innkjøp og garantier"] }).comparedAgainst, 2);
|
||||
});
|
||||
|
||||
test("an absent bucket on the candidate is not an error", () => {
|
||||
const r = checkSkeleton(distillate, {});
|
||||
assert.equal(r.verdict, "CLEAR");
|
||||
assert.deepEqual(r.hits, []);
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue