feat(tools): a re-measurable outline-reach instrument for K3

tests/test_outline_measure.py

@@ -0,0 +1,152 @@
+"""The outline-reach instrument: phase 2 of order 20260906T213322Z.
+
+Measures how far Arm D's outline rule reaches into a corpus, before and after
+the orphan check that deletes a third of what it proposes. The instrument
+exists because `docs/2026-09-04-k3-arm-c.md` published figures a reader could
+not re-derive: a number without a committed script cannot be reproduced, and a
+K3 row resting on one is an assertion rather than a measurement.
+
+The identity test below is the one that matters most. An instrument that
+re-implements the grammar it measures is measuring a SECOND definition, free to
+drift from the shipped one without a single test going red -- so the instrument
+imports `outline_lines`, `outline_runs` and `find_candidates` from the tool,
+and this file pins that they are the same objects.
+
+The negative control matters as much as the positive one, for the reason
+`test_cid_measure.py` states: an instrument reporting reach on a corpus that
+has none would inflate every number it ever produces. Here the control is
+sharper than "zero" -- it is zero WITH a nonzero denominator, because "found
+nothing" and "measured nothing" are different results and only one of them is
+evidence.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))
+
+import okf_outline_measure  # noqa: E402
+import okf_propose_segments  # noqa: E402
+
+OUTLINE_DOC = """1 Innledning
+
+Bakgrunn for prosjektet og omfanget.
+
+2 Krav
+
+Krav til seksjonering av bygget.
+
+3 Gjennomfoering
+
+Framdrift, faser og overlevering.
+"""
+
+NO_OUTLINE_DOC = """# Teknisk grunnlag
+
+Innledende tekst uten nummerering i det hele tatt.
+
+## 3.1 Brannkonsept
+
+To uavhengige roemningsveier fra hver branncelle.
+"""
+
+
+def test_the_instrument_and_the_tool_are_one_definition() -> None:
+    """Identity, not equality: the cheapest proof there is no second grammar."""
+    assert okf_outline_measure.outline_runs is okf_propose_segments.outline_runs
+    assert okf_outline_measure.outline_lines is okf_propose_segments.outline_lines
+    assert okf_outline_measure.find_candidates is okf_propose_segments.find_candidates
+
+
+def test_a_hand_computed_document_reports_its_boundaries_both_sides_of_the_gate() -> None:
+    """Three chapters, each with a body, so the gate deletes none of them."""
+    result = okf_outline_measure.measure_document(OUTLINE_DOC, "doc", run_length=3)
+    assert result.pre_gate_boundaries == 3
+    assert result.post_gate_boundaries == 3
+    assert result.arm_b_entries == 0
+    assert result.arm_d_entries == 3
+    assert result.deleted_arm_b == 0
+
+
+def test_the_gate_deletes_an_arm_b_heading_the_outline_immediately_follows() -> None:
+    """The 34 %-deletion mechanism, measured on a document small enough to check.
+
+    `# Teknisk grunnlag` is followed immediately by `1 Innledning`, so its body
+    is empty and the orphan check drops it. Arm B had one entry; Arm D has
+    three, and the one Arm B had is gone.
+    """
+    text = "# Teknisk grunnlag\n\n1 Innledning\n\nA.\n\n2 Krav\n\nB.\n\n3 Slutt\n\nC.\n"
+    result = okf_outline_measure.measure_document(text, "doc", run_length=3)
+    assert result.arm_b_entries == 1
+    assert result.deleted_arm_b == 1
+    assert result.post_gate_boundaries == 3
+
+
+def test_a_corpus_with_no_outline_reports_zero_with_a_nonzero_denominator() -> None:
+    """The negative control. Zero reach is only evidence if something was measured."""
+    result = okf_outline_measure.measure_document(NO_OUTLINE_DOC, "doc", run_length=3)
+    assert result.pre_gate_boundaries == 0
+    assert result.post_gate_boundaries == 0
+    # The denominator: the document WAS measured, and Arm B did find boundaries
+    # in it, so a zero here is the rule declining rather than the probe failing.
+    assert result.arm_b_entries > 0
+    assert result.arm_d_entries == result.arm_b_entries
+
+
+def test_an_empty_document_does_not_crash() -> None:
+    result = okf_outline_measure.measure_document("", "doc", run_length=3)
+    assert result.pre_gate_boundaries == 0
+    assert result.arm_b_entries == 0
+    assert result.arm_d_entries == 0
+    assert result.unique_paths == 0
+
+
+def test_the_sample_draw_reproduces_a_known_ordering() -> None:
+    """The draw is the method's, re-derived: hex SHA-256 of the NFC filename.
+
+    The expected list is computed by hand from the digests, not by calling the
+    function under test -- otherwise the assertion would only prove the code
+    agrees with itself.
+    """
+    names = [
+        "alfa.pdf",
+        "beta.pdf",
+        "gamma.pdf",
+        "delta.docx",
+        "epsilon.docx",
+        "zeta.xlsx",
+        "eta.xlsx",
+        "theta.pdf",
+    ]
+    drawn = okf_outline_measure.draw_sample(names, {"pdf": 2, "docx": 1, "xlsx": 1})
+    assert drawn == ["theta.pdf", "gamma.pdf", "eta.xlsx", "delta.docx"]
+
+
+def test_the_draw_normalises_to_nfc_before_hashing() -> None:
+    """macOS hands filenames over decomposed; the two forms hash differently.
+
+    Without this the same visual corpus would draw a different sample depending
+    on which normalisation the filenames arrived in -- the same defect the
+    library already fixed in `reduce_to_id_grammar`.
+    """
+    # Escapes, not literals: a source file is stored in ONE normalisation, so
+    # writing both forms as literals would silently make them the same string
+    # and the control below would be green for the wrong reason. `oe` is used
+    # because U+00F8 has no canonical decomposition at all -- picking it would
+    # make this test vacuous in a second, quieter way.
+    composed = "caf\u00e9.pdf"  # NFC: e-acute as one code point
+    decomposed = "cafe\u0301.pdf"  # NFD: `e` plus combining acute
+    assert composed != decomposed  # the control: they really are different strings
+    assert okf_outline_measure._draw_key(composed) == okf_outline_measure._draw_key(decomposed)
+    # And a second control: two names that are genuinely different still differ.
+    assert okf_outline_measure._draw_key("a.pdf") != okf_outline_measure._draw_key("b.pdf")
+
+
+def test_a_title_with_no_alphabetic_word_is_counted_as_junk() -> None:
+    """11 of the 95 surviving outline titles are junk; the report needs the count."""
+    text = "1 477 3 025\n\nA.\n\n2 Krav\n\nB.\n\n3 D L\n\nC.\n"
+    result = okf_outline_measure.measure_document(text, "doc", run_length=3)
+    assert result.post_gate_boundaries == 3
+    assert result.junk_titles == 2  # `477 3 025` and `D L` -- `Krav` is a word

tools/okf_outline_measure.py

@@ -0,0 +1,320 @@
+"""Measure how far Arm D's outline rule reaches into a corpus. Measure, don't build.
+
+Order 20260906T213322Z-1044411564-from-.claude, phase 2. `docs/2026-09-04-k3-arm-c.md`
+measured a SIZE answer to K3 and falsified it: Arm C changed 6 of 12 proposals
+and moved the four category counts by zero. The operator ruled on 2026-09-04
+that the next arm must be content-based. Arm D reads the document's OWN
+numbered outline; this instrument answers how far that rule reaches, and how
+much of its reach the shipping orphan check then deletes.
+
+It builds nothing. No new rule, no extractor, no bundle, no threshold -- the
+threshold is the operator's, and only once an arm moves the number.
+
+**It imports the shipped functions rather than re-implementing them.** An
+instrument carrying its own copy of the grammar measures a second definition
+that can drift from the tool's without a test going red, and every figure it
+publishes would then be about code nobody ships. `tests/test_outline_measure.py`
+pins the identity.
+
+It also carries the K3 SAMPLE DRAW, for the same reason: `8/12` is a published
+figure, and until now the draw existed only in an uncommitted script. A number
+whose script is not committed cannot be reproduced, which is the whole
+complaint this instrument answers.
+
+## What is an expectation here and what is not
+
+**Pre-gate `144` boundaries on `23/39` documents are the declared expectation.**
+They were measured this session against a replica that first reproduced Arm B's
+618 candidates exactly, so a different pre-gate total means the shipped rule is
+not the measured one.
+
+**Post-gate figures are reference values, printed beside the measured ones and
+never gating anything.** Declaring them would force a later reader to write a
+number down as true because a plan predicted it. What the orphan check deletes
+is a property of the shipped code, and this instrument's job is to report it.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import re
+import sys
+import unicodedata
+from dataclasses import dataclass, field
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from llm_ingestion_okf.errors import ExtractionError  # noqa: E402
+from llm_ingestion_okf.extract import extract_text  # noqa: E402
+from llm_ingestion_okf.materialize import reduce_to_id_grammar  # noqa: E402
+from okf_propose_segments import (  # noqa: E402
+    RULE_OUTLINE,
+    Candidate,
+    _segment_path,
+    find_candidates,
+    outline_lines,
+    outline_runs,
+)
+
+#: The gate length Arm D is measured at. Declared, not swept: the brief forbids
+#: a sweep, and 3 was chosen from the run-length distribution (328/37/18)
+#: before any reach figure was known.
+RUN_LENGTH = 3
+
+#: The K3 sample, declared in `docs/2026-09-02-k3-k4-k5-metode.md` and
+#: reproduced here rather than re-derived: n = 12, stratified 8 pdf / 3 docx /
+#: 1 xlsx, ordered within each stratum by the hex SHA-256 of the filename.
+SAMPLE_STRATA = {"pdf": 8, "docx": 3, "xlsx": 1}
+
+#: The declared pre-gate expectation. A different value means the shipped rule
+#: is not the rule that was measured, and no post-gate figure may be read.
+EXPECTED_PRE_GATE_BOUNDARIES = 144
+EXPECTED_PRE_GATE_DOCUMENTS = 23
+
+#: Reference values only -- measured this session against a replica, printed
+#: beside the shipped code's own numbers and never compared against them.
+REFERENCE_POST_GATE_BOUNDARIES = 95
+REFERENCE_POST_GATE_DOCUMENTS = 21
+REFERENCE_TOTAL_ENTRIES = 709
+REFERENCE_DELETED_ARM_B = 4
+
+#: "An alphabetic word" -- Unicode letters only, so a title made of digits and
+#: single letters (`477 3 025`, `D 1 L`) counts as junk. Digits and `_`
+#: excluded. THIS instrument's definition, not an upstream term.
+_ALPHA_WORD = re.compile(r"[^\W\d_]{2,}", re.UNICODE)
+
+
+def _draw_key(name: str) -> str:
+    """The draw's ordering key: hex SHA-256 of the NFC-normalised filename.
+
+    NFC first because macOS hands filenames over decomposed, so the same visual
+    corpus would otherwise draw a different sample depending on which form the
+    names arrived in.
+    """
+    return hashlib.sha256(unicodedata.normalize("NFC", name).encode("utf-8")).hexdigest()
+
+
+def draw_sample(names: list[str], strata: dict[str, int] | None = None) -> list[str]:
+    """The K3 draw: first k per format stratum, then canonical order over the sample."""
+    strata = SAMPLE_STRATA if strata is None else strata
+    picked: list[str] = []
+    for extension, count in strata.items():
+        pool = sorted(
+            (name for name in names if name.lower().endswith("." + extension)),
+            key=_draw_key,
+        )
+        picked.extend(pool[:count])
+    return sorted(picked, key=_draw_key)
+
+
+@dataclass(frozen=True)
+class DocumentMeasurement:
+    """One document, both sides of the orphan check."""
+
+    name: str
+    #: What the outline rule PROPOSES: the length of the last admitted run.
+    pre_gate_boundaries: int
+    #: What survives `find_candidates` -- the shipping orphan check included.
+    post_gate_boundaries: int
+    arm_b_entries: int
+    arm_d_entries: int
+    #: Arm B candidates present without the flag and absent with it, matched on
+    #: start offset. These are coverage the arm COSTS, and they are why a
+    #: corpus-wide entry delta alone would flatter the arm.
+    deleted_arm_b: int
+    unique_paths: int
+    junk_titles: int
+    span_sizes: list[int] = field(default_factory=list)
+
+    def line(self) -> str:
+        return (
+            f"| {self.name} | {self.pre_gate_boundaries} | {self.post_gate_boundaries} "
+            f"| {self.arm_b_entries} | {self.arm_d_entries} | {self.deleted_arm_b} "
+            f"| {self.unique_paths} | {self.junk_titles} |"
+        )
+
+
+@dataclass(frozen=True)
+class Row:
+    """A corpus file: measured, or a stated reason it was not."""
+
+    name: str
+    measurement: DocumentMeasurement | None
+    skip_reason: str | None
+
+    def line(self) -> str:
+        if self.measurement is None:
+            return f"| {self.name} | -- | -- | -- | -- | -- | -- | -- |"
+        return self.measurement.line()
+
+
+def _paths_for(candidates: list[Candidate], prefix: str) -> list[str]:
+    taken: set[str] = set()
+    return [_segment_path(candidate, taken, prefix) for candidate in candidates]
+
+
+def measure_document(text: str, name: str, run_length: int = RUN_LENGTH) -> DocumentMeasurement:
+    """Both sides of the gate for one document, from the SHIPPED functions."""
+    runs = outline_runs(outline_lines(text), run_length)
+    pre_gate = len(runs[-1]) if runs else 0
+
+    arm_b = find_candidates(text)
+    arm_d = find_candidates(text, outline_run=run_length)
+    outline_kept = [c for c in arm_d if c.rule == RULE_OUTLINE]
+
+    arm_b_starts = {c.start for c in arm_b}
+    arm_d_starts = {c.start for c in arm_d}
+    deleted = len(arm_b_starts - arm_d_starts)
+
+    prefix = reduce_to_id_grammar(Path(name).stem)
+    paths = _paths_for(arm_d, prefix)
+
+    junk = sum(1 for c in outline_kept if not _ALPHA_WORD.search(c.title))
+    return DocumentMeasurement(
+        name=name,
+        pre_gate_boundaries=pre_gate,
+        post_gate_boundaries=len(outline_kept),
+        arm_b_entries=len(arm_b),
+        arm_d_entries=len(arm_d),
+        deleted_arm_b=deleted,
+        unique_paths=len(set(paths)),
+        junk_titles=junk,
+        span_sizes=[c.end - c.start for c in arm_d],
+    )
+
+
+def run(corpus: Path, run_length: int = RUN_LENGTH) -> list[Row]:
+    rows: list[Row] = []
+    for path in sorted(corpus.iterdir(), key=lambda p: p.name):
+        if not path.is_file():
+            continue
+        try:
+            text = extract_text(path.name, path.read_bytes())
+        except ExtractionError as exc:
+            rows.append(Row(name=path.name, measurement=None, skip_reason=f"{exc.code}: {exc}"))
+            continue
+        rows.append(
+            Row(
+                name=path.name,
+                measurement=measure_document(text, path.name, run_length),
+                skip_reason=None,
+            )
+        )
+    return rows
+
+
+def _percentile(values: list[int], fraction: float) -> int:
+    if not values:
+        return 0
+    ordered = sorted(values)
+    index = min(len(ordered) - 1, int(fraction * (len(ordered) - 1) + 0.5))
+    return ordered[index]
+
+
+def render(corpus: Path, rows: list[Row], run_length: int = RUN_LENGTH) -> str:
+    measured = [row.measurement for row in rows if row.measurement is not None]
+    extractable = len(measured)
+    pre_total = sum(m.pre_gate_boundaries for m in measured)
+    post_total = sum(m.post_gate_boundaries for m in measured)
+    pre_docs = sum(1 for m in measured if m.pre_gate_boundaries > 0)
+    post_docs = sum(1 for m in measured if m.post_gate_boundaries > 0)
+    arm_b_total = sum(m.arm_b_entries for m in measured)
+    arm_d_total = sum(m.arm_d_entries for m in measured)
+    deleted_total = sum(m.deleted_arm_b for m in measured)
+    junk_total = sum(m.junk_titles for m in measured)
+    zero_entry = sum(1 for m in measured if m.arm_d_entries == 0)
+    unique_total = sum(m.unique_paths for m in measured)
+    spans = [size for m in measured for size in m.span_sizes]
+
+    names = [row.name for row in rows]
+    sample = draw_sample(names)
+    by_name = {m.name: m for m in measured}
+    reached = [n for n in sample if n in by_name and by_name[n].post_gate_boundaries > 0]
+
+    lines = [
+        f"# Arm D outline reach, {corpus}",
+        "",
+        f"Run-length gate: {run_length}. Declared, not swept.",
+        f"N = {len(rows)} (corpus directory file count). Extractable: {extractable}/{len(rows)}.",
+        "",
+        "## Pre-gate -- what the rule proposes (the DECLARED expectation)",
+        "",
+        f"- boundaries: **{pre_total}** (expected {EXPECTED_PRE_GATE_BOUNDARIES})",
+        f"- documents reached: **{pre_docs}/{extractable}** "
+        f"(expected {EXPECTED_PRE_GATE_DOCUMENTS}/{extractable})",
+        "",
+        "## Post-gate -- what survives the orphan check (MEASURED, not expected)",
+        "",
+        f"- boundaries: **{post_total}** (reference {REFERENCE_POST_GATE_BOUNDARIES})",
+        f"- documents reached: **{post_docs}/{extractable}** "
+        f"(reference {REFERENCE_POST_GATE_DOCUMENTS}/{extractable})",
+        f"- deleted by the gate: **{pre_total - post_total}** of {pre_total}",
+        "",
+        "## Entries, and what the arm costs",
+        "",
+        f"- Arm B entries: **{arm_b_total}**",
+        f"- Arm D entries: **{arm_d_total}** "
+        f"(delta {arm_d_total - arm_b_total:+d}, reference {REFERENCE_TOTAL_ENTRIES})",
+        f"- existing Arm B candidates deleted: **{deleted_total}** "
+        f"(reference {REFERENCE_DELETED_ARM_B})",
+        "",
+        "## The K3 sample",
+        "",
+        f"- sample reach: **{len(reached)}/{len(sample)}**",
+        "",
+        "## Quality counts, each with its denominator",
+        "",
+        f"- unique concept paths: **{unique_total}** of {arm_d_total} entries",
+        f"- documents with zero entries under Arm D: **{zero_entry}**/{extractable}",
+        f"- outline titles with no alphabetic word: **{junk_total}** of {post_total}",
+        "",
+        "## Span-size distribution, Arm D",
+        "",
+        f"- spans: {len(spans)}; min {min(spans) if spans else 0}; "
+        f"p50 {_percentile(spans, 0.5)}; p95 {_percentile(spans, 0.95)}; "
+        f"max {max(spans) if spans else 0}",
+        f"- spans under 200 chars: **{sum(1 for s in spans if s < 200)}** of {len(spans)}",
+        "",
+        "## Per document",
+        "",
+        "| file | pre-gate | post-gate | arm B | arm D | deleted | unique paths | junk titles |",
+        "|---|---|---|---|---|---|---|---|",
+    ]
+    lines.extend(row.line() for row in rows)
+    lines.extend(["", "## The K3 sample, in canonical draw order", ""])
+    for position, name in enumerate(sample):
+        measurement = by_name.get(name)
+        state = "not extractable" if measurement is None else f"{measurement.post_gate_boundaries}"
+        lines.append(f"- {position}: {name} -- outline boundaries: {state}")
+    return "\n".join(lines) + "\n"
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--corpus", type=Path, required=True, help="the directory to measure")
+    parser.add_argument("--report", type=Path, required=True, help="where to write the report")
+    parser.add_argument(
+        "--run-length",
+        type=int,
+        default=RUN_LENGTH,
+        metavar="N",
+        help="the ascending-run gate length Arm D is measured at (declared, not swept)",
+    )
+    args = parser.parse_args(argv)
+    if not args.corpus.is_dir():
+        print(
+            f"okf-outline-measure: FAILED - no corpus directory at {args.corpus}",
+            file=sys.stderr,
+        )
+        return 1
+    rows = run(args.corpus, args.run_length)
+    args.report.write_text(render(args.corpus, rows, args.run_length), encoding="utf-8", newline="")
+    print(f"okf-outline-measure: wrote {args.report}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())