"""What each candidate predicate would admit, measured by normalizing the SURFACE form onto a shape the CURRENT parser already accepts. The predicate is imported, never re-implemented: only the input's spelling is rewritten.""" import os, re, sys, collections sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) from llm_ingestion_guard import okf ROOT = os.path.join(os.path.dirname(__file__), "corpus", "okf", "bundles") def docs(): out = [] for dirpath, _d, fns in os.walk(ROOT): for fn in sorted(fns): if fn.endswith(".md") and fn not in ("index.md", "log.md"): out.append(os.path.join(dirpath, fn)) return sorted(out) def split_fm(text): lines = text.split("\n") if not lines or lines[0].strip() != "---": return None, None for i in range(1, len(lines)): if lines[i].strip() == "---": return lines[1:i], lines[i + 1:] return None, None TOPKEY = re.compile(r"^([A-Za-z0-9_]+):(.*)$") def n_flow_scalar_seq(fm): """P1: `k: [a, b]` where every element is a plain scalar -> indented block list.""" out = [] for raw in fm: m = TOPKEY.match(raw) if m and m.group(2).strip().startswith("[") and m.group(2).strip().endswith("]"): inner = m.group(2).strip()[1:-1] elems = [e.strip() for e in inner.split(",")] if inner and all(e and "{" not in e and "}" not in e and "[" not in e for e in elems): out.append("%s:" % m.group(1)) out.extend(" - %s" % e for e in elems) continue out.append(raw) return out def n_flush_block_seq(fm): """P2: a flush-left `- item` run under a bare `k:` -> the same run, indented.""" out, i, n = [], 0, len(fm) while i < n: raw = fm[i] m = TOPKEY.match(raw) if m and m.group(2).strip() == "" and i + 1 < n and fm[i + 1][:1] == "-": out.append(raw) i += 1 while i < n and (fm[i][:1] == "-" or fm[i][:1] in (" ", "\t")): out.append(" " + fm[i]) i += 1 continue out.append(raw) i += 1 return out def n_folded_scalar(fm): """P3: `k: text` continued on more-indented plain lines -> one joined line.""" out, i, n = [], 0, len(fm) while i < n: raw = fm[i] m = TOPKEY.match(raw) if m and m.group(2).strip() and not m.group(2).strip()[0] in "[{": acc = raw i += 1 while i < n and fm[i][:1] in (" ", "\t") and not fm[i].strip().startswith("- "): acc = acc.rstrip() + " " + fm[i].strip() i += 1 out.append(acc) continue out.append(raw) i += 1 return out def parses(fm, body): text = "---\n" + "\n".join(fm) + "\n---\n" + "\n".join(body) try: okf.parse_frontmatter(text) return None except okf.OKFFrontmatterError as exc: return str(exc) COMBOS = [ ("baseline (v1.3.0 as shipped)", []), ("P1 scalar flow sequence", [n_flow_scalar_seq]), ("P2 flush block sequence", [n_flush_block_seq]), ("P3 folded plain scalar", [n_folded_scalar]), ("P1+P3", [n_flow_scalar_seq, n_folded_scalar]), ("P2+P3", [n_flush_block_seq, n_folded_scalar]), ("P1+P2", [n_flow_scalar_seq, n_flush_block_seq]), ("P1+P2+P3", [n_flow_scalar_seq, n_flush_block_seq, n_folded_scalar]), ] def main(): paths = docs() print("denominator: %d documents (pin 3fcbb9f, parser HEAD)" % len(paths)) for name, fns in COMBOS: ok = 0 residual = collections.Counter() for p in paths: fm, body = split_fm(open(p, encoding="utf-8").read()) for fn in fns: fm = fn(fm) err = parses(fm, body) if err is None: ok += 1 else: residual[err.split(":")[0]] += 1 top = "; ".join("%dx %s" % (c, r) for r, c in residual.most_common(3)) print(" %-30s %2d/%d residual: %s" % (name, ok, len(paths), top or "-")) main() # --- diagnostic only (NOT a proposal): size the real binding constraint --- def n_block_mapping(fm): """D4: a top-level `k:` followed by indented `a: b` lines -> one flow mapping.""" out, i, n = [], 0, len(fm) while i < n: raw = fm[i] m = TOPKEY.match(raw) if m and m.group(2).strip() == "" and i + 1 < n and fm[i + 1][:1] in (" ", "\t") \ and not fm[i + 1].strip().startswith("- "): pairs = [] i += 1 while i < n and fm[i][:1] in (" ", "\t") and not fm[i].strip().startswith("- "): pairs.append(fm[i].strip()) i += 1 out.append("%s: { %s }" % (m.group(1), ", ".join(pairs))) continue out.append(raw) i += 1 return out print("\n--- diagnostic: what caps the corpus ABOVE the tags/description gap ---") for name, fns in [ ("D4 top-level block mapping", [n_block_mapping]), ("P1+P2+P3+D4", [n_flow_scalar_seq, n_flush_block_seq, n_folded_scalar, n_block_mapping]), ]: ok = 0 residual = collections.Counter() for p in docs(): fm, body = split_fm(open(p, encoding="utf-8").read()) for fn in fns: fm = fn(fm) err = parses(fm, body) if err is None: ok += 1 else: residual[err.split(":")[0]] += 1 top = "; ".join("%dx %s" % (c, r) for r, c in residual.most_common(3)) print(" %-30s %2d/53 residual: %s" % (name, ok, top or "-"))