fix(frontmatter): write a value a YAML reader reads back, and read both forms
K3-22. SPEC SS 11 point 1: "Every non-reserved `.md` file in the tree
contains a parseable YAML frontmatter block." Measured with PyYAML 6.0.3,
okf's own default K2 bundle failed safe_load on 41 of 455 blocks and the
R761 build on 1 of 2 763, every one a block scalar written verbatim.
Block (the profile emitter, every key): a value the K3-19 rule refuses as
plain is written double-quoted, `\` and `"` escaped; every other value keeps
its bytes, and a flow collection or an empty value is written as it stands.
The rule, now `profiles.yaml_block_plain`, agrees with PyYAML on every
top-level value in eleven measured trees (0 refused that it reads verbatim,
0 kept that it does not). Double, never single: 0 values in those trees are
`"`-wrapped and 11 193 are `'`-wrapped.
Flow (`sources`, Door A and Door B, and a run-stated flow value): the pinned
guard refuses ANY quote in a flow mapping (1.3.0, measured), so a leaf PyYAML
needs quoted has no form both read. `yaml_flow_plain` refuses it instead:
`,[]{}`, `?`, a quote, ": ", " #", a trailing `:`, a leading indicator -- a
leading `-` before a non-space excepted, which both readers take. The file
name is checked too, because it is the entry's `title` when the document
declares none. Existing codes: inbox_source_file_unaddressable,
inbox_source_title_unaddressable, source_reference_unquotable,
run_frontmatter_invalid.
Readers: parse_frontmatter, profiles' and structure's copies, and both
read_sources branches unquote a `"`-wrapped value (`\"` and `\\` decoded,
nothing else); `'`-wrapped values are untouched, and structure keeps the
single-quote rule it already had. The flow-mapping split is quote-aware, so
`{ title: "a, b" }` is one pair. The generated SKILL.md header goes through
the same block rule.
TWO K3-19 TESTS MOVED, deliberately: test_run_frontmatter built with
`sources=[{ resource: ...?languageCode=nb, ... }]`, the exact form PyYAML
refused on 2 761 of 2 761 frontmatters of K3-19's flagged build. The two
build tests now write an address without `?`; the flag-grammar test keeps
the `?` address (it only splits), and a new test holds that the build
refuses it with exit 2 and writes nothing.
1753 passed, 1 skipped (OKF_HTML_CORPUS, known). No golden moved.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
06e61a5acf
commit
ed0418f228
7 changed files with 288 additions and 79 deletions
|
|
@ -65,6 +65,7 @@ from .profiles import (
|
||||||
RESERVED_OKF_TYPE,
|
RESERVED_OKF_TYPE,
|
||||||
SEGMENTED_OKF_V0_2,
|
SEGMENTED_OKF_V0_2,
|
||||||
BundleProfile,
|
BundleProfile,
|
||||||
|
unquote_scalar,
|
||||||
)
|
)
|
||||||
|
|
||||||
#: The profile whose index policy reads a faceted, per-directory index -- the
|
#: The profile whose index policy reads a faceted, per-directory index -- the
|
||||||
|
|
@ -432,29 +433,52 @@ def _parse_flow_mappings(value: str) -> list[dict[str, str]] | None:
|
||||||
if not (item.startswith("{") and item.endswith("}")):
|
if not (item.startswith("{") and item.endswith("}")):
|
||||||
return None
|
return None
|
||||||
pairs: dict[str, str] = {}
|
pairs: dict[str, str] = {}
|
||||||
for field in item[1:-1].split(","):
|
# Split where a comma is not inside a `"`-quoted leaf, and unquote the
|
||||||
|
# leaf: a YAML reader reads `{ title: "a, b" }` as ONE pair (K3-22).
|
||||||
|
for field in _split_top_level(item[1:-1], "{", "}"):
|
||||||
key, separator, raw = field.partition(":")
|
key, separator, raw = field.partition(":")
|
||||||
if separator:
|
if separator:
|
||||||
pairs[key.strip()] = raw.strip()
|
pairs[key.strip()] = unquote_scalar(raw.strip())
|
||||||
mappings.append(pairs)
|
mappings.append(pairs)
|
||||||
return mappings
|
return mappings
|
||||||
|
|
||||||
|
|
||||||
def _split_top_level(body: str, opener: str, closer: str) -> list[str]:
|
def _split_top_level(body: str, opener: str, closer: str) -> list[str]:
|
||||||
"""Split on commas that are not inside a `{...}`."""
|
"""Split on commas that are not inside a `{...}` or a `"`-quoted scalar.
|
||||||
|
|
||||||
|
A `"` opens a quoted scalar only where a YAML value can START -- after
|
||||||
|
`:`, `,`, the opener, or at the beginning -- so a plain value carrying a
|
||||||
|
`"` in its middle is split exactly as it was before K3-22.
|
||||||
|
"""
|
||||||
parts: list[str] = []
|
parts: list[str] = []
|
||||||
depth = 0
|
depth = 0
|
||||||
current: list[str] = []
|
current: list[str] = []
|
||||||
|
quoted = escaped = False
|
||||||
|
previous = ""
|
||||||
for character in body:
|
for character in body:
|
||||||
|
current.append(character)
|
||||||
|
if quoted:
|
||||||
|
if escaped:
|
||||||
|
escaped = False
|
||||||
|
elif character == "\\":
|
||||||
|
escaped = True
|
||||||
|
elif character == '"':
|
||||||
|
quoted = False
|
||||||
|
previous = character
|
||||||
|
continue
|
||||||
|
if character == '"' and previous in ("", ":", ",", opener):
|
||||||
|
quoted = True
|
||||||
|
continue
|
||||||
if character == opener:
|
if character == opener:
|
||||||
depth += 1
|
depth += 1
|
||||||
elif character == closer:
|
elif character == closer:
|
||||||
depth -= 1
|
depth -= 1
|
||||||
if character == "," and depth == 0:
|
if character == "," and depth == 0:
|
||||||
|
current.pop()
|
||||||
parts.append("".join(current))
|
parts.append("".join(current))
|
||||||
current = []
|
current = []
|
||||||
continue
|
if not character.isspace():
|
||||||
current.append(character)
|
previous = character
|
||||||
parts.append("".join(current))
|
parts.append("".join(current))
|
||||||
return [part for part in parts if part.strip()]
|
return [part for part in parts if part.strip()]
|
||||||
|
|
||||||
|
|
@ -550,7 +574,7 @@ def read_sources(lines: Sequence[str]) -> tuple[tuple[Mapping[str, str], ...], b
|
||||||
key, separator, raw = item.partition(":")
|
key, separator, raw = item.partition(":")
|
||||||
if not separator:
|
if not separator:
|
||||||
return (), True
|
return (), True
|
||||||
entries[-1][key.strip()] = raw.strip()
|
entries[-1][key.strip()] = unquote_scalar(raw.strip())
|
||||||
if not entries:
|
if not entries:
|
||||||
return (), True
|
return (), True
|
||||||
return tuple(entries), True
|
return tuple(entries), True
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,16 @@ from .materialize import (
|
||||||
validate_ingested_at,
|
validate_ingested_at,
|
||||||
write_bytes,
|
write_bytes,
|
||||||
)
|
)
|
||||||
from .profiles import DEFAULT, BundleProfile, IndexEntry, ProvenancePolicy
|
from .profiles import (
|
||||||
|
DEFAULT,
|
||||||
|
BundleProfile,
|
||||||
|
IndexEntry,
|
||||||
|
ProvenancePolicy,
|
||||||
|
yaml_block_plain,
|
||||||
|
yaml_flow_collection,
|
||||||
|
yaml_flow_collection_plain,
|
||||||
|
yaml_flow_plain,
|
||||||
|
)
|
||||||
from .segmentation import (
|
from .segmentation import (
|
||||||
SegmentationPlan,
|
SegmentationPlan,
|
||||||
SegmentEntry,
|
SegmentEntry,
|
||||||
|
|
@ -213,7 +222,7 @@ def render_inbox_concept(
|
||||||
frontmatter[policy.offset_key] = _render_flow_list([str(offset) for offset in segment.span])
|
frontmatter[policy.offset_key] = _render_flow_list([str(offset) for offset in segment.span])
|
||||||
if segment.parent_id is not None:
|
if segment.parent_id is not None:
|
||||||
frontmatter["parent"] = segment.parent_id
|
frontmatter["parent"] = segment.parent_id
|
||||||
if segment.description is not None and _yaml_plain(segment.description):
|
if segment.description is not None and yaml_block_plain(segment.description):
|
||||||
# The SOURCE's words, carried by the plan, and written only where
|
# The SOURCE's words, carried by the plan, and written only where
|
||||||
# they read back verbatim. Absent is the source saying nothing, or
|
# they read back verbatim. Absent is the source saying nothing, or
|
||||||
# saying it in a form this line cannot carry -- never a summary
|
# saying it in a form this line cannot carry -- never a summary
|
||||||
|
|
@ -313,9 +322,12 @@ def validate_concept_frontmatter(
|
||||||
|
|
||||||
SPEC SS 4.1 lets a producer add any key and SS 11 forbids a consumer to
|
SPEC SS 4.1 lets a producer add any key and SS 11 forbids a consumer to
|
||||||
reject one, so the limits here are this package's own and each is a
|
reject one, so the limits here are this package's own and each is a
|
||||||
reader it has to survive: the value is written verbatim on ONE line
|
reader it has to survive: the value is written on ONE line because every
|
||||||
because every reader here is line-oriented, which rules out a line break
|
reader here is line-oriented, which rules out a line break and -- since
|
||||||
and -- since `parse_frontmatter` strips -- surrounding whitespace.
|
`parse_frontmatter` strips -- surrounding whitespace. A scalar goes out
|
||||||
|
plain where a YAML reader returns it verbatim and double-quoted otherwise;
|
||||||
|
a flow collection goes out as given, so every leaf in it must be one both a
|
||||||
|
YAML reader and the guard read back (K3-22).
|
||||||
"""
|
"""
|
||||||
written = _door_keys(profile)
|
written = _door_keys(profile)
|
||||||
for key, value in values.items():
|
for key, value in values.items():
|
||||||
|
|
@ -336,16 +348,17 @@ def validate_concept_frontmatter(
|
||||||
f"surrounding whitespace, got {value!r}",
|
f"surrounding whitespace, got {value!r}",
|
||||||
code="run_frontmatter_invalid",
|
code="run_frontmatter_invalid",
|
||||||
)
|
)
|
||||||
|
if yaml_flow_collection(value) and not yaml_flow_collection_plain(value):
|
||||||
|
raise MaterializationError(
|
||||||
|
f"frontmatter value for {key!r} is a flow collection a YAML reader and "
|
||||||
|
"the guard would not both read back as written: a leaf carrying `?`, a "
|
||||||
|
"quote, ': ', ' #', a trailing `:` or a leading indicator has no flow "
|
||||||
|
f"form both accept, got {value!r}",
|
||||||
|
code="run_frontmatter_invalid",
|
||||||
|
)
|
||||||
return dict(values)
|
return dict(values)
|
||||||
|
|
||||||
|
|
||||||
# The characters that would end a YAML flow mapping early, so a path carrying
|
|
||||||
# one would produce a `sources` list that parses as something other than what
|
|
||||||
# was written. The guard refuses a quoted scalar inside a flow mapping (1.3.0,
|
|
||||||
# measured), so escaping is not on the table -- validation is.
|
|
||||||
_FLOW_TERMINATORS = ",{}[]"
|
|
||||||
|
|
||||||
|
|
||||||
def _provenance_frontmatter(
|
def _provenance_frontmatter(
|
||||||
policy: ProvenancePolicy,
|
policy: ProvenancePolicy,
|
||||||
*,
|
*,
|
||||||
|
|
@ -360,22 +373,26 @@ def _provenance_frontmatter(
|
||||||
"which document", the locator answers "where in it", and a consumer is owed
|
"which document", the locator answers "where in it", and a consumer is owed
|
||||||
the first even when the second cannot be computed.
|
the first even when the second cannot be computed.
|
||||||
"""
|
"""
|
||||||
bad = [char for char in _FLOW_TERMINATORS if char in source_file]
|
# Validation, never quoting: plain is the one form of a flow-mapping leaf
|
||||||
if bad:
|
# that a YAML reader and the guard both read back verbatim -- the guard
|
||||||
|
# refuses any quote in a flow mapping (1.3.0, measured) -- so a value it
|
||||||
|
# cannot carry is refused rather than mangled (K3-22). The file name is
|
||||||
|
# checked too: it is the entry's `title` when the document declares none.
|
||||||
|
shown = title if title is not None else PurePosixPath(source_file).name
|
||||||
|
if not yaml_flow_plain(source_file) or (title is None and not yaml_flow_plain(shown)):
|
||||||
raise MaterializationError(
|
raise MaterializationError(
|
||||||
f"source_file {source_file!r} contains {bad[0]!r}, which would end the "
|
f"source_file {source_file!r} has no plain form in the `sources` flow "
|
||||||
"`sources` flow mapping early; this profile writes an address a "
|
"mapping that both a YAML reader and the guard read back verbatim; this "
|
||||||
"consumer can follow, and a path it cannot express is refused rather "
|
"profile writes an address a consumer can follow, and a path it cannot "
|
||||||
"than mangled",
|
"express is refused rather than mangled",
|
||||||
code="inbox_source_file_unaddressable",
|
code="inbox_source_file_unaddressable",
|
||||||
)
|
)
|
||||||
if title is not None and not _flow_expressible(title):
|
if title is not None and not yaml_flow_plain(title):
|
||||||
raise MaterializationError(
|
raise MaterializationError(
|
||||||
f"source title {title!r} cannot be written into the `sources` flow "
|
f"source title {title!r} cannot be written into the `sources` flow "
|
||||||
"mapping verbatim; refused rather than mangled",
|
"mapping verbatim; refused rather than mangled",
|
||||||
code="inbox_source_title_unaddressable",
|
code="inbox_source_title_unaddressable",
|
||||||
)
|
)
|
||||||
shown = title if title is not None else PurePosixPath(source_file).name
|
|
||||||
values = {policy.sources_key: f"[{{ resource: {source_file}, title: {shown} }}]"}
|
values = {policy.sources_key: f"[{{ resource: {source_file}, title: {shown} }}]"}
|
||||||
if units is None or span is None:
|
if units is None or span is None:
|
||||||
return values
|
return values
|
||||||
|
|
@ -395,35 +412,6 @@ def _provenance_frontmatter(
|
||||||
return values
|
return values
|
||||||
|
|
||||||
|
|
||||||
def _flow_expressible(value: str) -> bool:
|
|
||||||
"""Whether `value` survives as a plain scalar inside a flow mapping."""
|
|
||||||
return bool(value) and not any(char in value for char in f"{_FLOW_TERMINATORS}\n\r")
|
|
||||||
|
|
||||||
|
|
||||||
# What a YAML reader takes as syntax at the START of a plain scalar.
|
|
||||||
_YAML_INDICATORS = frozenset("-?:,[]{}#&*!|>'\"%@`")
|
|
||||||
|
|
||||||
|
|
||||||
def _yaml_plain(value: str) -> bool:
|
|
||||||
"""Whether `value` reads back verbatim as a plain scalar in a block mapping.
|
|
||||||
|
|
||||||
MEASURED ON R761: 217 of 2 024 first spec points carry `": "`, and PyYAML's
|
|
||||||
`safe_load` refused exactly those 217 concepts' frontmatter. Decided by
|
|
||||||
rule rather than by a parser, because this package's one runtime
|
|
||||||
dependency is the guard -- and over those 2 024 values the rule and PyYAML
|
|
||||||
agree on every one: 217 refused, 0 refused that PyYAML reads, 0 kept that
|
|
||||||
it does not.
|
|
||||||
"""
|
|
||||||
return (
|
|
||||||
bool(value)
|
|
||||||
and value[0] not in _YAML_INDICATORS
|
|
||||||
and ": " not in value
|
|
||||||
and " #" not in value
|
|
||||||
and not value.endswith(":")
|
|
||||||
and not any(char in value for char in "\t\n\r")
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _screened(gate: Gate, value: str | None) -> str | None:
|
def _screened(gate: Gate, value: str | None) -> str | None:
|
||||||
"""A value read from the DOCUMENT and persisted outside its screened body.
|
"""A value read from the DOCUMENT and persisted outside its screened body.
|
||||||
|
|
||||||
|
|
@ -462,7 +450,7 @@ def _declared_sources_title(identity: DeclaredIdentity | None, gate: Gate) -> st
|
||||||
candidates.append(identity.title)
|
candidates.append(identity.title)
|
||||||
for candidate in candidates:
|
for candidate in candidates:
|
||||||
kept = _screened(gate, candidate)
|
kept = _screened(gate, candidate)
|
||||||
if kept is not None and _flow_expressible(kept):
|
if kept is not None and yaml_flow_plain(kept):
|
||||||
return kept
|
return kept
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -29,7 +29,7 @@ from .manifest import (
|
||||||
generated_filename,
|
generated_filename,
|
||||||
load_manifest_bytes,
|
load_manifest_bytes,
|
||||||
)
|
)
|
||||||
from .profiles import DEFAULT, BundleProfile
|
from .profiles import DEFAULT, BundleProfile, unquote_scalar, yaml_flow_plain
|
||||||
from .render import render_fenced_block, render_table
|
from .render import render_fenced_block, render_table
|
||||||
|
|
||||||
_LOGGER = logging.getLogger(__name__)
|
_LOGGER = logging.getLogger(__name__)
|
||||||
|
|
@ -135,7 +135,10 @@ def parse_frontmatter(path: Path) -> dict[str, str]:
|
||||||
continue
|
continue
|
||||||
key, sep, value = line.partition(":")
|
key, sep, value = line.partition(":")
|
||||||
if sep:
|
if sep:
|
||||||
frontmatter[key.strip()] = value.strip()
|
# A `"`-wrapped value is how the emitter writes a scalar a YAML
|
||||||
|
# reader would refuse plain (K3-22); read back as that reader
|
||||||
|
# would. A `'`-wrapped one is returned as it stands.
|
||||||
|
frontmatter[key.strip()] = unquote_scalar(value.strip())
|
||||||
return frontmatter
|
return frontmatter
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -170,12 +173,6 @@ def _is_ingest_owned(path: Path, manifest_stem: str, *, profile: BundleProfile =
|
||||||
return reference.rsplit("@", 1)[0] == manifest_stem
|
return reference.rsplit("@", 1)[0] == manifest_stem
|
||||||
|
|
||||||
|
|
||||||
# The characters that terminate or restructure a YAML flow mapping. `:\s`
|
|
||||||
# catches a colon that would open a nested key; a colon inside `https://host`
|
|
||||||
# does not, and stays a plain scalar.
|
|
||||||
_FLOW_UNSAFE_RE = re.compile(r"[,\[\]{}]|:\s")
|
|
||||||
|
|
||||||
|
|
||||||
def _source_locator(source: Source) -> str:
|
def _source_locator(source: Source) -> str:
|
||||||
"""Where a manifest source points, per source type.
|
"""Where a manifest source points, per source type.
|
||||||
|
|
||||||
|
|
@ -238,11 +235,12 @@ def _render_sources(sources: Sequence[Source]) -> str:
|
||||||
for source in sources:
|
for source in sources:
|
||||||
locator = _source_locator(source)
|
locator = _source_locator(source)
|
||||||
for label, value in (("id", source.id), ("resource", locator)):
|
for label, value in (("id", source.id), ("resource", locator)):
|
||||||
if _FLOW_UNSAFE_RE.search(value):
|
if not yaml_flow_plain(value):
|
||||||
raise MaterializationError(
|
raise MaterializationError(
|
||||||
f"the source {label} {value!r} contains a character that would "
|
f"the source {label} {value!r} has no plain form in the `sources` "
|
||||||
"restructure the `sources` flow mapping (one of `,[]{}` or a "
|
"flow mapping that both a YAML reader and the guard read back "
|
||||||
"colon followed by whitespace) — refusing to emit a provenance "
|
"verbatim (`,[]{}`, `?`, a quote, ': ', ' #', a trailing `:` or a "
|
||||||
|
"leading YAML indicator) — refusing to emit a provenance "
|
||||||
"record that parses cleanly into something no one wrote",
|
"record that parses cleanly into something no one wrote",
|
||||||
code="source_reference_unquotable",
|
code="source_reference_unquotable",
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,159 @@ RESERVED_OKF_TYPE = "verdict"
|
||||||
_TIMESTAMP_FALLBACK_PAIR = frozenset({"timestamp", "generated"})
|
_TIMESTAMP_FALLBACK_PAIR = frozenset({"timestamp", "generated"})
|
||||||
|
|
||||||
|
|
||||||
|
# --- YAML scalar forms (K3-22) -----------------------------------------------
|
||||||
|
#
|
||||||
|
# SPEC SS 11 point 1 requires "a parseable YAML frontmatter block" and SS 4
|
||||||
|
# names no YAML version and no subset, so the reader a consumer has decides;
|
||||||
|
# PyYAML is the common one. The forms are decided by RULES here and never by a
|
||||||
|
# parser -- this package's one runtime dependency is the guard -- and the rules
|
||||||
|
# are validated against PyYAML in `tests/test_yaml_frontmatter.py`, both error
|
||||||
|
# directions counted.
|
||||||
|
|
||||||
|
# What a YAML reader takes as syntax at the START of a plain scalar.
|
||||||
|
_YAML_INDICATORS = frozenset("-?:,[]{}#&*!|>'\"%@`")
|
||||||
|
|
||||||
|
# What a plain scalar INSIDE a flow mapping may not carry, for one of the two
|
||||||
|
# readers a `sources` entry has to survive. `,[]{}` end it for both. `?` ends
|
||||||
|
# it for PyYAML, whose scanner stops a flow plain scalar there, so a URL with a
|
||||||
|
# query string fails `safe_load`. A quote anywhere is refused by the pinned
|
||||||
|
# guard, which admits no quoted leaf in a flow mapping (1.3.0, measured). So
|
||||||
|
# quoting cannot rescue a flow value: plain fails one reader, quoted the other.
|
||||||
|
_FLOW_UNSAFE = frozenset(",[]{}?'\"")
|
||||||
|
|
||||||
|
# A key inside a flow mapping, as the guard's `_KEY_RE` and our readers take it.
|
||||||
|
_FLOW_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$")
|
||||||
|
|
||||||
|
|
||||||
|
def yaml_block_plain(value: str) -> bool:
|
||||||
|
"""Whether `value` reads back verbatim as a plain scalar in a block mapping.
|
||||||
|
|
||||||
|
MEASURED ON R761 (K3-19): 217 of 2 024 first spec points carry `": "`, and
|
||||||
|
PyYAML's `safe_load` refused exactly those 217 concepts' frontmatter. K3-22
|
||||||
|
measured the same rule over every top-level value in eleven trees (the K2
|
||||||
|
default bundle, the R761 HTML and XML builds, a five-document project, this
|
||||||
|
repository's examples and fixtures, and five consumer bundles): 0 refused
|
||||||
|
that PyYAML returns verbatim, 0 kept that it does not. `" #"` is here
|
||||||
|
although PyYAML does not refuse it: `title: Kap #3` loads, as `Kap`.
|
||||||
|
"""
|
||||||
|
return (
|
||||||
|
bool(value)
|
||||||
|
and value[0] not in _YAML_INDICATORS
|
||||||
|
and ": " not in value
|
||||||
|
and " #" not in value
|
||||||
|
and not value.endswith(":")
|
||||||
|
and not any(char in value for char in "\t\n\r")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def yaml_flow_plain(value: str) -> bool:
|
||||||
|
"""Whether `value` reads back verbatim as a plain scalar inside a flow
|
||||||
|
mapping, for PyYAML AND the pinned guard.
|
||||||
|
|
||||||
|
The block rule plus `_FLOW_UNSAFE`, with one exception the block rule does
|
||||||
|
not make: a leading `-` followed by a non-space is an ordinary character in
|
||||||
|
a flow mapping for both readers (`del/-utkast.pdf`), and refusing it would
|
||||||
|
refuse an address both of them read. The block rule keeps refusing it
|
||||||
|
because it also decides which spec points are written at all (K3-19), and
|
||||||
|
in a block mapping a refusal only costs a pair of quotes.
|
||||||
|
"""
|
||||||
|
if not value or any(char in value for char in _FLOW_UNSAFE):
|
||||||
|
return False
|
||||||
|
rest = value[1:] if value[0] == "-" and value[1:2] not in ("", " ", "\t") else value
|
||||||
|
return yaml_block_plain(rest)
|
||||||
|
|
||||||
|
|
||||||
|
def yaml_flow_collection(value: str) -> bool:
|
||||||
|
"""Whether `value` is SHAPED as a flow collection (`[...]` or `{...}`).
|
||||||
|
|
||||||
|
Structure a producer built -- `source_offset: [0, 4]`, `generated: { by: x,
|
||||||
|
at: y }` -- is written as it stands; measured over the same eleven trees,
|
||||||
|
only structural keys carry this shape and no `title` does.
|
||||||
|
"""
|
||||||
|
return (value[:1], value[-1:]) in (("[", "]"), ("{", "}"))
|
||||||
|
|
||||||
|
|
||||||
|
def yaml_flow_collection_plain(value: str) -> bool:
|
||||||
|
"""Whether a flow collection parses, for PyYAML and the guard, into what
|
||||||
|
was written: one `{ key: leaf, ... }` mapping, or a `[...]` sequence of
|
||||||
|
such mappings or of scalars, every leaf `yaml_flow_plain`."""
|
||||||
|
if value[:1] == "{" and value[-1:] == "}":
|
||||||
|
return _flow_mapping_plain(value)
|
||||||
|
if not (value[:1] == "[" and value[-1:] == "]"):
|
||||||
|
return False
|
||||||
|
items: list[str] = []
|
||||||
|
depth = 0
|
||||||
|
current: list[str] = []
|
||||||
|
for char in value[1:-1]:
|
||||||
|
depth += {"{": 1, "}": -1}.get(char, 0)
|
||||||
|
if depth not in (0, 1):
|
||||||
|
return False
|
||||||
|
if char == "," and depth == 0:
|
||||||
|
items.append("".join(current).strip())
|
||||||
|
current = []
|
||||||
|
else:
|
||||||
|
current.append(char)
|
||||||
|
items.append("".join(current).strip())
|
||||||
|
if depth != 0:
|
||||||
|
return False
|
||||||
|
return all(
|
||||||
|
_flow_mapping_plain(item) if item[:1] == "{" else yaml_flow_plain(item) for item in items
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _flow_mapping_plain(item: str) -> bool:
|
||||||
|
if not (item[:1] == "{" and item[-1:] == "}"):
|
||||||
|
return False
|
||||||
|
inner = item[1:-1].strip()
|
||||||
|
if not inner:
|
||||||
|
return False
|
||||||
|
for entry in inner.split(","):
|
||||||
|
key, separator, leaf = entry.strip().partition(": ")
|
||||||
|
if not separator or not _FLOW_KEY.match(key) or not yaml_flow_plain(leaf.strip()):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def quote_scalar(value: str) -> str:
|
||||||
|
"""`value` as a double-quoted YAML scalar: `\\` and `"` escaped, nothing else.
|
||||||
|
|
||||||
|
Double and never single: over every bundle measured, 0 values carry a
|
||||||
|
surrounding `"` pair and 11 193 a surrounding `'` pair, so a reader that
|
||||||
|
unquotes `"` changes the meaning of no value already written.
|
||||||
|
"""
|
||||||
|
return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
||||||
|
|
||||||
|
|
||||||
|
def unquote_scalar(value: str) -> str:
|
||||||
|
"""A surrounding `"` pair removed and `\\"` / `\\\\` decoded; else `value`.
|
||||||
|
|
||||||
|
The two escapes `quote_scalar` writes are the two decoded. Any other
|
||||||
|
backslash sequence (`\\n`, `\\t`, `\\x..`, `\\u....`) is kept as written:
|
||||||
|
a YAML reader would decode it and this reader does not claim to. A
|
||||||
|
single-quoted value is returned as it stands -- `'1'` stays `'1'`.
|
||||||
|
"""
|
||||||
|
if len(value) < 2 or value[0] != '"' or value[-1] != '"':
|
||||||
|
return value
|
||||||
|
inner = value[1:-1]
|
||||||
|
out: list[str] = []
|
||||||
|
index = 0
|
||||||
|
while index < len(inner):
|
||||||
|
char = inner[index]
|
||||||
|
if char == "\\" and inner[index + 1 : index + 2] in ('"', "\\"):
|
||||||
|
out.append(inner[index + 1])
|
||||||
|
index += 2
|
||||||
|
continue
|
||||||
|
out.append(char)
|
||||||
|
index += 1
|
||||||
|
return "".join(out)
|
||||||
|
|
||||||
|
|
||||||
|
def block_scalar(value: str) -> str:
|
||||||
|
"""`value` as written after `key: ` in a block mapping: plain where a YAML
|
||||||
|
reader returns it verbatim, double-quoted otherwise."""
|
||||||
|
return value if yaml_block_plain(value) else quote_scalar(value)
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class TypeRejection:
|
class TypeRejection:
|
||||||
"""Why a profile refuses an `okf_type`, for the door to frame and raise.
|
"""Why a profile refuses an `okf_type`, for the door to frame and raise.
|
||||||
|
|
@ -258,11 +411,18 @@ class FrontmatterSchema:
|
||||||
than trusting insertion order is what makes a regeneration over the
|
than trusting insertion order is what makes a regeneration over the
|
||||||
same data byte-identical.
|
same data byte-identical.
|
||||||
|
|
||||||
|
A scalar a YAML reader would refuse or misread as plain is written
|
||||||
|
double-quoted (K3-22); every other value keeps its bytes. A flow
|
||||||
|
collection is written as it stands -- its leaves are validated where it
|
||||||
|
is built -- and so is an empty value.
|
||||||
|
|
||||||
Returns the lines only — the caller owns the `---` fences.
|
Returns the lines only — the caller owns the `---` fences.
|
||||||
"""
|
"""
|
||||||
named = [key for key in self.order if key in values]
|
named = [key for key in self.order if key in values]
|
||||||
tail = sorted(key for key in values if key not in self.order)
|
tail = sorted(key for key in values if key not in self.order)
|
||||||
return "\n".join(f"{key}: {self._render(key, values[key])}" for key in [*named, *tail])
|
return "\n".join(
|
||||||
|
f"{key}: {_emitted(self._render(key, values[key]))}" for key in [*named, *tail]
|
||||||
|
)
|
||||||
|
|
||||||
def _render(self, key: str, value: str) -> str:
|
def _render(self, key: str, value: str) -> str:
|
||||||
# §5 mandates whitespace-run collapse for `source_query` only
|
# §5 mandates whitespace-run collapse for `source_query` only
|
||||||
|
|
@ -274,6 +434,12 @@ class FrontmatterSchema:
|
||||||
return " ".join(value.split()) if key in self.collapsed_keys else value
|
return " ".join(value.split()) if key in self.collapsed_keys else value
|
||||||
|
|
||||||
|
|
||||||
|
def _emitted(value: str) -> str:
|
||||||
|
if not value or yaml_flow_collection(value):
|
||||||
|
return value
|
||||||
|
return block_scalar(value)
|
||||||
|
|
||||||
|
|
||||||
# The v0.1 ingest stamp. A literal rather than a configurable value: it is what
|
# The v0.1 ingest stamp. A literal rather than a configurable value: it is what
|
||||||
# every bundle this library has already written carries, and recognising it is
|
# every bundle this library has already written carries, and recognising it is
|
||||||
# what keeps those bundles re-runnable under a later profile.
|
# what keeps those bundles re-runnable under a later profile.
|
||||||
|
|
@ -380,7 +546,7 @@ def _split_frontmatter(text: str) -> tuple[dict[str, str], list[str]]:
|
||||||
continue
|
continue
|
||||||
key, sep, value = line.partition(":")
|
key, sep, value = line.partition(":")
|
||||||
if sep:
|
if sep:
|
||||||
head[key.strip()] = value.strip()
|
head[key.strip()] = unquote_scalar(value.strip())
|
||||||
return head, []
|
return head, []
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -57,7 +57,7 @@ from collections import Counter
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from . import consume as okf_consume
|
from . import consume as okf_consume
|
||||||
from .profiles import BundleProfile
|
from .profiles import BundleProfile, block_scalar
|
||||||
|
|
||||||
#: The template, resolved to a copy that exists wherever this module does.
|
#: The template, resolved to a copy that exists wherever this module does.
|
||||||
#:
|
#:
|
||||||
|
|
@ -389,7 +389,11 @@ def render(
|
||||||
bookkeeping=bookkeeping,
|
bookkeeping=bookkeeping,
|
||||||
breaking=breaking,
|
breaking=breaking,
|
||||||
)
|
)
|
||||||
header = f"---\nname: {name}\ndescription: {_description(bundle_id, total, ref)}\n---\n"
|
# Claude Code reads this header with a YAML reader, and `description`
|
||||||
|
# carries the root index's `bundle_id` raw -- a bundle this library did not
|
||||||
|
# build may call itself anything (K3-22).
|
||||||
|
description = block_scalar(_description(bundle_id, total, ref))
|
||||||
|
header = f"---\nname: {block_scalar(name)}\ndescription: {description}\n---\n"
|
||||||
return header + text, payload
|
return header + text, payload
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -35,6 +35,7 @@ from dataclasses import dataclass, field
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from .extract import strip_converter_attribute
|
from .extract import strip_converter_attribute
|
||||||
|
from .profiles import unquote_scalar
|
||||||
|
|
||||||
# A document number is either an alpha-prefixed identifier (`N500`, `V720`,
|
# A document number is either an alpha-prefixed identifier (`N500`, `V720`,
|
||||||
# `R610.4`) or a dotted numeric section (`4.2.1`). A BARE integer is
|
# `R610.4`) or a dotted numeric section (`4.2.1`). A BARE integer is
|
||||||
|
|
@ -100,8 +101,12 @@ DERIVABLE_FIELDS = frozenset({"title", "number", "parent", "references"})
|
||||||
def _unquote(value: str) -> str:
|
def _unquote(value: str) -> str:
|
||||||
# A producer quotes a scalar to keep YAML from retyping it (`version:
|
# A producer quotes a scalar to keep YAML from retyping it (`version:
|
||||||
# '2021'` is a string, not an integer). The quotes are the encoding, not
|
# '2021'` is a string, not an integer). The quotes are the encoding, not
|
||||||
# the value, and carrying them through would put them in the index.
|
# the value, and carrying them through would put them in the index. A
|
||||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
|
# `"`-wrapped value is decoded the way the emitter wrote it (K3-22); a
|
||||||
|
# `'`-wrapped one keeps this module's older rule, unchanged.
|
||||||
|
if len(value) >= 2 and value[0] == value[-1] == '"':
|
||||||
|
return unquote_scalar(value)
|
||||||
|
if len(value) >= 2 and value[0] == value[-1] == "'":
|
||||||
return value[1:-1]
|
return value[1:-1]
|
||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,12 @@ THE FORM IS NOT A STYLE CHOICE. The value is written verbatim on ONE line,
|
||||||
because this package's own readers are line-oriented and measured blind to a
|
because this package's own readers are line-oriented and measured blind to a
|
||||||
block-form `sources`. The flag splits on the FIRST `=` and only there: the
|
block-form `sources`. The flag splits on the FIRST `=` and only there: the
|
||||||
value a publisher's address needs carries `=` itself.
|
value a publisher's address needs carries `=` itself.
|
||||||
|
|
||||||
|
K3-22 moved one thing here, and it is a refusal: a flow value is written as
|
||||||
|
given, so every leaf in it must be one a YAML reader and the guard both read
|
||||||
|
back. The publisher's address with `?languageCode=nb` is not -- PyYAML refused
|
||||||
|
all 2 761 frontmatters K3-19's flagged build wrote with it -- and the build
|
||||||
|
tests below now write an address without `?`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
@ -43,6 +49,12 @@ OPAQUE = "0f1e2d3c-4b5a-6978-8796-a5b4c3d2e1f0.xml"
|
||||||
|
|
||||||
ADDRESS = "https://normer.example.test/api/sts/900?languageCode=nb"
|
ADDRESS = "https://normer.example.test/api/sts/900?languageCode=nb"
|
||||||
STATED_SOURCES = f"[{{ resource: {ADDRESS}, title: R900:2024 }}]"
|
STATED_SOURCES = f"[{{ resource: {ADDRESS}, title: R900:2024 }}]"
|
||||||
|
# K3-22: the address above carries `?`, which ends a plain scalar inside a
|
||||||
|
# PyYAML flow mapping, and the guard refuses the quoted form -- so no build may
|
||||||
|
# write it. The flag's GRAMMAR still takes it (it only splits); the build
|
||||||
|
# refuses it. A build that writes an address uses this one.
|
||||||
|
WRITABLE_ADDRESS = "https://normer.example.test/api/sts/900/nb"
|
||||||
|
WRITABLE_SOURCES = f"[{{ resource: {WRITABLE_ADDRESS}, title: R900:2024 }}]"
|
||||||
MARKDOWN = b"# Innledning\n\nTekst her.\n\n# Omfang\n\nMer tekst her.\n"
|
MARKDOWN = b"# Innledning\n\nTekst her.\n\n# Omfang\n\nMer tekst her.\n"
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -115,7 +127,7 @@ def test_every_concept_carries_the_stated_keys_once(tmp_path: Path) -> None:
|
||||||
"--frontmatter",
|
"--frontmatter",
|
||||||
"utgave=R900:2024",
|
"utgave=R900:2024",
|
||||||
"--frontmatter",
|
"--frontmatter",
|
||||||
f"sources={STATED_SOURCES}",
|
f"sources={WRITABLE_SOURCES}",
|
||||||
)
|
)
|
||||||
== 0
|
== 0
|
||||||
)
|
)
|
||||||
|
|
@ -130,7 +142,7 @@ def test_every_concept_carries_the_stated_keys_once(tmp_path: Path) -> None:
|
||||||
# the door would derive -- the document's own title on the STS file,
|
# the door would derive -- the document's own title on the STS file,
|
||||||
# the file name on the markdown one -- and never adds a second.
|
# the file name on the markdown one -- and never adds a second.
|
||||||
assert [line for line in lines if line.startswith("sources:")] == [
|
assert [line for line in lines if line.startswith("sources:")] == [
|
||||||
f"sources: {STATED_SOURCES}"
|
f"sources: {WRITABLE_SOURCES}"
|
||||||
], path
|
], path
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -138,11 +150,11 @@ def test_the_stated_address_reads_back_through_this_packages_own_readers(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
) -> None:
|
) -> None:
|
||||||
bundle = tmp_path / "bundle"
|
bundle = tmp_path / "bundle"
|
||||||
assert _main(_inbox(tmp_path), bundle, "--frontmatter", f"sources={STATED_SOURCES}") == 0
|
assert _main(_inbox(tmp_path), bundle, "--frontmatter", f"sources={WRITABLE_SOURCES}") == 0
|
||||||
for path in _concept_files(bundle):
|
for path in _concept_files(bundle):
|
||||||
assert parse_frontmatter(path)["sources"] == STATED_SOURCES
|
assert parse_frontmatter(path)["sources"] == WRITABLE_SOURCES
|
||||||
assert read_sources(_frontmatter_lines(path)) == (
|
assert read_sources(_frontmatter_lines(path)) == (
|
||||||
({"resource": ADDRESS, "title": "R900:2024"},),
|
({"resource": WRITABLE_ADDRESS, "title": "R900:2024"},),
|
||||||
True,
|
True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -227,6 +239,18 @@ def test_a_pair_that_would_not_read_back_verbatim_is_refused(tmp_path: Path, pai
|
||||||
assert not bundle.exists()
|
assert not bundle.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_stated_address_no_yaml_reader_reads_back_is_refused(
|
||||||
|
tmp_path: Path, capsys: pytest.CaptureFixture[str]
|
||||||
|
) -> None:
|
||||||
|
"""K3-19's own flagged build wrote this form, and 2 761 of 2 761 of its
|
||||||
|
frontmatters then failed `yaml.safe_load`. It is refused, never written
|
||||||
|
(K3-22): `?` ends a PyYAML flow scalar, and the guard refuses the quote."""
|
||||||
|
bundle = tmp_path / "bundle"
|
||||||
|
assert _main(_inbox(tmp_path), bundle, "--frontmatter", f"sources={STATED_SOURCES}") == 2
|
||||||
|
assert "sources" in capsys.readouterr().err
|
||||||
|
assert not bundle.exists()
|
||||||
|
|
||||||
|
|
||||||
def test_a_value_spanning_lines_is_refused_through_the_api(tmp_path: Path) -> None:
|
def test_a_value_spanning_lines_is_refused_through_the_api(tmp_path: Path) -> None:
|
||||||
with pytest.raises(IngestError) as caught:
|
with pytest.raises(IngestError) as caught:
|
||||||
cli.build(
|
cli.build(
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue