feat(m1): add strict frontmatter parser with documented grammar
This commit is contained in:
parent
8328d487a9
commit
a0874a5e17
2 changed files with 413 additions and 0 deletions
286
scripts/jobbsok_lib/frontmatter.py
Normal file
286
scripts/jobbsok_lib/frontmatter.py
Normal file
|
|
@ -0,0 +1,286 @@
|
|||
"""A strict, dependency-free frontmatter reader for kandidat.md and sak.md.
|
||||
|
||||
No YAML runtime is allowed here (risk H7), so this module hand-rolls a parser
|
||||
that sits directly on the hard-filter path: `lonn.gulv_nok` and
|
||||
`geografi.maks_reisetid_min` are read from here and compared numerically by
|
||||
scoring. A hand-rolled parser on that path is only safe if it is loud, so the
|
||||
one rule this module never breaks is that anything outside the grammar below
|
||||
raises :class:`FrontmatterError` carrying the offending line number. It never
|
||||
guesses, never applies a default, and never skips a line it did not understand.
|
||||
|
||||
Grammar
|
||||
-------
|
||||
|
||||
The whole accepted grammar, and nothing beyond it::
|
||||
|
||||
document := [ "---" NL entries "---" NL ] body
|
||||
entry := comment | blank | mapping | block
|
||||
comment := "#" ... -- only when "#" is the FIRST column
|
||||
mapping := KEY ":" SP scalar
|
||||
block := KEY ":" NL ( child | item )+
|
||||
child := " " KEY ":" SP scalar -- one level of nesting, no more
|
||||
item := " - " scalar -- a flat list, scalars only
|
||||
KEY := [A-Za-z_][A-Za-z0-9_]*
|
||||
scalar := integer | '"' text '"' | "'" text "'" | unquoted-text
|
||||
|
||||
Indentation is exactly zero or two spaces; a tab in the indentation is an
|
||||
error rather than a width the reader has to assume. A `#` anywhere but the
|
||||
first column is data -- `notat: rolle nr # 3` keeps its hash, because trimming
|
||||
it would silently rewrite what the operator wrote. Inside double quotes only
|
||||
``\\"`` and ``\\\\`` are escapes; single quotes take the text verbatim.
|
||||
A duplicate key is refused instead of resolving last-wins: two salary floors
|
||||
in one profile must stop the run, not pick one.
|
||||
|
||||
A document with no opening fence has no frontmatter -- that is an empty
|
||||
mapping and an untouched body, not an error. Rejecting a kandidat.md that
|
||||
lacks frontmatter is the section 5.1 contract check's job, not the reader's.
|
||||
|
||||
Preservation
|
||||
------------
|
||||
|
||||
:func:`render` writes values back exactly as they came in: no Unicode
|
||||
normalisation, no escaping of non-ASCII, no reordering. `paths.slug`
|
||||
normalises deliberately; this module must not, because the file on disk is
|
||||
the operator's own text and a round trip through here has to return the same
|
||||
bytes.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from . import paths
|
||||
|
||||
#: Keys are conservative identifiers: the frontmatter is a machine contract,
|
||||
#: not free text, and a key with a space in it is a typo worth stopping for.
|
||||
KEY_RE = re.compile(r"^([A-Za-z_][A-Za-z0-9_]*):(.*)$")
|
||||
|
||||
INTEGER_RE = re.compile(r"^-?\d+$")
|
||||
|
||||
FENCE = "---"
|
||||
|
||||
INDENT = " "
|
||||
|
||||
|
||||
class FrontmatterError(Exception):
|
||||
"""A line outside the grammar, reported with the line it was on.
|
||||
|
||||
``line`` is 1-based and counts from the start of the document, so it
|
||||
matches what an editor shows.
|
||||
"""
|
||||
|
||||
def __init__(self, line, message):
|
||||
self.line = line
|
||||
super().__init__("line %d: %s" % (line, message))
|
||||
|
||||
|
||||
def parse(text):
|
||||
"""Parse ``text`` into ``(metadata, body)``.
|
||||
|
||||
``metadata`` preserves key order. ``body`` is everything after the closing
|
||||
fence, byte for byte. A document without an opening fence returns
|
||||
``({}, text)``.
|
||||
"""
|
||||
if not text.startswith(FENCE + "\n") and text.rstrip("\n") != FENCE:
|
||||
return {}, text
|
||||
|
||||
lines = text.split("\n")
|
||||
closing = _find_closing_fence(lines)
|
||||
body = "\n".join(lines[closing + 1:])
|
||||
|
||||
return _parse_entries(lines[1:closing], first_lineno=2), body
|
||||
|
||||
|
||||
def render(metadata, body=""):
|
||||
"""Render ``metadata`` and ``body`` back into a document.
|
||||
|
||||
The inverse of :func:`parse` for any document written in the canonical
|
||||
form this function emits. Values are written as they are held: Norwegian
|
||||
characters go out as themselves, never escaped and never normalised.
|
||||
"""
|
||||
out = [FENCE]
|
||||
for key, value in metadata.items():
|
||||
if isinstance(value, dict):
|
||||
out.append("%s:" % key)
|
||||
out.extend("%s%s: %s" % (INDENT, k, _scalar_out(v)) for k, v in value.items())
|
||||
elif isinstance(value, list):
|
||||
out.append("%s:" % key)
|
||||
out.extend("%s- %s" % (INDENT, _scalar_out(item)) for item in value)
|
||||
else:
|
||||
out.append("%s: %s" % (key, _scalar_out(value)))
|
||||
out.append(FENCE)
|
||||
return "\n".join(out) + "\n" + body
|
||||
|
||||
|
||||
def read(root, *parts):
|
||||
"""Read and parse a file resolved under ``root`` via :func:`paths.safe_join`.
|
||||
|
||||
Going through ``safe_join`` rather than ``os.path.join`` means a caller
|
||||
that hands in a traversing path gets a refusal here too, not only in the
|
||||
scripts that remembered to check.
|
||||
"""
|
||||
target = paths.safe_join(root, *parts)
|
||||
with open(target, "r", encoding="utf-8") as handle:
|
||||
return parse(handle.read())
|
||||
|
||||
|
||||
def _find_closing_fence(lines):
|
||||
for index in range(1, len(lines)):
|
||||
if lines[index] == FENCE:
|
||||
return index
|
||||
raise FrontmatterError(
|
||||
1, "frontmatter fence opened here was never closed with %r" % FENCE
|
||||
)
|
||||
|
||||
|
||||
def _parse_entries(lines, first_lineno):
|
||||
metadata = {}
|
||||
open_key = None
|
||||
open_lineno = None
|
||||
block = None # None until the first child decides mapping or list
|
||||
|
||||
for offset, line in enumerate(lines):
|
||||
lineno = first_lineno + offset
|
||||
|
||||
if line.strip() == "":
|
||||
continue
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
|
||||
indent = _indent_width(line, lineno)
|
||||
content = line[indent:]
|
||||
|
||||
if indent == 0:
|
||||
if open_key is not None and block is None:
|
||||
raise _empty_block(open_key, open_lineno)
|
||||
open_key, open_lineno, block = _top_level(metadata, content, lineno)
|
||||
else:
|
||||
if open_key is None:
|
||||
raise FrontmatterError(
|
||||
lineno, "indented line has no parent key above it"
|
||||
)
|
||||
block = _child(metadata, open_key, block, content, lineno)
|
||||
|
||||
if open_key is not None and block is None:
|
||||
raise _empty_block(open_key, open_lineno)
|
||||
return metadata
|
||||
|
||||
|
||||
def _top_level(metadata, content, lineno):
|
||||
if content.startswith("- "):
|
||||
raise FrontmatterError(
|
||||
lineno, "list item at column 0 has no key to belong to"
|
||||
)
|
||||
match = KEY_RE.match(content)
|
||||
if match is None:
|
||||
raise FrontmatterError(
|
||||
lineno, "expected %r, got %r" % ("key: value", content)
|
||||
)
|
||||
key, rest = match.group(1), match.group(2)
|
||||
if key in metadata:
|
||||
raise FrontmatterError(lineno, "duplicate key %r" % key)
|
||||
|
||||
if rest.strip() == "":
|
||||
return key, lineno, None
|
||||
metadata[key] = _scalar_in(rest.strip(), lineno)
|
||||
return None, None, None
|
||||
|
||||
|
||||
def _child(metadata, open_key, block, content, lineno):
|
||||
if content.startswith("- "):
|
||||
if block == "mapping":
|
||||
raise FrontmatterError(
|
||||
lineno, "list item under %r, which already holds a mapping" % open_key
|
||||
)
|
||||
metadata.setdefault(open_key, [])
|
||||
metadata[open_key].append(_scalar_in(content[2:].strip(), lineno))
|
||||
return "list"
|
||||
|
||||
match = KEY_RE.match(content)
|
||||
if match is None:
|
||||
raise FrontmatterError(
|
||||
lineno, "expected %r or %r, got %r" % (" key: value", " - item", content)
|
||||
)
|
||||
if block == "list":
|
||||
raise FrontmatterError(
|
||||
lineno, "mapping entry under %r, which already holds a list" % open_key
|
||||
)
|
||||
key, rest = match.group(1), match.group(2)
|
||||
if rest.strip() == "":
|
||||
raise FrontmatterError(
|
||||
lineno, "%r nests deeper than one level, which the grammar does not allow" % key
|
||||
)
|
||||
nested = metadata.setdefault(open_key, {})
|
||||
if key in nested:
|
||||
raise FrontmatterError(lineno, "duplicate key %r under %r" % (key, open_key))
|
||||
nested[key] = _scalar_in(rest.strip(), lineno)
|
||||
return "mapping"
|
||||
|
||||
|
||||
def _indent_width(line, lineno):
|
||||
leading = line[: len(line) - len(line.lstrip(" \t"))]
|
||||
if "\t" in leading:
|
||||
raise FrontmatterError(
|
||||
lineno, "tab in the indentation; use exactly two spaces per level"
|
||||
)
|
||||
if len(leading) not in (0, 2):
|
||||
raise FrontmatterError(
|
||||
lineno,
|
||||
"indentation is %d spaces; the grammar allows 0 or 2" % len(leading),
|
||||
)
|
||||
return len(leading)
|
||||
|
||||
|
||||
def _empty_block(key, lineno):
|
||||
return FrontmatterError(
|
||||
lineno, "%r has no value and no indented block beneath it" % key
|
||||
)
|
||||
|
||||
|
||||
def _scalar_in(raw, lineno):
|
||||
if raw.startswith('"'):
|
||||
if len(raw) < 2 or not raw.endswith('"'):
|
||||
raise FrontmatterError(lineno, "unterminated double-quoted value")
|
||||
return _unescape(raw[1:-1])
|
||||
if raw.startswith("'"):
|
||||
if len(raw) < 2 or not raw.endswith("'"):
|
||||
raise FrontmatterError(lineno, "unterminated single-quoted value")
|
||||
return raw[1:-1]
|
||||
if INTEGER_RE.match(raw):
|
||||
return int(raw)
|
||||
return raw
|
||||
|
||||
|
||||
def _unescape(text):
|
||||
out = []
|
||||
index = 0
|
||||
while index < len(text):
|
||||
char = text[index]
|
||||
if char == "\\" and index + 1 < len(text) and text[index + 1] in '"\\':
|
||||
out.append(text[index + 1])
|
||||
index += 2
|
||||
continue
|
||||
out.append(char)
|
||||
index += 1
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _scalar_out(value):
|
||||
if isinstance(value, bool):
|
||||
# Booleans are not in the grammar; letting one through here would
|
||||
# write a value the reader cannot read back.
|
||||
raise ValueError("the grammar has no boolean; write the word you mean")
|
||||
if isinstance(value, int):
|
||||
return str(value)
|
||||
text = str(value)
|
||||
if _needs_quoting(text):
|
||||
return '"%s"' % text.replace("\\", "\\\\").replace('"', '\\"')
|
||||
return text
|
||||
|
||||
|
||||
def _needs_quoting(text):
|
||||
if text == "" or text != text.strip():
|
||||
return True
|
||||
if text[0] in "#\"'":
|
||||
return True
|
||||
# An unquoted digit string would come back as an int and stop being the
|
||||
# string it was written as.
|
||||
return bool(INTEGER_RE.match(text))
|
||||
Loading…
Add table
Add a link
Reference in a new issue