feat(m1): add append-only jsonl writer with folding reader

scripts/jobbsok_lib/jsonl.py

@@ -0,0 +1,207 @@
+"""The append-only store for logg.jsonl, beslutninger.jsonl and varianter.jsonl.
+
+Append-only is the whole design, and it has a consequence people usually
+discover too late: the schema is irreversible. There is no migration that can
+go back and add a field to lines already written, so `type` and `skjema` are
+on the very first line this module ever writes (risk H10). A reader that meets
+a line without them is meeting a file this module did not produce.
+
+Two writers, one file (risk H4). The CLI and the host MCP server can both be
+appending while the operator watches, so every record goes out as a single
+``os.write`` on a descriptor opened ``O_APPEND``. That is what makes the
+append indivisible; building the line first and writing it in one call is not
+a style choice here, it is the guarantee. Anything that split the record into
+two writes -- a buffered file object, a separate newline write -- would let a
+second writer land between them.
+
+Torn writes versus corruption (risk M5r). A process killed mid-append leaves a
+partial last line with no newline after it. That is recoverable and expected,
+so :func:`read_lines` skips it. A malformed line anywhere else is corruption,
+not a torn write, and it raises with its line number rather than being
+silently dropped -- the difference between "the last append did not finish"
+and "a record in the middle of your decision log is gone" is the difference
+this module refuses to blur.
+
+Timestamps carry an explicit UTC offset, always. Norwegian local time repeats
+02:30 on the last Sunday of October, so two records an hour apart share a wall
+clock and are told apart only by the offset. A naive timestamp is rejected on
+write, because by read time there is no way to recover which hour was meant.
+"""
+
+import datetime
+import json
+import os
+
+from . import paths
+
+#: The closed set of record kinds. `beslutning` is a sourcing decision,
+#: `korrigering` restates one that was wrong, `utfall` records how a case
+#: ended. Anything else is a caller bug, not a new kind.
+TYPES = ("beslutning", "korrigering", "utfall")
+
+#: Schema version stamped on every line. Bumping this is a fork of the format,
+#: never an in-place migration -- the file behind it cannot be rewritten.
+SKJEMA = 1
+
+#: Keys whose values are timestamps, wherever they appear. `ts` is the case
+#: log (build-brief 5.3), `dato` the decision log (5.4).
+TIMESTAMP_KEYS = ("ts", "dato")
+
+#: Fields belonging to the correction record itself, not to the decision it
+#: corrects. Folding must not smear these onto the target.
+_CORRECTION_OWN_FIELDS = ("type", "id", "korrigerer", "skjema")
+
+
+class JsonlError(Exception):
+    """A refusal from this module, with the line number when there is one."""
+
+    def __init__(self, message, line=None):
+        self.line = line
+        super().__init__("line %d: %s" % (line, message) if line else message)
+
+
+def append_line(path, obj, root=None):
+    """Append ``obj`` as exactly one line, in a single ``O_APPEND`` write.
+
+    Validation happens before the file is opened, so a rejected record leaves
+    no trace at all -- not an empty file, not a partial line.
+
+    ``root``, when given, resolves ``path`` under it with
+    :func:`paths.safe_join`, so a caller-supplied path that escapes the
+    workspace is refused here rather than in whichever script remembered to
+    check.
+    """
+    record = _validated(obj)
+    target = paths.safe_join(root, path) if root is not None else path
+    payload = (json.dumps(record, ensure_ascii=False) + "\n").encode("utf-8")
+
+    handle = os.open(target, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600)
+    try:
+        written = os.write(handle, payload)
+    finally:
+        os.close(handle)
+    if written != len(payload):
+        # Retrying the remainder would append a fragment behind whatever
+        # another writer put there in the meantime. Fail instead.
+        raise JsonlError(
+            "short write: %d of %d bytes reached %r" % (written, len(payload), target)
+        )
+    return record
+
+
+def read_lines(path):
+    """Read every record, skipping only a torn trailing line.
+
+    A missing file reads as no records -- an append-only log that has never
+    been appended to is empty, not broken.
+    """
+    if not os.path.exists(path):
+        return []
+    with open(path, "r", encoding="utf-8") as handle:
+        data = handle.read()
+    if data == "":
+        return []
+
+    lines = data.split("\n")
+    # A file ending in a newline leaves an empty final element; a file that
+    # does not was cut off mid-append, and only THAT last line may be skipped.
+    if lines and lines[-1] == "":
+        lines.pop()
+        torn_index = None
+    else:
+        torn_index = len(lines)
+
+    records = []
+    for index, line in enumerate(lines, start=1):
+        if line.strip() == "":
+            continue
+        try:
+            record = json.loads(line)
+        except ValueError as exc:
+            if index == torn_index:
+                continue
+            raise JsonlError("not valid JSON: %s" % exc, line=index)
+        if not isinstance(record, dict):
+            if index == torn_index:
+                continue
+            raise JsonlError("expected an object, got %s" % type(record).__name__, line=index)
+        records.append(record)
+    return records
+
+
+def fold(lines):
+    """Return the decision stream with corrections applied and outcomes removed.
+
+    A `korrigering` overwrites the fields it names on the `beslutning` it
+    points at, and contributes no row of its own -- the log keeps both, the
+    stream shows the corrected decision. An `utfall` is not a decision and is
+    excluded; build-brief 5.7 reads outcomes on its own terms.
+
+    The input is never mutated: the caller's records came off disk and stay as
+    they were read.
+    """
+    stream = []
+    by_id = {}
+    for record in lines:
+        kind = record.get("type")
+        if kind == "beslutning":
+            copy = dict(record)
+            stream.append(copy)
+            if "id" in copy:
+                by_id[copy["id"]] = copy
+        elif kind == "korrigering":
+            target_id = record.get("korrigerer")
+            if target_id not in by_id:
+                raise JsonlError(
+                    "correction %r names %r, which is not a decision above it"
+                    % (record.get("id"), target_id)
+                )
+            by_id[target_id].update(
+                {k: v for k, v in record.items() if k not in _CORRECTION_OWN_FIELDS}
+            )
+        elif kind == "utfall":
+            continue
+        else:
+            raise JsonlError("unknown record type %r" % (kind,))
+    return stream
+
+
+def instant(record):
+    """The record's timestamp as an aware :class:`datetime.datetime`."""
+    for key in TIMESTAMP_KEYS:
+        if key in record:
+            return _aware(record[key], key)
+    raise JsonlError("record has no timestamp; expected one of %r" % (TIMESTAMP_KEYS,))
+
+
+def by_time(lines):
+    """Sort records by the instant they name, not by the string that spells it."""
+    return sorted(lines, key=instant)
+
+
+def _validated(obj):
+    if not isinstance(obj, dict):
+        raise JsonlError("a record must be an object, got %s" % type(obj).__name__)
+    kind = obj.get("type")
+    if kind not in TYPES:
+        raise JsonlError("type %r is not one of %r" % (kind, TYPES))
+
+    record = dict(obj)
+    record.setdefault("skjema", SKJEMA)
+    for key in TIMESTAMP_KEYS:
+        if key in record:
+            _aware(record[key], key)
+    return record
+
+
+def _aware(value, key):
+    try:
+        moment = datetime.datetime.fromisoformat(value)
+    except (TypeError, ValueError):
+        raise JsonlError("%s=%r is not an ISO-8601 timestamp" % (key, value))
+    if moment.tzinfo is None or moment.utcoffset() is None:
+        raise JsonlError(
+            "%s=%r has no UTC offset; a naive timestamp cannot be ordered "
+            "across the October hour that happens twice" % (key, value)
+        )
+    return moment