207 lines
7.8 KiB
Python
207 lines
7.8 KiB
Python
"""The append-only store for logg.jsonl, beslutninger.jsonl and varianter.jsonl.
|
|
|
|
Append-only is the whole design, and it has a consequence people usually
|
|
discover too late: the schema is irreversible. There is no migration that can
|
|
go back and add a field to lines already written, so `type` and `skjema` are
|
|
on the very first line this module ever writes (risk H10). A reader that meets
|
|
a line without them is meeting a file this module did not produce.
|
|
|
|
Two writers, one file (risk H4). The CLI and the host MCP server can both be
|
|
appending while the operator watches, so every record goes out as a single
|
|
``os.write`` on a descriptor opened ``O_APPEND``. That is what makes the
|
|
append indivisible; building the line first and writing it in one call is not
|
|
a style choice here, it is the guarantee. Anything that split the record into
|
|
two writes -- a buffered file object, a separate newline write -- would let a
|
|
second writer land between them.
|
|
|
|
Torn writes versus corruption (risk M5r). A process killed mid-append leaves a
|
|
partial last line with no newline after it. That is recoverable and expected,
|
|
so :func:`read_lines` skips it. A malformed line anywhere else is corruption,
|
|
not a torn write, and it raises with its line number rather than being
|
|
silently dropped -- the difference between "the last append did not finish"
|
|
and "a record in the middle of your decision log is gone" is the difference
|
|
this module refuses to blur.
|
|
|
|
Timestamps carry an explicit UTC offset, always. Norwegian local time repeats
|
|
02:30 on the last Sunday of October, so two records an hour apart share a wall
|
|
clock and are told apart only by the offset. A naive timestamp is rejected on
|
|
write, because by read time there is no way to recover which hour was meant.
|
|
"""
|
|
|
|
import datetime
|
|
import json
|
|
import os
|
|
|
|
from . import paths
|
|
|
|
#: The closed set of record kinds. `beslutning` is a sourcing decision,
|
|
#: `korrigering` restates one that was wrong, `utfall` records how a case
|
|
#: ended. Anything else is a caller bug, not a new kind.
|
|
TYPES = ("beslutning", "korrigering", "utfall")
|
|
|
|
#: Schema version stamped on every line. Bumping this is a fork of the format,
|
|
#: never an in-place migration -- the file behind it cannot be rewritten.
|
|
SKJEMA = 1
|
|
|
|
#: Keys whose values are timestamps, wherever they appear. `ts` is the case
|
|
#: log (build-brief 5.3), `dato` the decision log (5.4).
|
|
TIMESTAMP_KEYS = ("ts", "dato")
|
|
|
|
#: Fields belonging to the correction record itself, not to the decision it
|
|
#: corrects. Folding must not smear these onto the target.
|
|
_CORRECTION_OWN_FIELDS = ("type", "id", "korrigerer", "skjema")
|
|
|
|
|
|
class JsonlError(Exception):
|
|
"""A refusal from this module, with the line number when there is one."""
|
|
|
|
def __init__(self, message, line=None):
|
|
self.line = line
|
|
super().__init__("line %d: %s" % (line, message) if line else message)
|
|
|
|
|
|
def append_line(path, obj, root=None):
|
|
"""Append ``obj`` as exactly one line, in a single ``O_APPEND`` write.
|
|
|
|
Validation happens before the file is opened, so a rejected record leaves
|
|
no trace at all -- not an empty file, not a partial line.
|
|
|
|
``root``, when given, resolves ``path`` under it with
|
|
:func:`paths.safe_join`, so a caller-supplied path that escapes the
|
|
workspace is refused here rather than in whichever script remembered to
|
|
check.
|
|
"""
|
|
record = _validated(obj)
|
|
target = paths.safe_join(root, path) if root is not None else path
|
|
payload = (json.dumps(record, ensure_ascii=False) + "\n").encode("utf-8")
|
|
|
|
handle = os.open(target, os.O_WRONLY | os.O_APPEND | os.O_CREAT, 0o600)
|
|
try:
|
|
written = os.write(handle, payload)
|
|
finally:
|
|
os.close(handle)
|
|
if written != len(payload):
|
|
# Retrying the remainder would append a fragment behind whatever
|
|
# another writer put there in the meantime. Fail instead.
|
|
raise JsonlError(
|
|
"short write: %d of %d bytes reached %r" % (written, len(payload), target)
|
|
)
|
|
return record
|
|
|
|
|
|
def read_lines(path):
|
|
"""Read every record, skipping only a torn trailing line.
|
|
|
|
A missing file reads as no records -- an append-only log that has never
|
|
been appended to is empty, not broken.
|
|
"""
|
|
if not os.path.exists(path):
|
|
return []
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
data = handle.read()
|
|
if data == "":
|
|
return []
|
|
|
|
lines = data.split("\n")
|
|
# A file ending in a newline leaves an empty final element; a file that
|
|
# does not was cut off mid-append, and only THAT last line may be skipped.
|
|
if lines and lines[-1] == "":
|
|
lines.pop()
|
|
torn_index = None
|
|
else:
|
|
torn_index = len(lines)
|
|
|
|
records = []
|
|
for index, line in enumerate(lines, start=1):
|
|
if line.strip() == "":
|
|
continue
|
|
try:
|
|
record = json.loads(line)
|
|
except ValueError as exc:
|
|
if index == torn_index:
|
|
continue
|
|
raise JsonlError("not valid JSON: %s" % exc, line=index)
|
|
if not isinstance(record, dict):
|
|
if index == torn_index:
|
|
continue
|
|
raise JsonlError("expected an object, got %s" % type(record).__name__, line=index)
|
|
records.append(record)
|
|
return records
|
|
|
|
|
|
def fold(lines):
|
|
"""Return the decision stream with corrections applied and outcomes removed.
|
|
|
|
A `korrigering` overwrites the fields it names on the `beslutning` it
|
|
points at, and contributes no row of its own -- the log keeps both, the
|
|
stream shows the corrected decision. An `utfall` is not a decision and is
|
|
excluded; build-brief 5.7 reads outcomes on its own terms.
|
|
|
|
The input is never mutated: the caller's records came off disk and stay as
|
|
they were read.
|
|
"""
|
|
stream = []
|
|
by_id = {}
|
|
for record in lines:
|
|
kind = record.get("type")
|
|
if kind == "beslutning":
|
|
copy = dict(record)
|
|
stream.append(copy)
|
|
if "id" in copy:
|
|
by_id[copy["id"]] = copy
|
|
elif kind == "korrigering":
|
|
target_id = record.get("korrigerer")
|
|
if target_id not in by_id:
|
|
raise JsonlError(
|
|
"correction %r names %r, which is not a decision above it"
|
|
% (record.get("id"), target_id)
|
|
)
|
|
by_id[target_id].update(
|
|
{k: v for k, v in record.items() if k not in _CORRECTION_OWN_FIELDS}
|
|
)
|
|
elif kind == "utfall":
|
|
continue
|
|
else:
|
|
raise JsonlError("unknown record type %r" % (kind,))
|
|
return stream
|
|
|
|
|
|
def instant(record):
|
|
"""The record's timestamp as an aware :class:`datetime.datetime`."""
|
|
for key in TIMESTAMP_KEYS:
|
|
if key in record:
|
|
return _aware(record[key], key)
|
|
raise JsonlError("record has no timestamp; expected one of %r" % (TIMESTAMP_KEYS,))
|
|
|
|
|
|
def by_time(lines):
|
|
"""Sort records by the instant they name, not by the string that spells it."""
|
|
return sorted(lines, key=instant)
|
|
|
|
|
|
def _validated(obj):
|
|
if not isinstance(obj, dict):
|
|
raise JsonlError("a record must be an object, got %s" % type(obj).__name__)
|
|
kind = obj.get("type")
|
|
if kind not in TYPES:
|
|
raise JsonlError("type %r is not one of %r" % (kind, TYPES))
|
|
|
|
record = dict(obj)
|
|
record.setdefault("skjema", SKJEMA)
|
|
for key in TIMESTAMP_KEYS:
|
|
if key in record:
|
|
_aware(record[key], key)
|
|
return record
|
|
|
|
|
|
def _aware(value, key):
|
|
try:
|
|
moment = datetime.datetime.fromisoformat(value)
|
|
except (TypeError, ValueError):
|
|
raise JsonlError("%s=%r is not an ISO-8601 timestamp" % (key, value))
|
|
if moment.tzinfo is None or moment.utcoffset() is None:
|
|
raise JsonlError(
|
|
"%s=%r has no UTC offset; a naive timestamp cannot be ordered "
|
|
"across the October hour that happens twice" % (key, value)
|
|
)
|
|
return moment
|