90 lines
3.7 KiB
Python
90 lines
3.7 KiB
Python
|
"""bench vet-window: a window's truth is the stored production judgment.
|
||
|
|
|
||
|
|
Bench truth decides which slots are distrusted inputs and which can be scored, so
|
||
|
|
it must be the very verdicts the corpus and renderers use -- not a private
|
||
|
|
re-judgment, which is how it twice drifted weaker than production before.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import os
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
import bench
|
||
|
|
from suvi import db, detectors, paths
|
||
|
|
|
||
|
|
T0 = 1715299200
|
||
|
|
BANDS = (171, 304)
|
||
|
|
|
||
|
|
|
||
|
|
def build(archive, tmp_path, verdicts):
|
||
|
|
"""Two slots x two bands of empty files, indexed; `verdicts` maps slot -> verdict
|
||
|
|
(None = unjudged) under a run made with the current rules."""
|
||
|
|
conn = db.connect(str(tmp_path / "index.sqlite"))
|
||
|
|
run = db.create_detector_run(conn, detectors.JUDGMENT_RUN, detectors.judgment_config())
|
||
|
|
rows = []
|
||
|
|
for i in range(2):
|
||
|
|
for band in BANDS:
|
||
|
|
when = T0 + i * paths.CADENCE
|
||
|
|
name = paths.FrameName(satellite=16, wavelength=band, t_start=when,
|
||
|
|
t_end=when + paths.CADENCE, version="1-0-2")
|
||
|
|
path = os.path.join(str(archive), name.relpath())
|
||
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||
|
|
open(path, "wb").close()
|
||
|
|
frame_id = db.upsert_frame(conn, name, name.relpath(), size_bytes=1)
|
||
|
|
verdict = verdicts.get((16, band, when), "good")
|
||
|
|
if verdict is not None:
|
||
|
|
rows.append((frame_id, verdict, "field_v1: bar" if verdict == "bad" else None,
|
||
|
|
None, 0))
|
||
|
|
db.record_detections(conn, run, rows)
|
||
|
|
conn.commit()
|
||
|
|
return conn, run
|
||
|
|
|
||
|
|
|
||
|
|
def vet(tmp_path, *extra):
|
||
|
|
return bench.main(["--db", str(tmp_path / "index.sqlite"), "vet-window",
|
||
|
|
"--start", str(T0), "--end", str(T0 + 2 * paths.CADENCE),
|
||
|
|
"--satellites", "16", "--wavelengths", ",".join(map(str, BANDS)),
|
||
|
|
"--name", "w", "--min-good-fraction", "0", *extra])
|
||
|
|
|
||
|
|
|
||
|
|
def truth(conn):
|
||
|
|
return {(r["wavelength"], r["t_start"]): (r["is_good"], r["vet_source"], r["reason"])
|
||
|
|
for r in conn.execute("SELECT f.wavelength, f.t_start, t.* FROM bench_truth t "
|
||
|
|
"JOIN frame f ON f.id = t.frame_id")}
|
||
|
|
|
||
|
|
|
||
|
|
def test_truth_is_the_stored_judgment(archive, tmp_path):
|
||
|
|
conn, run = build(archive, tmp_path, {(16, 304, T0 + paths.CADENCE): "bad"})
|
||
|
|
assert vet(tmp_path) == 0
|
||
|
|
got = truth(conn)
|
||
|
|
assert got[(304, T0 + paths.CADENCE)] == (0, f"judge run {run}", "field_v1: bar")
|
||
|
|
assert got[(171, T0)] == (1, f"judge run {run}", None)
|
||
|
|
assert len(got) == 4
|
||
|
|
|
||
|
|
|
||
|
|
def test_a_window_with_unjudged_frames_is_refused(archive, tmp_path, capsys):
|
||
|
|
conn, _ = build(archive, tmp_path, {(16, 171, T0): None})
|
||
|
|
assert vet(tmp_path) == 1
|
||
|
|
assert "no verdict" in capsys.readouterr().out
|
||
|
|
assert truth(conn) == {}
|
||
|
|
|
||
|
|
|
||
|
|
def test_no_judgment_under_the_current_rules_is_refused(archive, tmp_path, capsys):
|
||
|
|
"""Verdicts made under older rules are not truth for the current ones."""
|
||
|
|
conn, run = build(archive, tmp_path, {})
|
||
|
|
conn.execute("UPDATE detector_run SET config_json = '{\"older\": \"rules\"}' "
|
||
|
|
"WHERE id = ?", (run,))
|
||
|
|
conn.commit()
|
||
|
|
assert vet(tmp_path) == 1
|
||
|
|
assert "no production judgment" in capsys.readouterr().out
|
||
|
|
|
||
|
|
|
||
|
|
def test_an_explicit_run_is_used_when_named(archive, tmp_path):
|
||
|
|
conn, run = build(archive, tmp_path, {})
|
||
|
|
other = db.create_detector_run(conn, detectors.JUDGMENT_RUN, {"older": "rules"})
|
||
|
|
db.record_detections(conn, other, [(r["id"], "bad", "x", None, 0)
|
||
|
|
for r in conn.execute("SELECT id FROM frame")])
|
||
|
|
conn.commit()
|
||
|
|
assert vet(tmp_path, "--run", str(other)) == 0
|
||
|
|
assert all(is_good == 0 for is_good, _, _ in truth(conn).values())
|