1680 lines
69 KiB
Python
1680 lines
69 KiB
Python
#!/usr/bin/env python
|
|
"""Test bench for bad-frame detection and gap filling.
|
|
|
|
Takes a stretch of archive verified to be clean, damages it in known ways, and
|
|
measures how well each detector finds the damage and each filler repairs it.
|
|
|
|
The archive is never modified. Corrupted frames are written to a per-case overlay
|
|
directory and every read goes through :class:`suvi.cases.Overlay`, so a bench run
|
|
against live production data is safe.
|
|
|
|
Workflow::
|
|
|
|
bench.py select-window --start 2024-05-05 --end 2024-05-20
|
|
bench.py vet-window --name may2024 --start ... --end ...
|
|
bench.py make-case --window may2024 --name mixed10 --seed 42
|
|
bench.py detect --case mixed10
|
|
bench.py fill --case mixed10 --oracle
|
|
bench.py report --case mixed10
|
|
|
|
Each step records its results in the index, so steps are resumable and comparable
|
|
across runs.
|
|
"""
|
|
|
|
import argparse
|
|
import bisect
|
|
import datetime
|
|
import itertools
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
from collections import defaultdict
|
|
from multiprocessing import Pool
|
|
|
|
import numpy as np
|
|
from astropy.io import fits
|
|
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
|
|
from suvi import cases, corruptions, db, detectors, fillers, fitsio, metrics, paths, vfs
|
|
|
|
|
|
def parse_date(text):
|
|
"""Accept YYYY-MM-DD or YYYY-MM-DDTHH:MM, always UTC."""
|
|
for fmt in ("%Y-%m-%d", "%Y-%m-%dT%H:%M", "%Y-%m-%d %H:%M"):
|
|
try:
|
|
stamp = datetime.datetime.strptime(text, fmt)
|
|
return int(stamp.replace(tzinfo=datetime.timezone.utc).timestamp())
|
|
except ValueError:
|
|
continue
|
|
raise argparse.ArgumentTypeError(f"Not a date: {text!r}")
|
|
|
|
|
|
def format_time(timestamp):
|
|
return datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc).strftime(
|
|
"%Y-%m-%d %H:%M"
|
|
)
|
|
|
|
|
|
def parse_int_list(text):
|
|
return tuple(int(part) for part in text.split(",") if part.strip())
|
|
|
|
|
|
# ------------------------------------------------------------------- select-window
|
|
|
|
|
|
def cmd_select_window(args):
|
|
root = paths.data_root()
|
|
satellites = parse_int_list(args.satellites)
|
|
wavelengths = parse_int_list(args.wavelengths)
|
|
print(f"Scanning {root} from {format_time(args.start)} to {format_time(args.end)}")
|
|
found = cases.scan_window(root, satellites, wavelengths, args.start, args.end)
|
|
total = len(cases.timeline(args.start, args.end))
|
|
print(f" {len(found)} files across {total} slots\n")
|
|
|
|
runs = cases.find_runs(
|
|
found,
|
|
satellites,
|
|
wavelengths,
|
|
args.start,
|
|
args.end,
|
|
minimum=args.min_slots,
|
|
require_good_label=args.require_legacy_label,
|
|
)
|
|
if not runs:
|
|
print(f"No runs of >= {args.min_slots} slots complete on every requested band.")
|
|
print("Try fewer bands, one satellite, or a different date range.")
|
|
return 1
|
|
|
|
basis = "labelled good by the legacy filter" if args.require_legacy_label else "present"
|
|
print(f"Runs where every requested band and satellite is {basis}:")
|
|
print(f"{'slots':>7} {'hours':>8} window")
|
|
for length, start, end in runs[: args.limit]:
|
|
print(f"{length:>7} {length * paths.CADENCE / 3600:>8.1f} {format_time(start)} .. {format_time(end)}")
|
|
if not args.require_legacy_label:
|
|
print("\nCompleteness only -- run vet-window to verify quality from the headers.")
|
|
return 0
|
|
|
|
|
|
# ---------------------------------------------------------------------- vet-window
|
|
|
|
|
|
def _header_vet(path):
|
|
"""Independent check that a frame really is good, from its header."""
|
|
values, error = fitsio.scan_header(path)
|
|
if error:
|
|
return False, f"header unreadable: {error}"
|
|
if values.get("empty"):
|
|
return False, "EMPTY set"
|
|
if values.get("eclipse"):
|
|
return False, f"ECLIPSE={int(values['eclipse'])}"
|
|
return True, None
|
|
|
|
|
|
def cmd_vet_window(args):
|
|
root = paths.data_root()
|
|
satellites = parse_int_list(args.satellites)
|
|
wavelengths = parse_int_list(args.wavelengths)
|
|
conn = db.connect(args.db)
|
|
|
|
found = cases.scan_window(root, satellites, wavelengths, args.start, args.end)
|
|
slots = cases.timeline(args.start, args.end)
|
|
expected = len(slots) * len(satellites) * len(wavelengths)
|
|
print(f"Window has {len(found)}/{expected} frames present")
|
|
|
|
missing = [
|
|
(satellite, wavelength, time)
|
|
for time in slots
|
|
for satellite in satellites
|
|
for wavelength in wavelengths
|
|
if (satellite, wavelength, time) not in found
|
|
]
|
|
if missing and not args.allow_missing:
|
|
print(f"ERROR: {len(missing)} slots have no frame at all; refusing to vet.")
|
|
print(" Pick a window from select-window, or pass --allow-missing.")
|
|
return 1
|
|
|
|
# Ground truth comes from the headers, which the instrument wrote and which are
|
|
# independent of any pixel filter. The legacy filename labels are recorded for
|
|
# comparison but deliberately do not decide anything: they were written in May
|
|
# 2024 by a filter that was retuned that July, so they are neither current nor
|
|
# self-consistent. Treating them as truth would bake the old filter's mistakes
|
|
# into every score computed against this window.
|
|
print("Vetting from headers...")
|
|
disagreements = []
|
|
truth = {}
|
|
reliever = vfs.Reliever(label="vet")
|
|
for index, (slot, (path, label)) in enumerate(sorted(found.items())):
|
|
reliever.tick()
|
|
good_by_header, reason = _header_vet(path)
|
|
good_by_legacy = label == "f"
|
|
if good_by_header != good_by_legacy:
|
|
disagreements.append((slot, label, reason))
|
|
truth[slot] = (good_by_header, "header")
|
|
if args.progress and index % 2000 == 0 and index:
|
|
print(f" {index}/{len(found)}")
|
|
|
|
print(f" {len(disagreements)} frames where the legacy label disagrees with the header")
|
|
for slot, label, reason in disagreements[:10]:
|
|
print(f" {slot} legacy={label} header={reason or 'good'}")
|
|
|
|
good = sum(1 for is_good, _ in truth.values() if is_good)
|
|
print(f" {good}/{len(truth)} frames vetted good")
|
|
if good < len(truth) * args.min_good_fraction:
|
|
print(f"ERROR: only {good / len(truth):.1%} vetted good; window is not clean enough.")
|
|
return 1
|
|
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO bench_window (name, t_start, t_end, satellites, wavelengths, vetted_at, notes)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT (name) DO UPDATE SET
|
|
t_start = excluded.t_start, t_end = excluded.t_end,
|
|
satellites = excluded.satellites, wavelengths = excluded.wavelengths,
|
|
vetted_at = excluded.vetted_at, notes = excluded.notes
|
|
""",
|
|
(
|
|
args.name,
|
|
args.start,
|
|
args.end,
|
|
json.dumps(list(satellites)),
|
|
json.dumps(list(wavelengths)),
|
|
time.time(),
|
|
args.notes,
|
|
),
|
|
)
|
|
window_id = conn.execute(
|
|
"SELECT id FROM bench_window WHERE name = ?", (args.name,)
|
|
).fetchone()["id"]
|
|
conn.execute("DELETE FROM bench_truth WHERE window_id = ?", (window_id,))
|
|
|
|
for slot, (path, label) in sorted(found.items()):
|
|
name = paths.parse_frame_filename(os.path.basename(path))
|
|
relpath = name.relpath()
|
|
frame_id = db.upsert_frame(
|
|
conn, name, relpath, os.path.getsize(path), os.path.getmtime(path)
|
|
)
|
|
is_good, source = truth[slot]
|
|
conn.execute(
|
|
"INSERT OR REPLACE INTO bench_truth (window_id, frame_id, is_good, vet_source)"
|
|
" VALUES (?, ?, ?, ?)",
|
|
(window_id, frame_id, int(is_good), source),
|
|
)
|
|
conn.commit()
|
|
print(f"Wrote window '{args.name}' (id {window_id}) with {len(found)} frames")
|
|
|
|
if args.contact_sheet:
|
|
_write_contact_sheet(found, args.contact_sheet, satellites, wavelengths)
|
|
print(f"Contact sheet: {args.contact_sheet}")
|
|
print("Review it before trusting this window as ground truth.")
|
|
return 0
|
|
|
|
|
|
def _write_contact_sheet(found, path, satellites, wavelengths, columns=24, size=64):
|
|
"""A downsampled filmstrip of the window, for the one manual review pass."""
|
|
import matplotlib
|
|
|
|
matplotlib.use("Agg")
|
|
from matplotlib import pyplot as plt
|
|
|
|
satellite, wavelength = satellites[0], wavelengths[len(wavelengths) // 2]
|
|
series = sorted(slot for slot in found if slot[0] == satellite and slot[1] == wavelength)
|
|
if not series:
|
|
return
|
|
step = max(1, len(series) // (columns * 20))
|
|
picks = series[::step]
|
|
rows = (len(picks) + columns - 1) // columns
|
|
|
|
figure, axes = plt.subplots(rows, columns, figsize=(columns, rows))
|
|
axes = np.atleast_2d(axes)
|
|
vmin, vmax, gamma = metrics.DISPLAY_MAPPING[wavelength]
|
|
for index, slot in enumerate(picks):
|
|
axis = axes[index // columns][index % columns]
|
|
image, error = fitsio.read_image(found[slot][0])
|
|
if image is not None:
|
|
import cv2 as cv
|
|
|
|
thumb = cv.resize(np.nan_to_num(image), (size, size), interpolation=cv.INTER_AREA)
|
|
axis.imshow(metrics.to_display(thumb, wavelength), cmap="gray", vmin=0, vmax=1)
|
|
axis.set_title(format_time(slot[2])[5:], fontsize=3)
|
|
axis.axis("off")
|
|
for index in range(len(picks), rows * columns):
|
|
axes[index // columns][index % columns].axis("off")
|
|
figure.suptitle(f"g{satellite} {wavelength}A - {len(picks)} of {len(series)} frames")
|
|
figure.tight_layout()
|
|
figure.savefig(path, dpi=200)
|
|
plt.close(figure)
|
|
|
|
|
|
# ------------------------------------------------------------------------ make-case
|
|
|
|
|
|
def _load_window(conn, name):
|
|
row = conn.execute("SELECT * FROM bench_window WHERE name = ?", (name,)).fetchone()
|
|
if row is None:
|
|
raise SystemExit(f"No vetted window named {name!r}; run vet-window first.")
|
|
return row
|
|
|
|
|
|
def _build_overlay(conn, case_row, window_row):
|
|
"""Reconstruct a case's view of the archive from the index."""
|
|
root = paths.data_root()
|
|
satellites = tuple(json.loads(window_row["satellites"]))
|
|
wavelengths = tuple(json.loads(window_row["wavelengths"]))
|
|
archive = cases.scan_window(
|
|
root, satellites, wavelengths, window_row["t_start"], window_row["t_end"]
|
|
)
|
|
overrides, deleted = {}, set()
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT f.satellite, f.wavelength, f.t_start, i.mode, i.overlay_path
|
|
FROM bench_injection i JOIN frame f ON f.id = i.frame_id
|
|
WHERE i.case_id = ?
|
|
""",
|
|
(case_row["id"],),
|
|
).fetchall()
|
|
for row in rows:
|
|
slot = (row["satellite"], row["wavelength"], row["t_start"])
|
|
if row["mode"] == "delete":
|
|
deleted.add(slot)
|
|
elif row["overlay_path"]:
|
|
overrides[slot] = row["overlay_path"]
|
|
return cases.Overlay(archive=archive, overrides=overrides, deleted=frozenset(deleted))
|
|
|
|
|
|
def _write_corrupted(source_path, target_path, injection):
|
|
"""Materialise one corrupted frame into the overlay. Returns None on success."""
|
|
os.makedirs(os.path.dirname(target_path), exist_ok=True)
|
|
corruption = corruptions.CATALOG[injection.mode]
|
|
|
|
if corruption.kind == "file":
|
|
with open(source_path, "rb") as handle:
|
|
raw = handle.read()
|
|
damaged = corruptions.apply_file(injection.mode, raw, injection.seed, injection.severity)
|
|
with open(target_path, "wb") as handle:
|
|
handle.write(damaged)
|
|
return None
|
|
|
|
image, error = fitsio.read_image(source_path)
|
|
if image is None:
|
|
return f"could not read source: {error}"
|
|
|
|
donor = None
|
|
if corruption.needs_donor:
|
|
donor = np.roll(image, image.shape[0] // 3, axis=0) # a plausible other frame
|
|
|
|
damaged, overrides = corruptions.apply_array(
|
|
injection.mode, image, injection.seed, injection.severity, donor
|
|
)
|
|
if corruption.recompute_stats:
|
|
overrides = {**overrides, **corruptions.recomputed_stats(damaged)}
|
|
|
|
with fits.open(source_path) as hdus:
|
|
target = next(
|
|
(h for h in hdus if getattr(h, "data", None) is not None and h.data.ndim == 2), None
|
|
)
|
|
if target is None:
|
|
return "source has no image HDU"
|
|
target.data = damaged.astype(np.float32)
|
|
for key, value in overrides.items():
|
|
target.header[key] = value
|
|
hdus.writeto(target_path, overwrite=True, checksum=True)
|
|
return None
|
|
|
|
|
|
def _corrupt_one(job):
|
|
source_path, target_path, injection = job
|
|
try:
|
|
return injection.slot, _write_corrupted(source_path, target_path, injection)
|
|
except Exception as exc: # a corruption must never take the whole run down
|
|
return injection.slot, f"{type(exc).__name__}: {exc}"
|
|
|
|
|
|
def cmd_make_case(args):
|
|
conn = db.connect(args.db)
|
|
window = _load_window(conn, args.window)
|
|
satellites = tuple(json.loads(window["satellites"]))
|
|
wavelengths = tuple(json.loads(window["wavelengths"]))
|
|
root = paths.data_root()
|
|
|
|
archive = cases.scan_window(
|
|
root, satellites, wavelengths, window["t_start"], window["t_end"]
|
|
)
|
|
plan = cases.InjectionPlan(
|
|
fraction=args.fraction,
|
|
gap_lengths=parse_int_list(args.gap_lengths),
|
|
satellite_scope=args.satellite_scope,
|
|
wavelength_scope=args.wavelength_scope,
|
|
modes=tuple(args.modes.split(",")) if args.modes else cases.InjectionPlan.modes,
|
|
severity=(args.severity_low, args.severity_high),
|
|
gaps_per_length=args.gaps_per_length,
|
|
).validate()
|
|
|
|
window_times = sorted({slot[2] for slot in archive})
|
|
injections = cases.plan_injections(
|
|
plan, set(archive), satellites, wavelengths, args.seed
|
|
)
|
|
if not injections:
|
|
print("Plan produced no injections; raise --fraction or widen the window.")
|
|
return 1
|
|
|
|
overlay_dir = args.overlay_dir or os.path.join(
|
|
os.path.dirname(root), "bench", args.name
|
|
)
|
|
os.makedirs(overlay_dir, exist_ok=True)
|
|
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO bench_case (window_id, name, seed, plan_json, overlay_dir, created_at)
|
|
VALUES (?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT (name) DO UPDATE SET
|
|
window_id = excluded.window_id, seed = excluded.seed,
|
|
plan_json = excluded.plan_json, overlay_dir = excluded.overlay_dir,
|
|
created_at = excluded.created_at
|
|
""",
|
|
(
|
|
window["id"],
|
|
args.name,
|
|
args.seed,
|
|
json.dumps(plan.as_dict(), sort_keys=True),
|
|
overlay_dir,
|
|
time.time(),
|
|
),
|
|
)
|
|
case_id = conn.execute(
|
|
"SELECT id FROM bench_case WHERE name = ?", (args.name,)
|
|
).fetchone()["id"]
|
|
conn.execute("DELETE FROM bench_injection WHERE case_id = ?", (case_id,))
|
|
|
|
jobs, rows = [], []
|
|
for injection in injections:
|
|
source_path = archive[injection.slot][0]
|
|
name = paths.parse_frame_filename(os.path.basename(source_path))
|
|
frame_id = db.frame_id_by_slot(conn, *injection.slot)
|
|
if frame_id is None:
|
|
frame_id = db.upsert_frame(conn, name, name.relpath())
|
|
overlay_path = None
|
|
if injection.mode != "delete":
|
|
overlay_path = os.path.join(overlay_dir, name.relpath())
|
|
jobs.append((source_path, overlay_path, injection))
|
|
rows.append((case_id, frame_id, injection, overlay_path))
|
|
|
|
print(f"Planned {len(injections)} injections "
|
|
f"({sum(1 for i in injections if i.mode == 'delete')} deletions, {len(jobs)} corruptions)")
|
|
|
|
failures = {}
|
|
if jobs:
|
|
reliever = vfs.Reliever(label="make-case")
|
|
with Pool(args.workers, initializer=fitsio.quiet_astropy) as pool:
|
|
for index, (slot, error) in enumerate(pool.imap_unordered(_corrupt_one, jobs, 8)):
|
|
reliever.tick()
|
|
if error:
|
|
failures[slot] = error
|
|
if args.progress and index % 200 == 0 and index:
|
|
print(f" wrote {index}/{len(jobs)}")
|
|
if failures:
|
|
print(f"WARNING: {len(failures)} corruptions failed:")
|
|
for slot, error in list(failures.items())[:5]:
|
|
print(f" {slot}: {error}")
|
|
|
|
conn.executemany(
|
|
"""
|
|
INSERT INTO bench_injection
|
|
(case_id, frame_id, mode, severity, params_json, overlay_path, gap_index, gap_length)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
""",
|
|
[
|
|
(
|
|
case_id,
|
|
frame_id,
|
|
injection.mode,
|
|
injection.severity,
|
|
json.dumps({"seed": injection.seed}),
|
|
overlay_path,
|
|
injection.gap_index,
|
|
injection.gap_length,
|
|
)
|
|
for case_id, frame_id, injection, overlay_path in rows
|
|
if injection.slot not in failures
|
|
],
|
|
)
|
|
conn.commit()
|
|
|
|
affected = len({i.slot[2] for i in injections})
|
|
print(f"Case '{args.name}' (id {case_id}): {affected}/{len(window_times)} timestamps affected")
|
|
print(f"Overlay: {overlay_dir}")
|
|
return 0
|
|
|
|
|
|
# --------------------------------------------------------------------------- detect
|
|
|
|
|
|
def _read_features(job):
|
|
"""Load one frame's header, thumbnail, and single-frame verdicts."""
|
|
slot, path = job
|
|
if path is None:
|
|
return slot, None, None, None, 0
|
|
started = time.perf_counter_ns()
|
|
values, header_error = fitsio.scan_header(path)
|
|
features = detectors.FrameFeatures(slot=slot, header=values, error=header_error)
|
|
header_verdict = detectors.header_v1(features)
|
|
header_us = (time.perf_counter_ns() - started) // 1000
|
|
|
|
image, image_error = fitsio.read_image(path)
|
|
|
|
started = time.perf_counter_ns()
|
|
if image is None:
|
|
geometry_verdict = detectors.Verdict("bad", f"unreadable: {image_error}", {})
|
|
else:
|
|
geometry_verdict = detectors.geometry_v1(image, slot[1])
|
|
geometry_us = (time.perf_counter_ns() - started) // 1000
|
|
|
|
started = time.perf_counter_ns()
|
|
if image is None:
|
|
disc_verdict = detectors.Verdict("bad", f"unreadable: {image_error}", {})
|
|
else:
|
|
disc_verdict = detectors.disc_v1(image, slot[1], values)
|
|
disc_us = (time.perf_counter_ns() - started) // 1000
|
|
|
|
features.thumbnail = detectors.thumbnail(image) if image is not None else None
|
|
return (
|
|
slot,
|
|
features,
|
|
(header_verdict, header_us),
|
|
(geometry_verdict, geometry_us),
|
|
(disc_verdict, disc_us),
|
|
)
|
|
|
|
|
|
def cmd_detect(args):
|
|
fitsio.quiet_astropy()
|
|
conn = db.connect(args.db)
|
|
case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone()
|
|
if case is None:
|
|
raise SystemExit(f"No case named {args.case!r}")
|
|
window = conn.execute(
|
|
"SELECT * FROM bench_window WHERE id = ?", (case["window_id"],)
|
|
).fetchone()
|
|
satellites = tuple(json.loads(window["satellites"]))
|
|
wavelengths = tuple(json.loads(window["wavelengths"]))
|
|
overlay = _build_overlay(conn, case, window)
|
|
wanted = tuple(args.detectors.split(",")) if args.detectors else detectors.ALL_DETECTORS
|
|
|
|
runs = {}
|
|
for name in wanted:
|
|
runs[name] = db.create_detector_run(
|
|
conn, name, {"case": args.case}, notes=f"bench case {args.case}"
|
|
)
|
|
|
|
results = {name: {} for name in wanted}
|
|
elapsed = defaultdict(int)
|
|
|
|
for wavelength in wavelengths:
|
|
per_satellite = {}
|
|
for satellite in satellites:
|
|
series_slots = [
|
|
slot for slot in overlay.series(satellite, wavelength)
|
|
if slot not in overlay.deleted
|
|
]
|
|
jobs = [(slot, overlay.path(slot)) for slot in series_slots]
|
|
features_by_slot = {}
|
|
reliever = vfs.Reliever(label="detect")
|
|
with Pool(args.workers, initializer=fitsio.quiet_astropy) as pool:
|
|
for slot, features, header, geometry, disc in pool.imap(
|
|
_read_features, jobs, 8
|
|
):
|
|
reliever.tick()
|
|
if features is None:
|
|
continue
|
|
features_by_slot[slot] = features
|
|
for name, outcome in (
|
|
("header_v1", header),
|
|
("geometry_v1", geometry),
|
|
("disc_v1", disc),
|
|
):
|
|
if name in results:
|
|
results[name][slot] = outcome[0]
|
|
elapsed[name] += outcome[1]
|
|
|
|
ordered = [features_by_slot[s] for s in series_slots if s in features_by_slot]
|
|
per_satellite[satellite] = ordered
|
|
|
|
if "temporal_v1" in results and ordered:
|
|
started = time.perf_counter_ns()
|
|
verdicts = detectors.temporal_v1(ordered)
|
|
elapsed["temporal_v1"] += (time.perf_counter_ns() - started) // 1000
|
|
for features, verdict in zip(ordered, verdicts):
|
|
results["temporal_v1"][features.slot] = verdict
|
|
|
|
if "crosssat_v1" in results and len(satellites) >= 2:
|
|
first, second = satellites[0], satellites[1]
|
|
by_time = {
|
|
sat: {f.slot[2]: f for f in per_satellite.get(sat, [])}
|
|
for sat in (first, second)
|
|
}
|
|
temporal = results.get("temporal_v1", {})
|
|
started = time.perf_counter_ns()
|
|
for when in sorted(set(by_time[first]) | set(by_time[second])):
|
|
a, b = by_time[first].get(when), by_time[second].get(when)
|
|
verdict_a, verdict_b = detectors.crosssat_v1(
|
|
a,
|
|
b,
|
|
temporal.get(a.slot) if a else None,
|
|
temporal.get(b.slot) if b else None,
|
|
)
|
|
if a is not None:
|
|
results["crosssat_v1"][a.slot] = verdict_a
|
|
if b is not None:
|
|
results["crosssat_v1"][b.slot] = verdict_b
|
|
elapsed["crosssat_v1"] += (time.perf_counter_ns() - started) // 1000
|
|
print(f" {wavelength}A done")
|
|
|
|
for name, verdicts in results.items():
|
|
payload = []
|
|
for slot, verdict in verdicts.items():
|
|
frame_id = db.frame_id_by_slot(conn, *slot)
|
|
if frame_id is None:
|
|
continue
|
|
payload.append((frame_id, verdict.verdict, verdict.reason, verdict.scores, None))
|
|
db.record_detections(conn, runs[name], payload)
|
|
conn.commit()
|
|
flagged = sum(1 for v in verdicts.values() if v.verdict == "bad")
|
|
print(f"{name:>14}: {len(verdicts)} frames, {flagged} flagged bad, "
|
|
f"{elapsed[name] / max(len(verdicts), 1):.0f} us/frame")
|
|
return 0
|
|
|
|
|
|
# ----------------------------------------------------------------------------- fill
|
|
|
|
|
|
def _bad_slots(conn, case, source):
|
|
"""Slots the fill step should reconstruct, per oracle or per detector.
|
|
|
|
Detector runs are per case, so the run has to be looked up by this case's
|
|
config. Taking "the latest run of header_v1" would silently pick up whichever
|
|
case happened to be processed last.
|
|
"""
|
|
if source == "oracle":
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT f.satellite, f.wavelength, f.t_start
|
|
FROM bench_injection i JOIN frame f ON f.id = i.frame_id
|
|
WHERE i.case_id = ?
|
|
""",
|
|
(case["id"],),
|
|
).fetchall()
|
|
else:
|
|
run_id = db.latest_run_id(conn, source, {"case": case["name"]})
|
|
if run_id is None:
|
|
raise SystemExit(
|
|
f"No run of {source!r} for case {case['name']!r}; run detect first."
|
|
)
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT f.satellite, f.wavelength, f.t_start
|
|
FROM detection d JOIN frame f ON f.id = d.frame_id
|
|
WHERE d.run_id = ? AND d.verdict = 'bad'
|
|
""",
|
|
(run_id,),
|
|
).fetchall()
|
|
return {(row["satellite"], row["wavelength"], row["t_start"]) for row in rows}
|
|
|
|
|
|
def cmd_fill(args):
|
|
fitsio.quiet_astropy()
|
|
conn = db.connect(args.db)
|
|
case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone()
|
|
if case is None:
|
|
raise SystemExit(f"No case named {args.case!r}")
|
|
window = conn.execute(
|
|
"SELECT * FROM bench_window WHERE id = ?", (case["window_id"],)
|
|
).fetchone()
|
|
satellites = tuple(json.loads(window["satellites"]))
|
|
wavelengths = tuple(json.loads(window["wavelengths"]))
|
|
overlay = _build_overlay(conn, case, window)
|
|
|
|
source = "oracle" if args.oracle else args.from_detector
|
|
if source is None:
|
|
raise SystemExit("Pass either --oracle or --from-detector NAME")
|
|
bad = _bad_slots(conn, case, source)
|
|
print(f"Filling {len(bad)} slots identified by '{source}'")
|
|
|
|
gap_lengths = {}
|
|
for row in conn.execute(
|
|
"""
|
|
SELECT f.satellite, f.wavelength, f.t_start, i.gap_length
|
|
FROM bench_injection i JOIN frame f ON f.id = i.frame_id WHERE i.case_id = ?
|
|
""",
|
|
(case["id"],),
|
|
):
|
|
gap_lengths[(row["satellite"], row["wavelength"], row["t_start"])] = row["gap_length"]
|
|
|
|
wanted = tuple(args.fillers.split(",")) if args.fillers else tuple(fillers.FILLERS)
|
|
# The learned filler is joint across all six bands, so it cannot run inside a loop
|
|
# that handles one band at a time; it gets its own pass below.
|
|
per_band = tuple(name for name in wanted if name != "learned")
|
|
reliever = vfs.Reliever(label="fill")
|
|
scored = defaultdict(list)
|
|
# A slot whose truth frame will not read cannot be scored. Count them: silently
|
|
# dropping them yields metrics computed over whatever subset happened to work,
|
|
# with nothing in the output to say so. A degraded mount once reduced this to
|
|
# 12% coverage while the report looked entirely normal.
|
|
unreadable = 0
|
|
attempted = 0
|
|
payload = []
|
|
|
|
if not per_band:
|
|
# Only the learned filler was asked for. Its own pass below reads exactly what
|
|
# it needs; walking every band here would pull all 5,304 damaged slots and their
|
|
# neighbours off the mount to compute nothing at all.
|
|
print(" (skipping per-band pass: no per-band fillers requested)")
|
|
|
|
for wavelength in (wavelengths if per_band else ()):
|
|
truth_cache = {}
|
|
|
|
def truth_image(slot):
|
|
if slot not in truth_cache:
|
|
path = overlay.truth_path(slot)
|
|
image, _ = fitsio.read_image(path) if path else (None, None)
|
|
truth_cache[slot] = image
|
|
return truth_cache[slot]
|
|
|
|
|
|
# Times where every satellite has a good frame in this band. The two
|
|
# spacecraft observe simultaneously, so such a slot isolates the instrument
|
|
# difference with none of the Sun's own evolution folded in -- which is the
|
|
# only sound way to calibrate one against the other. See fillers.crosssat.
|
|
per_satellite = [
|
|
{
|
|
s[2]
|
|
for s in overlay.series(other, wavelength)
|
|
if s not in bad and overlay.path(s) is not None
|
|
}
|
|
for other in satellites
|
|
]
|
|
paired_times = sorted(set.intersection(*per_satellite)) if len(satellites) > 1 else []
|
|
|
|
for satellite in satellites:
|
|
series = overlay.series(satellite, wavelength)
|
|
good = [s for s in series if s not in bad and overlay.path(s) is not None]
|
|
good_times = [s[2] for s in good]
|
|
for slot in series:
|
|
if slot not in bad:
|
|
continue
|
|
reliever.tick()
|
|
attempted += 1
|
|
truth = truth_image(slot)
|
|
if truth is None:
|
|
unreadable += 1
|
|
continue
|
|
|
|
before = _nearest(good_times, slot[2], before=True)
|
|
after = _nearest(good_times, slot[2], before=False)
|
|
counterpart_slot = next(
|
|
(
|
|
(other, wavelength, slot[2])
|
|
for other in satellites
|
|
if other != satellite
|
|
and (other, wavelength, slot[2]) not in bad
|
|
and overlay.path((other, wavelength, slot[2])) is not None
|
|
),
|
|
None,
|
|
)
|
|
|
|
calibration = None
|
|
if counterpart_slot is not None and paired_times:
|
|
pair_time = _closest(paired_times, slot[2])
|
|
calibration = (
|
|
truth_image((counterpart_slot[0], wavelength, pair_time)),
|
|
truth_image((satellite, wavelength, pair_time)),
|
|
)
|
|
|
|
context = fillers.FillContext(
|
|
before=truth_image((satellite, wavelength, before)) if before else None,
|
|
dt_before=slot[2] - before if before else 0.0,
|
|
after=truth_image((satellite, wavelength, after)) if after else None,
|
|
dt_after=after - slot[2] if after else 0.0,
|
|
counterpart=truth_image(counterpart_slot) if counterpart_slot else None,
|
|
calibration=calibration,
|
|
header=_header_for(overlay, slot),
|
|
)
|
|
|
|
for filler_name in per_band:
|
|
filled = fillers.FILLERS[filler_name](context)
|
|
if filled is None:
|
|
continue
|
|
score = metrics.score_fill(
|
|
filled, truth, wavelength, gap_lengths.get(slot, context.gap_frames)
|
|
)
|
|
scored[filler_name].append(score)
|
|
frame_id = db.frame_id_by_slot(conn, *slot)
|
|
if frame_id is not None:
|
|
payload.append(
|
|
(case["id"], frame_id, filler_name, source,
|
|
json.dumps(score.as_dict()))
|
|
)
|
|
truth_cache.clear()
|
|
print(f" {wavelength}A done")
|
|
|
|
if "learned" in wanted:
|
|
learned_scores, learned_payload, learned_attempted = _fill_learned(
|
|
conn, case, overlay, satellites, wavelengths, bad, gap_lengths, source,
|
|
reliever, max_targets=args.max_targets,
|
|
)
|
|
scored["learned"] = learned_scores
|
|
payload.extend(learned_payload)
|
|
attempted = max(attempted, learned_attempted)
|
|
|
|
conn.executemany(
|
|
"""
|
|
INSERT INTO bench_fill_result (case_id, frame_id, filler, source, scores_json)
|
|
VALUES (?, ?, ?, ?, ?)
|
|
ON CONFLICT (case_id, frame_id, filler, source)
|
|
DO UPDATE SET scores_json = excluded.scores_json
|
|
""",
|
|
payload,
|
|
)
|
|
conn.commit()
|
|
|
|
covered = attempted - unreadable
|
|
print(f"\nScored {covered}/{attempted} slots"
|
|
+ (f" -- {unreadable} could not be read" if unreadable else ""))
|
|
if attempted and covered < attempted * 0.95:
|
|
print(f" WARNING: only {covered / attempted:.0%} of damaged slots were scored.")
|
|
print(" These numbers describe that subset, not the case. A mount")
|
|
print(" that ran out of file handles is the usual cause; re-run.")
|
|
print(f"\n{'filler':>16} {'n':>6} {'rmse':>10} {'log rmse':>10} {'psnr':>8} {'ssim':>7}")
|
|
for filler_name in wanted:
|
|
summary = metrics.summarise_fills(scored.get(filler_name, []))
|
|
if not summary:
|
|
print(f"{filler_name:>16} {'-- not applicable --':>44}")
|
|
continue
|
|
print(f"{filler_name:>16} {summary['n']:>6} {summary['rmse']:>10.4g} "
|
|
f"{summary['log_rmse']:>10.4g} {summary['psnr']:>8.2f} {summary['ssim']:>7.4f}")
|
|
return 0
|
|
|
|
|
|
def _stratified_targets(targets, gap_lengths, limit):
|
|
"""`limit` targets spread evenly over gap length, not the first `limit` found.
|
|
|
|
Fill quality is reported against gap length and varies by 10 dB across the range, so
|
|
a prefix of the sorted targets would sample whatever gap happened to be placed
|
|
earliest and report it as the model's overall quality.
|
|
"""
|
|
if not limit or limit >= len(targets):
|
|
return targets
|
|
by_length = defaultdict(list)
|
|
for target in targets:
|
|
satellite, when = target
|
|
lengths = [length for (s, _, t), length in gap_lengths.items()
|
|
if s == satellite and t == when]
|
|
by_length[max(lengths) if lengths else 0].append(target)
|
|
|
|
chosen = []
|
|
groups = [sorted(group) for _, group in sorted(by_length.items())]
|
|
position = 0
|
|
while len(chosen) < limit and any(position < len(g) for g in groups):
|
|
for group in groups:
|
|
if position < len(group) and len(chosen) < limit:
|
|
chosen.append(group[position])
|
|
position += 1
|
|
return sorted(chosen)
|
|
|
|
|
|
def _fill_learned(conn, case, overlay, satellites, wavelengths, bad, gap_lengths,
|
|
source, reliever, max_targets=None):
|
|
"""Run the learned filler, which reconstructs all six bands in one pass.
|
|
|
|
A separate pass because the model is joint across bands while the loop above is not.
|
|
Running it inside that loop would mean six forward passes per slot, five of whose
|
|
outputs are thrown away, and six times the stack assembly with it.
|
|
|
|
Returns (scores, rows, attempted) so the caller can merge it into its own tally.
|
|
"""
|
|
if fillers.load_learned() is None:
|
|
print("\nSkipping 'learned': no checkpoint "
|
|
f"(set {fillers.LEARNED_CHECKPOINT_ENV})")
|
|
return [], [], 0
|
|
|
|
targets = sorted({(slot[0], slot[2]) for slot in bad})
|
|
if max_targets:
|
|
targets = _stratified_targets(targets, gap_lengths, max_targets)
|
|
print(f"\nLearned filler: {len(targets)} (satellite, time) targets")
|
|
read = _band_reader(overlay, wavelengths, bad, reliever=reliever)
|
|
clean_times = _clean_times(overlay, satellites, wavelengths, bad)
|
|
# Times where *both* satellites are clean in every band: the calibration pairs
|
|
# the cross-satellite photometric transfer is fitted from. A target's own time
|
|
# is never in this list -- the target is a bad slot, so it is not clean.
|
|
paired_times = (sorted(set(clean_times.get(satellites[0], []))
|
|
& set(clean_times.get(satellites[1], [])))
|
|
if len(satellites) > 1 else [])
|
|
|
|
scores, rows = [], []
|
|
attempted = 0
|
|
unreadable = 0
|
|
for index, (satellite, when) in enumerate(targets):
|
|
stack = _build_stack(overlay, satellite, when, satellites, wavelengths, bad,
|
|
read, clean_times)
|
|
calibration = None
|
|
if paired_times:
|
|
other = next(s for s in satellites if s != satellite)
|
|
pair_time = _closest(paired_times, when)
|
|
pair = (read(other, pair_time), read(satellite, pair_time))
|
|
if pair[0] is not None and pair[1] is not None:
|
|
calibration = pair
|
|
filled = fillers.learned(fillers.FillContext(stack=stack,
|
|
calibration=calibration))
|
|
if filled is None:
|
|
unreadable += 1
|
|
continue
|
|
for band, wavelength in enumerate(wavelengths):
|
|
slot = (satellite, wavelength, when)
|
|
if slot not in bad:
|
|
continue
|
|
attempted += 1
|
|
path = overlay.truth_path(slot)
|
|
truth, _ = fitsio.read_image(path) if path else (None, None)
|
|
if truth is None:
|
|
unreadable += 1
|
|
continue
|
|
score = metrics.score_fill(filled[band], truth, wavelength,
|
|
gap_lengths.get(slot, 0))
|
|
scores.append(score)
|
|
frame_id = db.frame_id_by_slot(conn, *slot)
|
|
if frame_id is not None:
|
|
rows.append((case["id"], frame_id, "learned", source,
|
|
json.dumps(score.as_dict())))
|
|
if index and index % 200 == 0:
|
|
print(f" {index}/{len(targets)}", flush=True)
|
|
if unreadable:
|
|
print(f" {unreadable} targets could not be read or filled")
|
|
return scores, rows, attempted
|
|
|
|
|
|
def _clean_times(overlay, satellites, wavelengths, bad):
|
|
"""Per satellite, the times with a usable frame in every band.
|
|
|
|
Computed once for the whole case. Deriving it inside :func:`_build_stack` meant
|
|
sorting all 31,488 slots afresh for each of ~900 targets, which is most of a minute
|
|
spent recomputing the same list.
|
|
"""
|
|
times = {}
|
|
for satellite in satellites:
|
|
usable = defaultdict(int)
|
|
for slot in overlay.slots():
|
|
if slot[0] != satellite or slot in bad:
|
|
continue
|
|
if overlay.path(slot) is not None:
|
|
usable[slot[2]] += 1
|
|
# *Every* band must be clean. Counting a time as usable when only some
|
|
# bands were would hand out anchors marked 'available' that carry damaged
|
|
# pixels, and -- worse -- let a damaged band into the calibration pair the
|
|
# cross-satellite gain is fitted from.
|
|
times[satellite] = sorted(t for t, bands in usable.items()
|
|
if bands == len(wavelengths))
|
|
return times
|
|
|
|
|
|
def _build_stack(overlay, satellite, target_time, satellites, wavelengths, bad, read,
|
|
clean_times=None):
|
|
"""Assemble the multi-frame stack the learned filler fuses.
|
|
|
|
Mirrors `suvi.samples.stack_layout` so a model trained on shards meets the same
|
|
stack shape here: the same multi-scale offsets on both satellites, plus the nearest
|
|
usable frame in each direction as an anchor so a 300-slot gap is never empty.
|
|
|
|
Entries are **all six bands at once**, because the model is joint across them --
|
|
when one band is damaged the others usually are not, and that is much of what makes
|
|
the fusion work. `read(satellite, time)` returns a (6, H, W) array or None, and is
|
|
responsible for resolving damaged slots to their *damaged* pixels.
|
|
|
|
A slot the case damaged is included as **suspect**, carrying those pixels rather
|
|
than being dropped. Every hand-written filler discards such a frame outright; that
|
|
is the difference this stack exists to measure.
|
|
"""
|
|
from suvi import samples
|
|
|
|
def state_of(source, when):
|
|
"""'suspect' if any band of this slot was damaged, else 'available'."""
|
|
damaged = any((source, wl, when) in bad for wl in wavelengths)
|
|
return "suspect" if damaged else "available"
|
|
|
|
entries = []
|
|
for source, offset in samples.stack_layout(satellites, satellite):
|
|
when = target_time + offset * paths.CADENCE
|
|
if not any((source, wl, when) in overlay.archive for wl in wavelengths):
|
|
continue
|
|
image = read(source, when)
|
|
entries.append({
|
|
"image": image,
|
|
"state": "missing" if image is None else state_of(source, when),
|
|
"dt": float(when - target_time),
|
|
"same_satellite": source == satellite,
|
|
"slot": (source, when),
|
|
})
|
|
|
|
# Anchors, so the far end of a long outage is still reachable.
|
|
seen = {entry["dt"] for entry in entries if entry["same_satellite"]}
|
|
if clean_times is None:
|
|
clean_times = _clean_times(overlay, (satellite,), wavelengths, bad)
|
|
clean = clean_times.get(satellite, [])
|
|
for before in (True, False):
|
|
anchor = _nearest(clean, target_time, before=before)
|
|
if anchor is None or float(anchor - target_time) in seen:
|
|
continue
|
|
image = read(satellite, anchor)
|
|
if image is None:
|
|
continue
|
|
entries.append({
|
|
"image": image,
|
|
"state": "available",
|
|
"dt": float(anchor - target_time),
|
|
"same_satellite": True,
|
|
"slot": (satellite, anchor),
|
|
})
|
|
return entries
|
|
|
|
|
|
def _band_reader(overlay, wavelengths, bad, limit=48, reliever=None):
|
|
"""Reader of six-band slots for :func:`_build_stack`, with a bounded cache.
|
|
|
|
Damaged slots resolve through the overlay to their *damaged* pixels; a filler handed
|
|
the pristine frame for a slot the case corrupted would be reading the answer key.
|
|
|
|
Consecutive targets share most of their stack, so a small cache removes almost all
|
|
the repeat reads -- and it has to be bounded, because six bands of 1280x1280 float32
|
|
is 39 MB and an unbounded one would run this VM out of memory long before the case
|
|
finished.
|
|
"""
|
|
cache = {}
|
|
order = []
|
|
|
|
def read(satellite, when):
|
|
key = (satellite, when)
|
|
if key in cache:
|
|
return cache[key]
|
|
bands = []
|
|
for wavelength in wavelengths:
|
|
slot = (satellite, wavelength, when)
|
|
path = overlay.path(slot)
|
|
# Ticked per *read*, not per target. One target pulls ~90 frames through
|
|
# here, so ticking once per target would undercount by ninety-fold and the
|
|
# reliever would never reach its interval -- which is exactly how the mount
|
|
# runs out of file handles.
|
|
if reliever is not None:
|
|
reliever.tick()
|
|
image, _ = fitsio.read_image(path) if path else (None, None)
|
|
if image is None:
|
|
bands = None
|
|
break
|
|
bands.append(image)
|
|
value = np.stack(bands).astype(np.float32) if bands else None
|
|
cache[key] = value
|
|
order.append(key)
|
|
while len(order) > limit:
|
|
cache.pop(order.pop(0), None)
|
|
return value
|
|
|
|
return read
|
|
|
|
|
|
def _closest(times, target):
|
|
"""Closest time in sorted `times` to `target`, in either direction."""
|
|
index = bisect.bisect_left(times, target)
|
|
candidates = times[max(0, index - 1) : index + 1]
|
|
return min(candidates, key=lambda t: abs(t - target))
|
|
|
|
|
|
def _nearest(times, target, before=True):
|
|
"""Closest time in `times` strictly before or after `target`."""
|
|
if before:
|
|
candidates = [t for t in times if t < target]
|
|
return max(candidates) if candidates else None
|
|
candidates = [t for t in times if t > target]
|
|
return min(candidates) if candidates else None
|
|
|
|
|
|
def _header_for(overlay, slot):
|
|
path = overlay.truth_path(slot)
|
|
if path is None:
|
|
return {}
|
|
values, _ = fitsio.scan_header(path)
|
|
return values
|
|
|
|
|
|
# -------------------------------------------------------------------------- compare
|
|
|
|
|
|
def cmd_compare(args):
|
|
"""Fill quality by gap length, on one common set of frames.
|
|
|
|
Restricting to the frames a chosen filler actually scored is the point. A model
|
|
evaluated on 200 sampled targets and a baseline evaluated on 5,304 are not
|
|
comparable numbers, and putting them in the same table implies they are. With
|
|
--restrict-to the table is one set of frames scored several ways.
|
|
"""
|
|
conn = db.connect(args.db, readonly=True)
|
|
case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone()
|
|
if case is None:
|
|
raise SystemExit(f"No case named {args.case!r}")
|
|
|
|
gaps = {row["frame_id"]: row["gap_length"] for row in conn.execute(
|
|
"SELECT frame_id, gap_length FROM bench_injection WHERE case_id = ?", (case["id"],)
|
|
)}
|
|
|
|
rows = conn.execute(
|
|
"SELECT frame_id, filler, scores_json FROM bench_fill_result "
|
|
"WHERE case_id = ? AND source = ?", (case["id"], args.source)
|
|
).fetchall()
|
|
if not rows:
|
|
raise SystemExit(f"No fill results for case {args.case!r} from {args.source!r}")
|
|
|
|
by_filler = defaultdict(dict)
|
|
for row in rows:
|
|
by_filler[row["filler"]][row["frame_id"]] = json.loads(row["scores_json"])
|
|
|
|
common = None
|
|
if args.restrict_to:
|
|
if args.restrict_to not in by_filler:
|
|
raise SystemExit(f"No results for filler {args.restrict_to!r}")
|
|
common = set(by_filler[args.restrict_to])
|
|
for name, scores in by_filler.items():
|
|
common &= set(scores)
|
|
print(f"Restricted to the {len(common)} frames scored by "
|
|
f"every filler including '{args.restrict_to}'")
|
|
|
|
lengths = sorted({gaps.get(frame_id, 0)
|
|
for scores in by_filler.values() for frame_id in scores
|
|
if common is None or frame_id in common})
|
|
metric = args.metric
|
|
|
|
table = {}
|
|
for name, scores in by_filler.items():
|
|
cells = defaultdict(list)
|
|
for frame_id, values in scores.items():
|
|
if common is not None and frame_id not in common:
|
|
continue
|
|
cells[gaps.get(frame_id, 0)].append(values[metric])
|
|
table[name] = cells
|
|
|
|
print(f"\nCase '{args.case}', source '{args.source}' -- {metric} by gap length\n")
|
|
print(f"{'filler':>16} " + " ".join(f"{length:>7}" for length in lengths) + f" {'n':>7}")
|
|
order = [n for n in ("hold_last", "linear_blend", "optical_flow", "solar_rotation",
|
|
"crosssat", "learned") if n in table]
|
|
order += [n for n in sorted(table) if n not in order]
|
|
for name in order:
|
|
cells = table[name]
|
|
printed = []
|
|
for length in lengths:
|
|
values = cells.get(length)
|
|
printed.append(f"{sum(values) / len(values):>7.2f}" if values else f"{'--':>7}")
|
|
total = sum(len(v) for v in cells.values())
|
|
print(f"{name:>16} " + " ".join(printed) + f" {total:>7}")
|
|
|
|
# The bar the learned filler has to clear is the best of the others at each gap
|
|
# length, not any single one of them: the crossover is real and both sides win
|
|
# somewhere.
|
|
best = {}
|
|
for length in lengths:
|
|
options = {
|
|
name: sum(cells[length]) / len(cells[length])
|
|
for name, cells in table.items()
|
|
if name != "learned" and cells.get(length)
|
|
}
|
|
if options:
|
|
best[length] = max(options.items(), key=lambda kv: kv[1])
|
|
if best:
|
|
print("\n best non-learned: " + " ".join(
|
|
f"{length}:{name.split('_')[0]}({value:.2f})"
|
|
for length, (name, value) in best.items()))
|
|
if "learned" in table:
|
|
deltas = []
|
|
for length in lengths:
|
|
values = table["learned"].get(length)
|
|
if values and length in best:
|
|
deltas.append(f"{length}:{sum(values) / len(values) - best[length][1]:+.2f}")
|
|
print(" learned minus best: " + " ".join(deltas))
|
|
return 0
|
|
|
|
|
|
# --------------------------------------------------------------------------- report
|
|
|
|
|
|
def cmd_report(args):
|
|
conn = db.connect(args.db, readonly=True)
|
|
case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone()
|
|
if case is None:
|
|
raise SystemExit(f"No case named {args.case!r}")
|
|
window = conn.execute(
|
|
"SELECT * FROM bench_window WHERE id = ?", (case["window_id"],)
|
|
).fetchone()
|
|
|
|
plan = json.loads(case["plan_json"])
|
|
print(f"Case '{args.case}' on window '{window['name']}' "
|
|
f"({format_time(window['t_start'])} .. {format_time(window['t_end'])})")
|
|
print(f" plan: {plan}\n")
|
|
|
|
injected = {}
|
|
for row in conn.execute(
|
|
"""
|
|
SELECT f.satellite, f.wavelength, f.t_start, i.mode
|
|
FROM bench_injection i JOIN frame f ON f.id = i.frame_id WHERE i.case_id = ?
|
|
""",
|
|
(case["id"],),
|
|
):
|
|
injected[(row["satellite"], row["wavelength"], row["t_start"])] = row["mode"]
|
|
|
|
legacy_good = {
|
|
(row["satellite"], row["wavelength"], row["t_start"])
|
|
for row in conn.execute(
|
|
"""
|
|
SELECT f.satellite, f.wavelength, f.t_start FROM bench_truth t
|
|
JOIN frame f ON f.id = t.frame_id
|
|
WHERE t.window_id = ? AND t.is_good = 1
|
|
""",
|
|
(window["id"],),
|
|
)
|
|
}
|
|
|
|
print("DETECTION")
|
|
print(f"{'detector':>14} {'prec':>7} {'recall':>7} {'F1':>7} {'FPR':>8} "
|
|
f"{'unknown':>8} {'legacy-dis':>11}")
|
|
per_detector = {}
|
|
verdict_maps = {}
|
|
for name in detectors.ALL_DETECTORS:
|
|
run_id = db.latest_run_id(conn, name, {"case": args.case})
|
|
if run_id is None:
|
|
continue
|
|
verdicts = {}
|
|
for row in conn.execute(
|
|
"""
|
|
SELECT f.satellite, f.wavelength, f.t_start, d.verdict, d.reason
|
|
FROM detection d JOIN frame f ON f.id = d.frame_id WHERE d.run_id = ?
|
|
""",
|
|
(run_id,),
|
|
):
|
|
slot = (row["satellite"], row["wavelength"], row["t_start"])
|
|
verdicts[slot] = detectors.Verdict(row["verdict"], row["reason"], {})
|
|
# Deleted slots have no frame to judge, so they cannot be scored here.
|
|
scoreable = {s: v for s, v in verdicts.items() if injected.get(s) != "delete"}
|
|
score = metrics.score_detection(
|
|
scoreable,
|
|
{s: m for s, m in injected.items() if m != "delete"},
|
|
legacy_good,
|
|
)
|
|
per_detector[name] = score
|
|
verdict_maps[name] = verdicts
|
|
print(f"{name:>14} {score.precision:>7.3f} {score.recall:>7.3f} {score.f1:>7.3f} "
|
|
f"{score.false_positive_rate:>8.4f} {score.unknown:>8} "
|
|
f"{score.legacy_disagreements:>11}")
|
|
|
|
if args.combinations and len(per_detector) > 1:
|
|
print("\nDETECTOR COMBINATIONS (ranked by false-positive rate, then F1)")
|
|
print(f"{'policy':>9} {'detectors':>44} {'prec':>7} {'recall':>7} {'F1':>7} {'FPR':>8}")
|
|
scoreable_injected = {s: m for s, m in injected.items() if m != "delete"}
|
|
rows = []
|
|
names = sorted(per_detector)
|
|
for size in range(2, len(names) + 1):
|
|
for subset in itertools.combinations(names, size):
|
|
for policy in metrics.COMBINATION_POLICIES:
|
|
combined = metrics.combine_verdicts(
|
|
[verdict_maps[name] for name in subset], policy
|
|
)
|
|
combined = {
|
|
s: v for s, v in combined.items() if injected.get(s) != "delete"
|
|
}
|
|
score = metrics.score_detection(
|
|
combined, scoreable_injected, legacy_good
|
|
)
|
|
rows.append((score.false_positive_rate, -score.f1, policy, subset, score))
|
|
rows.sort(key=lambda r: (r[0] if np.isfinite(r[0]) else 9e9,
|
|
r[1] if np.isfinite(r[1]) else 9e9))
|
|
for _, _, policy, subset, score in rows[: args.combination_limit]:
|
|
label = "+".join(n.replace("_v1", "") for n in subset)
|
|
print(f"{policy:>9} {label:>44} {score.precision:>7.3f} {score.recall:>7.3f} "
|
|
f"{score.f1:>7.3f} {score.false_positive_rate:>8.4f}")
|
|
|
|
modes = sorted({m for m in injected.values() if m != "delete"})
|
|
if modes and per_detector:
|
|
print("\nRECALL BY CORRUPTION MODE")
|
|
header = f"{'mode':>18}" + "".join(f"{n[:11]:>13}" for n in per_detector)
|
|
print(header)
|
|
for mode in modes:
|
|
line = f"{mode:>18}"
|
|
for score in per_detector.values():
|
|
value = score.recall_by_mode.get(mode)
|
|
line += f"{value:>13.3f}" if value is not None else f"{'-':>13}"
|
|
print(line)
|
|
|
|
print("\nFILL")
|
|
rows = conn.execute(
|
|
"SELECT filler, source, scores_json FROM bench_fill_result WHERE case_id = ?",
|
|
(case["id"],),
|
|
).fetchall()
|
|
if not rows:
|
|
print(" (no fill results; run `bench.py fill` first)")
|
|
else:
|
|
grouped = defaultdict(list)
|
|
for row in rows:
|
|
payload = json.loads(row["scores_json"])
|
|
grouped[(row["source"], row["filler"])].append(
|
|
metrics.FillScore(**payload)
|
|
)
|
|
print(f"{'source':>10} {'filler':>16} {'n':>6} {'rmse':>10} {'psnr':>8} {'ssim':>7}")
|
|
for (source, filler_name), scores in sorted(grouped.items()):
|
|
summary = metrics.summarise_fills(scores)
|
|
print(f"{source:>10} {filler_name:>16} {summary['n']:>6} "
|
|
f"{summary['rmse']:>10.4g} {summary['psnr']:>8.2f} {summary['ssim']:>7.4f}")
|
|
|
|
print("\nFILL QUALITY BY GAP LENGTH (psnr, dB)")
|
|
gaps = sorted({s.gap_frames for scores in grouped.values() for s in scores})
|
|
print(f"{'source/filler':>28}" + "".join(f"{g:>8}" for g in gaps))
|
|
for (source, filler_name), scores in sorted(grouped.items()):
|
|
summary = metrics.summarise_fills(scores)
|
|
line = f"{source + '/' + filler_name:>28}"
|
|
for gap in gaps:
|
|
entry = summary["by_gap"].get(gap)
|
|
line += f"{entry['psnr']:>8.2f}" if entry else f"{'-':>8}"
|
|
print(line)
|
|
|
|
if args.json:
|
|
output = {
|
|
"case": args.case,
|
|
"window": window["name"],
|
|
"plan": plan,
|
|
"detection": {n: s.as_dict() for n, s in per_detector.items()},
|
|
}
|
|
with open(args.json, "w") as handle:
|
|
json.dump(output, handle, indent=2)
|
|
print(f"\nWrote {args.json}")
|
|
return 0
|
|
|
|
|
|
# --------------------------------------------------------------------------- render
|
|
|
|
#: Slots the current pipeline will carry forward from the last good frame. Beyond
|
|
#: this merger_FITS.py gives up and the timestamp produces no composite at all.
|
|
TODAY_MAX_TIME_GAP = 3
|
|
|
|
|
|
def _resolve_band(overlay, slot, truth_cache, use_truth=False):
|
|
"""Read one band's pixels, caching, or None if it cannot be read.
|
|
|
|
`use_truth` reads the original archive frame rather than what the case left in
|
|
its place. The pristine variant needs it: a slot the case *deleted* has no
|
|
overlay path at all, and resolving those through the overlay silently drops them
|
|
-- which cost the ground-truth pane 221 of its timestamps, exactly the ones the
|
|
other panes have to be compared against.
|
|
"""
|
|
key = (slot, use_truth)
|
|
if key not in truth_cache:
|
|
path = overlay.truth_path(slot) if use_truth else overlay.path(slot)
|
|
image, _ = fitsio.read_image(path) if path else (None, None)
|
|
truth_cache[key] = image
|
|
return truth_cache[key]
|
|
|
|
|
|
def _render_one(job):
|
|
"""Composite a single timestamp for one satellite. Runs in a worker."""
|
|
import merger_FITS
|
|
|
|
arrays, timestamp, out_path = job
|
|
try:
|
|
image = merger_FITS.composite_from_arrays(arrays, timestamp)
|
|
image.save(out_path, quality=95)
|
|
return out_path, None
|
|
except Exception as exc:
|
|
return out_path, f"{type(exc).__name__}: {exc}"
|
|
|
|
|
|
def cmd_render(args):
|
|
"""Produce the composite stream for one variant of a case.
|
|
|
|
Three variants, which together answer 'is the repair visible, and is it better
|
|
than what we do now':
|
|
|
|
* ``pristine`` -- the archive untouched. Ground truth.
|
|
* ``today`` -- what the current pipeline does with the injected damage: carry
|
|
the last good frame across gaps of up to TODAY_MAX_TIME_GAP slots, and emit
|
|
*nothing at all* for longer ones, which is what leaves ffmpeg_video.py padding
|
|
with black frames. A 300-slot gap produces 300 missing composites.
|
|
* ``new`` -- the chosen method: any(disc_v1 + header_v1) detection, filled
|
|
with optical_flow at every gap length.
|
|
"""
|
|
fitsio.quiet_astropy()
|
|
conn = db.connect(args.db)
|
|
case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone()
|
|
if case is None:
|
|
raise SystemExit(f"No case named {args.case!r}")
|
|
window = conn.execute(
|
|
"SELECT * FROM bench_window WHERE id = ?", (case["window_id"],)
|
|
).fetchone()
|
|
satellites = tuple(json.loads(window["satellites"]))
|
|
if args.satellites:
|
|
wanted = parse_int_list(args.satellites)
|
|
satellites = tuple(s for s in satellites if s in wanted)
|
|
if not satellites:
|
|
raise SystemExit(f"none of {wanted} are in this window")
|
|
wavelengths = tuple(json.loads(window["wavelengths"]))
|
|
if len(wavelengths) != 6:
|
|
raise SystemExit(f"a composite needs all six bands; window has {wavelengths}")
|
|
overlay = _build_overlay(conn, case, window)
|
|
|
|
# Pristine shows the window as it really is, so it reads the archive directly and
|
|
# treats nothing as damaged.
|
|
pristine = args.variant == "pristine"
|
|
bad = set() if pristine else _bad_slots(conn, case, args.detector)
|
|
times = sorted({slot[2] for slot in overlay.archive})
|
|
print(f"{args.variant}: {len(times)} timestamps x {len(satellites)} satellites, "
|
|
f"{len(bad)} slots treated as bad")
|
|
|
|
for satellite in satellites:
|
|
out_dir = os.path.join(args.out, args.variant, f"goes{satellite}")
|
|
os.makedirs(out_dir, exist_ok=True)
|
|
truth_cache = {}
|
|
jobs = []
|
|
skipped = 0
|
|
reliever = vfs.Reliever(label=f"render g{satellite}")
|
|
|
|
for when in times:
|
|
reliever.tick(len(wavelengths))
|
|
slots = [(satellite, band, when) for band in wavelengths]
|
|
arrays = []
|
|
usable = True
|
|
for slot in slots:
|
|
if not pristine and (slot in bad or overlay.path(slot) is None):
|
|
replacement, ok = _replacement_band(
|
|
args.variant, overlay, slot, bad, truth_cache, wavelengths
|
|
)
|
|
if not ok:
|
|
usable = False
|
|
break
|
|
arrays.append(replacement)
|
|
else:
|
|
image = _resolve_band(overlay, slot, truth_cache, use_truth=pristine)
|
|
if image is None:
|
|
usable = False
|
|
break
|
|
arrays.append(image)
|
|
|
|
if not usable:
|
|
# No composite for this timestamp -- exactly what the current
|
|
# pipeline does past its gap limit, and what ffmpeg then pads.
|
|
skipped += 1
|
|
continue
|
|
jobs.append((arrays, when, os.path.join(out_dir, f"Composite-{int(when)}.jpg")))
|
|
|
|
if len(jobs) >= args.batch:
|
|
_flush_renders(jobs, args.workers)
|
|
jobs.clear()
|
|
truth_cache.clear()
|
|
print(f" g{satellite}: {len(os.listdir(out_dir))} rendered, {skipped} skipped")
|
|
|
|
if jobs:
|
|
_flush_renders(jobs, args.workers)
|
|
print(f" g{satellite}: {len(os.listdir(out_dir))} composites, {skipped} timestamps skipped")
|
|
conn.close()
|
|
return 0
|
|
|
|
|
|
def _replacement_band(variant, overlay, slot, bad, truth_cache, wavelengths):
|
|
"""What to substitute for a band the case removed. Returns (array, usable)."""
|
|
satellite, band, when = slot
|
|
series = [s for s in overlay.series(satellite, band) if s not in bad]
|
|
times = [s[2] for s in series]
|
|
before = _nearest(times, when, before=True)
|
|
after = _nearest(times, when, before=False)
|
|
|
|
if variant == "today":
|
|
# merger_FITS carries the last good frame a few slots, then gives up.
|
|
if before is None or (when - before) // paths.CADENCE > TODAY_MAX_TIME_GAP:
|
|
return None, False
|
|
return _resolve_band(overlay, (satellite, band, before), truth_cache), True
|
|
|
|
context = fillers.FillContext(
|
|
before=_resolve_band(overlay, (satellite, band, before), truth_cache) if before else None,
|
|
dt_before=when - before if before else 0.0,
|
|
after=_resolve_band(overlay, (satellite, band, after), truth_cache) if after else None,
|
|
dt_after=after - when if after else 0.0,
|
|
)
|
|
filled = fillers.optical_flow(context)
|
|
return (filled, True) if filled is not None else (None, False)
|
|
|
|
|
|
def _flush_renders(jobs, workers):
|
|
failures = 0
|
|
with Pool(workers, initializer=fitsio.quiet_astropy) as pool:
|
|
for _, error in pool.imap_unordered(_render_one, jobs, 4):
|
|
failures += bool(error)
|
|
if failures:
|
|
print(f" WARNING: {failures} composites failed to render")
|
|
|
|
|
|
# ------------------------------------------------------------------------- examples
|
|
|
|
|
|
def cmd_examples(args):
|
|
"""Show the frames a detector got wrong, so a number becomes a picture.
|
|
|
|
A precision/recall table says how often a detector is wrong; it does not say
|
|
what it is wrong *about*. For `header_v1`, whose recall is low but precision
|
|
perfect, the interesting set is the false negatives -- damage that leaves
|
|
IMG_MEAN, ECLIPSE and EMPTY all looking normal.
|
|
"""
|
|
import matplotlib
|
|
|
|
matplotlib.use("Agg")
|
|
from matplotlib import pyplot as plt
|
|
|
|
fitsio.quiet_astropy()
|
|
conn = db.connect(args.db, readonly=True)
|
|
case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone()
|
|
if case is None:
|
|
raise SystemExit(f"No case named {args.case!r}")
|
|
window = conn.execute(
|
|
"SELECT * FROM bench_window WHERE id = ?", (case["window_id"],)
|
|
).fetchone()
|
|
overlay = _build_overlay(conn, case, window)
|
|
|
|
run_id = db.latest_run_id(conn, args.detector, {"case": args.case})
|
|
if run_id is None:
|
|
raise SystemExit(
|
|
f"No run of {args.detector!r} for case {args.case!r}; run detect first."
|
|
)
|
|
|
|
injected = {}
|
|
for row in conn.execute(
|
|
"""
|
|
SELECT f.satellite, f.wavelength, f.t_start, i.mode, i.severity
|
|
FROM bench_injection i JOIN frame f ON f.id = i.frame_id WHERE i.case_id = ?
|
|
""",
|
|
(case["id"],),
|
|
):
|
|
injected[(row["satellite"], row["wavelength"], row["t_start"])] = (
|
|
row["mode"], row["severity"]
|
|
)
|
|
|
|
wanted = []
|
|
for row in conn.execute(
|
|
"""
|
|
SELECT f.satellite, f.wavelength, f.t_start, d.verdict, d.reason, d.scores_json
|
|
FROM detection d JOIN frame f ON f.id = d.frame_id WHERE d.run_id = ?
|
|
""",
|
|
(run_id,),
|
|
):
|
|
slot = (row["satellite"], row["wavelength"], row["t_start"])
|
|
mode = injected.get(slot, (None, None))[0]
|
|
if mode == "delete":
|
|
continue # no frame to show
|
|
corrupted = mode is not None
|
|
if args.kind == "fn" and corrupted and row["verdict"] == "good":
|
|
wanted.append((slot, mode, row))
|
|
elif args.kind == "fp" and not corrupted and row["verdict"] == "bad":
|
|
wanted.append((slot, None, row))
|
|
|
|
print(f"{args.detector} {args.kind.upper()}s on case '{args.case}': {len(wanted)}")
|
|
if not wanted:
|
|
print(" none -- nothing to plot.")
|
|
return 0
|
|
|
|
by_mode = defaultdict(int)
|
|
for _, mode, _ in wanted:
|
|
by_mode[mode or "(uninjected)"] += 1
|
|
for mode, count in sorted(by_mode.items(), key=lambda kv: -kv[1]):
|
|
print(f" {count:>4} {mode}")
|
|
|
|
wanted = wanted[: args.limit]
|
|
columns = min(6, len(wanted))
|
|
rows = (len(wanted) + columns - 1) // columns
|
|
figure, axes = plt.subplots(rows, columns, figsize=(3 * columns, 3.4 * rows))
|
|
axes = np.atleast_2d(axes)
|
|
for index, (slot, mode, row) in enumerate(wanted):
|
|
axis = axes[index // columns][index % columns]
|
|
path = overlay.path(slot)
|
|
image, _ = fitsio.read_image(path) if path else (None, None)
|
|
if image is not None:
|
|
axis.imshow(metrics.to_display(image, slot[1]), cmap="gray", vmin=0, vmax=1)
|
|
scores = json.loads(row["scores_json"] or "{}")
|
|
shown = ", ".join(f"{k}={v:.3g}" for k, v in list(scores.items())[:3])
|
|
axis.set_title(
|
|
f"g{slot[0]} {slot[1]}A {format_time(slot[2])[5:]}\n"
|
|
f"{mode or 'clean'} -> {row['verdict']}\n{shown}",
|
|
fontsize=6,
|
|
)
|
|
axis.axis("off")
|
|
for index in range(len(wanted), rows * columns):
|
|
axes[index // columns][index % columns].axis("off")
|
|
figure.suptitle(f"{args.detector} {args.kind.upper()}s - case {args.case}")
|
|
figure.tight_layout()
|
|
figure.savefig(args.out, dpi=150)
|
|
plt.close(figure)
|
|
print(f" wrote {args.out}")
|
|
return 0
|
|
|
|
|
|
# ------------------------------------------------------------------------------ cli
|
|
|
|
|
|
def build_parser():
|
|
parser = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
parser.add_argument("--db", default=None, help="SQLite index (default: $SUVI_DB)")
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
def add_window_args(sub):
|
|
sub.add_argument("--start", type=parse_date, required=True)
|
|
sub.add_argument("--end", type=parse_date, required=True)
|
|
sub.add_argument("--satellites", default="16,18")
|
|
sub.add_argument("--wavelengths", default="94,131,171,195,284,304")
|
|
|
|
select = subparsers.add_parser("select-window", help="find clean stretches of archive")
|
|
add_window_args(select)
|
|
select.add_argument("--min-slots", type=int, default=120)
|
|
select.add_argument("--limit", type=int, default=10)
|
|
select.add_argument("--require-legacy-label", action="store_true",
|
|
help="also require the stale _f suffix (off by default)")
|
|
select.set_defaults(func=cmd_select_window)
|
|
|
|
vet = subparsers.add_parser("vet-window", help="verify and freeze a ground-truth window")
|
|
add_window_args(vet)
|
|
vet.add_argument("--name", required=True)
|
|
vet.add_argument("--notes", default=None)
|
|
vet.add_argument("--allow-missing", action="store_true")
|
|
vet.add_argument("--min-good-fraction", type=float, default=0.98)
|
|
vet.add_argument("--contact-sheet", default=None, help="write a filmstrip PNG for review")
|
|
vet.add_argument("--progress", action="store_true")
|
|
vet.set_defaults(func=cmd_vet_window)
|
|
|
|
make = subparsers.add_parser("make-case", help="inject damage into a vetted window")
|
|
make.add_argument("--window", required=True)
|
|
make.add_argument("--name", required=True)
|
|
make.add_argument("--seed", type=int, default=42)
|
|
make.add_argument("--fraction", type=float, default=0.10)
|
|
make.add_argument("--gap-lengths", default="1,2,3,5,10,30,60")
|
|
make.add_argument("--gaps-per-length", type=int, default=None,
|
|
help="place exactly this many gaps at each length, instead of "
|
|
"sizing the case by --fraction")
|
|
make.add_argument("--satellite-scope", default="mixed",
|
|
choices=("g16", "g18", "both", "mixed"))
|
|
make.add_argument("--wavelength-scope", default="all", choices=("all", "one"))
|
|
make.add_argument("--modes", default=None, help="comma-separated; default is all")
|
|
make.add_argument("--severity-low", type=float, default=0.5)
|
|
make.add_argument("--severity-high", type=float, default=1.0)
|
|
make.add_argument("--overlay-dir", default=None)
|
|
make.add_argument("--workers", type=int, default=8)
|
|
make.add_argument("--progress", action="store_true")
|
|
make.set_defaults(func=cmd_make_case)
|
|
|
|
detect = subparsers.add_parser("detect", help="run detectors over a case")
|
|
detect.add_argument("--case", required=True)
|
|
detect.add_argument("--detectors", default=None, help="comma-separated; default is all")
|
|
detect.add_argument("--workers", type=int, default=8)
|
|
detect.set_defaults(func=cmd_detect)
|
|
|
|
fill = subparsers.add_parser("fill", help="reconstruct flagged slots and score them")
|
|
fill.add_argument("--case", required=True)
|
|
fill.add_argument("--fillers", default=None, help="comma-separated; default is all")
|
|
fill.add_argument("--max-targets", type=int, default=None,
|
|
help="score only this many targets, spread evenly over gap "
|
|
"lengths -- for a quick read on a model mid-training")
|
|
group = fill.add_mutually_exclusive_group(required=True)
|
|
group.add_argument("--oracle", action="store_true",
|
|
help="use perfect knowledge of which slots are bad")
|
|
group.add_argument("--from-detector", default=None,
|
|
help="use a detector's output, showing end-to-end error")
|
|
fill.set_defaults(func=cmd_fill)
|
|
|
|
compare = subparsers.add_parser(
|
|
"compare", help="fill quality by gap length across fillers, on common frames")
|
|
compare.add_argument("--case", required=True)
|
|
compare.add_argument("--source", default="oracle")
|
|
compare.add_argument("--metric", default="psnr",
|
|
choices=("psnr", "ssim", "rmse", "log_rmse"))
|
|
compare.add_argument("--restrict-to", default=None,
|
|
help="score every filler on only the frames this one scored")
|
|
compare.set_defaults(func=cmd_compare)
|
|
|
|
report = subparsers.add_parser("report", help="summarise a case's results")
|
|
report.add_argument("--case", required=True)
|
|
report.add_argument("--json", default=None)
|
|
report.add_argument("--combinations", action="store_true",
|
|
help="also score every combination of detectors")
|
|
report.add_argument("--combination-limit", type=int, default=15)
|
|
report.set_defaults(func=cmd_report)
|
|
|
|
render = subparsers.add_parser(
|
|
"render", help="produce the composite stream for one variant of a case"
|
|
)
|
|
render.add_argument("--case", required=True)
|
|
render.add_argument("--variant", required=True,
|
|
choices=("pristine", "today", "new"))
|
|
render.add_argument("--out", required=True, help="root directory for the streams")
|
|
render.add_argument("--detector", default="oracle",
|
|
help="which slots to treat as bad ('oracle', or a run name)")
|
|
render.add_argument("--satellites", default=None,
|
|
help="restrict to these satellites (default: all in the window)")
|
|
render.add_argument("--workers", type=int, default=8)
|
|
render.add_argument("--batch", type=int, default=48,
|
|
help="composites per worker dispatch; bounds peak memory")
|
|
render.set_defaults(func=cmd_render)
|
|
|
|
examples = subparsers.add_parser(
|
|
"examples", help="plot the frames a detector got wrong"
|
|
)
|
|
examples.add_argument("--case", required=True)
|
|
examples.add_argument("--detector", required=True)
|
|
examples.add_argument("--kind", choices=("fp", "fn"), required=True)
|
|
examples.add_argument("--limit", type=int, default=12)
|
|
examples.add_argument("--out", default="examples.png")
|
|
examples.set_defaults(func=cmd_examples)
|
|
return parser
|
|
|
|
|
|
def main(argv=None):
|
|
args = build_parser().parse_args(argv)
|
|
return args.func(args) or 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|