jeremy/noaa-goes-visualization
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
noaa-goes-visualization/bench.py

1299 lines
52 KiB
Python
Raw Normal View History

Rewrote processing pipeline including bad-image detection with multiple overlapping methods (header + disc based detection seems best for now).
2026-09-01 22:36:13 -04:00
#!/usr/bin/env python
"""Test bench for bad-frame detection and gap filling.
Takes a stretch of archive verified to be clean, damages it in known ways, and
measures how well each detector finds the damage and each filler repairs it.
The archive is never modified. Corrupted frames are written to a per-case overlay
directory and every read goes through :class:`suvi.cases.Overlay`, so a bench run
against live production data is safe.
Workflow::
bench.py select-window --start 2024-05-05 --end 2024-05-20
bench.py vet-window --name may2024 --start ... --end ...
bench.py make-case --window may2024 --name mixed10 --seed 42
bench.py detect --case mixed10
bench.py fill --case mixed10 --oracle
bench.py report --case mixed10
Each step records its results in the index, so steps are resumable and comparable
across runs.
"""
import argparse
import datetime
import itertools
import json
import os
import sys
import time
from collections import defaultdict
from multiprocessing import Pool
import numpy as np
from astropy.io import fits
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from suvi import cases, corruptions, db, detectors, fillers, fitsio, metrics, paths, vfs
def parse_date(text):
"""Accept YYYY-MM-DD or YYYY-MM-DDTHH:MM, always UTC."""
for fmt in ("%Y-%m-%d", "%Y-%m-%dT%H:%M", "%Y-%m-%d %H:%M"):
try:
stamp = datetime.datetime.strptime(text, fmt)
return int(stamp.replace(tzinfo=datetime.timezone.utc).timestamp())
except ValueError:
continue
raise argparse.ArgumentTypeError(f"Not a date: {text!r}")
def format_time(timestamp):
return datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc).strftime(
"%Y-%m-%d %H:%M"
)
def parse_int_list(text):
return tuple(int(part) for part in text.split(",") if part.strip())
# ------------------------------------------------------------------- select-window
def cmd_select_window(args):
root = paths.data_root()
satellites = parse_int_list(args.satellites)
wavelengths = parse_int_list(args.wavelengths)
print(f"Scanning {root} from {format_time(args.start)} to {format_time(args.end)}")
found = cases.scan_window(root, satellites, wavelengths, args.start, args.end)
total = len(cases.timeline(args.start, args.end))
print(f" {len(found)} files across {total} slots\n")
runs = cases.find_runs(
found,
satellites,
wavelengths,
args.start,
args.end,
minimum=args.min_slots,
require_good_label=args.require_legacy_label,
)
if not runs:
print(f"No runs of >= {args.min_slots} slots complete on every requested band.")
print("Try fewer bands, one satellite, or a different date range.")
return 1
basis = "labelled good by the legacy filter" if args.require_legacy_label else "present"
print(f"Runs where every requested band and satellite is {basis}:")
print(f"{'slots':>7} {'hours':>8} window")
for length, start, end in runs[: args.limit]:
print(f"{length:>7} {length * paths.CADENCE / 3600:>8.1f} {format_time(start)} .. {format_time(end)}")
if not args.require_legacy_label:
print("\nCompleteness only -- run vet-window to verify quality from the headers.")
return 0
# ---------------------------------------------------------------------- vet-window
def _header_vet(path):
"""Independent check that a frame really is good, from its header."""
values, error = fitsio.scan_header(path)
if error:
return False, f"header unreadable: {error}"
if values.get("empty"):
return False, "EMPTY set"
if values.get("eclipse"):
return False, f"ECLIPSE={int(values['eclipse'])}"
return True, None
def cmd_vet_window(args):
root = paths.data_root()
satellites = parse_int_list(args.satellites)
wavelengths = parse_int_list(args.wavelengths)
conn = db.connect(args.db)
found = cases.scan_window(root, satellites, wavelengths, args.start, args.end)
slots = cases.timeline(args.start, args.end)
expected = len(slots) * len(satellites) * len(wavelengths)
print(f"Window has {len(found)}/{expected} frames present")
missing = [
(satellite, wavelength, time)
for time in slots
for satellite in satellites
for wavelength in wavelengths
if (satellite, wavelength, time) not in found
]
if missing and not args.allow_missing:
print(f"ERROR: {len(missing)} slots have no frame at all; refusing to vet.")
print(" Pick a window from select-window, or pass --allow-missing.")
return 1
# Ground truth comes from the headers, which the instrument wrote and which are
# independent of any pixel filter. The legacy filename labels are recorded for
# comparison but deliberately do not decide anything: they were written in May
# 2024 by a filter that was retuned that July, so they are neither current nor
# self-consistent. Treating them as truth would bake the old filter's mistakes
# into every score computed against this window.
print("Vetting from headers...")
disagreements = []
truth = {}
reliever = vfs.Reliever(label="vet")
for index, (slot, (path, label)) in enumerate(sorted(found.items())):
reliever.tick()
good_by_header, reason = _header_vet(path)
good_by_legacy = label == "f"
if good_by_header != good_by_legacy:
disagreements.append((slot, label, reason))
truth[slot] = (good_by_header, "header")
if args.progress and index % 2000 == 0 and index:
print(f" {index}/{len(found)}")
print(f" {len(disagreements)} frames where the legacy label disagrees with the header")
for slot, label, reason in disagreements[:10]:
print(f" {slot} legacy={label} header={reason or 'good'}")
good = sum(1 for is_good, _ in truth.values() if is_good)
print(f" {good}/{len(truth)} frames vetted good")
if good < len(truth) * args.min_good_fraction:
print(f"ERROR: only {good / len(truth):.1%} vetted good; window is not clean enough.")
return 1
conn.execute(
"""
INSERT INTO bench_window (name, t_start, t_end, satellites, wavelengths, vetted_at, notes)
VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (name) DO UPDATE SET
t_start = excluded.t_start, t_end = excluded.t_end,
satellites = excluded.satellites, wavelengths = excluded.wavelengths,
vetted_at = excluded.vetted_at, notes = excluded.notes
""",
(
args.name,
args.start,
args.end,
json.dumps(list(satellites)),
json.dumps(list(wavelengths)),
time.time(),
args.notes,
),
)
window_id = conn.execute(
"SELECT id FROM bench_window WHERE name = ?", (args.name,)
).fetchone()["id"]
conn.execute("DELETE FROM bench_truth WHERE window_id = ?", (window_id,))
for slot, (path, label) in sorted(found.items()):
name = paths.parse_frame_filename(os.path.basename(path))
relpath = name.relpath()
frame_id = db.upsert_frame(
conn, name, relpath, os.path.getsize(path), os.path.getmtime(path)
)
is_good, source = truth[slot]
conn.execute(
"INSERT OR REPLACE INTO bench_truth (window_id, frame_id, is_good, vet_source)"
" VALUES (?, ?, ?, ?)",
(window_id, frame_id, int(is_good), source),
)
conn.commit()
print(f"Wrote window '{args.name}' (id {window_id}) with {len(found)} frames")
if args.contact_sheet:
_write_contact_sheet(found, args.contact_sheet, satellites, wavelengths)
print(f"Contact sheet: {args.contact_sheet}")
print("Review it before trusting this window as ground truth.")
return 0
def _write_contact_sheet(found, path, satellites, wavelengths, columns=24, size=64):
"""A downsampled filmstrip of the window, for the one manual review pass."""
import matplotlib
matplotlib.use("Agg")
from matplotlib import pyplot as plt
satellite, wavelength = satellites[0], wavelengths[len(wavelengths) // 2]
series = sorted(slot for slot in found if slot[0] == satellite and slot[1] == wavelength)
if not series:
return
step = max(1, len(series) // (columns * 20))
picks = series[::step]
rows = (len(picks) + columns - 1) // columns
figure, axes = plt.subplots(rows, columns, figsize=(columns, rows))
axes = np.atleast_2d(axes)
vmin, vmax, gamma = metrics.DISPLAY_MAPPING[wavelength]
for index, slot in enumerate(picks):
axis = axes[index // columns][index % columns]
image, error = fitsio.read_image(found[slot][0])
if image is not None:
import cv2 as cv
thumb = cv.resize(np.nan_to_num(image), (size, size), interpolation=cv.INTER_AREA)
axis.imshow(metrics.to_display(thumb, wavelength), cmap="gray", vmin=0, vmax=1)
axis.set_title(format_time(slot[2])[5:], fontsize=3)
axis.axis("off")
for index in range(len(picks), rows * columns):
axes[index // columns][index % columns].axis("off")
figure.suptitle(f"g{satellite} {wavelength}A - {len(picks)} of {len(series)} frames")
figure.tight_layout()
figure.savefig(path, dpi=200)
plt.close(figure)
# ------------------------------------------------------------------------ make-case
def _load_window(conn, name):
row = conn.execute("SELECT * FROM bench_window WHERE name = ?", (name,)).fetchone()
if row is None:
raise SystemExit(f"No vetted window named {name!r}; run vet-window first.")
return row
def _build_overlay(conn, case_row, window_row):
"""Reconstruct a case's view of the archive from the index."""
root = paths.data_root()
satellites = tuple(json.loads(window_row["satellites"]))
wavelengths = tuple(json.loads(window_row["wavelengths"]))
archive = cases.scan_window(
root, satellites, wavelengths, window_row["t_start"], window_row["t_end"]
)
overrides, deleted = {}, set()
rows = conn.execute(
"""
SELECT f.satellite, f.wavelength, f.t_start, i.mode, i.overlay_path
FROM bench_injection i JOIN frame f ON f.id = i.frame_id
WHERE i.case_id = ?
""",
(case_row["id"],),
).fetchall()
for row in rows:
slot = (row["satellite"], row["wavelength"], row["t_start"])
if row["mode"] == "delete":
deleted.add(slot)
elif row["overlay_path"]:
overrides[slot] = row["overlay_path"]
return cases.Overlay(archive=archive, overrides=overrides, deleted=frozenset(deleted))
def _write_corrupted(source_path, target_path, injection):
"""Materialise one corrupted frame into the overlay. Returns None on success."""
os.makedirs(os.path.dirname(target_path), exist_ok=True)
corruption = corruptions.CATALOG[injection.mode]
if corruption.kind == "file":
with open(source_path, "rb") as handle:
raw = handle.read()
damaged = corruptions.apply_file(injection.mode, raw, injection.seed, injection.severity)
with open(target_path, "wb") as handle:
handle.write(damaged)
return None
image, error = fitsio.read_image(source_path)
if image is None:
return f"could not read source: {error}"
donor = None
if corruption.needs_donor:
donor = np.roll(image, image.shape[0] // 3, axis=0) # a plausible other frame
damaged, overrides = corruptions.apply_array(
injection.mode, image, injection.seed, injection.severity, donor
)
if corruption.recompute_stats:
overrides = {**overrides, **corruptions.recomputed_stats(damaged)}
with fits.open(source_path) as hdus:
target = next(
(h for h in hdus if getattr(h, "data", None) is not None and h.data.ndim == 2), None
)
if target is None:
return "source has no image HDU"
target.data = damaged.astype(np.float32)
for key, value in overrides.items():
target.header[key] = value
hdus.writeto(target_path, overwrite=True, checksum=True)
return None
def _corrupt_one(job):
source_path, target_path, injection = job
try:
return injection.slot, _write_corrupted(source_path, target_path, injection)
except Exception as exc: # a corruption must never take the whole run down
return injection.slot, f"{type(exc).__name__}: {exc}"
def cmd_make_case(args):
conn = db.connect(args.db)
window = _load_window(conn, args.window)
satellites = tuple(json.loads(window["satellites"]))
wavelengths = tuple(json.loads(window["wavelengths"]))
root = paths.data_root()
archive = cases.scan_window(
root, satellites, wavelengths, window["t_start"], window["t_end"]
)
plan = cases.InjectionPlan(
fraction=args.fraction,
gap_lengths=parse_int_list(args.gap_lengths),
satellite_scope=args.satellite_scope,
wavelength_scope=args.wavelength_scope,
modes=tuple(args.modes.split(",")) if args.modes else cases.InjectionPlan.modes,
severity=(args.severity_low, args.severity_high),
gaps_per_length=args.gaps_per_length,
).validate()
window_times = sorted({slot[2] for slot in archive})
injections = cases.plan_injections(
plan, set(archive), satellites, wavelengths, args.seed
)
if not injections:
print("Plan produced no injections; raise --fraction or widen the window.")
return 1
overlay_dir = args.overlay_dir or os.path.join(
os.path.dirname(root), "bench", args.name
)
os.makedirs(overlay_dir, exist_ok=True)
conn.execute(
"""
INSERT INTO bench_case (window_id, name, seed, plan_json, overlay_dir, created_at)
VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT (name) DO UPDATE SET
window_id = excluded.window_id, seed = excluded.seed,
plan_json = excluded.plan_json, overlay_dir = excluded.overlay_dir,
created_at = excluded.created_at
""",
(
window["id"],
args.name,
args.seed,
json.dumps(plan.as_dict(), sort_keys=True),
overlay_dir,
time.time(),
),
)
case_id = conn.execute(
"SELECT id FROM bench_case WHERE name = ?", (args.name,)
).fetchone()["id"]
conn.execute("DELETE FROM bench_injection WHERE case_id = ?", (case_id,))
jobs, rows = [], []
for injection in injections:
source_path = archive[injection.slot][0]
name = paths.parse_frame_filename(os.path.basename(source_path))
frame_id = db.frame_id_by_slot(conn, *injection.slot)
if frame_id is None:
frame_id = db.upsert_frame(conn, name, name.relpath())
overlay_path = None
if injection.mode != "delete":
overlay_path = os.path.join(overlay_dir, name.relpath())
jobs.append((source_path, overlay_path, injection))
rows.append((case_id, frame_id, injection, overlay_path))
print(f"Planned {len(injections)} injections "
f"({sum(1 for i in injections if i.mode == 'delete')} deletions, {len(jobs)} corruptions)")
failures = {}
if jobs:
reliever = vfs.Reliever(label="make-case")
with Pool(args.workers, initializer=fitsio.quiet_astropy) as pool:
for index, (slot, error) in enumerate(pool.imap_unordered(_corrupt_one, jobs, 8)):
reliever.tick()
if error:
failures[slot] = error
if args.progress and index % 200 == 0 and index:
print(f" wrote {index}/{len(jobs)}")
if failures:
print(f"WARNING: {len(failures)} corruptions failed:")
for slot, error in list(failures.items())[:5]:
print(f" {slot}: {error}")
conn.executemany(
"""
INSERT INTO bench_injection
(case_id, frame_id, mode, severity, params_json, overlay_path, gap_index, gap_length)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
[
(
case_id,
frame_id,
injection.mode,
injection.severity,
json.dumps({"seed": injection.seed}),
overlay_path,
injection.gap_index,
injection.gap_length,
)
for case_id, frame_id, injection, overlay_path in rows
if injection.slot not in failures
],
)
conn.commit()
affected = len({i.slot[2] for i in injections})
print(f"Case '{args.name}' (id {case_id}): {affected}/{len(window_times)} timestamps affected")
print(f"Overlay: {overlay_dir}")
return 0
# --------------------------------------------------------------------------- detect
def _read_features(job):
"""Load one frame's header, thumbnail, and single-frame verdicts."""
slot, path = job
if path is None:
return slot, None, None, None, 0
started = time.perf_counter_ns()
values, header_error = fitsio.scan_header(path)
features = detectors.FrameFeatures(slot=slot, header=values, error=header_error)
header_verdict = detectors.header_v1(features)
header_us = (time.perf_counter_ns() - started) // 1000
image, image_error = fitsio.read_image(path)
started = time.perf_counter_ns()
if image is None:
geometry_verdict = detectors.Verdict("bad", f"unreadable: {image_error}", {})
else:
geometry_verdict = detectors.geometry_v1(image, slot[1])
geometry_us = (time.perf_counter_ns() - started) // 1000
started = time.perf_counter_ns()
if image is None:
disc_verdict = detectors.Verdict("bad", f"unreadable: {image_error}", {})
else:
disc_verdict = detectors.disc_v1(image, slot[1], values)
disc_us = (time.perf_counter_ns() - started) // 1000
features.thumbnail = detectors.thumbnail(image) if image is not None else None
return (
slot,
features,
(header_verdict, header_us),
(geometry_verdict, geometry_us),
(disc_verdict, disc_us),
)
def cmd_detect(args):
fitsio.quiet_astropy()
conn = db.connect(args.db)
case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone()
if case is None:
raise SystemExit(f"No case named {args.case!r}")
window = conn.execute(
"SELECT * FROM bench_window WHERE id = ?", (case["window_id"],)
).fetchone()
satellites = tuple(json.loads(window["satellites"]))
wavelengths = tuple(json.loads(window["wavelengths"]))
overlay = _build_overlay(conn, case, window)
wanted = tuple(args.detectors.split(",")) if args.detectors else detectors.ALL_DETECTORS
runs = {}
for name in wanted:
runs[name] = db.create_detector_run(
conn, name, {"case": args.case}, notes=f"bench case {args.case}"
)
results = {name: {} for name in wanted}
elapsed = defaultdict(int)
for wavelength in wavelengths:
per_satellite = {}
for satellite in satellites:
series_slots = [
slot for slot in overlay.series(satellite, wavelength)
if slot not in overlay.deleted
]
jobs = [(slot, overlay.path(slot)) for slot in series_slots]
features_by_slot = {}
reliever = vfs.Reliever(label="detect")
with Pool(args.workers, initializer=fitsio.quiet_astropy) as pool:
for slot, features, header, geometry, disc in pool.imap(
_read_features, jobs, 8
):
reliever.tick()
if features is None:
continue
features_by_slot[slot] = features
for name, outcome in (
("header_v1", header),
("geometry_v1", geometry),
("disc_v1", disc),
):
if name in results:
results[name][slot] = outcome[0]
elapsed[name] += outcome[1]
ordered = [features_by_slot[s] for s in series_slots if s in features_by_slot]
per_satellite[satellite] = ordered
if "temporal_v1" in results and ordered:
started = time.perf_counter_ns()
verdicts = detectors.temporal_v1(ordered)
elapsed["temporal_v1"] += (time.perf_counter_ns() - started) // 1000
for features, verdict in zip(ordered, verdicts):
results["temporal_v1"][features.slot] = verdict
if "crosssat_v1" in results and len(satellites) >= 2:
first, second = satellites[0], satellites[1]
by_time = {
sat: {f.slot[2]: f for f in per_satellite.get(sat, [])}
for sat in (first, second)
}
temporal = results.get("temporal_v1", {})
started = time.perf_counter_ns()
for when in sorted(set(by_time[first]) | set(by_time[second])):
a, b = by_time[first].get(when), by_time[second].get(when)
verdict_a, verdict_b = detectors.crosssat_v1(
a,
b,
temporal.get(a.slot) if a else None,
temporal.get(b.slot) if b else None,
)
if a is not None:
results["crosssat_v1"][a.slot] = verdict_a
if b is not None:
results["crosssat_v1"][b.slot] = verdict_b
elapsed["crosssat_v1"] += (time.perf_counter_ns() - started) // 1000
print(f" {wavelength}A done")
for name, verdicts in results.items():
payload = []
for slot, verdict in verdicts.items():
frame_id = db.frame_id_by_slot(conn, *slot)
if frame_id is None:
continue
payload.append((frame_id, verdict.verdict, verdict.reason, verdict.scores, None))
db.record_detections(conn, runs[name], payload)
conn.commit()
flagged = sum(1 for v in verdicts.values() if v.verdict == "bad")
print(f"{name:>14}: {len(verdicts)} frames, {flagged} flagged bad, "
f"{elapsed[name] / max(len(verdicts), 1):.0f} us/frame")
return 0
# ----------------------------------------------------------------------------- fill
def _bad_slots(conn, case, source):
"""Slots the fill step should reconstruct, per oracle or per detector.
Detector runs are per case, so the run has to be looked up by this case's
config. Taking "the latest run of header_v1" would silently pick up whichever
case happened to be processed last.
"""
if source == "oracle":
rows = conn.execute(
"""
SELECT f.satellite, f.wavelength, f.t_start
FROM bench_injection i JOIN frame f ON f.id = i.frame_id
WHERE i.case_id = ?
""",
(case["id"],),
).fetchall()
else:
run_id = db.latest_run_id(conn, source, {"case": case["name"]})
if run_id is None:
raise SystemExit(
f"No run of {source!r} for case {case['name']!r}; run detect first."
)
rows = conn.execute(
"""
SELECT f.satellite, f.wavelength, f.t_start
FROM detection d JOIN frame f ON f.id = d.frame_id
WHERE d.run_id = ? AND d.verdict = 'bad'
""",
(run_id,),
).fetchall()
return {(row["satellite"], row["wavelength"], row["t_start"]) for row in rows}
def cmd_fill(args):
fitsio.quiet_astropy()
conn = db.connect(args.db)
case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone()
if case is None:
raise SystemExit(f"No case named {args.case!r}")
window = conn.execute(
"SELECT * FROM bench_window WHERE id = ?", (case["window_id"],)
).fetchone()
satellites = tuple(json.loads(window["satellites"]))
wavelengths = tuple(json.loads(window["wavelengths"]))
overlay = _build_overlay(conn, case, window)
source = "oracle" if args.oracle else args.from_detector
if source is None:
raise SystemExit("Pass either --oracle or --from-detector NAME")
bad = _bad_slots(conn, case, source)
print(f"Filling {len(bad)} slots identified by '{source}'")
gap_lengths = {}
for row in conn.execute(
"""
SELECT f.satellite, f.wavelength, f.t_start, i.gap_length
FROM bench_injection i JOIN frame f ON f.id = i.frame_id WHERE i.case_id = ?
""",
(case["id"],),
):
gap_lengths[(row["satellite"], row["wavelength"], row["t_start"])] = row["gap_length"]
wanted = tuple(args.fillers.split(",")) if args.fillers else tuple(fillers.FILLERS)
reliever = vfs.Reliever(label="fill")
scored = defaultdict(list)
# A slot whose truth frame will not read cannot be scored. Count them: silently
# dropping them yields metrics computed over whatever subset happened to work,
# with nothing in the output to say so. A degraded mount once reduced this to
# 12% coverage while the report looked entirely normal.
unreadable = 0
attempted = 0
payload = []
for wavelength in wavelengths:
truth_cache = {}
def truth_image(slot):
if slot not in truth_cache:
path = overlay.truth_path(slot)
image, _ = fitsio.read_image(path) if path else (None, None)
truth_cache[slot] = image
return truth_cache[slot]
for satellite in satellites:
series = overlay.series(satellite, wavelength)
good = [s for s in series if s not in bad and overlay.path(s) is not None]
good_times = [s[2] for s in good]
for slot in series:
if slot not in bad:
continue
reliever.tick()
attempted += 1
truth = truth_image(slot)
if truth is None:
unreadable += 1
continue
before = _nearest(good_times, slot[2], before=True)
after = _nearest(good_times, slot[2], before=False)
counterpart_slot = next(
(
(other, wavelength, slot[2])
for other in satellites
if other != satellite
and (other, wavelength, slot[2]) not in bad
and overlay.path((other, wavelength, slot[2])) is not None
),
None,
)
context = fillers.FillContext(
before=truth_image((satellite, wavelength, before)) if before else None,
dt_before=slot[2] - before if before else 0.0,
after=truth_image((satellite, wavelength, after)) if after else None,
dt_after=after - slot[2] if after else 0.0,
counterpart=truth_image(counterpart_slot) if counterpart_slot else None,
header=_header_for(overlay, slot),
)
for filler_name in wanted:
filled = fillers.FILLERS[filler_name](context)
if filled is None:
continue
score = metrics.score_fill(
filled, truth, wavelength, gap_lengths.get(slot, context.gap_frames)
)
scored[filler_name].append(score)
frame_id = db.frame_id_by_slot(conn, *slot)
if frame_id is not None:
payload.append(
(case["id"], frame_id, filler_name, source,
json.dumps(score.as_dict()))
)
truth_cache.clear()
print(f" {wavelength}A done")
conn.executemany(
"""
INSERT INTO bench_fill_result (case_id, frame_id, filler, source, scores_json)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT (case_id, frame_id, filler, source)
DO UPDATE SET scores_json = excluded.scores_json
""",
payload,
)
conn.commit()
covered = attempted - unreadable
print(f"\nScored {covered}/{attempted} slots"
+ (f" -- {unreadable} could not be read" if unreadable else ""))
if attempted and covered < attempted * 0.95:
print(f" WARNING: only {covered / attempted:.0%} of damaged slots were scored.")
print(" These numbers describe that subset, not the case. A mount")
print(" that ran out of file handles is the usual cause; re-run.")
print(f"\n{'filler':>16} {'n':>6} {'rmse':>10} {'log rmse':>10} {'psnr':>8} {'ssim':>7}")
for filler_name in wanted:
summary = metrics.summarise_fills(scored.get(filler_name, []))
if not summary:
print(f"{filler_name:>16} {'-- not applicable --':>44}")
continue
print(f"{filler_name:>16} {summary['n']:>6} {summary['rmse']:>10.4g} "
f"{summary['log_rmse']:>10.4g} {summary['psnr']:>8.2f} {summary['ssim']:>7.4f}")
return 0
def _nearest(times, target, before=True):
"""Closest time in `times` strictly before or after `target`."""
if before:
candidates = [t for t in times if t < target]
return max(candidates) if candidates else None
candidates = [t for t in times if t > target]
return min(candidates) if candidates else None
def _header_for(overlay, slot):
path = overlay.truth_path(slot)
if path is None:
return {}
values, _ = fitsio.scan_header(path)
return values
# --------------------------------------------------------------------------- report
def cmd_report(args):
conn = db.connect(args.db, readonly=True)
case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone()
if case is None:
raise SystemExit(f"No case named {args.case!r}")
window = conn.execute(
"SELECT * FROM bench_window WHERE id = ?", (case["window_id"],)
).fetchone()
plan = json.loads(case["plan_json"])
print(f"Case '{args.case}' on window '{window['name']}' "
f"({format_time(window['t_start'])} .. {format_time(window['t_end'])})")
print(f" plan: {plan}\n")
injected = {}
for row in conn.execute(
"""
SELECT f.satellite, f.wavelength, f.t_start, i.mode
FROM bench_injection i JOIN frame f ON f.id = i.frame_id WHERE i.case_id = ?
""",
(case["id"],),
):
injected[(row["satellite"], row["wavelength"], row["t_start"])] = row["mode"]
legacy_good = {
(row["satellite"], row["wavelength"], row["t_start"])
for row in conn.execute(
"""
SELECT f.satellite, f.wavelength, f.t_start FROM bench_truth t
JOIN frame f ON f.id = t.frame_id
WHERE t.window_id = ? AND t.is_good = 1
""",
(window["id"],),
)
}
print("DETECTION")
print(f"{'detector':>14} {'prec':>7} {'recall':>7} {'F1':>7} {'FPR':>8} "
f"{'unknown':>8} {'legacy-dis':>11}")
per_detector = {}
verdict_maps = {}
for name in detectors.ALL_DETECTORS:
run_id = db.latest_run_id(conn, name, {"case": args.case})
if run_id is None:
continue
verdicts = {}
for row in conn.execute(
"""
SELECT f.satellite, f.wavelength, f.t_start, d.verdict, d.reason
FROM detection d JOIN frame f ON f.id = d.frame_id WHERE d.run_id = ?
""",
(run_id,),
):
slot = (row["satellite"], row["wavelength"], row["t_start"])
verdicts[slot] = detectors.Verdict(row["verdict"], row["reason"], {})
# Deleted slots have no frame to judge, so they cannot be scored here.
scoreable = {s: v for s, v in verdicts.items() if injected.get(s) != "delete"}
score = metrics.score_detection(
scoreable,
{s: m for s, m in injected.items() if m != "delete"},
legacy_good,
)
per_detector[name] = score
verdict_maps[name] = verdicts
print(f"{name:>14} {score.precision:>7.3f} {score.recall:>7.3f} {score.f1:>7.3f} "
f"{score.false_positive_rate:>8.4f} {score.unknown:>8} "
f"{score.legacy_disagreements:>11}")
if args.combinations and len(per_detector) > 1:
print("\nDETECTOR COMBINATIONS (ranked by false-positive rate, then F1)")
print(f"{'policy':>9} {'detectors':>44} {'prec':>7} {'recall':>7} {'F1':>7} {'FPR':>8}")
scoreable_injected = {s: m for s, m in injected.items() if m != "delete"}
rows = []
names = sorted(per_detector)
for size in range(2, len(names) + 1):
for subset in itertools.combinations(names, size):
for policy in metrics.COMBINATION_POLICIES:
combined = metrics.combine_verdicts(
[verdict_maps[name] for name in subset], policy
)
combined = {
s: v for s, v in combined.items() if injected.get(s) != "delete"
}
score = metrics.score_detection(
combined, scoreable_injected, legacy_good
)
rows.append((score.false_positive_rate, -score.f1, policy, subset, score))
rows.sort(key=lambda r: (r[0] if np.isfinite(r[0]) else 9e9,
r[1] if np.isfinite(r[1]) else 9e9))
for _, _, policy, subset, score in rows[: args.combination_limit]:
label = "+".join(n.replace("_v1", "") for n in subset)
print(f"{policy:>9} {label:>44} {score.precision:>7.3f} {score.recall:>7.3f} "
f"{score.f1:>7.3f} {score.false_positive_rate:>8.4f}")
modes = sorted({m for m in injected.values() if m != "delete"})
if modes and per_detector:
print("\nRECALL BY CORRUPTION MODE")
header = f"{'mode':>18}" + "".join(f"{n[:11]:>13}" for n in per_detector)
print(header)
for mode in modes:
line = f"{mode:>18}"
for score in per_detector.values():
value = score.recall_by_mode.get(mode)
line += f"{value:>13.3f}" if value is not None else f"{'-':>13}"
print(line)
print("\nFILL")
rows = conn.execute(
"SELECT filler, source, scores_json FROM bench_fill_result WHERE case_id = ?",
(case["id"],),
).fetchall()
if not rows:
print(" (no fill results; run `bench.py fill` first)")
else:
grouped = defaultdict(list)
for row in rows:
payload = json.loads(row["scores_json"])
grouped[(row["source"], row["filler"])].append(
metrics.FillScore(**payload)
)
print(f"{'source':>10} {'filler':>16} {'n':>6} {'rmse':>10} {'psnr':>8} {'ssim':>7}")
for (source, filler_name), scores in sorted(grouped.items()):
summary = metrics.summarise_fills(scores)
print(f"{source:>10} {filler_name:>16} {summary['n']:>6} "
f"{summary['rmse']:>10.4g} {summary['psnr']:>8.2f} {summary['ssim']:>7.4f}")
print("\nFILL QUALITY BY GAP LENGTH (psnr, dB)")
gaps = sorted({s.gap_frames for scores in grouped.values() for s in scores})
print(f"{'source/filler':>28}" + "".join(f"{g:>8}" for g in gaps))
for (source, filler_name), scores in sorted(grouped.items()):
summary = metrics.summarise_fills(scores)
line = f"{source + '/' + filler_name:>28}"
for gap in gaps:
entry = summary["by_gap"].get(gap)
line += f"{entry['psnr']:>8.2f}" if entry else f"{'-':>8}"
print(line)
if args.json:
output = {
"case": args.case,
"window": window["name"],
"plan": plan,
"detection": {n: s.as_dict() for n, s in per_detector.items()},
}
with open(args.json, "w") as handle:
json.dump(output, handle, indent=2)
print(f"\nWrote {args.json}")
return 0
# --------------------------------------------------------------------------- render
#: Slots the current pipeline will carry forward from the last good frame. Beyond
#: this merger_FITS.py gives up and the timestamp produces no composite at all.
TODAY_MAX_TIME_GAP = 3
def _resolve_band(overlay, slot, truth_cache, use_truth=False):
"""Read one band's pixels, caching, or None if it cannot be read.
`use_truth` reads the original archive frame rather than what the case left in
its place. The pristine variant needs it: a slot the case *deleted* has no
overlay path at all, and resolving those through the overlay silently drops them
-- which cost the ground-truth pane 221 of its timestamps, exactly the ones the
other panes have to be compared against.
"""
key = (slot, use_truth)
if key not in truth_cache:
path = overlay.truth_path(slot) if use_truth else overlay.path(slot)
image, _ = fitsio.read_image(path) if path else (None, None)
truth_cache[key] = image
return truth_cache[key]
def _render_one(job):
"""Composite a single timestamp for one satellite. Runs in a worker."""
import merger_FITS
arrays, timestamp, out_path = job
try:
image = merger_FITS.composite_from_arrays(arrays, timestamp)
image.save(out_path, quality=95)
return out_path, None
except Exception as exc:
return out_path, f"{type(exc).__name__}: {exc}"
def cmd_render(args):
"""Produce the composite stream for one variant of a case.
Three variants, which together answer 'is the repair visible, and is it better
than what we do now':
* ``pristine`` -- the archive untouched. Ground truth.
* ``today`` -- what the current pipeline does with the injected damage: carry
the last good frame across gaps of up to TODAY_MAX_TIME_GAP slots, and emit
*nothing at all* for longer ones, which is what leaves ffmpeg_video.py padding
with black frames. A 300-slot gap produces 300 missing composites.
* ``new`` -- the chosen method: any(disc_v1 + header_v1) detection, filled
with optical_flow at every gap length.
"""
fitsio.quiet_astropy()
conn = db.connect(args.db)
case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone()
if case is None:
raise SystemExit(f"No case named {args.case!r}")
window = conn.execute(
"SELECT * FROM bench_window WHERE id = ?", (case["window_id"],)
).fetchone()
satellites = tuple(json.loads(window["satellites"]))
if args.satellites:
wanted = parse_int_list(args.satellites)
satellites = tuple(s for s in satellites if s in wanted)
if not satellites:
raise SystemExit(f"none of {wanted} are in this window")
wavelengths = tuple(json.loads(window["wavelengths"]))
if len(wavelengths) != 6:
raise SystemExit(f"a composite needs all six bands; window has {wavelengths}")
overlay = _build_overlay(conn, case, window)
# Pristine shows the window as it really is, so it reads the archive directly and
# treats nothing as damaged.
pristine = args.variant == "pristine"
bad = set() if pristine else _bad_slots(conn, case, args.detector)
times = sorted({slot[2] for slot in overlay.archive})
print(f"{args.variant}: {len(times)} timestamps x {len(satellites)} satellites, "
f"{len(bad)} slots treated as bad")
for satellite in satellites:
out_dir = os.path.join(args.out, args.variant, f"goes{satellite}")
os.makedirs(out_dir, exist_ok=True)
truth_cache = {}
jobs = []
skipped = 0
reliever = vfs.Reliever(label=f"render g{satellite}")
for when in times:
reliever.tick(len(wavelengths))
slots = [(satellite, band, when) for band in wavelengths]
arrays = []
usable = True
for slot in slots:
if not pristine and (slot in bad or overlay.path(slot) is None):
replacement, ok = _replacement_band(
args.variant, overlay, slot, bad, truth_cache, wavelengths
)
if not ok:
usable = False
break
arrays.append(replacement)
else:
image = _resolve_band(overlay, slot, truth_cache, use_truth=pristine)
if image is None:
usable = False
break
arrays.append(image)
if not usable:
# No composite for this timestamp -- exactly what the current
# pipeline does past its gap limit, and what ffmpeg then pads.
skipped += 1
continue
jobs.append((arrays, when, os.path.join(out_dir, f"Composite-{int(when)}.jpg")))
if len(jobs) >= args.batch:
_flush_renders(jobs, args.workers)
jobs.clear()
truth_cache.clear()
print(f" g{satellite}: {len(os.listdir(out_dir))} rendered, {skipped} skipped")
if jobs:
_flush_renders(jobs, args.workers)
print(f" g{satellite}: {len(os.listdir(out_dir))} composites, {skipped} timestamps skipped")
conn.close()
return 0
def _replacement_band(variant, overlay, slot, bad, truth_cache, wavelengths):
"""What to substitute for a band the case removed. Returns (array, usable)."""
satellite, band, when = slot
series = [s for s in overlay.series(satellite, band) if s not in bad]
times = [s[2] for s in series]
before = _nearest(times, when, before=True)
after = _nearest(times, when, before=False)
if variant == "today":
# merger_FITS carries the last good frame a few slots, then gives up.
if before is None or (when - before) // paths.CADENCE > TODAY_MAX_TIME_GAP:
return None, False
return _resolve_band(overlay, (satellite, band, before), truth_cache), True
context = fillers.FillContext(
before=_resolve_band(overlay, (satellite, band, before), truth_cache) if before else None,
dt_before=when - before if before else 0.0,
after=_resolve_band(overlay, (satellite, band, after), truth_cache) if after else None,
dt_after=after - when if after else 0.0,
)
filled = fillers.optical_flow(context)
return (filled, True) if filled is not None else (None, False)
def _flush_renders(jobs, workers):
failures = 0
with Pool(workers, initializer=fitsio.quiet_astropy) as pool:
for _, error in pool.imap_unordered(_render_one, jobs, 4):
failures += bool(error)
if failures:
print(f" WARNING: {failures} composites failed to render")
# ------------------------------------------------------------------------- examples
def cmd_examples(args):
"""Show the frames a detector got wrong, so a number becomes a picture.
A precision/recall table says how often a detector is wrong; it does not say
what it is wrong *about*. For `header_v1`, whose recall is low but precision
perfect, the interesting set is the false negatives -- damage that leaves
IMG_MEAN, ECLIPSE and EMPTY all looking normal.
"""
import matplotlib
matplotlib.use("Agg")
from matplotlib import pyplot as plt
fitsio.quiet_astropy()
conn = db.connect(args.db, readonly=True)
case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone()
if case is None:
raise SystemExit(f"No case named {args.case!r}")
window = conn.execute(
"SELECT * FROM bench_window WHERE id = ?", (case["window_id"],)
).fetchone()
overlay = _build_overlay(conn, case, window)
run_id = db.latest_run_id(conn, args.detector, {"case": args.case})
if run_id is None:
raise SystemExit(
f"No run of {args.detector!r} for case {args.case!r}; run detect first."
)
injected = {}
for row in conn.execute(
"""
SELECT f.satellite, f.wavelength, f.t_start, i.mode, i.severity
FROM bench_injection i JOIN frame f ON f.id = i.frame_id WHERE i.case_id = ?
""",
(case["id"],),
):
injected[(row["satellite"], row["wavelength"], row["t_start"])] = (
row["mode"], row["severity"]
)
wanted = []
for row in conn.execute(
"""
SELECT f.satellite, f.wavelength, f.t_start, d.verdict, d.reason, d.scores_json
FROM detection d JOIN frame f ON f.id = d.frame_id WHERE d.run_id = ?
""",
(run_id,),
):
slot = (row["satellite"], row["wavelength"], row["t_start"])
mode = injected.get(slot, (None, None))[0]
if mode == "delete":
continue # no frame to show
corrupted = mode is not None
if args.kind == "fn" and corrupted and row["verdict"] == "good":
wanted.append((slot, mode, row))
elif args.kind == "fp" and not corrupted and row["verdict"] == "bad":
wanted.append((slot, None, row))
print(f"{args.detector} {args.kind.upper()}s on case '{args.case}': {len(wanted)}")
if not wanted:
print(" none -- nothing to plot.")
return 0
by_mode = defaultdict(int)
for _, mode, _ in wanted:
by_mode[mode or "(uninjected)"] += 1
for mode, count in sorted(by_mode.items(), key=lambda kv: -kv[1]):
print(f" {count:>4} {mode}")
wanted = wanted[: args.limit]
columns = min(6, len(wanted))
rows = (len(wanted) + columns - 1) // columns
figure, axes = plt.subplots(rows, columns, figsize=(3 * columns, 3.4 * rows))
axes = np.atleast_2d(axes)
for index, (slot, mode, row) in enumerate(wanted):
axis = axes[index // columns][index % columns]
path = overlay.path(slot)
image, _ = fitsio.read_image(path) if path else (None, None)
if image is not None:
axis.imshow(metrics.to_display(image, slot[1]), cmap="gray", vmin=0, vmax=1)
scores = json.loads(row["scores_json"] or "{}")
shown = ", ".join(f"{k}={v:.3g}" for k, v in list(scores.items())[:3])
axis.set_title(
f"g{slot[0]} {slot[1]}A {format_time(slot[2])[5:]}\n"
f"{mode or 'clean'} -> {row['verdict']}\n{shown}",
fontsize=6,
)
axis.axis("off")
for index in range(len(wanted), rows * columns):
axes[index // columns][index % columns].axis("off")
figure.suptitle(f"{args.detector} {args.kind.upper()}s - case {args.case}")
figure.tight_layout()
figure.savefig(args.out, dpi=150)
plt.close(figure)
print(f" wrote {args.out}")
return 0
# ------------------------------------------------------------------------------ cli
def build_parser():
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--db", default=None, help="SQLite index (default: $SUVI_DB)")
subparsers = parser.add_subparsers(dest="command", required=True)
def add_window_args(sub):
sub.add_argument("--start", type=parse_date, required=True)
sub.add_argument("--end", type=parse_date, required=True)
sub.add_argument("--satellites", default="16,18")
sub.add_argument("--wavelengths", default="94,131,171,195,284,304")
select = subparsers.add_parser("select-window", help="find clean stretches of archive")
add_window_args(select)
select.add_argument("--min-slots", type=int, default=120)
select.add_argument("--limit", type=int, default=10)
select.add_argument("--require-legacy-label", action="store_true",
help="also require the stale _f suffix (off by default)")
select.set_defaults(func=cmd_select_window)
vet = subparsers.add_parser("vet-window", help="verify and freeze a ground-truth window")
add_window_args(vet)
vet.add_argument("--name", required=True)
vet.add_argument("--notes", default=None)
vet.add_argument("--allow-missing", action="store_true")
vet.add_argument("--min-good-fraction", type=float, default=0.98)
vet.add_argument("--contact-sheet", default=None, help="write a filmstrip PNG for review")
vet.add_argument("--progress", action="store_true")
vet.set_defaults(func=cmd_vet_window)
make = subparsers.add_parser("make-case", help="inject damage into a vetted window")
make.add_argument("--window", required=True)
make.add_argument("--name", required=True)
make.add_argument("--seed", type=int, default=42)
make.add_argument("--fraction", type=float, default=0.10)
make.add_argument("--gap-lengths", default="1,2,3,5,10,30,60")
make.add_argument("--gaps-per-length", type=int, default=None,
help="place exactly this many gaps at each length, instead of "
"sizing the case by --fraction")
make.add_argument("--satellite-scope", default="mixed",
choices=("g16", "g18", "both", "mixed"))
make.add_argument("--wavelength-scope", default="all", choices=("all", "one"))
make.add_argument("--modes", default=None, help="comma-separated; default is all")
make.add_argument("--severity-low", type=float, default=0.5)
make.add_argument("--severity-high", type=float, default=1.0)
make.add_argument("--overlay-dir", default=None)
make.add_argument("--workers", type=int, default=8)
make.add_argument("--progress", action="store_true")
make.set_defaults(func=cmd_make_case)
detect = subparsers.add_parser("detect", help="run detectors over a case")
detect.add_argument("--case", required=True)
detect.add_argument("--detectors", default=None, help="comma-separated; default is all")
detect.add_argument("--workers", type=int, default=8)
detect.set_defaults(func=cmd_detect)
fill = subparsers.add_parser("fill", help="reconstruct flagged slots and score them")
fill.add_argument("--case", required=True)
fill.add_argument("--fillers", default=None, help="comma-separated; default is all")
group = fill.add_mutually_exclusive_group(required=True)
group.add_argument("--oracle", action="store_true",
help="use perfect knowledge of which slots are bad")
group.add_argument("--from-detector", default=None,
help="use a detector's output, showing end-to-end error")
fill.set_defaults(func=cmd_fill)
report = subparsers.add_parser("report", help="summarise a case's results")
report.add_argument("--case", required=True)
report.add_argument("--json", default=None)
report.add_argument("--combinations", action="store_true",
help="also score every combination of detectors")
report.add_argument("--combination-limit", type=int, default=15)
report.set_defaults(func=cmd_report)
render = subparsers.add_parser(
"render", help="produce the composite stream for one variant of a case"
)
render.add_argument("--case", required=True)
render.add_argument("--variant", required=True,
choices=("pristine", "today", "new"))
render.add_argument("--out", required=True, help="root directory for the streams")
render.add_argument("--detector", default="oracle",
help="which slots to treat as bad ('oracle', or a run name)")
render.add_argument("--satellites", default=None,
help="restrict to these satellites (default: all in the window)")
render.add_argument("--workers", type=int, default=8)
render.add_argument("--batch", type=int, default=48,
help="composites per worker dispatch; bounds peak memory")
render.set_defaults(func=cmd_render)
examples = subparsers.add_parser(
"examples", help="plot the frames a detector got wrong"
)
examples.add_argument("--case", required=True)
examples.add_argument("--detector", required=True)
examples.add_argument("--kind", choices=("fp", "fn"), required=True)
examples.add_argument("--limit", type=int, default=12)
examples.add_argument("--out", default="examples.png")
examples.set_defaults(func=cmd_examples)
return parser
def main(argv=None):
args = build_parser().parse_args(argv)
return args.func(args) or 0
if __name__ == "__main__":
sys.exit(main())
Reference in a new issue Copy permalink