#!/usr/bin/env python """Test bench for bad-frame detection and gap filling. Takes a stretch of archive verified to be clean, damages it in known ways, and measures how well each detector finds the damage and each filler repairs it. The archive is never modified. Corrupted frames are written to a per-case overlay directory and every read goes through :class:`suvi.cases.Overlay`, so a bench run against live production data is safe. Workflow:: bench.py select-window --start 2024-05-05 --end 2024-05-20 bench.py vet-window --name may2024 --start ... --end ... bench.py make-case --window may2024 --name mixed10 --seed 42 bench.py detect --case mixed10 bench.py fill --case mixed10 --oracle bench.py report --case mixed10 Each step records its results in the index, so steps are resumable and comparable across runs. """ import argparse import bisect import datetime import itertools import json import os import sys import time from collections import OrderedDict, defaultdict from multiprocessing import Pool import numpy as np from astropy.io import fits sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from suvi import (availability, cases, corruptions, db, detectors, fillers, fitsio, metrics, paths) def parse_date(text): """Accept YYYY-MM-DD, YYYY-MM-DDTHH:MM (always UTC), or a raw unix timestamp. The raw form exists so a stored window can be re-vetted at its exact boundaries, which rarely fall on a midnight. """ if text.isdigit(): return int(text) for fmt in ("%Y-%m-%d", "%Y-%m-%dT%H:%M", "%Y-%m-%d %H:%M"): try: stamp = datetime.datetime.strptime(text, fmt) return int(stamp.replace(tzinfo=datetime.timezone.utc).timestamp()) except ValueError: continue raise argparse.ArgumentTypeError(f"Not a date: {text!r}") def format_time(timestamp): return datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc).strftime( "%Y-%m-%d %H:%M" ) def parse_int_list(text): return tuple(int(part) for part in text.split(",") if part.strip()) # ------------------------------------------------------------------- select-window def cmd_select_window(args): root = paths.data_root() satellites = parse_int_list(args.satellites) wavelengths = parse_int_list(args.wavelengths) print(f"Scanning {root} from {format_time(args.start)} to {format_time(args.end)}") found = cases.scan_window(root, satellites, wavelengths, args.start, args.end) total = len(cases.timeline(args.start, args.end)) print(f" {len(found)} files across {total} slots\n") runs = cases.find_runs( found, satellites, wavelengths, args.start, args.end, minimum=args.min_slots, require_good_label=args.require_legacy_label, ) if not runs: print(f"No runs of >= {args.min_slots} slots complete on every requested band.") print("Try fewer bands, one satellite, or a different date range.") return 1 basis = "labelled good by the legacy filter" if args.require_legacy_label else "present" print(f"Runs where every requested band and satellite is {basis}:") print(f"{'slots':>7} {'hours':>8} window") for length, start, end in runs[: args.limit]: print(f"{length:>7} {length * paths.CADENCE / 3600:>8.1f} {format_time(start)} .. {format_time(end)}") if not args.require_legacy_label: print("\nCompleteness only -- run vet-window to verify quality from the headers.") return 0 # ---------------------------------------------------------------------- vet-window def cmd_vet_window(args): """Record a window's ground truth from the stored production judgment. There used to be a private `_header_vet` here testing only EMPTY/ECLIPSE -- a weaker duplicate of the production detectors, and the reason a 10-hour dark stretch was once vetted 100% good -- and later a per-file `judge_file`, which still could not make the clean-neighbour check. Reading the stored run means bench truth cannot hold a different opinion from production again. """ root = paths.data_root() satellites = parse_int_list(args.satellites) wavelengths = parse_int_list(args.wavelengths) conn = db.connect(args.db) found = cases.scan_window(root, satellites, wavelengths, args.start, args.end) slots = cases.timeline(args.start, args.end) expected = len(slots) * len(satellites) * len(wavelengths) print(f"Window has {len(found)}/{expected} frames present") missing = [ (satellite, wavelength, time) for time in slots for satellite in satellites for wavelength in wavelengths if (satellite, wavelength, time) not in found ] if missing and not args.allow_missing: print(f"ERROR: {len(missing)} slots have no frame at all; refusing to vet.") print(" Pick a window from select-window, or pass --allow-missing.") return 1 # Ground truth is the stored production judgment (`filter_FITS.py judge`): the # same verdicts the corpus was selected and vetted by and the renderers read, # including the clean-neighbour check no single-file judgment can make. The # legacy filename labels are recorded for comparison but deliberately do not # decide anything: they were written in May 2024 by a filter that was retuned # that July, so they are neither current nor self-consistent. run_id = args.run or detectors.judgment_run(conn) if run_id is None: print("ERROR: no production judgment under the current rules; run " "`filter_FITS.py judge` over the window first.") return 1 source = f"judge run {run_id}" stored = { (row["satellite"], row["wavelength"], row["t_start"]): (row["verdict"], row["reason"]) for row in conn.execute( "SELECT f.satellite, f.wavelength, f.t_start, d.verdict, d.reason " "FROM frame f JOIN detection d ON d.frame_id = f.id AND d.run_id = ? " "WHERE f.t_start >= ? AND f.t_start < ?", (run_id, args.start, args.end)) } unjudged = [slot for slot in found if slot not in stored] if unjudged: print(f"ERROR: {len(unjudged)} frames in the window have no verdict in {source} " f"(e.g. {sorted(unjudged)[:3]}); judge the window first.") return 1 print(f"Vetting from {source}...") disagreements = [] truth = {} for slot, (path, label) in sorted(found.items()): verdict, reason = stored[slot] good = verdict != detectors.BAD if good != (label == "f"): disagreements.append((slot, label, reason)) truth[slot] = (good, source, None if good else reason) print(f" {len(disagreements)} frames where the legacy label disagrees with the judgment") for slot, label, reason in disagreements[:10]: print(f" {slot} legacy={label} judgment={reason or 'good'}") good = sum(1 for is_good, _, _ in truth.values() if is_good) print(f" {good}/{len(truth)} frames vetted good") if good < len(truth) * args.min_good_fraction: print(f"ERROR: only {good / len(truth):.1%} vetted good; window is not clean enough.") return 1 conn.execute( """ INSERT INTO bench_window (name, t_start, t_end, satellites, wavelengths, vetted_at, notes) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT (name) DO UPDATE SET t_start = excluded.t_start, t_end = excluded.t_end, satellites = excluded.satellites, wavelengths = excluded.wavelengths, vetted_at = excluded.vetted_at, notes = excluded.notes """, ( args.name, args.start, args.end, json.dumps(list(satellites)), json.dumps(list(wavelengths)), time.time(), args.notes, ), ) window_id = conn.execute( "SELECT id FROM bench_window WHERE name = ?", (args.name,) ).fetchone()["id"] conn.execute("DELETE FROM bench_truth WHERE window_id = ?", (window_id,)) for slot, (path, label) in sorted(found.items()): name = paths.parse_frame_filename(os.path.basename(path)) relpath = name.relpath() frame_id = db.upsert_frame( conn, name, relpath, os.path.getsize(path), os.path.getmtime(path) ) is_good, source, reason = truth[slot] conn.execute( "INSERT OR REPLACE INTO bench_truth " "(window_id, frame_id, is_good, vet_source, reason)" " VALUES (?, ?, ?, ?, ?)", (window_id, frame_id, int(is_good), source, reason), ) conn.commit() print(f"Wrote window '{args.name}' (id {window_id}) with {len(found)} frames") if args.contact_sheet: _write_contact_sheet(found, args.contact_sheet, satellites, wavelengths) print(f"Contact sheet: {args.contact_sheet}") print("Review it before trusting this window as ground truth.") return 0 def _write_contact_sheet(found, path, satellites, wavelengths, columns=24, size=64): """A downsampled filmstrip of the window, for the one manual review pass.""" import matplotlib matplotlib.use("Agg") from matplotlib import pyplot as plt satellite, wavelength = satellites[0], wavelengths[len(wavelengths) // 2] series = sorted(slot for slot in found if slot[0] == satellite and slot[1] == wavelength) if not series: return step = max(1, len(series) // (columns * 20)) picks = series[::step] rows = (len(picks) + columns - 1) // columns figure, axes = plt.subplots(rows, columns, figsize=(columns, rows)) axes = np.atleast_2d(axes) vmin, vmax, gamma = metrics.DISPLAY_MAPPING[wavelength] for index, slot in enumerate(picks): axis = axes[index // columns][index % columns] image, error = fitsio.read_image(found[slot][0]) if image is not None: import cv2 as cv thumb = cv.resize(np.nan_to_num(image), (size, size), interpolation=cv.INTER_AREA) axis.imshow(metrics.to_display(thumb, wavelength), cmap="gray", vmin=0, vmax=1) axis.set_title(format_time(slot[2])[5:], fontsize=3) axis.axis("off") for index in range(len(picks), rows * columns): axes[index // columns][index % columns].axis("off") figure.suptitle(f"g{satellite} {wavelength}A - {len(picks)} of {len(series)} frames") figure.tight_layout() figure.savefig(path, dpi=200) plt.close(figure) # ------------------------------------------------------------------------ make-case def _load_window(conn, name): row = conn.execute("SELECT * FROM bench_window WHERE name = ?", (name,)).fetchone() if row is None: raise SystemExit(f"No vetted window named {name!r}; run vet-window first.") return row def _build_overlay(conn, case_row, window_row): """Reconstruct a case's view of the archive from the index.""" root = paths.data_root() satellites = tuple(json.loads(window_row["satellites"])) wavelengths = tuple(json.loads(window_row["wavelengths"])) archive = cases.scan_window( root, satellites, wavelengths, window_row["t_start"], window_row["t_end"] ) overrides, deleted = {}, set() rows = conn.execute( """ SELECT f.satellite, f.wavelength, f.t_start, i.mode, i.overlay_path FROM bench_injection i JOIN frame f ON f.id = i.frame_id WHERE i.case_id = ? """, (case_row["id"],), ).fetchall() for row in rows: slot = (row["satellite"], row["wavelength"], row["t_start"]) if row["mode"] == "delete": deleted.add(slot) elif row["overlay_path"]: overrides[slot] = row["overlay_path"] return cases.Overlay(archive=archive, overrides=overrides, deleted=frozenset(deleted)) def _write_corrupted(source_path, target_path, injection): """Materialise one corrupted frame into the overlay. Returns None on success.""" os.makedirs(os.path.dirname(target_path), exist_ok=True) corruption = corruptions.CATALOG[injection.mode] if corruption.kind == "file": with open(source_path, "rb") as handle: raw = handle.read() damaged = corruptions.apply_file(injection.mode, raw, injection.seed, injection.severity) with open(target_path, "wb") as handle: handle.write(damaged) return None image, error = fitsio.read_image(source_path) if image is None: return f"could not read source: {error}" donor = None if corruption.needs_donor: donor = np.roll(image, image.shape[0] // 3, axis=0) # a plausible other frame damaged, overrides = corruptions.apply_array( injection.mode, image, injection.seed, injection.severity, donor ) if corruption.recompute_stats: overrides = {**overrides, **corruptions.recomputed_stats(damaged)} with fits.open(source_path) as hdus: target = next( (h for h in hdus if getattr(h, "data", None) is not None and h.data.ndim == 2), None ) if target is None: return "source has no image HDU" target.data = damaged.astype(np.float32) for key, value in overrides.items(): target.header[key] = value hdus.writeto(target_path, overwrite=True, checksum=True) return None def _corrupt_one(job): source_path, target_path, injection = job try: return injection.slot, _write_corrupted(source_path, target_path, injection) except Exception as exc: # a corruption must never take the whole run down return injection.slot, f"{type(exc).__name__}: {exc}" def cmd_make_case(args): conn = db.connect(args.db) window = _load_window(conn, args.window) satellites = tuple(json.loads(window["satellites"])) wavelengths = tuple(json.loads(window["wavelengths"])) root = paths.data_root() archive = cases.scan_window( root, satellites, wavelengths, window["t_start"], window["t_end"] ) if args.from_trace: from suvi import availability as availability_module model = availability_module.Availability.load(args.from_trace) injections, trace_span = cases.plan_from_trace( model, set(archive), satellites, wavelengths, args.seed, severity=(args.severity_low, args.severity_high), ) plan_json = json.dumps({"from_trace": args.from_trace, "trace_span": list(trace_span), "seed": args.seed}, sort_keys=True) deletions = sum(1 for i in injections if i.mode == "delete") print(f"trace window {trace_span}: {len(injections)} injections " f"({deletions} deletions, {len(injections) - deletions} corruptions)") else: plan = cases.InjectionPlan( fraction=args.fraction, gap_lengths=parse_int_list(args.gap_lengths), satellite_scope=args.satellite_scope, wavelength_scope=args.wavelength_scope, modes=tuple(args.modes.split(",")) if args.modes else cases.InjectionPlan.modes, severity=(args.severity_low, args.severity_high), gaps_per_length=args.gaps_per_length, ).validate() injections = cases.plan_injections( plan, set(archive), satellites, wavelengths, args.seed ) plan_json = json.dumps(plan.as_dict(), sort_keys=True) if not injections: print("Plan produced no injections; raise --fraction or widen the window.") return 1 overlay_dir = args.overlay_dir or os.path.join( os.path.dirname(root), "bench", args.name ) os.makedirs(overlay_dir, exist_ok=True) conn.execute( """ INSERT INTO bench_case (window_id, name, seed, plan_json, overlay_dir, created_at) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (name) DO UPDATE SET window_id = excluded.window_id, seed = excluded.seed, plan_json = excluded.plan_json, overlay_dir = excluded.overlay_dir, created_at = excluded.created_at """, ( window["id"], args.name, args.seed, plan_json, overlay_dir, time.time(), ), ) case_id = conn.execute( "SELECT id FROM bench_case WHERE name = ?", (args.name,) ).fetchone()["id"] conn.execute("DELETE FROM bench_injection WHERE case_id = ?", (case_id,)) jobs, rows = [], [] for injection in injections: source_path = archive[injection.slot][0] name = paths.parse_frame_filename(os.path.basename(source_path)) frame_id = db.frame_id_by_slot(conn, *injection.slot) if frame_id is None: frame_id = db.upsert_frame(conn, name, name.relpath()) overlay_path = None if injection.mode != "delete": overlay_path = os.path.join(overlay_dir, name.relpath()) jobs.append((source_path, overlay_path, injection)) rows.append((case_id, frame_id, injection, overlay_path)) print(f"Planned {len(injections)} injections " f"({sum(1 for i in injections if i.mode == 'delete')} deletions, {len(jobs)} corruptions)") failures = {} if jobs: with Pool(args.workers, initializer=fitsio.quiet_astropy) as pool: for index, (slot, error) in enumerate(pool.imap_unordered(_corrupt_one, jobs, 8)): if error: failures[slot] = error if args.progress and index % 200 == 0 and index: print(f" wrote {index}/{len(jobs)}") if failures: print(f"WARNING: {len(failures)} corruptions failed:") for slot, error in list(failures.items())[:5]: print(f" {slot}: {error}") conn.executemany( """ INSERT INTO bench_injection (case_id, frame_id, mode, severity, params_json, overlay_path, gap_index, gap_length) VALUES (?, ?, ?, ?, ?, ?, ?, ?) """, [ ( case_id, frame_id, injection.mode, injection.severity, json.dumps({"seed": injection.seed}), overlay_path, injection.gap_index, injection.gap_length, ) for case_id, frame_id, injection, overlay_path in rows if injection.slot not in failures ], ) conn.commit() affected = len({i.slot[2] for i in injections}) total_times = len({slot[2] for slot in archive}) print(f"Case '{args.name}' (id {case_id}): {affected}/{total_times} timestamps affected") print(f"Overlay: {overlay_dir}") return 0 # --------------------------------------------------------------------------- detect def _read_features(job): """Load one frame's header, thumbnail, and single-frame verdicts.""" slot, path = job if path is None: return slot, None, None, None, 0 started = time.perf_counter_ns() values, header_error = fitsio.scan_header(path) features = detectors.FrameFeatures(slot=slot, header=values, error=header_error) header_verdict = detectors.header_v1(features) header_us = (time.perf_counter_ns() - started) // 1000 image, image_error = fitsio.read_image(path) started = time.perf_counter_ns() if image is None: geometry_verdict = detectors.unreadable(image_error) else: geometry_verdict = detectors.geometry_v1(image, slot[1]) geometry_us = (time.perf_counter_ns() - started) // 1000 started = time.perf_counter_ns() if image is None: disc_verdict = detectors.unreadable(image_error) else: disc_verdict = detectors.disc_v1(image, slot[1], values) disc_us = (time.perf_counter_ns() - started) // 1000 features.thumbnail = detectors.thumbnail(image) if image is not None else None return ( slot, features, (header_verdict, header_us), (geometry_verdict, geometry_us), (disc_verdict, disc_us), ) def cmd_detect(args): fitsio.quiet_astropy() conn = db.connect(args.db) case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone() if case is None: raise SystemExit(f"No case named {args.case!r}") window = conn.execute( "SELECT * FROM bench_window WHERE id = ?", (case["window_id"],) ).fetchone() satellites = tuple(json.loads(window["satellites"])) wavelengths = tuple(json.loads(window["wavelengths"])) overlay = _build_overlay(conn, case, window) wanted = tuple(args.detectors.split(",")) if args.detectors else detectors.ALL_DETECTORS runs = {} for name in wanted: runs[name] = db.create_detector_run( conn, name, {"case": args.case}, notes=f"bench case {args.case}" ) results = {name: {} for name in wanted} elapsed = defaultdict(int) for wavelength in wavelengths: per_satellite = {} for satellite in satellites: series_slots = [ slot for slot in overlay.series(satellite, wavelength) if slot not in overlay.deleted ] jobs = [(slot, overlay.path(slot)) for slot in series_slots] features_by_slot = {} with Pool(args.workers, initializer=fitsio.quiet_astropy) as pool: for slot, features, header, geometry, disc in pool.imap( _read_features, jobs, 8 ): if features is None: continue features_by_slot[slot] = features for name, outcome in ( ("header_v1", header), ("geometry_v1", geometry), ("disc_v1", disc), ): if name in results: results[name][slot] = outcome[0] elapsed[name] += outcome[1] ordered = [features_by_slot[s] for s in series_slots if s in features_by_slot] per_satellite[satellite] = ordered if "temporal_v1" in results and ordered: started = time.perf_counter_ns() verdicts = detectors.temporal_v1(ordered) elapsed["temporal_v1"] += (time.perf_counter_ns() - started) // 1000 for features, verdict in zip(ordered, verdicts): results["temporal_v1"][features.slot] = verdict if "crosssat_v1" in results and len(satellites) >= 2: first, second = satellites[0], satellites[1] by_time = { sat: {f.slot[2]: f for f in per_satellite.get(sat, [])} for sat in (first, second) } temporal = results.get("temporal_v1", {}) started = time.perf_counter_ns() for when in sorted(set(by_time[first]) | set(by_time[second])): a, b = by_time[first].get(when), by_time[second].get(when) verdict_a, verdict_b = detectors.crosssat_v1( a, b, temporal.get(a.slot) if a else None, temporal.get(b.slot) if b else None, ) if a is not None: results["crosssat_v1"][a.slot] = verdict_a if b is not None: results["crosssat_v1"][b.slot] = verdict_b elapsed["crosssat_v1"] += (time.perf_counter_ns() - started) // 1000 print(f" {wavelength}A done") for name, verdicts in results.items(): payload = [] for slot, verdict in verdicts.items(): frame_id = db.frame_id_by_slot(conn, *slot) if frame_id is None: continue payload.append((frame_id, verdict.verdict, verdict.reason, verdict.scores, None)) db.record_detections(conn, runs[name], payload) conn.commit() flagged = sum(1 for v in verdicts.values() if v.verdict == "bad") print(f"{name:>14}: {len(verdicts)} frames, {flagged} flagged bad, " f"{elapsed[name] / max(len(verdicts), 1):.0f} us/frame") return 0 # ----------------------------------------------------------------------------- fill def _bad_slots(conn, case, source): """Slots the fill step should reconstruct, per oracle or per detector. Detector runs are per case, so the run has to be looked up by this case's config. Taking "the latest run of header_v1" would silently pick up whichever case happened to be processed last. """ if source == "oracle": rows = conn.execute( """ SELECT f.satellite, f.wavelength, f.t_start FROM bench_injection i JOIN frame f ON f.id = i.frame_id WHERE i.case_id = ? """, (case["id"],), ).fetchall() else: run_id = db.latest_run_id(conn, source, {"case": case["name"]}) if run_id is None: raise SystemExit( f"No run of {source!r} for case {case['name']!r}; run detect first." ) rows = conn.execute( """ SELECT f.satellite, f.wavelength, f.t_start FROM detection d JOIN frame f ON f.id = d.frame_id WHERE d.run_id = ? AND d.verdict = 'bad' """, (run_id,), ).fetchall() return {(row["satellite"], row["wavelength"], row["t_start"]) for row in rows} def _truth_bad_slots(conn, window_id): """Slots the window's own vetting found genuinely bad in the archive. These are distrusted as *inputs* -- excluded from clean times, calibration pairs, and withheld from stacks -- but are never fill *targets*: their "truth" frame is the bad frame itself, so a score against it is meaningless. """ rows = conn.execute( """ SELECT f.satellite, f.wavelength, f.t_start FROM bench_truth t JOIN frame f ON f.id = t.frame_id WHERE t.window_id = ? AND t.is_good = 0 """, (window_id,), ) return {(r["satellite"], r["wavelength"], r["t_start"]) for r in rows} def cmd_fill(args): fitsio.quiet_astropy() conn = db.connect(args.db) case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone() if case is None: raise SystemExit(f"No case named {args.case!r}") window = conn.execute( "SELECT * FROM bench_window WHERE id = ?", (case["window_id"],) ).fetchone() satellites = tuple(json.loads(window["satellites"])) wavelengths = tuple(json.loads(window["wavelengths"])) overlay = _build_overlay(conn, case, window) source = "oracle" if args.oracle else args.from_detector if source is None: raise SystemExit("Pass either --oracle or --from-detector NAME") truth_bad = _truth_bad_slots(conn, case["window_id"]) injected = _bad_slots(conn, case, source) targets_bad = injected - truth_bad bad = injected | truth_bad # distrusted as inputs from here on print(f"Filling {len(targets_bad)} slots identified by '{source}'") if truth_bad: print(f" {len(truth_bad)} vetted-bad archive slots distrusted as inputs; " f"{len(injected & truth_bad)} would-be targets excluded " "(no pristine truth to score against)") gap_lengths = {} for row in conn.execute( """ SELECT f.satellite, f.wavelength, f.t_start, i.gap_length FROM bench_injection i JOIN frame f ON f.id = i.frame_id WHERE i.case_id = ? """, (case["id"],), ): gap_lengths[(row["satellite"], row["wavelength"], row["t_start"])] = row["gap_length"] # The fill policy: a stretch where *neither* satellite holds some band clean # for more than --max-dual-gap slots is left as a gap, exactly as # ffmpeg_video's max_frame_interp leaves the single-satellite ones. Skipped # slots are counted apart from unreadable ones -- declining is policy, not # coverage loss. unfillable = unfillable_by_band( overlay, satellites, wavelengths, bad, cases.timeline(window["t_start"], window["t_end"]), args.max_dual_gap, ) declined = sum(1 for slot in bad if slot[2] in unfillable) if declined: # `bad` itself is NOT filtered: a slot past the policy limit is still bad # *input* -- damaged pixels must stay out of every other fill's stack # and out of the calibration pairs. Only its reconstruction is skipped. print(f" declining {declined} slots inside dual gaps longer than " f"{args.max_dual_gap} slots (policy: availability.MAX_DUAL_GAP)") wanted = tuple(args.fillers.split(",")) if args.fillers else tuple(fillers.FILLERS) # The learned filler is joint across all six bands, so it cannot run inside a loop # that handles one band at a time; it gets its own pass below. per_band = tuple(name for name in wanted if name != "learned") scored = defaultdict(list) # A slot whose truth frame will not read cannot be scored. Count them: silently # dropping them yields metrics computed over whatever subset happened to work, # with nothing in the output to say so. A degraded mount once reduced this to # 12% coverage while the report looked entirely normal. unreadable = 0 attempted = 0 payload = [] if "learned" in wanted: fillers.load_learned() # a missing checkpoint fails now, not hours in def persist(): """Commit scores as they accumulate. A crash after the per-band pass once cost six hours of fills because nothing was written until the very end.""" if not payload: return conn.executemany( """ INSERT INTO bench_fill_result (case_id, frame_id, filler, source, scores_json) VALUES (?, ?, ?, ?, ?) ON CONFLICT (case_id, frame_id, filler, source) DO UPDATE SET scores_json = excluded.scores_json """, payload, ) conn.commit() payload.clear() if not per_band: # Only the learned filler was asked for. Its own pass below reads exactly what # it needs; walking every band here would pull all 5,304 damaged slots and their # neighbours off the mount to compute nothing at all. print(" (skipping per-band pass: no per-band fillers requested)") for wavelength in (wavelengths if per_band else ()): truth_cache = {} def truth_image(slot): if slot not in truth_cache: path = overlay.truth_path(slot) image, _ = fitsio.read_image(path) if path else (None, None) truth_cache[slot] = image return truth_cache[slot] # Times where every satellite has a good frame in this band. The two # spacecraft observe simultaneously, so such a slot isolates the instrument # difference with none of the Sun's own evolution folded in -- which is the # only sound way to calibrate one against the other. See fillers.crosssat. per_satellite = [ { s[2] for s in overlay.series(other, wavelength) if s not in bad and overlay.path(s) is not None } for other in satellites ] paired_times = sorted(set.intersection(*per_satellite)) if len(satellites) > 1 else [] for satellite in satellites: series = overlay.series(satellite, wavelength) good = [s for s in series if s not in bad and overlay.path(s) is not None] good_times = [s[2] for s in good] for slot in series: if slot not in targets_bad or slot[2] in unfillable: continue attempted += 1 truth = truth_image(slot) if truth is None: unreadable += 1 continue before = _nearest(good_times, slot[2], before=True) after = _nearest(good_times, slot[2], before=False) counterpart_slot = next( ( (other, wavelength, slot[2]) for other in satellites if other != satellite and (other, wavelength, slot[2]) not in bad and overlay.path((other, wavelength, slot[2])) is not None ), None, ) calibration = None if counterpart_slot is not None and paired_times: pair_time = _closest(paired_times, slot[2]) calibration = ( truth_image((counterpart_slot[0], wavelength, pair_time)), truth_image((satellite, wavelength, pair_time)), ) context = fillers.FillContext( before=truth_image((satellite, wavelength, before)) if before else None, dt_before=slot[2] - before if before else 0.0, after=truth_image((satellite, wavelength, after)) if after else None, dt_after=after - slot[2] if after else 0.0, counterpart=truth_image(counterpart_slot) if counterpart_slot else None, calibration=calibration, header=_header_for(overlay, slot), ) for filler_name in per_band: filled = fillers.FILLERS[filler_name](context) if filled is None: continue score = metrics.score_fill( filled, truth, wavelength, gap_lengths.get(slot, context.gap_frames) ) scored[filler_name].append(score) frame_id = db.frame_id_by_slot(conn, *slot) if frame_id is not None: payload.append( (case["id"], frame_id, filler_name, source, json.dumps(score.as_dict())) ) truth_cache.clear() persist() print(f" {wavelength}A done") if "learned" in wanted: learned_scores, learned_payload, learned_attempted = _fill_learned( conn, case, overlay, satellites, wavelengths, bad, gap_lengths, source, max_targets=args.max_targets, unfillable=unfillable, label=args.learned_name, targets_bad=targets_bad, ) scored[args.learned_name] = learned_scores payload.extend(learned_payload) attempted = max(attempted, learned_attempted) persist() covered = attempted - unreadable print(f"\nScored {covered}/{attempted} slots" + (f" -- {unreadable} could not be read" if unreadable else "")) if attempted and covered < attempted * 0.95: print(f" WARNING: only {covered / attempted:.0%} of damaged slots were scored.") print(" These numbers describe that subset, not the case. A mount") print(" that ran out of file handles is the usual cause; re-run.") print(f"\n{'filler':>16} {'n':>6} {'rmse':>10} {'log rmse':>10} {'psnr':>8} {'ssim':>7}") for filler_name in (args.learned_name if name == "learned" else name for name in wanted): summary = metrics.summarise_fills(scored.get(filler_name, [])) if not summary: print(f"{filler_name:>16} {'-- not applicable --':>44}") continue print(f"{filler_name:>16} {summary['n']:>6} {summary['rmse']:>10.4g} " f"{summary['log_rmse']:>10.4g} {summary['psnr']:>8.2f} {summary['ssim']:>7.4f}") return 0 def _stratified_targets(targets, gap_lengths, limit): """`limit` targets spread evenly over gap length, not the first `limit` found. Fill quality is reported against gap length and varies by 10 dB across the range, so a prefix of the sorted targets would sample whatever gap happened to be placed earliest and report it as the model's overall quality. """ if not limit or limit >= len(targets): return targets by_length = defaultdict(list) for target in targets: satellite, when = target lengths = [length for (s, _, t), length in gap_lengths.items() if s == satellite and t == when] by_length[max(lengths) if lengths else 0].append(target) chosen = [] groups = [sorted(group) for _, group in sorted(by_length.items())] position = 0 while len(chosen) < limit and any(position < len(g) for g in groups): for group in groups: if position < len(group) and len(chosen) < limit: chosen.append(group[position]) position += 1 return sorted(chosen) def _fill_learned(conn, case, overlay, satellites, wavelengths, bad, gap_lengths, source, max_targets=None, unfillable=frozenset(), label="learned", targets_bad=None): """Run the learned filler, which reconstructs all six bands in one pass. A separate pass because the model is joint across bands while the loop above is not. Running it inside that loop would mean six forward passes per slot, five of whose outputs are thrown away, and six times the stack assembly with it. Returns (scores, rows, attempted) so the caller can merge it into its own tally. """ if fillers.load_learned() is None: print("\nSkipping 'learned': no checkpoint " f"(set {fillers.LEARNED_CHECKPOINT_ENV})") return [], [], 0 scorable = bad if targets_bad is None else targets_bad targets = sorted({(slot[0], slot[2]) for slot in scorable if slot[2] not in unfillable}) if max_targets: targets = _stratified_targets(targets, gap_lengths, max_targets) print(f"\nLearned filler: {len(targets)} (satellite, time) targets") joint = _LearnedFiller(overlay, satellites, wavelengths, bad) scores, rows = [], [] attempted = 0 unreadable = 0 for index, (satellite, when) in enumerate(targets): filled = joint.fill(satellite, when) if filled is None: unreadable += 1 continue for band, wavelength in enumerate(wavelengths): slot = (satellite, wavelength, when) if slot not in scorable: continue attempted += 1 if np.isnan(filled[band]).all(): unreadable += 1 # no frame in the stack holds this band continue path = overlay.truth_path(slot) truth, _ = fitsio.read_image(path) if path else (None, None) if truth is None: unreadable += 1 continue score = metrics.score_fill(filled[band], truth, wavelength, gap_lengths.get(slot, 0)) scores.append(score) frame_id = db.frame_id_by_slot(conn, *slot) if frame_id is not None: rows.append((case["id"], frame_id, label, source, json.dumps(score.as_dict()))) if index and index % 200 == 0: print(f" {index}/{len(targets)}", flush=True) if unreadable: print(f" {unreadable} targets could not be read or filled") return scores, rows, attempted class _LearnedFiller: """Joint six-band fills for one case: stack assembly, calibration, inference. One instance per case (or per satellite pass), shared between the scoring path (`_fill_learned`) and the render path (`cmd_render --variant learned`) so the two cannot drift apart in how they build the model's inputs. """ def __init__(self, overlay, satellites, wavelengths, bad): self.overlay = overlay self.satellites = tuple(satellites) self.wavelengths = tuple(wavelengths) self.read = _band_reader(overlay, wavelengths, bad) self.clean_times = _clean_times(overlay, satellites, wavelengths, bad) self.band_times = _band_clean_times(overlay, satellites, wavelengths, bad) # Times where *both* satellites are clean in every band: the calibration # pairs the cross-satellite photometric transfer is fitted from. A # target's own time is never in this list -- a target is a bad slot. self.paired_times = ( sorted(set(self.clean_times.get(satellites[0], [])) & set(self.clean_times.get(satellites[1], []))) if len(satellites) > 1 else [] ) def fill(self, satellite, when): """A (6, H, W) reconstruction for one slot, or None if nothing usable.""" loaded = fillers.load_learned() offsets = getattr(loaded[0], "offsets", None) if loaded else None stack = _build_stack(self.overlay, satellite, when, self.satellites, self.wavelengths, self.read, self.band_times, offsets=offsets) calibration = None if self.paired_times: other = next(s for s in self.satellites if s != satellite) pair_time = _closest(self.paired_times, when) pair = (self.read(other, pair_time), self.read(satellite, pair_time)) if _complete(pair[0]) and _complete(pair[1]): calibration = pair return fillers.learned(fillers.FillContext(stack=stack, calibration=calibration)) def _clean_times(overlay, satellites, wavelengths, bad): """Per satellite, the times with a usable frame in every band. Computed once for the whole case. Deriving it inside :func:`_build_stack` meant sorting all 31,488 slots afresh for each of ~900 targets, which is most of a minute spent recomputing the same list. """ times = {} for satellite in satellites: usable = defaultdict(int) for slot in overlay.slots(): if slot[0] != satellite or slot in bad: continue if overlay.path(slot) is not None: usable[slot[2]] += 1 # *Every* band must be clean. Counting a time as usable when only some # bands were would hand out anchors marked 'available' that carry damaged # pixels, and -- worse -- let a damaged band into the calibration pair the # cross-satellite gain is fitted from. times[satellite] = sorted(t for t, bands in usable.items() if bands == len(wavelengths)) return times def _band_clean_times(overlay, satellites, wavelengths, bad): """Per satellite and band, the sorted times that band is present and clean: {satellite: {wavelength: [times]}}. The per-band counterpart of :func:`_clean_times`, for anchors and the fill policy, where a band observed is a band usable whatever its frame's other bands are doing.""" times = {satellite: {wavelength: [] for wavelength in wavelengths} for satellite in satellites} for slot in overlay.slots(): satellite, wavelength, when = slot if (satellite in times and wavelength in times[satellite] and slot not in bad and overlay.path(slot) is not None): times[satellite][wavelength].append(when) for bands in times.values(): for series in bands.values(): series.sort() return times def unfillable_by_band(overlay, satellites, wavelengths, bad, timeline, limit): """Times inside a run of more than `limit` slots where no satellite holds some band clean: the slots the fill policy (:data:`availability.MAX_DUAL_GAP`) leaves black. Judged band by band. Requiring one satellite to hold all six bands made 2024-03-03 a day-long "dual outage" although GOES-16 lacked only 94/131 and GOES-18 only 304 -- every band was observed, and the stack now uses partial frames, so only a band nobody observed is a gap. """ band_times = _band_clean_times(overlay, satellites, wavelengths, bad) beyond = set() for wavelength in wavelengths: beyond |= availability.unfillable_times( timeline, {s: set(band_times[s][wavelength]) for s in satellites}, limit=limit, ) return beyond def _build_stack(overlay, satellite, target_time, satellites, wavelengths, read, band_times, offsets=None): """Assemble the multi-frame stack the learned filler fuses. Mirrors `suvi.samples.Sampler.build` so a model trained on shards meets the same stack here: the same multi-scale offsets on both satellites, the target's own slot at dt=0 holding whichever of its bands are usable (the renderer fills only the rest), and the nearest usable frame in each direction as an anchor so a 300-slot gap is never empty. Entries are **all six bands at once**, because the model is joint across them -- when one band is damaged the others usually are not, and that is much of what makes the fusion work. `read(satellite, time)` returns a (6, H, W) array or None (see :func:`_band_reader`), with every band it will not show the model -- absent, unreadable or judged bad -- as NaN. A slot missing some bands is still an entry: ``bands`` records which it holds, and the model uses the rest -- per band, not all or nothing. So every entry is `available` or `missing`. `band_times` is :func:`_band_clean_times` under the same judgment. """ from suvi import quality, samples entries = [] layout = samples.stack_layout(satellites, satellite, offsets or samples.OFFSETS) for source, offset in layout + [(satellite, 0)]: when = target_time + offset * paths.CADENCE if not any((source, wl, when) in overlay.archive for wl in wavelengths): continue image = read(source, when) entries.append({ "image": image, "bands": band_presence(image) if image is not None else (False,) * 6, "state": quality.AVAILABLE if image is not None else quality.MISSING, "dt": float(when - target_time), "same_satellite": source == satellite, "slot": (source, when), }) # Anchors, so the far end of a long outage is still reachable: per side, the # nearest slot where each band is clean -- usually one slot for all six, but a # band missing for a long stretch still gets its own. seen = {entry["dt"] for entry in entries if entry["same_satellite"]} clean = band_times.get(satellite, {}) anchors = set() for before in (True, False): for wavelength in wavelengths: anchor = _nearest(clean.get(wavelength, []), target_time, before=before) if anchor is not None and float(anchor - target_time) not in seen: anchors.add(anchor) for anchor in sorted(anchors): image = read(satellite, anchor) if image is None: continue entries.append({ "image": image, "bands": band_presence(image), "state": quality.AVAILABLE, "dt": float(anchor - target_time), "same_satellite": True, "slot": (satellite, anchor), }) return entries #: Bytes of decoded slots :func:`_band_reader` keeps (~6 full 1280px slots). The #: exponential ladder means consecutive targets share few entries -- a 973-target #: replay found ~24 distinct slots per target and a 48-slot cache serving 29% of #: reads -- so a large cache mostly held dead frames, ~1.9 GB of them. READER_BUDGET = 256 * 1024**2 def _band_reader(overlay, wavelengths, bad, budget=READER_BUDGET): """Reader of six-band slots for :func:`_build_stack`, with a bounded cache. Bands are read one by one: a band the slot lacks, that cannot be read, or that is judged bad (in `bad`) comes back all-NaN while the others keep their pixels, and only a slot with no usable band at all is None. Treating a slot as all-or-nothing threw away a counterpart's five good bands because its sixth was missing -- on 2024-03-03 GOES-16 lacked 94/131 and GOES-18 lacked 304 all day, so every band was observed and the model was handed nothing. A judged-bad band is withheld, as in training (:mod:`suvi.quality`): shown, its damaged pixels took blend weight -- on 2024-04-16/17 and 05-03, when most 94 and 131 frames were short-exposure static (N_LONG=0), enough to speckle the dark far field of every fill. Damage the judgment did not flag resolves through the overlay to its *damaged* pixels; a filler handed the pristine frame for a slot the case corrupted would be reading the answer key. Anchors and the calibration pair recur across a gap's targets, so a cache of recently used slots saves re-reading them; it is bounded by `budget` bytes, least recently used first, because six bands of 1280x1280 float32 is 39 MB. """ cache = OrderedDict() held = 0 def read(satellite, when): nonlocal held key = (satellite, when) if key in cache: cache.move_to_end(key) return cache[key] value = None for index, wavelength in enumerate(wavelengths): slot = (satellite, wavelength, when) path = overlay.path(slot) if slot not in bad else None image, _ = fitsio.read_image(path) if path else (None, None) if image is None: continue if value is None: # one array per slot, filled band by band value = np.full((len(wavelengths), *image.shape), np.nan, dtype=np.float32) value[index] = image cache[key] = value held += value.nbytes if value is not None else 0 while held > budget and len(cache) > 1: _, old = cache.popitem(last=False) held -= old.nbytes if old is not None else 0 return value return read def band_presence(image): """Which bands of a reader's slot hold pixels: a missing band is all-NaN.""" return tuple(bool(held) for held in ~np.isnan(image).all(axis=(1, 2))) def _complete(image): """A slot holding all six bands, as the calibration pair and truth need.""" return image is not None and all(band_presence(image)) def _closest(times, target): """Closest time in sorted `times` to `target`, in either direction.""" index = bisect.bisect_left(times, target) candidates = times[max(0, index - 1) : index + 1] return min(candidates, key=lambda t: abs(t - target)) def _nearest(times, target, before=True): """Closest time in `times` strictly before or after `target`.""" if before: candidates = [t for t in times if t < target] return max(candidates) if candidates else None candidates = [t for t in times if t > target] return min(candidates) if candidates else None def _header_for(overlay, slot): path = overlay.truth_path(slot) if path is None: return {} values, _ = fitsio.scan_header(path) return values # -------------------------------------------------------------------------- compare #: Gap-length bins, as (low, high) in slots with `None` for open-ended. #: Exact gap lengths make a table whose columns are whatever the case happened to #: inject -- on the trace-driven case that was twelve columns, eight of which #: rested on 5 to 21 independent targets and could not separate a tenth of a dB #: from noise. Doubling bins pool neighbouring lengths, which behave alike, so #: each column carries enough trials to mean something. GAP_BINS = ((1, 1), (2, 2), (3, 4), (5, 8), (9, 16), (17, 32), (33, 64), (65, 128), (129, 256), (257, 512), (513, None)) def bin_label(bounds): low, high = bounds if high is None: return f"{low}+" return str(low) if low == high else f"{low}-{high}" def gap_bin(length): """The bin a gap of `length` slots falls in, or None if it is below 1.""" for bounds in GAP_BINS: low, high = bounds if length >= low and (high is None or length <= high): return bounds return None def cmd_compare(args): """Fill quality by gap-length bin, on one common set of frames. Restricting to the frames a chosen filler actually scored is the point. A model evaluated on 200 sampled targets and a baseline evaluated on 5,304 are not comparable numbers, and putting them in the same table implies they are. With --restrict-to the table is one set of frames scored several ways. Two things the old exact-length table got wrong, both of which flattered small samples. It counted *band-slots*, but the six bands of one frame come from one instrument at one instant and are filled from a single stack, so they are one trial and not six -- the counts it printed were 6x the real sample size. And it showed only the mean, which hides whether a filler is reliable or merely averages well; a filler that is usually fine and occasionally catastrophic is the one that matters operationally, and it looks identical to a steady one in a column of means. So each cell is worst/mean/best, over `n` independent targets. """ conn = db.connect(args.db, readonly=True) case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone() if case is None: raise SystemExit(f"No case named {args.case!r}") gaps = {row["frame_id"]: row["gap_length"] for row in conn.execute( "SELECT frame_id, gap_length FROM bench_injection WHERE case_id = ?", (case["id"],) )} rows = conn.execute( "SELECT frame_id, filler, scores_json FROM bench_fill_result " "WHERE case_id = ? AND source = ?", (case["id"], args.source) ).fetchall() if not rows: raise SystemExit(f"No fill results for case {args.case!r} from {args.source!r}") by_filler = defaultdict(dict) for row in rows: by_filler[row["filler"]][row["frame_id"]] = json.loads(row["scores_json"]) common = None if args.restrict_to: if args.restrict_to not in by_filler: raise SystemExit(f"No results for filler {args.restrict_to!r}") common = set(by_filler[args.restrict_to]) for name, scores in by_filler.items(): common &= set(scores) print(f"Restricted to the {len(common)} frames scored by " f"every filler including '{args.restrict_to}'") metric = args.metric # frame_id -> (satellite, t_start): the six bands of one frame are one trial, # so trials are counted over distinct (satellite, time) targets. target_of = {row["id"]: (row["satellite"], row["t_start"]) for row in conn.execute("SELECT id, satellite, t_start FROM frame")} scored = defaultdict(dict) # (filler, bin) -> {frame_id: value} for name, scores in by_filler.items(): for frame_id, values in scores.items(): if common is not None and frame_id not in common: continue bounds = gap_bin(gaps.get(frame_id, 0)) if bounds is not None: scored[(name, bounds)][frame_id] = values[metric] names = set(by_filler) if args.fillers: names &= set(args.fillers.split(",")) order = [n for n in ("hold_last", "linear_blend", "optical_flow", "solar_rotation", "crosssat", "learned") if n in names] order += [n for n in sorted(names) if n not in order] # Every bin, including the empty ones: a column of '--' says the case never # injected a gap that long, which is a fact about the measurement worth # seeing next to the numbers rather than silently dropping. shown = list(GAP_BINS) # Each bin is narrowed to the frames *every* filler present in it scored, so # `n` is one number rather than a per-filler range and the rows in a column # describe the same work. Without this the counts differ by filler -- on the # trace-driven case optical_flow had 30 targets at gap 1 where v7 had 11 -- # and reading down such a column compares different problems. keep, counts, table = {}, [], defaultdict(dict) for bounds in shown: present = [n for n in order if scored.get((n, bounds))] frames = set(scored[(present[0], bounds)]) if present else set() for name in present[1:]: frames &= set(scored[(name, bounds)]) keep[bounds] = frames counts.append(str(len({target_of.get(f, f) for f in frames}))) for name in present: table[name][bounds] = [scored[(name, bounds)][f] for f in frames] dropped = {name: sum(len(scored[(name, b)]) for b in shown) - sum(len(table[name].get(b, ())) for b in shown) for name in order} width = 15 print(f"\nCase '{args.case}', source '{args.source}' -- {metric} " f"by gap length (worst / mean / best)\n") print(f"{'gap (slots)':>16} " + " ".join(f"{bin_label(b):>{width}}" for b in shown)) print(f"{'n trials':>16} " + " ".join(f"{c:>{width}}" for c in counts)) print(f"{'':>16} " + " ".join("-" * width for _ in shown)) for name in order: cells = table[name] printed = [] for bounds in shown: values = cells.get(bounds) if not values: printed.append(f"{'--':>{width}}") else: cell = (f"{min(values):.1f}/{sum(values) / len(values):.1f}" f"/{max(values):.1f}") printed.append(f"{cell:>{width}}") total = len({target_of.get(f, f) for b in shown for f in keep[b] if b in table[name]}) print(f"{name:>16} " + " ".join(printed) + f" {total:>6} total") if any(dropped.values()): print("\n Frames outside the common set, excluded so every column " "compares like with like: " + ", ".join(f"{name} {count}" for name, count in dropped.items() if count)) best = best_classical(table, shown) if best: print("\n best hand-written: " + " ".join( f"{bin_label(length)}:{name}({value:.2f})" for length, (name, value) in best.items())) for name, row in learned_margins(table, best).items(): print(f" {name} minus best: " + " ".join( f"{bin_label(length)}:{delta:+.2f}" for length, delta in row.items())) return 0 #: The hand-written fillers. Any other column in a comparison is a learned #: checkpoint, scored under its ``--learned-name`` (v7, v8, prior...). CLASSICAL = tuple(name for name in fillers.FILLERS if name != "learned") def best_classical(table, lengths): """Per gap bin, (name, mean) of the best hand-written filler: the bar a learned filler has to clear. It is the best of them at each length, not any single one -- the crossover is real and each wins somewhere.""" best = {} for length in lengths: options = {name: sum(cells[length]) / len(cells[length]) for name, cells in table.items() if name in CLASSICAL and cells.get(length)} if options: best[length] = max(options.items(), key=lambda kv: kv[1]) return best def learned_margins(table, best): """{learned column: {bin: its mean minus the best hand-written mean}}, in bin order, for the bins both have.""" margins = {} for name, cells in table.items(): if name in CLASSICAL: continue row = {length: sum(cells[length]) / len(cells[length]) - best[length][1] for length in best if cells.get(length)} if row: margins[name] = row return margins # --------------------------------------------------------------------------- report def cmd_report(args): conn = db.connect(args.db, readonly=True) case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone() if case is None: raise SystemExit(f"No case named {args.case!r}") window = conn.execute( "SELECT * FROM bench_window WHERE id = ?", (case["window_id"],) ).fetchone() plan = json.loads(case["plan_json"]) print(f"Case '{args.case}' on window '{window['name']}' " f"({format_time(window['t_start'])} .. {format_time(window['t_end'])})") print(f" plan: {plan}\n") injected = {} for row in conn.execute( """ SELECT f.satellite, f.wavelength, f.t_start, i.mode FROM bench_injection i JOIN frame f ON f.id = i.frame_id WHERE i.case_id = ? """, (case["id"],), ): injected[(row["satellite"], row["wavelength"], row["t_start"])] = row["mode"] legacy_good = { (row["satellite"], row["wavelength"], row["t_start"]) for row in conn.execute( """ SELECT f.satellite, f.wavelength, f.t_start FROM bench_truth t JOIN frame f ON f.id = t.frame_id WHERE t.window_id = ? AND t.is_good = 1 """, (window["id"],), ) } truth_bad = _truth_bad_slots(conn, window["id"]) print("DETECTION") print(f"{'detector':>14} {'prec':>7} {'recall':>7} {'F1':>7} {'FPR':>8} " f"{'unknown':>8} {'legacy-dis':>11}") per_detector = {} verdict_maps = {} for name in detectors.ALL_DETECTORS: run_id = db.latest_run_id(conn, name, {"case": args.case}) if run_id is None: continue verdicts = {} for row in conn.execute( """ SELECT f.satellite, f.wavelength, f.t_start, d.verdict, d.reason FROM detection d JOIN frame f ON f.id = d.frame_id WHERE d.run_id = ? """, (run_id,), ): slot = (row["satellite"], row["wavelength"], row["t_start"]) verdicts[slot] = detectors.Verdict(row["verdict"], row["reason"], {}) # Deleted slots have no frame to judge, and vetted-bad archive slots are # damage the case never injected: a detector that flags one is *right*, # so counting it as a false positive would punish correctness. scoreable = {s: v for s, v in verdicts.items() if injected.get(s) != "delete" and s not in truth_bad} score = metrics.score_detection( scoreable, {s: m for s, m in injected.items() if m != "delete"}, legacy_good, ) per_detector[name] = score verdict_maps[name] = verdicts print(f"{name:>14} {score.precision:>7.3f} {score.recall:>7.3f} {score.f1:>7.3f} " f"{score.false_positive_rate:>8.4f} {score.unknown:>8} " f"{score.legacy_disagreements:>11}") if args.combinations and len(per_detector) > 1: print("\nDETECTOR COMBINATIONS (ranked by false-positive rate, then F1)") print(f"{'policy':>9} {'detectors':>44} {'prec':>7} {'recall':>7} {'F1':>7} {'FPR':>8}") scoreable_injected = {s: m for s, m in injected.items() if m != "delete"} rows = [] names = sorted(per_detector) for size in range(2, len(names) + 1): for subset in itertools.combinations(names, size): for policy in metrics.COMBINATION_POLICIES: combined = metrics.combine_verdicts( [verdict_maps[name] for name in subset], policy ) combined = { s: v for s, v in combined.items() if injected.get(s) != "delete" and s not in truth_bad } score = metrics.score_detection( combined, scoreable_injected, legacy_good ) rows.append((score.false_positive_rate, -score.f1, policy, subset, score)) rows.sort(key=lambda r: (r[0] if np.isfinite(r[0]) else 9e9, r[1] if np.isfinite(r[1]) else 9e9)) for _, _, policy, subset, score in rows[: args.combination_limit]: label = "+".join(n.replace("_v1", "") for n in subset) print(f"{policy:>9} {label:>44} {score.precision:>7.3f} {score.recall:>7.3f} " f"{score.f1:>7.3f} {score.false_positive_rate:>8.4f}") modes = sorted({m for m in injected.values() if m != "delete"}) if modes and per_detector: print("\nRECALL BY CORRUPTION MODE") header = f"{'mode':>18}" + "".join(f"{n[:11]:>13}" for n in per_detector) print(header) for mode in modes: line = f"{mode:>18}" for score in per_detector.values(): value = score.recall_by_mode.get(mode) line += f"{value:>13.3f}" if value is not None else f"{'-':>13}" print(line) print("\nFILL") rows = conn.execute( "SELECT filler, source, scores_json FROM bench_fill_result WHERE case_id = ?", (case["id"],), ).fetchall() if not rows: print(" (no fill results; run `bench.py fill` first)") else: grouped = defaultdict(list) for row in rows: payload = json.loads(row["scores_json"]) grouped[(row["source"], row["filler"])].append( metrics.FillScore(**payload) ) print(f"{'source':>10} {'filler':>16} {'n':>6} {'rmse':>10} {'psnr':>8} {'ssim':>7}") for (source, filler_name), scores in sorted(grouped.items()): summary = metrics.summarise_fills(scores) print(f"{source:>10} {filler_name:>16} {summary['n']:>6} " f"{summary['rmse']:>10.4g} {summary['psnr']:>8.2f} {summary['ssim']:>7.4f}") print("\nFILL QUALITY BY GAP LENGTH (psnr, dB)") gaps = sorted({s.gap_frames for scores in grouped.values() for s in scores}) print(f"{'source/filler':>28}" + "".join(f"{g:>8}" for g in gaps)) for (source, filler_name), scores in sorted(grouped.items()): summary = metrics.summarise_fills(scores) line = f"{source + '/' + filler_name:>28}" for gap in gaps: entry = summary["by_gap"].get(gap) line += f"{entry['psnr']:>8.2f}" if entry else f"{'-':>8}" print(line) if args.json: output = { "case": args.case, "window": window["name"], "plan": plan, "detection": {n: s.as_dict() for n, s in per_detector.items()}, } with open(args.json, "w") as handle: json.dump(output, handle, indent=2) print(f"\nWrote {args.json}") return 0 # --------------------------------------------------------------------------- render #: Slots the current pipeline will carry forward from the last good frame. Beyond #: this merger_FITS.py gives up and the timestamp produces no composite at all. TODAY_MAX_TIME_GAP = 3 def _resolve_band(overlay, slot, truth_cache, use_truth=False): """Read one band's pixels, caching, or None if it cannot be read. `use_truth` reads the original archive frame rather than what the case left in its place. The pristine variant needs it: a slot the case *deleted* has no overlay path at all, and resolving those through the overlay silently drops them -- which cost the ground-truth pane 221 of its timestamps, exactly the ones the other panes have to be compared against. """ key = (slot, use_truth) if key not in truth_cache: path = overlay.truth_path(slot) if use_truth else overlay.path(slot) image, _ = fitsio.read_image(path) if path else (None, None) truth_cache[key] = image return truth_cache[key] def _render_one(job): """Composite a single timestamp for one satellite. Runs in a worker.""" import merger_FITS arrays, timestamp, out_path = job partial = out_path + ".partial" try: image = merger_FITS.composite_from_arrays(arrays, timestamp) # Renamed into place only once complete: every renderer resumes by # skipping composites that exist, so a file cut short by a kill would # otherwise be kept forever. image.save(partial, format="JPEG", quality=95) os.replace(partial, out_path) return out_path, None except Exception as exc: if os.path.exists(partial): os.remove(partial) return out_path, f"{type(exc).__name__}: {exc}" def cmd_render(args): """Produce the composite stream for one variant of a case. Three variants, which together answer 'is the repair visible, and is it better than what we do now': * ``pristine`` -- the archive untouched. Ground truth. * ``today`` -- what the current pipeline does with the injected damage: carry the last good frame across gaps of up to TODAY_MAX_TIME_GAP slots, and emit *nothing at all* for longer ones, which is what leaves ffmpeg_video.py padding with black frames. A 300-slot gap produces 300 missing composites. * ``new`` -- the chosen method: any(disc_v1 + header_v1) detection, filled with optical_flow at every gap length. * ``learned`` -- the trained fusion model: same detection, damaged timestamps reconstructed jointly across all six bands from the aligned two-satellite stack. Needs a checkpoint (SUVI_MODEL). * ``damaged`` -- the input as the fillers actually see it: damaged frames with their damaged pixels, deleted slots as holes, nothing repaired. The honest "before" pane for any comparison. """ fitsio.quiet_astropy() conn = db.connect(args.db, readonly=True) case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone() if case is None: raise SystemExit(f"No case named {args.case!r}") window = conn.execute( "SELECT * FROM bench_window WHERE id = ?", (case["window_id"],) ).fetchone() window_satellites = tuple(json.loads(window["satellites"])) satellites = window_satellites if args.satellites: wanted = parse_int_list(args.satellites) satellites = tuple(s for s in satellites if s in wanted) if not satellites: raise SystemExit(f"none of {wanted} are in this window") wavelengths = tuple(json.loads(window["wavelengths"])) if len(wavelengths) != 6: raise SystemExit(f"a composite needs all six bands; window has {wavelengths}") overlay = _build_overlay(conn, case, window) # Pristine shows the window as it really is, so it reads the archive directly and # treats nothing as damaged. pristine = args.variant == "pristine" damaged = args.variant == "damaged" # Vetted-bad archive slots are damage the case didn't inject but production # would still have to repair; the filling variants treat them the same way. bad = (set() if pristine else _bad_slots(conn, case, args.detector) | _truth_bad_slots(conn, case["window_id"])) times = sorted({slot[2] for slot in overlay.archive}) # The filling variants honour the same policy as `fill`: a dual gap longer # than --max-dual-gap slots stays a gap (skipped composite -> black frame), # like ffmpeg_video's max_frame_interp for the single-satellite case. Judged # against the *window's* satellites even when rendering only one, or a # one-satellite render would mistake every long one-sided gap for a dual one. unfillable = set() if not pristine and not damaged: unfillable = unfillable_by_band( overlay, window_satellites, wavelengths, bad, cases.timeline(window["t_start"], window["t_end"]), args.max_dual_gap, ) print(f"{args.variant}: {len(times)} timestamps x {len(satellites)} satellites, " f"{len(bad)} slots treated as bad" + (f", {len(unfillable)} timestamps beyond the dual-gap policy" if unfillable else "")) if args.variant == "learned" and fillers.load_learned() is None: raise SystemExit(f"variant 'learned' needs a checkpoint " f"(set {fillers.LEARNED_CHECKPOINT_ENV})") for satellite in satellites: out_dir = os.path.join(args.out, args.variant, f"goes{satellite}") os.makedirs(out_dir, exist_ok=True) truth_cache = {} jobs = [] skipped = 0 joint = (_LearnedFiller(overlay, satellites, wavelengths, bad) if args.variant == "learned" else None) for when in times: target_path = os.path.join(out_dir, f"Composite-{int(when)}.jpg") if os.path.exists(target_path): continue # crash-resume: composites are immutable if when in unfillable: skipped += 1 # policy: the gap stays a gap continue slots = [(satellite, band, when) for band in wavelengths] arrays = [] usable = True # The learned model reconstructs all six bands at once, so a damaged # timestamp gets one joint fill and its bands are sliced out below. joint_fill = None for band_index, slot in enumerate(slots): # The 'damaged' variant repairs nothing: overridden slots resolve to # their damaged pixels below, and deleted slots kill the composite, # which the encoder shows as a black frame -- the gap itself. if not pristine and not damaged and (slot in bad or overlay.path(slot) is None): if joint is not None: if joint_fill is None: joint_fill = joint.fill(satellite, when) if joint_fill is None or np.isnan(joint_fill[band_index]).all(): usable = False # nothing in the stack holds this band break arrays.append(joint_fill[band_index]) continue replacement, ok = _replacement_band( args.variant, overlay, slot, bad, truth_cache, wavelengths ) if not ok: usable = False break arrays.append(replacement) else: image = _resolve_band(overlay, slot, truth_cache, use_truth=pristine) if image is None: usable = False break arrays.append(image) if not usable: # No composite for this timestamp -- exactly what the current # pipeline does past its gap limit, and what ffmpeg then pads. skipped += 1 continue jobs.append((arrays, when, target_path)) if len(jobs) >= args.batch: _flush_renders(jobs, args.workers) jobs.clear() truth_cache.clear() print(f" g{satellite}: {len(os.listdir(out_dir))} rendered, {skipped} skipped") if jobs: _flush_renders(jobs, args.workers) print(f" g{satellite}: {len(os.listdir(out_dir))} composites, {skipped} timestamps skipped") conn.close() return 0 def _replacement_band(variant, overlay, slot, bad, truth_cache, wavelengths): """What to substitute for a band the case removed. Returns (array, usable).""" satellite, band, when = slot series = [s for s in overlay.series(satellite, band) if s not in bad] times = [s[2] for s in series] before = _nearest(times, when, before=True) after = _nearest(times, when, before=False) if variant == "today": # merger_FITS carries the last good frame a few slots, then gives up. if before is None or (when - before) // paths.CADENCE > TODAY_MAX_TIME_GAP: return None, False return _resolve_band(overlay, (satellite, band, before), truth_cache), True context = fillers.FillContext( before=_resolve_band(overlay, (satellite, band, before), truth_cache) if before else None, dt_before=when - before if before else 0.0, after=_resolve_band(overlay, (satellite, band, after), truth_cache) if after else None, dt_after=after - when if after else 0.0, ) filled = fillers.optical_flow(context) return (filled, True) if filled is not None else (None, False) def _flush_renders(jobs, workers): failures = 0 with Pool(workers, initializer=fitsio.quiet_astropy) as pool: for _, error in pool.imap_unordered(_render_one, jobs, 4): failures += bool(error) if failures: print(f" WARNING: {failures} composites failed to render") # ------------------------------------------------------------------------- examples def cmd_examples(args): """Show the frames a detector got wrong, so a number becomes a picture. A precision/recall table says how often a detector is wrong; it does not say what it is wrong *about*. For `header_v1`, whose recall is low but precision perfect, the interesting set is the false negatives -- damage that leaves IMG_MEAN, ECLIPSE and EMPTY all looking normal. """ import matplotlib matplotlib.use("Agg") from matplotlib import pyplot as plt fitsio.quiet_astropy() conn = db.connect(args.db, readonly=True) case = conn.execute("SELECT * FROM bench_case WHERE name = ?", (args.case,)).fetchone() if case is None: raise SystemExit(f"No case named {args.case!r}") window = conn.execute( "SELECT * FROM bench_window WHERE id = ?", (case["window_id"],) ).fetchone() overlay = _build_overlay(conn, case, window) run_id = db.latest_run_id(conn, args.detector, {"case": args.case}) if run_id is None: raise SystemExit( f"No run of {args.detector!r} for case {args.case!r}; run detect first." ) injected = {} for row in conn.execute( """ SELECT f.satellite, f.wavelength, f.t_start, i.mode, i.severity FROM bench_injection i JOIN frame f ON f.id = i.frame_id WHERE i.case_id = ? """, (case["id"],), ): injected[(row["satellite"], row["wavelength"], row["t_start"])] = ( row["mode"], row["severity"] ) wanted = [] for row in conn.execute( """ SELECT f.satellite, f.wavelength, f.t_start, d.verdict, d.reason, d.scores_json FROM detection d JOIN frame f ON f.id = d.frame_id WHERE d.run_id = ? """, (run_id,), ): slot = (row["satellite"], row["wavelength"], row["t_start"]) mode = injected.get(slot, (None, None))[0] if mode == "delete": continue # no frame to show corrupted = mode is not None if args.kind == "fn" and corrupted and row["verdict"] == "good": wanted.append((slot, mode, row)) elif args.kind == "fp" and not corrupted and row["verdict"] == "bad": wanted.append((slot, None, row)) print(f"{args.detector} {args.kind.upper()}s on case '{args.case}': {len(wanted)}") if not wanted: print(" none -- nothing to plot.") return 0 by_mode = defaultdict(int) for _, mode, _ in wanted: by_mode[mode or "(uninjected)"] += 1 for mode, count in sorted(by_mode.items(), key=lambda kv: -kv[1]): print(f" {count:>4} {mode}") wanted = wanted[: args.limit] columns = min(6, len(wanted)) rows = (len(wanted) + columns - 1) // columns figure, axes = plt.subplots(rows, columns, figsize=(3 * columns, 3.4 * rows)) axes = np.atleast_2d(axes) for index, (slot, mode, row) in enumerate(wanted): axis = axes[index // columns][index % columns] path = overlay.path(slot) image, _ = fitsio.read_image(path) if path else (None, None) if image is not None: axis.imshow(metrics.to_display(image, slot[1]), cmap="gray", vmin=0, vmax=1) scores = json.loads(row["scores_json"] or "{}") shown = ", ".join(f"{k}={v:.3g}" for k, v in list(scores.items())[:3]) axis.set_title( f"g{slot[0]} {slot[1]}A {format_time(slot[2])[5:]}\n" f"{mode or 'clean'} -> {row['verdict']}\n{shown}", fontsize=6, ) axis.axis("off") for index in range(len(wanted), rows * columns): axes[index // columns][index % columns].axis("off") figure.suptitle(f"{args.detector} {args.kind.upper()}s - case {args.case}") figure.tight_layout() figure.savefig(args.out, dpi=150) plt.close(figure) print(f" wrote {args.out}") return 0 # ------------------------------------------------------------------------------ cli def build_parser(): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("--db", default=None, help="SQLite index (default: $SUVI_DB)") subparsers = parser.add_subparsers(dest="command", required=True) def add_window_args(sub): sub.add_argument("--start", type=parse_date, required=True) sub.add_argument("--end", type=parse_date, required=True) sub.add_argument("--satellites", default="16,18") sub.add_argument("--wavelengths", default="94,131,171,195,284,304") select = subparsers.add_parser("select-window", help="find clean stretches of archive") add_window_args(select) select.add_argument("--min-slots", type=int, default=120) select.add_argument("--limit", type=int, default=10) select.add_argument("--require-legacy-label", action="store_true", help="also require the stale _f suffix (off by default)") select.set_defaults(func=cmd_select_window) vet = subparsers.add_parser("vet-window", help="verify and freeze a ground-truth window") add_window_args(vet) vet.add_argument("--name", required=True) vet.add_argument("--notes", default=None) vet.add_argument("--allow-missing", action="store_true") vet.add_argument("--min-good-fraction", type=float, default=0.98) vet.add_argument("--contact-sheet", default=None, help="write a filmstrip PNG for review") vet.add_argument("--run", type=int, default=None, help="judgment run id (default: the run made under the current rules)") vet.set_defaults(func=cmd_vet_window) make = subparsers.add_parser("make-case", help="inject damage into a vetted window") make.add_argument("--window", required=True) make.add_argument("--name", required=True) make.add_argument("--seed", type=int, default=42) make.add_argument("--fraction", type=float, default=0.10) make.add_argument("--gap-lengths", default="1,2,3,5,10,30,60") make.add_argument("--gaps-per-length", type=int, default=None, help="place exactly this many gaps at each length, instead of " "sizing the case by --fraction") make.add_argument("--satellite-scope", default="mixed", choices=("g16", "g18", "both", "mixed")) make.add_argument("--wavelength-scope", default="all", choices=("all", "one")) make.add_argument("--modes", default=None, help="comma-separated; default is all") make.add_argument("--severity-low", type=float, default=corruptions.SEVERITY_RANGE[0], help="floor low enough that subtle damage exists: penumbral " "eclipses live below the old 0.5 floor") make.add_argument("--severity-high", type=float, default=corruptions.SEVERITY_RANGE[1]) make.add_argument("--from-trace", default=None, help="availability.npz: overlay a sampled real availability " "window instead of the parametric gap plan") make.add_argument("--overlay-dir", default=None) make.add_argument("--workers", type=int, default=8) make.add_argument("--progress", action="store_true") make.set_defaults(func=cmd_make_case) detect = subparsers.add_parser("detect", help="run detectors over a case") detect.add_argument("--case", required=True) detect.add_argument("--detectors", default=None, help="comma-separated; default is all") detect.add_argument("--workers", type=int, default=8) detect.set_defaults(func=cmd_detect) fill = subparsers.add_parser("fill", help="reconstruct flagged slots and score them") fill.add_argument("--case", required=True) fill.add_argument("--fillers", default=None, help="comma-separated; default is all") fill.add_argument("--max-targets", type=int, default=None, help="score only this many targets, spread evenly over gap " "lengths -- for a quick read on a model mid-training") fill.add_argument("--learned-name", default="learned", help="store the learned filler's scores under this name, so " "two checkpoints can be compared in one table rather " "than the newer overwriting the older") fill.add_argument("--max-dual-gap", type=int, default=availability.MAX_DUAL_GAP, help="decline slots inside a simultaneous two-satellite gap " "longer than this many slots (default: " f"{availability.MAX_DUAL_GAP} = 24 hours)") group = fill.add_mutually_exclusive_group(required=True) group.add_argument("--oracle", action="store_true", help="use perfect knowledge of which slots are bad") group.add_argument("--from-detector", default=None, help="use a detector's output, showing end-to-end error") fill.set_defaults(func=cmd_fill) compare = subparsers.add_parser( "compare", help="fill quality by gap length across fillers, on common frames") compare.add_argument("--case", required=True) compare.add_argument("--source", default="oracle") compare.add_argument("--metric", default="psnr", choices=("psnr", "ssim", "rmse", "log_rmse")) compare.add_argument("--restrict-to", default=None, help="score every filler on only the frames this one scored") compare.add_argument("--fillers", default=None, help="comma-separated subset to compare; a filler with " "sparse coverage shrinks the common set for every " "other, so dropping it can widen the table") compare.set_defaults(func=cmd_compare) report = subparsers.add_parser("report", help="summarise a case's results") report.add_argument("--case", required=True) report.add_argument("--json", default=None) report.add_argument("--combinations", action="store_true", help="also score every combination of detectors") report.add_argument("--combination-limit", type=int, default=15) report.set_defaults(func=cmd_report) render = subparsers.add_parser( "render", help="produce the composite stream for one variant of a case" ) render.add_argument("--case", required=True) render.add_argument("--variant", required=True, choices=("pristine", "today", "new", "learned", "damaged")) render.add_argument("--out", required=True, help="root directory for the streams") render.add_argument("--detector", default="oracle", help="which slots to treat as bad ('oracle', or a run name)") render.add_argument("--satellites", default=None, help="restrict to these satellites (default: all in the window)") render.add_argument("--max-dual-gap", type=int, default=availability.MAX_DUAL_GAP, help="leave simultaneous two-satellite gaps longer than " "this many slots unfilled (black frames)") render.add_argument("--workers", type=int, default=8) render.add_argument("--batch", type=int, default=48, help="composites per worker dispatch; bounds peak memory") render.set_defaults(func=cmd_render) examples = subparsers.add_parser( "examples", help="plot the frames a detector got wrong" ) examples.add_argument("--case", required=True) examples.add_argument("--detector", required=True) examples.add_argument("--kind", choices=("fp", "fn"), required=True) examples.add_argument("--limit", type=int, default=12) examples.add_argument("--out", default="examples.png") examples.set_defaults(func=cmd_examples) return parser def main(argv=None): args = build_parser().parse_args(argv) return args.func(args) or 0 if __name__ == "__main__": sys.exit(main())