678 lines
31 KiB
Python
678 lines
31 KiB
Python
|
#!/usr/bin/env python
|
||
|
|
"""Score bad-frame detectors against REAL, hand-confirmed archive defects.
|
||
|
|
|
||
|
|
The bench scores detectors on damage it injected itself, against a notion of
|
||
|
|
"clean" produced by the detectors being scored. That is how the production pair
|
||
|
|
reached precision 1.000 on the bench while passing saturated bars, ghost discs and
|
||
|
|
short-exposure static in the real archive. This scores them on
|
||
|
|
``labels/real_defects.json`` instead (built by ``defect_audit.py``, confirmed by
|
||
|
|
eye), reporting recall per defect category and false positives separately on
|
||
|
|
ordinary and flare frames -- the flare frames being the ones a detector tuned to
|
||
|
|
catch brightness anomalies is most likely to throw away.
|
||
|
|
|
||
|
|
detector_eval.py measure --labels labels/real_defects.json --out AUDIT/eval.json
|
||
|
|
detector_eval.py score --labels labels/real_defects.json --features AUDIT/eval.json
|
||
|
|
detector_eval.py stored --labels labels/real_defects.json [--run N]
|
||
|
|
detector_eval.py dry-run --labels labels/real_defects.json --start T --end T --out new.json
|
||
|
|
detector_eval.py sheet --rows new.json --out sheet.jpg [--run N]
|
||
|
|
|
||
|
|
``measure`` reads every labelled frame with enough context around it for the
|
||
|
|
series detectors (+/-TEMPORAL_WINDOW frames on both satellites) and stores each
|
||
|
|
detector's verdict plus the raw far-field measurements. ``score`` is cheap and
|
||
|
|
repeatable: combinations (``a+b`` = bad if either says bad) and far-field
|
||
|
|
thresholds (``far_field@4``) are evaluated from the stored results, so trying a
|
||
|
|
rule does not mean re-reading the archive. ``stored`` scores a judgment run already
|
||
|
|
in the index (the production one by default) -- the check that what the archive-wide
|
||
|
|
judge stored is what the rules promised. ``dry-run`` judges stored measurements in
|
||
|
|
memory (nothing written) and lists every frame whose verdict changes against a
|
||
|
|
reference run; ``sheet`` draws such frames between their nearest good neighbours
|
||
|
|
for review. Read-only against archive and index.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import calendar
|
||
|
|
import collections
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
import time
|
||
|
|
from multiprocessing import Pool
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
|
||
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
|
|
|
||
|
|
from defect_audit import CLEAN, DEFECTS, NEUTRAL
|
||
|
|
from suvi import db, detectors, fitsio, paths
|
||
|
|
|
||
|
|
#: Labelled frames of one band closer than this many slots share a context read.
|
||
|
|
CLUSTER_GAP = 24
|
||
|
|
#: Context read either side of a cluster. Two windows, so a frame's temporal
|
||
|
|
#: neighbours have their own neighbours for the structural-churn baseline.
|
||
|
|
CONTEXT_SLOTS = 2 * detectors.TEMPORAL_WINDOW
|
||
|
|
#: Verdicts each labelled frame gets from `measure`.
|
||
|
|
FRAME_VERDICTS = ("header_v1", "header_v1_old", "lunar_v1", "disc_v1", "geometry_v1",
|
||
|
|
"field_v1")
|
||
|
|
#: Every verdict a measured frame can carry.
|
||
|
|
MEASURED = FRAME_VERDICTS + ("temporal_v1", "crosssat_v1", "legacy")
|
||
|
|
#: Judged from stored scores at scoring time, against the current limits, so
|
||
|
|
#: tuning a limit needs no new read of the archive.
|
||
|
|
FROM_SCORES = {"temporal_v2": ("temporal_v2_scores", "TEMPORAL2_LIMITS")}
|
||
|
|
#: Rows `score` prints when none are asked for: what production used when the 2024
|
||
|
|
#: render and the corpus were made, what it uses now, and the alternatives.
|
||
|
|
DEFAULT_ROWS = ("pair_old", "pair", "legacy", "temporal_v1", "temporal_v2", "field_v1",
|
||
|
|
"production", "production+temporal_v2")
|
||
|
|
#: Named combinations, expanded before evaluation.
|
||
|
|
ALIASES = {"pair": "header_v1+disc_v1", "pair_old": "header_v1_old+disc_v1",
|
||
|
|
"production": "header_v1+lunar_v1+disc_v1+field_v1"}
|
||
|
|
|
||
|
|
|
||
|
|
# ------------------------------------------------------------------- clustering
|
||
|
|
|
||
|
|
|
||
|
|
def clusters(labels, gap=CLUSTER_GAP, cadence=paths.CADENCE):
|
||
|
|
"""Group labelled frames by band into runs whose members are < `gap` slots apart.
|
||
|
|
|
||
|
|
Both satellites share a cluster, because the cross-satellite check needs the
|
||
|
|
other satellite's frames at the same instants anyway.
|
||
|
|
"""
|
||
|
|
by_band = collections.defaultdict(list)
|
||
|
|
for record in labels:
|
||
|
|
by_band[record["wavelength"]].append(record)
|
||
|
|
out = []
|
||
|
|
for wavelength in sorted(by_band):
|
||
|
|
members = sorted(by_band[wavelength], key=lambda r: r["t_start"])
|
||
|
|
current = [members[0]]
|
||
|
|
for record in members[1:]:
|
||
|
|
if record["t_start"] - current[-1]["t_start"] < gap * cadence:
|
||
|
|
current.append(record)
|
||
|
|
else:
|
||
|
|
out.append((wavelength, current))
|
||
|
|
current = [record]
|
||
|
|
out.append((wavelength, current))
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
# ---------------------------------------------------------------------- measure
|
||
|
|
|
||
|
|
|
||
|
|
def _frame(job):
|
||
|
|
"""Header, thumbnail, production verdict and (for labelled frames) every
|
||
|
|
single-frame verdict. Returns (features, production_good, record)."""
|
||
|
|
slot, path, labelled = job
|
||
|
|
cards, header_error = fitsio.read_header(path)
|
||
|
|
values = fitsio.header_metadata(cards)
|
||
|
|
refcrota = cards.get("REFCROTA")
|
||
|
|
refcrota = float(refcrota) if isinstance(refcrota, (int, float)) else None
|
||
|
|
image, image_error = fitsio.read_image(path)
|
||
|
|
features = detectors.FrameFeatures(
|
||
|
|
slot=slot, header=values, error=header_error or (None if image is not None else image_error),
|
||
|
|
thumbnail=detectors.thumbnail(image) if image is not None else None)
|
||
|
|
_, production, _ = detectors.judge_frame(slot, cards, header_error,
|
||
|
|
lambda: (image, image_error))
|
||
|
|
good = production.verdict != detectors.BAD
|
||
|
|
if not labelled:
|
||
|
|
return features, good, None
|
||
|
|
|
||
|
|
def pack(verdict):
|
||
|
|
return [verdict.verdict, verdict.reason]
|
||
|
|
|
||
|
|
old = dict(values)
|
||
|
|
old.pop("n_long", None)
|
||
|
|
header = detectors.FrameFeatures(slot=slot, header=values, error=header_error)
|
||
|
|
record = {
|
||
|
|
"header_v1": pack(detectors.header_v1(header)),
|
||
|
|
"header_v1_old": pack(detectors.header_v1(detectors.FrameFeatures(
|
||
|
|
slot=slot, header=old, error=header_error))),
|
||
|
|
"lunar_v1": pack(detectors.lunar_v1(cards)),
|
||
|
|
}
|
||
|
|
if image is None:
|
||
|
|
record["disc_v1"] = pack(detectors.unreadable(image_error))
|
||
|
|
record["geometry_v1"] = pack(detectors.unreadable(image_error))
|
||
|
|
record["field_v1"] = pack(detectors.unreadable(image_error))
|
||
|
|
record["far_field"] = record["noise"] = None
|
||
|
|
else:
|
||
|
|
record["disc_v1"] = pack(detectors.disc_v1(image, slot[1], header=values))
|
||
|
|
record["geometry_v1"] = pack(detectors.geometry_v1(image, slot[1]))
|
||
|
|
field = detectors.field_v1(image, slot[1], header=values, refcrota=refcrota)
|
||
|
|
record["field_v1"] = pack(field)
|
||
|
|
record["field_scores"] = field.scores
|
||
|
|
radius = values.get("diam_sun")
|
||
|
|
radius = radius / 2.0 if radius else None
|
||
|
|
record["far_field"] = detectors.far_field(image, radius)
|
||
|
|
record["noise"] = detectors.far_field_noise(image, radius)
|
||
|
|
return features, good, record
|
||
|
|
|
||
|
|
|
||
|
|
def measure_cluster(job):
|
||
|
|
"""Every verdict for one cluster's labelled frames, with series context."""
|
||
|
|
wavelength, members, streams = job
|
||
|
|
fitsio.quiet_astropy()
|
||
|
|
wanted = {(r["satellite"], r["wavelength"], r["t_start"]) for r in members}
|
||
|
|
frames, records, good = {}, {}, set()
|
||
|
|
for satellite, rows in streams.items():
|
||
|
|
for t_start, path in rows:
|
||
|
|
slot = (satellite, wavelength, t_start)
|
||
|
|
features, passed, record = _frame((slot, path, slot in wanted))
|
||
|
|
frames[slot] = features
|
||
|
|
if passed:
|
||
|
|
good.add(slot)
|
||
|
|
if record is not None:
|
||
|
|
records[slot] = record
|
||
|
|
|
||
|
|
temporal, temporal2 = {}, {}
|
||
|
|
for satellite in streams:
|
||
|
|
ordered = sorted((f for s, f in frames.items() if s[0] == satellite),
|
||
|
|
key=lambda f: f.t_start)
|
||
|
|
for features, verdict in zip(ordered, detectors.temporal_v1(ordered)):
|
||
|
|
temporal[features.slot] = verdict
|
||
|
|
for features, verdict in zip(ordered, detectors.temporal_v2(ordered, good)):
|
||
|
|
temporal2[features.slot] = verdict
|
||
|
|
|
||
|
|
out = []
|
||
|
|
for record in members:
|
||
|
|
slot = (record["satellite"], record["wavelength"], record["t_start"])
|
||
|
|
measured = records.get(slot)
|
||
|
|
if measured is None: # vanished from the archive
|
||
|
|
continue
|
||
|
|
verdict = temporal.get(slot)
|
||
|
|
measured["temporal_v1"] = [verdict.verdict, verdict.reason] if verdict else None
|
||
|
|
second = temporal2.get(slot)
|
||
|
|
measured["temporal_v2_scores"] = second.scores if second else None
|
||
|
|
other = [s for s in streams if s != slot[0]]
|
||
|
|
cross = None
|
||
|
|
if other:
|
||
|
|
counterpart = (other[0], wavelength, slot[2])
|
||
|
|
mine, theirs = detectors.crosssat_v1(
|
||
|
|
frames.get(slot), frames.get(counterpart),
|
||
|
|
temporal.get(slot), temporal.get(counterpart))
|
||
|
|
cross = [mine.verdict, mine.reason]
|
||
|
|
measured["crosssat_v1"] = cross
|
||
|
|
out.append({"satellite": slot[0], "wavelength": wavelength, "t_start": slot[2],
|
||
|
|
**measured})
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def legacy_verdicts(conn, labels):
|
||
|
|
"""The legacy pixel filter's recorded verdict per labelled slot, where it has one."""
|
||
|
|
out = {}
|
||
|
|
for record in labels:
|
||
|
|
row = conn.execute(
|
||
|
|
"SELECT d.verdict FROM detection d JOIN detector_run r ON r.id = d.run_id "
|
||
|
|
"JOIN frame f ON f.id = d.frame_id WHERE r.name = 'legacy_filter' AND "
|
||
|
|
"f.satellite = ? AND f.wavelength = ? AND f.t_start = ? LIMIT 1",
|
||
|
|
(record["satellite"], record["wavelength"], record["t_start"])).fetchone()
|
||
|
|
if row is not None:
|
||
|
|
out[(record["satellite"], record["wavelength"], record["t_start"])] = row[0]
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_measure(args):
|
||
|
|
with open(args.labels) as handle:
|
||
|
|
labels = json.load(handle)["labels"]
|
||
|
|
conn = db.connect(args.db, readonly=True)
|
||
|
|
root = paths.data_root()
|
||
|
|
jobs = []
|
||
|
|
for wavelength, members in clusters(labels):
|
||
|
|
t0 = members[0]["t_start"] - CONTEXT_SLOTS * paths.CADENCE
|
||
|
|
t1 = members[-1]["t_start"] + CONTEXT_SLOTS * paths.CADENCE
|
||
|
|
streams = {}
|
||
|
|
for satellite in (16, 18):
|
||
|
|
streams[satellite] = [
|
||
|
|
(row[0], paths.abspath(row[1], root)) for row in conn.execute(
|
||
|
|
"SELECT t_start, path FROM frame WHERE satellite = ? AND wavelength = ? "
|
||
|
|
"AND t_start >= ? AND t_start <= ? AND size_bytes >= ? ORDER BY t_start",
|
||
|
|
(satellite, wavelength, t0, t1, 100_000))]
|
||
|
|
jobs.append((wavelength, members, streams))
|
||
|
|
reads = sum(len(rows) for _, _, s in jobs for rows in s.values())
|
||
|
|
print(f"{len(labels)} labelled frames in {len(jobs)} clusters; {reads} frames to read",
|
||
|
|
flush=True)
|
||
|
|
legacy = legacy_verdicts(conn, labels)
|
||
|
|
|
||
|
|
results, done = [], 0
|
||
|
|
with Pool(args.workers, initializer=fitsio.quiet_astropy) as pool:
|
||
|
|
for batch in pool.imap_unordered(measure_cluster, jobs):
|
||
|
|
results.extend(batch)
|
||
|
|
done += 1
|
||
|
|
if done % 50 == 0:
|
||
|
|
print(f" {done}/{len(jobs)} clusters", flush=True)
|
||
|
|
for record in results:
|
||
|
|
verdict = legacy.get((record["satellite"], record["wavelength"], record["t_start"]))
|
||
|
|
record["legacy"] = [verdict, None] if verdict else None
|
||
|
|
os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)
|
||
|
|
with open(args.out, "w") as handle:
|
||
|
|
json.dump({"labels": os.path.abspath(args.labels), "frames": results}, handle)
|
||
|
|
print(f"wrote {len(results)} measured frames to {args.out}")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
def _refresh_one(job):
|
||
|
|
slot, path = job
|
||
|
|
fitsio.quiet_astropy()
|
||
|
|
return slot, _frame((slot, path, True))[2]
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_refresh(args):
|
||
|
|
"""Re-run only the single-frame detectors on the labelled frames, in place.
|
||
|
|
|
||
|
|
After a frame detector changes, the series context `measure` read is still
|
||
|
|
valid, so this reads each labelled frame once instead of every neighbour.
|
||
|
|
"""
|
||
|
|
with open(args.features) as handle:
|
||
|
|
stored = json.load(handle)
|
||
|
|
conn = db.connect(args.db, readonly=True)
|
||
|
|
root = paths.data_root()
|
||
|
|
jobs = []
|
||
|
|
for record in stored["frames"]:
|
||
|
|
row = conn.execute("SELECT path FROM frame WHERE satellite = ? AND wavelength = ? "
|
||
|
|
"AND t_start = ?", _slot(record)).fetchone()
|
||
|
|
if row is not None:
|
||
|
|
jobs.append((_slot(record), paths.abspath(row[0], root)))
|
||
|
|
by_slot = {_slot(r): r for r in stored["frames"]}
|
||
|
|
with Pool(args.workers, initializer=fitsio.quiet_astropy) as pool:
|
||
|
|
for slot, fresh in pool.imap_unordered(_refresh_one, jobs, 8):
|
||
|
|
by_slot[slot].update(fresh)
|
||
|
|
with open(args.features + ".tmp", "w") as handle:
|
||
|
|
json.dump(stored, handle)
|
||
|
|
os.replace(args.features + ".tmp", args.features)
|
||
|
|
print(f"refreshed {len(jobs)} frames in {args.features}")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
# ------------------------------------------------------------------------ score
|
||
|
|
|
||
|
|
|
||
|
|
def far_field_baselines(frames, labels_by_slot):
|
||
|
|
"""Median far-field brightness per band over frames labelled `normal`."""
|
||
|
|
per_band = collections.defaultdict(list)
|
||
|
|
for frame in frames:
|
||
|
|
if (labels_by_slot.get(_slot(frame)) == "normal"
|
||
|
|
and frame.get("far_field") is not None):
|
||
|
|
per_band[frame["wavelength"]].append(frame["far_field"])
|
||
|
|
return {band: float(np.median(values)) for band, values in per_band.items() if values}
|
||
|
|
|
||
|
|
|
||
|
|
def _slot(record):
|
||
|
|
return (record["satellite"], record["wavelength"], record["t_start"])
|
||
|
|
|
||
|
|
|
||
|
|
def says_bad(frame, term, baselines):
|
||
|
|
"""Whether one detector term calls `frame` bad. UNKNOWN and absent are not bad."""
|
||
|
|
if term.startswith("far_field@"):
|
||
|
|
factor = float(term.split("@", 1)[1])
|
||
|
|
base = baselines.get(frame["wavelength"])
|
||
|
|
value = frame.get("far_field")
|
||
|
|
return base is not None and value is not None and value > factor * base
|
||
|
|
if term in FROM_SCORES:
|
||
|
|
key, limits = FROM_SCORES[term]
|
||
|
|
scores = frame.get(key) or {}
|
||
|
|
return any(scores.get(name, 0.0) > limit
|
||
|
|
for name, limit in getattr(detectors, limits).items())
|
||
|
|
if term not in MEASURED:
|
||
|
|
raise KeyError(f"unknown detector term {term!r}")
|
||
|
|
verdict = frame.get(term)
|
||
|
|
return bool(verdict) and verdict[0] == detectors.BAD
|
||
|
|
|
||
|
|
|
||
|
|
def expand(row):
|
||
|
|
"""'pair+temporal_v1' -> ['header_v1', 'disc_v1', 'temporal_v1']."""
|
||
|
|
terms = []
|
||
|
|
for part in row.split("+"):
|
||
|
|
terms.extend(ALIASES.get(part, part).split("+"))
|
||
|
|
return terms
|
||
|
|
|
||
|
|
|
||
|
|
def score(frames, labels_by_slot, rows, baselines):
|
||
|
|
"""{row: {category: (caught, total)}} over every labelled category present."""
|
||
|
|
table = {}
|
||
|
|
for row in rows:
|
||
|
|
terms = expand(row)
|
||
|
|
counts = collections.defaultdict(lambda: [0, 0])
|
||
|
|
for frame in frames:
|
||
|
|
category = labels_by_slot.get(_slot(frame))
|
||
|
|
if category is None or category in NEUTRAL:
|
||
|
|
continue
|
||
|
|
counts[category][1] += 1
|
||
|
|
if any(says_bad(frame, term, baselines) for term in terms):
|
||
|
|
counts[category][0] += 1
|
||
|
|
table[row] = {k: tuple(v) for k, v in counts.items()}
|
||
|
|
return table
|
||
|
|
|
||
|
|
|
||
|
|
def summary(counts):
|
||
|
|
"""(defect recall, normal FPR, flare FPR) from one row's per-category counts."""
|
||
|
|
def rate(categories):
|
||
|
|
caught = sum(counts.get(c, (0, 0))[0] for c in categories)
|
||
|
|
total = sum(counts.get(c, (0, 0))[1] for c in categories)
|
||
|
|
return caught / total if total else float("nan")
|
||
|
|
return rate(DEFECTS), rate(("normal",)), rate(("flare_normal",))
|
||
|
|
|
||
|
|
|
||
|
|
def format_table(table):
|
||
|
|
categories = [c for c in DEFECTS + CLEAN
|
||
|
|
if any(c in counts for counts in table.values())]
|
||
|
|
head = f"{'detector':<32}" + "".join(f"{c[:9]:>10}" for c in categories)
|
||
|
|
head += f"{'RECALL':>9}{'FPR':>8}{'FPR flr':>9}"
|
||
|
|
lines = [head]
|
||
|
|
totals = next(iter(table.values()))
|
||
|
|
lines.append(f"{'(frames)':<32}" + "".join(f"{totals.get(c, (0, 0))[1]:>10}"
|
||
|
|
for c in categories))
|
||
|
|
for row, counts in table.items():
|
||
|
|
cells = "".join(
|
||
|
|
f"{counts[c][0] / counts[c][1]:>10.3f}" if counts.get(c, (0, 0))[1] else f"{'-':>10}"
|
||
|
|
for c in categories)
|
||
|
|
recall, fpr, flare = summary(counts)
|
||
|
|
lines.append(f"{row:<32}{cells}{recall:>9.3f}{fpr:>8.4f}{flare:>9.4f}")
|
||
|
|
return "\n".join(lines)
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_score(args):
|
||
|
|
with open(args.labels) as handle:
|
||
|
|
labels_by_slot = {_slot(r): r["category"] for r in json.load(handle)["labels"]}
|
||
|
|
with open(args.features) as handle:
|
||
|
|
frames = json.load(handle)["frames"]
|
||
|
|
baselines = far_field_baselines(frames, labels_by_slot)
|
||
|
|
rows = [r.strip() for r in args.rows.split(",")] if args.rows else list(DEFAULT_ROWS)
|
||
|
|
table = score(frames, labels_by_slot, rows, baselines)
|
||
|
|
print(f"{len(frames)} measured frames; recall per category, FPR on normal and flare frames")
|
||
|
|
print(format_table(table))
|
||
|
|
if args.misses:
|
||
|
|
terms = expand(args.misses)
|
||
|
|
missed = [f for f in frames if labels_by_slot.get(_slot(f)) in DEFECTS
|
||
|
|
and not any(says_bad(f, t, baselines) for t in terms)]
|
||
|
|
false = [f for f in frames if labels_by_slot.get(_slot(f)) in CLEAN
|
||
|
|
and any(says_bad(f, t, baselines) for t in terms)]
|
||
|
|
print(f"\n{args.misses}: {len(missed)} defects missed, {len(false)} clean frames rejected")
|
||
|
|
for f in missed + false:
|
||
|
|
base = baselines.get(f["wavelength"])
|
||
|
|
ratio = f["far_field"] / base if base and f.get("far_field") is not None else None
|
||
|
|
print(f" {labels_by_slot[_slot(f)]:<13} g{f['satellite']} {f['wavelength']:>3} "
|
||
|
|
f"{f['t_start']} far x{ratio if ratio is None else round(ratio, 1)}")
|
||
|
|
if args.out:
|
||
|
|
with open(args.out, "w") as handle:
|
||
|
|
json.dump({row: {c: list(v) for c, v in counts.items()}
|
||
|
|
for row, counts in table.items()}, handle, indent=1)
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
def score_verdicts(verdicts, labels):
|
||
|
|
"""Score verdicts against the labels.
|
||
|
|
|
||
|
|
`verdicts` maps (satellite, wavelength, t_start) to (verdict, reason). Returns
|
||
|
|
({category: (caught, total)}, missed, rejected): `missed` the defects passed and
|
||
|
|
`rejected` the clean frames rejected, each [(record, reason)]. Labelled frames
|
||
|
|
with no verdict (outside the judged range) are left out.
|
||
|
|
"""
|
||
|
|
counts = collections.defaultdict(lambda: [0, 0])
|
||
|
|
missed, rejected = [], []
|
||
|
|
for record in labels:
|
||
|
|
category = record["category"]
|
||
|
|
found = verdicts.get(_slot(record))
|
||
|
|
if category in NEUTRAL or found is None:
|
||
|
|
continue
|
||
|
|
verdict, reason = found
|
||
|
|
bad = verdict == detectors.BAD
|
||
|
|
counts[category][1] += 1
|
||
|
|
counts[category][0] += bad
|
||
|
|
if category in DEFECTS and not bad:
|
||
|
|
missed.append((record, reason))
|
||
|
|
elif category in CLEAN and bad:
|
||
|
|
rejected.append((record, reason))
|
||
|
|
return {k: tuple(v) for k, v in counts.items()}, missed, rejected
|
||
|
|
|
||
|
|
|
||
|
|
def stored_table(conn, run_id, labels):
|
||
|
|
"""``score_verdicts`` for a judgment run stored in the index."""
|
||
|
|
verdicts = {}
|
||
|
|
for record in labels:
|
||
|
|
row = conn.execute(
|
||
|
|
"SELECT d.verdict, d.reason FROM frame f JOIN detection d ON d.frame_id = f.id "
|
||
|
|
"AND d.run_id = ? WHERE f.satellite = ? AND f.wavelength = ? AND f.t_start = ?",
|
||
|
|
(run_id, record["satellite"], record["wavelength"], record["t_start"])).fetchone()
|
||
|
|
if row is not None:
|
||
|
|
verdicts[_slot(record)] = (row["verdict"], row["reason"])
|
||
|
|
return score_verdicts(verdicts, labels)
|
||
|
|
|
||
|
|
|
||
|
|
def print_scored(title, scored):
|
||
|
|
counts, missed, rejected = scored
|
||
|
|
print(f"{title}: {sum(total for _, total in counts.values())} labelled frames judged")
|
||
|
|
print(format_table({title: counts}))
|
||
|
|
for heading, rows in (("defects passed", missed), ("clean frames rejected", rejected)):
|
||
|
|
print(f"\n{heading}: {len(rows)}")
|
||
|
|
for record, reason in rows:
|
||
|
|
print(f" {record['category']:<13} g{record['satellite']} "
|
||
|
|
f"{record['wavelength']:>3} {_when(record['t_start'])} {reason or ''}")
|
||
|
|
|
||
|
|
|
||
|
|
def _when(t_start):
|
||
|
|
return time.strftime("%Y-%m-%d %H:%M", time.gmtime(t_start))
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_stored(args):
|
||
|
|
with open(args.labels) as handle:
|
||
|
|
labels = json.load(handle)["labels"]
|
||
|
|
conn = db.connect(args.db, readonly=True)
|
||
|
|
run_id = args.run or detectors.judgment_run(conn)
|
||
|
|
if run_id is None:
|
||
|
|
raise SystemExit("No production judgment under the current rules; pass --run")
|
||
|
|
print_scored(f"judgment run {run_id}", stored_table(conn, run_id, labels))
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
def dry_run(conn, measure_run, start, end, satellites):
|
||
|
|
"""The production judgment of [start, end) from stored measurements, in memory:
|
||
|
|
{slot: (verdict, source, reason)}. Nothing is written."""
|
||
|
|
import filter_FITS
|
||
|
|
|
||
|
|
margin = filter_FITS.JUDGE_MARGIN * paths.CADENCE
|
||
|
|
out = {}
|
||
|
|
for chunk in range(start, end, filter_FITS.JUDGE_CHUNK):
|
||
|
|
hi = min(chunk + filter_FITS.JUDGE_CHUNK, end)
|
||
|
|
for satellite in satellites:
|
||
|
|
streams = filter_FITS.load_measured(conn, measure_run, satellite,
|
||
|
|
chunk - margin, hi + margin)
|
||
|
|
for slot, source, verdict in filter_FITS.judge_measured(streams).values():
|
||
|
|
if chunk <= slot[2] < hi:
|
||
|
|
out[slot] = (verdict.verdict, source, verdict.reason)
|
||
|
|
return out
|
||
|
|
|
||
|
|
|
||
|
|
def run_verdicts(conn, run_id, start, end, satellites):
|
||
|
|
"""{slot: (verdict, reason)} of a stored run over [start, end)."""
|
||
|
|
marks = ",".join("?" * len(satellites))
|
||
|
|
return {(r["satellite"], r["wavelength"], r["t_start"]): (r["verdict"], r["reason"])
|
||
|
|
for r in conn.execute(
|
||
|
|
f"SELECT f.satellite, f.wavelength, f.t_start, d.verdict, d.reason FROM frame f "
|
||
|
|
f"JOIN detection d ON d.frame_id = f.id AND d.run_id = ? WHERE f.t_start >= ? "
|
||
|
|
f"AND f.t_start < ? AND f.satellite IN ({marks})",
|
||
|
|
(run_id, start, end, *satellites))}
|
||
|
|
|
||
|
|
|
||
|
|
def changes(verdicts, reference):
|
||
|
|
"""Frames whose verdict differs from a reference run: (newly_bad, newly_good),
|
||
|
|
each [(slot, reason)] in time order; frames the reference never judged are left
|
||
|
|
out."""
|
||
|
|
newly_bad, newly_good = [], []
|
||
|
|
for slot in sorted(verdicts, key=lambda s: (s[2], s[0], s[1])):
|
||
|
|
verdict, source, reason = verdicts[slot]
|
||
|
|
before = reference.get(slot)
|
||
|
|
if before is None:
|
||
|
|
continue
|
||
|
|
why = f"{source}: {reason}" if source and reason else reason
|
||
|
|
if verdict == detectors.BAD and before[0] != detectors.BAD:
|
||
|
|
newly_bad.append((slot, why))
|
||
|
|
elif verdict != detectors.BAD and before[0] == detectors.BAD:
|
||
|
|
newly_good.append((slot, f"was {before[1]}"))
|
||
|
|
return newly_bad, newly_good
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_dry_run(args):
|
||
|
|
with open(args.labels) as handle:
|
||
|
|
labels = json.load(handle)["labels"]
|
||
|
|
conn = db.connect(args.db, readonly=True)
|
||
|
|
measure_run = args.measure_run or detectors.measure_run(conn)
|
||
|
|
if measure_run is None:
|
||
|
|
raise SystemExit("No measurements under the current rules; pass --measure-run")
|
||
|
|
satellites = tuple(int(s) for s in args.satellites.split(","))
|
||
|
|
verdicts = dry_run(conn, measure_run, args.start, args.end, satellites)
|
||
|
|
print(f"{len(verdicts)} frames judged from measure run {measure_run}")
|
||
|
|
print_scored("dry run", score_verdicts(
|
||
|
|
{slot: (v, f"{src}: {why}" if src and why else why)
|
||
|
|
for slot, (v, src, why) in verdicts.items()}, labels))
|
||
|
|
newly_bad, newly_good = changes(
|
||
|
|
verdicts, run_verdicts(conn, args.reference, args.start, args.end, satellites))
|
||
|
|
print(f"\nagainst run {args.reference}: {len(newly_bad)} newly bad, "
|
||
|
|
f"{len(newly_good)} newly good")
|
||
|
|
tally = collections.Counter((why or "").split(":")[0] for _, why in newly_bad)
|
||
|
|
for source, count in tally.most_common():
|
||
|
|
print(f" newly bad by {source:<14}{count}")
|
||
|
|
if args.out:
|
||
|
|
rows = lambda pairs: [[s[0], s[1], _when(s[2]), why or ""] for s, why in pairs]
|
||
|
|
with open(args.out, "w") as handle:
|
||
|
|
json.dump({"newly_bad": rows(newly_bad), "newly_good": rows(newly_good)},
|
||
|
|
handle, indent=1)
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
#: Tile size of a review sheet, in pixels.
|
||
|
|
SHEET_TILE = 360
|
||
|
|
|
||
|
|
|
||
|
|
def _sheet_time(value):
|
||
|
|
if isinstance(value, (int, float)):
|
||
|
|
return int(value)
|
||
|
|
return calendar.timegm(time.strptime(value, "%Y-%m-%d %H:%M"))
|
||
|
|
|
||
|
|
|
||
|
|
def cmd_sheet(args):
|
||
|
|
"""A 3-column neighbour sheet: each row the previous good frame of the same band,
|
||
|
|
the frame under review, and the next good frame -- at any distance, skipping
|
||
|
|
frames judged bad and every frame listed for review or exclusion."""
|
||
|
|
from PIL import Image, ImageDraw
|
||
|
|
|
||
|
|
from suvi import metrics
|
||
|
|
|
||
|
|
with open(args.rows) as handle:
|
||
|
|
rows = json.load(handle)
|
||
|
|
rows = rows.get(args.key, []) if isinstance(rows, dict) else rows
|
||
|
|
rows = [(int(r[0]), int(r[1]), _sheet_time(r[2]), r[3] if len(r) > 3 else "")
|
||
|
|
for r in rows[:args.limit]]
|
||
|
|
excluded = {(s, w, t) for s, w, t, _ in rows}
|
||
|
|
if args.exclude:
|
||
|
|
with open(args.exclude) as handle:
|
||
|
|
extra = json.load(handle)
|
||
|
|
extra = extra.get(args.key, []) if isinstance(extra, dict) else extra
|
||
|
|
excluded |= {(int(r[0]), int(r[1]), _sheet_time(r[2])) for r in extra}
|
||
|
|
conn = db.connect(args.db, readonly=True)
|
||
|
|
run_id = args.run or detectors.judgment_run(conn)
|
||
|
|
if run_id is None:
|
||
|
|
raise SystemExit("No production judgment under the current rules; pass --run")
|
||
|
|
|
||
|
|
def lookup(satellite, wavelength, t_start, direction):
|
||
|
|
if direction == 0:
|
||
|
|
row = conn.execute(
|
||
|
|
"SELECT f.t_start, f.path, d.verdict FROM frame f LEFT JOIN detection d "
|
||
|
|
"ON d.frame_id = f.id AND d.run_id = ? WHERE f.satellite = ? AND "
|
||
|
|
"f.wavelength = ? AND f.t_start = ?",
|
||
|
|
(run_id, satellite, wavelength, t_start)).fetchone()
|
||
|
|
return tuple(row) if row else None
|
||
|
|
op, order = ("<", "DESC") if direction < 0 else (">", "ASC")
|
||
|
|
for row in conn.execute(
|
||
|
|
f"SELECT f.t_start, f.path, d.verdict FROM frame f JOIN detection d "
|
||
|
|
f"ON d.frame_id = f.id AND d.run_id = ? WHERE f.satellite = ? AND "
|
||
|
|
f"f.wavelength = ? AND f.t_start {op} ? AND d.verdict = 'good' "
|
||
|
|
f"ORDER BY f.t_start {order} LIMIT 500",
|
||
|
|
(run_id, satellite, wavelength, t_start)):
|
||
|
|
if (satellite, wavelength, row[0]) not in excluded:
|
||
|
|
return tuple(row)
|
||
|
|
return None
|
||
|
|
|
||
|
|
tile, head = SHEET_TILE, 40
|
||
|
|
sheet = Image.new("RGB", (3 * tile, len(rows) * (tile + head)), (40, 40, 40))
|
||
|
|
for index, (satellite, wavelength, t_start, note) in enumerate(rows):
|
||
|
|
for column, direction in enumerate((-1, 0, 1)):
|
||
|
|
cell = Image.new("RGB", (tile, tile + head))
|
||
|
|
draw = ImageDraw.Draw(cell)
|
||
|
|
found = lookup(satellite, wavelength, t_start, direction)
|
||
|
|
if found is None:
|
||
|
|
draw.text((6, 6), "no good frame found", fill=(255, 80, 80))
|
||
|
|
else:
|
||
|
|
when, relpath, verdict = found
|
||
|
|
image, _ = fitsio.read_image(paths.abspath(relpath))
|
||
|
|
if image is not None:
|
||
|
|
shown = np.clip(np.nan_to_num(metrics.to_display(
|
||
|
|
np.asarray(image, np.float32), wavelength)), 0, 1)
|
||
|
|
cell.paste(Image.fromarray((shown * 255).astype(np.uint8))
|
||
|
|
.resize((tile, tile)).convert("RGB"), (0, head))
|
||
|
|
gap = round((when - t_start) / paths.CADENCE)
|
||
|
|
draw.text((6, 4), f"G{satellite} {wavelength}A {_when(when)} judged {verdict}",
|
||
|
|
fill=(255, 255, 0))
|
||
|
|
draw.text((6, 20), (f"REVIEW: {note}" if direction == 0 else
|
||
|
|
f"{gap:+d} slots")[:56], fill=(160, 220, 255))
|
||
|
|
sheet.paste(cell, (column * tile, index * (tile + head)))
|
||
|
|
sheet.save(args.out, quality=90)
|
||
|
|
print(f"{args.out}: {len(rows)} rows")
|
||
|
|
return 0
|
||
|
|
|
||
|
|
|
||
|
|
def main(argv=None):
|
||
|
|
parser = argparse.ArgumentParser(description=__doc__,
|
||
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
||
|
|
measure = sub.add_parser("measure", help="read labelled frames and run every detector")
|
||
|
|
measure.add_argument("--labels", required=True)
|
||
|
|
measure.add_argument("--out", required=True)
|
||
|
|
measure.add_argument("--workers", type=int, default=4)
|
||
|
|
measure.add_argument("--db", default=None)
|
||
|
|
measure.set_defaults(func=cmd_measure)
|
||
|
|
refresh = sub.add_parser("refresh", help="re-run the single-frame detectors in place")
|
||
|
|
refresh.add_argument("--features", required=True)
|
||
|
|
refresh.add_argument("--workers", type=int, default=4)
|
||
|
|
refresh.add_argument("--db", default=None)
|
||
|
|
refresh.set_defaults(func=cmd_refresh)
|
||
|
|
scoring = sub.add_parser("score", help="tabulate recall and false positives")
|
||
|
|
scoring.add_argument("--labels", required=True)
|
||
|
|
scoring.add_argument("--features", required=True)
|
||
|
|
scoring.add_argument("--rows", default=None,
|
||
|
|
help="comma-separated detectors; '+' combines (bad if any says bad)")
|
||
|
|
scoring.add_argument("--misses", default=None,
|
||
|
|
help="list the defects this row misses and clean frames it rejects")
|
||
|
|
scoring.add_argument("--out", default=None)
|
||
|
|
scoring.set_defaults(func=cmd_score)
|
||
|
|
run = sub.add_parser("stored", help="score a judgment run stored in the index")
|
||
|
|
run.add_argument("--labels", required=True)
|
||
|
|
run.add_argument("--run", type=int, default=None,
|
||
|
|
help="detector run id (default: the production judgment)")
|
||
|
|
run.add_argument("--db", default=None)
|
||
|
|
run.set_defaults(func=cmd_stored)
|
||
|
|
dry = sub.add_parser("dry-run", help="judge stored measurements in memory and score")
|
||
|
|
dry.add_argument("--labels", required=True)
|
||
|
|
dry.add_argument("--start", type=int, required=True)
|
||
|
|
dry.add_argument("--end", type=int, required=True)
|
||
|
|
dry.add_argument("--satellites", default="16,18")
|
||
|
|
dry.add_argument("--measure-run", type=int, default=None)
|
||
|
|
dry.add_argument("--reference", type=int, default=46,
|
||
|
|
help="stored run to list changed verdicts against (default 46)")
|
||
|
|
dry.add_argument("--out", default=None, help="JSON of newly bad / newly good rows")
|
||
|
|
dry.add_argument("--db", default=None)
|
||
|
|
dry.set_defaults(func=cmd_dry_run)
|
||
|
|
sheet = sub.add_parser("sheet", help="draw a 3-column neighbour sheet")
|
||
|
|
sheet.add_argument("--rows", required=True,
|
||
|
|
help="JSON rows [[satellite, band, 'YYYY-MM-DD HH:MM', note]]")
|
||
|
|
sheet.add_argument("--key", default="newly_bad",
|
||
|
|
help="list to draw when the rows file is a dry-run output")
|
||
|
|
sheet.add_argument("--exclude", default=None, help="more rows never used as neighbours")
|
||
|
|
sheet.add_argument("--limit", type=int, default=40)
|
||
|
|
sheet.add_argument("--run", type=int, default=None,
|
||
|
|
help="judgment run deciding what is good (default: production)")
|
||
|
|
sheet.add_argument("--out", required=True)
|
||
|
|
sheet.add_argument("--db", default=None)
|
||
|
|
sheet.set_defaults(func=cmd_sheet)
|
||
|
|
args = parser.parse_args(argv)
|
||
|
|
return args.func(args)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
sys.exit(main())
|