#!/usr/bin/env python """Score bad-frame detectors against REAL, hand-confirmed archive defects. The bench scores detectors on damage it injected itself, against a notion of "clean" produced by the detectors being scored. That is how the production pair reached precision 1.000 on the bench while passing saturated bars, ghost discs and short-exposure static in the real archive. This scores them on ``labels/real_defects.json`` instead (built by ``defect_audit.py``, confirmed by eye), reporting recall per defect category and false positives separately on ordinary and flare frames -- the flare frames being the ones a detector tuned to catch brightness anomalies is most likely to throw away. detector_eval.py measure --labels labels/real_defects.json --out AUDIT/eval.json detector_eval.py score --labels labels/real_defects.json --features AUDIT/eval.json detector_eval.py stored --labels labels/real_defects.json [--run N] detector_eval.py dry-run --labels labels/real_defects.json --start T --end T --out new.json detector_eval.py sheet --rows new.json --out sheet.jpg [--run N] ``measure`` reads every labelled frame with enough context around it for the series detectors (+/-TEMPORAL_WINDOW frames on both satellites) and stores each detector's verdict plus the raw far-field measurements. ``score`` is cheap and repeatable: combinations (``a+b`` = bad if either says bad) and far-field thresholds (``far_field@4``) are evaluated from the stored results, so trying a rule does not mean re-reading the archive. ``stored`` scores a judgment run already in the index (the production one by default) -- the check that what the archive-wide judge stored is what the rules promised. ``dry-run`` judges stored measurements in memory (nothing written) and lists every frame whose verdict changes against a reference run; ``sheet`` draws such frames between their nearest good neighbours for review. Read-only against archive and index. """ import argparse import calendar import collections import json import os import sys import time from multiprocessing import Pool import numpy as np sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from defect_audit import CLEAN, DEFECTS, NEUTRAL from suvi import db, detectors, fitsio, paths #: Labelled frames of one band closer than this many slots share a context read. CLUSTER_GAP = 24 #: Context read either side of a cluster. Two windows, so a frame's temporal #: neighbours have their own neighbours for the structural-churn baseline. CONTEXT_SLOTS = 2 * detectors.TEMPORAL_WINDOW #: Verdicts each labelled frame gets from `measure`. FRAME_VERDICTS = ("header_v1", "header_v1_old", "lunar_v1", "disc_v1", "geometry_v1", "field_v1") #: Every verdict a measured frame can carry. MEASURED = FRAME_VERDICTS + ("temporal_v1", "crosssat_v1", "legacy") #: Judged from stored scores at scoring time, against the current limits, so #: tuning a limit needs no new read of the archive. FROM_SCORES = {"temporal_v2": ("temporal_v2_scores", "TEMPORAL2_LIMITS")} #: Rows `score` prints when none are asked for: what production used when the 2024 #: render and the corpus were made, what it uses now, and the alternatives. DEFAULT_ROWS = ("pair_old", "pair", "legacy", "temporal_v1", "temporal_v2", "field_v1", "production", "production+temporal_v2") #: Named combinations, expanded before evaluation. ALIASES = {"pair": "header_v1+disc_v1", "pair_old": "header_v1_old+disc_v1", "production": "header_v1+lunar_v1+disc_v1+field_v1"} # ------------------------------------------------------------------- clustering def clusters(labels, gap=CLUSTER_GAP, cadence=paths.CADENCE): """Group labelled frames by band into runs whose members are < `gap` slots apart. Both satellites share a cluster, because the cross-satellite check needs the other satellite's frames at the same instants anyway. """ by_band = collections.defaultdict(list) for record in labels: by_band[record["wavelength"]].append(record) out = [] for wavelength in sorted(by_band): members = sorted(by_band[wavelength], key=lambda r: r["t_start"]) current = [members[0]] for record in members[1:]: if record["t_start"] - current[-1]["t_start"] < gap * cadence: current.append(record) else: out.append((wavelength, current)) current = [record] out.append((wavelength, current)) return out # ---------------------------------------------------------------------- measure def _frame(job): """Header, thumbnail, production verdict and (for labelled frames) every single-frame verdict. Returns (features, production_good, record).""" slot, path, labelled = job cards, header_error = fitsio.read_header(path) values = fitsio.header_metadata(cards) refcrota = cards.get("REFCROTA") refcrota = float(refcrota) if isinstance(refcrota, (int, float)) else None image, image_error = fitsio.read_image(path) features = detectors.FrameFeatures( slot=slot, header=values, error=header_error or (None if image is not None else image_error), thumbnail=detectors.thumbnail(image) if image is not None else None) _, production, _ = detectors.judge_frame(slot, cards, header_error, lambda: (image, image_error)) good = production.verdict != detectors.BAD if not labelled: return features, good, None def pack(verdict): return [verdict.verdict, verdict.reason] old = dict(values) old.pop("n_long", None) header = detectors.FrameFeatures(slot=slot, header=values, error=header_error) record = { "header_v1": pack(detectors.header_v1(header)), "header_v1_old": pack(detectors.header_v1(detectors.FrameFeatures( slot=slot, header=old, error=header_error))), "lunar_v1": pack(detectors.lunar_v1(cards)), } if image is None: record["disc_v1"] = pack(detectors.unreadable(image_error)) record["geometry_v1"] = pack(detectors.unreadable(image_error)) record["field_v1"] = pack(detectors.unreadable(image_error)) record["far_field"] = record["noise"] = None else: record["disc_v1"] = pack(detectors.disc_v1(image, slot[1], header=values)) record["geometry_v1"] = pack(detectors.geometry_v1(image, slot[1])) field = detectors.field_v1(image, slot[1], header=values, refcrota=refcrota) record["field_v1"] = pack(field) record["field_scores"] = field.scores radius = values.get("diam_sun") radius = radius / 2.0 if radius else None record["far_field"] = detectors.far_field(image, radius) record["noise"] = detectors.far_field_noise(image, radius) return features, good, record def measure_cluster(job): """Every verdict for one cluster's labelled frames, with series context.""" wavelength, members, streams = job fitsio.quiet_astropy() wanted = {(r["satellite"], r["wavelength"], r["t_start"]) for r in members} frames, records, good = {}, {}, set() for satellite, rows in streams.items(): for t_start, path in rows: slot = (satellite, wavelength, t_start) features, passed, record = _frame((slot, path, slot in wanted)) frames[slot] = features if passed: good.add(slot) if record is not None: records[slot] = record temporal, temporal2 = {}, {} for satellite in streams: ordered = sorted((f for s, f in frames.items() if s[0] == satellite), key=lambda f: f.t_start) for features, verdict in zip(ordered, detectors.temporal_v1(ordered)): temporal[features.slot] = verdict for features, verdict in zip(ordered, detectors.temporal_v2(ordered, good)): temporal2[features.slot] = verdict out = [] for record in members: slot = (record["satellite"], record["wavelength"], record["t_start"]) measured = records.get(slot) if measured is None: # vanished from the archive continue verdict = temporal.get(slot) measured["temporal_v1"] = [verdict.verdict, verdict.reason] if verdict else None second = temporal2.get(slot) measured["temporal_v2_scores"] = second.scores if second else None other = [s for s in streams if s != slot[0]] cross = None if other: counterpart = (other[0], wavelength, slot[2]) mine, theirs = detectors.crosssat_v1( frames.get(slot), frames.get(counterpart), temporal.get(slot), temporal.get(counterpart)) cross = [mine.verdict, mine.reason] measured["crosssat_v1"] = cross out.append({"satellite": slot[0], "wavelength": wavelength, "t_start": slot[2], **measured}) return out def legacy_verdicts(conn, labels): """The legacy pixel filter's recorded verdict per labelled slot, where it has one.""" out = {} for record in labels: row = conn.execute( "SELECT d.verdict FROM detection d JOIN detector_run r ON r.id = d.run_id " "JOIN frame f ON f.id = d.frame_id WHERE r.name = 'legacy_filter' AND " "f.satellite = ? AND f.wavelength = ? AND f.t_start = ? LIMIT 1", (record["satellite"], record["wavelength"], record["t_start"])).fetchone() if row is not None: out[(record["satellite"], record["wavelength"], record["t_start"])] = row[0] return out def cmd_measure(args): with open(args.labels) as handle: labels = json.load(handle)["labels"] conn = db.connect(args.db, readonly=True) root = paths.data_root() jobs = [] for wavelength, members in clusters(labels): t0 = members[0]["t_start"] - CONTEXT_SLOTS * paths.CADENCE t1 = members[-1]["t_start"] + CONTEXT_SLOTS * paths.CADENCE streams = {} for satellite in (16, 18): streams[satellite] = [ (row[0], paths.abspath(row[1], root)) for row in conn.execute( "SELECT t_start, path FROM frame WHERE satellite = ? AND wavelength = ? " "AND t_start >= ? AND t_start <= ? AND size_bytes >= ? ORDER BY t_start", (satellite, wavelength, t0, t1, 100_000))] jobs.append((wavelength, members, streams)) reads = sum(len(rows) for _, _, s in jobs for rows in s.values()) print(f"{len(labels)} labelled frames in {len(jobs)} clusters; {reads} frames to read", flush=True) legacy = legacy_verdicts(conn, labels) results, done = [], 0 with Pool(args.workers, initializer=fitsio.quiet_astropy) as pool: for batch in pool.imap_unordered(measure_cluster, jobs): results.extend(batch) done += 1 if done % 50 == 0: print(f" {done}/{len(jobs)} clusters", flush=True) for record in results: verdict = legacy.get((record["satellite"], record["wavelength"], record["t_start"])) record["legacy"] = [verdict, None] if verdict else None os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) with open(args.out, "w") as handle: json.dump({"labels": os.path.abspath(args.labels), "frames": results}, handle) print(f"wrote {len(results)} measured frames to {args.out}") return 0 def _refresh_one(job): slot, path = job fitsio.quiet_astropy() return slot, _frame((slot, path, True))[2] def cmd_refresh(args): """Re-run only the single-frame detectors on the labelled frames, in place. After a frame detector changes, the series context `measure` read is still valid, so this reads each labelled frame once instead of every neighbour. """ with open(args.features) as handle: stored = json.load(handle) conn = db.connect(args.db, readonly=True) root = paths.data_root() jobs = [] for record in stored["frames"]: row = conn.execute("SELECT path FROM frame WHERE satellite = ? AND wavelength = ? " "AND t_start = ?", _slot(record)).fetchone() if row is not None: jobs.append((_slot(record), paths.abspath(row[0], root))) by_slot = {_slot(r): r for r in stored["frames"]} with Pool(args.workers, initializer=fitsio.quiet_astropy) as pool: for slot, fresh in pool.imap_unordered(_refresh_one, jobs, 8): by_slot[slot].update(fresh) with open(args.features + ".tmp", "w") as handle: json.dump(stored, handle) os.replace(args.features + ".tmp", args.features) print(f"refreshed {len(jobs)} frames in {args.features}") return 0 # ------------------------------------------------------------------------ score def far_field_baselines(frames, labels_by_slot): """Median far-field brightness per band over frames labelled `normal`.""" per_band = collections.defaultdict(list) for frame in frames: if (labels_by_slot.get(_slot(frame)) == "normal" and frame.get("far_field") is not None): per_band[frame["wavelength"]].append(frame["far_field"]) return {band: float(np.median(values)) for band, values in per_band.items() if values} def _slot(record): return (record["satellite"], record["wavelength"], record["t_start"]) def says_bad(frame, term, baselines): """Whether one detector term calls `frame` bad. UNKNOWN and absent are not bad.""" if term.startswith("far_field@"): factor = float(term.split("@", 1)[1]) base = baselines.get(frame["wavelength"]) value = frame.get("far_field") return base is not None and value is not None and value > factor * base if term in FROM_SCORES: key, limits = FROM_SCORES[term] scores = frame.get(key) or {} return any(scores.get(name, 0.0) > limit for name, limit in getattr(detectors, limits).items()) if term not in MEASURED: raise KeyError(f"unknown detector term {term!r}") verdict = frame.get(term) return bool(verdict) and verdict[0] == detectors.BAD def expand(row): """'pair+temporal_v1' -> ['header_v1', 'disc_v1', 'temporal_v1'].""" terms = [] for part in row.split("+"): terms.extend(ALIASES.get(part, part).split("+")) return terms def score(frames, labels_by_slot, rows, baselines): """{row: {category: (caught, total)}} over every labelled category present.""" table = {} for row in rows: terms = expand(row) counts = collections.defaultdict(lambda: [0, 0]) for frame in frames: category = labels_by_slot.get(_slot(frame)) if category is None or category in NEUTRAL: continue counts[category][1] += 1 if any(says_bad(frame, term, baselines) for term in terms): counts[category][0] += 1 table[row] = {k: tuple(v) for k, v in counts.items()} return table def summary(counts): """(defect recall, normal FPR, flare FPR) from one row's per-category counts.""" def rate(categories): caught = sum(counts.get(c, (0, 0))[0] for c in categories) total = sum(counts.get(c, (0, 0))[1] for c in categories) return caught / total if total else float("nan") return rate(DEFECTS), rate(("normal",)), rate(("flare_normal",)) def format_table(table): categories = [c for c in DEFECTS + CLEAN if any(c in counts for counts in table.values())] head = f"{'detector':<32}" + "".join(f"{c[:9]:>10}" for c in categories) head += f"{'RECALL':>9}{'FPR':>8}{'FPR flr':>9}" lines = [head] totals = next(iter(table.values())) lines.append(f"{'(frames)':<32}" + "".join(f"{totals.get(c, (0, 0))[1]:>10}" for c in categories)) for row, counts in table.items(): cells = "".join( f"{counts[c][0] / counts[c][1]:>10.3f}" if counts.get(c, (0, 0))[1] else f"{'-':>10}" for c in categories) recall, fpr, flare = summary(counts) lines.append(f"{row:<32}{cells}{recall:>9.3f}{fpr:>8.4f}{flare:>9.4f}") return "\n".join(lines) def cmd_score(args): with open(args.labels) as handle: labels_by_slot = {_slot(r): r["category"] for r in json.load(handle)["labels"]} with open(args.features) as handle: frames = json.load(handle)["frames"] baselines = far_field_baselines(frames, labels_by_slot) rows = [r.strip() for r in args.rows.split(",")] if args.rows else list(DEFAULT_ROWS) table = score(frames, labels_by_slot, rows, baselines) print(f"{len(frames)} measured frames; recall per category, FPR on normal and flare frames") print(format_table(table)) if args.misses: terms = expand(args.misses) missed = [f for f in frames if labels_by_slot.get(_slot(f)) in DEFECTS and not any(says_bad(f, t, baselines) for t in terms)] false = [f for f in frames if labels_by_slot.get(_slot(f)) in CLEAN and any(says_bad(f, t, baselines) for t in terms)] print(f"\n{args.misses}: {len(missed)} defects missed, {len(false)} clean frames rejected") for f in missed + false: base = baselines.get(f["wavelength"]) ratio = f["far_field"] / base if base and f.get("far_field") is not None else None print(f" {labels_by_slot[_slot(f)]:<13} g{f['satellite']} {f['wavelength']:>3} " f"{f['t_start']} far x{ratio if ratio is None else round(ratio, 1)}") if args.out: with open(args.out, "w") as handle: json.dump({row: {c: list(v) for c, v in counts.items()} for row, counts in table.items()}, handle, indent=1) return 0 def score_verdicts(verdicts, labels): """Score verdicts against the labels. `verdicts` maps (satellite, wavelength, t_start) to (verdict, reason). Returns ({category: (caught, total)}, missed, rejected): `missed` the defects passed and `rejected` the clean frames rejected, each [(record, reason)]. Labelled frames with no verdict (outside the judged range) are left out. """ counts = collections.defaultdict(lambda: [0, 0]) missed, rejected = [], [] for record in labels: category = record["category"] found = verdicts.get(_slot(record)) if category in NEUTRAL or found is None: continue verdict, reason = found bad = verdict == detectors.BAD counts[category][1] += 1 counts[category][0] += bad if category in DEFECTS and not bad: missed.append((record, reason)) elif category in CLEAN and bad: rejected.append((record, reason)) return {k: tuple(v) for k, v in counts.items()}, missed, rejected def stored_table(conn, run_id, labels): """``score_verdicts`` for a judgment run stored in the index.""" verdicts = {} for record in labels: row = conn.execute( "SELECT d.verdict, d.reason FROM frame f JOIN detection d ON d.frame_id = f.id " "AND d.run_id = ? WHERE f.satellite = ? AND f.wavelength = ? AND f.t_start = ?", (run_id, record["satellite"], record["wavelength"], record["t_start"])).fetchone() if row is not None: verdicts[_slot(record)] = (row["verdict"], row["reason"]) return score_verdicts(verdicts, labels) def print_scored(title, scored): counts, missed, rejected = scored print(f"{title}: {sum(total for _, total in counts.values())} labelled frames judged") print(format_table({title: counts})) for heading, rows in (("defects passed", missed), ("clean frames rejected", rejected)): print(f"\n{heading}: {len(rows)}") for record, reason in rows: print(f" {record['category']:<13} g{record['satellite']} " f"{record['wavelength']:>3} {_when(record['t_start'])} {reason or ''}") def _when(t_start): return time.strftime("%Y-%m-%d %H:%M", time.gmtime(t_start)) def cmd_stored(args): with open(args.labels) as handle: labels = json.load(handle)["labels"] conn = db.connect(args.db, readonly=True) run_id = args.run or detectors.judgment_run(conn) if run_id is None: raise SystemExit("No production judgment under the current rules; pass --run") print_scored(f"judgment run {run_id}", stored_table(conn, run_id, labels)) return 0 def dry_run(conn, measure_run, start, end, satellites): """The production judgment of [start, end) from stored measurements, in memory: {slot: (verdict, source, reason)}. Nothing is written.""" import filter_FITS margin = filter_FITS.JUDGE_MARGIN * paths.CADENCE out = {} for chunk in range(start, end, filter_FITS.JUDGE_CHUNK): hi = min(chunk + filter_FITS.JUDGE_CHUNK, end) for satellite in satellites: streams = filter_FITS.load_measured(conn, measure_run, satellite, chunk - margin, hi + margin) for slot, source, verdict in filter_FITS.judge_measured(streams).values(): if chunk <= slot[2] < hi: out[slot] = (verdict.verdict, source, verdict.reason) return out def run_verdicts(conn, run_id, start, end, satellites): """{slot: (verdict, reason)} of a stored run over [start, end).""" marks = ",".join("?" * len(satellites)) return {(r["satellite"], r["wavelength"], r["t_start"]): (r["verdict"], r["reason"]) for r in conn.execute( f"SELECT f.satellite, f.wavelength, f.t_start, d.verdict, d.reason FROM frame f " f"JOIN detection d ON d.frame_id = f.id AND d.run_id = ? WHERE f.t_start >= ? " f"AND f.t_start < ? AND f.satellite IN ({marks})", (run_id, start, end, *satellites))} def changes(verdicts, reference): """Frames whose verdict differs from a reference run: (newly_bad, newly_good), each [(slot, reason)] in time order; frames the reference never judged are left out.""" newly_bad, newly_good = [], [] for slot in sorted(verdicts, key=lambda s: (s[2], s[0], s[1])): verdict, source, reason = verdicts[slot] before = reference.get(slot) if before is None: continue why = f"{source}: {reason}" if source and reason else reason if verdict == detectors.BAD and before[0] != detectors.BAD: newly_bad.append((slot, why)) elif verdict != detectors.BAD and before[0] == detectors.BAD: newly_good.append((slot, f"was {before[1]}")) return newly_bad, newly_good def cmd_dry_run(args): with open(args.labels) as handle: labels = json.load(handle)["labels"] conn = db.connect(args.db, readonly=True) measure_run = args.measure_run or detectors.measure_run(conn) if measure_run is None: raise SystemExit("No measurements under the current rules; pass --measure-run") satellites = tuple(int(s) for s in args.satellites.split(",")) verdicts = dry_run(conn, measure_run, args.start, args.end, satellites) print(f"{len(verdicts)} frames judged from measure run {measure_run}") print_scored("dry run", score_verdicts( {slot: (v, f"{src}: {why}" if src and why else why) for slot, (v, src, why) in verdicts.items()}, labels)) newly_bad, newly_good = changes( verdicts, run_verdicts(conn, args.reference, args.start, args.end, satellites)) print(f"\nagainst run {args.reference}: {len(newly_bad)} newly bad, " f"{len(newly_good)} newly good") tally = collections.Counter((why or "").split(":")[0] for _, why in newly_bad) for source, count in tally.most_common(): print(f" newly bad by {source:<14}{count}") if args.out: rows = lambda pairs: [[s[0], s[1], _when(s[2]), why or ""] for s, why in pairs] with open(args.out, "w") as handle: json.dump({"newly_bad": rows(newly_bad), "newly_good": rows(newly_good)}, handle, indent=1) return 0 #: Tile size of a review sheet, in pixels. SHEET_TILE = 360 def _sheet_time(value): if isinstance(value, (int, float)): return int(value) return calendar.timegm(time.strptime(value, "%Y-%m-%d %H:%M")) def cmd_sheet(args): """A 3-column neighbour sheet: each row the previous good frame of the same band, the frame under review, and the next good frame -- at any distance, skipping frames judged bad and every frame listed for review or exclusion.""" from PIL import Image, ImageDraw from suvi import metrics with open(args.rows) as handle: rows = json.load(handle) rows = rows.get(args.key, []) if isinstance(rows, dict) else rows rows = [(int(r[0]), int(r[1]), _sheet_time(r[2]), r[3] if len(r) > 3 else "") for r in rows[:args.limit]] excluded = {(s, w, t) for s, w, t, _ in rows} if args.exclude: with open(args.exclude) as handle: extra = json.load(handle) extra = extra.get(args.key, []) if isinstance(extra, dict) else extra excluded |= {(int(r[0]), int(r[1]), _sheet_time(r[2])) for r in extra} conn = db.connect(args.db, readonly=True) run_id = args.run or detectors.judgment_run(conn) if run_id is None: raise SystemExit("No production judgment under the current rules; pass --run") def lookup(satellite, wavelength, t_start, direction): if direction == 0: row = conn.execute( "SELECT f.t_start, f.path, d.verdict FROM frame f LEFT JOIN detection d " "ON d.frame_id = f.id AND d.run_id = ? WHERE f.satellite = ? AND " "f.wavelength = ? AND f.t_start = ?", (run_id, satellite, wavelength, t_start)).fetchone() return tuple(row) if row else None op, order = ("<", "DESC") if direction < 0 else (">", "ASC") for row in conn.execute( f"SELECT f.t_start, f.path, d.verdict FROM frame f JOIN detection d " f"ON d.frame_id = f.id AND d.run_id = ? WHERE f.satellite = ? AND " f"f.wavelength = ? AND f.t_start {op} ? AND d.verdict = 'good' " f"ORDER BY f.t_start {order} LIMIT 500", (run_id, satellite, wavelength, t_start)): if (satellite, wavelength, row[0]) not in excluded: return tuple(row) return None tile, head = SHEET_TILE, 40 sheet = Image.new("RGB", (3 * tile, len(rows) * (tile + head)), (40, 40, 40)) for index, (satellite, wavelength, t_start, note) in enumerate(rows): for column, direction in enumerate((-1, 0, 1)): cell = Image.new("RGB", (tile, tile + head)) draw = ImageDraw.Draw(cell) found = lookup(satellite, wavelength, t_start, direction) if found is None: draw.text((6, 6), "no good frame found", fill=(255, 80, 80)) else: when, relpath, verdict = found image, _ = fitsio.read_image(paths.abspath(relpath)) if image is not None: shown = np.clip(np.nan_to_num(metrics.to_display( np.asarray(image, np.float32), wavelength)), 0, 1) cell.paste(Image.fromarray((shown * 255).astype(np.uint8)) .resize((tile, tile)).convert("RGB"), (0, head)) gap = round((when - t_start) / paths.CADENCE) draw.text((6, 4), f"G{satellite} {wavelength}A {_when(when)} judged {verdict}", fill=(255, 255, 0)) draw.text((6, 20), (f"REVIEW: {note}" if direction == 0 else f"{gap:+d} slots")[:56], fill=(160, 220, 255)) sheet.paste(cell, (column * tile, index * (tile + head))) sheet.save(args.out, quality=90) print(f"{args.out}: {len(rows)} rows") return 0 def main(argv=None): parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = parser.add_subparsers(dest="command", required=True) measure = sub.add_parser("measure", help="read labelled frames and run every detector") measure.add_argument("--labels", required=True) measure.add_argument("--out", required=True) measure.add_argument("--workers", type=int, default=4) measure.add_argument("--db", default=None) measure.set_defaults(func=cmd_measure) refresh = sub.add_parser("refresh", help="re-run the single-frame detectors in place") refresh.add_argument("--features", required=True) refresh.add_argument("--workers", type=int, default=4) refresh.add_argument("--db", default=None) refresh.set_defaults(func=cmd_refresh) scoring = sub.add_parser("score", help="tabulate recall and false positives") scoring.add_argument("--labels", required=True) scoring.add_argument("--features", required=True) scoring.add_argument("--rows", default=None, help="comma-separated detectors; '+' combines (bad if any says bad)") scoring.add_argument("--misses", default=None, help="list the defects this row misses and clean frames it rejects") scoring.add_argument("--out", default=None) scoring.set_defaults(func=cmd_score) run = sub.add_parser("stored", help="score a judgment run stored in the index") run.add_argument("--labels", required=True) run.add_argument("--run", type=int, default=None, help="detector run id (default: the production judgment)") run.add_argument("--db", default=None) run.set_defaults(func=cmd_stored) dry = sub.add_parser("dry-run", help="judge stored measurements in memory and score") dry.add_argument("--labels", required=True) dry.add_argument("--start", type=int, required=True) dry.add_argument("--end", type=int, required=True) dry.add_argument("--satellites", default="16,18") dry.add_argument("--measure-run", type=int, default=None) dry.add_argument("--reference", type=int, default=46, help="stored run to list changed verdicts against (default 46)") dry.add_argument("--out", default=None, help="JSON of newly bad / newly good rows") dry.add_argument("--db", default=None) dry.set_defaults(func=cmd_dry_run) sheet = sub.add_parser("sheet", help="draw a 3-column neighbour sheet") sheet.add_argument("--rows", required=True, help="JSON rows [[satellite, band, 'YYYY-MM-DD HH:MM', note]]") sheet.add_argument("--key", default="newly_bad", help="list to draw when the rows file is a dry-run output") sheet.add_argument("--exclude", default=None, help="more rows never used as neighbours") sheet.add_argument("--limit", type=int, default=40) sheet.add_argument("--run", type=int, default=None, help="judgment run deciding what is good (default: production)") sheet.add_argument("--out", required=True) sheet.add_argument("--db", default=None) sheet.set_defaults(func=cmd_sheet) args = parser.parse_args(argv) return args.func(args) if __name__ == "__main__": sys.exit(main())