355 lines
13 KiB
Python
355 lines
13 KiB
Python
|
"""Scoring for detection and fill quality.
|
||
|
|
|
||
|
|
Detection scoring keeps two things apart that are easy to conflate:
|
||
|
|
|
||
|
|
* a **false positive** -- the detector flagged a slot the bench did not corrupt, and
|
||
|
|
the window's ground truth says was good;
|
||
|
|
* a **legacy disagreement** -- the detector flagged a slot the old filter passed.
|
||
|
|
|
||
|
|
On a vetted window the second is a candidate *find*, not an error: the old filter is
|
||
|
|
what we are trying to beat, so scoring its misses against a new detector would
|
||
|
|
penalise exactly the improvement we want.
|
||
|
|
|
||
|
|
Fill scoring is reported both in radiance and after the display mapping the videos
|
||
|
|
actually use, plus a temporal term. A frame can score well on per-frame PSNR and
|
||
|
|
still read as a visible stutter at 60 fps, so flicker is measured explicitly.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from dataclasses import dataclass, field
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
from skimage.metrics import structural_similarity
|
||
|
|
|
||
|
|
#: Display mapping used by merger_FITS.py to turn radiance into pixels, by band.
|
||
|
|
#: (vmin, vmax, gamma). Fill error is reported through this because it is what the
|
||
|
|
#: eye sees -- an error in the dim corona matters far less than one on the disc.
|
||
|
|
DISPLAY_MAPPING = {
|
||
|
|
94: (0.050, 8.0, 0.375),
|
||
|
|
131: (0.05, 8.0, 0.40),
|
||
|
|
171: (0.100, 20.0, 0.425),
|
||
|
|
195: (0.10, 30.0, 0.45),
|
||
|
|
284: (0.100, 40.0, 0.475),
|
||
|
|
304: (0.1, 90.0, 0.5),
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def to_display(image, wavelength):
|
||
|
|
"""Map radiance to the 0-1 display range merger_FITS.py renders with."""
|
||
|
|
vmin, vmax, gamma = DISPLAY_MAPPING.get(wavelength, (0.0, 1.0, 1.0))
|
||
|
|
clean = np.nan_to_num(np.asarray(image, dtype=np.float64), nan=0.0)
|
||
|
|
return np.clip((clean - vmin) / vmax, 0.0, 1.0) ** gamma
|
||
|
|
|
||
|
|
|
||
|
|
# ------------------------------------------------------------------------ detection
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class DetectionScore:
|
||
|
|
"""Confusion counts and derived rates for one detector on one bench case."""
|
||
|
|
|
||
|
|
true_positives: int = 0
|
||
|
|
false_positives: int = 0
|
||
|
|
false_negatives: int = 0
|
||
|
|
true_negatives: int = 0
|
||
|
|
unknown: int = 0
|
||
|
|
#: Slots the detector flagged that the legacy filter had passed and the bench
|
||
|
|
#: did not corrupt. Reported, never counted as errors.
|
||
|
|
legacy_disagreements: int = 0
|
||
|
|
#: Recall broken down by corruption mode, so a detector that only catches
|
||
|
|
#: blackouts cannot hide behind a good overall number.
|
||
|
|
recall_by_mode: dict = field(default_factory=dict)
|
||
|
|
elapsed_us: int = 0
|
||
|
|
frames: int = 0
|
||
|
|
|
||
|
|
@property
|
||
|
|
def precision(self):
|
||
|
|
flagged = self.true_positives + self.false_positives
|
||
|
|
return self.true_positives / flagged if flagged else float("nan")
|
||
|
|
|
||
|
|
@property
|
||
|
|
def recall(self):
|
||
|
|
actual = self.true_positives + self.false_negatives
|
||
|
|
return self.true_positives / actual if actual else float("nan")
|
||
|
|
|
||
|
|
@property
|
||
|
|
def f1(self):
|
||
|
|
precision, recall = self.precision, self.recall
|
||
|
|
if not np.isfinite(precision) or not np.isfinite(recall) or precision + recall == 0:
|
||
|
|
return float("nan")
|
||
|
|
return 2 * precision * recall / (precision + recall)
|
||
|
|
|
||
|
|
@property
|
||
|
|
def false_positive_rate(self):
|
||
|
|
negatives = self.false_positives + self.true_negatives
|
||
|
|
return self.false_positives / negatives if negatives else float("nan")
|
||
|
|
|
||
|
|
@property
|
||
|
|
def microseconds_per_frame(self):
|
||
|
|
return self.elapsed_us / self.frames if self.frames else float("nan")
|
||
|
|
|
||
|
|
def as_dict(self):
|
||
|
|
return {
|
||
|
|
"true_positives": self.true_positives,
|
||
|
|
"false_positives": self.false_positives,
|
||
|
|
"false_negatives": self.false_negatives,
|
||
|
|
"true_negatives": self.true_negatives,
|
||
|
|
"unknown": self.unknown,
|
||
|
|
"legacy_disagreements": self.legacy_disagreements,
|
||
|
|
"precision": self.precision,
|
||
|
|
"recall": self.recall,
|
||
|
|
"f1": self.f1,
|
||
|
|
"false_positive_rate": self.false_positive_rate,
|
||
|
|
"us_per_frame": self.microseconds_per_frame,
|
||
|
|
"recall_by_mode": dict(self.recall_by_mode),
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def score_detection(verdicts, injected, legacy_good=frozenset(), elapsed_us=0):
|
||
|
|
"""Score detector output against what the bench actually injected.
|
||
|
|
|
||
|
|
`verdicts` maps slot -> Verdict. `injected` maps slot -> corruption mode name
|
||
|
|
for every slot the bench damaged (a deleted slot has no frame to judge, so it is
|
||
|
|
not scored here). `legacy_good` is the set of slots the old filter passed.
|
||
|
|
|
||
|
|
'unknown' verdicts are counted separately and excluded from precision and recall
|
||
|
|
rather than folded into 'good' -- a detector that abstains has not made an error,
|
||
|
|
but it has not made a call either, and hiding that would flatter it.
|
||
|
|
"""
|
||
|
|
score = DetectionScore(elapsed_us=elapsed_us, frames=len(verdicts))
|
||
|
|
by_mode = {}
|
||
|
|
|
||
|
|
for slot, verdict in verdicts.items():
|
||
|
|
mode = injected.get(slot)
|
||
|
|
corrupted = mode is not None
|
||
|
|
|
||
|
|
if verdict.verdict == "unknown":
|
||
|
|
score.unknown += 1
|
||
|
|
if corrupted:
|
||
|
|
by_mode.setdefault(mode, [0, 0])[1] += 1
|
||
|
|
continue
|
||
|
|
|
||
|
|
flagged = verdict.verdict == "bad"
|
||
|
|
if corrupted:
|
||
|
|
stats = by_mode.setdefault(mode, [0, 0])
|
||
|
|
stats[1] += 1
|
||
|
|
if flagged:
|
||
|
|
score.true_positives += 1
|
||
|
|
stats[0] += 1
|
||
|
|
else:
|
||
|
|
score.false_negatives += 1
|
||
|
|
elif flagged:
|
||
|
|
score.false_positives += 1
|
||
|
|
if slot in legacy_good:
|
||
|
|
score.legacy_disagreements += 1
|
||
|
|
else:
|
||
|
|
score.true_negatives += 1
|
||
|
|
|
||
|
|
score.recall_by_mode = {
|
||
|
|
mode: (caught / total if total else float("nan"))
|
||
|
|
for mode, (caught, total) in sorted(by_mode.items())
|
||
|
|
}
|
||
|
|
return score
|
||
|
|
|
||
|
|
|
||
|
|
#: How several detectors' verdicts are folded into one.
|
||
|
|
COMBINATION_POLICIES = ("any", "all", "majority")
|
||
|
|
|
||
|
|
|
||
|
|
def combine_verdicts(verdict_maps, policy="any"):
|
||
|
|
"""Fold several detectors' verdicts into one per slot.
|
||
|
|
|
||
|
|
Which detectors to run in production is a trade-off between recall and the
|
||
|
|
false-positive rate, and the only honest way to choose is to score the
|
||
|
|
combinations the same way as the individuals. Because `detection` rows already
|
||
|
|
store a verdict per frame, this is pure post-processing.
|
||
|
|
|
||
|
|
* ``any`` -- bad if any detector says bad. Maximum recall; false positives
|
||
|
|
accumulate across detectors.
|
||
|
|
* ``all`` -- bad only if every detector that voted says bad. Minimum false
|
||
|
|
positives, and the right shape for a near-zero FP budget.
|
||
|
|
* ``majority`` -- bad if more than half the votes say bad.
|
||
|
|
|
||
|
|
``unknown`` abstains rather than voting. A detector that declines to judge must
|
||
|
|
not be silently counted as saying "good": under ``all`` that would let one
|
||
|
|
abstention veto a real detection, and under ``any`` it would inflate recall. A
|
||
|
|
slot where every detector abstains is itself ``unknown``.
|
||
|
|
|
||
|
|
`verdict_maps` is a sequence of {slot: Verdict}. Returns {slot: Verdict}.
|
||
|
|
"""
|
||
|
|
if policy not in COMBINATION_POLICIES:
|
||
|
|
raise ValueError(f"Unknown policy {policy!r}; expected one of {COMBINATION_POLICIES}")
|
||
|
|
if not verdict_maps:
|
||
|
|
return {}
|
||
|
|
|
||
|
|
from .detectors import BAD, GOOD, UNKNOWN, Verdict
|
||
|
|
|
||
|
|
combined = {}
|
||
|
|
slots = set()
|
||
|
|
for verdicts in verdict_maps:
|
||
|
|
slots.update(verdicts)
|
||
|
|
|
||
|
|
for slot in slots:
|
||
|
|
votes = []
|
||
|
|
reasons = []
|
||
|
|
for verdicts in verdict_maps:
|
||
|
|
verdict = verdicts.get(slot)
|
||
|
|
if verdict is None or verdict.verdict == UNKNOWN:
|
||
|
|
continue
|
||
|
|
votes.append(verdict.verdict == BAD)
|
||
|
|
if verdict.verdict == BAD and verdict.reason:
|
||
|
|
reasons.append(verdict.reason)
|
||
|
|
|
||
|
|
if not votes:
|
||
|
|
combined[slot] = Verdict(UNKNOWN, "no detector judged this frame", {})
|
||
|
|
continue
|
||
|
|
|
||
|
|
if policy == "any":
|
||
|
|
is_bad = any(votes)
|
||
|
|
elif policy == "all":
|
||
|
|
is_bad = all(votes)
|
||
|
|
else:
|
||
|
|
is_bad = sum(votes) * 2 > len(votes)
|
||
|
|
|
||
|
|
combined[slot] = Verdict(
|
||
|
|
BAD if is_bad else GOOD,
|
||
|
|
"; ".join(reasons[:3]) if is_bad else None,
|
||
|
|
{"votes_bad": float(sum(votes)), "votes_total": float(len(votes))},
|
||
|
|
)
|
||
|
|
return combined
|
||
|
|
|
||
|
|
|
||
|
|
def precision_recall_curve(scores, labels):
|
||
|
|
"""Precision/recall across every threshold of a continuous detector score.
|
||
|
|
|
||
|
|
`scores` are higher-is-more-suspicious. Returns (thresholds, precision, recall).
|
||
|
|
"""
|
||
|
|
scores = np.asarray(scores, dtype=np.float64)
|
||
|
|
labels = np.asarray(labels, dtype=bool)
|
||
|
|
finite = np.isfinite(scores)
|
||
|
|
scores, labels = scores[finite], labels[finite]
|
||
|
|
if scores.size == 0 or not labels.any():
|
||
|
|
return np.array([]), np.array([]), np.array([])
|
||
|
|
|
||
|
|
order = np.argsort(-scores)
|
||
|
|
scores, labels = scores[order], labels[order]
|
||
|
|
true_positives = np.cumsum(labels)
|
||
|
|
flagged = np.arange(1, scores.size + 1)
|
||
|
|
precision = true_positives / flagged
|
||
|
|
recall = true_positives / labels.sum()
|
||
|
|
return scores, precision, recall
|
||
|
|
|
||
|
|
|
||
|
|
def average_precision(scores, labels):
|
||
|
|
"""Area under the precision-recall curve; the threshold-free summary."""
|
||
|
|
_, precision, recall = precision_recall_curve(scores, labels)
|
||
|
|
if recall.size == 0:
|
||
|
|
return float("nan")
|
||
|
|
return float(np.sum(np.diff(np.concatenate([[0.0], recall])) * precision))
|
||
|
|
|
||
|
|
|
||
|
|
# ----------------------------------------------------------------------------- fill
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class FillScore:
|
||
|
|
"""Reconstruction error for one filled frame."""
|
||
|
|
|
||
|
|
rmse: float
|
||
|
|
mae: float
|
||
|
|
log_rmse: float
|
||
|
|
psnr: float
|
||
|
|
ssim: float
|
||
|
|
gap_frames: int = 0
|
||
|
|
wavelength: int = 0
|
||
|
|
|
||
|
|
def as_dict(self):
|
||
|
|
return {
|
||
|
|
"rmse": self.rmse,
|
||
|
|
"mae": self.mae,
|
||
|
|
"log_rmse": self.log_rmse,
|
||
|
|
"psnr": self.psnr,
|
||
|
|
"ssim": self.ssim,
|
||
|
|
"gap_frames": self.gap_frames,
|
||
|
|
"wavelength": self.wavelength,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def score_fill(filled, truth, wavelength, gap_frames=0):
|
||
|
|
"""Compare a reconstructed frame against the frame that was withheld.
|
||
|
|
|
||
|
|
Radiance errors (rmse/mae) are dominated by the bright disc; the log term keeps
|
||
|
|
the faint corona visible in the score; psnr and ssim are computed after the
|
||
|
|
display mapping, because that is the image a viewer actually sees.
|
||
|
|
"""
|
||
|
|
filled = np.nan_to_num(np.asarray(filled, dtype=np.float64), nan=0.0)
|
||
|
|
truth = np.nan_to_num(np.asarray(truth, dtype=np.float64), nan=0.0)
|
||
|
|
if filled.shape != truth.shape:
|
||
|
|
raise ValueError(f"shape mismatch: {filled.shape} vs {truth.shape}")
|
||
|
|
|
||
|
|
residual = filled - truth
|
||
|
|
rmse = float(np.sqrt(np.mean(residual**2)))
|
||
|
|
mae = float(np.mean(np.abs(residual)))
|
||
|
|
|
||
|
|
log_residual = np.log1p(np.clip(filled, 0, None)) - np.log1p(np.clip(truth, 0, None))
|
||
|
|
log_rmse = float(np.sqrt(np.mean(log_residual**2)))
|
||
|
|
|
||
|
|
shown_fill, shown_truth = to_display(filled, wavelength), to_display(truth, wavelength)
|
||
|
|
display_mse = float(np.mean((shown_fill - shown_truth) ** 2))
|
||
|
|
psnr = float("inf") if display_mse == 0 else float(10.0 * np.log10(1.0 / display_mse))
|
||
|
|
ssim = float(structural_similarity(shown_truth, shown_fill, data_range=1.0))
|
||
|
|
|
||
|
|
return FillScore(rmse, mae, log_rmse, psnr, ssim, gap_frames, wavelength)
|
||
|
|
|
||
|
|
|
||
|
|
def temporal_flicker(sequence, truth_sequence, wavelength):
|
||
|
|
"""How much the reconstruction's frame-to-frame motion departs from reality.
|
||
|
|
|
||
|
|
Per-frame PSNR is blind to the artifact that matters most in a 60 fps video: a
|
||
|
|
filled run can match each frame tolerably and still freeze then jump. This
|
||
|
|
compares the *rate of change* rather than the frames, in display space.
|
||
|
|
|
||
|
|
Returns mean absolute difference of successive-frame deltas; 0 is perfect.
|
||
|
|
"""
|
||
|
|
if len(sequence) != len(truth_sequence):
|
||
|
|
raise ValueError("sequences must be the same length")
|
||
|
|
if len(sequence) < 2:
|
||
|
|
return float("nan")
|
||
|
|
|
||
|
|
shown = [to_display(frame, wavelength) for frame in sequence]
|
||
|
|
shown_truth = [to_display(frame, wavelength) for frame in truth_sequence]
|
||
|
|
deltas = [np.mean(np.abs(shown[i] - shown[i - 1])) for i in range(1, len(shown))]
|
||
|
|
truth_deltas = [
|
||
|
|
np.mean(np.abs(shown_truth[i] - shown_truth[i - 1])) for i in range(1, len(shown_truth))
|
||
|
|
]
|
||
|
|
return float(np.mean(np.abs(np.array(deltas) - np.array(truth_deltas))))
|
||
|
|
|
||
|
|
|
||
|
|
def summarise_fills(scores):
|
||
|
|
"""Aggregate per-frame fill scores, overall and by gap length."""
|
||
|
|
if not scores:
|
||
|
|
return {}
|
||
|
|
by_gap = {}
|
||
|
|
for score in scores:
|
||
|
|
by_gap.setdefault(score.gap_frames, []).append(score)
|
||
|
|
|
||
|
|
def mean(values):
|
||
|
|
finite = [v for v in values if np.isfinite(v)]
|
||
|
|
return float(np.mean(finite)) if finite else float("nan")
|
||
|
|
|
||
|
|
return {
|
||
|
|
"n": len(scores),
|
||
|
|
"rmse": mean([s.rmse for s in scores]),
|
||
|
|
"log_rmse": mean([s.log_rmse for s in scores]),
|
||
|
|
"psnr": mean([s.psnr for s in scores]),
|
||
|
|
"ssim": mean([s.ssim for s in scores]),
|
||
|
|
"by_gap": {
|
||
|
|
gap: {
|
||
|
|
"n": len(group),
|
||
|
|
"rmse": mean([s.rmse for s in group]),
|
||
|
|
"psnr": mean([s.psnr for s in group]),
|
||
|
|
"ssim": mean([s.ssim for s in group]),
|
||
|
|
}
|
||
|
|
for gap, group in sorted(by_gap.items())
|
||
|
|
},
|
||
|
|
}
|