jeremy/noaa-goes-visualization
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
noaa-goes-visualization/suvi/detectors.py

742 lines
30 KiB
Python
Raw Permalink Blame History

"""Bad-frame detectors.
Every detector is a pure function: data in, a :class:`Verdict` out. Nothing here
renames a file, writes a plot, or touches the database -- that is the driver's job.
This is what lets the same code run in production (``filter_FITS.py``) and under the
test bench, and lets several detectors judge the same frame independently.
Two shapes of detector:
* **Frame-level** (:func:`header_v1`, :func:`geometry_v1`) judge one frame alone.
* **Series-level** (:func:`temporal_v1`, :func:`crosssat_v1`) judge a frame in the
context of its neighbours in time, or of the other satellite at the same instant.
They work from :class:`FrameFeatures` -- cached header values plus a small
thumbnail -- rather than full 1280x1280 arrays, which keeps a whole day of context
in memory at once.
Every detector reports continuous ``scores`` alongside its verdict. The driver
stores those, so re-tuning a threshold later is a database query rather than another
pass over the archive.
"""
from dataclasses import dataclass, field
import cv2 as cv
import numpy as np
# --------------------------------------------------------------------------- types
@dataclass(frozen=True)
class Verdict:
"""One detector's judgement of one frame."""
verdict: str # 'good' | 'bad' | 'unknown'
reason: str | None = None
scores: dict = field(default_factory=dict)
GOOD = "good"
BAD = "bad"
UNKNOWN = "unknown"
@dataclass
class FrameFeatures:
"""The cheap summary of a frame that series-level detectors work from."""
slot: tuple # (satellite, wavelength, t_start)
header: dict = field(default_factory=dict)
#: Read error from fitsio, if the frame could not be parsed at all.
error: str | None = None
#: Small float32 image (see THUMBNAIL_SIZE), or None if not loaded.
thumbnail: np.ndarray | None = None
@property
def t_start(self):
return self.slot[2]
@property
def wavelength(self):
return self.slot[1]
#: Series detectors compare thumbnails, not full frames. 128px preserves the disc,
#: active regions and gross structure while making a day of context cheap to hold.
THUMBNAIL_SIZE = 128
def thumbnail(image, size=THUMBNAIL_SIZE):
"""Downsample a frame for structural comparison. NaNs become zero."""
clean = np.nan_to_num(np.asarray(image, dtype=np.float32), nan=0.0, posinf=0.0, neginf=0.0)
return cv.resize(clean, (size, size), interpolation=cv.INTER_AREA)
# ------------------------------------------------------------------ header_v1
#: Plausible radiance range of a healthy frame, per passband, in W m-2 sr-1.
#:
#: Deliberately wide -- roughly a factor of five outside the observed annual range of
#: good frames. Sampled across 24 days of 2024 on both satellites, the *rejected*
#: frames' IMG_MEAN distribution sits almost entirely inside the good one (171A: good
#: 0.288-0.452, rejected median 0.352), so a tight bound here would cost enormous
#: numbers of false positives while catching little. What this rule is actually for
#: is gross dropout and blowout -- eclipse frames run 1e-4, three orders of magnitude
#: below the floor -- and it is priced accordingly.
HEADER_MEAN_BOUNDS = {
94: (0.004, 0.41),
131: (0.005, 0.39),
171: (0.057, 2.3),
195: (0.15, 5.4),
284: (0.13, 7.0),
304: (0.29, 20.0),
}
#: Solar diameter in pixels; varies ~3% over the year with Earth-Sun distance.
DIAM_SUN_RANGE = (740.0, 800.0)
#: How far the recorded sun centre may sit from the image centre, in pixels.
MAX_CRPIX_OFFSET = 8.0
def header_v1(features, mean_bounds=None, diam_range=DIAM_SUN_RANGE,
max_crpix_offset=MAX_CRPIX_OFFSET):
"""Judge a frame from its header alone -- no pixel decompression.
The instrument already reports most of what matters: ``ECLIPSE`` marks frames
taken through Earth's shadow, ``EMPTY`` marks frames built from no source
imagery, and ``IMG_MEAN``/``IMG_SDEV`` summarise the radiance. On sampled
archive days every frame the pixel filter rejected also carried ``ECLIPSE=2``,
at roughly a fifth of the cost of decompressing the image.
``DEGRADED`` is recorded as a score but is deliberately *not* a rejection rule.
It is set for reasons unrelated to a frame's usability: across sampled days the
195A band carried ``DEGRADED=T`` on 40/40 good frames on 2024-01-20 and 0/40 on
2024-07-04, on both satellites. Rejecting on it would discard entire months of
that band. The bench can re-weight it from the stored score if that turns out
to be worth doing.
"""
mean_bounds = mean_bounds or HEADER_MEAN_BOUNDS
header = features.header
scores = {}
if features.error:
return Verdict(BAD, f"unreadable: {features.error}", {"read_ok": 0.0})
if not header:
return Verdict(UNKNOWN, "no header cached", scores)
for field_name in ("img_mean", "img_sdev", "diam_sun", "eclipse", "degraded"):
if field_name in header:
scores[field_name] = float(header[field_name])
if header.get("empty"):
return Verdict(BAD, "EMPTY flag set", scores)
eclipse = header.get("eclipse")
if eclipse:
return Verdict(BAD, f"ECLIPSE flag set ({int(eclipse)})", scores)
# A frame built from no source images carries no information regardless of what
# the radiance statistics happen to say.
if header.get("num_imgs") is not None and header["num_imgs"] < 1:
scores["num_imgs"] = float(header["num_imgs"])
return Verdict(BAD, "no source images", scores)
# The band the file claims must match the band its name puts it in, or the
# archive has a misfiled frame and every downstream threshold is wrong.
declared = header.get("wavelnth")
if declared is not None and int(declared) != features.wavelength:
scores["wavelnth"] = float(declared)
return Verdict(BAD, f"wavelength {int(declared)} != {features.wavelength}", scores)
mean = header.get("img_mean")
if mean is not None:
low, high = mean_bounds.get(features.wavelength, (0.0, np.inf))
if mean < low:
return Verdict(BAD, f"IMG_MEAN {mean:.3g} below {low:g}", scores)
if mean > high:
return Verdict(BAD, f"IMG_MEAN {mean:.3g} above {high:g}", scores)
sdev = header.get("img_sdev")
if sdev is not None and sdev <= 0.0:
return Verdict(BAD, "zero variance", scores)
diameter = header.get("diam_sun")
if diameter is not None and not (diam_range[0] <= diameter <= diam_range[1]):
return Verdict(BAD, f"DIAM_SUN {diameter:.1f} outside {diam_range}", scores)
crpix1, crpix2 = header.get("crpix1"), header.get("crpix2")
if crpix1 is not None and crpix2 is not None:
offset = max(abs(crpix1 - 640.5), abs(crpix2 - 640.5))
scores["crpix_offset"] = float(offset)
if offset > max_crpix_offset:
return Verdict(BAD, f"sun centre offset {offset:.1f}px", scores)
return Verdict(GOOD, None, scores)
# ---------------------------------------------------------------- geometry_v1
#: Per-band radiance ceiling used to normalise the image before shape analysis.
#: Index order matches paths.WAVELENGTHS.
GEOMETRY_THRESHOLDS = {94: 0.050, 131: 0.10, 171: 1.00, 195: 1.40, 284: 1.0, 304: 2.50}
EXPECTED_DIMS = 1280
HALF_DIMS = EXPECTED_DIMS // 2
VALID_RADII = (383, 394)
AVG_RADIUS = (VALID_RADII[0] + VALID_RADII[1]) // 2
MAX_RADIUS_ERROR = 80
MAX_CENTER_SKEW = 7
RATIO_ABOVE_THRESH_MAX = 0.4
RATIO_ABOVE_THRESH_MIN = 0.07
EDGE_THRESH = 0.98
MAX_GOF = 0.2
_ideal_axis_cache = {}
def ideal_disc_axis():
"""Column-mean profile of a perfect solar disc, used as the shape reference."""
if "axis" not in _ideal_axis_cache:
disc = np.zeros((HALF_DIMS, HALF_DIMS))
cv.circle(disc, (HALF_DIMS // 2, HALF_DIMS // 2), AVG_RADIUS // 2, 1, -1)
_ideal_axis_cache["axis"] = np.average(disc, 0)
return _ideal_axis_cache["axis"]
def geometry_v1(image, wavelength):
"""The pre-existing pixel-geometry filter, preserved as the baseline.
Normalises against a per-band radiance ceiling, then checks the fraction of
saturated pixels, the intensity-weighted centroid, the radius implied by the
98th-percentile cumulative edges, and the fit against an ideal disc profile.
Deliberately kept behaviourally identical to the original so bench numbers
describe what the pipeline actually did, with two known weaknesses left intact
for the bench to quantify:
* the centre check is one-sided -- ``HALF_DIMS//2 - centre > skew`` catches a
disc displaced up/left but not one displaced down/right;
* non-finite pixels propagate as NaN through every comparison, and ``NaN > x``
is False, so an all-NaN frame passes every check.
The one deliberate change: the original raised on an unexpected image size and
left the file unlabelled. Here that is a ``bad`` verdict, because "no verdict"
is not a usable baseline to score against.
"""
scores = {}
if image is None:
return Verdict(BAD, "no image data", scores)
if image.shape[0] != EXPECTED_DIMS or image.shape[1] != EXPECTED_DIMS:
return Verdict(BAD, f"unexpected dimensions {image.shape}", scores)
threshold = GEOMETRY_THRESHOLDS.get(wavelength)
if threshold is None:
return Verdict(UNKNOWN, f"no threshold for band {wavelength}", scores)
data = cv.resize(
np.asarray(image, dtype=np.float32),
dsize=(HALF_DIMS, HALF_DIMS),
interpolation=cv.INTER_LINEAR,
)
above = data > threshold
normalised = np.copy(data)
normalised[above] = threshold
normalised /= threshold
ratio = np.count_nonzero(above) / data.shape[0] / data.shape[1]
scores["ratio_above_thresh"] = float(ratio)
if ratio > RATIO_ABOVE_THRESH_MAX:
return Verdict(BAD, f"ratio_above_thresh {ratio:.2f} too high", scores)
if ratio < RATIO_ABOVE_THRESH_MIN:
return Verdict(BAD, f"ratio_above_thresh {ratio:.2f} too low", scores)
xavg = np.average(normalised, 0)
yavg = np.average(normalised, 1)
axis = list(range(HALF_DIMS))
centre_x = np.average(axis, 0, xavg)
centre_y = np.average(axis, 0, yavg)
scores["centre_x"] = float(centre_x)
scores["centre_y"] = float(centre_y)
good_centre = not (
(HALF_DIMS // 2 - centre_x > MAX_CENTER_SKEW)
or (HALF_DIMS // 2 - centre_y > MAX_CENTER_SKEW)
)
high_x = np.argmax(np.cumsum(xavg) > (np.sum(xavg) * EDGE_THRESH))
low_x = len(xavg) - np.argmax(np.cumsum(np.flip(xavg)) > (np.sum(xavg) * EDGE_THRESH))
high_y = np.argmax(np.cumsum(yavg) > (np.sum(yavg) * EDGE_THRESH))
low_y = len(yavg) - np.argmax(np.cumsum(np.flip(yavg)) > (np.sum(yavg) * EDGE_THRESH))
# Not halved: the image is already downsampled by two in each dimension.
radius = ((high_x - low_x) + (high_y - low_y)) / 2.0
scores["radius"] = float(radius)
good_radius = abs(radius - AVG_RADIUS) <= MAX_RADIUS_ERROR
reference = ideal_disc_axis()
gof_x = np.sum(np.abs(xavg - reference)) / HALF_DIMS
gof_y = np.sum(np.abs(yavg - reference)) / HALF_DIMS
scores["gof_x"] = float(gof_x)
scores["gof_y"] = float(gof_y)
good_fit = not (gof_x > MAX_GOF or gof_y > MAX_GOF)
if good_centre and good_radius and good_fit:
return Verdict(GOOD, None, scores)
failed = [
name
for name, ok in (("centre", good_centre), ("radius", good_radius), ("fit", good_fit))
if not ok
]
return Verdict(BAD, "failed " + "+".join(failed), scores)
# -------------------------------------------------------------------- disc_v1
#: Angular samples in the polar resampling. The measurements are all averages over
#: this axis, so more angles cost little and reduce noise.
DISC_ANGLES = 720
#: Working resolution. Halving 1280 keeps the limb several pixels wide while making
#: the polar transform cheap.
DISC_SIZE = 640
#: Radial bands, in units of the expected solar radius, used to characterise the
#: profile. The on-disc band avoids both the centre (where limb darkening and
#: filaments live) and the limb itself; the off-limb band sits outside the corona's
#: steepest falloff but inside the frame -- 1.45R of a ~193px radius is 280px, within
#: the 320px half-width.
DISC_ON_BAND = (0.30, 0.70)
DISC_OFF_BAND = (1.25, 1.45)
#: Where the limb is allowed to be found, again in units of expected radius.
DISC_SEARCH_BAND = (0.75, 1.25)
def disc_profile(image, expected_radius, size=DISC_SIZE, angles=DISC_ANGLES):
"""Measure the solar disc from its azimuthally averaged radial profile.
Returns a dict of continuous measurements, or with None values where the frame
is too degenerate to measure. Separated from the verdict logic so that
``filter_FITS.py calibrate`` can gather the same numbers over a sample.
Averaging over angle is the entire point. An active region is localised in
angle, so it barely shifts the averaged profile; the existing ``geometry_v1``
averages along image rows and columns instead, which a bright region shifts
directly -- the cause of its 36.9% rejection rate.
`expected_radius` is in pixels *at the working resolution*, i.e. DIAM_SUN / 4.
"""
empty = {"limb_contrast": None, "radius_ratio": None, "limb_width": None}
if image is None or image.ndim != 2 or min(image.shape) < 16:
return empty
if not expected_radius or not np.isfinite(expected_radius) or expected_radius <= 0:
return empty
working = cv.resize(
np.nan_to_num(np.asarray(image, dtype=np.float32), nan=0.0, posinf=0.0, neginf=0.0),
(size, size),
interpolation=cv.INTER_AREA,
)
# Radiance spans orders of magnitude; log compresses a flare so it cannot
# dominate the profile the way it dominates a linear mean.
working = np.log1p(np.clip(working, 0.0, None))
max_radius = size / 2.0
polar = cv.warpPolar(
working,
(int(max_radius), angles),
(size / 2.0, size / 2.0),
max_radius,
cv.WARP_POLAR_LINEAR,
)
profile = polar.mean(axis=0)
radii = np.arange(profile.size, dtype=np.float64)
scale = expected_radius
def band(low, high):
mask = (radii > low * scale) & (radii < high * scale)
return profile[mask] if mask.any() else np.array([])
on_values, off_values = band(*DISC_ON_BAND), band(*DISC_OFF_BAND)
if on_values.size < 3 or off_values.size < 3:
return empty
on_disc, off_limb = float(np.median(on_values)), float(np.median(off_values))
if not np.isfinite(on_disc) or not np.isfinite(off_limb):
return empty
if on_disc <= 0:
# No signal at all where the disc should be. This is a conclusion, not a
# failure to measure: report zero contrast so the caller rejects the frame
# rather than abstaining on it.
return {"limb_contrast": 0.0, "radius_ratio": None, "limb_width": None}
contrast = (on_disc - off_limb) / on_disc
measured = {"limb_contrast": float(contrast), "radius_ratio": None, "limb_width": None}
if on_disc <= off_limb:
# No radial falloff at all: there is no disc here to measure the size of.
return measured
# The limb is the steepest descent of the smoothed profile. Taken on the
# *averaged* profile rather than per angle: fitting each angle separately failed
# outright on frames where too few angles yielded a usable edge.
smoothed = cv.GaussianBlur(profile.astype(np.float32).reshape(1, -1), (9, 1), 0).ravel()
derivative = np.gradient(smoothed.astype(np.float64))
low, high = DISC_SEARCH_BAND
window = (radii >= low * scale) & (radii <= high * scale)
if window.sum() < 5:
return measured
indices = np.flatnonzero(window)
local = derivative[indices]
trough = int(indices[np.argmin(local)])
depth = derivative[trough]
if not np.isfinite(depth) or depth >= 0:
return measured # no descending edge: not a disc
measured["radius_ratio"] = float(trough / scale)
# Width of the descent at half its depth. A disc displaced by d smears the
# averaged limb across roughly 2d, so this is what makes the measurement
# sensitive to decentring without ever having to locate a centre.
half = depth / 2.0
left = trough
while left > 0 and derivative[left - 1] <= half:
left -= 1
right = trough
last = derivative.size - 1
while right < last and derivative[right + 1] <= half:
right += 1
measured["limb_width"] = float((right - left + 1) / scale)
return measured
#: Per-band acceptance ranges for the disc_v1 measurements.
#:
#: Generated by ``filter_FITS.py calibrate``; regenerate and paste the output here
#: rather than hand-editing. Derived from 400 frames per band sampled evenly across
#: the archive's whole time range, with 0/400 sample failures in every band.
#:
#: The bounds sit well outside the observed spread of good frames, because the
#: false-positive budget for this detector is <0.1%: the method it replaces was
#: discarding 36.9% of perfectly good frames, which is the entire reason it exists.
#: Only the diagnostic side of each measurement is bounded -- see CALIBRATION_SIDES
#: in filter_FITS.py.
#:
#: Note what the wide contrast floors mean in practice. Good frames in most bands
#: reach as low as 0.11-0.22 contrast, so a floor tight enough to catch a partially
#: degraded frame would reject real data. At this budget limb_contrast therefore
#: catches total signal loss (all-zero, NaN, uniform, which measure exactly 0.0) and
#: little else; size and shape faults are caught by the other two measurements.
#: 171A is the exception, where good frames never drop below 0.68.
#:
#: A measurement whose calibrated bounds cannot achieve the budget is set to None
#: here, which records it as a score without letting it fail a frame.
DISC_BOUNDS = {
94: {
"limb_contrast": (0.02, 1.0),
"radius_ratio": (0.9631, 1.1176),
"limb_width": (0.0, 0.1217),
},
131: {
"limb_contrast": (0.02, 1.0),
"radius_ratio": (0.9578, 1.0854),
"limb_width": (0.0, 0.1011),
},
171: {
"limb_contrast": (0.3183, 1.0),
"radius_ratio": (0.9616, 1.1062),
"limb_width": (0.0, 0.1375),
},
195: {
"limb_contrast": (0.02, 1.0),
"radius_ratio": (0.9676, 1.1167),
"limb_width": (0.0, 0.1704),
},
284: {
"limb_contrast": (0.02, 1.0),
"radius_ratio": (0.9548, 1.1489),
"limb_width": (0.0, 0.3177),
},
304: {
"limb_contrast": (0.02, 1.0),
"radius_ratio": (0.9548, 1.0813),
"limb_width": (0.0, 0.1016),
},
}
def disc_v1(image, wavelength, header=None, bounds=None):
"""Verify that the frame holds a correctly sized, centred, sharp solar disc.
Replaces ``geometry_v1``'s intensity-weighted centroid, which a large active
region drags off centre -- rejecting entire days of good data during exactly the
solar activity most worth watching.
Everything measured here is an average over angle, so where the bright regions
sit on the disc does not move it. What it deliberately cannot see is rotation:
``rotate`` and ``yaw_flip`` leave every azimuthally averaged quantity unchanged,
and one frame carries no absolute rotation reference. Those belong to
``temporal_v1`` and ``crosssat_v1``.
The expected radius comes from the frame's own ``DIAM_SUN``, which varies by
3.3% over the year with the Earth-Sun distance.
"""
bounds = bounds or DISC_BOUNDS
if image is None:
return Verdict(BAD, "no image data", {})
diameter = (header or {}).get("diam_sun")
if not diameter:
return Verdict(UNKNOWN, "no DIAM_SUN in header", {})
# DIAM_SUN is a full diameter at full resolution; halve for radius, halve again
# for the working downsample.
expected_radius = float(diameter) / 4.0
measured = disc_profile(image, expected_radius)
scores = {k: v for k, v in measured.items() if v is not None}
if measured["limb_contrast"] is None:
return Verdict(UNKNOWN, "radial profile not measurable", scores)
limits = bounds.get(wavelength)
if limits is None:
return Verdict(UNKNOWN, f"no disc bounds for band {wavelength}", scores)
failed = []
for name, limit in limits.items():
if limit is None:
continue # recorded as a score, not used to judge
value = measured.get(name)
if value is None:
failed.append(f"{name} unmeasurable")
elif not (limit[0] <= value <= limit[1]):
failed.append(f"{name} {value:.3f} outside [{limit[0]:.3f}, {limit[1]:.3f}]")
if failed:
return Verdict(BAD, "; ".join(failed), scores)
return Verdict(GOOD, None, scores)
# ---------------------------------------------------------------- temporal_v1
#: Frames either side used to build the local reference. At a 4-minute cadence,
#: +/-6 spans about 50 minutes -- long enough to be robust, short enough that real
#: solar evolution does not dominate.
TEMPORAL_WINDOW = 6
#: Robust z-score above which a frame's brightness is judged anomalous.
TEMPORAL_BRIGHTNESS_Z = 8.0
#: Structural difference from neighbours, relative to the neighbours' own churn.
TEMPORAL_STRUCTURE_RATIO = 6.0
#: Below this relative difference two consecutive frames are considered identical.
FROZEN_TOLERANCE = 1e-6
def _robust_z(values, index, window):
"""Median-absolute-deviation z-score of values[index] against its neighbours."""
low = max(0, index - window)
high = min(len(values), index + window + 1)
neighbours = np.array(
[values[i] for i in range(low, high) if i != index and np.isfinite(values[i])]
)
if neighbours.size < 3:
return None
median = np.median(neighbours)
mad = np.median(np.abs(neighbours - median))
if mad <= 0:
# A perfectly steady neighbourhood: fall back to the spread, and treat an
# exactly-constant one as uninformative rather than infinitely sensitive.
spread = neighbours.std()
if spread <= 0:
return 0.0 if values[index] == median else np.inf
return abs(values[index] - median) / spread
return abs(values[index] - median) / (1.4826 * mad)
def temporal_v1(series, window=TEMPORAL_WINDOW, brightness_z=TEMPORAL_BRIGHTNESS_Z,
structure_ratio=TEMPORAL_STRUCTURE_RATIO,
frozen_tolerance=FROZEN_TOLERANCE):
"""Judge each frame against its neighbours in the same band and satellite.
Catches what single-frame checks cannot: a frame that is individually plausible
but inconsistent with the minutes either side of it -- a frozen duplicate, a
brightness step, a substituted frame from another time.
`series` must be ordered by time and come from one (satellite, wavelength)
stream; gaps are fine, they simply widen the neighbourhood in wall-clock terms.
Returns one Verdict per input frame.
"""
count = len(series)
means = [
f.header.get("img_mean", np.nan) if not f.error else np.nan for f in series
]
verdicts = []
# Structural churn between consecutive thumbnails, when they are available.
diffs = [np.nan] * count
for i in range(1, count):
a, b = series[i - 1].thumbnail, series[i].thumbnail
if a is not None and b is not None and a.shape == b.shape:
scale = float(np.abs(a).mean() + np.abs(b).mean()) / 2.0
diffs[i] = float(np.abs(a - b).mean()) / scale if scale > 0 else np.nan
for i, frame in enumerate(series):
scores = {}
if frame.error:
verdicts.append(Verdict(BAD, f"unreadable: {frame.error}", {"read_ok": 0.0}))
continue
z = _robust_z(means, i, window)
if z is not None:
scores["brightness_z"] = float(z)
# A frame identical to its predecessor means the feed stalled.
if i > 0 and np.isfinite(diffs[i]) and diffs[i] < frozen_tolerance:
scores["frame_diff"] = float(diffs[i])
verdicts.append(Verdict(BAD, "identical to previous frame", scores))
continue
neighbour_diffs = np.array(
[
diffs[j]
for j in range(max(1, i - window), min(count, i + window + 1))
if j != i and np.isfinite(diffs[j])
]
)
if np.isfinite(diffs[i]) and neighbour_diffs.size >= 3:
typical = float(np.median(neighbour_diffs))
scores["frame_diff"] = float(diffs[i])
scores["frame_diff_ratio"] = float(diffs[i] / typical) if typical > 0 else 0.0
if typical > 0 and diffs[i] / typical > structure_ratio:
verdicts.append(
Verdict(BAD, f"structural jump {diffs[i] / typical:.1f}x typical", scores)
)
continue
if z is not None and z > brightness_z:
verdicts.append(Verdict(BAD, f"brightness z={z:.1f}", scores))
continue
if z is None and not np.isfinite(diffs[i]):
verdicts.append(Verdict(UNKNOWN, "insufficient context", scores))
continue
verdicts.append(Verdict(GOOD, None, scores))
return verdicts
# ---------------------------------------------------------------- crosssat_v1
#: Relative disagreement between satellites above which one of them is wrong.
CROSSSAT_MAX_DIFF = 0.35
#: Alignment shift beyond which the two views cannot be meaningfully compared.
#: Geostationary parallax is under ~7px at 2.5 arcsec/px; more means a pointing fault.
CROSSSAT_MAX_SHIFT = 12.0
#: Plausible range for the calibration gain between the two flight models. Wide
#: enough to absorb genuine instrument differences, narrow enough that a blackout or
#: a saturation blowout cannot be rescaled into agreement.
CROSSSAT_GAIN_RANGE = (0.2, 5.0)
def _gain_match(source, reference):
"""Least-squares gain and offset putting `source` on `reference`'s scale.
The two instruments are different flight models with different responses, so a
raw difference conflates calibration with genuine disagreement.
Returns (matched, gain, offset). The gain is returned rather than hidden
because matching is otherwise *too* effective: a frame uniformly scaled by 1e-4
fits perfectly after rescaling, so a detector that only looked at the residual
would call a total blackout a match. The size of the correction is itself
evidence.
"""
x = source.ravel().astype(np.float64)
y = reference.ravel().astype(np.float64)
finite = np.isfinite(x) & np.isfinite(y)
if finite.sum() < 16:
return source, 1.0, 0.0
x, y = x[finite], y[finite]
variance = float(((x - x.mean()) ** 2).sum())
if variance <= 0:
return source, 1.0, 0.0
gain = float(((x - x.mean()) * (y - y.mean())).sum() / variance)
offset = float(y.mean() - gain * x.mean())
return source * gain + offset, gain, offset
def align_shift(a, b):
"""Sub-pixel translation between two thumbnails, via phase correlation."""
if a is None or b is None or a.shape != b.shape:
return None
window = cv.createHanningWindow((a.shape[1], a.shape[0]), cv.CV_64F)
(dx, dy), _ = cv.phaseCorrelate(
np.nan_to_num(a.astype(np.float64)), np.nan_to_num(b.astype(np.float64)), window
)
return float(np.hypot(dx, dy))
def crosssat_v1(features_a, features_b, temporal_a=None, temporal_b=None,
max_diff=CROSSSAT_MAX_DIFF, max_shift=CROSSSAT_MAX_SHIFT,
gain_range=CROSSSAT_GAIN_RANGE):
"""Compare the two satellites' view of the same instant.
GOES-16 and GOES-18 observe the same Sun from geostationary orbit, so at 1 AU
their views differ only by a few pixels of parallax and by instrument
calibration. Sustained disagreement therefore means one of them is wrong.
Deciding *which* one needs outside information: `temporal_a`/`temporal_b` are the
corresponding :func:`temporal_v1` verdicts, and whichever frame also disagrees
with its own history is blamed. With no tiebreak available both are reported
``unknown`` rather than guessing -- a wrong attribution would discard a good
frame and keep a bad one.
Returns (verdict_a, verdict_b).
"""
if features_a is None or features_b is None:
return Verdict(UNKNOWN, "no counterpart", {}), Verdict(UNKNOWN, "no counterpart", {})
if features_a.error or features_b.error:
# Frame-level errors are not this detector's job; defer rather than double-count.
return Verdict(UNKNOWN, "counterpart unreadable", {}), Verdict(
UNKNOWN, "counterpart unreadable", {}
)
a, b = features_a.thumbnail, features_b.thumbnail
if a is None or b is None or a.shape != b.shape:
return Verdict(UNKNOWN, "no comparable thumbnail", {}), Verdict(
UNKNOWN, "no comparable thumbnail", {}
)
scores = {}
shift = align_shift(a, b)
if shift is not None:
scores["align_shift"] = shift
matched, gain, offset = _gain_match(b, a)
scale = float(np.abs(a).mean() + np.abs(matched).mean()) / 2.0
difference = float(np.abs(a - matched).mean()) / scale if scale > 0 else np.inf
scores["cross_diff"] = difference
scores["cross_gain"] = gain
scores["cross_offset"] = offset
# A correction this large means the two frames are not on comparable scales at
# all, whatever the residual says once it has been applied.
gain_ok = gain_range[0] <= gain <= gain_range[1]
agree = difference <= max_diff and gain_ok and (shift is None or shift <= max_shift)
if agree:
return Verdict(GOOD, None, scores), Verdict(GOOD, None, scores)
reason = f"cross-satellite disagreement {difference:.2f}"
if not gain_ok:
reason = f"cross-satellite gain {gain:.3g} outside {gain_range}"
elif shift is not None and shift > max_shift:
reason = f"cross-satellite misalignment {shift:.1f}px"
a_suspect = temporal_a is not None and temporal_a.verdict == BAD
b_suspect = temporal_b is not None and temporal_b.verdict == BAD
if a_suspect and not b_suspect:
return Verdict(BAD, reason, scores), Verdict(GOOD, None, scores)
if b_suspect and not a_suspect:
return Verdict(GOOD, None, scores), Verdict(BAD, reason, scores)
return Verdict(UNKNOWN, reason + " (source unclear)", scores), Verdict(
UNKNOWN, reason + " (source unclear)", scores
)
#: Detectors the drivers and bench expose by name.
FRAME_DETECTORS = {"header_v1": header_v1, "geometry_v1": geometry_v1, "disc_v1": disc_v1}
SERIES_DETECTORS = {"temporal_v1": temporal_v1, "crosssat_v1": crosssat_v1}
ALL_DETECTORS = tuple(FRAME_DETECTORS) + tuple(SERIES_DETECTORS)
Reference in a new issue View git blame Copy permalink