noaa-goes-visualization/suvi/segments.py

"""Pristine contiguous segments: the unit the training corpus is built from.

A segment is a maximal run of slots in which *both* satellites are GOOD -- all
six wavelengths present and detector-clean -- taken from the archive profile
(:mod:`suvi.availability`) and then verified against the extracted shards.  The
model trains only on targets whose entire input span lies inside one segment, so
every frame it is fed is pre-verified pristine and every hole or damage it sees
is simulated, against complete ground truth.  Day boundaries mean nothing here;
the minimum length is the model's own input requirement, ``2*span + 1`` frames.

Segments are stored as ``[t0, t1)`` unix-time pairs in ``segments.json`` beside
the shard manifest, and the train/val/test split assigns whole segments, so the
two sides of a split can never share a contiguous stretch of Sun.
"""

import datetime
import json
import os

import numpy as np

from . import paths


def from_availability(model, minimum_slots):
    """Maximal jointly-pristine runs of at least `minimum_slots`, as (t0, t1)."""
    pristine = model.pristine()
    edges = np.diff(pristine.astype(np.int8))
    starts = list(np.flatnonzero(edges == 1) + 1)
    ends = list(np.flatnonzero(edges == -1) + 1)
    if pristine.size and pristine[0]:
        starts.insert(0, 0)
    if pristine.size and pristine[-1]:
        ends.append(pristine.size)
    # int(), not numpy's: these bounds are written to JSON and compared against
    # shard keys, and a stray int64 breaks json.dump at the very end of a
    # multi-hour build.
    return [
        (int(model.t0 + a * model.cadence), int(model.t0 + b * model.cadence))
        for a, b in zip(starts, ends)
        if (b - a) >= minimum_slots
    ]


def decorrelation_slots(cadence=paths.CADENCE):
    """Frames after which a segment is showing an essentially different Sun.

    Half a synodic rotation: by then the hemisphere that was facing us has gone
    round the back and the one that was hidden is in view, so two frames that far
    apart share no surface features.  This is the honest scale for "how much
    diversity does one contiguous segment hold" -- a segment shorter than this is
    one evolving view of one hemisphere, and truncating it below this length
    discards genuinely new Sun rather than redundancy.

    Equatorial synodic rotation is the fastest and so the conservative choice:
    26.1 days, giving ~4,700 frames at the 4-minute cadence.  Higher latitudes
    take half again as long.
    """
    from . import fillers

    period_days = 360.0 / fillers.rotation_rate(0.0, synodic=True)
    return int(period_days / 2 * 86400 / cadence)


def days_touched(segments, cadence=paths.CADENCE):
    """Every calendar day any segment's slots fall on -- the days to extract."""
    days = set()
    for t0, t1 in segments:
        for t in range(t0, t1, cadence):
            days.add(datetime.datetime.fromtimestamp(
                t, datetime.timezone.utc).strftime("%Y-%m-%d"))
    return sorted(days)


def verify(segments, usable, cadence=paths.CADENCE, minimum_slots=1):
    """Split segments wherever extraction disagrees with the profile.

    The profile is header-only; ``disc_v1`` runs at extraction and can reject a
    slot inside a nominally pristine run (measured ~0.3%).  `usable(t)` says
    whether every band on every satellite actually made it into the shards at
    `t`.  Any slot that did not splits its segment; fragments shorter than
    `minimum_slots` are dropped.  The corpus is only final after this pass.
    """
    verified = []
    for t0, t1 in segments:
        start = t0
        for t in range(t0, t1, cadence):
            if not usable(t):
                if (t - start) // cadence >= minimum_slots:
                    verified.append((start, t))
                start = t + cadence
        if (t1 - start) // cadence >= minimum_slots:
            verified.append((start, t1))
    return verified


def targets(segments, span_slots, cadence=paths.CADENCE):
    """Every slot time at least `span_slots` from both ends of its segment."""
    margin = span_slots * cadence
    out = []
    for t0, t1 in segments:
        out.extend(range(t0 + margin, t1 - margin, cadence))
    return out


def split(segments, seed, val_targets, test_targets, span_slots,
          cadence=paths.CADENCE):
    """Assign whole segments to val/test until each holds enough targets.

    Deterministic in `seed`.  Splitting by segment, not by day or slot, is the
    leakage rule: two targets in one contiguous run share most of their context,
    so a run must land entirely on one side.
    """
    order = list(segments)
    np.random.default_rng(seed).shuffle(order)

    def mass(segment):
        t0, t1 = segment
        return max((t1 - t0) // cadence - 2 * span_slots, 0)

    splits = {"val": [], "test": [], "train": []}
    need = {"val": val_targets, "test": test_targets}
    for segment in order:
        for name in ("val", "test"):
            if need[name] > 0:
                splits[name].append(tuple(segment))
                need[name] -= mass(segment)
                break
        else:
            splits["train"].append(tuple(segment))
    return {name: sorted(chosen) for name, chosen in splits.items()}


def containing(segments, t):
    """The segment holding time `t`, or None."""
    for t0, t1 in segments:
        if t0 <= t < t1:
            return (t0, t1)
    return None


def save(path, segments):
    payload = {"segments": [[int(a), int(b)] for a, b in sorted(segments)]}
    # Whole-file write, then rename: a half-written segments.json is worse than
    # none, because the next run would load it and train on a truncated corpus.
    with open(path + ".tmp", "w") as handle:
        json.dump(payload, handle)
    os.replace(path + ".tmp", path)


def load(path):
    with open(path) as handle:
        return [tuple(pair) for pair in json.load(handle)["segments"]]