"""Cheap FITS header reads and file-integrity checks. The archive's science keywords (``DEGRADED``, ``ECLIPSE``, ``IMG_MEAN``, ...) live in the tile-compressed image extension's header, in the first ~18 KB of a 1.7 MB file. Reading just those bytes and parsing the 80-column cards by hand is ~5x faster than letting astropy open the file (4.1 ms vs 22.2 ms per file measured on this archive), which is the difference between a two-hour and a ten-hour pass over the full set. astropy is still used for pixel data, where it earns its keep. Every function reports trouble with the *file* as a structured error string rather than raising, so a truncated or corrupt file is a *finding* the detector can score, not an exception that kills a worker. Trouble with the *file system* is the opposite: a share that refuses to open one more file (errno 23, while another job walked the whole share) says nothing about the frame, and once was stored as 1,754 "unreadable" frames. Those raise :class:`TransientReadError`, for the caller to retry or fail on. """ import errno import logging import os import warnings import numpy as np from astropy.io import fits from astropy.utils.exceptions import AstropyUserWarning from . import db def quiet_astropy(): """Silence astropy's per-file chatter about damaged files. astropy reports conditions like "File may have been truncated" through both its own logger and ``warnings``, either of which floods stderr when scanning millions of files -- and damaged files are exactly what a bench run is full of. Those conditions are already surfaced per-frame by ``read_header``, ``read_image`` and ``verify_datasums``, so nothing is lost by muting them. Call once per process; worker pools must call it in each child. """ logging.getLogger("astropy").setLevel(logging.ERROR) warnings.simplefilter("ignore", AstropyUserWarning) #: FITS records are a whole number of 2880-byte blocks. BLOCK = 2880 CARD = 80 #: How far into a file to look for the image HDU header. The real ones end by #: ~17 KB; the cap stops a malformed file from pulling megabytes into memory. MAX_HEADER_BYTES = BLOCK * 16 #: Sentinel some SUVI products use for undefined pixels. ZBLANK = -999999 def _parse_value(raw): """Parse a FITS card value, honouring quoted strings and comment separators.""" raw = raw.strip() if not raw: return None if raw[0] == "'": # String value: scan for the closing quote, where '' is a literal quote. out = [] i = 1 while i < len(raw): if raw[i] == "'": if i + 1 < len(raw) and raw[i + 1] == "'": out.append("'") i += 2 continue break out.append(raw[i]) i += 1 return "".join(out).strip() # Non-string: everything before the comment separator. value = raw.split("/", 1)[0].strip() if value in ("T", "F"): return value == "T" if not value: return None try: return int(value) except ValueError: pass try: # FITS permits Fortran-style 'D' exponents. return float(value.replace("D", "E").replace("d", "e")) except ValueError: return value def _parse_header_blocks(buf, pos): """Parse cards from `pos` until the END card. Returns (cards, next_pos), or (None, reason) if END is not reached. """ cards = {} while pos + BLOCK <= len(buf): block = buf[pos : pos + BLOCK] pos += BLOCK for i in range(0, BLOCK, CARD): card = block[i : i + CARD] key = card[:8].rstrip() if key == b"END": return cards, pos # Only fixed-format valued cards; COMMENT/HISTORY/CONTINUE lack the '= '. if card[8:10] == b"= ": try: name = key.decode("ascii").strip() value = _parse_value(card[10:].decode("ascii")) except UnicodeDecodeError: continue if name and name not in cards: cards[name] = value return None, "header has no END card within the scanned region" def _data_bytes(cards): """Size of an HDU's data unit, in bytes, rounded up to whole blocks.""" naxis = cards.get("NAXIS", 0) if not isinstance(naxis, int) or naxis <= 0: return 0 count = 1 for axis in range(1, naxis + 1): length = cards.get(f"NAXIS{axis}") if not isinstance(length, int) or length < 0: return 0 count *= length bitpix = cards.get("BITPIX", 8) if not isinstance(bitpix, int): return 0 size = count * abs(bitpix) // 8 size += cards.get("PCOUNT", 0) or 0 return ((size + BLOCK - 1) // BLOCK) * BLOCK def _is_image_hdu(cards): """Whether this HDU carries the image we care about. SUVI L2 stores the image tile-compressed inside a BINTABLE, flagged by ZIMAGE; its real dimensions are in ZNAXIS, not NAXIS. A plain IMAGE extension or a primary HDU with data also counts. """ if cards.get("ZIMAGE") is True: return True return isinstance(cards.get("NAXIS"), int) and cards["NAXIS"] > 0 #: errno values that mean the file system refused the read, not that the file is bad. TRANSIENT_ERRNOS = frozenset({errno.ENFILE, errno.EMFILE, errno.EAGAIN, errno.EINTR, errno.ETIMEDOUT, errno.ESTALE, errno.ECONNRESET}) class TransientReadError(OSError): """The file system refused a read; retry later. Never a verdict on the file.""" def _transient(exc): """`exc` as a TransientReadError if it is one, else None.""" while exc is not None: if isinstance(exc, OSError) and exc.errno in TRANSIENT_ERRNOS: return TransientReadError(exc.errno, exc.strerror, getattr(exc, "filename", None)) exc = exc.__cause__ or exc.__context__ return None def read_header(path, max_bytes=MAX_HEADER_BYTES): """Read the image HDU's header without decompressing pixels. Returns (cards, error). On failure cards is whatever was parsed (possibly empty) and error is a short human-readable reason. Raises TransientReadError when the file system refused the read. """ try: size = os.path.getsize(path) with open(path, "rb") as handle: buf = handle.read(max_bytes) except OSError as exc: transient = _transient(exc) if transient is not None: raise transient from exc return {}, f"unreadable: {exc.strerror or exc}" if size == 0: return {}, "empty file" if len(buf) < BLOCK: return {}, f"truncated: {size} bytes is less than one {BLOCK}-byte block" if not buf.startswith(b"SIMPLE ="): return {}, "not a FITS file: missing SIMPLE keyword" if size % BLOCK: # Worth reporting even though the header may still parse: it means the file # was cut short mid-transfer, which is exactly the download failure mode. truncation = f"truncated: {size} bytes is not a multiple of {BLOCK}" else: truncation = None pos = 0 while True: cards, result = _parse_header_blocks(buf, pos) if cards is None: return {}, result pos = result if _is_image_hdu(cards): return cards, truncation skip = _data_bytes(cards) if pos + skip + BLOCK > len(buf): return cards, truncation or "no image HDU within the scanned region" pos += skip def header_metadata(cards): """Project parsed cards onto the ``header`` table's columns. Booleans become 0/1 so they sort and filter naturally in SQL; anything of an unexpected type is dropped rather than stored, so a corrupt card cannot poison a numeric column. """ values = {} for field, keyword, sqltype in db.HEADER_COLUMNS: raw = cards.get(keyword) if raw is None: continue if isinstance(raw, bool): values[field] = int(raw) elif sqltype == "TEXT": values[field] = str(raw) elif sqltype == "INTEGER": values[field] = int(raw) if isinstance(raw, (int, float)) else None elif isinstance(raw, (int, float)): values[field] = float(raw) return {k: v for k, v in values.items() if v is not None} def scan_header(path): """Read and project a file's header in one step. Returns (values, error).""" cards, error = read_header(path) return header_metadata(cards), error def read_image(path): """Read the decompressed image array. Returns (data, error). Returns a float32 array with the ZBLANK sentinel converted to NaN, so callers do not have to remember that -999999 means "no data" rather than a radiance. Detects damage only insofar as astropy refuses to decode it; a file whose image HDU survives intact reads fine even if later bytes are missing. Pair this with ``verify_datasums`` when integrity matters rather than availability. Raises TransientReadError when the file system refused the read. """ try: with fits.open(path, memmap=False) as hdus: data = None for hdu in hdus: if getattr(hdu, "data", None) is not None and hdu.data.ndim == 2: data = hdu.data break if data is None: # The blank/corrupt-HDU case filter_FITS.py caught as IndexError. return None, "no 2-D image array in file" except Exception as exc: # astropy raises a wide variety on malformed input transient = _transient(exc) if transient is not None: raise transient from exc return None, f"{type(exc).__name__}: {exc}" data = np.asarray(data, dtype=np.float32) blank = data == ZBLANK if blank.any(): data = data.copy() data[blank] = np.nan return data, None def datasum(payload): """FITS-standard 32-bit ones-complement checksum of a data unit.""" if len(payload) % 4: payload = payload + b"\x00" * (4 - len(payload) % 4) total = int(np.frombuffer(payload, dtype=">u4").astype(np.uint64).sum()) while total >> 32: # fold the carries back in total = (total & 0xFFFFFFFF) + (total >> 32) return total def verify_datasums(path): """Check every HDU's stored DATASUM against the bytes on disk. Catches silent corruption that a size check misses. Computed over the raw file, deliberately not via astropy: astropy re-serialises a tile-compressed HDU before summing, so it reports a mismatch for every healthy SUVI file. Returns an error string, or None if all present DATASUMs agree. """ try: with open(path, "rb") as handle: buf = handle.read() except OSError as exc: return f"unreadable: {exc.strerror or exc}" if len(buf) < BLOCK or not buf.startswith(b"SIMPLE ="): return "not a FITS file" pos = 0 checked = 0 while pos < len(buf): cards, result = _parse_header_blocks(buf, pos) if cards is None: return result size = _data_bytes(cards) if result + size > len(buf): return "truncated: HDU data unit runs past end of file" stored = cards.get("DATASUM") if stored is not None: try: expected = int(str(stored).strip()) except ValueError: return f"malformed DATASUM keyword: {stored!r}" if datasum(buf[result : result + size]) != expected: name = cards.get("EXTNAME") or ("PRIMARY" if pos == 0 else "extension") return f"datasum mismatch in {name}" checked += 1 pos = result + size return None if checked else "no DATASUM keywords present"