"""Keeping the frame index in step with the archive, without walking it. The index exists so that nothing has to traverse 2.65M files to answer "what frames do we have". Traversal is not merely slow here: the archive sits on a virtiofs mount where the daemon holds a host file descriptor per inode the guest has looked up, so a full walk exhausts the host's descriptors and the mount then refuses *every* open until the guest drops its dentry cache. One full pass cost 152,193 renames to ENFILE before this was understood. So the index is maintained three ways, in descending order of preference: 1. **At the source.** ``puller_fits.py`` records each frame as it downloads it, so new data enters the index with no filesystem traversal at all. 2. **By directory mtime** (:func:`reconcile`). A directory's mtime changes whenever an entry is added or removed, so comparing it against ``dir_scan`` is an exact test for "did anything change in here". The archive averages ~360 frames per day-directory, so this checks ~7,400 stats instead of 2.65M lookups -- a 360x reduction, and the difference between an operation this mount sustains and one that breaks it. 3. **A full rebuild**, only for a cold start or when the index is suspect. Only step 3 is expensive, and after the first one it should never be needed again. """ import os import time from . import db, paths, vfs def day_directories(root, satellites=paths.SATELLITES, wavelengths=paths.WAVELENGTHS, years=None): """Yield (relpath, abspath) for every day-directory that exists. Enumerated by descending the year/month/day structure rather than walking, so the cost is one listing per year and month directory -- a couple of thousand small reads -- and no per-file lookups at all. """ for satellite in satellites: for wavelength in wavelengths: band_rel = f"goes{satellite:d}/l2/data/suvi-l2-ci{wavelength:03d}" band_abs = os.path.join(root, *band_rel.split("/")) for year in _subdirs(band_abs): if not year.isdigit() or (years and int(year) not in years): continue year_abs = os.path.join(band_abs, year) for month in _subdirs(year_abs): month_abs = os.path.join(year_abs, month) for day in _subdirs(month_abs): yield ( f"{band_rel}/{year}/{month}/{day}", os.path.join(month_abs, day), ) def _subdirs(path): """Immediate subdirectory names. A directory that does not exist yields nothing; one that exists but cannot be read raises. The distinction is essential: swallowing the error makes an unreadable tree look like an empty one, and :func:`reconcile` would then conclude its contents had been deleted. That is not hypothetical -- an ENFILE partway through enumeration made five of six bands look absent, and 205,618 index rows were dropped as "vanished" before this was caught. """ try: with os.scandir(path) as entries: return sorted(entry.name for entry in entries if entry.is_dir()) except (FileNotFoundError, NotADirectoryError): return [] def scan_directory(conn, dir_relpath, dir_abspath): """Bring one directory's frames into the index. Returns (added, removed, duplicates, examined). `examined` is the number of frame files looked at, which is what drives how close the mount is to running out of file handles -- see suvi.vfs. Reconciles in both directions: files that appeared are inserted, and index rows whose files have gone are deleted, so the index stays authoritative rather than merely append-only. """ try: with os.scandir(dir_abspath) as entries: found = {} for entry in entries: name = paths.parse_frame_filename(entry.name) if name is None: continue try: stat = entry.stat() found[entry.name] = (name, stat.st_size, stat.st_mtime) except OSError: found[entry.name] = (name, None, None) except OSError: return 0, 0, [], 0 examined = len(found) found, duplicates = _resolve_slot_collisions(found) known = db.frames_in_dir(conn, dir_relpath) # Remove departed rows *before* inserting, so a file renamed within a directory # -- which is exactly what the un-rename migration does -- does not momentarily # have two rows claiming one observation slot and trip the unique constraint. gone = [frame_id for filename, frame_id in known.items() if filename not in found] if gone: db.delete_frames(conn, gone) added = 0 for filename, (name, size, mtime) in found.items(): db.upsert_frame(conn, name, f"{dir_relpath}/{filename}", size, mtime) if filename not in known: added += 1 return added, len(gone), duplicates, examined def _resolve_slot_collisions(found): """Keep one file per observation slot; return the rest as duplicates. Two files can claim the same (satellite, band, time) when an older filter labelled a frame repeatedly -- the archive holds 1,742 such pairs, a clean ``...v1-0-1.fits`` beside a ``...v1-0-1_f_f_f.fits`` of byte-identical content, left when the puller re-downloaded a frame it could no longer find under its published name. The index cannot hold both, and crashing on them would block indexing the entire archive over a handful of stale copies. Prefer the file NOAA actually published: unlabelled first, shortest name to break ties. """ by_slot = {} for filename, (name, _, _) in found.items(): by_slot.setdefault(name.slot, []).append(filename) duplicates = [] for names in by_slot.values(): if len(names) < 2: continue canonical = min( names, key=lambda n: (found[n][0].label is not None, len(n), n) ) for other in names: if other != canonical: duplicates.append(other) found.pop(other, None) return found, duplicates def reconcile(conn, root=None, satellites=paths.SATELLITES, wavelengths=paths.WAVELENGTHS, years=None, force=False, progress=None, relief=True): """Update the index from the archive, reading only what changed. Returns a summary dict. With `force`, every directory is re-read regardless of its recorded mtime -- the escape hatch for when the index is suspected wrong. A steady-state run touches almost nothing and needs no special care. A cold build is different: it reads every file in the archive, which is exactly the traversal that exhausts this mount's file handles, so it hands them back periodically (see suvi.vfs). Pass ``relief=False`` to disable that. """ root = root or paths.data_root() recorded = {} if force else db.dir_mtimes(conn) checked = changed = added = removed = scanned = 0 next_relief = vfs.RELIEF_INTERVAL duplicates = [] seen = set() pending = [] started = time.time() for dir_relpath, dir_abspath in day_directories(root, satellites, wavelengths, years): seen.add(dir_relpath) checked += 1 try: mtime = os.stat(dir_abspath).st_mtime except OSError: continue if not force and recorded.get(dir_relpath) == mtime: continue # nothing added or removed since we last looked changed += 1 new, lost, dupes, examined = scan_directory(conn, dir_relpath, dir_abspath) added += new removed += lost duplicates.extend(f"{dir_relpath}/{name}" for name in dupes) # Record the mtime we actually observed, not one read afterwards, so a write # racing this scan leaves the directory looking stale and gets picked up next # time rather than being silently skipped forever. pending.append((dir_relpath, mtime, len(db.frames_in_dir(conn, dir_relpath)))) if len(pending) >= 200: db.record_dir_scans(conn, pending) conn.commit() pending.clear() scanned += examined if relief and scanned >= next_relief: vfs.release_handles() next_relief = scanned + vfs.RELIEF_INTERVAL if progress and changed % 100 == 0: progress(f" {changed} changed of {checked} checked, +{added}/-{removed}") if pending: db.record_dir_scans(conn, pending) # Directories that have vanished entirely. # # Absence from `seen` is not sufficient evidence to delete anything: enumeration # can come up short for reasons that have nothing to do with the data, and the # cost of being wrong is destroying index rows for files that are still present. # Confirm each one is genuinely gone before acting on it. vanished = [] for dir_relpath in recorded: if dir_relpath in seen: continue if os.path.isdir(paths.abspath(dir_relpath, root)): continue # still there; enumeration simply missed it vanished.append(dir_relpath) for dir_relpath in vanished: stale = db.frames_in_dir(conn, dir_relpath) if stale: db.delete_frames(conn, list(stale.values())) removed += len(stale) if vanished: db.forget_dir_scans(conn, vanished) conn.commit() return { "directories_checked": checked, "directories_changed": changed, "directories_vanished": len(vanished), "frames_added": added, "frames_removed": removed, "duplicate_slots": duplicates, "seconds": time.time() - started, } def record_downloaded(conn, local_path, root=None): """Index a frame the puller has just written. The cheapest path of all: the downloader already knows the file exists, so the index can learn about it without anybody looking at the filesystem. Returns the frame id, or None if the file is not a SUVI frame. """ root = root or paths.data_root() name = paths.parse_frame_filename(os.path.basename(local_path)) if name is None: return None try: stat = os.stat(local_path) size, mtime = stat.st_size, stat.st_mtime except OSError: size = mtime = None return db.upsert_frame(conn, name, name.relpath(), size, mtime)