254 lines
10 KiB
Python
254 lines
10 KiB
Python
|
"""Keeping the frame index in step with the archive, without walking it.
|
||
|
|
|
||
|
|
The index exists so that nothing has to traverse 2.65M files to answer "what frames
|
||
|
|
do we have". Traversal is not merely slow here: the archive sits on a virtiofs mount
|
||
|
|
where the daemon holds a host file descriptor per inode the guest has looked up, so a
|
||
|
|
full walk exhausts the host's descriptors and the mount then refuses *every* open
|
||
|
|
until the guest drops its dentry cache. One full pass cost 152,193 renames to ENFILE
|
||
|
|
before this was understood.
|
||
|
|
|
||
|
|
So the index is maintained three ways, in descending order of preference:
|
||
|
|
|
||
|
|
1. **At the source.** ``puller_fits.py`` records each frame as it downloads it, so
|
||
|
|
new data enters the index with no filesystem traversal at all.
|
||
|
|
2. **By directory mtime** (:func:`reconcile`). A directory's mtime changes whenever
|
||
|
|
an entry is added or removed, so comparing it against ``dir_scan`` is an exact
|
||
|
|
test for "did anything change in here". The archive averages ~360 frames per
|
||
|
|
day-directory, so this checks ~7,400 stats instead of 2.65M lookups -- a 360x
|
||
|
|
reduction, and the difference between an operation this mount sustains and one
|
||
|
|
that breaks it.
|
||
|
|
3. **A full rebuild**, only for a cold start or when the index is suspect.
|
||
|
|
|
||
|
|
Only step 3 is expensive, and after the first one it should never be needed again.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import os
|
||
|
|
import time
|
||
|
|
|
||
|
|
from . import db, paths, vfs
|
||
|
|
|
||
|
|
|
||
|
|
def day_directories(root, satellites=paths.SATELLITES, wavelengths=paths.WAVELENGTHS,
|
||
|
|
years=None):
|
||
|
|
"""Yield (relpath, abspath) for every day-directory that exists.
|
||
|
|
|
||
|
|
Enumerated by descending the year/month/day structure rather than walking, so the
|
||
|
|
cost is one listing per year and month directory -- a couple of thousand small
|
||
|
|
reads -- and no per-file lookups at all.
|
||
|
|
"""
|
||
|
|
for satellite in satellites:
|
||
|
|
for wavelength in wavelengths:
|
||
|
|
band_rel = f"goes{satellite:d}/l2/data/suvi-l2-ci{wavelength:03d}"
|
||
|
|
band_abs = os.path.join(root, *band_rel.split("/"))
|
||
|
|
for year in _subdirs(band_abs):
|
||
|
|
if not year.isdigit() or (years and int(year) not in years):
|
||
|
|
continue
|
||
|
|
year_abs = os.path.join(band_abs, year)
|
||
|
|
for month in _subdirs(year_abs):
|
||
|
|
month_abs = os.path.join(year_abs, month)
|
||
|
|
for day in _subdirs(month_abs):
|
||
|
|
yield (
|
||
|
|
f"{band_rel}/{year}/{month}/{day}",
|
||
|
|
os.path.join(month_abs, day),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _subdirs(path):
|
||
|
|
"""Immediate subdirectory names.
|
||
|
|
|
||
|
|
A directory that does not exist yields nothing; one that exists but cannot be
|
||
|
|
read raises. The distinction is essential: swallowing the error makes an
|
||
|
|
unreadable tree look like an empty one, and :func:`reconcile` would then conclude
|
||
|
|
its contents had been deleted. That is not hypothetical -- an ENFILE partway
|
||
|
|
through enumeration made five of six bands look absent, and 205,618 index rows
|
||
|
|
were dropped as "vanished" before this was caught.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
with os.scandir(path) as entries:
|
||
|
|
return sorted(entry.name for entry in entries if entry.is_dir())
|
||
|
|
except (FileNotFoundError, NotADirectoryError):
|
||
|
|
return []
|
||
|
|
|
||
|
|
|
||
|
|
def scan_directory(conn, dir_relpath, dir_abspath):
|
||
|
|
"""Bring one directory's frames into the index.
|
||
|
|
|
||
|
|
Returns (added, removed, duplicates, examined). `examined` is the number of
|
||
|
|
frame files looked at, which is what drives how close the mount is to running
|
||
|
|
out of file handles -- see suvi.vfs.
|
||
|
|
|
||
|
|
Reconciles in both directions: files that appeared are inserted, and index rows
|
||
|
|
whose files have gone are deleted, so the index stays authoritative rather than
|
||
|
|
merely append-only.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
with os.scandir(dir_abspath) as entries:
|
||
|
|
found = {}
|
||
|
|
for entry in entries:
|
||
|
|
name = paths.parse_frame_filename(entry.name)
|
||
|
|
if name is None:
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
stat = entry.stat()
|
||
|
|
found[entry.name] = (name, stat.st_size, stat.st_mtime)
|
||
|
|
except OSError:
|
||
|
|
found[entry.name] = (name, None, None)
|
||
|
|
except OSError:
|
||
|
|
return 0, 0, [], 0
|
||
|
|
|
||
|
|
examined = len(found)
|
||
|
|
found, duplicates = _resolve_slot_collisions(found)
|
||
|
|
|
||
|
|
known = db.frames_in_dir(conn, dir_relpath)
|
||
|
|
|
||
|
|
# Remove departed rows *before* inserting, so a file renamed within a directory
|
||
|
|
# -- which is exactly what the un-rename migration does -- does not momentarily
|
||
|
|
# have two rows claiming one observation slot and trip the unique constraint.
|
||
|
|
gone = [frame_id for filename, frame_id in known.items() if filename not in found]
|
||
|
|
if gone:
|
||
|
|
db.delete_frames(conn, gone)
|
||
|
|
|
||
|
|
added = 0
|
||
|
|
for filename, (name, size, mtime) in found.items():
|
||
|
|
db.upsert_frame(conn, name, f"{dir_relpath}/{filename}", size, mtime)
|
||
|
|
if filename not in known:
|
||
|
|
added += 1
|
||
|
|
return added, len(gone), duplicates, examined
|
||
|
|
|
||
|
|
|
||
|
|
def _resolve_slot_collisions(found):
|
||
|
|
"""Keep one file per observation slot; return the rest as duplicates.
|
||
|
|
|
||
|
|
Two files can claim the same (satellite, band, time) when an older filter
|
||
|
|
labelled a frame repeatedly -- the archive holds 1,742 such pairs, a clean
|
||
|
|
``...v1-0-1.fits`` beside a ``...v1-0-1_f_f_f.fits`` of byte-identical content,
|
||
|
|
left when the puller re-downloaded a frame it could no longer find under its
|
||
|
|
published name. The index cannot hold both, and crashing on them would block
|
||
|
|
indexing the entire archive over a handful of stale copies. Prefer the file
|
||
|
|
NOAA actually published: unlabelled first, shortest name to break ties.
|
||
|
|
"""
|
||
|
|
by_slot = {}
|
||
|
|
for filename, (name, _, _) in found.items():
|
||
|
|
by_slot.setdefault(name.slot, []).append(filename)
|
||
|
|
|
||
|
|
duplicates = []
|
||
|
|
for names in by_slot.values():
|
||
|
|
if len(names) < 2:
|
||
|
|
continue
|
||
|
|
canonical = min(
|
||
|
|
names, key=lambda n: (found[n][0].label is not None, len(n), n)
|
||
|
|
)
|
||
|
|
for other in names:
|
||
|
|
if other != canonical:
|
||
|
|
duplicates.append(other)
|
||
|
|
found.pop(other, None)
|
||
|
|
return found, duplicates
|
||
|
|
|
||
|
|
|
||
|
|
def reconcile(conn, root=None, satellites=paths.SATELLITES, wavelengths=paths.WAVELENGTHS,
|
||
|
|
years=None, force=False, progress=None, relief=True):
|
||
|
|
"""Update the index from the archive, reading only what changed.
|
||
|
|
|
||
|
|
Returns a summary dict. With `force`, every directory is re-read regardless of
|
||
|
|
its recorded mtime -- the escape hatch for when the index is suspected wrong.
|
||
|
|
|
||
|
|
A steady-state run touches almost nothing and needs no special care. A cold
|
||
|
|
build is different: it reads every file in the archive, which is exactly the
|
||
|
|
traversal that exhausts this mount's file handles, so it hands them back
|
||
|
|
periodically (see suvi.vfs). Pass ``relief=False`` to disable that.
|
||
|
|
"""
|
||
|
|
root = root or paths.data_root()
|
||
|
|
recorded = {} if force else db.dir_mtimes(conn)
|
||
|
|
|
||
|
|
checked = changed = added = removed = scanned = 0
|
||
|
|
next_relief = vfs.RELIEF_INTERVAL
|
||
|
|
duplicates = []
|
||
|
|
seen = set()
|
||
|
|
pending = []
|
||
|
|
started = time.time()
|
||
|
|
|
||
|
|
for dir_relpath, dir_abspath in day_directories(root, satellites, wavelengths, years):
|
||
|
|
seen.add(dir_relpath)
|
||
|
|
checked += 1
|
||
|
|
try:
|
||
|
|
mtime = os.stat(dir_abspath).st_mtime
|
||
|
|
except OSError:
|
||
|
|
continue
|
||
|
|
if not force and recorded.get(dir_relpath) == mtime:
|
||
|
|
continue # nothing added or removed since we last looked
|
||
|
|
|
||
|
|
changed += 1
|
||
|
|
new, lost, dupes, examined = scan_directory(conn, dir_relpath, dir_abspath)
|
||
|
|
added += new
|
||
|
|
removed += lost
|
||
|
|
duplicates.extend(f"{dir_relpath}/{name}" for name in dupes)
|
||
|
|
# Record the mtime we actually observed, not one read afterwards, so a write
|
||
|
|
# racing this scan leaves the directory looking stale and gets picked up next
|
||
|
|
# time rather than being silently skipped forever.
|
||
|
|
pending.append((dir_relpath, mtime, len(db.frames_in_dir(conn, dir_relpath))))
|
||
|
|
if len(pending) >= 200:
|
||
|
|
db.record_dir_scans(conn, pending)
|
||
|
|
conn.commit()
|
||
|
|
pending.clear()
|
||
|
|
scanned += examined
|
||
|
|
if relief and scanned >= next_relief:
|
||
|
|
vfs.release_handles()
|
||
|
|
next_relief = scanned + vfs.RELIEF_INTERVAL
|
||
|
|
if progress and changed % 100 == 0:
|
||
|
|
progress(f" {changed} changed of {checked} checked, +{added}/-{removed}")
|
||
|
|
|
||
|
|
if pending:
|
||
|
|
db.record_dir_scans(conn, pending)
|
||
|
|
|
||
|
|
# Directories that have vanished entirely.
|
||
|
|
#
|
||
|
|
# Absence from `seen` is not sufficient evidence to delete anything: enumeration
|
||
|
|
# can come up short for reasons that have nothing to do with the data, and the
|
||
|
|
# cost of being wrong is destroying index rows for files that are still present.
|
||
|
|
# Confirm each one is genuinely gone before acting on it.
|
||
|
|
vanished = []
|
||
|
|
for dir_relpath in recorded:
|
||
|
|
if dir_relpath in seen:
|
||
|
|
continue
|
||
|
|
if os.path.isdir(paths.abspath(dir_relpath, root)):
|
||
|
|
continue # still there; enumeration simply missed it
|
||
|
|
vanished.append(dir_relpath)
|
||
|
|
|
||
|
|
for dir_relpath in vanished:
|
||
|
|
stale = db.frames_in_dir(conn, dir_relpath)
|
||
|
|
if stale:
|
||
|
|
db.delete_frames(conn, list(stale.values()))
|
||
|
|
removed += len(stale)
|
||
|
|
if vanished:
|
||
|
|
db.forget_dir_scans(conn, vanished)
|
||
|
|
conn.commit()
|
||
|
|
|
||
|
|
return {
|
||
|
|
"directories_checked": checked,
|
||
|
|
"directories_changed": changed,
|
||
|
|
"directories_vanished": len(vanished),
|
||
|
|
"frames_added": added,
|
||
|
|
"frames_removed": removed,
|
||
|
|
"duplicate_slots": duplicates,
|
||
|
|
"seconds": time.time() - started,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def record_downloaded(conn, local_path, root=None):
|
||
|
|
"""Index a frame the puller has just written.
|
||
|
|
|
||
|
|
The cheapest path of all: the downloader already knows the file exists, so the
|
||
|
|
index can learn about it without anybody looking at the filesystem. Returns the
|
||
|
|
frame id, or None if the file is not a SUVI frame.
|
||
|
|
"""
|
||
|
|
root = root or paths.data_root()
|
||
|
|
name = paths.parse_frame_filename(os.path.basename(local_path))
|
||
|
|
if name is None:
|
||
|
|
return None
|
||
|
|
try:
|
||
|
|
stat = os.stat(local_path)
|
||
|
|
size, mtime = stat.st_size, stat.st_mtime
|
||
|
|
except OSError:
|
||
|
|
size = mtime = None
|
||
|
|
return db.upsert_frame(conn, name, name.relpath(), size, mtime)
|