noaa-goes-visualization/reclaim.py

#!/usr/bin/env python3
"""Keep the guest's inode cache below the ceiling that breaks the archive mount.

The archive is a virtiofs re-export of an SMB share: guest -> virtiofs -> virtiofsd on
the host -> cifs.ko -> SMB3 -> NAS.  The middle hop is the problem.  virtiofsd holds one
open host file descriptor per inode the guest has looked up, and the guest's dentry cache
is bounded only by memory pressure -- on a machine with free memory it simply grows.  When
virtiofsd runs out of descriptors the mount returns ENFILE ("too many open files in
system") for *every* subsequent open, and does not recover on its own.  That surfaces
across the whole machine, because anything on the share -- including the venv interpreter
the pipeline runs from -- stops being executable.

Run as a daemon this watches the count and drops caches only when it approaches the
ceiling.  That ordering is the whole point: reclaiming *early* is not free, because every
evicted dentry is an SMB round trip to re-establish, so the goal is to stay as warm as
possible for as long as possible and intervene just before the wall.

    reclaim.py                 # report, then reclaim if we are near the ceiling
    reclaim.py --check         # report only, change nothing (works unprivileged)
    sudo reclaim.py --force    # reclaim now, whatever the count says
    sudo reclaim.py --install  # install and enable the watchdog service

**This file deliberately imports nothing from `suvi`, and nothing outside the standard
library.**  It has to run in exactly the situation it exists for, and in that situation
the archive mount refuses every open -- which means the venv interpreter (which lives on
the mount), the `suvi` package, and even Python's scan of the working directory all
fail with ENFILE before any of this code runs.  Recovery went like this once:

    $ ./.venv/bin/python reclaim.py
    bash: ./.venv/bin/python: Too many open files in system
    $ /usr/bin/python3 -c ...
    OSError: [Errno 23] Too many open files in system: '.../scripts'

So: no package imports, and run it off the root filesystem with an isolated interpreter,
which keeps the mount out of `sys.path` entirely.  ``--install`` puts a copy where that
is true and writes the systemd unit that runs it.

On the metric.  An earlier version of this script decided from ``SReclaimable``, on the
reasoning that a cached dentry plus inode is on the order of a kilobyte.  That is true per
object but useless as a signal: slab pages are partially free, so objects accumulate
without the page count moving.  Measured on this machine, a walk of 40,000 archive files
added ~16,000 inodes and moved ``SReclaimable`` by 2 MiB.  The inode count itself is the
signal, and it is exact.

On the lever.  ``drop_caches`` is the only thing that reliably sheds dentries.  The
gentler option was tried and measured: graduated memory-cgroup pressure freed 15 MiB of
slab per 1 GiB of ``memory.reclaim`` requested, so shedding a useful number of inodes
would mean evicting gigabytes of page cache -- on a share this slow, far more expensive
than the problem.  So the design is a blunt instrument fired rarely, rather than a
gentle one running constantly, and that is why it needs root.
"""

import argparse
import errno
import json
import os
import shutil
import signal
import sys
import threading
import time

#: The number of inodes the guest may cache before the mount refuses opens.  Half of
#: virtiofsd's own default ``RLIMIT_NOFILE`` (1,000,000): on 2026-10-09 the mount
#: refused opens (ENFILE) while the watchdog held the count at 75% of a million, as
#: other jobs walked the whole share, so the real limit sits lower than the default
#: suggests.  Still a guess until proven -- see `Watchdog.probe`, which lowers it on
#: evidence.  If you can read the host, the true value is
#: `grep 'Max open files' /proc/$(pgrep virtiofsd)/limits`.
DEFAULT_CEILING = 500_000
#: Act at this fraction of the ceiling, and not a moment sooner.  Measured on this
#: archive, the cache fills at ~890 inodes/s under a serial `find` and ~4,000/s under a
#: six-way parallel stat, which is the shape of the pipeline's own bulk passes.  The
#: 250,000-inode gap the default leaves is therefore about a minute of the worst load
#: this machine can produce -- twelve poll intervals -- and below the mark ordinary work
#: keeps its cache entirely, which is the point.
HIGH_FRACTION = 0.75
#: Start probing for a wrongly-estimated ceiling at this fraction of `high`.
PROBE_FRACTION = 0.9
#: Seconds between polls.  Two small /proc reads; the cost is not measurable.
DEFAULT_INTERVAL = 5.0
#: Never drop twice inside this window, so a job that re-fills the cache immediately
#: cannot turn the watchdog into a thrash loop.
DEFAULT_COOLDOWN = 60.0
#: Seconds between "nothing happened" log lines.  These carry the peak seen since the
#: last one, which is what tells you whether `high` can be raised.
DEFAULT_HEARTBEAT = 3600.0
#: How long to wait for the probe open() before giving up on it.
PROBE_TIMEOUT = 5.0

CONFIG_PATH = "/etc/suvi-reclaimd.conf"
STATE_PATH = "/var/lib/suvi-reclaimd/state.json"
INSTALL_PATH = "/usr/local/sbin/suvi-reclaimd"
SERVICE_PATH = "/etc/systemd/system/suvi-reclaimd.service"
SERVICE_NAME = "suvi-reclaimd"

#: A small file on the share, opened to find out whether the mount is refusing opens.
#: Any file will do; it only has to be on the affected mount and cheap to read.
DEFAULT_PROBE_PATH = "/media/shared/NOAA_GOES/scripts/pytest.ini"

SERVICE_UNIT = """\
[Unit]
Description=Keep the guest dentry cache below virtiofsd's descriptor ceiling
Documentation=file://{source}
After=media-shared.mount

[Service]
ExecStart=/usr/bin/python3 -I {target} --daemon
Restart=always
RestartSec=5
Nice=5

[Install]
WantedBy=multi-user.target
"""

DEFAULT_CONFIG = f"""\
# Settings for the inode-cache watchdog (reclaim.py --daemon).  Delete a line to take
# its built-in default.  Restart with: systemctl restart {SERVICE_NAME}

# Descriptors virtiofsd can hold, and so inodes the guest may cache before the mount
# starts refusing opens.  {DEFAULT_CEILING} is half virtiofsd's own default
# RLIMIT_NOFILE, after the mount refused opens below 750,000; if you can read the
# host, the true figure is
#     grep 'Max open files' /proc/$(pgrep virtiofsd)/limits
# The watchdog lowers this on its own if it is ever refused an open, and never raises
# it -- so a value that turns out to be too high costs one event, not a standing risk.
ceiling = {DEFAULT_CEILING}

# Where to act.  Defaults to {HIGH_FRACTION:g} * ceiling.  Below it nothing happens at all, which
# is what keeps the cache warm; the point of dropping is to survive, not to stay tidy.
# Measured on this archive: the cache fills at ~890 inodes/s under a serial walk and
# ~4,000/s under a six-way parallel one, so the default leaves about a minute of the
# worst load this machine produces.  Raise it once the heartbeat's peak line shows what
# you actually reach -- more cache is faster, and every eviction is an SMB round trip
# to undo.
# high = {int(DEFAULT_CEILING * HIGH_FRACTION)}

# interval = {DEFAULT_INTERVAL:g}      # seconds between polls; two small /proc reads
# cooldown = {DEFAULT_COOLDOWN:g}     # seconds; stops a refilling job causing a thrash loop
# heartbeat = {DEFAULT_HEARTBEAT:g}  # seconds between "peak since last report" lines; 0 to silence

# A small file on the affected mount, opened to tell a full mount from a wedged one.
# probe_path = {DEFAULT_PROBE_PATH}
"""


def log(message):
    """One line to stdout, which under systemd is the journal."""
    print(message, flush=True)


# ------------------------------------------------------------------------- the metric


def fuse_inodes(path="/proc/slabinfo"):
    """Cached FUSE inodes from /proc/slabinfo, or None if it cannot be read.

    This is the exact number of inodes virtiofsd is holding a descriptor for, which is
    the quantity that runs out.  /proc/slabinfo is root-only, so unprivileged callers
    fall back to `total_inodes`.
    """
    try:
        with open(path) as handle:
            for line in handle:
                fields = line.split()
                # name, active_objs, num_objs, objsize, ...
                if len(fields) > 1 and fields[0] == "fuse_inode":
                    return int(fields[1])
    except (OSError, ValueError):
        pass
    return None


def total_inodes(path="/proc/sys/fs/inode-nr"):
    """Every inode the kernel has cached, across all filesystems.

    Coarser than `fuse_inodes` -- it counts the root filesystem and the squashfs snaps
    too -- but those are a stable baseline of a few tens of thousands against a ceiling
    of a million, so as a trigger it is more than good enough.  Unlike /proc/slabinfo it
    is world-readable, which is what lets `--check` work without sudo.
    """
    try:
        with open(path) as handle:
            return int(handle.read().split()[0])
    except (OSError, ValueError, IndexError):
        return 0


def cached_inodes():
    """The best available inode count, and where it came from."""
    exact = fuse_inodes()
    if exact is not None:
        return exact, "fuse_inode"
    return total_inodes(), "inode-nr"


# ------------------------------------------------------------------------- the lever


def drop_caches():
    """Ask the kernel to free dentries and inodes.  True if it worked.

    Needs root.  Only *unused* dentries go: anything with an open file or a working
    directory on it survives, so what this discards is the cold residue of traversal
    rather than the working set.
    """
    try:
        with open("/proc/sys/vm/drop_caches", "w") as handle:
            handle.write("2\n")  # 2 = dentries and inodes; page cache is not the issue
        return True
    except OSError:
        return False


#: Held for the duration of a probe.  A probe that never returns leaves it held, which
#: is how `probe` knows not to start a second one.
_probe_lock = threading.Lock()


def probe(path, timeout=PROBE_TIMEOUT):
    """Try to open `path`.  Returns the errno that stopped us, or 0 on success.

    Runs in a thread rather than inline because the failure this looks for can also
    present as a mount that stops answering altogether, and the watchdog must not be the
    thing that hangs.  A task blocked in the filesystem cannot reliably be interrupted,
    so the thread is simply abandoned -- and because at most one is ever outstanding, a
    wedged mount costs one thread rather than one per poll.  A probe still stuck from
    last time is itself an answer: the mount is not well.
    """
    # Captured, not looked up again in the thread: an abandoned probe can outlive any
    # rebinding of the global, and releasing whatever happens to be there by then would
    # unlock a lock this call never took.
    lock = _probe_lock
    if not lock.acquire(blocking=False):
        return errno.ENFILE
    result = {}

    def attempt():
        try:
            descriptor = os.open(path, os.O_RDONLY)
            os.close(descriptor)
            result["errno"] = 0
        except OSError as error:
            result["errno"] = error.errno or 0
        except BaseException:
            result["errno"] = 0
        finally:
            lock.release()

    thread = threading.Thread(target=attempt, daemon=True)
    thread.start()
    thread.join(timeout)
    # A probe that has not answered within the timeout is treated as exhaustion.  If it
    # answers a moment later it releases the lock, so this corrects itself next poll.
    return result.get("errno", errno.ENFILE)


# ------------------------------------------------------------------- config and state


def _coerce(value, kind, minimum):
    """Parse one config value, or raise ValueError.  Bounds keep a typo from disarming
    the watchdog -- an interval of 0 would spin, a ceiling of 0 would drop constantly."""
    parsed = kind(value)
    if parsed < minimum:
        raise ValueError(f"{parsed} is below the minimum {minimum}")
    return parsed


#: name -> (type, minimum).  Anything outside this is rejected rather than ignored, so a
#: misspelt key is a loud failure rather than a silently unapplied setting.
CONFIG_FIELDS = {
    "ceiling": (int, 1000),
    "high": (int, 1000),
    "interval": (float, 0.1),
    "cooldown": (float, 0.0),
    "heartbeat": (float, 0.0),
    "probe_path": (str, None),
}


def load_config(path=CONFIG_PATH):
    """Read `key=value` lines.  A missing file means defaults; a bad one is fatal."""
    config = {}
    try:
        with open(path) as handle:
            lines = handle.readlines()
    except FileNotFoundError:
        return config
    except OSError as error:
        raise SystemExit(f"cannot read {path}: {error}")

    for number, line in enumerate(lines, 1):
        line = line.split("#", 1)[0].strip()
        if not line:
            continue
        if "=" not in line:
            raise SystemExit(f"{path}:{number}: expected key=value")
        key, _, value = line.partition("=")
        key, value = key.strip(), value.strip()
        if key not in CONFIG_FIELDS:
            raise SystemExit(f"{path}:{number}: unknown setting {key!r}")
        kind, minimum = CONFIG_FIELDS[key]
        if kind is str:
            config[key] = value
            continue
        try:
            config[key] = _coerce(value, kind, minimum)
        except ValueError as error:
            raise SystemExit(f"{path}:{number}: bad value for {key}: {error}")
    return config


def load_state(path=STATE_PATH):
    """What we have learned about the ceiling.  Corruption is not worth dying over."""
    try:
        with open(path) as handle:
            state = json.load(handle)
        if not isinstance(state, dict):
            return {}
        return state
    except (OSError, ValueError):
        return {}


def save_state(state, path=STATE_PATH):
    """Persist learned state, best effort.  Written via a temporary file so a crash
    mid-write cannot leave a truncated file that reads back as no knowledge at all."""
    try:
        os.makedirs(os.path.dirname(path), exist_ok=True)
        temporary = path + ".tmp"
        with open(temporary, "w") as handle:
            json.dump(state, handle)
        os.replace(temporary, path)
        return True
    except OSError:
        return False


# ------------------------------------------------------------------------ the watchdog


class Watchdog:
    """Polls the inode count and drops caches when it nears the ceiling.

    Everything it does is driven by one number, and it does nothing at all while that
    number is comfortable.  That is deliberate: the cache is worth keeping, and on a
    share where a lookup is an SMB round trip, reclaiming early costs real time.
    """

    def __init__(self, config=None, state_path=STATE_PATH):
        config = dict(config or {})
        self.state_path = state_path
        state = load_state(state_path)

        self.ceiling = config.get("ceiling", DEFAULT_CEILING)
        # A ceiling learned the hard way beats a configured guess, but only downwards:
        # see `observe_exhaustion`.
        learned = state.get("ceiling")
        if isinstance(learned, int) and 0 < learned < self.ceiling:
            self.ceiling = learned

        #: An explicit `high` is honoured as-is; a derived one follows the ceiling down
        #: when the ceiling is corrected, which is the behaviour you want from a default.
        self.explicit_high = config.get("high")
        self.interval = config.get("interval", DEFAULT_INTERVAL)
        self.cooldown = config.get("cooldown", DEFAULT_COOLDOWN)
        self.heartbeat = config.get("heartbeat", DEFAULT_HEARTBEAT)
        self.probe_path = config.get("probe_path", DEFAULT_PROBE_PATH)

        self.last_drop = None
        self.peak = 0
        self.drops = 0
        self.running = True

    @property
    def high(self):
        if self.explicit_high is not None:
            return self.explicit_high
        return int(self.ceiling * HIGH_FRACTION)

    def observe_exhaustion(self, count):
        """Record that the mount refused an open at `count` inodes.

        The correction only ever lowers the ceiling.  A watchdog that could raise its own
        limit would creep towards the wall looking for it, and finding it is exactly the
        outcome this exists to prevent.
        """
        if count <= 0 or count >= self.ceiling:
            return False
        self.ceiling = count
        state = load_state(self.state_path)
        state["ceiling"] = count
        state["observed_at"] = time.time()
        save_state(state, self.state_path)
        log(f"mount refused an open at {count:,} inodes: "
            f"ceiling lowered to {count:,}, high water now {self.high:,}")
        return True

    def reclaim(self, count, why):
        """Drop caches and report what it bought."""
        if not drop_caches():
            # The only way this fails is lack of privilege, which will not resolve on
            # its own.  Take the cooldown anyway so a misconfigured service warns once a
            # minute instead of once a poll.
            self.last_drop = time.monotonic()
            log(f"WARNING: need root to drop caches, and {count:,} inodes is {why}")
            return False
        after, _ = cached_inodes()
        self.last_drop = time.monotonic()
        self.drops += 1
        log(f"reclaimed ({why}): {count:,} -> {after:,} inodes, "
            f"high water {self.high:,} of {self.ceiling:,} ceiling")
        return True

    def step(self):
        """One poll.  True if it reclaimed."""
        count, _ = cached_inodes()
        self.peak = max(self.peak, count)

        if count < self.high * PROBE_FRACTION:
            return False

        # Close enough to matter: find out whether the ceiling estimate is still good
        # before relying on it.  This costs one open() and only happens up here.
        failure = probe(self.probe_path)
        if failure in (errno.ENFILE, errno.EMFILE):
            self.observe_exhaustion(count)
            return self.reclaim(count, "mount is refusing opens")

        if count < self.high:
            return False
        if self.last_drop is not None and \
                time.monotonic() - self.last_drop < self.cooldown:
            return False
        return self.reclaim(count, f"at or above the {self.high:,} high water mark")

    def stop(self, *_):
        self.running = False

    def run(self):
        """Poll until told to stop."""
        signal.signal(signal.SIGTERM, self.stop)
        signal.signal(signal.SIGINT, self.stop)
        _, source = cached_inodes()
        log(f"watching {source}: high water {self.high:,} of {self.ceiling:,} ceiling, "
            f"every {self.interval:g}s")

        next_beat = time.monotonic() + self.heartbeat
        while self.running:
            self.step()
            now = time.monotonic()
            if self.heartbeat and now >= next_beat:
                log(f"peak {self.peak:,} inodes since last report "
                    f"({self.peak * 100 // max(self.ceiling, 1)}% of ceiling), "
                    f"{self.drops} reclaim(s) total")
                self.peak, next_beat = 0, now + self.heartbeat
            time.sleep(self.interval)
        log("stopping")
        return 0


# --------------------------------------------------------------------------- one-shots


def report(watchdog):
    """Print where we stand, and return the count."""
    count, source = cached_inodes()
    share = count * 100.0 / max(watchdog.ceiling, 1)
    log(f"{count:,} cached inodes ({source}), {share:.1f}% of the "
        f"{watchdog.ceiling:,} ceiling; high water {watchdog.high:,}")
    return count


def install(target=INSTALL_PATH, service=SERVICE_PATH, config=CONFIG_PATH):
    """Copy this script off the share and write the unit and config that run it."""
    source = os.path.abspath(__file__)
    target = os.path.abspath(os.path.expanduser(target))
    if os.path.realpath(target) == os.path.realpath(source):
        raise SystemExit("refusing to install over the original")

    os.makedirs(os.path.dirname(target), exist_ok=True)
    shutil.copyfile(source, target)
    os.chmod(target, 0o755)
    log(f"installed {target}")

    with open(service, "w") as handle:
        handle.write(SERVICE_UNIT.format(source=source, target=target))
    log(f"wrote {service}")

    # Never overwrite a tuned config -- the whole point of the heartbeat's peak line is
    # that the operator raises `high` once they know what this machine actually reaches.
    if config and not os.path.exists(config):
        with open(config, "w") as handle:
            handle.write(DEFAULT_CONFIG)
        log(f"wrote {config}")
    elif config:
        log(f"kept existing {config}")
    log(f"  enable it with:  sudo systemctl enable --now {SERVICE_NAME}")
    log(f"  watch it with:   journalctl -fu {SERVICE_NAME}")
    log(f"  by hand:         /usr/bin/python3 -I {target}")
    return target


def main(argv=None):
    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--daemon", action="store_true",
                        help="poll and reclaim near the ceiling; this is the service")
    parser.add_argument("--check", action="store_true",
                        help="report only, reclaim nothing; nonzero if above high water")
    parser.add_argument("--force", action="store_true",
                        help="reclaim even if the count looks fine")
    parser.add_argument("--install", action="store_true",
                        help=f"copy to {INSTALL_PATH} and write the systemd unit")
    parser.add_argument("--config", default=CONFIG_PATH, help="settings file")
    args = parser.parse_args(argv)

    if args.install:
        install()
        return 0

    watchdog = Watchdog(load_config(args.config))

    if args.daemon:
        return watchdog.run()

    count = report(watchdog)
    if args.check:
        return 0 if count < watchdog.high else 1

    if count < watchdog.high and not args.force:
        log(f"  below the {watchdog.high:,} high water mark; nothing to do.  "
            f"--force reclaims anyway.")
        return 0
    if os.geteuid() != 0:
        log("  reclaiming needs root: re-run under sudo, or install the watchdog "
            f"with `sudo {os.path.basename(__file__)} --install`")
        return 1
    watchdog.reclaim(count, "asked to" if args.force else "above the high water mark")
    return 0


if __name__ == "__main__":
    sys.exit(main())