noaa-goes-visualization/reclaim.py

#!/usr/bin/env python3
"""Hand the archive's file handles back to the host.

The archive is on a virtiofs share whose daemon holds a host file descriptor per inode
the guest has looked up.  Anything that reads a lot of the archive leaves those inodes
cached, and the handles with them, until something forces the guest to evict them -- at
which point the daemon can run out and the mount starts refusing *every* open, which
surfaces across the whole machine as "too many open files in system".

The pipeline's own bulk jobs reclaim as they go and again when they finish, so this
should rarely be needed.  It exists for when something else has loaded the cache:

    reclaim.py            # report, then reclaim if needed
    reclaim.py --check    # report only, change nothing
    sudo reclaim.py       # uses drop_caches: instant, and far gentler

**This file deliberately imports nothing from `suvi`, and nothing outside the standard
library.**  It has to run in exactly the situation it exists for, and in that situation
the archive mount refuses every open -- which means the venv interpreter (which lives on
the mount), the `suvi` package, and even Python's scan of the working directory all
fail with ENFILE before any of this code runs.  Recovery went like this once:

    $ ./.venv/bin/python reclaim.py
    bash: ./.venv/bin/python: Too many open files in system
    $ /usr/bin/python3 -c ...
    OSError: [Errno 23] Too many open files in system: '.../scripts'

So: no package imports, and run it off the root filesystem with an isolated interpreter,
which keeps the mount out of `sys.path` entirely:

    /usr/bin/python3 -I ~/reclaim.py --force

`--install` writes that copy for you, to a path that is not on the share.

Without root the only lever is memory pressure, which means briefly allocating several
GiB.  Running under sudo avoids that entirely.
"""

import argparse
import os
import shutil
import sys

#: Roughly the slab size at which the daemon is likely near a 1M descriptor limit.  A
#: cached dentry plus inode is on the order of a kilobyte, so a gigabyte of reclaimable
#: slab is on the order of a million inodes.
CONCERN_GIB = 1.0
#: Leave at least this much memory free while applying pressure.
RESERVE_GIB = 2.0
#: Never allocate more than this in total.
MAX_GIB = 48
#: Below this, the cache is already small enough that pressure achieves nothing.
FLOOR_KB = 1024 * 1024


def meminfo(key):
    try:
        with open("/proc/meminfo") as handle:
            for line in handle:
                if line.startswith(key):
                    return int(line.split()[1])
    except OSError:
        pass
    return 0


def reclaimable_kb():
    return meminfo("SReclaimable")


def available_gib():
    return meminfo("MemAvailable") / (1024 * 1024)


def drop_caches():
    """The direct path: ask the kernel to drop dentries and inodes.  Needs root."""
    try:
        with open("/proc/sys/vm/drop_caches", "w") as handle:
            handle.write("2\n")
        return True
    except OSError:
        return False


def release_handles(reserve_gib=RESERVE_GIB, max_gib=MAX_GIB, floor_kb=FLOOR_KB):
    """Force the guest to evict cached inodes, so the daemon can close their handles."""
    if drop_caches():
        return True
    before = reclaimable_kb()
    if before <= 0:
        return False
    if before < floor_kb:
        return True
    target = before // 2
    blocks = []
    try:
        for _ in range(int(max_gib)):
            if reclaimable_kb() <= target:
                return True
            if available_gib() <= reserve_gib:
                break
            blocks.append(bytearray(1024**3))
    except MemoryError:
        pass
    finally:
        blocks.clear()
    return reclaimable_kb() <= target


def report(prefix):
    slab = reclaimable_kb() / (1024 * 1024)
    print(f"{prefix:>8}: {slab:.2f} GiB reclaimable slab "
          f"(~{slab:.1f}M cached inodes), {available_gib():.1f} GiB available")
    return slab


def install(destination):
    """Copy this script somewhere that is not on the share it rescues."""
    destination = os.path.abspath(os.path.expanduser(destination))
    if os.path.realpath(destination) == os.path.realpath(os.path.abspath(__file__)):
        raise SystemExit("refusing to install over the original")
    shutil.copyfile(os.path.abspath(__file__), destination)
    os.chmod(destination, 0o755)
    print(f"Installed to {destination}")
    print(f"  when the mount is refusing opens, run:  /usr/bin/python3 -I {destination}")
    return destination


def main(argv=None):
    parser = argparse.ArgumentParser(description=__doc__,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--check", action="store_true", help="report only, reclaim nothing")
    parser.add_argument("--force", action="store_true", help="reclaim even if it looks fine")
    parser.add_argument("--install", nargs="?", const="~/reclaim.py", default=None,
                        metavar="PATH",
                        help="copy this script off the share so it survives ENFILE")
    args = parser.parse_args(argv)

    if args.install:
        install(args.install)
        return 0

    before = report("before")
    if args.check:
        print("  (check only; nothing reclaimed)")
        return 0 if before < CONCERN_GIB else 1

    if before < CONCERN_GIB and not args.force:
        print(f"  below {CONCERN_GIB} GiB; nothing to do.  Use --force to reclaim anyway.")
        return 0

    if os.geteuid() != 0:
        print("  not root, so using memory pressure; run under sudo for the direct path")
    freed = release_handles()
    after = report("after")
    print(f"  {'reclaimed' if freed else 'no change'}: {before - after:+.2f} GiB")
    return 0


if __name__ == "__main__":
    sys.exit(main())