200 lines
8.3 KiB
Python
200 lines
8.3 KiB
Python
"""Working with the archive's virtiofs mount without exhausting it.
|
|
|
|
The archive sits on a virtiofs share whose daemon holds a host file descriptor for
|
|
every inode the guest has looked up. Touching a few hundred thousand files fills the
|
|
guest's dentry cache, the daemon runs out of descriptors, and the mount then returns
|
|
ENFILE ("too many open files in system") for *any* subsequent open -- reads, writes,
|
|
even starting a Python interpreter stored on it. It does not recover on its own.
|
|
|
|
The durable fix is host-side (``virtiofsd --inode-file-handles=prefer``, which stores
|
|
compact handles instead of descriptors). Until that is in place, anything that
|
|
enumerates a large part of the archive has to periodically hand the descriptors back,
|
|
which means persuading the guest kernel to evict dentries so it sends FORGET.
|
|
|
|
Note that ``vm.vfs_cache_pressure`` does **not** do this. It biases which caches the
|
|
kernel drops once it has decided to reclaim; it does not cause reclaim. On a machine
|
|
with free memory the kernel never feels pressure, so the dentry cache grows unbounded
|
|
whatever that setting says. Reclaim has to be asked for explicitly.
|
|
"""
|
|
|
|
import atexit
|
|
|
|
#: Operations between reclaims during a long traversal. Low enough that the mount
|
|
#: never approaches its ceiling, high enough that the cost is amortised.
|
|
#:
|
|
#: Was 12,000, which proved too coarse. The mount refuses opens somewhere around 2 GiB
|
|
#: of reclaimable slab -- roughly two million cached inodes -- and a job that touches
|
|
#: 25,000 frames with several reads each can cross that between two ticks, which is how
|
|
#: a `fill` run left the whole machine unable to exec anything off the share. 6,000 is
|
|
#: about a quarter of the way to the ceiling per interval, so a single missed tick is
|
|
#: not enough to reach it.
|
|
RELIEF_INTERVAL = 6_000
|
|
|
|
#: NOTE: reclaim.py deliberately duplicates the reclaim logic below rather than
|
|
#: importing it. That is not an oversight. This module lives on the very share whose
|
|
#: exhaustion it addresses, so when the mount starts refusing opens neither this file
|
|
#: nor the venv interpreter can be read at all -- the recovery tool has to stand alone
|
|
#: on the root filesystem. Keep the two in step by hand; there is not much of either.
|
|
|
|
|
|
def drop_caches():
|
|
"""Ask the kernel to free dentries and inodes. True if it worked.
|
|
|
|
Needs root, and is worth having it: this is direct and instant, where the
|
|
fallback has to allocate tens of gigabytes to provoke the same reclaim. Run
|
|
long traversals under sudo and they will take this path.
|
|
"""
|
|
try:
|
|
with open("/proc/sys/vm/drop_caches", "w") as handle:
|
|
handle.write("2\n") # 2 = dentries and inodes; page cache is not the issue
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
def reclaimable_kb():
|
|
"""Size of the reclaimable slab, which is where dentries and inodes live."""
|
|
try:
|
|
with open("/proc/meminfo") as handle:
|
|
for line in handle:
|
|
if line.startswith("SReclaimable:"):
|
|
return int(line.split()[1])
|
|
except (OSError, ValueError, IndexError):
|
|
pass
|
|
return 0
|
|
|
|
|
|
def available_gib():
|
|
"""Memory the kernel thinks can be handed out without swapping."""
|
|
try:
|
|
with open("/proc/meminfo") as handle:
|
|
for line in handle:
|
|
if line.startswith("MemAvailable:"):
|
|
return int(line.split()[1]) / (1024 * 1024)
|
|
except (OSError, ValueError, IndexError):
|
|
pass
|
|
return 0.0
|
|
|
|
|
|
#: Memory left unclaimed by the fallback, in GiB. Pressure has to be genuine to
|
|
#: make the kernel reclaim slab at all, so this is a *reserve* rather than a
|
|
#: fraction: capping at some proportion of available memory always leaves headroom,
|
|
#: the kernel never feels squeezed, and the allocation frees nothing while still
|
|
#: costing the machine several gigabytes -- worse than not trying.
|
|
RELIEF_RESERVE_GIB = 2.0
|
|
#: Absolute cap, so a machine with vast memory does not get an unbounded allocation.
|
|
RELIEF_MAX_GIB = 48
|
|
|
|
|
|
#: Below this much reclaimable slab there is nothing worth freeing -- it is the
|
|
#: working set of live processes, not archive inodes. Pushing memory down to the
|
|
#: reserve to shave a few hundred megabytes off it costs the machine far more than
|
|
#: it gains, and during a long job this runs every RELIEF_INTERVAL items.
|
|
RELIEF_FLOOR_KB = 1024 * 1024 # 1 GiB
|
|
|
|
|
|
def release_handles(budget_gib=None, reserve_gib=RELIEF_RESERVE_GIB,
|
|
floor_kb=RELIEF_FLOOR_KB):
|
|
"""Force the kernel to shrink its dentry/inode cache. True if anything was freed.
|
|
|
|
Prefers ``drop_caches``; falls back to allocating memory until the reclaimable
|
|
slab actually shrinks, or until only `reserve_gib` remains available.
|
|
|
|
Two things make the fallback awkward, both learned the hard way:
|
|
|
|
* A *fixed* budget silently does nothing once the machine has free memory --
|
|
10 GiB against 17 GiB free reclaims zero while reporting success.
|
|
* A budget capped at a *fraction* of available memory has the same failure for
|
|
the same reason: it never applies real pressure. 14 GiB of a 23 GiB
|
|
allowance freed nothing at all.
|
|
|
|
So it pushes until the kernel actually gives ground, stopping at a fixed reserve
|
|
rather than a proportion, and reports honestly whether the slab moved.
|
|
"""
|
|
if drop_caches():
|
|
return True
|
|
|
|
before = reclaimable_kb()
|
|
if before <= 0:
|
|
return False
|
|
if before < floor_kb:
|
|
return True # already small; nothing here is worth the memory churn
|
|
|
|
target = before // 2
|
|
if budget_gib is None:
|
|
budget_gib = RELIEF_MAX_GIB
|
|
blocks = []
|
|
try:
|
|
for _ in range(int(budget_gib)):
|
|
if reclaimable_kb() <= target:
|
|
return True
|
|
if available_gib() <= reserve_gib:
|
|
break # as much pressure as is safe to apply
|
|
blocks.append(bytearray(1024 * 1024 * 1024))
|
|
except MemoryError:
|
|
pass
|
|
finally:
|
|
blocks.clear()
|
|
return reclaimable_kb() <= target
|
|
|
|
|
|
class Reliever:
|
|
"""Hands file handles back periodically during a bulk traversal.
|
|
|
|
Every long pass over the archive -- indexing, detecting, filling, rendering --
|
|
accumulates dentries that pin handles in the host's virtiofs daemon, and the
|
|
mount eventually refuses *every* open, taking unrelated software on the machine
|
|
down with it. Reclaim was originally wired only into the migration and the
|
|
index build; the bench jobs, which run for hours over tens of thousands of
|
|
frames, had none, and duly exhausted the mount.
|
|
|
|
Bulk loops construct one of these and call :meth:`tick` per item.
|
|
"""
|
|
|
|
def __init__(self, interval=RELIEF_INTERVAL, enabled=True, label="", on_exit=True):
|
|
self.interval = interval
|
|
self.enabled = enabled
|
|
self.label = label
|
|
self.on_exit = on_exit
|
|
self.seen = 0
|
|
self.releases = 0
|
|
self._next = interval
|
|
self._registered = False
|
|
|
|
def tick(self, count=1):
|
|
"""Record `count` items processed, reclaiming if enough have gone by."""
|
|
self.seen += count
|
|
if self.enabled and self.on_exit and not self._registered:
|
|
# Reclaiming only *during* a run leaves the machine loaded once it ends:
|
|
# the inodes a traversal cached stay pinned, and with them the host's
|
|
# file handles, until something else forces reclaim. Every bulk job has
|
|
# to hand them back when it finishes, not merely while it runs.
|
|
atexit.register(self.finish)
|
|
self._registered = True
|
|
if not self.enabled or self.seen < self._next:
|
|
return False
|
|
self._next = self.seen + self.interval
|
|
self._release("after {} items".format(self.seen))
|
|
return True
|
|
|
|
def finish(self):
|
|
"""Reclaim once at the end of a run. Idempotent; safe to call twice."""
|
|
if not self.enabled or self.seen == 0:
|
|
return False
|
|
self.enabled = False # nothing more to do for this traversal
|
|
self._release("on finish, {} items".format(self.seen))
|
|
return True
|
|
|
|
def _release(self, why):
|
|
freed = release_handles()
|
|
self.releases += 1
|
|
if self.label:
|
|
print(f" [{self.label}] reclaimed handles {why} "
|
|
f"({'ok' if freed else 'no change'})", flush=True)
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
self.finish()
|
|
return False
|