546 lines
22 KiB
Python
546 lines
22 KiB
Python
#!/usr/bin/env python3
|
|
"""Keep the guest's inode cache below the ceiling that breaks the archive mount.
|
|
|
|
The archive is a virtiofs re-export of an SMB share: guest -> virtiofs -> virtiofsd on
|
|
the host -> cifs.ko -> SMB3 -> NAS. The middle hop is the problem. virtiofsd holds one
|
|
open host file descriptor per inode the guest has looked up, and the guest's dentry cache
|
|
is bounded only by memory pressure -- on a machine with free memory it simply grows. When
|
|
virtiofsd runs out of descriptors the mount returns ENFILE ("too many open files in
|
|
system") for *every* subsequent open, and does not recover on its own. That surfaces
|
|
across the whole machine, because anything on the share -- including the venv interpreter
|
|
the pipeline runs from -- stops being executable.
|
|
|
|
Run as a daemon this watches the count and drops caches only when it approaches the
|
|
ceiling. That ordering is the whole point: reclaiming *early* is not free, because every
|
|
evicted dentry is an SMB round trip to re-establish, so the goal is to stay as warm as
|
|
possible for as long as possible and intervene just before the wall.
|
|
|
|
reclaim.py # report, then reclaim if we are near the ceiling
|
|
reclaim.py --check # report only, change nothing (works unprivileged)
|
|
sudo reclaim.py --force # reclaim now, whatever the count says
|
|
sudo reclaim.py --install # install and enable the watchdog service
|
|
|
|
**This file deliberately imports nothing from `suvi`, and nothing outside the standard
|
|
library.** It has to run in exactly the situation it exists for, and in that situation
|
|
the archive mount refuses every open -- which means the venv interpreter (which lives on
|
|
the mount), the `suvi` package, and even Python's scan of the working directory all
|
|
fail with ENFILE before any of this code runs. Recovery went like this once:
|
|
|
|
$ ./.venv/bin/python reclaim.py
|
|
bash: ./.venv/bin/python: Too many open files in system
|
|
$ /usr/bin/python3 -c ...
|
|
OSError: [Errno 23] Too many open files in system: '.../scripts'
|
|
|
|
So: no package imports, and run it off the root filesystem with an isolated interpreter,
|
|
which keeps the mount out of `sys.path` entirely. ``--install`` puts a copy where that
|
|
is true and writes the systemd unit that runs it.
|
|
|
|
On the metric. An earlier version of this script decided from ``SReclaimable``, on the
|
|
reasoning that a cached dentry plus inode is on the order of a kilobyte. That is true per
|
|
object but useless as a signal: slab pages are partially free, so objects accumulate
|
|
without the page count moving. Measured on this machine, a walk of 40,000 archive files
|
|
added ~16,000 inodes and moved ``SReclaimable`` by 2 MiB. The inode count itself is the
|
|
signal, and it is exact.
|
|
|
|
On the lever. ``drop_caches`` is the only thing that reliably sheds dentries. The
|
|
gentler option was tried and measured: graduated memory-cgroup pressure freed 15 MiB of
|
|
slab per 1 GiB of ``memory.reclaim`` requested, so shedding a useful number of inodes
|
|
would mean evicting gigabytes of page cache -- on a share this slow, far more expensive
|
|
than the problem. So the design is a blunt instrument fired rarely, rather than a
|
|
gentle one running constantly, and that is why it needs root.
|
|
"""
|
|
|
|
import argparse
|
|
import errno
|
|
import json
|
|
import os
|
|
import shutil
|
|
import signal
|
|
import sys
|
|
import threading
|
|
import time
|
|
|
|
#: The number of inodes the guest may cache before the mount refuses opens. Half of
|
|
#: virtiofsd's own default ``RLIMIT_NOFILE`` (1,000,000): on 2026-10-09 the mount
|
|
#: refused opens (ENFILE) while the watchdog held the count at 75% of a million, as
|
|
#: other jobs walked the whole share, so the real limit sits lower than the default
|
|
#: suggests. Still a guess until proven -- see `Watchdog.probe`, which lowers it on
|
|
#: evidence. If you can read the host, the true value is
|
|
#: `grep 'Max open files' /proc/$(pgrep virtiofsd)/limits`.
|
|
DEFAULT_CEILING = 500_000
|
|
#: Act at this fraction of the ceiling, and not a moment sooner. Measured on this
|
|
#: archive, the cache fills at ~890 inodes/s under a serial `find` and ~4,000/s under a
|
|
#: six-way parallel stat, which is the shape of the pipeline's own bulk passes. The
|
|
#: 250,000-inode gap the default leaves is therefore about a minute of the worst load
|
|
#: this machine can produce -- twelve poll intervals -- and below the mark ordinary work
|
|
#: keeps its cache entirely, which is the point.
|
|
HIGH_FRACTION = 0.75
|
|
#: Start probing for a wrongly-estimated ceiling at this fraction of `high`.
|
|
PROBE_FRACTION = 0.9
|
|
#: Seconds between polls. Two small /proc reads; the cost is not measurable.
|
|
DEFAULT_INTERVAL = 5.0
|
|
#: Never drop twice inside this window, so a job that re-fills the cache immediately
|
|
#: cannot turn the watchdog into a thrash loop.
|
|
DEFAULT_COOLDOWN = 60.0
|
|
#: Seconds between "nothing happened" log lines. These carry the peak seen since the
|
|
#: last one, which is what tells you whether `high` can be raised.
|
|
DEFAULT_HEARTBEAT = 3600.0
|
|
#: How long to wait for the probe open() before giving up on it.
|
|
PROBE_TIMEOUT = 5.0
|
|
|
|
CONFIG_PATH = "/etc/suvi-reclaimd.conf"
|
|
STATE_PATH = "/var/lib/suvi-reclaimd/state.json"
|
|
INSTALL_PATH = "/usr/local/sbin/suvi-reclaimd"
|
|
SERVICE_PATH = "/etc/systemd/system/suvi-reclaimd.service"
|
|
SERVICE_NAME = "suvi-reclaimd"
|
|
|
|
#: A small file on the share, opened to find out whether the mount is refusing opens.
|
|
#: Any file will do; it only has to be on the affected mount and cheap to read.
|
|
DEFAULT_PROBE_PATH = "/media/shared/NOAA_GOES/scripts/pytest.ini"
|
|
|
|
SERVICE_UNIT = """\
|
|
[Unit]
|
|
Description=Keep the guest dentry cache below virtiofsd's descriptor ceiling
|
|
Documentation=file://{source}
|
|
After=media-shared.mount
|
|
|
|
[Service]
|
|
ExecStart=/usr/bin/python3 -I {target} --daemon
|
|
Restart=always
|
|
RestartSec=5
|
|
Nice=5
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
"""
|
|
|
|
DEFAULT_CONFIG = f"""\
|
|
# Settings for the inode-cache watchdog (reclaim.py --daemon). Delete a line to take
|
|
# its built-in default. Restart with: systemctl restart {SERVICE_NAME}
|
|
|
|
# Descriptors virtiofsd can hold, and so inodes the guest may cache before the mount
|
|
# starts refusing opens. {DEFAULT_CEILING} is half virtiofsd's own default
|
|
# RLIMIT_NOFILE, after the mount refused opens below 750,000; if you can read the
|
|
# host, the true figure is
|
|
# grep 'Max open files' /proc/$(pgrep virtiofsd)/limits
|
|
# The watchdog lowers this on its own if it is ever refused an open, and never raises
|
|
# it -- so a value that turns out to be too high costs one event, not a standing risk.
|
|
ceiling = {DEFAULT_CEILING}
|
|
|
|
# Where to act. Defaults to {HIGH_FRACTION:g} * ceiling. Below it nothing happens at all, which
|
|
# is what keeps the cache warm; the point of dropping is to survive, not to stay tidy.
|
|
# Measured on this archive: the cache fills at ~890 inodes/s under a serial walk and
|
|
# ~4,000/s under a six-way parallel one, so the default leaves about a minute of the
|
|
# worst load this machine produces. Raise it once the heartbeat's peak line shows what
|
|
# you actually reach -- more cache is faster, and every eviction is an SMB round trip
|
|
# to undo.
|
|
# high = {int(DEFAULT_CEILING * HIGH_FRACTION)}
|
|
|
|
# interval = {DEFAULT_INTERVAL:g} # seconds between polls; two small /proc reads
|
|
# cooldown = {DEFAULT_COOLDOWN:g} # seconds; stops a refilling job causing a thrash loop
|
|
# heartbeat = {DEFAULT_HEARTBEAT:g} # seconds between "peak since last report" lines; 0 to silence
|
|
|
|
# A small file on the affected mount, opened to tell a full mount from a wedged one.
|
|
# probe_path = {DEFAULT_PROBE_PATH}
|
|
"""
|
|
|
|
|
|
def log(message):
|
|
"""One line to stdout, which under systemd is the journal."""
|
|
print(message, flush=True)
|
|
|
|
|
|
# ------------------------------------------------------------------------- the metric
|
|
|
|
|
|
def fuse_inodes(path="/proc/slabinfo"):
|
|
"""Cached FUSE inodes from /proc/slabinfo, or None if it cannot be read.
|
|
|
|
This is the exact number of inodes virtiofsd is holding a descriptor for, which is
|
|
the quantity that runs out. /proc/slabinfo is root-only, so unprivileged callers
|
|
fall back to `total_inodes`.
|
|
"""
|
|
try:
|
|
with open(path) as handle:
|
|
for line in handle:
|
|
fields = line.split()
|
|
# name, active_objs, num_objs, objsize, ...
|
|
if len(fields) > 1 and fields[0] == "fuse_inode":
|
|
return int(fields[1])
|
|
except (OSError, ValueError):
|
|
pass
|
|
return None
|
|
|
|
|
|
def total_inodes(path="/proc/sys/fs/inode-nr"):
|
|
"""Every inode the kernel has cached, across all filesystems.
|
|
|
|
Coarser than `fuse_inodes` -- it counts the root filesystem and the squashfs snaps
|
|
too -- but those are a stable baseline of a few tens of thousands against a ceiling
|
|
of a million, so as a trigger it is more than good enough. Unlike /proc/slabinfo it
|
|
is world-readable, which is what lets `--check` work without sudo.
|
|
"""
|
|
try:
|
|
with open(path) as handle:
|
|
return int(handle.read().split()[0])
|
|
except (OSError, ValueError, IndexError):
|
|
return 0
|
|
|
|
|
|
def cached_inodes():
|
|
"""The best available inode count, and where it came from."""
|
|
exact = fuse_inodes()
|
|
if exact is not None:
|
|
return exact, "fuse_inode"
|
|
return total_inodes(), "inode-nr"
|
|
|
|
|
|
# ------------------------------------------------------------------------- the lever
|
|
|
|
|
|
def drop_caches():
|
|
"""Ask the kernel to free dentries and inodes. True if it worked.
|
|
|
|
Needs root. Only *unused* dentries go: anything with an open file or a working
|
|
directory on it survives, so what this discards is the cold residue of traversal
|
|
rather than the working set.
|
|
"""
|
|
try:
|
|
with open("/proc/sys/vm/drop_caches", "w") as handle:
|
|
handle.write("2\n") # 2 = dentries and inodes; page cache is not the issue
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
#: Held for the duration of a probe. A probe that never returns leaves it held, which
|
|
#: is how `probe` knows not to start a second one.
|
|
_probe_lock = threading.Lock()
|
|
|
|
|
|
def probe(path, timeout=PROBE_TIMEOUT):
|
|
"""Try to open `path`. Returns the errno that stopped us, or 0 on success.
|
|
|
|
Runs in a thread rather than inline because the failure this looks for can also
|
|
present as a mount that stops answering altogether, and the watchdog must not be the
|
|
thing that hangs. A task blocked in the filesystem cannot reliably be interrupted,
|
|
so the thread is simply abandoned -- and because at most one is ever outstanding, a
|
|
wedged mount costs one thread rather than one per poll. A probe still stuck from
|
|
last time is itself an answer: the mount is not well.
|
|
"""
|
|
# Captured, not looked up again in the thread: an abandoned probe can outlive any
|
|
# rebinding of the global, and releasing whatever happens to be there by then would
|
|
# unlock a lock this call never took.
|
|
lock = _probe_lock
|
|
if not lock.acquire(blocking=False):
|
|
return errno.ENFILE
|
|
result = {}
|
|
|
|
def attempt():
|
|
try:
|
|
descriptor = os.open(path, os.O_RDONLY)
|
|
os.close(descriptor)
|
|
result["errno"] = 0
|
|
except OSError as error:
|
|
result["errno"] = error.errno or 0
|
|
except BaseException:
|
|
result["errno"] = 0
|
|
finally:
|
|
lock.release()
|
|
|
|
thread = threading.Thread(target=attempt, daemon=True)
|
|
thread.start()
|
|
thread.join(timeout)
|
|
# A probe that has not answered within the timeout is treated as exhaustion. If it
|
|
# answers a moment later it releases the lock, so this corrects itself next poll.
|
|
return result.get("errno", errno.ENFILE)
|
|
|
|
|
|
# ------------------------------------------------------------------- config and state
|
|
|
|
|
|
def _coerce(value, kind, minimum):
|
|
"""Parse one config value, or raise ValueError. Bounds keep a typo from disarming
|
|
the watchdog -- an interval of 0 would spin, a ceiling of 0 would drop constantly."""
|
|
parsed = kind(value)
|
|
if parsed < minimum:
|
|
raise ValueError(f"{parsed} is below the minimum {minimum}")
|
|
return parsed
|
|
|
|
|
|
#: name -> (type, minimum). Anything outside this is rejected rather than ignored, so a
|
|
#: misspelt key is a loud failure rather than a silently unapplied setting.
|
|
CONFIG_FIELDS = {
|
|
"ceiling": (int, 1000),
|
|
"high": (int, 1000),
|
|
"interval": (float, 0.1),
|
|
"cooldown": (float, 0.0),
|
|
"heartbeat": (float, 0.0),
|
|
"probe_path": (str, None),
|
|
}
|
|
|
|
|
|
def load_config(path=CONFIG_PATH):
|
|
"""Read `key=value` lines. A missing file means defaults; a bad one is fatal."""
|
|
config = {}
|
|
try:
|
|
with open(path) as handle:
|
|
lines = handle.readlines()
|
|
except FileNotFoundError:
|
|
return config
|
|
except OSError as error:
|
|
raise SystemExit(f"cannot read {path}: {error}")
|
|
|
|
for number, line in enumerate(lines, 1):
|
|
line = line.split("#", 1)[0].strip()
|
|
if not line:
|
|
continue
|
|
if "=" not in line:
|
|
raise SystemExit(f"{path}:{number}: expected key=value")
|
|
key, _, value = line.partition("=")
|
|
key, value = key.strip(), value.strip()
|
|
if key not in CONFIG_FIELDS:
|
|
raise SystemExit(f"{path}:{number}: unknown setting {key!r}")
|
|
kind, minimum = CONFIG_FIELDS[key]
|
|
if kind is str:
|
|
config[key] = value
|
|
continue
|
|
try:
|
|
config[key] = _coerce(value, kind, minimum)
|
|
except ValueError as error:
|
|
raise SystemExit(f"{path}:{number}: bad value for {key}: {error}")
|
|
return config
|
|
|
|
|
|
def load_state(path=STATE_PATH):
|
|
"""What we have learned about the ceiling. Corruption is not worth dying over."""
|
|
try:
|
|
with open(path) as handle:
|
|
state = json.load(handle)
|
|
if not isinstance(state, dict):
|
|
return {}
|
|
return state
|
|
except (OSError, ValueError):
|
|
return {}
|
|
|
|
|
|
def save_state(state, path=STATE_PATH):
|
|
"""Persist learned state, best effort. Written via a temporary file so a crash
|
|
mid-write cannot leave a truncated file that reads back as no knowledge at all."""
|
|
try:
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
temporary = path + ".tmp"
|
|
with open(temporary, "w") as handle:
|
|
json.dump(state, handle)
|
|
os.replace(temporary, path)
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
# ------------------------------------------------------------------------ the watchdog
|
|
|
|
|
|
class Watchdog:
|
|
"""Polls the inode count and drops caches when it nears the ceiling.
|
|
|
|
Everything it does is driven by one number, and it does nothing at all while that
|
|
number is comfortable. That is deliberate: the cache is worth keeping, and on a
|
|
share where a lookup is an SMB round trip, reclaiming early costs real time.
|
|
"""
|
|
|
|
def __init__(self, config=None, state_path=STATE_PATH):
|
|
config = dict(config or {})
|
|
self.state_path = state_path
|
|
state = load_state(state_path)
|
|
|
|
self.ceiling = config.get("ceiling", DEFAULT_CEILING)
|
|
# A ceiling learned the hard way beats a configured guess, but only downwards:
|
|
# see `observe_exhaustion`.
|
|
learned = state.get("ceiling")
|
|
if isinstance(learned, int) and 0 < learned < self.ceiling:
|
|
self.ceiling = learned
|
|
|
|
#: An explicit `high` is honoured as-is; a derived one follows the ceiling down
|
|
#: when the ceiling is corrected, which is the behaviour you want from a default.
|
|
self.explicit_high = config.get("high")
|
|
self.interval = config.get("interval", DEFAULT_INTERVAL)
|
|
self.cooldown = config.get("cooldown", DEFAULT_COOLDOWN)
|
|
self.heartbeat = config.get("heartbeat", DEFAULT_HEARTBEAT)
|
|
self.probe_path = config.get("probe_path", DEFAULT_PROBE_PATH)
|
|
|
|
self.last_drop = None
|
|
self.peak = 0
|
|
self.drops = 0
|
|
self.running = True
|
|
|
|
@property
|
|
def high(self):
|
|
if self.explicit_high is not None:
|
|
return self.explicit_high
|
|
return int(self.ceiling * HIGH_FRACTION)
|
|
|
|
def observe_exhaustion(self, count):
|
|
"""Record that the mount refused an open at `count` inodes.
|
|
|
|
The correction only ever lowers the ceiling. A watchdog that could raise its own
|
|
limit would creep towards the wall looking for it, and finding it is exactly the
|
|
outcome this exists to prevent.
|
|
"""
|
|
if count <= 0 or count >= self.ceiling:
|
|
return False
|
|
self.ceiling = count
|
|
state = load_state(self.state_path)
|
|
state["ceiling"] = count
|
|
state["observed_at"] = time.time()
|
|
save_state(state, self.state_path)
|
|
log(f"mount refused an open at {count:,} inodes: "
|
|
f"ceiling lowered to {count:,}, high water now {self.high:,}")
|
|
return True
|
|
|
|
def reclaim(self, count, why):
|
|
"""Drop caches and report what it bought."""
|
|
if not drop_caches():
|
|
# The only way this fails is lack of privilege, which will not resolve on
|
|
# its own. Take the cooldown anyway so a misconfigured service warns once a
|
|
# minute instead of once a poll.
|
|
self.last_drop = time.monotonic()
|
|
log(f"WARNING: need root to drop caches, and {count:,} inodes is {why}")
|
|
return False
|
|
after, _ = cached_inodes()
|
|
self.last_drop = time.monotonic()
|
|
self.drops += 1
|
|
log(f"reclaimed ({why}): {count:,} -> {after:,} inodes, "
|
|
f"high water {self.high:,} of {self.ceiling:,} ceiling")
|
|
return True
|
|
|
|
def step(self):
|
|
"""One poll. True if it reclaimed."""
|
|
count, _ = cached_inodes()
|
|
self.peak = max(self.peak, count)
|
|
|
|
if count < self.high * PROBE_FRACTION:
|
|
return False
|
|
|
|
# Close enough to matter: find out whether the ceiling estimate is still good
|
|
# before relying on it. This costs one open() and only happens up here.
|
|
failure = probe(self.probe_path)
|
|
if failure in (errno.ENFILE, errno.EMFILE):
|
|
self.observe_exhaustion(count)
|
|
return self.reclaim(count, "mount is refusing opens")
|
|
|
|
if count < self.high:
|
|
return False
|
|
if self.last_drop is not None and \
|
|
time.monotonic() - self.last_drop < self.cooldown:
|
|
return False
|
|
return self.reclaim(count, f"at or above the {self.high:,} high water mark")
|
|
|
|
def stop(self, *_):
|
|
self.running = False
|
|
|
|
def run(self):
|
|
"""Poll until told to stop."""
|
|
signal.signal(signal.SIGTERM, self.stop)
|
|
signal.signal(signal.SIGINT, self.stop)
|
|
_, source = cached_inodes()
|
|
log(f"watching {source}: high water {self.high:,} of {self.ceiling:,} ceiling, "
|
|
f"every {self.interval:g}s")
|
|
|
|
next_beat = time.monotonic() + self.heartbeat
|
|
while self.running:
|
|
self.step()
|
|
now = time.monotonic()
|
|
if self.heartbeat and now >= next_beat:
|
|
log(f"peak {self.peak:,} inodes since last report "
|
|
f"({self.peak * 100 // max(self.ceiling, 1)}% of ceiling), "
|
|
f"{self.drops} reclaim(s) total")
|
|
self.peak, next_beat = 0, now + self.heartbeat
|
|
time.sleep(self.interval)
|
|
log("stopping")
|
|
return 0
|
|
|
|
|
|
# --------------------------------------------------------------------------- one-shots
|
|
|
|
|
|
def report(watchdog):
|
|
"""Print where we stand, and return the count."""
|
|
count, source = cached_inodes()
|
|
share = count * 100.0 / max(watchdog.ceiling, 1)
|
|
log(f"{count:,} cached inodes ({source}), {share:.1f}% of the "
|
|
f"{watchdog.ceiling:,} ceiling; high water {watchdog.high:,}")
|
|
return count
|
|
|
|
|
|
def install(target=INSTALL_PATH, service=SERVICE_PATH, config=CONFIG_PATH):
|
|
"""Copy this script off the share and write the unit and config that run it."""
|
|
source = os.path.abspath(__file__)
|
|
target = os.path.abspath(os.path.expanduser(target))
|
|
if os.path.realpath(target) == os.path.realpath(source):
|
|
raise SystemExit("refusing to install over the original")
|
|
|
|
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
shutil.copyfile(source, target)
|
|
os.chmod(target, 0o755)
|
|
log(f"installed {target}")
|
|
|
|
with open(service, "w") as handle:
|
|
handle.write(SERVICE_UNIT.format(source=source, target=target))
|
|
log(f"wrote {service}")
|
|
|
|
# Never overwrite a tuned config -- the whole point of the heartbeat's peak line is
|
|
# that the operator raises `high` once they know what this machine actually reaches.
|
|
if config and not os.path.exists(config):
|
|
with open(config, "w") as handle:
|
|
handle.write(DEFAULT_CONFIG)
|
|
log(f"wrote {config}")
|
|
elif config:
|
|
log(f"kept existing {config}")
|
|
log(f" enable it with: sudo systemctl enable --now {SERVICE_NAME}")
|
|
log(f" watch it with: journalctl -fu {SERVICE_NAME}")
|
|
log(f" by hand: /usr/bin/python3 -I {target}")
|
|
return target
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(
|
|
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
parser.add_argument("--daemon", action="store_true",
|
|
help="poll and reclaim near the ceiling; this is the service")
|
|
parser.add_argument("--check", action="store_true",
|
|
help="report only, reclaim nothing; nonzero if above high water")
|
|
parser.add_argument("--force", action="store_true",
|
|
help="reclaim even if the count looks fine")
|
|
parser.add_argument("--install", action="store_true",
|
|
help=f"copy to {INSTALL_PATH} and write the systemd unit")
|
|
parser.add_argument("--config", default=CONFIG_PATH, help="settings file")
|
|
args = parser.parse_args(argv)
|
|
|
|
if args.install:
|
|
install()
|
|
return 0
|
|
|
|
watchdog = Watchdog(load_config(args.config))
|
|
|
|
if args.daemon:
|
|
return watchdog.run()
|
|
|
|
count = report(watchdog)
|
|
if args.check:
|
|
return 0 if count < watchdog.high else 1
|
|
|
|
if count < watchdog.high and not args.force:
|
|
log(f" below the {watchdog.high:,} high water mark; nothing to do. "
|
|
f"--force reclaims anyway.")
|
|
return 0
|
|
if os.geteuid() != 0:
|
|
log(" reclaiming needs root: re-run under sudo, or install the watchdog "
|
|
f"with `sudo {os.path.basename(__file__)} --install`")
|
|
return 1
|
|
watchdog.reclaim(count, "asked to" if args.force else "above the high water mark")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|