"""Tests for the inode-cache watchdog. The property that matters is restraint: this thing runs as root on a timer and drops the whole machine's dentry cache, so it must do nothing at all until the count genuinely approaches the ceiling, and it must never talk itself into a higher ceiling than the one it has evidence for. Most of what follows is about the *absence* of action. """ import errno import json import os import threading import pytest import reclaim @pytest.fixture(autouse=True) def fresh_probe_lock(monkeypatch): """A probe the watchdog abandoned holds the lock until it returns, by design. Tests must not inherit that from each other.""" monkeypatch.setattr(reclaim, "_probe_lock", threading.Lock()) SLABINFO = """\ slabinfo - version: 2.1 # name ext4_inode_cache 40312 41800 1096 29 8 : tunables 0 0 0 fuse_inode 812345 900000 832 19 4 : tunables 0 0 0 dentry 38423 45150 208 19 1 : tunables 0 0 0 """ def write(tmp_path, name, text): path = tmp_path / name path.write_text(text) return str(path) # --------------------------------------------------------------------------- the metric def test_fuse_inodes_reads_the_slab_line(tmp_path): assert reclaim.fuse_inodes(write(tmp_path, "slabinfo", SLABINFO)) == 812345 def test_fuse_inodes_is_none_when_the_line_is_absent(tmp_path): """No fuse_inode line means no FUSE mounts, which is not the same as zero cached -- it means this metric does not apply and the caller must fall back.""" trimmed = "\n".join(l for l in SLABINFO.splitlines() if "fuse_inode" not in l) assert reclaim.fuse_inodes(write(tmp_path, "slabinfo", trimmed)) is None def test_fuse_inodes_is_none_when_unreadable(tmp_path): """The usual case: /proc/slabinfo is root-only and we are not root.""" assert reclaim.fuse_inodes(str(tmp_path / "nope")) is None def test_fuse_inodes_survives_a_garbled_line(tmp_path): assert reclaim.fuse_inodes(write(tmp_path, "s", "fuse_inode notanumber\n")) is None def test_total_inodes_reads_the_first_field(tmp_path): assert reclaim.total_inodes(write(tmp_path, "inode-nr", "64282\t1\n")) == 64282 def test_total_inodes_is_zero_when_malformed(tmp_path): assert reclaim.total_inodes(write(tmp_path, "inode-nr", "")) == 0 assert reclaim.total_inodes(write(tmp_path, "i2", "x y\n")) == 0 assert reclaim.total_inodes(str(tmp_path / "missing")) == 0 def test_total_inodes_reads_the_real_procfs(): assert reclaim.total_inodes() > 0 def test_cached_inodes_prefers_the_exact_source(monkeypatch): monkeypatch.setattr(reclaim, "fuse_inodes", lambda *a: 500) monkeypatch.setattr(reclaim, "total_inodes", lambda *a: 999) assert reclaim.cached_inodes() == (500, "fuse_inode") def test_cached_inodes_falls_back_when_not_root(monkeypatch): monkeypatch.setattr(reclaim, "fuse_inodes", lambda *a: None) monkeypatch.setattr(reclaim, "total_inodes", lambda *a: 999) assert reclaim.cached_inodes() == (999, "inode-nr") # ---------------------------------------------------------------------------- the probe def test_probe_succeeds_on_a_readable_file(tmp_path): assert reclaim.probe(write(tmp_path, "f", "x")) == 0 def test_probe_reports_the_errno_that_stopped_it(tmp_path): assert reclaim.probe(str(tmp_path / "absent")) == errno.ENOENT def test_probe_treats_a_mount_that_does_not_answer_as_exhaustion(tmp_path): """A FIFO with no writer blocks in open() exactly as a wedged mount does. The watchdog must not be the thing that hangs.""" fifo = str(tmp_path / "fifo") os.mkfifo(fifo) try: assert reclaim.probe(fifo, timeout=0.2) == errno.ENFILE finally: # Let the abandoned reader finish, so it does not outlive the test. os.close(os.open(fifo, os.O_WRONLY | os.O_NONBLOCK)) def test_probe_does_not_pile_up_threads_on_a_wedged_mount(tmp_path): """One abandoned probe is the budget. A probe still outstanding from last time is itself the answer, so the next poll must not start another thread.""" assert reclaim._probe_lock.acquire(blocking=False), "lock leaked from another test" try: assert reclaim.probe(str(tmp_path)) == errno.ENFILE finally: reclaim._probe_lock.release() # --------------------------------------------------------------------------- the config def test_config_defaults_when_the_file_is_absent(tmp_path): assert reclaim.load_config(str(tmp_path / "none.conf")) == {} def test_config_parses_values_comments_and_blanks(tmp_path): path = write(tmp_path, "c.conf", "# a comment\n\nceiling = 400000\n" "interval=2.5 # trailing\nprobe_path=/tmp/x\n") assert reclaim.load_config(path) == { "ceiling": 400000, "interval": 2.5, "probe_path": "/tmp/x"} def test_config_rejects_an_unknown_setting(tmp_path): """Silently ignoring a misspelt key would leave the watchdog on defaults while the operator believed otherwise, which is the worst of both.""" path = write(tmp_path, "c.conf", "cieling=400000\n") with pytest.raises(SystemExit, match="unknown setting"): reclaim.load_config(path) def test_config_rejects_a_value_that_would_disarm_the_watchdog(tmp_path): """An interval of zero spins; a ceiling of zero drops caches forever.""" with pytest.raises(SystemExit, match="interval"): reclaim.load_config(write(tmp_path, "a.conf", "interval=0\n")) with pytest.raises(SystemExit, match="ceiling"): reclaim.load_config(write(tmp_path, "b.conf", "ceiling=0\n")) def test_config_rejects_a_line_without_a_value(tmp_path): with pytest.raises(SystemExit, match="expected key=value"): reclaim.load_config(write(tmp_path, "c.conf", "ceiling\n")) # ---------------------------------------------------------------------------- the state def test_corrupt_state_is_tolerated(tmp_path): assert reclaim.load_state(write(tmp_path, "s.json", "{not json")) == {} assert reclaim.load_state(write(tmp_path, "s2.json", "[1, 2]")) == {} assert reclaim.load_state(str(tmp_path / "absent.json")) == {} def test_state_round_trips(tmp_path): path = str(tmp_path / "sub" / "state.json") assert reclaim.save_state({"ceiling": 123}, path) is True assert reclaim.load_state(path)["ceiling"] == 123 # ------------------------------------------------------------------------- the watchdog @pytest.fixture def watchdog(tmp_path, monkeypatch): """A watchdog with a known ceiling, whose lever and probe are recorded not fired.""" made = reclaim.Watchdog({"ceiling": 100_000, "cooldown": 60.0}, state_path=str(tmp_path / "state.json")) made.dropped = [] monkeypatch.setattr(reclaim, "drop_caches", lambda: made.dropped.append(1) or True) monkeypatch.setattr(reclaim, "probe", lambda *a, **k: 0) return made def at(monkeypatch, count): monkeypatch.setattr(reclaim, "cached_inodes", lambda: (count, "test")) def test_high_water_defaults_to_a_fraction_of_the_ceiling(watchdog): assert watchdog.high == 75_000 def test_an_explicit_high_water_overrides_the_fraction(tmp_path): made = reclaim.Watchdog({"ceiling": 100_000, "high": 90_000}, state_path=str(tmp_path / "s.json")) assert made.high == 90_000 def test_it_does_nothing_while_the_cache_is_comfortable(watchdog, monkeypatch): """The whole point. Reclaiming early throws away cache that costs SMB round trips to rebuild, so below the mark the watchdog must be inert.""" at(monkeypatch, 40_000) assert watchdog.step() is False assert watchdog.dropped == [] def test_it_does_not_even_probe_while_comfortable(watchdog, monkeypatch): probes = [] monkeypatch.setattr(reclaim, "probe", lambda *a, **k: probes.append(1) or 0) at(monkeypatch, 40_000) watchdog.step() assert probes == [] def test_it_reclaims_at_the_high_water_mark(watchdog, monkeypatch): at(monkeypatch, 75_000) assert watchdog.step() is True assert len(watchdog.dropped) == 1 def test_it_respects_the_cooldown(watchdog, monkeypatch): """A job that refills the cache immediately must not turn this into a thrash loop.""" at(monkeypatch, 80_000) assert watchdog.step() is True assert watchdog.step() is False assert len(watchdog.dropped) == 1 def test_the_cooldown_expires(watchdog, monkeypatch): at(monkeypatch, 80_000) watchdog.step() watchdog.last_drop -= watchdog.cooldown + 1 assert watchdog.step() is True assert len(watchdog.dropped) == 2 def test_it_tracks_the_peak_for_the_heartbeat(watchdog, monkeypatch): """The peak is what tells the operator whether the mark can be raised.""" at(monkeypatch, 40_000) watchdog.step() at(monkeypatch, 61_000) watchdog.step() at(monkeypatch, 50_000) watchdog.step() assert watchdog.peak == 61_000 def test_it_reports_failure_when_it_cannot_drop(watchdog, monkeypatch): monkeypatch.setattr(reclaim, "drop_caches", lambda: False) at(monkeypatch, 80_000) assert watchdog.step() is False def test_it_backs_off_when_it_cannot_drop(watchdog, monkeypatch): """Lack of privilege does not resolve on its own, so a misconfigured service must warn once a cooldown rather than once a poll.""" attempts = [] monkeypatch.setattr(reclaim, "drop_caches", lambda: attempts.append(1) or False) at(monkeypatch, 80_000) watchdog.step() watchdog.step() watchdog.step() assert len(attempts) == 1 # ------------------------------------------------------------- learning the real ceiling def test_enfile_near_the_mark_lowers_the_ceiling_and_reclaims(watchdog, monkeypatch): """The configured ceiling is a guess. Being refused an open is proof it was wrong, and the only proof available from inside the guest.""" monkeypatch.setattr(reclaim, "probe", lambda *a, **k: errno.ENFILE) at(monkeypatch, 70_000) # above 0.9*high, below high: would not have acted assert watchdog.step() is True assert watchdog.ceiling == 70_000 assert watchdog.high == 52_500 assert len(watchdog.dropped) == 1 def test_emfile_counts_as_exhaustion_too(watchdog, monkeypatch): monkeypatch.setattr(reclaim, "probe", lambda *a, **k: errno.EMFILE) at(monkeypatch, 70_000) assert watchdog.step() is True assert watchdog.ceiling == 70_000 def test_an_unrelated_probe_failure_is_not_exhaustion(watchdog, monkeypatch): """A missing probe file must not be mistaken for a wedged mount.""" monkeypatch.setattr(reclaim, "probe", lambda *a, **k: errno.ENOENT) at(monkeypatch, 70_000) assert watchdog.step() is False assert watchdog.ceiling == 100_000 def test_the_ceiling_is_never_raised(watchdog): """A watchdog that could raise its own limit would creep towards the wall looking for it, and finding it is the outcome this exists to prevent.""" assert watchdog.observe_exhaustion(150_000) is False assert watchdog.ceiling == 100_000 assert watchdog.observe_exhaustion(100_000) is False assert watchdog.ceiling == 100_000 assert watchdog.observe_exhaustion(0) is False assert watchdog.ceiling == 100_000 def test_a_learned_ceiling_survives_a_restart(tmp_path): path = str(tmp_path / "state.json") first = reclaim.Watchdog({"ceiling": 100_000}, state_path=path) first.observe_exhaustion(60_000) assert json.load(open(path))["ceiling"] == 60_000 second = reclaim.Watchdog({"ceiling": 100_000}, state_path=path) assert second.ceiling == 60_000 assert second.high == 45_000 def test_a_learned_ceiling_does_not_override_a_lower_configured_one(tmp_path): path = str(tmp_path / "state.json") reclaim.save_state({"ceiling": 90_000}, path) made = reclaim.Watchdog({"ceiling": 50_000}, state_path=path) assert made.ceiling == 50_000 def test_an_explicit_high_water_does_not_follow_the_ceiling_down(tmp_path): """A derived mark should track a corrected ceiling; one the operator set by hand is their decision to revisit.""" path = str(tmp_path / "state.json") made = reclaim.Watchdog({"ceiling": 100_000, "high": 80_000}, state_path=path) made.observe_exhaustion(90_000) assert made.high == 80_000 # ---------------------------------------------------------------------------- the CLI def test_check_is_quiet_and_succeeds_below_the_mark(monkeypatch, tmp_path): at(monkeypatch, 1000) monkeypatch.setattr(reclaim, "drop_caches", lambda: pytest.fail("reclaimed")) assert reclaim.main(["--check", "--config", str(tmp_path / "none.conf")]) == 0 def test_check_fails_above_the_mark(monkeypatch, tmp_path): at(monkeypatch, 10_000_000) monkeypatch.setattr(reclaim, "drop_caches", lambda: pytest.fail("reclaimed")) assert reclaim.main(["--check", "--config", str(tmp_path / "none.conf")]) == 1 def test_a_plain_run_below_the_mark_changes_nothing(monkeypatch, tmp_path): at(monkeypatch, 1000) monkeypatch.setattr(reclaim, "drop_caches", lambda: pytest.fail("reclaimed")) assert reclaim.main(["--config", str(tmp_path / "none.conf")]) == 0 def test_force_reclaims_below_the_mark(monkeypatch, tmp_path): calls = [] at(monkeypatch, 1000) monkeypatch.setattr(reclaim, "drop_caches", lambda: calls.append(1) or True) monkeypatch.setattr(reclaim.os, "geteuid", lambda: 0) assert reclaim.main(["--force", "--config", str(tmp_path / "none.conf")]) == 0 assert calls == [1] def test_force_without_root_fails_loudly_rather_than_pretending(monkeypatch, tmp_path): """The old fallback allocated tens of gigabytes to fake memory pressure. Saying so and exiting nonzero is better than quietly costing the machine its memory.""" at(monkeypatch, 1000) monkeypatch.setattr(reclaim, "drop_caches", lambda: pytest.fail("reclaimed")) monkeypatch.setattr(reclaim.os, "geteuid", lambda: 1000) assert reclaim.main(["--force", "--config", str(tmp_path / "none.conf")]) == 1 # ------------------------------------------------------------------------- the installer def test_install_writes_the_script_the_unit_and_a_config(tmp_path): target = str(tmp_path / "sbin" / "suvi-reclaimd") service = str(tmp_path / "suvi-reclaimd.service") config = str(tmp_path / "suvi-reclaimd.conf") reclaim.install(target, service, config) assert os.access(target, os.X_OK) assert open(target).read() == open(reclaim.__file__).read() unit = open(service).read() assert "--daemon" in unit and target in unit # The interpreter must be the system one, not the venv: the venv lives on the very # mount this recovers, and cannot be executed when that mount is refusing opens. assert "/usr/bin/python3 -I" in unit # The config it writes has to be one it can read back. assert reclaim.load_config(config) == {"ceiling": reclaim.DEFAULT_CEILING} def test_install_never_clobbers_a_tuned_config(tmp_path): """The operator raises `high` from what the heartbeat reports. Reinstalling after an update must not silently put that back to the default.""" config = str(tmp_path / "suvi-reclaimd.conf") with open(config, "w") as handle: handle.write("high = 912345\n") reclaim.install(str(tmp_path / "sbin" / "d"), str(tmp_path / "u"), config) assert reclaim.load_config(config) == {"high": 912345} def test_install_refuses_to_overwrite_the_original(tmp_path): with pytest.raises(SystemExit, match="refusing"): reclaim.install(reclaim.__file__, str(tmp_path / "unit"))