From be30ddc9af2b39b9d6cb4796d7af21699c6b819f Mon Sep 17 00:00:00 2001 From: Jeremy Karst Date: Mon, 12 Oct 2026 00:14:10 -0400 Subject: [PATCH] Tail-check duplicates, re-measure benchmarks and fix CI's release-only test failures - Duplicates tab: groups are confirmed by each file's last 8 KiB and keyed by head+tail; unchecked files are never shown. False-positive duplicates drop from 3.4% to 0.1%, and indexing is unchanged. - Encrypted search cache cap raised to 192 MiB at 150 B per file: a keystroke at 1M files takes 59 ms, down from 99 ms. - Case-insensitive filename matching uses a memchr2 scan (1.18x). - Benches run again on the compact index, seed with the indexer's 2 GiB WAL cap (peak scratch 6.2 GB -> 2.45 GB), and every recorded figure is re-measured. - A fresh index no longer logs a rebuild warning. - CI: the sandbox memory test's allocation survives release builds, and --toggle retries an interrupted read. --- DESIGN.md | 18 +- config_example.toml | 33 +- crates/quicksearch-core/Cargo.toml | 5 +- crates/quicksearch-core/benches/arms/mod.rs | 8 +- crates/quicksearch-core/benches/index.rs | 10 +- .../quicksearch-core/benches/page_geometry.rs | 22 +- crates/quicksearch-core/benches/search.rs | 56 +- .../quicksearch-core/benches/search_perf.rs | 74 +-- crates/quicksearch-core/examples/hashprobe.rs | 133 ++++- crates/quicksearch-core/src/config/mod.rs | 35 +- crates/quicksearch-core/src/config/tests.rs | 32 +- crates/quicksearch-core/src/db/open.rs | 39 +- crates/quicksearch-core/src/db/open_tests.rs | 46 +- crates/quicksearch-core/src/db/schema.rs | 152 ++--- .../src/file_handling/records.rs | 9 +- crates/quicksearch-core/src/indexing/mod.rs | 1 + .../quicksearch-core/src/indexing/pipeline.rs | 43 +- crates/quicksearch-core/src/indexing/tests.rs | 47 ++ crates/quicksearch-core/src/platform_tests.rs | 4 +- crates/quicksearch-core/src/query/pattern.rs | 189 +++++- .../quicksearch-core/src/search/duplicates.rs | 544 ++++++++++++++++-- crates/quicksearch-core/src/search/mod.rs | 14 +- crates/quicksearch-core/src/security.rs | 4 +- crates/quicksearch-core/src/testutil.rs | 40 +- crates/quicksearch-core/tests/search_cache.rs | 6 +- crates/quicksearch-gui/src/activate.rs | 20 +- crates/quicksearch-gui/src/app.rs | 26 +- crates/quicksearch-gui/src/app/tests.rs | 4 +- crates/quicksearch-gui/src/app/verify.rs | 8 +- crates/quicksearch-gui/src/backend.rs | 214 ++++++- crates/quicksearch-gui/src/duplicates_tab.rs | 88 +-- .../src/duplicates_tab/tests.rs | 142 ++++- crates/quicksearch-gui/src/help_tab.rs | 8 +- crates/quicksearch-gui/src/settings_tab.rs | 7 +- .../quicksearch-gui/src/settings_tab/tests.rs | 4 +- crates/quicksearch-gui/src/tips.rs | 12 +- crates/quicksearch-gui/src/tutorial.rs | 4 +- 37 files changed, 1687 insertions(+), 414 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 24b7130..4f79e62 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -101,13 +101,19 @@ readable with the old password. changes again. Sub-second times would cost a rebuild and gain nothing on FAT, which keeps two-second times, and the size already catches appends and most replacements. -- Duplicate groups are a strong suspicion, not a certainty: grouping hashes +- Duplicate groups are a strong suspicion, not a certainty: indexing hashes only each file's size and head, which the walk reads anyway to detect the - file's type, so it costs little. The Duplicates tab compares files byte for - byte on request and never deletes or modifies them; it tells you where to - look. Hard links and bind-mounted copies of one file group as copies, and - deleting one frees nothing; verification calls them identical, which they - are. + file's type, so it costs little. Only for the groups it lists, the + Duplicates tab also reads each member's tail, and a listed group is the + files sharing both head and tail, so the extra reads are paid where + precision matters. On 86k real files at 8 KiB, the head alone misgroups + 3.4% of them and head and tail 0.1%. Nothing is listed before its tail is + checked: a file whose tail cannot be read is left out, and so is a group + it leaves with one file. A hidden group is remembered by head and tail. The + Duplicates tab compares files byte for byte on request and never deletes or + modifies them; it tells you where to look. Hard links and bind-mounted + copies of one file group as copies, and deleting one frees nothing; + verification calls them identical, which they are. - Only one instance may open an index for writing where the filesystem supports locks. A second launch raises the running window instead, and refuses if it cannot rather than risk corruption; where the filesystem diff --git a/config_example.toml b/config_example.toml index 052bbc5..ee2c3d1 100644 --- a/config_example.toml +++ b/config_example.toml @@ -71,10 +71,13 @@ ignore_patterns = [".git", "node_modules", "*.tmp", ".venv", "venv"] [processing] # Bytes read from the start of each file to hash it (with its size) for # duplicate detection and to detect its type. Files of equal size with -# identical heads are reported as duplicates even if they differ later; -# verify on the Duplicates tab before deleting. 8192 is the default; 16384 -# is also reasonable. Changing it rebuilds the index. Clamped to -# 262..1048576. +# identical heads hash alike: 3.4% of the files on a measured disk at 8192, +# 1.6% at 16384. The Duplicates tab also reads this many bytes from the end +# of each file it lists and splits apart files whose ends differ, which +# brings those to 0.1% and 0.06%. Files that differ only in between are +# still listed together, so verify on the Duplicates tab before deleting. +# 8192 is the default; 16384 is also reasonable. Changing it rebuilds the +# index. Clamped to 262..1048576. hash_length = 8192 # Most extracted text stored per file (bytes). Clamped to 1..16777216. maximum_text_size = 262144 @@ -178,15 +181,15 @@ debounce_ms = 150 live_results = true # Memory, in MiB, the search connection keeps index pages in, held while # searching and released after a long idle. 0 (recommended) sizes it from the -# index: about 168 bytes per indexed file, capped at 128 MiB. +# index: about 150 bytes per indexed file, capped at 192 MiB. # # It matters for an encrypted index, which must decrypt every page the cache -# misses (at 600k files: 127 ms per keystroke with too small a cache, 34 ms -# with enough). An unencrypted index gets a flat 16 MiB; more measured no -# faster. +# misses (at 1M files: 99 ms per keystroke with too small a cache, 59 ms with +# enough). An unencrypted index gets a flat 16 MiB to save memory, though one +# large enough to hold its file list measured 10 to 25% faster. # # Set it by hand only for deeply nested folders (wider rows) or indexes over -# ~800k files, where the automatic cap is too low. Clamped to 16..1024. +# ~1.3M files, where the automatic cap is too low. Clamped to 16..1024. cache_size_mib = 0 # Which columns the Search tab shows; the right-click menu of any column @@ -219,9 +222,11 @@ rank = true # copies, this is the key to use. exclude_patterns = [] # Groups dismissed with "Hide this group", for groups that only look like -# duplicates, since grouping reads just each file's size and first -# processing.hash_length bytes. Stored by content hash (lowercase hex), so a -# hidden group stays hidden when its files move and returns if their contents -# change. Changing processing.hash_length rebuilds the index and strands these -# entries; the tab's Clear button empties the list. +# duplicates, since grouping reads just each file's size and its first and +# last processing.hash_length bytes. Stored by content (the head hash, then +# the tail digest, in lowercase hex), so a hidden group stays hidden when its +# files move and returns if their contents change. Entries saved by earlier +# versions hold the head hash alone and no longer match any group. Changing +# processing.hash_length rebuilds the index and strands these entries; the +# tab's Clear button empties the list. hidden_groups = [] diff --git a/crates/quicksearch-core/Cargo.toml b/crates/quicksearch-core/Cargo.toml index 745232f..4a96237 100644 --- a/crates/quicksearch-core/Cargo.toml +++ b/crates/quicksearch-core/Cargo.toml @@ -86,8 +86,9 @@ regex = "1" # regex passes ever see. Transitive already (regex). regex-syntax = "0.8" # SIMD substring search for the full-text passes: std's Two-Way searcher has -# no vector prefilter (`benches/search.rs`, `substring`). Transitive already -# (regex, globset, chardetng). +# no vector prefilter (`benches/search.rs`, `substring`). `memchr2` also finds +# the filename pass's case-insensitive candidates (`filename_ladder`). +# Transitive already (regex, globset, chardetng). memchr = "2" # `Local`, for the user's calendar days in `modified:`: the OS zone rules # (tzdata on Unix, the Win32 API on Windows) place midnight, DST gaps and diff --git a/crates/quicksearch-core/benches/arms/mod.rs b/crates/quicksearch-core/benches/arms/mod.rs index 062d68f..813e3dc 100644 --- a/crates/quicksearch-core/benches/arms/mod.rs +++ b/crates/quicksearch-core/benches/arms/mod.rs @@ -107,7 +107,13 @@ pub fn header(arm: &Arm) -> i64 { /// produced it. The middle two rows separate those. pub fn attribution(arm: &Arm) { let like = format!("%{}%", BODY_TERM); - let match_expr = format!("text: \"{}\"", BODY_TERM); + // Built the way the cascade builds it: a quoted phrase is refused by the + // default `detail=none` index, so it is an AND of trigrams there. + let match_expr = quicksearch_core::search::prefilter::fts_literal( + BODY_TERM, + db::repo::fts_positions(&arm.open_search()), + ) + .expect("BODY_TERM clears the trigram floor"); let shapes: [(&str, &str, &str); 4] = [ ( diff --git a/crates/quicksearch-core/benches/index.rs b/crates/quicksearch-core/benches/index.rs index cec4f71..2748640 100644 --- a/crates/quicksearch-core/benches/index.rs +++ b/crates/quicksearch-core/benches/index.rs @@ -103,9 +103,9 @@ mod text_pipeline { } } -/// The MIME sniff, once per new or changed file: ~180 ns on a text head -/// against ~2.2 µs for the `classify` next to it, so its allocations are not -/// worth removing. This group keeps that ratio visible. +/// The MIME sniff, once per new or changed file: ~120 ns on a text head +/// against ~2.5–3.3 µs for the `classify` next to it, so its allocations are +/// not worth removing. This group keeps that ratio visible. mod mime_sniff { use super::*; use std::path::Path; @@ -302,8 +302,8 @@ mod directory_event { /// The same directory's own `Modify`: a `chmod`, a `touch`, an xattr, one /// of the directory events a `chmod -R` sends, and on Windows any change /// among its entries. Its files report their own changes, so it walks - /// nothing. Measured: 1.16 ms → 1.13 µs median; before, it walked the - /// tree as a `Create` does, at `unchanged`'s cost. + /// nothing. Measured: 1.16 ms → about 761 ns median; before, it walked + /// the tree as a `Create` does, at `unchanged`'s cost. #[divan::bench] fn modified(bencher: Bencher) { let (dir, db) = Scratch::db("bench-modified-event"); diff --git a/crates/quicksearch-core/benches/page_geometry.rs b/crates/quicksearch-core/benches/page_geometry.rs index 57f558a..ac3e9d0 100644 --- a/crates/quicksearch-core/benches/page_geometry.rs +++ b/crates/quicksearch-core/benches/page_geometry.rs @@ -11,10 +11,11 @@ //! cargo bench -p quicksearch-core --bench page_geometry //! ``` //! -//! Point `TMPDIR` at real storage; see `arms::warn_if_tmpdir_unset`. The -//! matrix wanted ~2 GB of scratch at a time, since each arm is dropped once -//! measured; that predates the seed's uncapped WAL (`testutil::seed_index`), -//! which needs room on top. +//! Point `TMPDIR` at real storage; see `arms::warn_if_tmpdir_unset`. Each +//! arm is dropped once measured, so scratch peaks at one 1M-file arm, about +//! 2.4 GB: its ~300 MiB database plus a WAL that `testutil::seed_index` +//! checkpoints as a default-config run does, at `maximum_wal_size` (2 GiB; +//! less on a volume with under ~8 GiB free). //! //! # Settled: FTS5 record size //! @@ -48,11 +49,8 @@ //! //! The plain arm is the noise floor. Cold is SQLite-cold: a fresh //! connection, so its cache starts empty. Nothing evicts the OS cache, so no -//! tier is storage-cold. Warm runs at the cache ceiling each arm ships with; -//! the sweep on `db::schema::PAGE_SIZE` predates that and ran at the 16 MiB -//! floor, below the `files` table, so its warm ranking prices per-page work, -//! not a warm keystroke at the shipped ceiling. Report the tiers separately; -//! do not average them. +//! tier is storage-cold. Warm runs at the cache ceiling each arm ships with. +//! Report the tiers separately; do not average them. mod arms; @@ -66,9 +64,9 @@ use quicksearch_core::testutil::{Arm, SeedSpec}; /// room. const SWEPT: [i64; 7] = [1024, 2048, 4096, 8192, 16384, 32768, 65536]; -/// `QSB_PGSZ_SIZES=8192,16384` narrows the sweep. A full run takes ~45 -/// minutes, nearly all of it seeding, so re-asking one question should not -/// re-ask all of them. +/// `QSB_PGSZ_SIZES=8192,16384` narrows the sweep. A full run takes about 5 +/// minutes, most of it seeding, so re-asking one question should not re-ask +/// all of them. fn swept() -> Vec { match std::env::var("QSB_PGSZ_SIZES") { Ok(list) => list diff --git a/crates/quicksearch-core/benches/search.rs b/crates/quicksearch-core/benches/search.rs index 0e0152e..105b37a 100644 --- a/crates/quicksearch-core/benches/search.rs +++ b/crates/quicksearch-core/benches/search.rs @@ -32,7 +32,7 @@ fn literal(term: &str) -> TermPattern { } /// Decompressing the stored body, the first thing every full-text row does. -/// Reusing one context and pre-sizing the output is ~4x faster at 1 KiB, the +/// Reusing one context and pre-sizing the output is 5.1x faster at 1 KiB, the /// size most documents are; `DocDecoder` exists for that. mod zstd_decode { use super::*; @@ -54,7 +54,7 @@ mod zstd_decode { /// Turning decompressed bytes into a `&str`: `from_utf8_lossy` copies and /// scan-validates; `String::from_utf8` vector-validates and *moves*. A -/// 16-28x gap; `DocDecoder` borrows instead. +/// 14-16x gap; `DocDecoder` borrows instead. mod utf8 { use super::*; @@ -116,8 +116,8 @@ mod fold { /// Substring search: std's Two-Way against `memmem`'s SIMD prefilter. The /// miss case matters most (a pass verifies far more rows than it accepts): -/// 111 µs against 2.4 µs at 256 KiB. `match_indices` stays as the guard: if -/// the two converge, the SIMD path has stopped being selected. +/// 92 µs against 4.7 µs at 256 KiB (20x). `match_indices` stays as the +/// guard: if the two converge, the SIMD path has stopped being selected. mod substring { use super::*; @@ -134,8 +134,9 @@ mod substring { bencher.bench(|| finder.find_iter(divan::black_box(text)).count()); } - /// A `Finder` built per call: measured indistinguishable from a hoisted - /// one, since the precompute is O(needle) and a term is a few bytes. + /// A `Finder` built per call: +14 ns against a hoisted one at 1 KiB and + /// indistinguishable from 16 KiB up, since the precompute is O(needle) + /// and a term is a few bytes. #[divan::bench(args = corpus::SIZES)] fn memmem_per_call_miss(bencher: Bencher, size: usize) { let text = corpus::text(size, 0).as_bytes(); @@ -226,10 +227,19 @@ mod snippet_extract { } } -/// The filename pass's per-row ladder over 2000 realistic rows. The two -/// obvious fixes each measure *worse* alone (reused fold buffer, prebuilt -/// `Finder`); only together do they win, and only by ~1.2x: short strings -/// do not behave like document bodies. +/// The filename pass's per-row ladder over 2000 realistic rows. `current` is +/// the shipped `find_first`, which jumps between spellings of the term's +/// first byte with `memchr2`. Adopting that took it from 53.4 µs to 45.4 µs, +/// 1.18x (2026-10-11, mimalloc, fastest sample, median of 3 runs), with the +/// other arms within 4%. Folding first does not catch up: the two obvious +/// fixes each measure *worse* alone (reused fold buffer, prebuilt `Finder`), +/// and together they only tie it. Short strings do not behave like document +/// bodies. +/// +/// The other arms are local copies, which skip `find_first`'s per-call cost +/// (about 1.5 ns, 6 µs a run). The `memchr2` copy that prompted the change +/// measured 1.39x where the shipped call gained 1.18x, so gate a change on +/// `current` before and after it, not on a copy. mod filename_ladder { use super::*; @@ -324,26 +334,22 @@ mod filename_ladder { }); } - /// memchr2 candidate scan: measured, not worth it on short haystacks. + /// The per-byte scan `find_first` ran until 2026-10-11: every start + /// tested, `needle` already lowercased. The `memchr2` scan that replaced + /// it was first rejected under glibc as not worth it on short haystacks; + /// re-measured under mimalloc, its copy here ran 38.5 µs against + /// `current`'s 53.4 µs. This copy runs 47.9 µs. #[divan::bench] - fn find_first_ci_memchr2(bencher: Bencher) { + fn find_first_ci_bytewise(bencher: Bencher) { let rows = corpus::rows(); let needle = corpus::NEEDLE.as_bytes(); fn find(hay: &[u8], needle: &[u8]) -> Option { - let (lo, up) = ( - needle[0].to_ascii_lowercase(), - needle[0].to_ascii_uppercase(), - ); + let first = needle[0]; let last = hay.len().checked_sub(needle.len())?; - let mut at = 0usize; - while at <= last { - let i = at + memchr::memchr2(lo, up, &hay[at..=last])?; - if hay[i..i + needle.len()].eq_ignore_ascii_case(needle) { - return Some(i); - } - at = i + 1; - } - None + (0..=last).find(|&i| { + hay[i].to_ascii_lowercase() == first + && hay[i..i + needle.len()].eq_ignore_ascii_case(needle) + }) } bencher.bench(|| { let mut found = 0usize; diff --git a/crates/quicksearch-core/benches/search_perf.rs b/crates/quicksearch-core/benches/search_perf.rs index ee72864..b2066d4 100644 --- a/crates/quicksearch-core/benches/search_perf.rs +++ b/crates/quicksearch-core/benches/search_perf.rs @@ -15,8 +15,6 @@ //! characters, unpredicated below that). So the working set scales with file //! count, not document volume, and the corpus dimension below shows it. //! -//! Keyed figures predate `HMAC_MODE = Off`; see `db::schema::HMAC_MODE`. -//! //! Printed rather than asserted: shared-box timings are not stable enough for //! a pass/fail gate, and a flaky perf gate gets muted rather than fixed. //! @@ -25,47 +23,47 @@ //! cargo bench -p quicksearch-core --bench search_perf //! ``` //! -//! Point `TMPDIR` at real storage (see `arms::warn_if_tmpdir_unset`); the -//! 1M-file arms were ~580 MB each, before the seed's uncapped WAL -//! (`testutil::seed_index`), which needs room on top. Budget ~20 minutes, -//! nearly all of it seeding. +//! Point `TMPDIR` at real storage (see `arms::warn_if_tmpdir_unset`). One arm +//! is held at a time, and a 1M-file arm peaks at about 2.5 GB of scratch: its +//! 344 MiB database plus a WAL that `testutil::seed_index` checkpoints as a +//! default-config run does, at `maximum_wal_size` (2 GiB; less on a volume +//! with under ~8 GiB free). Budget about 2 minutes, most of it seeding. //! //! # What it found //! -//! Warm search, the mean keystroke of one settled pass, -//! `schema::PAGE_SIZE` = 8192, rows at a realistic width (139 B; see -//! [`spec`]), on the earlier fixed grid of 1–256 MiB (powers of two plus 24, -//! 48 and 96): +//! Warm search, the mean keystroke of one settled pass, medians of 3 runs, +//! `schema::PAGE_SIZE` = 8192, `HMAC_MODE = Off`, rows at a realistic width +//! (137 B; see [`spec`]), on [`cache_sizes`]'s grid (2026-10-11): //! //! | corpus | `files` table | keyed, under knee | keyed, at knee | knee | ratio | //! |---|---|---|---|---|---| -//! | 200k | 26.5 MiB | 41.4 ms (≤24 MiB) | **10.7 ms** | 32 MiB | ≤1.21x | -//! | 600k | 79.5 MiB | 121.2 ms (≤64 MiB) | **33.7 ms** | 96 MiB | ≤1.21x | -//! | 1M | 132.4 MiB | 199.6 ms (≤128 MiB) | **58.3 ms** | 256 MiB | ≤1.93x | +//! | 200k | 26.1 MiB | 18.1–18.4 ms (≤1.0x) | **12.1 ms** | 27.4 MiB | 1.05x | +//! | 600k | 78.3 MiB | 57.8–61.3 ms (≤1.0x) | **35.6 ms** | 82.2 MiB | 1.05x | +//! | 1M | 130.4 MiB | 85.3–99.0 ms (≤1.0x) | **60.0 ms** | 136.9 MiB | 1.05x | //! -//! 1. The knee is at most 1.21x the `files` table: the grid brackets it in -//! (24, 32] MiB at 200k and (64, 96] at 600k, and the two agree by -//! construction, since corpus and grid both step by 3. 1M's lies in -//! (128, 256], where 1.21x predicts 160. Hence 139 B/row × 1.21 = 168 B, -//! `schema::SEARCH_CACHE_BYTES_PER_FILE`, which errs large. -//! 2. Below the knee an encrypted index is 3.4–3.9x slower, and the step is a -//! cliff, not a slope: 600k keyed ran 121–130 ms at every ceiling up to -//! 64 MiB and 33.7 ms at 96. -//! 3. Plain has no knee. Its widest spread was 1.35x and it is not monotonic -//! (32 and 48 MiB measured slower than 1 MiB at 600k): run-to-run noise, -//! not a curve. A miss on a plain index is a `memcpy` from the OS cache; on -//! a keyed one it is an AES-CBC decrypt. Hence -//! `schema::SEARCH_CACHE_PLAIN_MIB`, flat. +//! 1. The knee is at 1.05x the `files` table on every corpus: the table +//! itself is under it. Hence 137 B/row × 1.05 = 144 B, plus about 4% of +//! margin: 150 B, `schema::SEARCH_CACHE_BYTES_PER_FILE`. +//! 2. Below the knee an encrypted index is 1.5–1.7x slower (3.4–3.9x under +//! HMAC-SHA512), and the step is a cliff, not a slope: 600k keyed ran +//! 58–61 ms at every ceiling up to the table and 35.6 ms at 1.05x. +//! 3. Plain has no cliff, but it runs 10–25% faster once the cache holds the +//! `files` table: 13.0 → 10.7–12.1 ms at 200k, 43.0 → 34.3–36.6 ms at +//! 600k, 63.6 → 57.5–59.2 ms at 1M, from its flat 16 MiB to 1.05x the +//! table and up. A miss on a plain index is a `memcpy` from the OS cache; +//! on a keyed one it is an AES-CBC decrypt. `schema::SEARCH_CACHE_PLAIN_MIB` +//! stays flat by choice, for memory. //! //! Row width is half the answer. The default seed's narrow rows (`hash` NULL, //! a 16-character `/seed/NNN/` parent) measure 69.5 B, half of a realistic //! 139 B; calibrating on them would under-size every cache by two and put //! every user back under the knee. //! -//! The 128 MiB automatic cap (`schema::SEARCH_CACHE_MAX_MIB`) binds at ~800k -//! files. At 1M the index wants ~160 MiB and gets 128: 205 ms per keystroke -//! against the 58 ms available at 256. `[search] cache_size_mib` is how a -//! user trades memory for that speed. +//! The 192 MiB automatic cap (`schema::SEARCH_CACHE_MAX_MIB`) binds at +//! ~1.34M files, and the knee passes it at ~1.4M. The cap was 128 MiB, under +//! 1M's knee: 99.0 ms per keystroke there, against 58.9 ms at the 143 MiB it +//! now derives. Past ~1.4M, `[search] cache_size_mib` is how a user trades +//! memory for that speed. mod arms; @@ -75,9 +73,9 @@ use quicksearch_core::db::schema::recommended_search_cache_mib; use quicksearch_core::search::SearchOptions; use quicksearch_core::testutil::{run_cascade, Arm, SeedSpec}; -/// Two corpora under the automatic cap and one past it -/// (`schema::SEARCH_CACHE_MAX_MIB` binds at ~800k files). If the working set -/// is `files`, the knee moves with the corpus. +/// Three corpora under the automatic cap (`schema::SEARCH_CACHE_MAX_MIB` +/// binds at ~1.34M files). If the working set is `files`, the knee moves with +/// the corpus. const CORPORA: [usize; 3] = [200_000, 600_000, 1_000_000]; /// Cache ceilings to sweep, as multiples of the arm's own `files` table. The @@ -122,9 +120,13 @@ const SEQUENCE: [&str; 4] = ["quar", "quart", "quartz", "quartzi"]; const KNEE_TOLERANCE: f64 = 1.10; /// How much worse the worst ceiling must be than the best before the curve -/// counts as a knee. A real one is a 4x step (keyed at 200k: 41.4 ms below -/// 32 MiB, 10.7 ms at or above). [`KNEE_TOLERANCE`] alone would find a knee -/// in a plain arm's ~1.3x run-to-run noise, so this gate sits well above it. +/// counts as a knee. A real one was a 3.4–3.9x step under HMAC-SHA512 and is +/// 1.5–1.7x with `HMAC_MODE = Off` (keyed at 200k: 18.1 ms at the table, +/// 12.1 ms at 1.05x), right at this gate. A plain arm's spread, its 10–25% +/// gain once the table fits plus noise, measured 1.13–1.65x, so the two +/// overlap: across the 2026-10-11 runs one keyed arm missed its knee at 1.50x +/// and two plain arms reported one. Read a knee off several runs, not one. +/// [`KNEE_TOLERANCE`] alone would find a knee in every plain arm. const KNEE_MIN_SPREAD: f64 = 1.5; fn enabled() -> bool { diff --git a/crates/quicksearch-core/examples/hashprobe.rs b/crates/quicksearch-core/examples/hashprobe.rs index 776f79e..92f4fb5 100644 --- a/crates/quicksearch-core/examples/hashprobe.rs +++ b/crates/quicksearch-core/examples/hashprobe.rs @@ -4,7 +4,9 @@ //! so one number decides both. Collisions are measured over `1..32 KiB` //! against the 32 KiB baseline, which is right by definition; `--verify` //! reads the baseline's groups byte for byte and reports how many of those -//! were wrong too. +//! were wrong too. The `+tail` columns apply the Duplicates tab's tail check +//! (`search::split_by_tail`) at each sample size, the tail as long as the +//! head. //! //! ```text //! cargo build -p quicksearch-core --example hashprobe --release @@ -25,7 +27,7 @@ mod common; use std::collections::HashMap; use std::fs::File; -use std::io::Read; +use std::io::{Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -68,6 +70,10 @@ struct Scan { size: u64, /// `sha256(size ‖ head[..n])` for each `n` in [`SWEEP`]. digests: [Sha; SWEEP.len()], + /// For each `n`, `sha256` of the bytes `search::split_by_tail` reads + /// with a tail of `n`: from `max(n, size - n)` to the end. Zero for a + /// file no larger than `n`, which the head hashed whole. + tails: [Sha; SWEEP.len()], /// Bit `i`: the MIME guessed from `SWEEP[i]` bytes differs from the /// baseline's, since a short sample also classifies on less evidence. mime_differs: u8, @@ -87,22 +93,43 @@ struct ScanErrors { irregular: u64, } -/// Read one file's head (`min(size, 32 KiB)`) once and derive every sample -/// size's answer from it, so the six answers are consistent with each other -/// as six separate runs could not be. +/// Read one file's head (`min(size, 32 KiB)`) and the tail past it (at most +/// the last 32 KiB) once, and derive every sample size's answer from them, so +/// the six answers are consistent with each other as six separate runs could +/// not be. fn scan_one(path: &Path, size: u64, config: &Config, registry: &Registry) -> Option { - let mut head = vec![0u8; size.min(SWEEP[BASE] as u64) as usize]; + let base = SWEEP[BASE] as u64; + let mut head = vec![0u8; size.min(base) as usize]; let mut file = File::open(path).ok()?; // `read_exact`, like the indexer: a file that shrank between the stat and // this read is not a file we know the size of, so it is not scanned. file.read_exact(&mut head).ok()?; + // Every smaller sample's tail lies inside this one or the head: below + // `2 × base` the two are the whole file, and above it every tail is a + // suffix of this one. + let rest_start = base.max(size.saturating_sub(base)); + let mut rest = vec![0u8; size.saturating_sub(rest_start) as usize]; + if !rest.is_empty() { + file.seek(SeekFrom::Start(rest_start)).ok()?; + file.read_exact(&mut rest).ok()?; + } let mut digests = [[0u8; 32]; SWEEP.len()]; - for (slot, n) in digests.iter_mut().zip(SWEEP) { + let mut tails = [[0u8; 32]; SWEEP.len()]; + for ((slot, tail), n) in digests.iter_mut().zip(&mut tails).zip(SWEEP) { let mut hasher = Sha256::new(); hasher.update(size.to_le_bytes()); hasher.update(&head[..head.len().min(n)]); slot.copy_from_slice(&hasher.finalize()); + + let n = n as u64; + if size > n { + let start = n.max(size - n); + let mut hasher = Sha256::new(); + hasher.update(&head[head.len().min(start as usize)..]); + hasher.update(&rest[start.saturating_sub(rest_start) as usize..]); + tail.copy_from_slice(&hasher.finalize()); + } } let base_mime = guess_mime_from_head(path, &head); @@ -124,6 +151,7 @@ fn scan_one(path: &Path, size: u64, config: &Config, registry: &Registry) -> Opt Some(Scan { size, digests, + tails, mime_differs, text_differs, extractable, @@ -322,6 +350,20 @@ fn partition(scans: &[Scan], which: usize) -> Vec> { by_digest.into_values().collect() } +/// [`partition`], each group then split by its members' tails as +/// `search::split_by_tail` splits it with `tail_len = SWEEP[which]`. The +/// size is already in the head digest. +fn partition_with_tail(scans: &[Scan], which: usize) -> Vec> { + let mut by_digest: HashMap<(Sha, Sha), Vec> = HashMap::new(); + for (i, scan) in scans.iter().enumerate() { + by_digest + .entry((scan.digests[which], scan.tails[which])) + .or_default() + .push(i as u32); + } + by_digest.into_values().collect() +} + fn baseline_ids(scans: &[Scan]) -> Vec { let mut ids = vec![0u32; scans.len()]; let mut next: HashMap = HashMap::new(); @@ -334,21 +376,41 @@ fn baseline_ids(scans: &[Scan]) -> Vec { struct Sweep { rows: Vec, + /// The same, after the tail check. + tailed: Vec, } impl Sweep { /// Assert the invariants that hold by construction: the baseline's /// false-positive columns are zero, and every "wrong" column is - /// non-increasing as the sample widens. A probe that reports a plausible - /// wrong number is worse than none. + /// non-increasing as the sample widens. The tail check only splits, so + /// its rows obey the same and never report more than the head alone + /// (head and tail at `2n` cover what they cover at `n`). A probe that + /// reports a plausible wrong number is worse than none. fn check_invariants(&self) { - let base = &self.rows[BASE]; + Sweep::check_rows(&self.rows); + Sweep::check_rows(&self.tailed); + for (head, tailed) in self.rows.iter().zip(&self.tailed) { + assert!(tailed.files <= head.files, "the tail check only splits"); + assert!( + tailed.fp_files <= head.fp_files, + "the tail check only splits" + ); + assert!( + tailed.fp_pairs <= head.fp_pairs, + "the tail check only splits" + ); + } + } + + fn check_rows(rows: &[Collisions]) { + let base = &rows[BASE]; assert_eq!(base.fp_groups, 0, "the baseline defines truth"); assert_eq!(base.fp_files, 0, "the baseline defines truth"); assert_eq!(base.fp_pairs, 0, "the baseline defines truth"); assert_eq!(base.overstated_bytes, 0, "the baseline defines truth"); - for pair in self.rows.windows(2) { + for pair in rows.windows(2) { let (small, large) = (&pair[0], &pair[1]); assert!( large.files <= small.files, @@ -441,6 +503,9 @@ fn collide(roots: &[String], workers: usize, ignore: &[String], verify_budget: O rows: (0..SWEEP.len()) .map(|i| tally(&partition(&scans, i), &base, &sizes)) .collect(), + tailed: (0..SWEEP.len()) + .map(|i| tally(&partition_with_tail(&scans, i), &base, &sizes)) + .collect(), }; sweep.check_invariants(); @@ -450,12 +515,12 @@ fn collide(roots: &[String], workers: usize, ignore: &[String], verify_budget: O sample_name(SWEEP[BASE]) ); println!( - "| sample | dup groups | dup files | reported pairs | FP groups | FP files | FP pairs | FP rate (pairs) | FP rate (files) | overstated |" + "| sample | dup groups | dup files | reported pairs | FP groups | FP files | FP pairs | FP rate (pairs) | FP rate (files) | overstated | FP files +tail | FP rate (files) +tail | overstated +tail |" ); - println!("|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|"); - for (i, row) in sweep.rows.iter().enumerate() { + println!("|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|"); + for (i, (row, tailed)) in sweep.rows.iter().zip(&sweep.tailed).enumerate() { println!( - "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |", + "| {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} | {} |", sample_name(SWEEP[i]), row.groups, row.files, @@ -466,6 +531,9 @@ fn collide(roots: &[String], workers: usize, ignore: &[String], verify_budget: O pct(row.fp_pairs, row.reported_pairs), pct(row.fp_files, files), mib(row.overstated_bytes), + tailed.fp_files, + pct(tailed.fp_files, files), + mib(tailed.overstated_bytes), ); } @@ -833,6 +901,41 @@ mod tests { assert_eq!(total.overstated_bytes, 20); } + /// Each sample's tail digest covers the bytes `split_by_tail` reads, + /// wherever the size falls against the head and the 32 KiB read past it. + #[test] + fn tail_digests_cover_what_the_tail_check_reads() { + let dir = quicksearch_core::testutil::scratch_dir("hashprobe-tails"); + let (config, registry) = (Config::default(), Registry::default_set()); + let base = SWEEP[BASE]; + for size in [ + 1, + 1024, + 1025, + 3000, + base, + base + 1, + base + 5000, + 2 * base, + 2 * base + 1, + 5 * base + 7, + ] { + let body: Vec = (0..size).map(|i| (i * 31 % 251) as u8).collect(); + let path = dir.join(format!("{size}.bin")); + std::fs::write(&path, &body).unwrap(); + let scan = scan_one(&path, size as u64, &config, ®istry).expect("scanned"); + for (tail, n) in scan.tails.iter().zip(SWEEP) { + let want: Sha = if size > n { + Sha256::digest(&body[n.max(size - n)..]).into() + } else { + [0; 32] + }; + assert_eq!(*tail, want, "size {size}, tail {n}"); + } + } + std::fs::remove_dir_all(&dir).ok(); + } + /// The hardlink-farm case: the pair count is quadratic, so it must be /// computed, not enumerated. #[test] diff --git a/crates/quicksearch-core/src/config/mod.rs b/crates/quicksearch-core/src/config/mod.rs index 0a6acc2..9e6eb13 100644 --- a/crates/quicksearch-core/src/config/mod.rs +++ b/crates/quicksearch-core/src/config/mod.rs @@ -75,7 +75,10 @@ pub struct IndexingConfig { #[serde(default)] pub struct ProcessingConfig { /// Bytes read from the head of each new or changed file: the identity - /// hash window, the MIME sniff window, and the inline-extract cap. + /// hash window, the MIME sniff window, and the inline-extract cap. Also + /// the tail the Duplicates tab reads from each listed file + /// ([`crate::search::split_by_tail`]): at the default 8 KiB, head alone + /// misgroups 3.44% of 86k real files and head and tail 0.10%. /// Changing it invalidates stored hashes and rebuilds the index: at once /// in automatic mode, otherwise when confirmed or at the next run. /// `examples/hashprobe.rs` measures what a different value would trade. @@ -194,13 +197,16 @@ pub struct DuplicatesConfig { /// is not counted, and a group left with fewer than two members is not /// listed at all. pub exclude_patterns: Vec, - /// Content hashes of groups dismissed with "Hide this group", lowercase - /// hex (see [`crate::search::DuplicateGroup::hash_hex`]). Keyed by hash - /// rather than by path so a hidden group stays hidden when its files are - /// renamed or moved, and comes back when their contents change. + /// Identities of groups dismissed with "Hide this group": head hash then + /// tail digest, lowercase hex (see + /// [`crate::search::DuplicateGroup::identity`]). Keyed by content rather + /// than by path so a hidden group stays hidden when its files are renamed + /// or moved, and comes back when their contents change. Anything else, + /// such as the head hash alone that earlier versions saved, is dropped + /// when the config loads. /// - /// The hash covers `processing.hash_length` bytes, so changing that - /// setting (which rebuilds the index anyway) strands every entry here. + /// Both cover `processing.hash_length` bytes, so changing that setting + /// (which rebuilds the index anyway) strands every entry here. pub hidden_groups: Vec, } @@ -375,6 +381,12 @@ fn default_ignore_patterns() -> Vec { patterns.into_iter().map(str::to_string).collect() } +/// Whether `id` is shaped like [`crate::search::DuplicateGroup::identity`]: +/// two SHA-256 digests, head then tail, as 128 lowercase hex digits. +fn is_group_identity(id: &str) -> bool { + id.len() == 128 && id.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')) +} + /// The home directory, else `.`. A home whose name is not valid Unicode /// gives no root: the walk skips such paths silently, and a lossy spelling /// must never become a database key. @@ -548,6 +560,15 @@ impl Config { for warning in cfg.clamp_out_of_range() { crate::log_warn!("config: {}", warning); } + let hidden = cfg.duplicates.hidden_groups.len(); + cfg.duplicates.hidden_groups.retain(|id| is_group_identity(id)); + if cfg.duplicates.hidden_groups.len() < hidden { + crate::log_warn!( + "config: dropped {} hidden duplicate group(s) that are not a head and tail \ + identity (saved by an older version); hide them again if they come back", + hidden - cfg.duplicates.hidden_groups.len() + ); + } IgnoreSet::compile(&cfg.indexing.ignore_patterns) .map_err(|e| format!("Invalid ignore pattern in {}: {}", path.display(), e))?; Ok(cfg) diff --git a/crates/quicksearch-core/src/config/tests.rs b/crates/quicksearch-core/src/config/tests.rs index c88f06a..7b9fdd5 100644 --- a/crates/quicksearch-core/src/config/tests.rs +++ b/crates/quicksearch-core/src/config/tests.rs @@ -36,6 +36,33 @@ fn fresh_install_defaults_to_home_as_only_root() { assert_eq!(partial.paths.indexing_paths, cfg.paths.indexing_paths); } +/// Only head and tail identities survive a load: the head hash alone that +/// earlier versions saved, and anything malformed, would match no group yet +/// still count as hidden. +#[test] +fn hidden_groups_that_are_not_identities_are_dropped_on_load() { + let dir = tmp_dir(); + let path = dir.join("config.toml"); + let identity = "ab".repeat(32) + &"cd".repeat(32); + let stale = [ + "ab".repeat(32), // head hash alone, as saved before + "AB".repeat(64), // uppercase + "zz".repeat(64), // not hex + "ab".repeat(64) + "a", // one digit too many + String::new(), + ]; + let mut entries: Vec = stale.to_vec(); + entries.insert(2, identity.clone()); + let list = entries + .iter() + .map(|e| format!("\"{e}\"")) + .collect::>() + .join(", "); + fs::write(&path, format!("[duplicates]\nhidden_groups = [{list}]\n")).unwrap(); + let cfg = Config::load_from(&path).unwrap(); + assert_eq!(cfg.duplicates.hidden_groups, [identity]); +} + #[test] fn missing_file_created_with_defaults() { let dir = tmp_dir(); @@ -1232,7 +1259,7 @@ fn ui_bookkeeping_fields_are_soft_knobs() { c.duplicates.exclude_patterns = vec!["*.iso".to_string()] }), ("duplicates.hidden_groups", |c| { - c.duplicates.hidden_groups = vec!["ab".repeat(32)] + c.duplicates.hidden_groups = vec!["ab".repeat(64)] }), ]; for (label, mutate) in cases { @@ -1255,7 +1282,8 @@ fn newer_fields_round_trip() { cfg.ui.color_scheme = "light".to_string(); cfg.search.fuzzy_max_edits = 4; cfg.duplicates.exclude_patterns = vec!["*.iso".to_string()]; - cfg.duplicates.hidden_groups = vec!["ab".repeat(32)]; + // A group's identity: its head hash, then its tail digest. + cfg.duplicates.hidden_groups = vec!["ab".repeat(32) + &"cd".repeat(32)]; cfg.save().unwrap(); let loaded = Config::load_from(&path).unwrap(); assert_eq!( diff --git a/crates/quicksearch-core/src/db/open.rs b/crates/quicksearch-core/src/db/open.rs index c1b1cca..e0ff4c0 100644 --- a/crates/quicksearch-core/src/db/open.rs +++ b/crates/quicksearch-core/src/db/open.rs @@ -78,22 +78,28 @@ pub(crate) fn open_or_recreate_keyed( // Neither the page size nor the page reserve changes in place, so a // file that opened only under an older profile is rebuilt whatever // its schema says. - ProfileMatch::Previous(found) => { - format!("was built with {} and this build uses {}", found, want) - } + ProfileMatch::Previous(found) => Some(format!( + "was built with {} and this build uses {}", + found, want + )), ProfileMatch::Current if db_matches_current(&conn, processing)? => { narrow_index_files(&path); conn.execute_batch(PRAGMAS_FAST) .map_err(|e| format!("Failed to apply pragmas: {}", e))?; return Ok(conn); } - ProfileMatch::Current => "does not match current schema".to_string(), + // A path just created, or an empty file: nothing is rebuilt, so + // creating the schema is not worth a warning. + ProfileMatch::Current if holds_no_schema(&conn)? => None, + ProfileMatch::Current => Some("does not match current schema".to_string()), }; - crate::log_warn!( - "database at {} {}; rebuilding. Existing rows will be re-scanned on next indexing run.", - db_path, - why - ); + if let Some(why) = why { + crate::log_warn!( + "database at {} {}; rebuilding. Existing rows will be re-scanned on next indexing run.", + db_path, + why + ); + } let conn = wipe_and_reopen(conn, &path, key, want)?; apply_current_schema(&conn, processing, key, want)?; Ok(conn) @@ -269,9 +275,11 @@ pub fn index_needs_rebuild(db_path: &str) -> bool { if matched != ProfileMatch::Current { return true; } - // Only `Ok` answers count: an `Err` means we could not tell. + // Only `Ok` answers count: an `Err` means we could not tell. An empty + // file is only given the schema, which is no reset. matches!(schema_version_current(&conn), Ok(false)) && matches!(is_ours_or_empty(&conn), Ok(true)) + && matches!(holds_no_schema(&conn), Ok(false)) } /// Serialises SQLCipher's process-wide `cipher_default_*` settings, which @@ -475,6 +483,17 @@ fn is_ours_or_empty(conn: &Connection) -> Result { Ok(!any) } +/// Whether the file holds no schema at all: a path just created, or an empty +/// file. `sqlite_%` names are excluded, as in [`is_ours_or_empty`]. +fn holds_no_schema(conn: &Connection) -> Result { + conn.query_row( + "SELECT NOT EXISTS (SELECT 1 FROM sqlite_master WHERE name NOT LIKE 'sqlite_%')", + [], + |r| r.get(0), + ) + .map_err(|e| format!("read sqlite_master: {}", e)) +} + /// The refusal message, naming a few of the tables in the way. fn foreign_database_message(conn: &Connection) -> Result { let mut stmt = conn diff --git a/crates/quicksearch-core/src/db/open_tests.rs b/crates/quicksearch-core/src/db/open_tests.rs index 2900422..92bb2b3 100644 --- a/crates/quicksearch-core/src/db/open_tests.rs +++ b/crates/quicksearch-core/src/db/open_tests.rs @@ -111,6 +111,45 @@ fn reopen_is_idempotent() { assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string()); } +/// Rebuild warnings in the process log that name `p`. Every scratch path is +/// unique, so lines other tests log at the same time cannot match. +fn rebuild_warnings(p: &str) -> usize { + crate::log::snapshot() + .iter() + .filter(|line| { + line.level == crate::log::Level::Warn + && line.text.contains(p) + && line.text.contains("rebuilding") + }) + .count() +} + +/// A new index has nothing to rebuild, so creating one must not warn of a +/// schema mismatch, as it did in every CI run and on the GUI's first launch. +/// A real mismatch still warns. +#[test] +fn only_a_real_schema_mismatch_warns_of_a_rebuild() { + let (_dir, fresh) = db(); + drop(open_or_recreate(&fresh, &Default::default()).unwrap()); + assert_eq!(rebuild_warnings(&fresh), 0, "a path that did not exist"); + + let (_dir, keyed) = db(); + drop(open_or_recreate_keyed(&keyed, &Default::default(), Some(&test_key(0xa3))).unwrap()); + assert_eq!(rebuild_warnings(&keyed), 0, "a new encrypted index"); + + let (_dir, empty) = db(); + std::fs::write(&empty, b"").unwrap(); + let conn = open_or_recreate(&empty, &Default::default()).unwrap(); + assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string()); + assert_eq!(rebuild_warnings(&empty), 0, "an empty file"); + + let (_dir, old) = db(); + write_v1_index(&old, "a.txt"); + let conn = open_or_recreate(&old, &Default::default()).unwrap(); + assert_eq!(file_count(&conn), 0, "the mismatch is still rebuilt"); + assert_eq!(rebuild_warnings(&old), 1, "a real schema mismatch"); +} + #[test] fn older_versioned_db_is_wiped_and_recreated() { let (_dir, p) = db(); @@ -391,8 +430,13 @@ type Fixture = fn(&str); /// announces a wipe that is not happening. #[test] fn index_needs_rebuild_only_when_the_schema_really_differs() { - let cases: [(Fixture, bool, &str); 7] = [ + let cases: [(Fixture, bool, &str); 8] = [ (|_| {}, false, "no file yet is a fresh install, not a reset"), + ( + |p| std::fs::write(p, b"").unwrap(), + false, + "an empty file holds nothing to reset", + ), ( |p| drop(open_or_recreate(p, &Default::default()).unwrap()), false, diff --git a/crates/quicksearch-core/src/db/schema.rs b/crates/quicksearch-core/src/db/schema.rs index 68d3607..0b8de8d 100644 --- a/crates/quicksearch-core/src/db/schema.rs +++ b/crates/quicksearch-core/src/db/schema.rs @@ -55,22 +55,25 @@ pub const PRAGMAS_INCREMENTAL: &str = " /// Cache ceiling for an unencrypted index, whatever its size. /// -/// `benches/search_perf.rs` swept 1–256 MiB on 200k–1M files and found no -/// knee on a plain index, where a page-cache miss is a `memcpy` from the OS -/// cache. Only SQLCipher makes a miss expensive: it caches pages decrypted, -/// so a miss pays the AES-CBC again. +/// `benches/search_perf.rs` swept 1–256 MiB on 200k–1M files. A plain index, +/// where a page-cache miss is a `memcpy` from the OS cache, has no cliff, but +/// it runs 10–25% faster once the cache holds the `files` table (600k: 43.0 ms +/// per keystroke at 16 MiB, 34.4 ms at 1.2x the table). It is kept at a flat +/// 16 MiB by choice, for memory. Only SQLCipher makes a miss expensive: it +/// caches pages decrypted, so a miss pays the AES-CBC again. pub const SEARCH_CACHE_PLAIN_MIB: i64 = 16; /// Floor and ceiling for the derived value. The floor is the smallest the /// sweep found sufficient. /// -/// The cap limits resident memory and binds at about 800k files, beyond which -/// an encrypted index sits under its knee: at 1M files (a 132 MiB `files` -/// table) a keystroke costs 205 ms at 128 MiB against 58 ms at 256. Trading a -/// third of a gigabyte for a 3.5x faster session is the user's call, through -/// [`SEARCH_CACHE_OVERRIDE_MAX_MIB`]. +/// The cap limits resident memory. It binds at about 1.34M files, and the knee +/// (1.05x the `files` table, about 144 B per file) passes it at about 1.4M, +/// beyond which an encrypted index sits under its knee and runs 1.5–1.7x +/// slower. Trading memory for that speed is the user's call, through +/// [`SEARCH_CACHE_OVERRIDE_MAX_MIB`]. The cap was 128 MiB, which left 1M files +/// under their knee; the figures are at [`SEARCH_CACHE_BYTES_PER_FILE`]. pub const SEARCH_CACHE_MIN_MIB: i64 = 16; -pub const SEARCH_CACHE_MAX_MIB: i64 = 128; +pub const SEARCH_CACHE_MAX_MIB: i64 = 192; /// Ceiling on an explicit `[search] cache_size_mib`, well above the automatic /// cap: the override exists for a large encrypted index whose knee is past @@ -82,17 +85,25 @@ pub const SEARCH_CACHE_OVERRIDE_MAX_MIB: i64 = 1024; const _: () = assert!(SEARCH_CACHE_OVERRIDE_MAX_MIB > SEARCH_CACHE_MAX_MIB); const _: () = assert!(SEARCH_CACHE_MIN_MIB <= SEARCH_CACHE_MAX_MIB); -/// Cache bytes to allow per indexed file: 139 bytes per `files` row times the -/// 1.21x headroom the sweep found sufficient. +/// Cache bytes to allow per indexed file: 137 bytes per `files` row times +/// 1.05, where the knee sits, is 144 B; about 4% of margin makes 150. /// /// Every keystroke rescans the `files` table (filename queries are a `LIKE` /// scan with no FTS), so the working set is that table and the warm curve's -/// knee sits at the first ceiling that holds it. The sweep, the knee per -/// corpus and the row-width caveat are in `benches/search_perf.rs`. The -/// figures predate `HMAC_MODE = Off` (see [`HMAC_MODE`]), which makes this -/// conservative. 139 B/row assumes ordinary path lengths; a much deeper tree -/// wants more, which `[search] cache_size_mib` covers. -pub const SEARCH_CACHE_BYTES_PER_FILE: i64 = 168; +/// knee sits at the first ceiling that holds it. Keyed warm keystroke, +/// `HMAC_MODE = Off`, medians of 3 (`benches/search_perf.rs`, 2026-10-11): +/// +/// | files | `files` table | knee | 168 B, 128 MiB cap | 150 B, 192 MiB cap | +/// |---|---|---|---|---| +/// | 200k | 26.1 MiB | 27.4 MiB | 14.0 ms at 32 MiB | 12.0 ms at 28 MiB | +/// | 1M | 130.4 MiB | 136.9 MiB | 99.0 ms at 128 MiB | 58.9 ms at 143 MiB | +/// +/// Both 200k ceilings are past the knee, so their difference is noise. At 1M +/// the ceilings from the knee to 256 MiB measured 56–64 ms. The sweep, the +/// knee per corpus and the row-width caveat are in `benches/search_perf.rs`. +/// 137 B/row assumes ordinary path lengths; rows over ~4% wider, as in a much +/// deeper tree, sit under the knee, which `[search] cache_size_mib` covers. +pub const SEARCH_CACHE_BYTES_PER_FILE: i64 = 150; /// The cache ceiling this index wants, in MiB. pub fn recommended_search_cache_mib(files: i64, keyed: bool) -> i64 { @@ -215,27 +226,26 @@ CREATE TABLE config_validation ( /// /// 8192 rather than the SQLite and SQLCipher default of 4096: /// `benches/page_geometry.rs` swept 1024→65536 on a keyed 200k-file index, -/// with every search connection at the 16 MiB cache floor (`seed_index` -/// recorded no root counts then), and this balances two opposing costs: +/// `HMAC_MODE = Off`, each search connection at its shipped ceiling, medians +/// of 3 runs (2026-10-11): /// -/// | page | size | `files` scan | scattered rows | cold name | -/// |---|---|---|---|---| -/// | 1024 | 179.9 MiB | 221 ms | 51 ms | 79.4 ms | -/// | 4096 | 154.8 MiB | 100 ms | 43 ms | 44.4 ms | -/// | **8192** | **151.4 MiB** | **55 ms** | **45 ms** | **35.7 ms** | -/// | 16384 | 151.4 MiB | 37 ms | 58 ms | 34.2 ms | -/// | 65536 | 158.4 MiB | 23 ms | 61 ms | 38.1 ms | +/// | page | size | seed rows/s | `files` scan | scattered rows | cold name | warm total | +/// |---|---|---|---|---|---|---| +/// | 1024 | 72.3 MiB | 54,054 | 21.1 ms | 1.12 ms | 39.3 ms | 154.3 ms | +/// | 2048 | 61.5 MiB | 60,606 | 13.7 ms | 0.93 ms | 24.9 ms | 133.9 ms | +/// | 4096 | 60.9 MiB | 62,500 | 11.6 ms | 1.08 ms | 20.6 ms | 121.4 ms | +/// | **8192** | **59.2 MiB** | **66,667** | **12.7 ms** | **1.44 ms** | **18.5 ms** | **117.4 ms** | +/// | 16384 | 58.6 MiB | 66,667 | 10.4 ms | 2.22 ms | 19.8 ms | 110.2 ms | +/// | 32768 | 58.7 MiB | 58,824 | 10.0 ms | 3.74 ms | 18.4 ms | 118.6 ms | +/// | 65536 | 59.1 MiB | 54,054 | 9.1 ms | 4.51 ms | 18.7 ms | 114.5 ms | /// /// A scattered row fetch decrypts a whole page to read one ~130-byte row, so -/// it wants a small page and degrades past 8192 (43 ms → 73 ms by 32768). The -/// sequential `files` scan behind every filename query wants a large one: the -/// bytes decrypted stay ~9 MiB either way while per-page codec and pager work -/// falls. 8192 keeps scattered reads within 3% of their optimum, halves the -/// scan, and ties for the smallest file. -/// -/// The keyed columns predate `HMAC_MODE = Off` (see [`HMAC_MODE`]). Both costs -/// are per-page work, so dropping the authenticator scales them together, and -/// the plain arm picked 8192 too. +/// it wants a small page and degrades from 2048 up (0.93 ms → 4.51 ms by +/// 65536). The sequential `files` scan behind every filename query wants a +/// large one: the bytes decrypted stay ~9 MiB either way while per-page codec +/// and pager work falls. Below 4096 the file grows and the scan and both +/// query totals slow; from 4096 up the curve is flat. 8192 seeds fastest +/// (tied with 16384) and ties for the best cold name. /// /// **Changing this is not a free edit**; see [`PROFILES_PREVIOUS`]. pub const PAGE_SIZE: i64 = 8192; @@ -314,42 +324,51 @@ impl HmacMode { /// /// # What it buys /// -/// `benches/cipher_hmac.rs`, 200k files, all four arms in one process, every -/// search connection at the 16 MiB cache floor (`seed_index` recorded no root -/// counts then), below the `files` table. Warm total is six cascade shapes -/// summed, so it prices per-page work, not a keystroke at the shipped -/// ceiling; its "body (common)" shape filled the display limit from file -/// names and never reached the full-text pass: +/// `benches/cipher_hmac.rs`, 200k files, all four arms in one process, +/// medians of 3 runs (2026-10-11). Each search connection ran at its arm's +/// shipped ceiling: 16 MiB plain, and 32 MiB keyed, which like today's 28 MiB +/// holds the 19.3 MiB `files` table. Cold and warm totals are six cascade +/// shapes summed, cold on a fresh connection: /// -/// | arm | seed | warm total | duplicates | size | -/// |---|---|---|---|---| -/// | plain (no password) | 30.9 s | 125.6 ms | 206 ms | 151.5 MiB | -/// | **keyed, HMAC off** | **37.7 s** | **146.2 ms** | **221 ms** | **151.5 MiB** | -/// | keyed, HMAC_SHA256 | 40.0 s | 209.0 ms | 250 ms | 150.6 MiB | -/// | keyed, HMAC_SHA512 | 41.3 s | 259.5 ms | 262 ms | 151.4 MiB | +/// | arm | seed | cold total | warm total | duplicates | size | +/// |---|---|---|---|---|---| +/// | plain (no password) | 2.9 s | 142.1 ms | 125.5 ms | 52 ms | 59.1 MiB | +/// | **keyed, HMAC off** | **2.9 s** | **165.1 ms** | **110.6 ms** | **58 ms** | **59.2 MiB** | +/// | keyed, HMAC_SHA256 | 3.2 s | 237.3 ms | 110.6 ms | 74 ms | 59.4 MiB | +/// | keyed, HMAC_SHA512 | 3.4 s | 290.9 ms | 115.9 ms | 84 ms | 59.6 MiB | /// -/// Against SQLCipher's default that is 1.78x on warm search, and it takes -/// encrypted-over-plain from 2.07x to 1.16x, 84% of the cost of having a -/// password. A filename query went 32.6 ms → 13.8 ms and a rare body term -/// 39.8 → 18.6. +/// Cold is the case that matters: QuickSearch is usually launched fresh from +/// its shortcut, searched and closed, so nearly every search starts on a cold +/// connection. That is where the HMAC is paid, on every page decrypted, as it +/// is in the duplicate scan and the seed. Cold, encrypted over plain is 1.16x +/// with the HMAC off against 2.05x under SHA-512, so `Off` removes 85% of the +/// cold cost of having a password: a cold filename query went 38.7 ms → +/// 18.3 ms (plain 14.1) and a cold rare body term 49.5 → 24.3 (plain 20.2). +/// Only in a long-running session, once the cache holds what a keystroke +/// reads, are the keyed arms within noise of each other (the plain arm's flat +/// 16 MiB sits under its table, so it re-reads pages from the OS cache). /// -/// SHA-256 recovers only 1.24x of the 2.07x despite halving the digest, +/// At 1M files, with the keyed arms capped at 128 MiB (above their 97 MiB +/// `files` table), the shape holds: cold total 825 ms off against 1428 ms +/// under SHA-512 (plain 627), duplicates 292 against 432 ms, seed 27.3 against +/// 41.8 s, and warm 535–569 ms on every keyed arm. +/// +/// SHA-256 recovers only 1.23x of the 2.05x despite halving the digest, /// because the digest is not the cost: `sqlcipher_openssl_hmac` calls /// `EVP_MAC_fetch(NULL, "HMAC", NULL)`, `EVP_MAC_CTX_new` and an /// `EVP_MAC_init` that fetches the digest by name, two OpenSSL 3 provider /// lookups per page whichever digest is chosen. Only `Off` removes them. /// -/// Size is unmoved (151.5 against 151.4 MiB): [`fts_pgsz_for`] re-derives the +/// Size barely moves (59.2 against 59.6 MiB): [`fts_pgsz_for`] re-derives the /// record size from the new reserve, so the 64 bytes handed back per page go /// into leaves. /// /// # Figures taken under HMAC-SHA512 /// -/// [`PAGE_SIZE`]'s sweep, [`SEARCH_CACHE_BYTES_PER_FILE`], and the keyed -/// figures in `benches/search_perf.rs`, `benches/page_geometry.rs` and -/// `search::IDLE_RELEASE` were measured under HMAC-SHA512 and not re-swept. -/// A keyed page miss is now cheaper, so those keyed costs are overstated and -/// the cache sizing errs large. +/// Only the FTS5 record-size table in `benches/page_geometry.rs`, whose keyed +/// columns predate `Off`. [`PAGE_SIZE`]'s sweep, the search cache sizing +/// ([`SEARCH_CACHE_BYTES_PER_FILE`], [`SEARCH_CACHE_MAX_MIB`]) and +/// `search::IDLE_RELEASE` were re-measured under `Off`. /// /// # Changing it is not a free edit /// @@ -553,8 +572,8 @@ mod tests { } } - /// The sweep found no knee on a plain index at any corpus size, so this - /// must not grow with one. + /// A plain index has no cliff to stay above, and its ceiling is held flat + /// by choice, for memory, so this must not grow with the corpus. #[test] fn a_plain_index_gets_the_same_ceiling_at_every_size() { for files in [0, 1_000, 200_000, 1_000_000, 50_000_000] { @@ -597,12 +616,13 @@ mod tests { assert!(large <= SEARCH_CACHE_MAX_MIB); } - /// The two corpora `benches/search_perf.rs` measured under the automatic - /// cap, against the knee it found for each. Below the knee a keyed search - /// ran 3.4–3.9x slower (HMAC-SHA512), so the recommendation must reach it. + /// The corpora `benches/search_perf.rs` measured, against the knee it + /// found for each: 1.05x the `files` table, in whole MiB rounded up. Below + /// the knee a keyed search ran 1.5–1.7x slower, so the recommendation must + /// reach it. #[test] fn the_recommendation_clears_every_measured_knee() { - for (files, knee_mib) in [(200_000, 32), (600_000, 96)] { + for (files, knee_mib) in [(200_000, 28), (600_000, 83), (1_000_000, 137)] { let got = recommended_search_cache_mib(files, true); assert!( got >= knee_mib, @@ -627,7 +647,7 @@ mod tests { ); } - /// An explicit setting must be able to exceed the automatic cap: past ~800k + /// An explicit setting must be able to exceed the automatic cap: past ~1.4M /// files the derived value is capped below the measured knee, and the /// override is the only way to reach it. #[test] diff --git a/crates/quicksearch-core/src/file_handling/records.rs b/crates/quicksearch-core/src/file_handling/records.rs index 9ef3b2c..89738a7 100644 --- a/crates/quicksearch-core/src/file_handling/records.rs +++ b/crates/quicksearch-core/src/file_handling/records.rs @@ -103,9 +103,12 @@ pub(crate) fn mtime_secs(meta: &std::fs::Metadata) -> u64 { pub type FileHash = [u8; 32]; /// Identify a file as `sha256(size || first hash_length bytes)`, leaving the -/// head bytes in `head` for MIME sniffing. The cost is a collision class: -/// same-size files with identical heads read as duplicates -/// (`examples/hashprobe.rs` measures it); `crate::verify` is the way out. +/// head bytes in `head` for MIME sniffing. Head only, for indexing speed: the +/// walk reads these bytes anyway. The cost is a collision class: same-size +/// files with identical heads hash alike, 3.44% of 86k real files at 8 KiB +/// (`examples/hashprobe.rs` measures it). The Duplicates tab's tail check +/// ([`crate::search::split_by_tail`]) brings that to 0.10%, and +/// `crate::verify` is the byte-exact way out. /// /// Returned with the opened handle's metadata, which is where the hashed size /// comes from: a record built from it describes the file that was read, not diff --git a/crates/quicksearch-core/src/indexing/mod.rs b/crates/quicksearch-core/src/indexing/mod.rs index 2263436..dd9c6e3 100644 --- a/crates/quicksearch-core/src/indexing/mod.rs +++ b/crates/quicksearch-core/src/indexing/mod.rs @@ -17,6 +17,7 @@ mod progress; #[cfg(test)] mod tests; +pub(crate) use pipeline::WalGuard; pub use progress::{ overall_progress, MaintenanceStep, OverallProgress, ReconcileProgress, RootPhase, RootProgress, }; diff --git a/crates/quicksearch-core/src/indexing/pipeline.rs b/crates/quicksearch-core/src/indexing/pipeline.rs index 8321f96..5a97450 100644 --- a/crates/quicksearch-core/src/indexing/pipeline.rs +++ b/crates/quicksearch-core/src/indexing/pipeline.rs @@ -781,8 +781,9 @@ fn rotated(v: &mut [T], round: usize) -> impl Iterator { /// holds a read mark, and a run keeps a reader per root from start to /// finish; an explicit checkpoint retries that lock under `busy_timeout` and /// wins. So the writer forces one each time the log grows by `cap`, and -/// stops the run when the volume nears [`DISK_FLOOR`]. -struct WalGuard { +/// stops the run when the volume nears [`DISK_FLOOR`]. `testutil::seed_index` +/// holds one too, so a seeded index's log is bounded as a run's is. +pub(crate) struct WalGuard { path: String, /// 0 when the log is unbounded. cap: u64, @@ -791,8 +792,8 @@ struct WalGuard { } impl WalGuard { - fn new(config: &Config, db_path: &str) -> WalGuard { - let configured = match config.processing.maximum_wal_size { + pub(crate) fn new(processing: &crate::config::ProcessingConfig, db_path: &str) -> WalGuard { + let configured = match processing.maximum_wal_size { 0 => 0, n => n.max(crate::config::MINIMUM_WAL_SIZE), }; @@ -804,25 +805,32 @@ impl WalGuard { } } - /// Checkpoint if a round that made progress left the log past `next_at`, - /// then stop the run if the disk is nearly full. + /// Whether the log has grown past `next_at`. + pub(crate) fn due(&self) -> bool { + self.cap != 0 && wal_len(&self.path) >= self.next_at + } + + /// Checkpoint now, then re-arm from what is on disk: a lost race costs + /// one attempt per further `cap` of growth, not one per round. + pub(crate) fn checkpoint(&mut self, conn: &Connection) { + if let Err(e) = crate::db::repo::checkpoint_truncate(conn) { + crate::log_warn!("{}", e); + } + self.next_at = wal_len(&self.path) + self.cap; + } + + /// [`Self::checkpoint`] if a round that made progress left the log past + /// `next_at`, then stop the run if the disk is nearly full. fn after_round(&mut self, cx: &RunCx<'_>, progressed: bool) -> Result<(), String> { // Called after `publish_status`: the checkpoint may block for // `busy_timeout`, and it must not sit in front of `stop_indexing`, // hence the stop-flag check. - if self.cap == 0 - || !progressed - || cx.stop_flag.load(Ordering::Relaxed) - || wal_len(&self.path) < self.next_at - { + if !progressed || cx.stop_flag.load(Ordering::Relaxed) || !self.due() { return Ok(()); } { let _maintaining = cx.maintaining(MaintenanceStep::Checkpoint); - let conn = crate::lock_ok(&cx.conn_mutex); - if let Err(e) = crate::db::repo::checkpoint_truncate(&conn) { - crate::log_warn!("{}", e); - } + self.checkpoint(&crate::lock_ok(&cx.conn_mutex)); } // Only here, not every round: the log is at its largest, and the // checkpoint above has just returned whatever it could, so what is @@ -840,9 +848,6 @@ impl WalGuard { )); } } - // Re-armed from what is on disk: a lost race costs one attempt per - // further `cap` of growth, not one per round. - self.next_at = wal_len(&self.path) + self.cap; Ok(()) } } @@ -980,7 +985,7 @@ impl IndexingService { let mut cleanup_done = false; let mut round = 0usize; let mut last_census = Instant::now(); - let mut wal = WalGuard::new(config, db_path); + let mut wal = WalGuard::new(&config.processing, db_path); // Each round serves every walking root one slice, then one extracting // root one slice. Walk workers stall once their channel fills, so diff --git a/crates/quicksearch-core/src/indexing/tests.rs b/crates/quicksearch-core/src/indexing/tests.rs index ca6bda0..a8718a4 100644 --- a/crates/quicksearch-core/src/indexing/tests.rs +++ b/crates/quicksearch-core/src/indexing/tests.rs @@ -1134,3 +1134,50 @@ fn the_wal_cap_is_bounded_by_the_volume() { "disabled checkpoints on a roomy disk are effectively never" ); } + +/// `testutil::seed_index` bounds its log through [`WalGuard`]'s two halves +/// rather than through a run's `after_round`, so they are checked on their +/// own: not due under the cap, due past it, and re-armed a cap of growth +/// later once the checkpoint has emptied the log. +#[test] +fn the_wal_guard_checkpoints_once_the_log_passes_its_cap() { + let (_dir, db) = crate::testutil::Scratch::db("walguard"); + let db = db.to_str().unwrap(); + let processing = crate::config::ProcessingConfig { + // Raised to the floor, which no amount of free space lowers. + maximum_wal_size: 1, + ..Default::default() + }; + let conn = db::open_or_recreate(db, &processing).unwrap(); + conn.execute_batch("PRAGMA wal_autocheckpoint = 0; CREATE TABLE ballast(b BLOB);") + .unwrap(); + let wal_bytes = || std::fs::metadata(format!("{}-wal", db)).map_or(0, |m| m.len()); + // One commit of about a MiB each. + let grow = |mib: usize| { + for _ in 0..mib { + conn.execute("INSERT INTO ballast VALUES (zeroblob(1048576))", []) + .unwrap(); + } + }; + + let mut wal = WalGuard::new(&processing, db); + assert!(!wal.due(), "a fresh log is not due"); + grow(8); + assert!(!wal.due(), "half the cap is not due"); + grow(12); + assert!( + wal_bytes() > crate::config::MINIMUM_WAL_SIZE, + "the fixture must pass the cap" + ); + assert!(wal.due(), "a log past the cap is due"); + + wal.checkpoint(&conn); + assert_eq!( + wal_bytes(), + 0, + "with no reader, the checkpoint empties the log" + ); + assert!(!wal.due(), "re-armed from the emptied log"); + grow(8); + assert!(!wal.due(), "the next cap is counted from the checkpoint"); +} diff --git a/crates/quicksearch-core/src/platform_tests.rs b/crates/quicksearch-core/src/platform_tests.rs index cf36e6a..25c702c 100644 --- a/crates/quicksearch-core/src/platform_tests.rs +++ b/crates/quicksearch-core/src/platform_tests.rs @@ -759,7 +759,9 @@ fn limited_child() { return; } let bytes = ask.parse::().expect("MiB") << 20; - let held = vec![1u8; bytes]; + // `black_box`, or a release build drops the unread allocation and the + // over-limit child exits cleanly. + let held = std::hint::black_box(vec![1u8; bytes]); assert_eq!(held.len(), bytes); } diff --git a/crates/quicksearch-core/src/query/pattern.rs b/crates/quicksearch-core/src/query/pattern.rs index 67d28c8..b95cd95 100644 --- a/crates/quicksearch-core/src/query/pattern.rs +++ b/crates/quicksearch-core/src/query/pattern.rs @@ -236,20 +236,29 @@ impl TermPattern { } /// ASCII-case-insensitive [`str::find`] without folding the haystack; - /// `needle` must already be lowercased. The returned index is always a - /// `char` boundary: a `&str`'s first byte is never a continuation byte, - /// and ASCII-only case differences keep the end a boundary too. + /// `needle` must already be lowercased. `memchr2` jumps to each spelling + /// of the needle's first byte, and only those candidates are compared: + /// 1.18x faster than testing every byte (`benches/search.rs`, + /// `filename_ladder`). The returned index is always a `char` boundary: a + /// `&str`'s first byte is never a continuation byte, and ASCII-only case + /// differences keep the end a boundary too. fn find_ascii_ci(hay: &str, needle: &str) -> Option { - if needle.is_empty() { - return Some(0); - } let (h, n) = (hay.as_bytes(), needle.as_bytes()); - let first = n[0]; - h.len().checked_sub(n.len()).and_then(|last| { - (0..=last).find(|&i| { - h[i].to_ascii_lowercase() == first && h[i..i + n.len()].eq_ignore_ascii_case(n) - }) - }) + let Some(&first) = n.first() else { + return Some(0); + }; + // Equal to `first` when it has no ASCII case partner. + let upper = first.to_ascii_uppercase(); + let last = h.len().checked_sub(n.len())?; + let mut at = 0; + while at <= last { + let i = at + memchr::memchr2(first, upper, &h[at..=last])?; + if h[i..i + n.len()].eq_ignore_ascii_case(n) { + return Some(i); + } + at = i + 1; + } + None } /// Leftmost match as a byte range; folded offsets are valid in the original. @@ -528,6 +537,162 @@ mod tests { assert_eq!(p.find_first("my report.pdf", false), None); } + /// What a literal's `find_first(text, true)` must return: the leftmost + /// match in the ASCII-folded haystack. Folding keeps byte length, so the + /// offsets carry over. + fn folded_reference(folded_hay: &str, folded_needle: &str) -> Option> { + folded_hay + .find(folded_needle) + .map(|at| at..at + folded_needle.len()) + } + + fn literal(term: &str) -> TermPattern { + TermPattern::build(&[part(term, false)]).unwrap() + } + + /// Every string of up to `max` characters over `alphabet`. + fn strings_over(alphabet: &[char], max: usize) -> Vec { + let mut all = vec![String::new()]; + let mut level = vec![String::new()]; + for _ in 0..max { + level = level + .iter() + .flat_map(|s| alphabet.iter().map(move |c| format!("{s}{c}"))) + .collect(); + all.extend(level.iter().cloned()); + } + all + } + + /// The candidate scan against the reference over every short string: both + /// cases of letters, separators, `é`/`É` (one lead byte, different + /// continuations), and KELVIN SIGN and long s, which fold to `k` and `s` + /// under Unicode rules only and so must not match here. + #[test] + fn ascii_fold_find_agrees_with_a_folded_reference() { + const ALPHABET: [char; 12] = [ + 'a', 'A', 'b', 'B', 'k', 's', '-', '/', 'é', 'É', '\u{212A}', 'ſ', + ]; + let patterns: Vec<(TermPattern, String)> = strings_over(&ALPHABET, 2) + .into_iter() + .skip(1) + .map(|n| (literal(&n), n.to_ascii_lowercase())) + .collect(); + for hay in strings_over(&ALPHABET, 4) { + let folded = hay.to_ascii_lowercase(); + for (p, needle) in &patterns { + assert_eq!( + p.find_first(&hay, true), + folded_reference(&folded, needle), + "{needle:?} in {hay:?}" + ); + } + } + } + + #[test] + fn ascii_fold_find_edges() { + let find = |term: &str, hay: &str| literal(term).find_first(hay, true); + // An empty field, and a term longer than the field. + assert_eq!(find("a", ""), None); + assert_eq!(find("quartzite", "QUARTZ"), None); + assert_eq!(find("quartz", "QuArTz"), Some(0..6)); + // The last start a scan may take. + assert_eq!(find("txt", "notes.TXT"), Some(6..9)); + assert_eq!(find("ab", "aaaa"), None); + // A failed candidate resumes one byte on, inside the repeat. + assert_eq!(find("aab", "aaaab"), Some(2..5)); + assert_eq!(find("AAB", "aAaAb"), Some(2..5)); + // The leftmost of several spellings. + assert_eq!(find("Report", "x-REPORT-report"), Some(2..8)); + // First bytes with no case partner. + assert_eq!(find("-01", "x-A-01"), Some(3..6)); + assert_eq!(find(".txt", "a.TXT"), Some(1..5)); + assert_eq!(find("/q", r"C:\Users/Q"), Some(8..10)); + // ASCII folding only: `É` is not `é`, and KELVIN SIGN is not `k`. + assert_eq!(find("été", "ÉTÉ été"), Some(6..11)); + assert_eq!(find("café", "CAFÉ"), None); + assert_eq!(find("café", "CAFé"), Some(0..5)); + assert_eq!(find("kilo", "\u{212A}ILO"), None); + assert_eq!(find("é", "aÉé"), Some(3..5)); + assert_eq!(find("b", "éB"), Some(2..3)); + // Unreachable through `find_first` (an empty term is `Empty`). + assert_eq!(TermPattern::find_ascii_ci("abc", ""), Some(0)); + } + + /// The ladder in `search/cascade/passes.rs` `pass_filename` (ranks 1–4 on + /// the name, 9–10 on the path), with the case-insensitive find swappable. + fn ladder( + p: &TermPattern, + name: &str, + path: &str, + find_ci: &dyn Fn(&str) -> Option>, + ) -> Option<(u8, Range)> { + let with_paths = p.literal_char_count() >= crate::search::prefilter::TRIGRAM_FLOOR; + if p.whole_match(name, false) { + Some((1, 0..name.len())) + } else if p.whole_match(name, true) { + Some((2, 0..name.len())) + } else if let Some(r) = p.find_first(name, false) { + Some((3, r)) + } else if let Some(r) = find_ci(name) { + Some((4, r)) + } else if !with_paths { + None + } else if let Some(r) = p.find_first(path, false) { + Some((9, r)) + } else { + find_ci(path).map(|r| (10, r)) + } + } + + #[test] + fn filename_tiers_agree_with_a_folded_reference() { + let rows = [ + ("report.txt", "/home/u/report.txt"), + ("REPORT.TXT", "/home/u/REPORT.TXT"), + ("Report", "/home/u/Report"), + ("my-report-2024.pdf", "/data/my-report-2024.pdf"), + ("MY_REPORT.pdf", "/data/MY_REPORT.pdf"), + ("notes.txt", "/srv/Report/notes.txt"), + ("notes.txt", "/srv/REPORT/notes.txt"), + ("notes.txt", r"C:\Users\u\Report\notes.txt"), + ("résumé-report.odt", "/home/u/résumé-report.odt"), + ("RÉSUMÉ.odt", "/home/u/RÉSUMÉ.odt"), + ("rrreport", "/x/rrreport"), + ("rep", "/rep"), + ("", "/"), + ]; + let terms = [ + "report", + "Report", + "REPORT", + "rep", + "rrep", + "résumé", + "Résumé", + "RÉSUMÉ", + "report/notes", + r"Report\notes", + ".txt", + "t", + ]; + let mut ranks = std::collections::BTreeSet::new(); + for term in terms { + let p = literal(term); + let needle = term.to_ascii_lowercase(); + for (name, path) in rows { + let shipped = ladder(&p, name, path, &|f| p.find_first(f, true)); + let reference = ladder(&p, name, path, &|f| { + folded_reference(&f.to_ascii_lowercase(), &needle) + }); + assert_eq!(shipped, reference, "{term:?} against {path:?}"); + ranks.extend(shipped.map(|(rank, _)| rank)); + } + } + assert_eq!(ranks.into_iter().collect::>(), [1, 2, 3, 4, 9, 10]); + } + #[test] fn regex_defaults_case_insensitive_with_optout() { let r = RegexQuery::new("foo\\d+").unwrap(); diff --git a/crates/quicksearch-core/src/search/duplicates.rs b/crates/quicksearch-core/src/search/duplicates.rs index 730f3af..1dcb0a4 100644 --- a/crates/quicksearch-core/src/search/duplicates.rs +++ b/crates/quicksearch-core/src/search/duplicates.rs @@ -19,29 +19,58 @@ //! The index exists only once a run has finished (see //! [`crate::db::repo::build_hash_index`]); before that the same query sorts //! the table, slower but with the same answer. +//! +//! The hash covers only each file's size and head, so these are +//! [`HeadGroup`]s, not for display: [`split_by_tail`] reads their tails and +//! returns the [`DuplicateGroup`]s a caller shows. use std::cmp::Ordering; -use std::collections::BinaryHeap; +use std::collections::{BinaryHeap, HashMap}; +use std::io::{Read, Seek, SeekFrom}; +use std::path::Path; +use std::sync::atomic::{self, AtomicBool}; use rusqlite::{params, Connection, OptionalExtension}; +use sha2::{Digest, Sha256}; use crate::db; +/// One member of a group: `(file_id, name, path, size, mtime)`. +pub type Member = (i64, String, String, u64, i64); + +/// Files sharing a head hash, as the index groups them. +#[derive(Debug, Clone, PartialEq)] +pub struct HeadGroup { + pub hash: Vec, + /// Path-ordered. Price a group from these: the counts it was ranked by + /// came from an earlier statement, with no transaction holding the two + /// together. + pub members: Vec, +} + +/// Files sharing a head hash and a tail digest: what [`split_by_tail`] makes +/// of a [`HeadGroup`], and the only kind of group to show. #[derive(Debug, Clone, PartialEq)] pub struct DuplicateGroup { + /// The head hash every member was indexed with. pub hash: Vec, - /// `(file_id, name, path, size, mtime)` per member, path-ordered. Price a - /// group from these: the counts it was ranked by came from an earlier - /// statement, with no transaction holding the two together. - pub members: Vec<(i64, String, String, u64, i64)>, + /// `sha256` of the bytes each member holds past its head. A file the head + /// hashed whole holds none, so its digest is of nothing, and such files + /// group by head alone. + pub tail: [u8; 32], + /// Path-ordered, as in [`HeadGroup::members`]. + pub members: Vec, } impl DuplicateGroup { /// The group's identity, for a UI that wants to remember one: lowercase - /// hex of the hash. Stable across renames and moves, because the hash - /// covers the content and nothing about where the files live. - pub fn hash_hex(&self) -> String { - crate::security::hex_encode(&self.hash) + /// hex of the head hash, then of the tail digest. Stable across renames + /// and moves, because both cover content and nothing about where the + /// files live, and distinct for each group one head hash splits into. + pub fn identity(&self) -> String { + let mut id = crate::security::hex_encode(&self.hash); + id.push_str(&crate::security::hex_encode(&self.tail)); + id } } @@ -68,14 +97,15 @@ impl PartialOrd for Ranked { } } -/// The `limit` hash groups with the most reclaimable bytes, largest first. -/// Groups of one, NULL-hash rows and zero-size files are excluded. +/// The `limit` head-hash groups with the most reclaimable bytes, largest +/// first; see [`split_by_tail`] before showing them. Groups of one, NULL-hash +/// rows and zero-size files are excluded. /// -/// Grouping is by hash alone, so hard links and bind-mounted copies of one +/// Grouping is by content alone, so hard links and bind-mounted copies of one /// file are listed as copies, and deleting one frees nothing. That is /// documented rather than detected: they really are byte-identical, which /// is all verification claims. -pub fn find_duplicate_groups(db_path: &str, limit: u32) -> Result, String> { +pub fn find_duplicate_groups(db_path: &str, limit: u32) -> Result, String> { if limit == 0 { return Ok(Vec::new()); } @@ -158,7 +188,7 @@ fn rank_candidates( /// Turn the winners into full groups: the hash they were ranked by, then their /// members. Nothing here is proportional to the size of the index. -fn hydrate(conn: &Connection, winners: Vec) -> Result, String> { +fn hydrate(conn: &Connection, winners: Vec) -> Result, String> { let mut hash_stmt = conn .prepare("SELECT hash FROM files WHERE id = ?1") .map_err(|e| e.to_string())?; @@ -196,11 +226,132 @@ fn hydrate(conn: &Connection, winners: Vec) -> Result, + tail_len: usize, + never_open: impl Fn(&Path) -> bool, + cancel: &AtomicBool, +) -> Vec { + let tail_len = tail_len as u64; + let mut buf = Vec::new(); + let mut ranked: Vec<(Ranked, DuplicateGroup)> = Vec::with_capacity(groups.len()); + for group in groups { + let mut by_tail: HashMap<[u8; 32], Vec> = HashMap::new(); + for member in group.members { + if cancel.load(atomic::Ordering::Relaxed) { + break; + } + let path = Path::new(&member.2); + if let Ok(Some(tail)) = tail_digest(path, member.3, tail_len, &never_open, &mut buf) { + by_tail.entry(tail).or_default().push(member); + } + } + for (tail, members) in by_tail { + if members.len() < 2 { + continue; + } + let sub = DuplicateGroup { + hash: group.hash.clone(), + tail, + members, + }; + ranked.push((rank(&sub.members), sub)); + } + } + ranked.sort_unstable_by(|a, b| b.0.cmp(&a.0)); + ranked.into_iter().map(|(_, group)| group).collect() +} + +/// `sha256` of the bytes a member holds past its head, or `Ok(None)` when its +/// size has moved since indexing. A file the head hashed whole holds none and +/// is not opened. Reads into `buf`, which is resized to the tail. +fn tail_digest( + path: &Path, + size: u64, + tail_len: u64, + never_open: &dyn Fn(&Path) -> bool, + buf: &mut Vec, +) -> std::io::Result> { + if size <= tail_len { + return Ok(Some(Sha256::digest(b"").into())); + } + if never_open(path) { + return Err(std::io::ErrorKind::PermissionDenied.into()); + } + // Non-blocking, so a FIFO renamed over a member is refused rather than + // waited on; the size comes from the handle, so it is the file read. + let (mut file, meta) = crate::platform::open_regular_file(path)?; + if meta.len() != size { + return Ok(None); + } + let start = tail_len.max(size - tail_len); + file.seek(SeekFrom::Start(start))?; + buf.clear(); + buf.resize((size - start) as usize, 0); + file.read_exact(buf)?; + Ok(Some(Sha256::digest(&buf[..]).into())) +} + +/// [`rank_candidates`]'s order for a group in hand: priced by its lowest +/// rowid's size, as the scan priced it. +fn rank(members: &[Member]) -> Ranked { + let (rowid, size) = members + .iter() + .min_by_key(|m| m.0) + .map_or((0, 0), |m| (m.0, m.3.min(i64::MAX as u64) as i64)); + Ranked { + redundant: size.saturating_mul(members.len() as i64 - 1), + rowid, + } +} + #[cfg(test)] mod tests { use super::*; @@ -262,7 +413,7 @@ mod tests { } /// Each group as `(first member's name, member count)`, in listed order. - fn shape(groups: &[DuplicateGroup]) -> Vec<(&str, usize)> { + fn shape(groups: &[HeadGroup]) -> Vec<(&str, usize)> { groups .iter() .map(|g| (g.members[0].1.as_str(), g.members.len())) @@ -323,34 +474,6 @@ mod tests { ); } - /// The identity a UI remembers a dismissed group by must be the hash - /// itself, spelled one way, or a hidden group comes back under a - /// different spelling. - #[test] - fn a_group_spells_its_hash_the_one_way() { - let (_dir, p) = seed_db(); - let groups = find_duplicate_groups(&p, 10).unwrap(); - for group in &groups { - let hex = group.hash_hex(); - assert_eq!(hex.len(), group.hash.len() * 2); - assert!(hex - .chars() - .all(|c| c.is_ascii_hexdigit() && !c.is_uppercase())); - } - let again = find_duplicate_groups(&p, 10).unwrap(); - let spelled = |gs: &[DuplicateGroup]| gs.iter().map(|g| g.hash_hex()).collect::>(); - assert_eq!( - spelled(&groups), - spelled(&again), - "an unchanged index names its groups the same way twice" - ); - assert_ne!( - groups[0].hash_hex(), - groups[1].hash_hex(), - "two groups must not answer to one name" - ); - } - #[test] fn zero_limit_answers_without_opening_the_index() { assert!(find_duplicate_groups("/nonexistent/index.sqlite", 0) @@ -474,4 +597,337 @@ mod tests { "the candidate scan must not touch table rows; plan was {plan:?}" ); } + + // --- split_by_tail ------------------------------------------------------ + + /// The head the hand-built groups below were "indexed" with. + const TAIL: usize = 8; + + /// A group as `find_duplicate_groups` hands one over: one member per body, + /// written under `dir`, indexed at its length, rowids from `first`. + fn on_disk(dir: &Path, first: i64, bodies: &[Vec]) -> HeadGroup { + let members = bodies + .iter() + .zip(first..) + .map(|(body, id)| { + let name = format!("f{id}.bin"); + let path = dir.join(&name); + std::fs::write(&path, body).unwrap(); + let path = path.to_str().unwrap().to_string(); + (id, name, path, body.len() as u64, 1_700_000_000) + }) + .collect(); + HeadGroup { + hash: vec![first as u8; 32], + members, + } + } + + /// `len` bytes of one head, ending in `last`. + fn ending(len: usize, last: u8) -> Vec { + let mut body = vec![b'h'; len]; + body[len - 1] = last; + body + } + + /// The tail digest a file holding `body` is grouped by. + fn tail_of(body: &[u8]) -> [u8; 32] { + let start = TAIL.max(body.len().saturating_sub(TAIL)).min(body.len()); + Sha256::digest(&body[start..]).into() + } + + /// `group` as `split_by_tail` lists it when every member checks out. + fn checked(group: &HeadGroup, tail: [u8; 32]) -> DuplicateGroup { + DuplicateGroup { + hash: group.hash.clone(), + tail, + members: group.members.clone(), + } + } + + fn split(groups: &[HeadGroup]) -> Vec { + split_by_tail(groups.to_vec(), TAIL, |_| false, &AtomicBool::new(false)) + } + + /// Each group's member rowids, in listed order. + fn ids(groups: &[DuplicateGroup]) -> Vec> { + groups + .iter() + .map(|g| g.members.iter().map(|m| m.0).collect()) + .collect() + } + + fn path_of(group: &HeadGroup, i: usize) -> &Path { + Path::new(&group.members[i].2) + } + + #[test] + fn a_different_tail_splits_the_group() { + let dir = Scratch::dir("tails-differ"); + let bodies = [ending(24, b'a'), ending(24, b'a'), ending(24, b'b')]; + let group = on_disk(&dir, 1, &bodies); + let refined = split(std::slice::from_ref(&group)); + assert_eq!(ids(&refined), [[1, 2]], "the odd one out leaves, alone"); + assert_eq!(refined[0].hash, group.hash, "a part keeps the head hash"); + assert_eq!(refined[0].tail, tail_of(&bodies[0])); + } + + #[test] + fn a_matching_tail_keeps_the_group_together() { + let dir = Scratch::dir("tails-match"); + let body = ending(24, b'a'); + let group = on_disk(&dir, 1, &[body.clone(), body.clone()]); + assert_eq!( + split(std::slice::from_ref(&group)), + [checked(&group, tail_of(&body))] + ); + } + + /// Two real sets of copies that start alike: a group each, named apart, + /// so hiding one cannot hide the other. + #[test] + fn two_copy_sets_sharing_a_head_are_two_groups() { + let dir = Scratch::dir("tails-two-sets"); + let (a, b) = (ending(24, b'a'), ending(24, b'b')); + let group = on_disk(&dir, 1, &[a.clone(), b.clone(), a.clone(), b.clone()]); + let refined = split(std::slice::from_ref(&group)); + assert_eq!( + ids(&refined), + [[1, 3], [2, 4]], + "tied on waste, so the lower rowid lists first" + ); + assert_eq!( + (refined[0].tail, refined[1].tail), + (tail_of(&a), tail_of(&b)) + ); + assert!(refined.iter().all(|g| g.hash == group.hash)); + assert_ne!( + refined[0].identity(), + refined[1].identity(), + "two groups must not answer to one name" + ); + } + + /// The identity a UI remembers a dismissed group by is spelled one way, + /// or a hidden group comes back under a different spelling. + #[test] + fn a_group_s_identity_is_its_head_then_its_tail() { + let dir = Scratch::dir("tails-identity"); + let groups = [ + on_disk(&dir, 1, &vec![ending(24, b'a'); 2]), + on_disk(&dir, 10, &vec![ending(4, b'b'); 2]), + ]; + let refined = split(&groups); + assert_eq!(ids(&refined), [[1, 2], [10, 11]]); + for group in &refined { + let id = group.identity(); + assert_eq!(id.len(), (group.hash.len() + 32) * 2); + assert!(id + .chars() + .all(|c| c.is_ascii_hexdigit() && !c.is_uppercase())); + } + assert!( + refined[0].identity().starts_with(&"01".repeat(32)), + "the head hash first: {}", + refined[0].identity() + ); + assert_eq!( + refined[1].identity(), + format!( + "{}{}", + "0a".repeat(32), + // sha256 of nothing: a file the head hashed whole. + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ) + ); + let named = |gs: &[DuplicateGroup]| gs.iter().map(|g| g.identity()).collect::>(); + assert_eq!( + named(&refined), + named(&split(&groups)), + "unchanged files name their groups the same way twice" + ); + } + + /// The head hashed these whole, so there is nothing left to read, and they + /// group by head alone. Their files have since grown apart or gone: a read + /// would see that and drop every member. + #[test] + fn files_the_head_hashed_whole_group_by_head_unread() { + let dir = Scratch::dir("tails-small"); + let mut at_tail = on_disk(&dir, 1, &[ending(20, b'a'), ending(20, b'b')]); + let mut under = on_disk(&dir, 10, &[ending(20, b'c'), ending(20, b'd')]); + for m in &mut at_tail.members { + m.3 = TAIL as u64; + } + for m in &mut under.members { + m.3 = 3; + } + std::fs::remove_file(path_of(&under, 1)).unwrap(); + let groups = [at_tail, under]; + let nothing = tail_of(&[]); + let whole: Vec<_> = groups.iter().map(|g| checked(g, nothing)).collect(); + assert_eq!(split(&groups), whole); + assert_eq!( + split_by_tail(groups.to_vec(), TAIL, |_| true, &AtomicBool::new(false)), + whole, + "nothing is opened, so never_open has nothing to refuse" + ); + } + + #[test] + fn a_member_whose_size_changed_is_dropped() { + let dir = Scratch::dir("tails-resized"); + let three = on_disk(&dir, 1, &vec![ending(24, b'a'); 3]); + let pair = on_disk(&dir, 10, &[ending(24, b'b'), ending(24, b'b')]); + // Grown since indexing; the head hash still matches what it was. + std::fs::write(path_of(&three, 2), ending(30, b'a')).unwrap(); + std::fs::write(path_of(&pair, 1), ending(23, b'b')).unwrap(); + assert_eq!( + ids(&split(&[three, pair])), + [[1, 2]], + "the changed members leave, and a pair down to one is not listed" + ); + } + + /// A copy whose tail cannot be read is not listed, and a group it leaves + /// with one copy is not listed at all. + #[test] + fn an_unreadable_member_is_dropped() { + let dir = Scratch::dir("tails-unreadable"); + let three = on_disk(&dir, 1, &vec![ending(24, b'a'); 3]); + let pair = on_disk(&dir, 10, &vec![ending(24, b'b'); 2]); + let groups = [three.clone(), pair.clone()]; + let listed = |never_open: &dyn Fn(&Path) -> bool| { + ids(&split_by_tail( + groups.to_vec(), + TAIL, + never_open, + &AtomicBool::new(false), + )) + }; + let everything = [vec![1, 2, 3], vec![10, 11]]; + assert_eq!(listed(&|_| false), everything); + + std::fs::remove_file(path_of(&three, 1)).unwrap(); + std::fs::remove_file(path_of(&pair, 0)).unwrap(); + assert_eq!(listed(&|_| false), [[1, 3]], "deleted since indexing"); + std::fs::write(path_of(&three, 1), ending(24, b'a')).unwrap(); + std::fs::write(path_of(&pair, 0), ending(24, b'b')).unwrap(); + + let refused = [path_of(&three, 2), path_of(&pair, 1)]; + assert_eq!( + listed(&|p: &Path| refused.contains(&p)), + [[1, 2]], + "a path never_open names" + ); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let locked = [path_of(&three, 0), path_of(&pair, 0)]; + let chmod = |mode| { + for path in locked { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap(); + } + }; + chmod(0o000); + // Root reads them anyway, and then there is nothing to prove. + let readable = std::fs::File::open(locked[0]).is_ok(); + let refined = listed(&|_| false); + chmod(0o644); + if !readable { + assert_eq!(refined, [[2, 3]], "permission denied"); + } + } + assert_eq!(listed(&|_| false), everything, "readable again"); + } + + #[test] + fn splitting_re_ranks() { + let dir = Scratch::dir("tails-rerank"); + // 300 reclaimable bytes as one group, but two pairs of 100 each. + let four = on_disk( + &dir, + 1, + &[ + ending(100, b'a'), + ending(100, b'a'), + ending(100, b'b'), + ending(100, b'b'), + ], + ); + let pair = on_disk(&dir, 10, &[ending(150, b'c'), ending(150, b'c')]); + assert_eq!( + ids(&split(&[four, pair])), + [vec![10, 11], vec![1, 2], vec![3, 4]], + "150 bytes before 100, and the lower rowid first on the tie" + ); + } + + /// At `2 × TAIL` the head and the tail are the whole file between them; + /// one byte longer and the byte between them is read by neither. + #[test] + fn the_tail_starts_where_the_head_ends() { + let dir = Scratch::dir("tails-boundary"); + let differing_at = |len: usize, at: usize| { + let mut other = vec![b'h'; len]; + other[at] = b'x'; + [vec![b'h'; len], other] + }; + let last_of_head = [on_disk(&dir, 1, &differing_at(2 * TAIL, TAIL - 1))]; + assert_eq!( + ids(&split(&last_of_head)), + [[1, 2]], + "the head's own bytes are not read again" + ); + let first_of_tail = [on_disk(&dir, 10, &differing_at(2 * TAIL, TAIL))]; + assert!(split(&first_of_tail).is_empty(), "the tail starts at TAIL"); + let middle = [on_disk(&dir, 20, &differing_at(2 * TAIL + 1, TAIL))]; + assert_eq!(ids(&split(&middle)), [[20, 21]], "left for Verify"); + } + + #[cfg(unix)] + #[test] + fn a_fifo_swapped_in_is_dropped_without_blocking() { + use std::os::unix::ffi::OsStrExt; + let dir = Scratch::dir("tails-fifo"); + let group = on_disk(&dir, 1, &vec![ending(24, b'a'); 3]); + let fifo = path_of(&group, 2); + std::fs::remove_file(fifo).unwrap(); + let c_path = std::ffi::CString::new(fifo.as_os_str().as_bytes()).unwrap(); + assert_eq!(unsafe { libc::mkfifo(c_path.as_ptr(), 0o600) }, 0); + + let (tx, rx) = std::sync::mpsc::channel(); + let groups = vec![group.clone()]; + std::thread::spawn(move || tx.send(split(&groups)).unwrap()); + let refined = rx + .recv_timeout(std::time::Duration::from_secs(30)) + .expect("the tail check blocked opening a FIFO"); + assert_eq!(ids(&refined), [[1, 2]], "refused, so not listed"); + } + + /// Members not yet read when `cancel` rises are dropped, never listed on + /// their head's word; what was read before it still counts. + #[test] + fn a_raised_cancel_lists_nothing_unchecked() { + let dir = Scratch::dir("tails-cancel"); + let groups = [ + on_disk(&dir, 1, &vec![ending(24, b'a'); 3]), + on_disk(&dir, 10, &vec![ending(24, b'b'); 2]), + ]; + let raised = AtomicBool::new(true); + assert!(split_by_tail(groups.to_vec(), TAIL, |_| false, &raised).is_empty()); + + // Raised as the second member is opened, which is still read. + let cancel = AtomicBool::new(false); + let second = path_of(&groups[0], 1); + let raise_at_second = |p: &Path| { + if p == second { + cancel.store(true, atomic::Ordering::Relaxed); + } + false + }; + let refined = split_by_tail(groups.to_vec(), TAIL, raise_at_second, &cancel); + assert_eq!(ids(&refined), [[1, 2]]); + } } diff --git a/crates/quicksearch-core/src/search/mod.rs b/crates/quicksearch-core/src/search/mod.rs index c4d1ba7..57fcc91 100644 --- a/crates/quicksearch-core/src/search/mod.rs +++ b/crates/quicksearch-core/src/search/mod.rs @@ -23,16 +23,16 @@ use crate::query::split::split_for_cascade; use crate::snippet::Snippet; pub use cascade::Outcome; -pub use duplicates::{find_duplicate_groups, DuplicateGroup}; +pub use duplicates::{find_duplicate_groups, split_by_tail, DuplicateGroup, HeadGroup}; /// Idle window before the connection is released. /// -/// Long, because a release drops the search cache -/// ([`db::schema::pragmas_search`]) and the next keystroke pays to refill it: -/// on a 200k-file index the first query after a release costs 129 ms -/// unencrypted and 192 ms encrypted, against ~10 ms warm, because a keyed -/// refill decrypts every page. (The keyed figure predates `HMAC_MODE = Off`; -/// see [`db::schema::HMAC_MODE`].) +/// Long, so a pause in a typing session never pays to refill the search cache +/// ([`db::schema::pragmas_search`]) that a release drops. The refill is +/// modest: the first keystroke on a fresh connection costs about 15 ms +/// unencrypted and 23 ms encrypted on a 200k-file index, against 12–14 ms +/// warm, and 71 against 36 ms encrypted at 600k, where a refill decrypts +/// every page it reads (`benches/search_perf.rs`, 2026-10-11). /// /// Removals do not wait on it: the service registers with [`db::hold_index`], /// so [`db::remove_index_files`] has it send [`WorkerMsg::ReleaseConnection`] diff --git a/crates/quicksearch-core/src/security.rs b/crates/quicksearch-core/src/security.rs index 21e273e..7a6efc0 100644 --- a/crates/quicksearch-core/src/security.rs +++ b/crates/quicksearch-core/src/security.rs @@ -77,8 +77,8 @@ pub fn derive_key(password: &str, salt: &[u8; SALT_LEN]) -> IndexKey { IndexKey(out) } -/// The crate's only hex encoder, also how a duplicate group spells its hash -/// ([`crate::search::DuplicateGroup::hash_hex`]). +/// The crate's only hex encoder, also how a duplicate group spells its +/// identity ([`crate::search::DuplicateGroup::identity`]). pub(crate) fn hex_encode(bytes: &[u8]) -> String { let mut s = String::with_capacity(bytes.len() * 2); for b in bytes { diff --git a/crates/quicksearch-core/src/testutil.rs b/crates/quicksearch-core/src/testutil.rs index bd515f1..d38f69b 100644 --- a/crates/quicksearch-core/src/testutil.rs +++ b/crates/quicksearch-core/src/testutil.rs @@ -411,11 +411,17 @@ impl Default for SeedSpec { /// Seed an index with synthetic rows. Shared by the measurement harnesses so /// they all describe the same corpus. /// -/// Written under a run's write regime: FTS5's write settings before the first -/// row, autocheckpoint off, the run's finalize merge after the last commit. -/// It records the root counts a completed run would, under `/seed`, so a -/// search connection opens at the cache ceiling it ships with for this -/// many files. +/// Written under a default-config run's write regime: FTS5's write settings +/// before the first row, autocheckpoint off, the run's WAL cap between +/// commits (`indexing::WalGuard`: the default `maximum_wal_size`, lowered as +/// a run lowers it on a volume short of space), the run's tail checkpoint +/// after the last commit, then its finalize merge. It records the root counts a completed run would, +/// under `/seed`, so a search connection opens at the cache ceiling it ships +/// with for this many files. +/// +/// Scratch therefore peaks near the cap plus the database, not at every +/// commit's pages piled into one log; `commit_every = 0` is one transaction, +/// which no checkpoint can split. pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) { use crate::db::repo::{insert_file, store_pending_text, NewFile, RootCounts}; use crate::file_handling::{fts_begin_bulk_write, fts_finalize_after_text_indexing}; @@ -429,7 +435,9 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) { if let Some(hmac) = spec.hmac { crate::db::set_hmac_mode_override(hmac); } - let conn = crate::db::open_or_recreate(path.to_str().unwrap(), &Default::default()).unwrap(); + let processing = crate::config::ProcessingConfig::default(); + let db_path = path.to_str().unwrap(); + let conn = crate::db::open_or_recreate(db_path, &processing).unwrap(); // Before the first row: `pgsz` decides how leaves are built, so setting it // afterwards would only affect segments merged later. if let Some(pgsz) = spec.pgsz { @@ -440,10 +448,11 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) { .unwrap(); } fts_begin_bulk_write(&conn); - // As a run's writer has it. The WAL then grows until the closing - // checkpoint, since nothing here caps it the way a run's writer does. + // As a run's writer has it, which is safe only with the run's own bound + // on the log beside it. conn.execute_batch("PRAGMA wal_autocheckpoint = 0;") .unwrap(); + let mut wal = crate::indexing::WalGuard::new(&processing, db_path); let mut rng = Lcg::new(0x5eed); let ext_mix = if spec.ext_mix.is_empty() { EXT_PLAIN @@ -535,10 +544,17 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) { } if spec.commit_every > 0 && (i + 1) % spec.commit_every == 0 { tx.commit().unwrap(); + // Where a run's writer checks, between its rounds' commits. + if wal.due() { + wal.checkpoint(&conn); + } tx = conn.unchecked_transaction().unwrap(); } } tx.commit().unwrap(); + // The run's first tail checkpoint (`checkpoint_tail`), so the merge + // writes onto an empty log. + wal.checkpoint(&conn); fts_finalize_after_text_indexing(&conn); let files = spec.files as i64; crate::db::repo::set_root_counts(&conn, "/seed", RootCounts { files, fts }).unwrap(); @@ -722,10 +738,10 @@ pub struct Arm { /// reserve, which the header cannot be read without. pub hmac: Option, pub path: PathBuf, - /// How long [`seed_index`] took: rows, zstd bodies and FTS postings, then - /// the merge, counts, hash index and checkpoint. Not an indexing rate: it - /// has no walk or extraction, its WAL grows uncapped until the closing - /// checkpoint, and each arm is one sample taken in sweep order. + /// How long [`seed_index`] took: rows, zstd bodies and FTS postings with + /// the run's WAL checkpoints among them, then the merge, counts, hash + /// index and closing checkpoint. Not an indexing rate: it has no walk or + /// extraction, and each arm is one sample taken in sweep order. pub seeded_in: std::time::Duration, } diff --git a/crates/quicksearch-core/tests/search_cache.rs b/crates/quicksearch-core/tests/search_cache.rs index 7f9b844..07e722b 100644 --- a/crates/quicksearch-core/tests/search_cache.rs +++ b/crates/quicksearch-core/tests/search_cache.rs @@ -19,7 +19,7 @@ use quicksearch_core::db::schema::{ use quicksearch_core::testutil::{measurement_key, seed_index, Scratch, SeedSpec}; /// Enough files that the recommendation clears the floor and is derived rather -/// than clamped: at 168 B/file, 16 MiB is reached around 100k. `seed_index` +/// than clamped: at 150 B/file, 16 MiB is reached around 112k. `seed_index` /// stores that count, as a run over the index would. Seeded without content: /// the FTS write is the slow part and this measures nothing about it. const FILES: usize = 150_000; @@ -128,8 +128,8 @@ fn the_search_connection_opens_at_the_ceiling_it_was_sized_for() { db::set_search_cache_override(None); // --- plain, automatic ------------------------------------------------- - // Same corpus, no key: the sweep found no knee unencrypted, so this must - // be flat regardless of how large the index is. + // Same corpus, no key: an unencrypted index has no cliff and is held at a + // flat ceiling by choice, for memory, however large the index is. let (_plain_dir, plain) = Scratch::db("searchcache-plain"); let plain_path = plain.to_string_lossy().into_owned(); db::set_process_key(None); diff --git a/crates/quicksearch-gui/src/activate.rs b/crates/quicksearch-gui/src/activate.rs index c6be410..c182002 100644 --- a/crates/quicksearch-gui/src/activate.rs +++ b/crates/quicksearch-gui/src/activate.rs @@ -228,11 +228,23 @@ mod imp { return Signal::Unanswered; } // The reply distinguishes "delivered" from "wrote into a - // socket nobody reads". EOF means the peer went away. + // socket nobody reads". EOF means the peer went away. A read cut + // short by a signal is retried for the rest of the timeout: with + // SO_RCVTIMEO set, Linux fails it with EINTR after a stop and + // continue even when no handler is installed. + let deadline = std::time::Instant::now() + IO_TIMEOUT; let mut ack = [0u8; 1]; - match stream.read(&mut ack) { - Ok(1) => Signal::Delivered, - _ => Signal::Unanswered, + loop { + match stream.read(&mut ack) { + Ok(1) => return Signal::Delivered, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => { + let left = deadline.saturating_duration_since(std::time::Instant::now()); + if left.is_zero() || stream.set_read_timeout(Some(left)).is_err() { + return Signal::Unanswered; + } + } + _ => return Signal::Unanswered, + } } } diff --git a/crates/quicksearch-gui/src/app.rs b/crates/quicksearch-gui/src/app.rs index 50872bc..9f2065e 100644 --- a/crates/quicksearch-gui/src/app.rs +++ b/crates/quicksearch-gui/src/app.rs @@ -304,11 +304,7 @@ impl QuickSearchApp { fn start_duplicates_scan(&mut self, ctx: &egui::Context) { self.dups.state = DupState::Loading; let cfg = self.cfg.clone(); - let limit = self.backend.start_duplicates(&cfg, ctx.clone()); - // 0 means a scan was already running, whose limit still stands. - if limit > 0 { - self.dups.scan_limit = limit; - } + self.backend.start_duplicates(&cfg, ctx.clone()); } /// Drops a duplicate listing the index has moved out from under (a finished @@ -317,7 +313,7 @@ impl QuickSearchApp { fn invalidate_duplicates(&mut self, ctx: &egui::Context) { match dup_invalidation(self.tab, &self.dups.state) { DupInvalidation::Nothing => {} - DupInvalidation::WhenItLands => self.rescan_when_dups_land = true, + DupInvalidation::WhenItLands => self.rescan_after_dup_scan(), DupInvalidation::Rescan => self.start_duplicates_scan(ctx), DupInvalidation::Drop => self.dups.state = DupState::NotLoaded, } @@ -332,13 +328,20 @@ impl QuickSearchApp { fn drop_duplicates(&mut self) { match dup_invalidation(self.tab, &self.dups.state) { DupInvalidation::Nothing => {} - DupInvalidation::WhenItLands => self.rescan_when_dups_land = true, + DupInvalidation::WhenItLands => self.rescan_after_dup_scan(), DupInvalidation::Rescan | DupInvalidation::Drop => { self.dups.state = DupState::NotLoaded } } } + /// The running scan's answer is stale: cut its tail check short, and scan + /// again once it lands. + fn rescan_after_dup_scan(&mut self) { + self.rescan_when_dups_land = true; + self.backend.cancel_duplicates(); + } + /// Every switch goes through here, including guard-delayed ones. fn switch_tab(&mut self, ctx: &egui::Context, to: Tab) { if self.tab == to { @@ -546,11 +549,12 @@ impl QuickSearchApp { } self.backend.reindex_live_paths(touched); // Duplicates worker. - if let Some(rx) = &self.backend.dup_job { + if let Some(job) = &self.backend.dup_job { use std::sync::mpsc::TryRecvError; - let done = match rx.try_recv() { - Ok(Ok(groups)) => Some(DupState::Loaded(crate::duplicates_tab::LoadedGroups::new( - groups, + let done = match job.rx.try_recv() { + Ok(Ok(scan)) => Some(DupState::Loaded(crate::duplicates_tab::LoadedGroups::new( + scan.groups, + scan.limited_to, ))), Ok(Err(e)) => Some(DupState::Error(e)), Err(TryRecvError::Empty) => None, diff --git a/crates/quicksearch-gui/src/app/tests.rs b/crates/quicksearch-gui/src/app/tests.rs index eb33870..0807c58 100644 --- a/crates/quicksearch-gui/src/app/tests.rs +++ b/crates/quicksearch-gui/src/app/tests.rs @@ -166,7 +166,7 @@ fn only_an_empty_duplicates_tab_scans_on_arrival() { )); for held in [ DupState::Loading, - DupState::Loaded(crate::duplicates_tab::LoadedGroups::new(Vec::new())), + DupState::Loaded(crate::duplicates_tab::LoadedGroups::new(Vec::new(), None)), DupState::Error("boom".into()), ] { assert!( @@ -190,7 +190,7 @@ fn only_an_empty_duplicates_tab_scans_on_arrival() { /// index can change out from under it has to reach the tab. #[test] fn an_index_that_changed_invalidates_what_the_tab_holds() { - let loaded = || DupState::Loaded(crate::duplicates_tab::LoadedGroups::new(Vec::new())); + let loaded = || DupState::Loaded(crate::duplicates_tab::LoadedGroups::new(Vec::new(), None)); // On screen: the user is looking at it, so redo it now. assert_eq!( diff --git a/crates/quicksearch-gui/src/app/verify.rs b/crates/quicksearch-gui/src/app/verify.rs index de5c7e3..394f2d8 100644 --- a/crates/quicksearch-gui/src/app/verify.rs +++ b/crates/quicksearch-gui/src/app/verify.rs @@ -1,6 +1,6 @@ //! Byte-for-byte verification of one duplicate group, and the modal that -//! reports it. Grouping only hashes each file's size and head, so this is the -//! second opinion, asked for before what usually follows it: deletion. +//! reports it. Grouping only hashes each file's size, head and tail, so this +//! is the second opinion, asked for before what usually follows it: deletion. use super::*; @@ -173,8 +173,8 @@ pub(crate) fn verify_modal(ctx: &egui::Context, modal: &VerifyModal) -> bool { ui.colored_label(color, summary_line(report)); if !identical { ui.label( - "Grouping only reads each file's size and how it begins; \ - this read every byte.", + "Grouping only reads each file's size and how it begins and \ + ends; this read every byte.", ); // Answers the next question: the files look the same when // opened, so how can they differ? Named formats, because diff --git a/crates/quicksearch-gui/src/backend.rs b/crates/quicksearch-gui/src/backend.rs index 5932dab..8089059 100644 --- a/crates/quicksearch-gui/src/backend.rs +++ b/crates/quicksearch-gui/src/backend.rs @@ -3,7 +3,8 @@ //! //! The duplicates scan and the verification run on throwaway threads fired //! by user actions. The verification can hold a large group's worth of file -//! handles, so it carries a cancel flag and shutdown raises it. +//! handles and the scan reads a file per listed member, so each carries a +//! cancel flag and shutdown raises both. use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; @@ -12,7 +13,9 @@ use std::sync::{mpsc, Arc}; use quicksearch_core::config::Config; use quicksearch_core::coordinator::IndexCoordinator; use quicksearch_core::live::{LiveUpdate, LiveWatcher}; -use quicksearch_core::search::{DuplicateGroup, SearchService, SearchUpdate}; +use quicksearch_core::search::{ + find_duplicate_groups, split_by_tail, DuplicateGroup, SearchService, SearchUpdate, +}; use quicksearch_core::shutdown; use quicksearch_core::verify::{verify_identical, VerifyUpdate}; @@ -29,9 +32,32 @@ impl VerifyJob { } } +/// A finished duplicates scan. +pub struct DupScan { + pub groups: Vec, + /// The fetch came back full at this many groups, so the index may hold + /// more. Taken before the tail check, which can leave fewer listed. + pub limited_to: Option, +} + +/// A duplicates scan in flight. It sends one answer, once the tail check is +/// done, so no group reaches the tab on its head hash alone. Cancelling drops +/// the members the tail check has not reached, so an answer about to be +/// discarded lands sooner. +pub struct DupJob { + pub rx: mpsc::Receiver>, + cancel: Arc, +} + +impl DupJob { + pub fn cancel(&self) { + self.cancel.store(true, Ordering::Relaxed); + } +} + /// Duplicate groups a scan is asked for. Every group listed costs one row -/// fetch and one member query to hydrate, so this is a real cost, not a -/// display cap. +/// fetch and one member query to hydrate, and a file read per member for the +/// tail check, so this is a real cost, not a display cap. pub const DUP_SCAN_LIMIT: u32 = 500; /// …and the most the hidden set may add to it. const DUP_HIDDEN_ALLOWANCE: u32 = 4_500; @@ -49,7 +75,7 @@ pub struct Backend { pub coordinator: Arc, pub search: Option, pub search_rx: mpsc::Receiver, - pub dup_job: Option, String>>>, + pub dup_job: Option, pub verify_job: Option, /// Watches the on-screen results; `None` only after [`Backend::shutdown`]. pub live: Option, @@ -152,24 +178,21 @@ impl Backend { } /// Ignored while a scan is already running: a second one would re-read the - /// whole hash index for an answer the first is about to produce. Returns - /// the limit it asked for, which tells the tab a full page from a - /// complete one; 0 when nothing was started. - pub fn start_duplicates(&mut self, config: &Config, ctx: egui::Context) -> u32 { + /// whole hash index for an answer the first is about to produce. + pub fn start_duplicates(&mut self, config: &Config, ctx: egui::Context) { if self.dup_job.is_some() { - return 0; + return; } let limit = dup_scan_limit(config.duplicates.hidden_groups.len()); - let (tx, rx) = mpsc::channel(); - let db = config.resolved_database_path(); - std::thread::spawn(move || { - let result = - quicksearch_core::search::find_duplicate_groups(&db.to_string_lossy(), limit); - let _ = tx.send(result); - ctx.request_repaint(); - }); - self.dup_job = Some(rx); - limit + self.dup_job = Some(spawn_dup_scan(config.clone(), limit, ctx)); + } + + /// Hurry the running duplicates scan, if any, whose answer is about to be + /// discarded. It still lands, and `dup_job` clears then. + pub fn cancel_duplicates(&self) { + if let Some(job) = &self.dup_job { + job.cancel(); + } } /// Compare every member against the first, byte for byte, on a worker @@ -205,6 +228,7 @@ impl Backend { /// holds the closing window about a second, never indefinitely. pub fn shutdown(&mut self) { self.cancel_verify(); + self.cancel_duplicates(); if let Some(search) = self.search.take() { search.shutdown(); } @@ -214,3 +238,153 @@ impl Backend { self.coordinator.shutdown(); } } + +/// Run [`scan_duplicates`] on a thread of its own, which sends its answer and +/// nothing else. +fn spawn_dup_scan(config: Config, limit: u32, ctx: egui::Context) -> DupJob { + let (tx, rx) = mpsc::channel(); + let cancel = Arc::new(AtomicBool::new(false)); + let worker_cancel = cancel.clone(); + std::thread::spawn(move || { + let _ = tx.send(scan_duplicates(&config, limit, &worker_cancel)); + ctx.request_repaint(); + }); + DupJob { rx, cancel } +} + +/// One duplicates scan, as its thread runs it: the `limit` groups wasting the +/// most, then the tail check on just those. Groups it dissolves are not +/// replaced, so fewer than `limit` may be listed. +fn scan_duplicates(config: &Config, limit: u32, cancel: &AtomicBool) -> Result { + let db = config.resolved_database_path(); + let groups = find_duplicate_groups(&db.to_string_lossy(), limit)?; + let limited_to = (groups.len() as u32 >= limit).then_some(limit); + // See `watch_live` for why the index must not be opened. + let groups = split_by_tail( + groups, + config.processing.hash_length, + |path| config.is_index_file(path), + cancel, + ); + Ok(DupScan { groups, limited_to }) +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::mpsc::RecvTimeoutError; + use std::time::Duration; + + use quicksearch_core::indexing::{IndexingService, IndexingStatus}; + use quicksearch_core::search::duplicates::Member; + use quicksearch_core::testutil::{touch, wait_for, Scratch}; + + /// Each group's member names, in listed order. + fn names<'a>(groups: impl IntoIterator>) -> Vec> { + groups + .into_iter() + .map(|members| members.iter().map(|m| m.1.as_str()).collect()) + .collect() + } + + fn listed(groups: &[DuplicateGroup]) -> Vec> { + names(groups.iter().map(|g| &g.members)) + } + + /// An indexed root holding two pairs whose members share a head: `fake_*`, + /// whose tails differ, and `real_*`, real copies. The scratch guards come + /// back with the config. + fn indexed_pairs(tag: &str) -> (Scratch, Scratch, Config) { + let root = Scratch::canonical(&format!("{tag}-root")); + let (db_dir, db) = Scratch::db(&format!("{tag}-db")); + let mut config = Config::default(); + config.paths.indexing_paths = vec![root.to_string_lossy().into_owned()]; + config.paths.database_path = db.to_string_lossy().into_owned(); + config.processing.hash_length = 1024; + // The biggest waste by head alone, but the tails differ. + let mut fake = vec![b'f'; 4000]; + touch(&root.join("fake_a.bin"), &fake); + fake[3999] = b'g'; + touch(&root.join("fake_b.bin"), &fake); + touch(&root.join("real_a.bin"), &[b'r'; 3000]); + touch(&root.join("real_b.bin"), &[b'r'; 3000]); + + let service = IndexingService::new(); + service + .start_indexing( + config.paths.indexing_paths.clone(), + config.paths.database_path.clone(), + config.clone(), + ) + .unwrap(); + wait_for("the run to end", Duration::from_secs(120), || { + matches!( + service.get_status(), + IndexingStatus::Idle | IndexingStatus::Error(_) + ) + }); + assert!(matches!(service.get_status(), IndexingStatus::Idle)); + let head_only = find_duplicate_groups(&config.paths.database_path, 10).unwrap(); + assert_eq!( + names(head_only.iter().map(|g| &g.members)), + [["fake_a.bin", "fake_b.bin"], ["real_a.bin", "real_b.bin"]], + "the head alone groups both pairs" + ); + (root, db_dir, config) + } + + /// The tail check reads only the groups the fetch returned: one it + /// dissolves leaves a gap rather than pulling in the next group. + #[test] + fn the_scan_tail_checks_only_the_groups_it_fetched() { + let (_root, _db_dir, config) = indexed_pairs("dup-scan"); + let cancel = AtomicBool::new(false); + let top = scan_duplicates(&config, 1, &cancel).unwrap(); + assert!( + top.groups.is_empty(), + "the one group fetched was dissolved and not replaced: {:?}", + listed(&top.groups) + ); + assert_eq!(top.limited_to, Some(1), "the fetch itself was full"); + + let all = scan_duplicates(&config, 10, &cancel).unwrap(); + assert_eq!(listed(&all.groups), [["real_a.bin", "real_b.bin"]]); + assert_eq!(all.limited_to, None); + } + + /// The scan thread sends the tab one answer, the tail-checked one, and + /// then hangs up: no head-only group is ever on the channel, even for a + /// moment, and a cancelled scan lists nothing it did not check. + #[test] + fn nothing_reaches_the_tab_before_the_tail_check() { + let (_root, _db_dir, config) = indexed_pairs("dup-send"); + let head_only = find_duplicate_groups(&config.paths.database_path, 10).unwrap(); + let checked = split_by_tail( + head_only, + config.processing.hash_length, + |_| false, + &AtomicBool::new(false), + ); + + let job = spawn_dup_scan(config.clone(), 10, egui::Context::default()); + let first = job.rx.recv_timeout(Duration::from_secs(60)); + let sent = first.expect("the scan sent nothing").unwrap(); + assert_eq!(sent.groups, checked, "what was sent was not checked"); + assert_eq!(listed(&sent.groups), [["real_a.bin", "real_b.bin"]]); + assert!( + matches!( + job.rx.recv_timeout(Duration::from_secs(60)), + Err(RecvTimeoutError::Disconnected) + ), + "the scan sent a second message" + ); + + let cancelled = scan_duplicates(&config, 10, &AtomicBool::new(true)).unwrap(); + assert!( + cancelled.groups.is_empty(), + "a cancelled scan listed what it never read: {:?}", + listed(&cancelled.groups) + ); + } +} diff --git a/crates/quicksearch-gui/src/duplicates_tab.rs b/crates/quicksearch-gui/src/duplicates_tab.rs index 2b2f815..c4c97c7 100644 --- a/crates/quicksearch-gui/src/duplicates_tab.rs +++ b/crates/quicksearch-gui/src/duplicates_tab.rs @@ -1,14 +1,17 @@ //! The Duplicates tab: groups of files sharing a content hash. //! //! Grouping is a suspicion, not a verdict: the hash covers each file's size -//! and its first `processing.hash_length` bytes and nothing else. The tab says +//! and its first `processing.hash_length` bytes, and the scan then splits +//! groups whose members end differently and leaves out any member it cannot +//! read (`search::split_by_tail`); nothing in between is read. The tab says //! so where someone about to delete something will read it, and offers the //! byte-for-byte verification that settles it. //! //! Two ways to stop seeing a group again: exclude paths by glob (a folder that -//! is *meant* to hold copies), or hide one group by its hash (a head-hash -//! false positive). Both are persisted, and both are applied to the listing -//! already in hand rather than by going back to the database. +//! is *meant* to hold copies), or hide one group by its identity, head hash +//! and tail digest (a false positive that differs only in between). Both are +//! persisted, and both are applied to the listing already in hand rather than +//! by going back to the database. use std::collections::HashSet; @@ -55,18 +58,24 @@ fn extension_of(name: &str) -> String { } /// What [`LoadedGroups::rebuild`] orders an extension listing by: no extension -/// last, then the extension, then the biggest waste, then the hash to settle -/// whatever is left. -type ExtensionKey<'a> = (bool, &'a str, std::cmp::Reverse, &'a [u8]); +/// last, then the extension, then the biggest waste, then the identity to +/// settle whatever is left. +type ExtensionKey<'a> = ( + bool, + &'a str, + std::cmp::Reverse, + (&'a [u8], &'a [u8; 32]), +); fn extension_key<'a>(row: &'a Row, groups: &'a [DuplicateGroup]) -> ExtensionKey<'a> { + let group = &groups[row.group]; ( // Extensionless groups last: a category of their own, and not an // interesting one. row.extension.is_empty(), row.extension.as_str(), std::cmp::Reverse(row.redundant), - groups[row.group].hash.as_slice(), + (group.hash.as_slice(), &group.tail), ) } @@ -131,18 +140,18 @@ impl DupFilters { } } - fn hide(&mut self, hash_hex: String) { - if self.hidden.contains(&hash_hex) { + fn hide(&mut self, identity: String) { + if self.hidden.contains(&identity) { return; } - self.config.hidden_groups.push(hash_hex); + self.config.hidden_groups.push(identity); self.recompile(); } - fn unhide(&mut self, hash_hex: &str) { + fn unhide(&mut self, identity: &str) { self.config .hidden_groups - .retain(|h| !h.trim().eq_ignore_ascii_case(hash_hex)); + .retain(|h| !h.trim().eq_ignore_ascii_case(identity)); self.recompile(); } @@ -156,7 +165,7 @@ impl DupFilters { } fn is_hidden(&self, group: &DuplicateGroup) -> bool { - !self.hidden.is_empty() && self.hidden.contains(&group.hash_hex()) + !self.hidden.is_empty() && self.hidden.contains(&group.identity()) } } @@ -200,6 +209,9 @@ fn totals(group: &DuplicateGroup, members: &[usize]) -> (i64, i64, i64) { /// frame, on a list that repaints at 20 Hz for as long as the tab is open. pub struct LoadedGroups { groups: Vec, + /// The scan's fetch came back full at this many groups: there may be + /// more. See [`crate::backend::DupScan`]. + limited_to: Option, /// The visible groups, in display order. rows: Vec, /// Groups the filters kept off screen, for the line that says so. @@ -209,9 +221,10 @@ pub struct LoadedGroups { } impl LoadedGroups { - pub fn new(groups: Vec) -> LoadedGroups { + pub fn new(groups: Vec, limited_to: Option) -> LoadedGroups { LoadedGroups { groups, + limited_to, rows: Vec::new(), filtered_out: 0, built_for: None, @@ -287,9 +300,6 @@ pub struct DuplicatesTab { pub filters: DupFilters, /// List the groups the filters hid, so hiding one is reversible. show_hidden: bool, - /// What the last scan was asked for; a result of exactly this many groups - /// is a truncated one. See [`crate::backend::Backend::start_duplicates`]. - pub scan_limit: u32, /// The exclusion being typed. draft: String, } @@ -307,7 +317,8 @@ pub struct DuplicatesActions { const VERIFY_LABEL: &str = "Verify copies are identical…"; const VERIFY_TIP: &str = "Reads every file in the group in full and compares them byte for \ - byte. Grouping only reads each file's size and how it begins."; + byte. Grouping only reads each file's size and how it begins \ + and ends."; const HIDE_LABEL: &str = "Hide this group"; const UNHIDE_LABEL: &str = "Unhide this group"; const NO_GROUPS: &str = "No duplicate files found."; @@ -331,7 +342,6 @@ impl DuplicatesTab { sort: DupSort::default(), filters: DupFilters::new(filters), show_hidden: false, - scan_limit: 0, draft: String::new(), } } @@ -415,19 +425,19 @@ impl DuplicatesTab { } DupState::Loaded(loaded) => { // "Limited to", not "showing": the scan asks for extra to - // cover the hidden set, so the number it found is not always - // the number on screen. The filtered-out line below accounts - // for the difference. - if loaded.groups.len() as u32 >= self.scan_limit && self.scan_limit > 0 { + // cover the hidden set, and the tail check dissolves some of + // what it found, so the number is not the number on screen. + // The filtered-out line below accounts for the hidden ones. + if let Some(limit) = loaded.limited_to { ui.label(hint(match self.sort { DupSort::Reclaimable => { - format!("Limited to the {} largest groups.", self.scan_limit) + format!("Limited to the {} largest groups.", limit) } // Said plainly: this is not every .raw file you own, // it is the biggest groups put in that order. DupSort::Extension => format!( "Limited to the {} largest groups, then ordered by extension.", - self.scan_limit + limit ), })); } @@ -443,8 +453,10 @@ impl DuplicatesTab { .show(ui, |ui| { for row in &loaded.rows { let group = &loaded.groups[row.group]; + // Keyed by identity, so a group stays open + // through a rescan that lists it elsewhere. let header = egui::CollapsingHeader::new(&row.title) - .id_salt(row.group) + .id_salt((&group.hash, &group.tail)) .show(ui, |ui| { for &j in &row.members { member_row(ui, group, row, j, verify_open, &mut picked); @@ -463,12 +475,12 @@ impl DuplicatesTab { match picked { Some(Pick::Verify(paths)) => actions.verify = Some(paths), - Some(Pick::Hide(hex)) => { - self.filters.hide(hex); + Some(Pick::Hide(identity)) => { + self.filters.hide(identity); edited = true; } - Some(Pick::Unhide(hex)) => { - self.filters.unhide(&hex); + Some(Pick::Unhide(identity)) => { + self.filters.unhide(&identity); edited = true; } Some(Pick::Exclude(pattern)) => { @@ -585,10 +597,10 @@ fn caution_banner(ui: &mut egui::Ui, hash_length: usize) { ui.spacing_mut().item_spacing.x = 0.0; ui.label(egui::RichText::new("These are suspected duplicates. ").color(p.orange)); ui.label(format!( - "Files are grouped by their size and their first {}, which is all \ - indexing reads. That is not proof that they are identical: \ - right-click a group and choose \"{}\" to compare every byte before \ - you delete anything.", + "Files are grouped by their size and their first {0}, which is all \ + indexing reads, then split apart where their last {0} differ. That \ + is not proof that they are identical: right-click a group and \ + choose \"{1}\" to compare every byte before you delete anything.", human_size(hash_length as u64), VERIFY_LABEL.trim_end_matches('…'), )); @@ -646,11 +658,11 @@ fn group_menu( } ui.separator(); if hide_entry(ui, row.hidden) { - let hex = group.hash_hex(); + let identity = group.identity(); *picked = Some(if row.hidden { - Pick::Unhide(hex) + Pick::Unhide(identity) } else { - Pick::Hide(hex) + Pick::Hide(identity) }); } } diff --git a/crates/quicksearch-gui/src/duplicates_tab/tests.rs b/crates/quicksearch-gui/src/duplicates_tab/tests.rs index b67ae70..26403d2 100644 --- a/crates/quicksearch-gui/src/duplicates_tab/tests.rs +++ b/crates/quicksearch-gui/src/duplicates_tab/tests.rs @@ -1,5 +1,11 @@ use super::*; +use std::sync::atomic::AtomicBool; + +use quicksearch_core::config::Config; +use quicksearch_core::search::{split_by_tail, HeadGroup}; +use quicksearch_core::testutil::Scratch; + use crate::test_ui::{click_at, click_with, painted_text, painted_text_center, run_panel}; const SCREEN: egui::Vec2 = egui::vec2(1000.0, 700.0); @@ -9,6 +15,7 @@ const HASH_LENGTH: usize = 8192; fn group(paths: &[&str]) -> DuplicateGroup { DuplicateGroup { hash: vec![0xab; 32], + tail: [0xcd; 32], members: paths .iter() .enumerate() @@ -22,8 +29,7 @@ fn group(paths: &[&str]) -> DuplicateGroup { fn tab(groups: Vec, filters: DuplicatesConfig) -> DuplicatesTab { let mut tab = DuplicatesTab::new(&filters); - tab.scan_limit = crate::backend::DUP_SCAN_LIMIT; - tab.state = DupState::Loaded(LoadedGroups::new(groups)); + tab.state = DupState::Loaded(LoadedGroups::new(groups, None)); tab } @@ -214,6 +220,22 @@ fn an_empty_result_says_so_rather_than_showing_an_empty_list() { ); } +/// Whether the listing was cut short is the fetch's to say: the tail check +/// can leave a full fetch listing fewer groups than it asked for. +#[test] +fn a_full_fetch_says_it_was_limited_however_many_are_listed() { + let ctx = crate::test_ui::ctx(); + let limited = |limited_to: Option| { + let mut tab = DuplicatesTab::new(&DuplicatesConfig::default()); + tab.state = DupState::Loaded(LoadedGroups::new(vec![group(&PATHS)], limited_to)); + painted_text(&frame(&ctx, &mut tab, false, Vec::new()).0) + .iter() + .any(|t| t.contains("Limited to the 500 largest groups")) + }; + assert!(limited(Some(500)), "one group left of a full fetch"); + assert!(!limited(None), "a fetch short of its limit was everything"); +} + // --- Ordering --------------------------------------------------------------- /// A group named by its first member, with the waste that decides the @@ -240,7 +262,7 @@ fn names(loaded: &LoadedGroups) -> Vec { /// The names of the groups, in the order the tab would list them. fn listed(groups: Vec, sort: DupSort) -> Vec { let filters = DupFilters::new(&DuplicatesConfig::default()); - let mut loaded = LoadedGroups::new(groups); + let mut loaded = LoadedGroups::new(groups, None); loaded.rebuild(sort, &filters, false); names(&loaded) } @@ -312,7 +334,7 @@ fn the_order_returns_when_the_choice_does() { sized_group("middling.txt", 100), ]; let filters = DupFilters::new(&DuplicatesConfig::default()); - let mut loaded = LoadedGroups::new(groups); + let mut loaded = LoadedGroups::new(groups, None); loaded.rebuild(DupSort::Extension, &filters, false); loaded.rebuild(DupSort::Reclaimable, &filters, false); assert_eq!(names(&loaded), ["big.txt", "small.jpg", "middling.txt"]); @@ -541,15 +563,15 @@ fn an_unparseable_pattern_is_reported_rather_than_silently_dropped() { fn hiding_a_group_takes_it_off_the_list_and_is_persisted() { let ctx = crate::test_ui::ctx(); let mut tab = loaded(&PATHS); - let hash = group(&PATHS).hash_hex(); + let identity = group(&PATHS).identity(); let header = header_of(&ctx, &mut tab); context_menu_on(&ctx, &mut tab, false, &header); let actions = click_menu_entry(&ctx, &mut tab, HIDE_LABEL); assert_eq!( actions.save_filters.map(|f| f.hidden_groups), - Some(vec![hash]), - "hiding is not remembered by content hash" + Some(vec![identity]), + "hiding is not remembered by head and tail" ); let painted = painted_text(&frame(&ctx, &mut tab, false, Vec::new()).0); assert!(painted.contains(&ALL_FILTERED.to_string()), "{painted:?}"); @@ -566,7 +588,7 @@ fn a_group_hidden_in_the_config_is_not_listed() { let ctx = crate::test_ui::ctx(); let mut tab = tab( vec![group(&PATHS), sized_group("keep.txt", 40)], - hiding(&[group(&PATHS).hash_hex()]), + hiding(&[group(&PATHS).identity()]), ); let out = frame(&ctx, &mut tab, false, Vec::new()).0; let listed = headers(&out); @@ -584,7 +606,7 @@ fn a_group_hidden_in_the_config_is_not_listed() { #[test] fn show_hidden_lists_them_again_and_offers_the_way_back() { let ctx = crate::test_ui::ctx(); - let mut tab = tab(vec![group(&PATHS)], hiding(&[group(&PATHS).hash_hex()])); + let mut tab = tab(vec![group(&PATHS)], hiding(&[group(&PATHS).identity()])); let (out, _) = frame(&ctx, &mut tab, false, Vec::new()); let toggle = painted_text_center(&out, "Show hidden").expect("no Show hidden control"); let (out, _) = frame(&ctx, &mut tab, false, click_at(toggle)); @@ -610,7 +632,7 @@ fn clear_empties_the_hidden_list() { let ctx = crate::test_ui::ctx(); let mut tab = tab( vec![group(&PATHS)], - hiding(&[group(&PATHS).hash_hex(), "00".repeat(32)]), + hiding(&[group(&PATHS).identity(), "00".repeat(64)]), ); let (out, _) = frame(&ctx, &mut tab, false, Vec::new()); let clear = painted_text_center(&out, "Clear").expect("no Clear button"); @@ -622,18 +644,108 @@ fn clear_empties_the_hidden_list() { assert_eq!(headers(&out).len(), 1, "the group did not come back"); } -/// A hash spelled in capitals by a hand-edited config still means that group. +/// An identity spelled in capitals by a hand-edited config still means that +/// group. #[test] -fn hidden_hashes_are_matched_case_insensitively() { +fn hidden_identities_are_matched_case_insensitively() { let ctx = crate::test_ui::ctx(); let mut tab = tab( vec![group(&PATHS)], - hiding(&[group(&PATHS).hash_hex().to_uppercase()]), + hiding(&[group(&PATHS).identity().to_uppercase()]), ); let painted = painted_text(&frame(&ctx, &mut tab, false, Vec::new()).0); assert!(painted.contains(&ALL_FILTERED.to_string()), "{painted:?}"); } +/// A head hash alone, as earlier versions saved a hidden group, names no +/// group now: every listed group is a head and a tail. +#[test] +fn a_head_only_entry_hides_nothing() { + let ctx = crate::test_ui::ctx(); + let head_only = group(&PATHS).identity()[..64].to_string(); + let mut tab = tab(vec![group(&PATHS)], hiding(&[head_only])); + let out = frame(&ctx, &mut tab, false, Vec::new()).0; + assert_eq!(headers(&out).len(), 1, "an old entry hid a group"); +} + +/// The head the files below were "indexed" with. +const TAIL: usize = 16; + +/// Two sets of copies under `dir` that start alike and end apart, as the tail +/// check lists them: `a1.bin` and `a2.bin`, then `b1.bin` and `b2.bin`. +fn sharing_a_head(dir: &std::path::Path) -> Vec { + let members = ["a1.bin", "a2.bin", "b1.bin", "b2.bin"] + .iter() + .enumerate() + .map(|(i, name)| { + let mut body = vec![b'h'; 3 * TAIL]; + body[3 * TAIL - 1] = name.as_bytes()[0]; + let path = dir.join(name); + std::fs::write(&path, &body).unwrap(); + let path = path.to_str().unwrap().to_string(); + let size = body.len() as u64; + (i as i64, name.to_string(), path, size, 1_700_000_000) + }) + .collect(); + let head = HeadGroup { + hash: vec![0xab; 32], + members, + }; + split_by_tail(vec![head], TAIL, |_| false, &AtomicBool::new(false)) +} + +/// Hides whichever group is listed first; what the tab hands back to save. +fn hide_the_first(ctx: &egui::Context, tab: &mut DuplicatesTab) -> DuplicatesConfig { + let header = header_of(ctx, tab); + context_menu_on(ctx, tab, false, &header); + click_menu_entry(ctx, tab, HIDE_LABEL) + .save_filters + .expect("the hide was not handed back to be saved") +} + +/// One head hash, two real sets of copies: hiding one leaves the other. +#[test] +fn hiding_one_of_two_groups_sharing_a_head_leaves_the_other() { + let ctx = crate::test_ui::ctx(); + let dir = Scratch::dir("dup-tab-shared-head"); + let groups = sharing_a_head(&dir); + assert_eq!(groups.len(), 2, "the tail check did not split them"); + let mut tab = tab(groups.clone(), DuplicatesConfig::default()); + + let saved = hide_the_first(&ctx, &mut tab); + assert_eq!(saved.hidden_groups, [groups[0].identity()]); + let listed = headers(&frame(&ctx, &mut tab, false, Vec::new()).0); + assert_eq!(listed.len(), 1, "{listed:?}"); + assert!( + listed[0].contains("b1.bin"), + "the wrong one is listed: {listed:?}" + ); +} + +/// The hidden set is written to the config and read back at the next start, +/// where the identity it holds must still name that group, and only it. +#[test] +fn a_hidden_group_stays_hidden_through_a_saved_config() { + let ctx = crate::test_ui::ctx(); + let dir = Scratch::dir("dup-tab-hidden-config"); + let groups = sharing_a_head(&dir); + let mut before = tab(groups.clone(), DuplicatesConfig::default()); + let path = dir.join("config.toml"); + let config = Config { + duplicates: hide_the_first(&ctx, &mut before), + source: Some(path.clone()), + ..Config::default() + }; + config.save().unwrap(); + + let loaded = Config::load_from(&path).unwrap(); + assert_eq!(loaded.duplicates.hidden_groups, [groups[0].identity()]); + let mut after = tab(groups, loaded.duplicates); + let listed = headers(&frame(&ctx, &mut after, false, Vec::new()).0); + assert_eq!(listed.len(), 1, "{listed:?}"); + assert!(listed[0].contains("b1.bin"), "{listed:?}"); +} + // --- The two together ------------------------------------------------------- /// The extension listing orders on what the titles say, which after an @@ -654,7 +766,7 @@ fn the_extension_order_follows_the_recomputed_waste() { ); let filters = DupFilters::new(&excluding(&["/keep-out/*"])); - let mut loaded = LoadedGroups::new(vec![big, small]); + let mut loaded = LoadedGroups::new(vec![big, small], None); loaded.rebuild(DupSort::Extension, &filters, false); assert_eq!( names(&loaded), @@ -672,7 +784,7 @@ fn the_extension_order_follows_the_recomputed_waste() { #[test] fn an_unchanged_listing_is_not_rebuilt() { let filters = DupFilters::new(&DuplicatesConfig::default()); - let mut loaded = LoadedGroups::new(vec![group(&PATHS)]); + let mut loaded = LoadedGroups::new(vec![group(&PATHS)], None); loaded.rebuild(DupSort::Reclaimable, &filters, false); let built = loaded.built_for; loaded.rows.clear(); diff --git a/crates/quicksearch-gui/src/help_tab.rs b/crates/quicksearch-gui/src/help_tab.rs index d2e65eb..f2df7c6 100644 --- a/crates/quicksearch-gui/src/help_tab.rs +++ b/crates/quicksearch-gui/src/help_tab.rs @@ -341,9 +341,11 @@ fn duplicates_section(ui: &mut egui::Ui) { ui.add_space(4.0); ui.label( "A group is a strong suspicion, not a verdict: files are grouped by \ - their size and a hash of their first few kilobytes, and the rest is \ - never read. Before deleting anything, right-click the group and choose \ - Verify copies are identical to compare every byte.", + their size and a hash of their first few kilobytes, then split apart \ + where their last few kilobytes differ, and the rest is never read. \ + A file whose end cannot be read is left out. Before deleting \ + anything, right-click the group and choose Verify copies are \ + identical to compare every byte.", ); ui.add_space(8.0); diff --git a/crates/quicksearch-gui/src/settings_tab.rs b/crates/quicksearch-gui/src/settings_tab.rs index fcaf76c..cb58b60 100644 --- a/crates/quicksearch-gui/src/settings_tab.rs +++ b/crates/quicksearch-gui/src/settings_tab.rs @@ -735,13 +735,14 @@ fn search_cache_hint(config: &Config, indexed_files: Option) -> Option SEARCH_CACHE_MAX_MIB { diff --git a/crates/quicksearch-gui/src/settings_tab/tests.rs b/crates/quicksearch-gui/src/settings_tab/tests.rs index 62a61a6..f0278d4 100644 --- a/crates/quicksearch-gui/src/settings_tab/tests.rs +++ b/crates/quicksearch-gui/src/settings_tab/tests.rs @@ -524,14 +524,14 @@ fn the_search_cache_hint_explains_the_automatic_value() { // came from. cfg.security.password_protected = true; let keyed = search_cache_hint(&cfg, Some(200_000)).expect("a hint"); - assert!(keyed.contains("32 MiB"), "{}", keyed); + assert!(keyed.contains("28 MiB"), "{}", keyed); assert!(keyed.contains("200,000"), "{}", keyed); // Encrypted and past it: says so, and says what the index actually wants, // or the override cannot be found by the people who need it. let capped = search_cache_hint(&cfg, Some(2_000_000)).expect("a hint"); assert!( - capped.contains("128 MiB") && capped.contains("320 MiB"), + capped.contains("192 MiB") && capped.contains("286 MiB"), "the capped hint must name both the cap and the want: {}", capped ); diff --git a/crates/quicksearch-gui/src/tips.rs b/crates/quicksearch-gui/src/tips.rs index 5cfbf04..26f0473 100644 --- a/crates/quicksearch-gui/src/tips.rs +++ b/crates/quicksearch-gui/src/tips.rs @@ -133,7 +133,9 @@ pub static HASH_LENGTH: Tip = Tip { body: "How many bytes from the start of each file QuickSearch reads to \ recognise it. They make the fingerprint the Duplicates tab groups \ files by, tell what kind of file it is, and for a small text file \ - are the whole text.\n\n\ + are the whole text. The Duplicates tab also reads as many bytes \ + from the end of each file it lists, and splits apart files that \ + only start alike.\n\n\ Reading more is more reliable but slower, especially over a \ network.", examples: &[ @@ -268,13 +270,13 @@ pub static DISPLAY_LIMIT: Tip = Tip { pub static SEARCH_CACHE: Tip = Tip { title: "Search cache", body: "Memory, in MiB, that searching keeps index pages in. 0 sizes it \ - for your index automatically, up to 128 MiB; the value it picks is \ + for your index automatically, up to 192 MiB; the value it picks is \ shown below.\n\n\ It matters on an encrypted index, which must decrypt every page the \ cache misses: a cache smaller than the one shown can make searches \ - about four times slower. An unencrypted index gets a small fixed \ - cache, because a larger one measures no faster. The memory is \ - released after a long idle.", + about one and a half times slower. An unencrypted index gets a small fixed \ + cache to save memory; a larger one makes its searches at most about a \ + quarter faster. The memory is released after a long idle.", examples: &[ "0, the default, is right unless your folders are nested unusually deep.", "a fixed value to cap the memory QuickSearch holds, at the cost of slower \ diff --git a/crates/quicksearch-gui/src/tutorial.rs b/crates/quicksearch-gui/src/tutorial.rs index b593659..68f9599 100644 --- a/crates/quicksearch-gui/src/tutorial.rs +++ b/crates/quicksearch-gui/src/tutorial.rs @@ -179,8 +179,8 @@ const PAGES: &[Page] = &[ extension if you prefer.", "It is a quick way to find the same download sitting in three \ places. It only shows the groups; deleting is up to you.", - "For speed, files are matched on size and their first few \ - kilobytes, so a group is a strong suspicion, not a certainty. \ + "For speed, files are matched on size and their first and last \ + few kilobytes, so a group is a strong suspicion, not a certainty. \ Right-click a group and choose \"Verify copies are identical\" to \ compare every byte before you delete anything.", ],