diff --git a/Cargo.lock b/Cargo.lock index 27d20a0..143d5aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2119,6 +2119,15 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libmimalloc-sys" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +dependencies = [ + "cc", +] + [[package]] name = "libredox" version = "0.1.16" @@ -2280,6 +2289,15 @@ dependencies = [ "hex", ] +[[package]] +name = "mimalloc" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "mime" version = "0.3.17" @@ -3090,7 +3108,7 @@ dependencies = [ [[package]] name = "quicksearch-core" -version = "1.1.5" +version = "1.1.6" dependencies = [ "argon2", "cfb", @@ -3105,9 +3123,11 @@ dependencies = [ "id3", "infer", "libc", + "libmimalloc-sys", "lofty", "memchr", "metaflac", + "mimalloc", "mime_guess", "notify", "pdf-extract", @@ -3130,7 +3150,7 @@ dependencies = [ [[package]] name = "quicksearch-gui" -version = "1.1.5" +version = "1.1.6" dependencies = [ "ashpd", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 687ba2d..96c2e32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ pdf-extract = { path = "vendor/pdf-extract" } # Patched unbounded reads which co rtf-parser = { path = "vendor/rtf-parser" } # Patched a parsing error which occurs on UTF-16 characters [workspace.package] -version = "1.1.5" +version = "1.1.6" edition = "2021" license = "GPL-3.0-or-later" authors = ["Jeremy "] diff --git a/README.md b/README.md index dc2ef31..906e5fa 100644 --- a/README.md +++ b/README.md @@ -348,14 +348,20 @@ rebuilds the index — there is no in-place conversion. - The key is derived as `Argon2id(password, salt)`; the salt is written to `config.toml` when the password is set (it is unique, not secret, and required — keep it with the config if you copy a protected setup). +- Pages are AES-256-CBC at an 8192-byte page size, with SQLCipher's + per-page HMAC **deliberately disabled** — it only detects tampering by + someone who could already read the indexed files directly, and it costs + 1.78x on search. Confidentiality is unchanged. - **Remember on this device** stores the derived key (never the password) in the OS keychain — Secret Service/KWallet on Linux, Credential Manager on Windows — and skips the prompt. Without a keychain daemon the option quietly falls back to prompting. - **Show database key** asks for the password, then shows the raw SQLCipher - key as `0x…` (64 hex digits) with a copy button, for opening the index in - other SQLCipher tools. That key alone reads the index, so treat a copy of - it as carefully as the password. + key as `0x…` (64 hex digits) with a copy button, alongside the page size + and HMAC setting another tool has to be given — on SQLCipher's defaults + the index decrypts to noise and every tool calls a correct key wrong. + That key alone reads the index, so treat a copy of it as carefully as + the password. - Scripts can set `QUICKSEARCH_PASSWORD` for non-interactive terminal search. Environment variables are readable by other processes of the same user (`/proc//environ`) — prefer the keychain. @@ -516,7 +522,12 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. an optimize pass: checkpoint, VACUUM if the file has at least 20% slack, `PRAGMA optimize`, checkpoint again. Progress streams through a polled `IndexingStatus` (`Optimizing` during that pass, `Preparing` - for everything before the first file is walked). + for everything before the first file is walked). The upkeep a run does + *between* files — WAL checkpoints, the stale-row sweep, FTS merges, + the per-root recount — blocks the writer for as long as it takes, so + each announces itself as a `MaintenanceStep` on the published run + rather than leaving the per-file counters frozen and reading as a + hang. - **Scope reconciliation** (`scope.rs`): the index is a cache of what a walk under the configured roots would produce, so a configuration change is a difference between the two, not a reason to start over. diff --git a/config_example.toml b/config_example.toml index 69dd7ed..42fe302 100644 --- a/config_example.toml +++ b/config_example.toml @@ -98,12 +98,20 @@ writer_turn_slice_ms = 100 fts_update_batch_size = 1000 # How large the write-ahead log (index.sqlite-wal) may grow during an # indexing run before the indexer forces a checkpoint (bytes); left -# alone, the log grows for the whole run. A stall-frequency knob, not a -# safety one: on a volume short of space the indexer checkpoints sooner -# than asked and stops the run with an error before the disk fills -# (which would kill the process with SIGBUS through SQLite's mmap'd -# wal-index). 0 disables it; any other value below 16777216 is raised to it. -maximum_wal_size = 536870912 +# alone, the log grows for the whole run. Not a safety knob: on a volume +# short of space the indexer checkpoints sooner than asked and stops the +# run with an error before the disk fills (which would kill the process +# with SIGBUS through SQLite's mmap'd wal-index). 0 disables it; any +# other value below 16777216 is raised to it. +# +# Both directions cost. A checkpoint blocks indexing for its whole +# copy-back, so a low value stalls the run often. A high one is paid by +# readers instead: SQLite searches the log before every page it fetches +# from the index, so a larger log slows searches running alongside a +# run, and it lengthens recovery after a crash or a force-quit. The +# default trades toward fewer stalls; lower it if searching while +# indexing matters more than the run finishing quickly. +maximum_wal_size = 2147483648 # FTS5 tokenizer: 'trigram' (substring matching, the default; gets # remove_diacritics 1 appended), 'unicode61', 'porter', or a full FTS5 # option string. See https://www.sqlite.org/fts5.html#tokenizers @@ -151,6 +159,12 @@ search_hotkey = "Ctrl+Shift+F" # light/dark setting would need the session's D-Bus settings portal, so it # is deliberately not offered. color_scheme = "dark" +# Written by QuickSearch, not by you: whether the Settings tab shows the +# technical settings alongside the everyday ones. The checkbox at the top of +# that tab writes it immediately, without an Apply. Off, the tab hides the +# byte budgets, the tokenizer, the database path and the other knobs whose +# defaults suit almost every installation. +show_advanced_settings = false # Written by QuickSearch, not by you: whether the short introduction shown # on a brand-new installation has been dismissed. Absent means this config # predates that introduction - an installation that upgraded into this @@ -180,6 +194,24 @@ debounce_ms = 150 # so this works whether or not indexing is running, and the index is # brought up to date for those files. Editing the query drops the watches. live_results = true +# Memory the search connection keeps database pages in, in MiB, held for the +# length of a search session and released after a long idle. +# +# 0 sizes it from the index and is almost always right. Every keystroke +# rescans the whole file list, so what has to stay resident is that list: +# roughly 168 bytes per indexed file, capped at 128 MiB automatically. +# +# It matters on an *encrypted* index, which must decrypt any page the cache +# does not already hold. Measured at 600k files: 127 ms per keystroke with a +# cache too small, 34 ms once it fit. An unencrypted index reads a miss +# straight from the operating system and is given a flat 16 MiB, because +# sweeping 1 MiB to 256 MiB on one measured no faster than noise. +# +# Set it explicitly only when the automatic value is wrong for your tree — +# deeply nested folders make wider rows and want more — or when your index is +# over ~800k files, where the automatic cap lands below what it wants. An +# explicit value may exceed that cap; it is clamped to 16..1024 on load. +cache_size_mib = 0 # Which columns the Search tab shows; the right-click menu of any column # header and Settings → Search both write here immediately, without an diff --git a/crates/quicksearch-core/Cargo.toml b/crates/quicksearch-core/Cargo.toml index e255ba6..f12085b 100644 --- a/crates/quicksearch-core/Cargo.toml +++ b/crates/quicksearch-core/Cargo.toml @@ -28,6 +28,21 @@ probe = [] # no runtime library dependencies. With no `PRAGMA key` applied, SQLCipher # behaves identically to stock SQLite, so unencrypted indexes are unaffected. rusqlite = { version = "0.39", features = ["bundled-sqlcipher-vendored-openssl"] } +# The allocator every binary here installs; see `platform::Allocator`. +# +# glibc gives each thread a 64 MiB arena and never shrinks one below its +# high-water mark, so a multi-million-file run settled at 985 MB RSS with +# 871 MB of anonymous slack that `malloc_trim` could not coalesce. +# `glibc.malloc.arena_max=2` cut that to 146 MB but made indexing dramatically +# slower, because two arenas serialise every worker's allocations. mimalloc +# has no such trade: per-thread heaps with no lock on the fast path, and freed +# segments are decommitted rather than parked. +# +# `mimalloc` supplies the `GlobalAlloc` type; `libmimalloc-sys` is what links +# the C library `platform::release_free_heap` calls `mi_collect` from. Both +# compile C, which this build already needs for SQLCipher and OpenSSL. +mimalloc = "0.1" +libmimalloc-sys = "0.1" argon2 = { version = "0.5", features = ["zeroize"] } zeroize = { version = "1", features = ["derive"] } getrandom = "0.2" @@ -161,10 +176,10 @@ harness = false name = "index" harness = false -# Plain-main measurement probes, env-gated (QSB_SEARCH_PERF / QSB_SEARCH_ALLOC) -# so a bare `cargo bench` doesn't pay their seed cost. snippet_perf, which -# compared the pre-schema-v3 FTS shape against v3, is gone: the question is -# decided (v3 won on every axis) — don't re-measure it. +# Plain-main measurement probes, env-gated (QSB_SEARCH_PERF / QSB_SEARCH_ALLOC +# / QSB_INDEX_ALLOC) so a bare `cargo bench` doesn't pay their seed cost. +# snippet_perf, which compared the pre-schema-v3 FTS shape against v3, is +# gone: the question is decided (v3 won on every axis) — don't re-measure it. [[bench]] name = "search_perf" harness = false @@ -172,3 +187,25 @@ harness = false [[bench]] name = "search_alloc" harness = false + +# Per-file allocator traffic through the walk and the extractors. Compiles +# `tests/corpus/` for its fixtures, so it needs the dev-dependency writers. +[[bench]] +name = "index_alloc" +harness = false + +# QSB_PGSZ: the speed half of the page-geometry question that +# `tests/encrypted_perf.rs` gates the size half of. Sweeps database page size +# against FTS5's record size over corpora up to 1M files, so it is opt-in like +# the two above — and wants TMPDIR pointed at real storage, not tmpfs. +[[bench]] +name = "page_geometry" +harness = false + +# QSB_HMAC: prices SQLCipher's per-page authenticator (off / SHA-256 / SHA-512) +# against a plain index, and is what `db::schema::HMAC_MODE` is set from. Same +# corpora and same seed cost as page_geometry, so it is opt-in for the same +# reason. +[[bench]] +name = "cipher_hmac" +harness = false diff --git a/crates/quicksearch-core/benches/cipher_hmac.rs b/crates/quicksearch-core/benches/cipher_hmac.rs new file mode 100644 index 0000000..f7c35b0 --- /dev/null +++ b/crates/quicksearch-core/benches/cipher_hmac.rs @@ -0,0 +1,389 @@ +//! What SQLCipher's per-page authenticator costs, so the build can decide +//! whether to keep one. +//! +//! ```text +//! TMPDIR=/media/shared/qs-scratch QSB_HMAC=1 \ +//! cargo bench -p quicksearch-core --bench cipher_hmac +//! ``` +//! +//! `TMPDIR` is not optional in spirit, for the reason +//! `benches/page_geometry.rs` gives at length: a tmpfs `/tmp` cannot produce a +//! page fetch that was not already in RAM, and the scale tier will not fit +//! besides. Point it at real storage. +//! +//! # The question +//! +//! The cipher is not a choice. SQLCipher 4 removed `PRAGMA cipher` and the +//! provider hard-codes AES-256-CBC, so the only lever the build has is the +//! HMAC — and that lever is worth pulling on because the index holds text read +//! out of files the same user can already read. Anything positioned to *tamper* +//! with the index could read the originals instead, so per-page authentication +//! defends very little while being paid on every page read and every page +//! write. An unprotected index has never had any, either. +//! +//! Three modes, and the reason the middle one is not obviously pointless: +//! +//! | mode | reserve | per page | +//! |---|---|---| +//! | `Sha512` | 80 | SQLCipher's default | +//! | `Sha256` | 48 | SHA-NI on Zen and Ice Lake+, and 32 bytes of page back | +//! | `Off` | 16 | no authenticator at all | +//! +//! The reserve matters twice: it is page space the rows do not get, and +//! `db::schema::fts_pgsz_for` derives FTS5's record size from it, so each arm +//! also gets a differently-shaped leaf. +//! +//! **The write path is the one to watch.** `sqlcipher_openssl_hmac` calls +//! `EVP_MAC_fetch(NULL, "HMAC", NULL)`, `EVP_MAC_CTX_new` and an +//! `EVP_MAC_init` that fetches the digest *by name* — two OpenSSL 3 provider +//! lookups per page, on top of the hash itself. That fixed cost is paid +//! whichever digest is selected, which is why `Sha256` may buy far less than +//! its digest speed suggests, and why `Off` may buy far more. +//! +//! # Reading it +//! +//! The plain arm is the noise floor, not a candidate: it is what the product +//! does with no password set. Rank the three keyed arms against each other and +//! against it. + +use std::time::{Duration, Instant}; + +use quicksearch_core::db; +use quicksearch_core::db::schema::HmacMode; +use quicksearch_core::query::split::split_for_cascade; +use quicksearch_core::search::{cascade, find_duplicate_groups, SearchHit, SearchOptions}; +use quicksearch_core::testutil::{cache_stats, Arm, SeedSpec, BODY_TERM, NEEDLE}; +use rusqlite::Connection; + +/// Newest-to-oldest is deliberate: `Off` is the candidate, `Sha512` the +/// incumbent, and reporting the candidate first makes the table read as a +/// comparison against what ships rather than a sweep with no thesis. +const MODES: [HmacMode; 3] = [HmacMode::Off, HmacMode::Sha256, HmacMode::Sha512]; + +/// The shape tier — all four arms, cheap enough to run every time. +const SHAPE_FILES: usize = 200_000; +/// The confirmation tier, where the working set stops fitting the OS cache and +/// real reads enter. `QSB_HMAC_SHAPE_ONLY=1` skips it. +const SCALE_FILES: usize = 1_000_000; + +const CONTENT_EVERY: usize = 8; + +/// Commit in slices, as a production run does: each commit flushes an FTS5 +/// segment, so a single enormous transaction would not resemble one — and the +/// write path is half of what this bench is for. +const COMMIT_EVERY: usize = 5_000; + +/// Best-of-N. The minimum is the run least disturbed by whatever else is on +/// the box, which is the honest figure for a comparison. +const RUNS: u32 = 5; + +/// The workloads, in the order they are reported. One common word leads: its +/// posting lists are long, where the rare terms stop at the display limit +/// having touched very little. +const WORKLOADS: [(&str, &str, bool); 6] = [ + ("body (common)", "planning", false), + ("body (rare)", BODY_TERM, false), + ("name", NEEDLE, false), + ("fuzzy", "quartzlte", true), + ("wildcard", "quart*", false), + ("regex", "regex:quart[sz]ite", false), +]; + +fn enabled() -> bool { + std::env::var("QSB_HMAC").is_ok() +} + +fn shape_only() -> bool { + std::env::var("QSB_HMAC_SHAPE_ONLY").is_ok() +} + +fn spec(files: usize, hmac: Option) -> SeedSpec { + SeedSpec { + files, + content_every: CONTENT_EVERY, + dup_every: 5, + commit_every: COMMIT_EVERY, + hmac, + ..SeedSpec::default() + } +} + +fn mib(bytes: u64) -> f64 { + bytes as f64 / (1024.0 * 1024.0) +} + +/// One arm's identity: `None` is the plain floor, `Some` a keyed mode. +fn arms(tag: &str) -> Vec<(String, String, bool, Option)> { + let mut out = vec![( + "plain (no password)".to_string(), + format!("{}-plain", tag), + false, + None, + )]; + for mode in MODES { + out.push(( + format!("keyed, HMAC {}", mode.label()), + format!("{}-{}", tag, mode.label()), + true, + Some(mode), + )); + } + out +} + +fn main() { + if !enabled() { + eprintln!("skipping: set QSB_HMAC=1 to run"); + return; + } + if std::env::var_os("TMPDIR").is_none() { + eprintln!( + "warning: TMPDIR unset — scratch goes to {}. If that is tmpfs, \ + every 'cold' figure below is RAM and the scale tier may not fit.", + std::env::temp_dir().display() + ); + } + + tier(SHAPE_FILES, "shape"); + if shape_only() { + println!("\n(QSB_HMAC_SHAPE_ONLY set — skipping the scale tier)"); + return; + } + tier(SCALE_FILES, "scale"); +} + +/// Every arm at one corpus size, seeded and dropped one at a time so only one +/// index is resident. +fn tier(files: usize, tag: &str) { + println!( + "\n######## {} tier: {} files, {} with content ########", + tag, + files, + files / CONTENT_EVERY + ); + let mut summary: Vec<(String, f64, f64, f64, u64)> = Vec::new(); + for (what, suffix, keyed, hmac) in arms(tag) { + let arm = Arm::seed(&what, &suffix, keyed, &spec(files, hmac)); + let (warm_total, dup) = report(&arm); + summary.push(( + what, + arm.seeded_in.as_secs_f64(), + warm_total, + dup.as_secs_f64(), + arm.size_bytes(), + )); + arm.discard(); + } + + // The whole bench in one table, because the per-arm blocks above are too + // far apart on a terminal to compare by eye. + println!("\n---- {} tier summary ----", tag); + println!( + "{:<24}{:>12}{:>12}{:>12}{:>12}", + "arm", "seed", "warm total", "duplicates", "size" + ); + let baseline = summary.first().map(|s| (s.1, s.2, s.3)).unwrap_or_default(); + for (what, seeded, warm_total, dup, size) in &summary { + println!( + "{:<24}{:>12}{:>12}{:>12}{:>12}", + what, + format!("{:.1} s", seeded), + format!("{:.1} ms", warm_total * 1000.0), + format!("{:.0} ms", dup * 1000.0), + format!("{:.1} MiB", mib(*size)), + ); + } + println!( + "\n{:<24}{:>12}{:>12}{:>12}", + "over plain", "seed", "warm total", "duplicates" + ); + for (what, seeded, warm_total, dup, _) in &summary { + println!( + "{:<24}{:>12}{:>12}{:>12}", + what, + format!("{:.2}x", seeded / baseline.0), + format!("{:.2}x", warm_total / baseline.1), + format!("{:.2}x", dup / baseline.2), + ); + } +} + +/// Everything measured about one arm. Returns `(warm query total, duplicate +/// scan)` — the two numbers the summary ranks on. +fn report(arm: &Arm) -> (f64, Duration) { + let (leaf, overflow) = arm.fts_pages(); + println!( + "\n=== {} === {:.1} MiB on disk, files table {:.1} MiB, \ + fts {} leaf / {} overflow, written in {:.1?} ({:.0} rows/s)", + arm.what, + mib(arm.size_bytes()), + mib(arm.table_bytes("files")), + leaf, + overflow, + arm.seeded_in, + seeded_rows(arm) as f64 / arm.seeded_in.as_secs_f64(), + ); + assert_eq!( + overflow, 0, + "{}: FTS5 leaves overflowed, so this arm is measuring a broken \ + derivation rather than its authenticator", + arm.what + ); + + attribution(arm); + + println!( + "{:<16}{:>12}{:>12}{:>12}{:>10}", + "workload", "cold", "warm", "cold miss", "hits" + ); + let conn = arm.open_search(); + let mut warm_total = 0.0; + for (what, query, fuzzy) in WORKLOADS { + let (cold_time, misses, hits) = cold(arm, query, fuzzy); + let warm_time = warm(&conn, query, fuzzy); + warm_total += warm_time.as_secs_f64(); + println!( + "{:<16}{:>12}{:>12}{:>12}{:>10}", + what, + format!("{:.2?}", cold_time), + format!("{:.2?}", warm_time), + misses, + hits + ); + } + drop(conn); + + (warm_total, duplicates(arm)) +} + +/// `find_duplicate_groups` is the read shape with the most pages per unit of +/// answer — a full `idx_files_hash` scan — so it is where a per-page cost +/// shows up most plainly. It opens its own connection, so the process key and +/// profile have to be installed at *call* time. +fn duplicates(arm: &Arm) -> Duration { + let db_path = arm.path.to_string_lossy().into_owned(); + arm.with_key(|| { + let mut best = Duration::MAX; + for _ in 0..RUNS { + let start = Instant::now(); + let groups = find_duplicate_groups(&db_path, 200).expect("duplicate scan"); + assert!(!groups.is_empty(), "the seed must contain duplicate groups"); + best = best.min(start.elapsed()); + } + best + }) +} + +fn seeded_rows(arm: &Arm) -> i64 { + let conn = arm.open_search(); + conn.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) + .unwrap_or(0) +} + +/// **Where the page fetches go**, so a difference between arms lands on the +/// table that caused it. The shapes are the ones `search/cascade/passes.rs` +/// issues; see `benches/page_geometry.rs`, which uses the same four. +fn attribution(arm: &Arm) { + let like = format!("%{}%", BODY_TERM); + let match_expr = format!("text: \"{}\"", BODY_TERM); + + let shapes: [(&str, &str, &str); 4] = [ + ( + "pass A: files scan", + "SELECT COUNT(*) FROM files f WHERE f.name LIKE ?1 ESCAPE '\\'", + "like", + ), + ( + " FTS postings only", + "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH ?1", + "match", + ), + ( + " + files rowid join", + "SELECT COUNT(*) FROM searchabletext \ + JOIN files f ON f.id = searchabletext.rowid \ + WHERE searchabletext MATCH ?1", + "match", + ), + ( + "pass B: + the bodies", + "SELECT SUM(LENGTH(dt.text_zstd)) FROM searchabletext \ + JOIN files f ON f.id = searchabletext.rowid \ + LEFT JOIN documents_text dt ON dt.file_id = f.id \ + WHERE searchabletext MATCH ?1", + "match", + ), + ]; + + println!( + "{:<24}{:>12}{:>12}{:>14}", + "cold page misses", "misses", "time", "MiB decrypted" + ); + for (what, sql, param) in shapes { + // A fresh connection per shape: the miss count is only meaningful from + // an empty cache. + let conn = arm.open_search(); + let bound: &str = if param == "like" { &like } else { &match_expr }; + let before = cache_stats(&conn).1; + let start = Instant::now(); + conn.query_row(sql, [bound], |r| r.get::<_, Option>(0)) + .expect("attribution shape runs"); + let elapsed = start.elapsed(); + let misses = cache_stats(&conn).1 - before; + let page = arm.page_size.unwrap_or(db::schema::PAGE_SIZE); + println!( + "{:<24}{:>12}{:>12}{:>14.1}", + what, + misses, + format!("{:.2?}", elapsed), + (misses * page) as f64 / (1024.0 * 1024.0) + ); + } +} + +/// Run one query, counting hits rather than keeping them — holding the +/// `SearchHit`s would measure the allocator instead of the scan. +fn run_query(conn: &Connection, query: &str, fuzzy: bool) -> (Duration, usize) { + let split = split_for_cascade(query).expect("query parses"); + let options = SearchOptions { + fuzzy, + ..SearchOptions::default() + }; + let latest = std::sync::atomic::AtomicU64::new(1); + let mut hits = 0usize; + let mut sink = |batch: Vec| hits += batch.len(); + let start = Instant::now(); + cascade::run(conn, &split, &options, 1, &latest, &mut sink).expect("cascade runs"); + (start.elapsed(), hits) +} + +/// Best of `RUNS`, each on a **fresh** connection, so SQLite's page cache +/// starts empty and every page the query wants is a miss — the regime where a +/// per-page authenticator is paid rather than skipped. +fn cold(arm: &Arm, query: &str, fuzzy: bool) -> (Duration, i64, usize) { + let mut best = Duration::MAX; + let mut misses = 0; + let mut hits = 0; + for _ in 0..RUNS { + let conn = arm.open_search(); + let before = cache_stats(&conn).1; + let (elapsed, n) = run_query(&conn, query, fuzzy); + if elapsed < best { + best = elapsed; + misses = cache_stats(&conn).1 - before; + } + hits = n; + } + (best, misses, hits) +} + +/// Best of `RUNS` on one connection after a priming run — the steady state of +/// a typing session, which is what almost every real search is. +fn warm(conn: &Connection, query: &str, fuzzy: bool) -> Duration { + run_query(conn, query, fuzzy); + let mut best = Duration::MAX; + for _ in 0..RUNS { + best = best.min(run_query(conn, query, fuzzy).0); + } + best +} diff --git a/crates/quicksearch-core/benches/corpus/mod.rs b/crates/quicksearch-core/benches/corpus/mod.rs index 0779191..af17940 100644 --- a/crates/quicksearch-core/benches/corpus/mod.rs +++ b/crates/quicksearch-core/benches/corpus/mod.rs @@ -47,7 +47,7 @@ pub fn document(size: usize, hits: usize) -> String { next_plant = next_plant.saturating_add(stride); continue; } - out.push_str(WORDS[lcg.next() as usize % WORDS.len()]); + out.push_str(WORDS[lcg.next_u64() as usize % WORDS.len()]); out.push(' '); } out @@ -119,8 +119,8 @@ static ROWS: LazyLock> = LazyLock::new(|| { let mut lcg = Lcg::new(0xd00d); (0..2000) .map(|i| { - let w1 = WORDS[lcg.next() as usize % WORDS.len()]; - let w2 = WORDS[lcg.next() as usize % WORDS.len()]; + let w1 = WORDS[lcg.next_u64() as usize % WORDS.len()]; + let w2 = WORDS[lcg.next_u64() as usize % WORDS.len()]; let name = format!("{}-{}-{:05}.txt", w1, w2, i); // Mixed case in the directory portion, so the folded tiers resolve. let path = format!("/home/user/Documents/Quartzite/{:03}/{}", i % 40, name); @@ -144,7 +144,7 @@ pub fn text_head() -> &'static [u8] { pub fn binary_head() -> &'static [u8] { static HEAD: LazyLock> = LazyLock::new(|| { let mut lcg = Lcg::new(0xbeef); - (0..8 << 10).map(|_| (lcg.next() & 0xff) as u8).collect() + (0..8 << 10).map(|_| (lcg.next_u64() & 0xff) as u8).collect() }); &HEAD } diff --git a/crates/quicksearch-core/benches/index_alloc.rs b/crates/quicksearch-core/benches/index_alloc.rs new file mode 100644 index 0000000..88be57e --- /dev/null +++ b/crates/quicksearch-core/benches/index_alloc.rs @@ -0,0 +1,481 @@ +//! What indexing one file *allocates*. `index.rs` answers "how long does a +//! step take"; `extractprobe` answers "how much RAM does a pool need"; +//! this answers "how many trips to the allocator does one file cost, and how +//! big is the transient peak behind it". +//! +//! Only Rust-side allocations are counted — SQLite mallocs directly and is +//! invisible — which leaves precisely the walk's and the extractors' own +//! churn. The fixtures are the shared extraction corpus (`tests/corpus/`), +//! so every format QuickSearch claims appears exactly once, written by a +//! library that is not the one reading it back. +//! +//! Reading the numbers: **allocs** is per-file allocator traffic and is what +//! a scratch buffer removes; **bytes** is churn; **peak** is the transient +//! high-water one file reaches, and is what multiplies by the worker count +//! (`walk::thread_count_for`, one pool per root). A `peak` far above the +//! file's own size is amplification inside a parser. +//! +//! Gated by `QSB_INDEX_ALLOC`: +//! +//! ```text +//! QSB_INDEX_ALLOC=1 cargo bench -p quicksearch-core --bench index_alloc +//! ``` + +use std::alloc::{GlobalAlloc, Layout}; + +// What `Counting` wraps: the allocator the shipped binaries install, or the +// figures describe a build nobody runs. See `platform::Allocator`. +use quicksearch_core::platform::Allocator as Inner; +use std::cell::Cell; +use std::path::Path; + +use quicksearch_core::config::Config; +use quicksearch_core::db::repo::{self, DocEncoder}; +use quicksearch_core::extract::{Registry, Scratch}; +use quicksearch_core::file_handling::{decide_content, prepare_file_record}; +use quicksearch_core::mime; + +// The corpus lives with the tests that assert on its content; this reads the +// same fixtures rather than growing a second, drifting set. +#[path = "../tests/corpus/mod.rs"] +mod corpus; + +// --------------------------------------------------------------------------- +// The counting allocator +// --------------------------------------------------------------------------- + +// Counters are per thread — load-bearing: everything measured here is +// synchronous on the main thread, so "this thread" *is* "the region", and a +// background thread (a lazily-spawned pool inside a parser, say) cannot +// silently land in someone else's total. +// +// `const`-initialized `Cell`s: a lazy initializer would allocate from inside +// the allocator, and a destructor can panic during thread teardown — exactly +// when the last deallocations happen. +// `LIVE` and `PEAK` are **signed**: cross-thread frees drive a balance +// legitimately negative; held unsigned it reads ~1.8e19 and `PEAK.max` +// latches there forever. Counts are `u64` (they only rise), balances `i64`. +thread_local! { + static ALLOCS: Cell = const { Cell::new(0) }; + static REALLOCS: Cell = const { Cell::new(0) }; + static BYTES: Cell = const { Cell::new(0) }; + static LIVE: Cell = const { Cell::new(0) }; + static PEAK: Cell = const { Cell::new(0) }; +} + +#[inline] +fn get(counter: &'static std::thread::LocalKey>) -> u64 { + counter.try_with(Cell::get).unwrap_or(0) +} + +#[inline] +fn bump(counter: &'static std::thread::LocalKey>, by: u64) -> u64 { + counter + .try_with(|c| { + let v = c.get().wrapping_add(by); + c.set(v); + v + }) + .unwrap_or(0) +} + +#[inline] +fn get_live(counter: &'static std::thread::LocalKey>) -> i64 { + counter.try_with(Cell::get).unwrap_or(0) +} + +#[inline] +fn bump_live(by: i64) -> i64 { + LIVE.try_with(|c| { + let v = c.get().wrapping_add(by); + c.set(v); + v + }) + .unwrap_or(0) +} + +#[inline] +fn note_peak(live: i64) { + PEAK.try_with(|p| p.set(p.get().max(live))).ok(); +} + +/// [`Inner`], with counters; a failed allocation is not counted, so the +/// totals describe memory that really existed. +struct Counting; + +#[inline] +fn note_alloc(size: usize) { + bump(&ALLOCS, 1); + bump(&BYTES, size as u64); + note_peak(bump_live(size as i64)); +} + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let p = unsafe { Inner.alloc(layout) }; + if !p.is_null() { + note_alloc(layout.size()); + } + p + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let p = unsafe { Inner.alloc_zeroed(layout) }; + if !p.is_null() { + note_alloc(layout.size()); + } + p + } + + /// Cross-thread frees drive this negative; see the `thread_local!` note. + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + bump_live(-(layout.size() as i64)); + unsafe { Inner.dealloc(ptr, layout) } + } + + /// Counted as a resize: a doubling `Vec` is one buffer, not twelve — the + /// difference this harness exists to show. Only growth adds to traffic. + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let p = unsafe { Inner.realloc(ptr, layout, new_size) }; + if !p.is_null() { + bump(&REALLOCS, 1); + let (old, new) = (layout.size() as u64, new_size as u64); + bump(&BYTES, new.saturating_sub(old)); + note_peak(bump_live(new as i64 - old as i64)); + } + p + } +} + +#[global_allocator] +static ALLOCATOR: Counting = Counting; + +#[derive(Clone, Copy)] +struct Counters { + allocs: u64, + reallocs: u64, + bytes: u64, + /// Signed; see the `thread_local!` note. + live: i64, +} + +#[derive(Clone, Copy, Default)] +struct Usage { + allocs: u64, + reallocs: u64, + bytes: u64, + peak: u64, +} + +impl Counters { + /// Snapshot and re-arm the peak tracker at the current live figure, so + /// the following high-water mark belongs to the measured region. + fn start() -> Counters { + let live = get_live(&LIVE); + PEAK.with(|p| p.set(live)); + Counters { + allocs: get(&ALLOCS), + reallocs: get(&REALLOCS), + bytes: get(&BYTES), + live, + } + } + + fn since(&self) -> Usage { + Usage { + allocs: get(&ALLOCS).wrapping_sub(self.allocs), + reallocs: get(&REALLOCS).wrapping_sub(self.reallocs), + bytes: get(&BYTES).wrapping_sub(self.bytes), + // Above the region's starting live figure, so a no-op reads zero. + peak: (get_live(&PEAK) - self.live).max(0) as u64, + } + } +} + +/// Measure one closure, discarding whatever it produced *inside* the region +/// so the drop is charged to it too — a result kept alive would report the +/// peak of the next case instead. +fn measure(f: impl FnOnce() -> T) -> Usage { + let start = Counters::start(); + drop(std::hint::black_box(f())); + start.since() +} + +// --------------------------------------------------------------------------- +// The measurement +// --------------------------------------------------------------------------- + +fn enabled() -> bool { + std::env::var("QSB_INDEX_ALLOC").is_ok() +} + +fn kib(bytes: u64) -> String { + format!("{:.1}", bytes as f64 / 1024.0) +} + +fn main() { + // The accounting is verified before anything is printed: a harness that + // silently stopped counting would read as a spectacular optimization. + the_counters_track_real_allocations(); + if !enabled() { + eprintln!("skipping: set QSB_INDEX_ALLOC=1 to run"); + return; + } + let (dir, samples) = corpus::build("index-alloc"); + let config = Config::default(); + let registry = Registry::default_set(); + + extraction_traffic_per_file(&samples, &config, ®istry); + walk_traffic_per_file(&samples, &config, ®istry); + compression_traffic_per_chunk(&config); + + let _ = std::fs::remove_dir_all(&dir); +} + +/// The content pass's half: `decide_content` is what a `qs-extract` worker +/// runs, and it is the whole of the per-file cost outside the connection. +fn extraction_traffic_per_file(samples: &[corpus::Sample], config: &Config, registry: &Registry) { + println!( + "extraction — decide_content, the content worker's per-file work\n\n\ + {:<10} {:>10} {:>10} {:>10} {:>12} {:>12} {:>8}", + "format", "file (KiB)", "allocs", "reallocs", "bytes (KiB)", "peak (KiB)", "amp" + ); + + // One scratch for the whole table, as a `qs-extract` worker holds one + // for a whole pass: the figures below are a worker's *steady state*, not + // its first file. + let mut scratch = Scratch::new(config); + let mut rows: Vec<(&str, u64, Usage)> = Vec::new(); + for sample in samples { + let path = sample.path.to_string_lossy().into_owned(); + let size = std::fs::metadata(&sample.path).map(|m| m.len()).unwrap_or(0); + let Some(mime) = sniff(&sample.path, config.processing.hash_length) else { + continue; + }; + // Warm once: a format's first call may build a lazy static the file + // after it does not pay for. + let _ = decide_content(&path, Some(mime), registry, config, &mut scratch); + let usage = measure(|| decide_content(&path, Some(mime), registry, config, &mut scratch)); + rows.push((sample.label, size, usage)); + } + + for (label, size, usage) in &rows { + println!( + "{:<10} {:>10} {:>10} {:>10} {:>12} {:>12} {:>8}", + label, + kib(*size), + usage.allocs, + usage.reallocs, + kib(usage.bytes), + kib(usage.peak), + // How far above the file's own size the transient peak reached. + // This is the figure that multiplies by the worker count. + match size { + 0 => "-".to_string(), + n => format!("{:.1}x", usage.peak as f64 / *n as f64), + } + ); + } + println!( + "\n maximum_text_file_size {} KiB, maximum_text_size {} KiB — what every\n \ + extractor's ceilings are now derived from.\n", + config.processing.maximum_text_file_size / 1024, + config.processing.maximum_text_size / 1024, + ); +} + +/// The walk's half: one `stat`'s worth of metadata in, one finished record +/// out — hashing, MIME sniffing and the inline-text shortcut included. +fn walk_traffic_per_file(samples: &[corpus::Sample], config: &Config, registry: &Registry) { + println!( + "walk — prepare_file_record, the walk worker's per-file work\n\n\ + {:<10} {:>10} {:>10} {:>10} {:>12} {:>12}", + "format", "file (KiB)", "allocs", "reallocs", "bytes (KiB)", "peak (KiB)" + ); + + // One scratch for the whole table; see `extraction_traffic_per_file`. + let mut scratch = Scratch::new(config); + for sample in samples { + let path = sample.path.to_string_lossy().into_owned(); + let Ok(meta) = std::fs::metadata(&sample.path) else { + continue; + }; + let _ = prepare_file_record(&path, &meta, config, registry, &mut scratch); + let usage = measure(|| prepare_file_record(&path, &meta, config, registry, &mut scratch)); + println!( + "{:<10} {:>10} {:>10} {:>10} {:>12} {:>12}", + sample.label, + kib(meta.len()), + usage.allocs, + usage.reallocs, + kib(usage.bytes), + kib(usage.peak), + ); + } + println!( + "\n A file under hash_length ({} KiB) is extracted here rather than by the\n \ + content pass, so its row carries inline text.\n", + config.processing.hash_length / 1024 + ); +} + +/// Rows per writer chunk — `batch::STORE_CHUNK`, the unit `compress_bodies` +/// is handed. +const CHUNK: usize = 32; + +/// The writer's half that runs *outside* the connection lock. +/// +/// **The zstd context does not appear here**: `zstd::bulk::Compressor::new` +/// allocates through zstd's own C allocator, not Rust's, so it is invisible +/// to this harness and its cost is CPU only (`benches/index.rs`, group +/// `zstd_encode`). What this measures is the part that *is* Rust-side — one +/// output `Vec` per row against one arena per chunk. +fn compression_traffic_per_chunk(config: &Config) { + // `maximum_text_size` is the worst case a row can carry. + let doc = lipsum(config.processing.maximum_text_size); + println!( + "compression — one {}-row chunk of {} KiB documents\n\n{:<24} {:>10} {:>10} {:>12} {:>12}", + CHUNK, + kib(doc.len() as u64), + "shape", + "allocs", + "reallocs", + "bytes (KiB)", + "peak (KiB)" + ); + + let per_row = measure(|| { + (0..CHUNK) + .map(|_| repo::encode_one(&doc, true).expect("encode")) + .collect::>() + }); + // What the writer does now: one encoder and one arena for the chunk. + let arena = measure(|| { + let mut enc = DocEncoder::new().expect("encoder"); + let mut arena = Vec::new(); + (0..CHUNK) + .map(|_| enc.encode_into(&doc, &mut arena).expect("encode")) + .collect::>() + .len() + }); + + for (shape, usage) in [("a Vec per row", per_row), ("one arena, chunk", arena)] { + println!( + "{:<24} {:>10} {:>10} {:>12} {:>12}", + shape, + usage.allocs, + usage.reallocs, + kib(usage.bytes), + kib(usage.peak), + ); + } + println!( + "\n The arena is reused across every chunk a writer call handles, so after the\n \ + first its growth is zero too. The zstd context is invisible here — see above.\n" + ); +} + +/// Head bytes read for the MIME sniff — the same window the walk uses, so +/// this classifies files exactly as a run would. +fn sniff(path: &Path, hash_length: usize) -> Option<&'static str> { + use std::io::Read; + let mut f = std::fs::File::open(path).ok()?; + let mut head = vec![0u8; hash_length]; + let n = f.read(&mut head).ok()?; + head.truncate(n); + mime::guess_mime_from_head(path, &head) +} + +/// Deterministic filler; the compression figures must not move between runs. +fn lipsum(size: usize) -> String { + const WORDS: &[&str] = &[ + "lorem", "ipsum", "dolor", "consectetur", "adipiscing", "tempor", "incididunt", "labore", + ]; + let mut lcg = quicksearch_core::testutil::Lcg::new(0x5eed); + let mut out = String::with_capacity(size + 16); + while out.len() < size { + out.push_str(WORDS[lcg.next_u64() as usize % WORDS.len()]); + out.push(' '); + } + out.truncate(size); + out +} + +/// The accounting itself, verified unconditionally at startup. +fn the_counters_track_real_allocations() { + let start = Counters::start(); + let mut v: Vec = Vec::new(); + // `push` in a loop on purpose: the growth is what is being measured, and + // the `resize`/`vec![0; n]` clippy asks for would allocate once with no + // reallocs at all, so the assertion below could never fail. + #[allow(clippy::same_item_push)] + for _ in 0..64 * 1024 { + v.push(0); + } + let grown = start.since(); + assert_eq!(grown.allocs, 1, "a doubling Vec is one allocation"); + assert!(grown.reallocs > 0, "and several resizes"); + assert!( + grown.peak >= 64 * 1024, + "peak {} should cover the grown buffer", + grown.peak + ); + + // Dropping returns the bytes, so a later region's peak is not inflated. + let before_drop = get_live(&LIVE); + drop(v); + assert!( + get_live(&LIVE) < before_drop, + "dealloc must decrement live bytes" + ); + + // The signedness the scheme turns on: sink the balance below zero, as a + // cross-thread free really does, and a peak must still be reported — + // unsigned, `max` latches on ~1.8e19 forever. + bump_live(-(1 << 20)); + let negative = Counters::start(); + assert!(negative.live < 0, "the balance is genuinely negative"); + let mut grow: Vec = Vec::with_capacity(32 * 1024); + grow.push(1); + let seen = negative.since().peak; + drop(grow); + assert!( + seen >= 32 * 1024, + "a negative live balance swallowed the peak: {}", + seen + ); + bump_live(1 << 20); // put back what was sunk, so later regions start clean + + // `measure` must charge the value's *drop* to its own region, or every + // per-file peak here would belong to the case after it. + let held = measure(|| Vec::::with_capacity(1 << 20)); + assert!(held.peak >= 1 << 20, "the allocation is inside the region"); + let after = Counters::start(); + std::hint::black_box(1u64 + 1); + assert_eq!(after.since().peak, 0, "and it was freed before the next one"); + + // Another thread allocating hard must not touch this thread's counters. + // Spawn and join sit *outside* the region: `spawn` boxes its closure on + // the calling thread and `join` frees it there; a barrier hands control + // across without allocating. + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let child = { + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); // the region is open + let noisy: Vec = (0..10_000).map(|i| format!("allocation {}", i)).collect(); + std::hint::black_box(noisy.len()); + barrier.wait(); // the noise is done + }) + }; + let quiet_across_threads = Counters::start(); + barrier.wait(); + barrier.wait(); + let leaked = quiet_across_threads.since(); + child.join().expect("the noisy thread finishes"); + assert_eq!( + (leaked.allocs, leaked.bytes), + (0, 0), + "another thread's allocations must not be charged to this region" + ); +} diff --git a/crates/quicksearch-core/benches/page_geometry.rs b/crates/quicksearch-core/benches/page_geometry.rs new file mode 100644 index 0000000..e5a22a8 --- /dev/null +++ b/crates/quicksearch-core/benches/page_geometry.rs @@ -0,0 +1,390 @@ +//! What the index's page geometry costs, on disk and in query time. +//! +//! Two levers, swept together because they are coupled: the **database page +//! size** (`db::schema::PAGE_SIZE`) and FTS5's **record size** +//! (`db::schema::fts_pgsz_for`). The second is derived from the first, so +//! neither can be moved alone — at a page size of 8192 an FTS5 record built +//! for 4096 leaves half of every page empty. +//! +//! ```text +//! TMPDIR=/media/shared/qs-scratch QSB_PGSZ=1 \ +//! cargo bench -p quicksearch-core --bench page_geometry +//! ``` +//! +//! `TMPDIR` is not optional in spirit. `testutil::scratch_dir` builds on +//! `std::env::temp_dir()`, and a `/tmp` that is tmpfs cannot produce a page +//! fetch that was not already in RAM — it would price the one regime this is +//! not trying to characterise, and a 1M-file arm would not fit besides. Point +//! it at real storage. The matrix wants ~2 GB at a time (each arm is dropped +//! once measured) and around ten minutes, most of it seeding. +//! +//! # Settled: FTS5 record size +//! +//! SQLCipher reserves part of every page for its IV and any authenticator, so +//! a keyed page holds `page − reserve − 35` bytes inline. FTS5's default +//! record of 4050 was chosen for a *plain* 4096 page and missed that by 71 +//! bytes under the 80-byte reserve of the day, which sent every full leaf to +//! an overflow page. Measured at 120k files before `fts_pgsz_for` existed, and +//! while `db::schema::HMAC_MODE` was still HMAC-SHA512 — at today's 16-byte +//! reserve the miss is smaller, but the derivation is what makes it zero at +//! *every* page size: +//! +//! | | plain 4050 | plain shipped | keyed 4050 | keyed shipped | +//! |---|---|---|---|---| +//! | bulk write | 3.87 s | 4.04 s | 7.34 s | 7.13 s | +//! | size | 156.4 MiB | 156.4 MiB | **170.2 MiB** | **159.1 MiB** | +//! | fts overflow pages | 0 | 0 | **27381** | 0 | +//! | cold `chalcedony` | 11.35 ms | 11.37 ms | 29.91 ms | 29.57 ms | +//! | warm `chalcedony` | 9.29 ms | 9.24 ms | 9.41 ms | 9.27 ms | +//! +//! A disk-space fix (1.089x → 1.011x encrypted-over-plain), not a speed fix: +//! both indexing and search moved less than the harness's own noise floor. +//! `tests/encrypted_perf.rs` gates the size half of that and is the reason +//! this bench does not re-measure it. +//! +//! # Open: database page size +//! +//! A keyed index decrypts a whole page to read one row out of it. If the +//! expensive fetches are *scattered* single rows, a smaller page cuts that +//! work in proportion, and — `cache_size` being a byte ceiling — lets the same +//! 32 MiB hold four times as many distinct rows. Pulling the other way, the +//! `files` scan behind every filename query is sequential and wants large +//! pages, and the reserve costs proportionally more of a small page: at the +//! 80 bytes of the HMAC-SHA512 era that was 2% of a 4096-byte page against +//! 7.8% of a 1024-byte one, and at today's 16 it is 0.4% against 1.6%. +//! +//! [`attribution`] settles which of those a query actually does, by counting +//! page-cache misses per query shape rather than inferring them from timings. +//! +//! # Reading it +//! +//! **The plain arm is the noise floor**, and at the 200k tier a just-seeded +//! index is small enough that the OS page cache serves nearly all of it — so +//! those figures price decrypt work with little I/O in them. The 1M tier +//! exceeds what stays cached, and is where real reads enter: storage reads in +//! ≥4 KiB blocks whatever the page size, so a sub-4K page cuts decryption but +//! not I/O. The two tiers are reported separately for that reason; do not +//! average them. + +use std::time::{Duration, Instant}; + +use quicksearch_core::db; +use quicksearch_core::query::split::split_for_cascade; +use quicksearch_core::search::{cascade, SearchHit, SearchOptions}; +use quicksearch_core::testutil::{cache_stats, Arm, SeedSpec, BODY_TERM, NEEDLE}; +use rusqlite::Connection; + +/// Page sizes to sweep, up to `SQLITE_MAX_PAGE_SIZE`. 512 is excluded: with +/// SQLCipher's 80-byte reserve its usable size falls under SQLite's 480-byte +/// floor. FTS5 caps its own record size at 64 KiB and rejects anything larger, +/// so 65536 is the last size where `fts_pgsz_for` still has room. +const SWEPT: [i64; 7] = [1024, 2048, 4096, 8192, 16384, 32768, 65536]; + +/// `QSB_PGSZ_SIZES=8192,16384` narrows the sweep; a full run is ~45 minutes, +/// nearly all of it seeding, so re-asking one question should not re-ask all +/// of them. +fn swept() -> Vec { + match std::env::var("QSB_PGSZ_SIZES") { + Ok(list) => list + .split(',') + .map(|s| s.trim().parse().expect("QSB_PGSZ_SIZES wants integers")) + .collect(), + Err(_) => SWEPT.to_vec(), + } +} + +/// `QSB_PGSZ_SHAPE_ONLY=1` skips the large corpora. +fn shape_only() -> bool { + std::env::var("QSB_PGSZ_SHAPE_ONLY").is_ok() +} + +/// The shape tier — every page size, cheap enough to run them all. +const SHAPE_FILES: usize = 200_000; +/// The confirmation tiers, run only for the baseline and the shape tier's +/// winner. 1M is where the working set stops fitting in the OS cache. +const SCALE_FILES: [usize; 2] = [600_000, 1_000_000]; + +const CONTENT_EVERY: usize = 8; + +/// Commit in slices, as a production run does: each commit flushes an FTS5 +/// segment, so a single enormous transaction would not resemble one. +const COMMIT_EVERY: usize = 5_000; + +/// Best-of-N. The minimum is the run least disturbed by whatever else is on +/// the box, which is the honest figure for a comparison. +const RUNS: u32 = 5; + +/// The workloads, in the order they are reported. One word from +/// `testutil::WORDS` leads: its posting lists are long, where the rare terms +/// stop at the display limit having touched very little. +const WORKLOADS: [(&str, &str, bool); 6] = [ + ("body (common)", "planning", false), + ("body (rare)", BODY_TERM, false), + ("name", NEEDLE, false), + ("fuzzy", "quartzlte", true), + ("wildcard", "quart*", false), + ("regex", "regex:quart[sz]ite", false), +]; + +fn enabled() -> bool { + std::env::var("QSB_PGSZ").is_ok() +} + +fn spec(files: usize, page_size: i64) -> SeedSpec { + SeedSpec { + files, + content_every: CONTENT_EVERY, + dup_every: 5, + commit_every: COMMIT_EVERY, + page_size: Some(page_size), + ..SeedSpec::default() + } +} + +fn mib(bytes: u64) -> f64 { + bytes as f64 / (1024.0 * 1024.0) +} + +fn main() { + if !enabled() { + eprintln!("skipping: set QSB_PGSZ=1 to run"); + return; + } + if std::env::var_os("TMPDIR").is_none() { + eprintln!( + "warning: TMPDIR unset — scratch goes to {}. If that is tmpfs, \ + every 'cold' figure below is RAM and the large tiers may not fit.", + std::env::temp_dir().display() + ); + } + + let best = shape_tier(); + if shape_only() { + println!("\n(QSB_PGSZ_SHAPE_ONLY set — skipping the large corpora)"); + return; + } + scale_tier(best); +} + +/// Every page size at [`SHAPE_FILES`], plain and keyed. Returns the keyed page +/// size with the lowest total warm time — the metric that matters, because a +/// session re-queries on every keystroke. +fn shape_tier() -> i64 { + println!( + "\n######## shape tier: {} files, {} with content ########", + SHAPE_FILES, + SHAPE_FILES / CONTENT_EVERY + ); + let mut best = (db::schema::PAGE_SIZE, f64::MAX); + for page_size in swept() { + for keyed in [false, true] { + let arm = Arm::seed( + format!("{} {}", if keyed { "keyed" } else { "plain" }, page_size), + &format!("pgsz-{}-{}", page_size, keyed), + keyed, + &spec(SHAPE_FILES, page_size), + ); + let warm_total = report(&arm); + if keyed && warm_total < best.1 { + best = (page_size, warm_total); + } + arm.discard(); + } + } + println!( + "\n>>> lowest keyed warm total at page_size {} ({:.1} ms across {} workloads)", + best.0, + best.1 * 1000.0, + WORKLOADS.len() + ); + best.0 +} + +/// The baseline and the winner only, at the larger corpora. +fn scale_tier(best: i64) { + let mut sizes = vec![db::schema::PAGE_SIZE]; + if best != db::schema::PAGE_SIZE { + sizes.push(best); + } + for files in SCALE_FILES { + println!( + "\n######## scale tier: {} files, {} with content ########", + files, + files / CONTENT_EVERY + ); + for page_size in &sizes { + for keyed in [false, true] { + let arm = Arm::seed( + format!("{} {}", if keyed { "keyed" } else { "plain" }, page_size), + &format!("pgsz-{}-{}-{}", files, page_size, keyed), + keyed, + &spec(files, *page_size), + ); + report(&arm); + arm.discard(); + } + } + } +} + +/// Everything measured about one arm. Returns its total warm query time in +/// seconds, the metric [`shape_tier`] ranks on. +fn report(arm: &Arm) -> f64 { + let (leaf, overflow) = arm.fts_pages(); + let files_bytes = arm.table_bytes("files"); + println!( + "\n=== {} === {:.1} MiB on disk, files table {:.1} MiB, \ + fts {} leaf / {} overflow, written in {:.1?} ({:.0} rows/s)", + arm.what, + mib(arm.size_bytes()), + mib(files_bytes), + leaf, + overflow, + arm.seeded_in, + seeded_rows(arm) as f64 / arm.seeded_in.as_secs_f64(), + ); + + attribution(arm); + + println!( + "{:<16}{:>12}{:>12}{:>12}{:>10}", + "workload", "cold", "warm", "cold miss", "hits" + ); + let conn = arm.open_search(); + let mut warm_total = 0.0; + for (what, query, fuzzy) in WORKLOADS { + let (cold_time, misses, hits) = cold(arm, query, fuzzy); + let warm_time = warm(&conn, query, fuzzy); + warm_total += warm_time.as_secs_f64(); + println!( + "{:<16}{:>12}{:>12}{:>12}{:>10}", + what, + format!("{:.2?}", cold_time), + format!("{:.2?}", warm_time), + misses, + hits + ); + } + warm_total +} + +fn seeded_rows(arm: &Arm) -> i64 { + let conn = arm.open_search(); + conn.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) + .unwrap_or(0) +} + +/// **Where the page fetches go.** Each shape runs on its own fresh connection +/// and reports the misses it caused, so the cost lands on the table that +/// caused it rather than on whichever query happened to be slow. +/// +/// The shapes are the ones `search/cascade/passes.rs` actually issues: pass A +/// is a `files` scan with no FTS in it at all, and pass B's FTS MATCH is +/// joined straight back to `files` by rowid and to `documents_text` for the +/// body — so each posting costs a random row seek and a blob read on top of +/// the posting list that produced it. The middle two rows separate those. +fn attribution(arm: &Arm) { + let term = BODY_TERM; + let like = format!("%{}%", term); + let match_expr = format!("text: \"{}\"", term); + + let shapes: [(&str, &str, &str); 4] = [ + ( + "pass A: files scan", + "SELECT COUNT(*) FROM files f WHERE f.name LIKE ?1 ESCAPE '\\'", + "like", + ), + ( + " FTS postings only", + "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH ?1", + "match", + ), + ( + " + files rowid join", + "SELECT COUNT(*) FROM searchabletext \ + JOIN files f ON f.id = searchabletext.rowid \ + WHERE searchabletext MATCH ?1", + "match", + ), + ( + "pass B: + the bodies", + "SELECT SUM(LENGTH(dt.text_zstd)) FROM searchabletext \ + JOIN files f ON f.id = searchabletext.rowid \ + LEFT JOIN documents_text dt ON dt.file_id = f.id \ + WHERE searchabletext MATCH ?1", + "match", + ), + ]; + + println!( + "{:<24}{:>12}{:>12}{:>14}", + "cold page misses", "misses", "time", "MiB decrypted" + ); + for (what, sql, param) in shapes { + // A fresh connection per shape: the miss count is only meaningful + // from an empty cache. + let conn = arm.open_search(); + let bound: &str = if param == "like" { &like } else { &match_expr }; + let before = cache_stats(&conn).1; + let start = Instant::now(); + conn.query_row(sql, [bound], |r| r.get::<_, Option>(0)) + .expect("attribution shape runs"); + let elapsed = start.elapsed(); + let misses = cache_stats(&conn).1 - before; + let page = arm.page_size.unwrap_or(db::schema::PAGE_SIZE); + println!( + "{:<24}{:>12}{:>12}{:>14.1}", + what, + misses, + format!("{:.2?}", elapsed), + (misses * page) as f64 / (1024.0 * 1024.0) + ); + } +} + +/// Run one query, counting hits rather than keeping them — holding the +/// `SearchHit`s would measure the allocator instead of the scan. +fn run_query(conn: &Connection, query: &str, fuzzy: bool) -> (Duration, usize) { + let split = split_for_cascade(query).expect("query parses"); + let options = SearchOptions { + fuzzy, + ..SearchOptions::default() + }; + let latest = std::sync::atomic::AtomicU64::new(1); + let mut hits = 0usize; + let mut sink = |batch: Vec| hits += batch.len(); + let start = Instant::now(); + cascade::run(conn, &split, &options, 1, &latest, &mut sink).expect("cascade runs"); + (start.elapsed(), hits) +} + +/// Best of `RUNS`, each on a **fresh** connection, so SQLite's page cache +/// starts empty and every page the query wants is a miss. Returns the miss +/// count alongside, which is what makes the timing interpretable. +fn cold(arm: &Arm, query: &str, fuzzy: bool) -> (Duration, i64, usize) { + let mut best = Duration::MAX; + let mut misses = 0; + let mut hits = 0; + for _ in 0..RUNS { + let conn = arm.open_search(); + let before = cache_stats(&conn).1; + let (elapsed, n) = run_query(&conn, query, fuzzy); + if elapsed < best { + best = elapsed; + misses = cache_stats(&conn).1 - before; + } + hits = n; + } + (best, misses, hits) +} + +/// Best of `RUNS` on one connection after a priming run — the steady state of +/// a typing session, which is what almost every real search is. +fn warm(conn: &Connection, query: &str, fuzzy: bool) -> Duration { + run_query(conn, query, fuzzy); + let mut best = Duration::MAX; + for _ in 0..RUNS { + best = best.min(run_query(conn, query, fuzzy).0); + } + best +} diff --git a/crates/quicksearch-core/benches/search.rs b/crates/quicksearch-core/benches/search.rs index ee51d6e..f445243 100644 --- a/crates/quicksearch-core/benches/search.rs +++ b/crates/quicksearch-core/benches/search.rs @@ -336,10 +336,7 @@ mod filename_ladder { let last = hay.len().checked_sub(needle.len())?; let mut at = 0usize; while at <= last { - let Some(off) = memchr::memchr2(lo, up, &hay[at..=last]) else { - return None; - }; - let i = at + off; + let i = at + memchr::memchr2(lo, up, &hay[at..=last])?; if hay[i..i + needle.len()].eq_ignore_ascii_case(needle) { return Some(i); } diff --git a/crates/quicksearch-core/benches/search_alloc.rs b/crates/quicksearch-core/benches/search_alloc.rs index e413937..8ce77b9 100644 --- a/crates/quicksearch-core/benches/search_alloc.rs +++ b/crates/quicksearch-core/benches/search_alloc.rs @@ -14,7 +14,11 @@ //! QSB_SEARCH_ALLOC=1 cargo bench -p quicksearch-core --bench search_alloc //! ``` -use std::alloc::{GlobalAlloc, Layout, System}; +use std::alloc::{GlobalAlloc, Layout}; + +// What `Counting` wraps: the allocator the shipped binaries install, or the +// figures describe a build nobody runs. See `platform::Allocator`. +use quicksearch_core::platform::Allocator as Inner; use std::cell::Cell; use std::sync::atomic::AtomicU64; use std::time::{Duration, Instant}; @@ -83,7 +87,7 @@ fn note_peak(live: i64) { PEAK.try_with(|p| p.set(p.get().max(live))).ok(); } -/// `System`, with counters; a failed allocation is not counted, so the +/// [`Inner`], with counters; a failed allocation is not counted, so the /// totals describe memory that really existed. struct Counting; @@ -96,7 +100,7 @@ fn note_alloc(size: usize) { unsafe impl GlobalAlloc for Counting { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - let p = unsafe { System.alloc(layout) }; + let p = unsafe { Inner.alloc(layout) }; if !p.is_null() { note_alloc(layout.size()); } @@ -104,7 +108,7 @@ unsafe impl GlobalAlloc for Counting { } unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - let p = unsafe { System.alloc_zeroed(layout) }; + let p = unsafe { Inner.alloc_zeroed(layout) }; if !p.is_null() { note_alloc(layout.size()); } @@ -114,13 +118,13 @@ unsafe impl GlobalAlloc for Counting { /// Cross-thread frees drive this negative; see the `thread_local!` note. unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { bump_live(-(layout.size() as i64)); - unsafe { System.dealloc(ptr, layout) } + unsafe { Inner.dealloc(ptr, layout) } } /// Counted as a resize: a doubling `Vec` is one buffer, not twelve — the /// difference this harness exists to show. Only growth adds to traffic. unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { - let p = unsafe { System.realloc(ptr, layout, new_size) }; + let p = unsafe { Inner.realloc(ptr, layout, new_size) }; if !p.is_null() { bump(&REALLOCS, 1); let (old, new) = (layout.size() as u64, new_size as u64); @@ -322,8 +326,8 @@ fn allocation_traffic_per_query() { .expect("open the seeded index"); println!( - "{:<16} {:>12} {:>10} {:>12} {:>12} {:>7} {:>9} {}", - "case", "allocs", "reallocs", "bytes (MiB)", "peak (MiB)", "hits", "time", "passes" + "{:<16} {:>12} {:>10} {:>12} {:>12} {:>7} {:>9} passes", + "case", "allocs", "reallocs", "bytes (MiB)", "peak (MiB)", "hits", "time" ); for case in CASES { // Warm once, then measure: a cold first query would report SQLite's @@ -355,6 +359,10 @@ fn allocation_traffic_per_query() { fn the_counters_track_real_allocations() { let start = Counters::start(); let mut v: Vec = Vec::new(); + // `push` in a loop on purpose: the growth is what is being measured, and + // the `resize`/`vec![0; n]` clippy asks for would allocate once with no + // reallocs at all, so the assertion below could never fail. + #[allow(clippy::same_item_push)] for _ in 0..64 * 1024 { v.push(0); } diff --git a/crates/quicksearch-core/benches/search_perf.rs b/crates/quicksearch-core/benches/search_perf.rs index 9b809e5..b3e8030 100644 --- a/crates/quicksearch-core/benches/search_perf.rs +++ b/crates/quicksearch-core/benches/search_perf.rs @@ -1,54 +1,156 @@ -//! What a warm page cache is worth to search, and how big it has to be: -//! warm beats cold, and 8 MiB ([`PRAGMAS_SEARCH`]) is enough — the two -//! claims holding one connection across requests rests on. +//! How big the search connection's page cache has to be, and what it is worth +//! — the two claims `db::schema::PRAGMAS_SEARCH` and +//! `search::IDLE_RELEASE` rest on. //! -//! Queries run as a keystroke sequence (`q`, `qu`, `qui`, `quic`); the -//! second and later queries are the number that matters. +//! Queries run as a keystroke sequence (`quar`, `quart`, `quartz`, +//! `quartzi`), because that is what a real session is: the user types, and +//! every keystroke re-runs the search. **The warm column is the one that +//! decides the constant.** A cold query happens once, on the first keystroke +//! after `IDLE_RELEASE` drops the connection; a warm one happens on every +//! keystroke after it. //! -//! The encrypted column is where a smaller cache hurts first: a page-cache -//! miss costs an AES decrypt plus an HMAC verify, not a `memcpy`. +//! What has to stay resident is the **`files` table**, not the FTS index. +//! `search/cascade/passes.rs` answers filename queries (ranks 1–4, 9–10) with +//! `SELECT … FROM files f WHERE f.name LIKE '%…%'` — a full table scan, no FTS +//! at all — and the fuzzy pass scans it again with `WHERE 1=1`. So the working +//! set scales with **file count**, not with document volume, and the corpus +//! dimension below is what makes that visible. //! -//! Printed rather than asserted: shared-box timings are not stable enough -//! for a pass/fail gate, and a flaky perf gate gets muted rather than -//! fixed. Gated by `QSB_SEARCH_PERF`: +//! The keyed rows are where an undersized cache hurts first: a page-cache miss +//! costs an AES-CBC decrypt, not a `memcpy`. +//! +//! **Every keyed figure recorded below was taken while +//! `db::schema::HMAC_MODE` was HMAC-SHA512**, so a miss then also cost a +//! per-page verify. It no longer does, and `benches/cipher_hmac.rs` measured +//! that as 1.78x on warm search — so the knees found here are deeper than the +//! ones a re-sweep would find. `schema::SEARCH_CACHE_BYTES_PER_FILE` says the +//! same thing from the other side: it is now conservative, and re-running this +//! bench is what would tighten it. +//! +//! Printed rather than asserted: shared-box timings are not stable enough for +//! a pass/fail gate, and a flaky perf gate gets muted rather than fixed. //! //! ```text -//! QSB_SEARCH_PERF=1 cargo bench -p quicksearch-core --bench search_perf +//! TMPDIR=/media/shared/qs-scratch QSB_SEARCH_PERF=1 \ +//! cargo bench -p quicksearch-core --bench search_perf //! ``` +//! +//! `TMPDIR` wants real storage: the 1M-file arms are ~580 MB each and a tmpfs +//! `/tmp` would turn every miss into a RAM copy. Budget ~20 minutes, nearly +//! all of it seeding. +//! +//! # What it found +//! +//! Warm search, best of the settled session, `schema::PAGE_SIZE` = 8192, rows +//! at a realistic width (139 B — see [`spec`]): +//! +//! | corpus | `files` table | keyed, under knee | keyed, at knee | knee | ratio | +//! |---|---|---|---|---|---| +//! | 200k | 26.5 MiB | 41.4 ms (≤24 MiB) | **10.7 ms** | 32 MiB | 1.21x | +//! | 600k | 79.5 MiB | 121.2 ms (≤64 MiB) | **33.7 ms** | 96 MiB | 1.21x | +//! | 1M | 132.4 MiB | 199.6 ms (≤128 MiB) | **58.3 ms** | 256 MiB | ≤1.93x | +//! +//! 1. **The knee is 1.21x the `files` table, at every corpus.** Not +//! approximately: 26.5→32 and 79.5→96 both land on it, and 1M's true knee +//! is somewhere in (128, 256] where 1.21x predicts 160. It tracks *file +//! count*, not document volume, because what every keystroke rescans is +//! `files` (see the pass-A note above), never the FTS index. That product — +//! 139 B/row × 1.21 — is `schema::SEARCH_CACHE_BYTES_PER_FILE`. +//! 2. **Below the knee an encrypted index is 3.4–3.9x slower**, and the step +//! is a cliff, not a slope: 600k measured 121–130 ms at every ceiling from +//! 1 to 64 MiB and 33.7 ms at 96. +//! 3. **Plain has no knee.** Its widest spread was 1.35x and it is not even +//! monotonic — 32 and 48 MiB measured slower than 1 MiB at 600k — which is +//! run-to-run noise, not a curve. A miss it takes is a `memcpy` from the OS +//! cache; a miss the keyed arm takes is an AES-CBC decrypt (plus, when +//! these were measured, an HMAC-SHA512 verify). Hence +//! `schema::SEARCH_CACHE_PLAIN_MIB`, flat. +//! +//! **Row width is half the answer and was nearly missed.** The narrow rows the +//! search harnesses used to seed — `hash` NULL, a 16-character parent — are +//! 69.5 B, exactly half of a realistic 139 B. Calibrating against those would +//! have under-sized every cache by two and put every user back under the knee. +//! +//! **The 128 MiB automatic cap binds at ~800k files.** At 1M the derived value +//! is capped at 128 while the index wants ~160: 205 ms per keystroke against +//! the 58 ms available. That is the memory-versus-speed trade +//! `schema::SEARCH_CACHE_MAX_MIB` documents, and +//! `[search] cache_size_mib` is how a user takes the other side of it. use std::time::{Duration, Instant}; -use quicksearch_core::db::set_process_key; use quicksearch_core::query::split::split_for_cascade; use quicksearch_core::search::{cascade, SearchHit, SearchOptions}; -use quicksearch_core::security::IndexKey; -use quicksearch_core::testutil::{scratch_db, seed_index, SeedSpec}; +use quicksearch_core::testutil::{Arm, SeedSpec}; use rusqlite::Connection; -/// Large enough that the b-tree has interior levels and FTS several -/// segments — below that everything fits in any cache and says nothing. -const NUM_FILES: usize = 200_000; +/// Corpora chosen to bracket `PRAGMAS_SEARCH`: at 200k the `files` table is +/// ~19 MiB and fits inside 32 MiB, at 1M it is ~98 MiB and cannot. If the +/// working set really is `files`, the knee moves between these two. +const CORPORA: [usize; 3] = [200_000, 600_000, 1_000_000]; -/// Cache ceilings: `-40960` is what every read connection used to take, -/// `-8192` is `PRAGMAS_SEARCH`, and `-1024` is deliberately too small — the -/// curve needs a visible floor for "8 MiB is enough" to be a measurement. -const CACHE_SIZES: [i64; 6] = [-40960, -32768, -16384, -8192, -4096, -1024]; +/// Cache ceilings in KiB (negative is KiB; positive would be a page count). +/// 1 MiB is deliberately far too small — the curve needs a visible floor for +/// any "enough" to be a measurement — and 256 MiB is past anything shippable, +/// so a knee inside the range is a knee and not the edge of the sweep. +/// +/// 24, 48 and 96 MiB break the powers of two. Without them the knee can only +/// be located to the next power up, which pins the cache-over-`files` ratio no +/// tighter than (1.0, 1.93] — too loose to derive a constant from. +const CACHE_SIZES: [i64; 12] = [ + -1024, -2048, -4096, -8192, -16384, -24576, -32768, -49152, -65536, -98304, -131072, -262144, +]; + +/// What `PRAGMAS_SEARCH` ships with, marked in the output so the curve can be +/// read against it without counting columns. +const SHIPPED_CACHE: i64 = -32768; const SEQUENCE: [&str; 4] = ["quar", "quart", "quartz", "quartzi"]; +/// A cache is "enough" once warm is within this of the best warm on the same +/// arm — used to *locate* a knee, once there is one to locate. +const KNEE_TOLERANCE: f64 = 1.10; + +/// How much worse the worst ceiling must be than the best before the curve is +/// called a knee at all. +/// +/// A real one is unmistakable: keyed at 200k ran 41.4 ms flat below 32 MiB and +/// 10.7 ms at or above it, monotonically, a 4x step. A plain arm at the same +/// corpus wanders over about 1.3x and is not even monotonic — 32 and 48 MiB +/// measured *slower* than 1 MiB — which is run-to-run noise wearing the shape +/// of a curve. At 1.10 the locator happily reports a knee in that noise, so +/// the gate to being a knee is set well above it. +const KNEE_MIN_SPREAD: f64 = 1.5; + fn enabled() -> bool { std::env::var("QSB_SEARCH_PERF").is_ok() } -fn seed(path: &std::path::Path) { - seed_index( - path, - &SeedSpec { - files: NUM_FILES, - body_words: 60, - ..SeedSpec::default() - }, - ); +/// A **realistically wide** `files` row, which is the whole calibration. +/// +/// The default seed stores `hash` NULL and a 16-character `/seed/NNN/` parent; +/// a real row carries a 32-byte content hash and a parent nested several +/// directories deep, and `parent` is stored per row. Since the working set +/// *is* the `files` table, calibrating a cache constant against the narrow +/// shape would under-size it by roughly the ratio between them — the bytes per +/// row are printed per arm so that ratio stays visible rather than assumed. +fn spec(files: usize) -> SeedSpec { + SeedSpec { + files, + // ~2 KB documents, the default: a corpus of tiny ones would make the + // full-text pass look free when it is the cascade's most expensive. + commit_every: 5_000, + // A hash on every row, as a real index has once hashing has run. + dup_every: 2, + // `/seed/NNN/word/word/word/word/word/` — about 50 characters, which + // is an ordinary depth for a document tree. + dir_depth: 6, + ..SeedSpec::default() + } +} + +fn mib(bytes: u64) -> f64 { + bytes as f64 / (1024.0 * 1024.0) } /// Run one query; hits are counted, not kept — holding 200k `SearchHit`s @@ -67,41 +169,125 @@ fn time_query(conn: &Connection, query: &str) -> (Duration, usize) { (start.elapsed(), count) } -/// Open at an explicit cache ceiling — `open_search_reader` deliberately -/// does not expose one, and comparing ceilings is the whole point. -fn open_at(path: &std::path::Path, cache_size: i64) -> Connection { - let conn = Connection::open(path).unwrap(); - conn.execute_batch(&format!( - "PRAGMA busy_timeout = 5000; - PRAGMA cache_size = {}; - PRAGMA temp_store = MEMORY; - PRAGMA foreign_keys = ON;", - cache_size - )) - .unwrap(); - conn +/// One arm at one cache ceiling: the first keystroke on a fresh connection, +/// then the steady state after a priming pass. +/// +/// The connection comes from the production `open_search_reader` with only +/// `cache_size` overridden, so the two key states differ by the key and +/// nothing else — the old version opened the plain arm with a raw +/// `Connection::open`, which skipped the key path entirely and made the two +/// columns incomparable. +fn measure(arm: &Arm, cache_size: i64) -> (Duration, Duration, usize) { + let conn = arm.open_search(); + conn.execute_batch(&format!("PRAGMA cache_size = {};", cache_size)) + .unwrap(); + + let (cold, hits) = time_query(&conn, SEQUENCE[0]); + // A priming pass, so "warm" is a settled session rather than the three + // keystrokes after the first. + for query in SEQUENCE { + time_query(&conn, query); + } + let total: Duration = SEQUENCE.iter().map(|q| time_query(&conn, q).0).sum(); + (cold, total / SEQUENCE.len() as u32, hits) } -fn run_matrix(label: &str, path: &std::path::Path) { - println!("\n=== {} ===", label); +fn run_matrix(arm: &Arm, files: usize) { + let files_bytes = arm.table_bytes("files"); + // Bytes per row is the constant `schema::recommended_search_cache_mib` is + // built on, so it is printed rather than left to be inferred from the + // table size and the corpus. + println!( + "\n=== {} === {:.1} MiB on disk, files table {:.1} MiB ({:.0} B/row), \ + seeded in {:.1?}", + arm.what, + mib(arm.size_bytes()), + mib(files_bytes), + files_bytes as f64 / files as f64, + arm.seeded_in + ); println!( "{:>12} {:>10} {:>10} {:>10} {:>8}", - "cache_size", "cold", "warm avg", "warm best", "hits" + "cache", "cold", "warm", "vs best", "hits" ); - for cache_size in CACHE_SIZES { - let conn = open_at(path, cache_size); - let (cold, hits) = time_query(&conn, SEQUENCE[0]); - let mut warm = Vec::new(); - for query in &SEQUENCE[1..] { - warm.push(time_query(&conn, query).0); - } - let avg = warm.iter().sum::() / warm.len() as u32; - let best = warm.iter().min().copied().unwrap_or_default(); + + let rows: Vec<(i64, Duration, Duration, usize)> = CACHE_SIZES + .iter() + .map(|&cache_size| { + let (cold, warm, hits) = measure(arm, cache_size); + (cache_size, cold, warm, hits) + }) + .collect(); + + let best = rows + .iter() + .map(|(_, _, warm, _)| *warm) + .min() + .unwrap_or_default(); + for (cache_size, cold, warm, hits) in &rows { + let ratio = warm.as_secs_f64() / best.as_secs_f64(); println!( - "{:>12} {:>9.1?} {:>9.1?} {:>9.1?} {:>8}", - cache_size, cold, avg, best, hits + "{:>9} MiB{} {:>9.1?} {:>9.1?} {:>9.2}x {:>8}", + -cache_size / 1024, + if *cache_size == SHIPPED_CACHE { + " *" + } else { + " " + }, + cold, + warm, + ratio, + hits ); } + + // A curve without a real step is the normal shape for a *plain* arm, and + // naming the low point of its noise a knee would invent a result. + let worst = rows + .iter() + .map(|(_, _, warm, _)| *warm) + .max() + .unwrap_or_default(); + let spread = worst.as_secs_f64() / best.as_secs_f64(); + if spread < KNEE_MIN_SPREAD { + println!( + "no knee — warm spans only {:.2}x across the whole sweep, under \ + the {:.1}x a step has to clear; files table is {:.1} MiB", + spread, + KNEE_MIN_SPREAD, + mib(files_bytes) + ); + return; + } + + // Otherwise the knee is the smallest ceiling still within tolerance of the + // best. Reported against the `files` table because that is the quantity it + // should track — if the two move together across corpora, the constant + // should be written in terms of file count. `max` on a negative KiB + // ceiling is the *smallest* cache. + let knee = rows + .iter() + .filter(|(_, _, warm, _)| warm.as_secs_f64() <= best.as_secs_f64() * KNEE_TOLERANCE) + .map(|(cache_size, _, _, _)| *cache_size) + .max() + .unwrap_or(SHIPPED_CACHE); + // The ratio is the number the constant is derived from: how much cache one + // byte of `files` needs. Reported per arm so the derivation can be checked + // against every corpus rather than fitted to one. + println!( + "knee at {} MiB (within {:.0}% of best warm); files table is {:.1} MiB; \ + ratio {:.2}x; shipped ceiling is {} MiB{}", + -knee / 1024, + (KNEE_TOLERANCE - 1.0) * 100.0, + mib(files_bytes), + (-knee * 1024) as f64 / files_bytes as f64, + -SHIPPED_CACHE / 1024, + if knee < SHIPPED_CACHE { + " — TOO SMALL for this corpus" + } else { + "" + } + ); } fn main() { @@ -109,52 +295,27 @@ fn main() { eprintln!("skipping: set QSB_SEARCH_PERF=1 to run"); return; } - unencrypted(); - encrypted(); -} - -/// The headline comparison, to be *read* when [`PRAGMAS_SEARCH`] is questioned. -fn unencrypted() { - let plain = scratch_db("searchperf-plain"); - let seeded = Instant::now(); - seed(&plain); - println!( - "seeded {} rows in {:.1?} ({} MiB on disk)", - NUM_FILES, - seeded.elapsed(), - std::fs::metadata(&plain).map(|m| m.len()).unwrap_or(0) / (1024 * 1024) - ); - run_matrix("unencrypted", &plain); -} - -/// Runs after [`unencrypted`]: [`set_process_key`] is process-global, so the -/// plain index must be opened before any key is set. -fn encrypted() { - set_process_key(Some( - IndexKey::from_hex(&"42".repeat(32)).expect("valid 32-byte key"), - )); - let enc = scratch_db("searchperf-enc"); - seed(&enc); - - println!("\n(encrypted: every cache miss costs an AES-CBC + HMAC-SHA512 per page)"); - // Both orders: a difference that survives reversing them is a property - // of the ceiling, not of when it was measured. - let mut order: Vec = CACHE_SIZES.to_vec(); - order.extend(CACHE_SIZES.iter().rev()); - for cache_size in order { - let conn = quicksearch_core::db::open_existing(&enc.to_string_lossy(), false).unwrap(); - conn.execute_batch(&format!("PRAGMA cache_size = {};", cache_size)) - .unwrap(); - let (cold, hits) = time_query(&conn, SEQUENCE[0]); - let mut warm = Vec::new(); - for query in &SEQUENCE[1..] { - warm.push(time_query(&conn, query).0); - } - let avg = warm.iter().sum::() / warm.len() as u32; - println!( - "{:>12} cold {:>9.1?} warm avg {:>9.1?} hits {}", - cache_size, cold, avg, hits + if std::env::var_os("TMPDIR").is_none() { + eprintln!( + "warning: TMPDIR unset — scratch goes to {}. If that is tmpfs the \ + large arms will not fit and every miss is a RAM copy.", + std::env::temp_dir().display() ); } - set_process_key(None); + for files in CORPORA { + for keyed in [false, true] { + let arm = Arm::seed( + format!( + "{} {}k files", + if keyed { "keyed" } else { "plain" }, + files / 1000 + ), + &format!("searchperf-{}-{}", files, keyed), + keyed, + &spec(files), + ); + run_matrix(&arm, files); + arm.discard(); + } + } } diff --git a/crates/quicksearch-core/examples/extractprobe.rs b/crates/quicksearch-core/examples/extractprobe.rs index 8d6d6f6..b411036 100644 --- a/crates/quicksearch-core/examples/extractprobe.rs +++ b/crates/quicksearch-core/examples/extractprobe.rs @@ -23,7 +23,11 @@ //! its arena free lists. With one worker the per-file table is exact — the //! largest entries are the files that would spike a real run. -use std::alloc::{GlobalAlloc, Layout, System}; +use std::alloc::{GlobalAlloc, Layout}; + +// What `Counting` wraps: the allocator the shipped binaries install, or the +// figures describe a build nobody runs. See `platform::Allocator`. +use quicksearch_core::platform::Allocator as Inner; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -35,7 +39,7 @@ mod common; // Allocation accounting // --------------------------------------------------------------------------- -/// `System`, counting — per binary, so the shipped `quicksearch` is +/// [`Inner`], counting — per binary, so the shipped `quicksearch` is /// untouched. `PEAK_LIVE` is the high-water of live bytes: unlike RSS it /// cannot be inflated by the allocator declining to return pages. struct Counting; @@ -53,14 +57,14 @@ fn note(live: u64) { unsafe impl GlobalAlloc for Counting { unsafe fn alloc(&self, l: Layout) -> *mut u8 { - let p = unsafe { System.alloc(l) }; + let p = unsafe { Inner.alloc(l) }; if !p.is_null() { note(LIVE.fetch_add(l.size() as u64, Ordering::Relaxed) + l.size() as u64); } p } unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 { - let p = unsafe { System.alloc_zeroed(l) }; + let p = unsafe { Inner.alloc_zeroed(l) }; if !p.is_null() { note(LIVE.fetch_add(l.size() as u64, Ordering::Relaxed) + l.size() as u64); } @@ -68,10 +72,10 @@ unsafe impl GlobalAlloc for Counting { } unsafe fn dealloc(&self, p: *mut u8, l: Layout) { LIVE.fetch_sub(l.size() as u64, Ordering::Relaxed); - unsafe { System.dealloc(p, l) } + unsafe { Inner.dealloc(p, l) } } unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 { - let q = unsafe { System.realloc(p, l, new) }; + let q = unsafe { Inner.realloc(p, l, new) }; if !q.is_null() { let (old, new) = (l.size() as u64, new as u64); let live = if new >= old { @@ -103,7 +107,7 @@ use quicksearch_core::testutil::mib; /// Head bytes read for the MIME sniff — the same window the walk uses, so /// this probe classifies files exactly as a run would. -fn sniff(path: &Path, hash_length: usize) -> Option { +fn sniff(path: &Path, hash_length: usize) -> Option<&'static str> { use std::io::Read; let mut f = std::fs::File::open(path).ok()?; let mut head = vec![0u8; hash_length]; @@ -114,7 +118,7 @@ fn sniff(path: &Path, hash_length: usize) -> Option { struct Candidate { path: String, - mime: String, + mime: &'static str, size: u64, } @@ -137,7 +141,7 @@ fn candidates(dir: &Path, config: &Config, registry: &Registry) -> Vec = (0..replicas).flat_map(|_| found.iter()).collect(); let next = AtomicUsize::new(0); - let worst: Mutex> = Mutex::new(Vec::new()); + let worst: Mutex> = Mutex::new(Vec::new()); let per_file = workers == 1; let start = Instant::now(); @@ -203,13 +207,16 @@ fn main() { for _ in 0..workers { let (queue, next, worst) = (&queue, &next, &worst); let (registry, config) = (registry.clone(), config.clone()); + // One per worker, as the content pass does: the per-file figures + // below are a worker's steady state, not its first file. + let mut scratch = quicksearch_core::extract::Scratch::new(&config); s.spawn(move || loop { let i = next.fetch_add(1, Ordering::Relaxed); let Some(c) = queue.get(i) else { return }; if per_file { take_mark(); } - let outcome = decide_content(&c.path, Some(&c.mime), ®istry, &config); + let outcome = decide_content(&c.path, Some(c.mime), ®istry, &config, &mut scratch); if per_file { let cost = take_mark(); let text = @@ -217,7 +224,7 @@ fn main() { let mut w = worst .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - w.push((cost, c.path.clone(), c.size, c.mime.clone())); + w.push((cost, c.path.clone(), c.size, c.mime)); // Kept small so the table itself is not the peak. w.sort_by_key(|(cost, ..)| std::cmp::Reverse(*cost)); w.truncate(12); diff --git a/crates/quicksearch-core/examples/hashprobe.rs b/crates/quicksearch-core/examples/hashprobe.rs index d9e7934..884143c 100644 --- a/crates/quicksearch-core/examples/hashprobe.rs +++ b/crates/quicksearch-core/examples/hashprobe.rs @@ -113,7 +113,7 @@ fn scan_one(path: &Path, size: u64, config: &Config, registry: &Registry) -> Opt } let extractable = size <= config.processing.maximum_text_file_size - && content_extractable(path, base_mime.as_deref(), config, registry); + && content_extractable(path, base_mime, config, registry); Some(Scan { size, @@ -768,7 +768,7 @@ mod tests { members: 10, baseline_groups: 4, reported_pairs: 45, - fp_pairs: 45 - (3 + 1 + 0 + 6), + fp_pairs: 45 - ((3 + 1) + 6), overstated_bytes: 3 * 64, }, }, diff --git a/crates/quicksearch-core/examples/indexprobe.rs b/crates/quicksearch-core/examples/indexprobe.rs index c6b2665..f7756d1 100644 --- a/crates/quicksearch-core/examples/indexprobe.rs +++ b/crates/quicksearch-core/examples/indexprobe.rs @@ -31,7 +31,11 @@ mod common; -use std::alloc::{GlobalAlloc, Layout, System}; +use std::alloc::{GlobalAlloc, Layout}; + +// What `Counting` wraps: the allocator the shipped binaries install, or the +// throughput figures describe a build nobody runs. +use quicksearch_core::platform::Allocator as Inner; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; @@ -41,7 +45,7 @@ use common::{evict, mib, Io}; // Allocation accounting // --------------------------------------------------------------------------- -/// `System`, counting — per binary, so the shipped `quicksearch` is +/// [`Inner`], counting — per binary, so the shipped `quicksearch` is /// untouched. Global atomics, not `search_alloc`'s per-thread `Cell`s: the /// work spreads over several pools and nothing else runs here, so a global /// count is exactly the run. The contended RMW is fine when both sides of a @@ -63,14 +67,14 @@ fn note_alloc(size: usize) { unsafe impl GlobalAlloc for Counting { unsafe fn alloc(&self, l: Layout) -> *mut u8 { - let p = unsafe { System.alloc(l) }; + let p = unsafe { Inner.alloc(l) }; if !p.is_null() { note_alloc(l.size()); } p } unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 { - let p = unsafe { System.alloc_zeroed(l) }; + let p = unsafe { Inner.alloc_zeroed(l) }; if !p.is_null() { note_alloc(l.size()); } @@ -78,10 +82,10 @@ unsafe impl GlobalAlloc for Counting { } unsafe fn dealloc(&self, p: *mut u8, l: Layout) { LIVE.fetch_sub(l.size() as u64, Ordering::Relaxed); - unsafe { System.dealloc(p, l) } + unsafe { Inner.dealloc(p, l) } } unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 { - let q = unsafe { System.realloc(p, l, new) }; + let q = unsafe { Inner.realloc(p, l, new) }; if !q.is_null() { let (old, new) = (l.size() as u64, new as u64); ALLOC_BYTES.fetch_add(new.saturating_sub(old), Ordering::Relaxed); diff --git a/crates/quicksearch-core/examples/memprobe.rs b/crates/quicksearch-core/examples/memprobe.rs index 8099d23..bbbab2b 100644 --- a/crates/quicksearch-core/examples/memprobe.rs +++ b/crates/quicksearch-core/examples/memprobe.rs @@ -30,6 +30,11 @@ //! naming what its run-scoped structures hold, which is the half `smaps` //! cannot answer: a mapping is "heap", never "the stale-candidate list". +// Settled RSS is the whole point of this probe, and it is a property of the +// allocator, so it must be the one the shipped binaries install. +#[global_allocator] +static GLOBAL: quicksearch_core::platform::Allocator = quicksearch_core::platform::Allocator; + use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; diff --git a/crates/quicksearch-core/examples/oleprobe.rs b/crates/quicksearch-core/examples/oleprobe.rs index 4b04ba4..d0627ab 100644 --- a/crates/quicksearch-core/examples/oleprobe.rs +++ b/crates/quicksearch-core/examples/oleprobe.rs @@ -8,15 +8,23 @@ use std::path::Path; -use quicksearch_core::extract::{Extractor, Registry}; +use quicksearch_core::config::Config; +use quicksearch_core::extract::{Extractor, Registry, Scratch}; fn main() { let mut failures = 0; + // The extractor is called directly, by extension, so a file whose MIME + // the sniff would get wrong still says what the parser makes of it. + let config = Config::default(); + let mut scratch = Scratch::new(&config); for arg in std::env::args().skip(1) { let path = Path::new(&arg); println!("=== {} ===", path.display()); - match quicksearch_core::extract::office::OfficeExtractor.extract(path) { - Ok(text) => { + let mut text = String::new(); + match quicksearch_core::extract::office::OfficeExtractor + .extract(path, &mut text, &mut scratch) + { + Ok(()) => { println!("{} chars", text.chars().count()); let preview: String = text.chars().take(400).collect(); println!("{}", preview); diff --git a/crates/quicksearch-core/examples/rssprobe.rs b/crates/quicksearch-core/examples/rssprobe.rs index 31440e0..fa88ce5 100644 --- a/crates/quicksearch-core/examples/rssprobe.rs +++ b/crates/quicksearch-core/examples/rssprobe.rs @@ -17,6 +17,10 @@ //! `glibc arenas` line says whether an anonymous figure is live data or //! retention that `malloc_trim(3)` could return. +// The GUI's idle footprint is an allocator property too; see `memprobe`. +#[global_allocator] +static GLOBAL: quicksearch_core::platform::Allocator = quicksearch_core::platform::Allocator; + use quicksearch_core::testutil::{mib, size_class}; use std::collections::HashMap; use std::path::PathBuf; diff --git a/crates/quicksearch-core/examples/walkprobe.rs b/crates/quicksearch-core/examples/walkprobe.rs index 8aa8cf7..853f7b9 100644 --- a/crates/quicksearch-core/examples/walkprobe.rs +++ b/crates/quicksearch-core/examples/walkprobe.rs @@ -60,6 +60,8 @@ fn main() { fn serial(root: &str, config: &Config, existing: &DirRows) -> (usize, usize) { let ignore = IgnoreSet::compile(&[]).unwrap(); let registry = Registry::default_set(); + // One for the whole walk, as a walk worker holds one. + let mut scratch = quicksearch_core::extract::Scratch::new(config); let (mut seen, mut prepared) = (0, 0); for entry in filtered_walk(root, false, false, &ignore, &UnreadableDirs::default()) { seen += 1; @@ -83,7 +85,7 @@ fn serial(root: &str, config: &Config, existing: &DirRows) -> (usize, usize) { .map(|n| n.to_string_lossy().into_owned()) .unwrap_or_default(); if classify_for_indexing(&name, mtime, existing) != FileIndexAction::Skip - && prepare_file_record(&path, &meta, config, ®istry).is_some() + && prepare_file_record(&path, &meta, config, ®istry, &mut scratch).is_some() { prepared += 1; } diff --git a/crates/quicksearch-core/src/config/mod.rs b/crates/quicksearch-core/src/config/mod.rs index 2dfb447..71187b6 100644 --- a/crates/quicksearch-core/src/config/mod.rs +++ b/crates/quicksearch-core/src/config/mod.rs @@ -88,6 +88,19 @@ pub struct ProcessingConfig { /// checkpoint, in bytes. `0` disables; else raised to [`MINIMUM_WAL_SIZE`]. /// Needed because autocheckpoint can only *reset* the log when no reader /// is mid-query, and a run keeps a reader per root busy throughout. + /// + /// **Both directions cost something, which is why the default is neither + /// end of the range.** A checkpoint blocks the writer for its whole + /// copy-back and has to evict every per-root reader first, so a low value + /// stalls indexing often. A high one is paid by *readers*: SQLite searches + /// the log before every page it fetches from the database file, one hash + /// block per 4096 frames, and a page that is not in the log is charged for + /// all of them — so a larger log slows the walk prefetchers and every + /// search run alongside indexing. It also lengthens WAL recovery after an + /// unclean exit, which is read and checksummed frame by frame. + /// + /// The default trades toward fewer stalls; lower it if searching during a + /// run matters more than the run finishing quickly. pub maximum_wal_size: u64, pub tokenize: String, /// When `true` (default), extracted text is stored zstd-compressed in @@ -124,6 +137,12 @@ pub struct SearchConfig { /// Watch the visible search results and show renames, deletions and /// content changes as they happen. See [`crate::live`]. pub live_results: bool, + /// Page cache held by the search connection, in MiB. **`0` derives it from + /// the index** — see [`crate::db::schema::recommended_search_cache_mib`], + /// which sizes it to hold the `files` table because that is what every + /// keystroke rescans. Set it only when the derived value is wrong for your + /// tree; the GUI shows the recommendation next to the field. + pub cache_size_mib: usize, pub columns: ColumnsConfig, } @@ -200,7 +219,7 @@ impl Default for ProcessingConfig { batch_size: 500, writer_turn_slice_ms: 100, fts_update_batch_size: 1000, - maximum_wal_size: 1024 * 1024 * 512, + maximum_wal_size: 1024 * 1024 * 1024 * 2, tokenize: "trigram".to_string(), store_text_for_snippets: true, } @@ -216,6 +235,8 @@ impl Default for SearchConfig { results_per_page: 100, debounce_ms: 150, live_results: true, + // Derived from the index; see the field's doc comment. + cache_size_mib: 0, columns: ColumnsConfig::default(), } } @@ -273,6 +294,16 @@ pub struct UiConfig { /// typed-out enum would fail to deserialize and take the whole config /// file down with it. pub color_scheme: String, + /// Whether the Settings tab shows the technical settings as well as the + /// everyday ones. Off is the default: most of that tab is byte budgets and + /// indexer internals that a person who indexed their home folder will + /// never need, and cannot evaluate without already knowing how the indexer + /// works. + /// + /// A view preference, not a setting the rest of the program reads — it is + /// written the moment the box is ticked, without an Apply, the way the + /// column picker is. + pub show_advanced_settings: bool, /// Whether the first-start tour has been dismissed. `None` means the key /// predates the tour, so only a config this version *created* is offered /// it. @@ -292,6 +323,7 @@ impl Default for UiConfig { watch_cap_warned_roots: Vec::new(), search_hotkey: "Ctrl+Shift+F".to_string(), color_scheme: "dark".to_string(), + show_advanced_settings: false, // `Some(false)`, not `None`: `None` is reserved for a file that // predates the key. tutorial_seen: Some(false), @@ -451,6 +483,21 @@ impl Config { clamp("[search] display_limit", &mut display_limit, 1, 1_000_000); self.search.display_limit = display_limit as usize; + // 0 is the automatic setting and must survive the clamp; anything else + // is held to the range the sweep found useful — under the floor is + // slower than automatic would be, over the cap is resident memory for + // nothing. + if self.search.cache_size_mib != 0 { + let mut cache = self.search.cache_size_mib as u64; + clamp( + "[search] cache_size_mib", + &mut cache, + crate::db::schema::SEARCH_CACHE_MIN_MIB as u64, + crate::db::schema::SEARCH_CACHE_OVERRIDE_MAX_MIB as u64, + ); + self.search.cache_size_mib = cache as usize; + } + clamp( "[processing] maximum_text_file_size", &mut self.processing.maximum_text_file_size, @@ -458,6 +505,19 @@ impl Config { 4 * 1024 * 1024 * 1024, ); + // Not just the stored text: it is what the extractors size their + // buffers from (`extract::Limits`), and those are held per worker + // across pools. 16 MiB is far above any document worth full-text + // indexing whole and keeps the derived inflation budget sane. + let mut text_size = self.processing.maximum_text_size as u64; + clamp( + "[processing] maximum_text_size", + &mut text_size, + 1, + 16 * 1024 * 1024, + ); + self.processing.maximum_text_size = text_size as usize; + // Below 262 bytes `infer`'s longest magic-number matcher cannot run. let mut hash_length = self.processing.hash_length as u64; clamp( diff --git a/crates/quicksearch-core/src/config/tests.rs b/crates/quicksearch-core/src/config/tests.rs index c55cb88..0caef86 100644 --- a/crates/quicksearch-core/src/config/tests.rs +++ b/crates/quicksearch-core/src/config/tests.rs @@ -782,7 +782,10 @@ fn salt_bytes_validates_hostile_configs() { #[test] fn ui_bookkeeping_fields_are_soft_knobs() { let base = Config::default(); - let cases: [(&str, fn(&mut Config)); 2] = [ + // Named so the closures below coerce to fn pointers and share one array + // type; without an annotation each would be its own anonymous type. + type Knob = (&'static str, fn(&mut Config)); + let cases: [Knob; 2] = [ ("watch_cap_warned_roots", |c| { c.ui.watch_cap_warned_roots = vec!["/media/ApolloStore".to_string()] }), diff --git a/crates/quicksearch-core/src/content.rs b/crates/quicksearch-core/src/content.rs index bd620cd..a8a3e71 100644 --- a/crates/quicksearch-core/src/content.rs +++ b/crates/quicksearch-core/src/content.rs @@ -30,22 +30,34 @@ const FEED_PAGE: usize = 128; #[derive(Debug)] pub struct ExtractedRow { pub file_id: i64, - /// The `files.name` the FTS row is indexed under. - pub name: String, + /// The path buffer the feeder built, carried through rather than split: + /// see [`crate::db::repo::RowPath`]. + path: crate::db::repo::RowPath, pub outcome: ContentOutcome, } -#[derive(Debug)] -struct Pending { - file_id: i64, - name: String, - path: String, - mime: Option, +impl ExtractedRow { + pub fn new( + file_id: i64, + path: crate::db::repo::RowPath, + outcome: ContentOutcome, + ) -> ExtractedRow { + ExtractedRow { + file_id, + path, + outcome, + } + } + + /// The `files.name` the FTS row is indexed under. + pub fn name(&self) -> &str { + self.path.name() + } } #[derive(Default)] struct Queue { - rows: Vec, + rows: Vec, /// Feeder mid-query, holding rows in neither the queue nor a worker; /// without it a worker could see an empty queue between two pages and /// declare the pass finished early. @@ -66,7 +78,7 @@ struct Shared { impl Shared { /// Claim a row. `None` only when the queue is empty *and* the feeder is /// finished — at that instant nobody is left who could add another row. - fn take(&self) -> Option { + fn take(&self) -> Option { let mut q = crate::lock_ok(&self.queue); loop { if q.done { @@ -109,7 +121,7 @@ impl Shared { /// Publish a page and clear the in-flight flag together, under one lock — /// the indivisibility [`Shared::take`]'s end-of-pass test relies on. - fn finish_feed(&self, rows: Vec, last_page: bool) { + fn finish_feed(&self, rows: Vec, last_page: bool) { let mut q = crate::lock_ok(&self.queue); // Reversed: `take` pops from the back, and rows should reach workers // in id order so a partial run leaves a contiguous prefix done. @@ -225,19 +237,10 @@ fn feeder(shared: &Shared, db_path: &str, mut cursor: ExtractCursor, config: &Co } }; let last_page = page.len() < FEED_PAGE; - if let Some((id, _, _, _)) = page.last() { - cursor.last_id = *id; + if let Some(row) = page.last() { + cursor.last_id = row.file_id; } - let rows = page - .into_iter() - .map(|(file_id, name, path, mime)| Pending { - file_id, - name, - path, - mime, - }) - .collect(); - shared.finish_feed(rows, last_page); + shared.finish_feed(page, last_page); count_now(&conn, &cursor); if last_page { return; @@ -256,16 +259,25 @@ fn worker( stop_flag: &Arc, stats: &WorkerStats, ) { + // One per worker, for the whole pass: the container and stream buffers + // inside it are what every extraction stages through. + let mut scratch = crate::extract::Scratch::new(config); while let Some(row) = shared.take() { let _busy = stats.enter(); if stop_flag.load(Ordering::Relaxed) { shared.shutdown(); return; } - let outcome = decide_content(&row.path, row.mime.as_deref(), registry, config); + let outcome = decide_content( + row.path.as_str(), + row.mime.as_deref(), + registry, + config, + &mut scratch, + ); let sent = tx.send(ExtractedRow { file_id: row.file_id, - name: row.name, + path: row.path, outcome, }); if sent.is_err() { diff --git a/crates/quicksearch-core/src/db/key.rs b/crates/quicksearch-core/src/db/key.rs index a7b0255..467d29b 100644 --- a/crates/quicksearch-core/src/db/key.rs +++ b/crates/quicksearch-core/src/db/key.rs @@ -3,6 +3,7 @@ use std::sync::RwLock; +use super::schema::{HmacMode, Profile}; use crate::security::IndexKey; static PROCESS_KEY: RwLock> = RwLock::new(None); @@ -32,3 +33,85 @@ pub fn process_key_hex() -> Option { .as_ref() .map(|k| k.to_hex()) } + +static PAGE_SIZE: std::sync::atomic::AtomicI64 = + std::sync::atomic::AtomicI64::new(super::schema::PAGE_SIZE); + +/// Override the page size every subsequent open applies, for +/// `benches/page_geometry.rs` to sweep it. A process-global for the same +/// reason [`set_process_key`] is one: a keyed file's page size cannot be read +/// off the file — the header is ciphertext until SQLCipher has been told the +/// size — so it has to be known before the open, not derived during it. +/// +/// **Measurement only.** Production never calls this, and an index seeded +/// under an override must be *opened* under the same one or it will not +/// decrypt. +#[doc(hidden)] +pub fn set_page_size_override(page_size: i64) { + PAGE_SIZE.store(page_size, std::sync::atomic::Ordering::Relaxed); +} + +/// `HmacMode` as an atom. The discriminants are private to this pair of +/// functions and never reach disk — the *reserve* is what the format records. +static HMAC_MODE: std::sync::atomic::AtomicU8 = + std::sync::atomic::AtomicU8::new(encode_hmac(super::schema::HMAC_MODE)); + +const fn encode_hmac(mode: HmacMode) -> u8 { + match mode { + HmacMode::Off => 0, + HmacMode::Sha256 => 1, + HmacMode::Sha512 => 2, + } +} + +fn decode_hmac(byte: u8) -> HmacMode { + match byte { + 0 => HmacMode::Off, + 1 => HmacMode::Sha256, + _ => HmacMode::Sha512, + } +} + +/// Override the per-page authenticator every subsequent open applies, for +/// `benches/cipher_hmac.rs` to sweep it. A process-global for exactly the +/// reason [`set_page_size_override`] is one, and with the same warning: the +/// mode decides the page reserve, so a keyed file written under one and opened +/// under another decrypts to noise. +/// +/// **Measurement only.** Production never calls this. +#[doc(hidden)] +pub fn set_hmac_mode_override(mode: HmacMode) { + HMAC_MODE.store(encode_hmac(mode), std::sync::atomic::Ordering::Relaxed); +} + +/// The layout this open should apply: [`super::schema::PROFILE`] unless a +/// measurement harness has overridden part of it. +pub(crate) fn current_profile() -> Profile { + Profile { + page_size: PAGE_SIZE.load(std::sync::atomic::Ordering::Relaxed), + hmac: decode_hmac(HMAC_MODE.load(std::sync::atomic::Ordering::Relaxed)), + } +} + +/// `0` means "derive it from the index"; see [`set_search_cache_override`]. +static SEARCH_CACHE_MIB: std::sync::atomic::AtomicI64 = std::sync::atomic::AtomicI64::new(0); + +/// Override the search connection's cache ceiling, in MiB, from +/// `[search] cache_size_mib`. `None` restores the derived value. +/// +/// A process-global for the same reason the key is one: the search `Worker` +/// (`crate::search`) holds no `Config` — options travel per request in +/// `SearchOptions`, and this is a property of the *connection*, which the +/// worker opens and reopens on its own. Install it wherever the key is +/// installed, and again when settings are saved. +pub fn set_search_cache_override(cache_mib: Option) { + SEARCH_CACHE_MIB.store(cache_mib.unwrap_or(0), std::sync::atomic::Ordering::Relaxed); +} + +/// The configured override, or `None` to derive one. +pub(crate) fn search_cache_override() -> Option { + match SEARCH_CACHE_MIB.load(std::sync::atomic::Ordering::Relaxed) { + 0 => None, + mib => Some(mib), + } +} diff --git a/crates/quicksearch-core/src/db/mod.rs b/crates/quicksearch-core/src/db/mod.rs index 5cc74a4..b8b5945 100644 --- a/crates/quicksearch-core/src/db/mod.rs +++ b/crates/quicksearch-core/src/db/mod.rs @@ -11,7 +11,10 @@ pub mod open; pub mod repo; pub mod schema; -pub use key::{process_key_hex, set_process_key}; +pub use key::{ + process_key_hex, set_hmac_mode_override, set_page_size_override, set_process_key, + set_search_cache_override, +}; pub use open::{ index_needs_rebuild, key_mismatch_parts, open_existing, open_or_recreate, verify_process_key, KeyMismatch, CURRENT_SCHEMA_VERSION, FOREIGN_DB_PREFIX, KEY_MISMATCH_PREFIX, diff --git a/crates/quicksearch-core/src/db/open.rs b/crates/quicksearch-core/src/db/open.rs index b6b6cac..9e70af9 100644 --- a/crates/quicksearch-core/src/db/open.rs +++ b/crates/quicksearch-core/src/db/open.rs @@ -7,8 +7,9 @@ use std::path::Path; use rusqlite::{params, Connection, OpenFlags, OptionalExtension}; use super::schema::{ - effective_tokenizer, fts_create_sql, PRAGMAS_FAST, PRAGMAS_INCREMENTAL, PRAGMAS_MAINTENANCE, - PRAGMAS_READONLY, PRAGMAS_SEARCH, PRAGMAS_WALK_READER, SCHEMA_CURRENT, + effective_tokenizer, fts_create_sql, fts_set_pgsz, pragmas_search, + recommended_search_cache_mib, Profile, PRAGMAS_FAST, PRAGMAS_INCREMENTAL, PRAGMAS_MAINTENANCE, + PRAGMAS_READONLY, PRAGMAS_WALK_READER, SCHEMA_CURRENT, }; use crate::security::IndexKey; @@ -20,7 +21,16 @@ pub const KEY_MISMATCH_PREFIX: &str = "KEY_MISMATCH: "; /// Bump on any schema change — and on classifier changes: `files.mime`, /// `files.type` and `content_state` are computed at walk time and never /// re-derived for unchanged files, so only the wipe applies them everywhere. -pub const CURRENT_SCHEMA_VERSION: u32 = 8; +/// +/// v11 is a cipher-profile change, not a table change: [`schema::HMAC_MODE`] +/// became `Off`, which moves the page reserve and with it FTS5's record size. +/// A *keyed* index would have been condemned anyway — the profile retry in +/// [`open_probed`] is what spots it, since the version cannot be read off a +/// file that will not decrypt — so this bump is what brings **unprotected** +/// indexes along, on the release boundary rather than piecemeal. +/// +/// [`schema::HMAC_MODE`]: super::schema::HMAC_MODE +pub const CURRENT_SCHEMA_VERSION: u32 = 11; /// Open `db_path`; on any schema/tokenizer mismatch, delete the file and /// recreate it empty — callers will need to re-index. @@ -40,30 +50,91 @@ pub(crate) fn open_or_recreate_keyed( .map_err(|e| format!("Failed to create database dir {}: {}", dir.display(), e))?; } } - let conn = Connection::open(db_path) - .map_err(|e| format!("Failed to open database at {}: {}", db_path, e))?; - // Before a single row is written: SQLite creates the file 0644 (inherited - // by `-wal`/`-shm`), and the index holds the full text of files whose own - // permissions are 0600. - crate::platform::restrict_to_owner(&path); - key_and_probe(&conn, db_path, key)?; + let want = super::key::current_profile(); + let (conn, matched) = open_probed(db_path, key, want, |p| { + let conn = + Connection::open(p).map_err(|e| format!("Failed to open database at {}: {}", p, e))?; + // Before a single row is written: SQLite creates the file 0644 + // (inherited by `-wal`/`-shm`), and the index holds the full text of + // files whose own permissions are 0600. + crate::platform::restrict_to_owner(Path::new(p)); + Ok(conn) + })?; conn.execute_batch(PRAGMAS_FAST) .map_err(|e| format!("Failed to apply pragmas: {}", e))?; - if db_matches_current(&conn, tokenizer)? { - return Ok(conn); + match matched { + // The profile is fixed for a file's life — neither the page size nor + // the page reserve can be changed in place — so a file that only + // opened under an older one has to be rebuilt whatever its schema says. + ProfileMatch::Previous(found) => crate::log_warn!( + "database at {} was built with {} and this build uses {}; \ + rebuilding. Existing rows will be re-scanned on next indexing run.", + db_path, + found, + want + ), + ProfileMatch::Current => { + if db_matches_current(&conn, tokenizer)? { + return Ok(conn); + } + crate::log_warn!( + "database at {} does not match current schema; rebuilding. \ + Existing rows will be re-scanned on next indexing run.", + db_path + ); + } } - - crate::log_warn!( - "database at {} does not match current schema; rebuilding. \ - Existing rows will be re-scanned on next indexing run.", - db_path - ); - let conn = wipe_and_reopen(conn, &path, key)?; - apply_current_schema(&conn, tokenizer)?; + let conn = wipe_and_reopen(conn, &path, key, want)?; + apply_current_schema(&conn, tokenizer, key, want)?; Ok(conn) } +/// Which layout the file on disk answered to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ProfileMatch { + Current, + /// Opened only under this [`super::schema::PROFILES_PREVIOUS`] entry. + Previous(Profile), +} + +/// Open through `make`, apply the key and `profile`, and probe. On a keyed +/// file that will not decrypt, retry under each [`PROFILES_PREVIOUS`] entry +/// before giving up — a profile change is otherwise indistinguishable from a +/// wrong password, and answering it with "wrong password" would be a lie the +/// user cannot act on. +fn open_probed( + db_path: &str, + key: Option<&IndexKey>, + profile: Profile, + make: impl Fn(&str) -> Result, +) -> Result<(Connection, ProfileMatch), String> { + let conn = make(db_path)?; + let failure = match key_and_probe(&conn, db_path, key, profile) { + Ok(()) => return Ok((conn, ProfileMatch::Current)), + Err(e) => e, + }; + // Only a profile mismatch is worth retrying, and only when a key is what + // makes the layout undiscoverable. An unencrypted file reports its own + // page size and has no reserve, so a failure there is a real one. + if key.is_none() || !failure.starts_with(KEY_MISMATCH_PREFIX) { + return Err(failure); + } + for previous in super::schema::PROFILES_PREVIOUS { + if *previous == profile { + continue; + } + // A fresh connection: after a failed decrypt the pager has already + // formed an opinion about the file, and both `cipher_page_size` and + // the HMAC pragmas are only honoured before the first read. + let retry = make(db_path)?; + if key_and_probe(&retry, db_path, key, *previous).is_ok() { + return Ok((retry, ProfileMatch::Previous(*previous))); + } + } + Err(failure) +} + /// Open an *existing* index: any schema mismatch is an error instead of a /// wipe. Every *consumer* uses this; only the indexer's own write path uses /// [`open_or_recreate`]. @@ -81,8 +152,25 @@ pub fn open_walk_reader(db_path: &str) -> Result { } /// The search worker's connection, held across requests. +/// +/// The only profile whose cache ceiling is not a constant: it has to hold the +/// `files` table, which every keystroke rescans, and that scales with the +/// index. The order matters — the connection is opened on the read-only +/// profile, the ceiling is worked out *from* it, and only then is the search +/// profile applied over the top. `PRAGMA cache_size` is settable at any time, +/// so the brief moment on the smaller ceiling costs one `sqlite_stat1` read. pub fn open_search_reader(db_path: &str) -> Result { - open_profiled(db_path, false, PRAGMAS_SEARCH) + let conn = open_profiled(db_path, false, PRAGMAS_READONLY)?; + let cache_mib = super::key::search_cache_override().unwrap_or_else(|| { + // No stats yet means a fresh or never-optimised index; the floor is + // right for one, and `repo::maintain` will have run by the time an + // index is large enough for it to be wrong. + let files = super::repo::analyzed_file_count(&conn).unwrap_or(0); + recommended_search_cache_mib(files, super::key::process_key().is_some()) + }); + conn.execute_batch(&pragmas_search(cache_mib)) + .map_err(|e| format!("Failed to apply search pragmas: {}", e))?; + Ok(conn) } /// The coordinator's write connection for watcher events and reconciles. @@ -121,13 +209,24 @@ fn open_keyed_with_pragmas( } else { OpenFlags::SQLITE_OPEN_READ_ONLY }; - let conn = Connection::open_with_flags(db_path, flags) - .map_err(|e| format!("Failed to open database at {}: {}", db_path, e))?; - key_and_probe(&conn, db_path, key)?; + let (conn, matched) = open_probed(db_path, key, super::key::current_profile(), |p| { + Connection::open_with_flags(p, flags) + .map_err(|e| format!("Failed to open database at {}: {}", p, e)) + })?; conn.execute_batch(pragmas) .map_err(|e| format!("Failed to apply pragmas: {}", e))?; - if !schema_version_current(&conn)? { + // A file that answered only to a previous profile is one `open_or_recreate` + // is about to wipe, so a consumer must be turned away now. The version + // check beside it is not enough on its own, and the independence is the + // point: nothing forces a profile change to come with a schema bump, and + // an unbumped one would leave this reading a file back perfectly while the + // indexer replaces the inode under it. Both conditions, so neither has to + // be remembered. + // + // The refusal is deliberately the re-index one and not + // [`KEY_MISMATCH_PREFIX`]: the password was right. + if !schema_version_current(&conn)? || matched != ProfileMatch::Current { return Err(format!( "index at {} is not a compatible QuickSearch index (schema v{} expected); \ refusing to modify it. Re-index to rebuild.", @@ -148,14 +247,25 @@ pub fn verify_process_key(db_path: &str) -> Result<(), String> { /// `false` for anything this cannot positively establish: announcing a reset /// that is not happening would be worse than saying nothing. pub fn index_needs_rebuild(db_path: &str) -> bool { - let Ok(conn) = Connection::open_with_flags( + let opened = open_probed( db_path, - OpenFlags::SQLITE_OPEN_NO_MUTEX | OpenFlags::SQLITE_OPEN_READ_ONLY, - ) else { + super::key::process_key().as_ref(), + super::key::current_profile(), + |p| { + Connection::open_with_flags( + p, + OpenFlags::SQLITE_OPEN_NO_MUTEX | OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|e| format!("Failed to open database at {}: {}", p, e)) + }, + ); + let Ok((conn, matched)) = opened else { return false; }; - if key_and_probe(&conn, db_path, super::key::process_key().as_ref()).is_err() { - return false; + // A file under an older profile is certain to be rebuilt: neither the page + // size nor the page reserve can be changed in place. + if matched != ProfileMatch::Current { + return true; } // Only `Ok(false)`: an `Err` means we could not tell. matches!(schema_version_current(&conn), Ok(false)) @@ -164,31 +274,88 @@ pub fn index_needs_rebuild(db_path: &str) -> bool { pub(crate) fn verify_key(db_path: &str, key: Option<&IndexKey>) -> Result<(), String> { // Read-only and no CREATE: verifying a key must never bring a database // into existence, and must never modify one. - let conn = Connection::open_with_flags( - db_path, - OpenFlags::SQLITE_OPEN_NO_MUTEX | OpenFlags::SQLITE_OPEN_READ_ONLY, - ) - .map_err(|e| format!("Failed to open database at {}: {}", db_path, e))?; - key_and_probe(&conn, db_path, key) + // + // The profile is discarded: this answers the *key* question, and a key + // that opens the file under an older profile is the right key. Telling a + // user their password is wrong because their index predates a page-size or + // HMAC change would be the worst answer available. + open_probed(db_path, key, super::key::current_profile(), |p| { + Connection::open_with_flags( + p, + OpenFlags::SQLITE_OPEN_NO_MUTEX | OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|e| format!("Failed to open database at {}: {}", p, e)) + }) + .map(|_| ()) } -/// Apply the SQLCipher key (if any) and force the first page off disk. +/// SQLCipher keeps its `cipher_default_*` settings in process-wide statics, +/// and [`key_and_probe`] has to write them to select an [`HmacMode`]. This +/// covers the window between writing them and the `PRAGMA key` that consumes +/// them, so two threads opening under different profiles cannot interleave. /// -/// Ordering is load-bearing twice over: SQLCipher requires `PRAGMA key` -/// before anything else touches the file, and the probe must run before any -/// schema comparison so a wrong key surfaces as [`KEY_MISMATCH_PREFIX`] — -/// never as a "schema mismatch" that [`open_or_recreate`] answers by wiping. -/// The raw-key `x'…'` form bypasses SQLCipher's per-connection PBKDF2. -fn key_and_probe(conn: &Connection, db_path: &str, key: Option<&IndexKey>) -> Result<(), String> { +/// Held for the length of one pragma batch and never across a query, so it +/// costs a connection setup, not a search. +static CIPHER_DEFAULTS: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// Apply the SQLCipher key (if any) and the profile, then force the first +/// page off disk. +/// +/// Ordering is load-bearing four times over: the HMAC mode has to be chosen +/// *before* `PRAGMA key`, `PRAGMA key` has to precede anything else that +/// touches the file, `cipher_page_size` has to follow the key and precede the +/// first read, and the probe must run before any schema comparison so a wrong +/// key surfaces as [`KEY_MISMATCH_PREFIX`] — never as a "schema mismatch" that +/// [`open_or_recreate`] answers by wiping. The raw-key `x'…'` form bypasses +/// SQLCipher's per-connection PBKDF2. +/// +/// # Why the HMAC mode goes first, as a *default* +/// +/// The obvious spelling — `PRAGMA cipher_use_hmac = OFF` after the key — is +/// silently ignored. `sqlite3BtreeSetPageSize` will only ever *raise* a page +/// reserve (`if( nReserve, + profile: Profile, +) -> Result<(), String> { if let Some(key) = key { // `cipher_log_level = NONE` mutes SQLCipher's stderr HMAC trace on // wrong-password attempts; it must follow `PRAGMA key`, which has to - // be the first statement on the connection. - conn.execute_batch(&format!( - "PRAGMA key = \"x'{}'\"; PRAGMA cipher_log_level = NONE;", - key.to_hex() - )) - .map_err(|e| format!("Failed to apply encryption key: {}", e))?; + // be the first statement to touch the file. + let guard = crate::lock_ok(&CIPHER_DEFAULTS); + let applied = conn.execute_batch(&format!( + "{} PRAGMA key = \"x'{}'\"; PRAGMA cipher_log_level = NONE; \ + PRAGMA cipher_page_size = {};", + profile.hmac.default_pragmas(), + key.to_hex(), + profile.page_size + )); + drop(guard); + applied.map_err(|e| format!("Failed to apply encryption key: {}", e))?; + } else { + conn.execute_batch(&format!("PRAGMA page_size = {};", profile.page_size)) + .map_err(|e| format!("Failed to apply page size: {}", e))?; } match conn.query_row("SELECT count(*) FROM sqlite_master", [], |r| { r.get::<_, i64>(0) @@ -402,13 +569,16 @@ fn db_matches_current(conn: &Connection, tokenizer: &str) -> Result, + profile: Profile, ) -> Result { drop(conn); // Before the delete, even if the removal fails partway. @@ -438,18 +608,30 @@ fn wipe_and_reopen( let conn = Connection::open(path) .map_err(|e| format!("Failed to reopen database after rebuild: {}", e))?; crate::platform::restrict_to_owner(path); - key_and_probe(&conn, &path.to_string_lossy(), key)?; + key_and_probe(&conn, &path.to_string_lossy(), key, profile)?; conn.execute_batch(PRAGMAS_FAST) .map_err(|e| format!("Failed to apply pragmas after rebuild: {}", e))?; Ok(conn) } -fn apply_current_schema(conn: &Connection, tokenizer: &str) -> Result<(), String> { +fn apply_current_schema( + conn: &Connection, + tokenizer: &str, + key: Option<&IndexKey>, + profile: Profile, +) -> Result<(), String> { conn.execute_batch(SCHEMA_CURRENT) .map_err(|e| format!("Failed to create current schema tables: {}", e))?; let fts = fts_create_sql(tokenizer); conn.execute_batch(&fts) .map_err(|e| format!("Failed to create searchabletext: {}", e))?; + // Only here: FTS5's leaf size has to suit the page size and the reserve + // this file was built with. Deciding it once at creation is sound because + // the profile is fixed for the file's life — toggling password protection + // always wipes and rebuilds, so the stored geometry cannot outlive its key + // state. + fts_set_pgsz(conn, profile, key.is_some()) + .map_err(|e| format!("Failed to set searchabletext pgsz: {}", e))?; let now = crate::log::now_unix(); let effective = effective_tokenizer(tokenizer); diff --git a/crates/quicksearch-core/src/db/open_tests.rs b/crates/quicksearch-core/src/db/open_tests.rs index 4fd03a7..e004daa 100644 --- a/crates/quicksearch-core/src/db/open_tests.rs +++ b/crates/quicksearch-core/src/db/open_tests.rs @@ -457,9 +457,17 @@ fn schema_mismatch_under_key_wipes_and_recreates_encrypted() { let p = tmp_db_path(); let key = test_key(0xa1); { + // Built through `key_and_probe` under an explicit previous profile, + // not with a bare `PRAGMA key`. SQLCipher's `cipher_default_*` + // settings are process globals that `key_and_probe` writes, so a bare + // key here would inherit whatever another test in this binary last + // installed and could land the fixture under a layout + // `PROFILES_PREVIOUS` does not list — which reads as a wrong password. + // Naming the layout is also what the fixture means: this is a file + // from an older build. + let previous = crate::db::schema::PROFILES_PREVIOUS[0]; let conn = Connection::open(&p).unwrap(); - conn.execute_batch(&format!("PRAGMA key = \"x'{}'\";", key.to_hex())) - .unwrap(); + key_and_probe(&conn, p.to_str().unwrap(), Some(&key), previous).unwrap(); conn.execute( "CREATE TABLE schema_info (key TEXT PRIMARY KEY, value TEXT NOT NULL)", [], @@ -572,7 +580,6 @@ fn a_fresh_index_and_its_sidecars_are_owner_only() { fn maintain_reads_its_pragmas_on_a_keyed_index() { let p = tmp_db_path(); let key = test_key(0xc3); - let dir = p.parent().unwrap().to_string_lossy().into_owned(); { let conn = open_or_recreate_keyed(p.to_str().unwrap(), "trigram", Some(&key)).unwrap(); conn.execute( @@ -585,7 +592,7 @@ fn maintain_reads_its_pragmas_on_a_keyed_index() { .unwrap(); // What matters is an answer, not an error; a tiny index has no slack. assert_eq!( - crate::db::repo::maintain(&conn, &dir), + crate::db::repo::maintain(&conn, p.to_str().unwrap()), Ok(false), "maintain must not fail on a keyed index" ); @@ -600,6 +607,197 @@ fn maintain_reads_its_pragmas_on_a_keyed_index() { std::fs::remove_file(&p).ok(); } +/// Write enough text that FTS5 emits several *full* leaves. A corpus of tiny +/// documents fits in one part-filled leaf, never reaches the inline-payload +/// limit, and would let the tests below pass on a broken geometry. +fn seed_searchable_text(conn: &Connection) { + let words = crate::testutil::WORDS; + conn.execute_batch("BEGIN").unwrap(); + for doc in 0..200usize { + let body: Vec<&str> = (0..150) + .map(|w| words[(doc * 31 + w * 7) % words.len()]) + .collect(); + conn.execute( + "INSERT INTO searchabletext(rowid, text) VALUES (?1, ?2)", + params![doc as i64 + 1, body.join(" ")], + ) + .unwrap(); + } + conn.execute_batch("COMMIT").unwrap(); +} + +/// `(leaf, overflow)` page counts for the FTS5 data table. +fn fts_page_types(conn: &Connection) -> (i64, i64) { + let count = |pagetype: &str| -> i64 { + conn.query_row( + "SELECT COUNT(*) FROM dbstat \ + WHERE name = 'searchabletext_data' AND pagetype = ?1", + params![pagetype], + |r| r.get(0), + ) + .unwrap() + }; + (count("leaf"), count("overflow")) +} + +fn fts_stored_pgsz(conn: &Connection) -> Option { + conn.query_row( + "SELECT v FROM searchabletext_config WHERE k = 'pgsz'", + [], + |r| r.get(0), + ) + .optional() + .unwrap() +} + +/// The inline payload limit for the shipped profile: `page − reserve − 35`, +/// with SQLCipher's page reserve when keyed. +fn max_inline(keyed: bool) -> i64 { + use crate::db::schema::PROFILE; + PROFILE.page_size - PROFILE.reserve(keyed) - 35 +} + +/// Assert one index's FTS5 leaves are inline. An overflowed leaf is a second +/// page fetch and decrypt on every read of it, and it is silent — nothing but +/// the page counts shows it. +fn assert_leaves_inline(conn: &Connection, keyed: bool) { + let (leaf, overflow) = fts_page_types(conn); + assert!( + leaf > 20, + "the corpus must fill real leaves for this to test anything: {} leaves", + leaf + ); + assert_eq!( + overflow, + 0, + "{} of {} FTS5 leaves spilled to overflow pages — the derived pgsz is \ + back above the {}-byte inline limit", + overflow, + leaf, + max_inline(keyed) + ); + // The limit itself, in case `dbstat` is ever unavailable or lies. + let widest: i64 = conn + .query_row( + "SELECT MAX(LENGTH(block)) FROM searchabletext_data", + [], + |r| r.get(0), + ) + .unwrap(); + assert!( + widest <= max_inline(keyed), + "a {}-byte record cannot sit inline under a {}-byte limit", + widest, + max_inline(keyed) + ); +} + +/// SQLCipher reserves bytes of every page for its IV and any authenticator, so +/// a keyed index's inline payload limit is that much lower than a plain one's +/// at the same page size. FTS5's own default record size ignores that and, +/// before `fts_pgsz_for`, sent **every** full keyed leaf to an overflow page — +/// 12% more file, and a second decrypt per leaf read. +#[test] +fn keyed_fts_leaves_stay_inline() { + let p = tmp_db_path(); + let key = test_key(0xd4); + let conn = open_or_recreate_keyed(p.to_str().unwrap(), "trigram", Some(&key)).unwrap(); + assert_eq!( + fts_stored_pgsz(&conn), + Some(crate::db::schema::fts_pgsz_for( + crate::db::schema::PROFILE, + true + )), + "a keyed index must pin pgsz at creation" + ); + + seed_searchable_text(&conn); + assert_leaves_inline(&conn, true); + + drop(conn); + std::fs::remove_file(&p).ok(); +} + +/// The derivation is only worth anything if it clears the limit it is derived +/// from, at every size `benches/page_geometry.rs` sweeps. Arithmetic only — +/// the round trip through a real file is the test below. +#[test] +fn derived_fts_pgsz_fits_inline_at_every_swept_page_size() { + use crate::db::schema::{fts_pgsz_for, HmacMode, Profile}; + for page_size in [1024i64, 2048, 4096, 8192, 16384, 32768, 65536] { + // Every HMAC mode, not just the shipped one: the reserve moves with it + // and the derivation has to clear the limit under all of them, or + // `benches/cipher_hmac.rs` would be measuring overflow rather than + // authentication. + for hmac in [HmacMode::Off, HmacMode::Sha256, HmacMode::Sha512] { + let profile = Profile { page_size, hmac }; + for keyed in [false, true] { + // What a table leaf holds inline, and what FTS5 actually writes. + let max_inline = page_size - profile.reserve(keyed) - 35; + let widest_record = fts_pgsz_for(profile, keyed) + 2; + assert!( + widest_record <= max_inline, + "page {} hmac={:?} keyed={}: a {}-byte record does not fit in {}", + page_size, + hmac, + keyed, + widest_record, + max_inline + ); + // A pgsz so small the leaves stop holding useful runs would be + // a different bug, and a silent one. + assert!( + widest_record * 2 > max_inline, + "page {} hmac={:?} keyed={}: {} wastes over half of a {}-byte leaf", + page_size, + hmac, + keyed, + widest_record, + max_inline + ); + } + } + } + // Spelled out so a change to the formula has to be deliberate. The keyed + // value is quoted per mode, because that is the number the on-disk format + // depends on. + let at = |page_size, hmac| Profile { page_size, hmac }; + assert_eq!(fts_pgsz_for(at(4096, HmacMode::Sha512), true), 3970); + assert_eq!(fts_pgsz_for(at(4096, HmacMode::Sha256), true), 4002); + assert_eq!(fts_pgsz_for(at(4096, HmacMode::Off), true), 4034); + assert_eq!( + fts_pgsz_for(at(4096, HmacMode::Sha512), false), + 4050, + "FTS5's own default; a plain file has no reserve, so the mode is moot" + ); +} + +/// The mirror. A plain index gets a *larger* record than a keyed one at the +/// same page size, because it has no reserve to give up — the two differ by +/// exactly that, and both have to land inline. +#[test] +fn a_plain_index_gets_the_derived_pgsz_for_its_page_size() { + use crate::db::schema::{fts_pgsz_for, PROFILE}; + let p = tmp_db_path(); + let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + assert_eq!( + fts_stored_pgsz(&conn), + Some(fts_pgsz_for(PROFILE, false)), + "an unencrypted index gets the derivation for its page size" + ); + assert_eq!( + fts_pgsz_for(PROFILE, false) - fts_pgsz_for(PROFILE, true), + PROFILE.hmac.reserve(), + "the plain and keyed records differ by exactly the reserve" + ); + + seed_searchable_text(&conn); + assert_leaves_inline(&conn, false); + + drop(conn); + std::fs::remove_file(&p).ok(); +} + /// A typo naming another application's database used to delete it on the /// next indexing run, because "no `schema_info`" read as "an old index of /// ours". diff --git a/crates/quicksearch-core/src/db/repo.rs b/crates/quicksearch-core/src/db/repo.rs index c1bf510..d05dd88 100644 --- a/crates/quicksearch-core/src/db/repo.rs +++ b/crates/quicksearch-core/src/db/repo.rs @@ -255,20 +255,52 @@ const ZSTD_LEVEL: i32 = 3; /// Reusable compression context for the `documents_text` sidecar; one per /// batch. Measured (`benches/index.rs`, group `zstd_encode`). -pub struct DocEncoder(zstd::bulk::Compressor<'static>); +pub struct DocEncoder { + ctx: zstd::bulk::Compressor<'static>, + /// One row's compressed output, reused. **`zstd`'s `WriteBuf` for `Vec` + /// writes from offset 0 and sets the length** — it overwrites rather + /// than appends — so a body cannot be compressed straight into a shared + /// arena. It lands here and is copied across, which still costs no + /// allocation once both buffers have grown. + row: Vec, +} impl DocEncoder { pub fn new() -> Result { - zstd::bulk::Compressor::new(ZSTD_LEVEL) - .map(DocEncoder) - .map_err(|e| format!("zstd encoder: {}", e)) + Ok(DocEncoder { + ctx: zstd::bulk::Compressor::new(ZSTD_LEVEL) + .map_err(|e| format!("zstd encoder: {}", e))?, + row: Vec::new(), + }) } pub fn encode(&mut self, text: &str) -> Result, String> { - self.0 + self.ctx .compress(text.as_bytes()) .map_err(|e| format!("zstd encode: {}", e)) } + + /// Append the compressed form of `text` to `arena`, returning where it + /// landed. The mirror of [`DocDecoder`]'s reused buffer on the write + /// side: a whole batch's bodies share one allocation instead of taking + /// one `Vec` each, which at a chunk per commit was an allocation per + /// indexed document. + pub fn encode_into( + &mut self, + text: &str, + arena: &mut Vec, + ) -> Result, String> { + self.row.clear(); + // `compress_to_buffer` writes into the buffer's capacity and fails + // rather than growing it, so the room has to be there first. + self.row.reserve(zstd::zstd_safe::compress_bound(text.len())); + self.ctx + .compress_to_buffer(text.as_bytes(), &mut self.row) + .map_err(|e| format!("zstd encode: {}", e))?; + let start = arena.len(); + arena.extend_from_slice(&self.row); + Ok(start..arena.len()) + } } /// Compress one body, for the writers that handle a single row. @@ -447,17 +479,57 @@ pub fn dir_rows( Ok(out) } +/// A stored path that remembers where its `name` begins — the whole +/// `parent + name` in **one** buffer rather than the two or three strings +/// carrying both halves would cost. The content pass moves one of these per +/// file from the feeder to the writer, so the saving is per indexed file. +#[derive(Debug)] +pub struct RowPath { + path: String, + name_at: usize, +} + +impl RowPath { + /// Join the two halves the index stores into one buffer. The pending-page + /// query builds its own in place; this is for everyone assembling a row + /// from parts they already hold. + pub fn new(parent: &str, name: &str) -> RowPath { + let mut path = String::with_capacity(parent.len() + name.len()); + path.push_str(parent); + let name_at = path.len(); + path.push_str(name); + RowPath { path, name_at } + } + + pub fn as_str(&self) -> &str { + &self.path + } + + /// The `files.name` half. + pub fn name(&self) -> &str { + // `name_at` is the parent's length, taken as the buffer was built; a + // stored parent is by construction a prefix of the path. + &self.path[self.name_at..] + } +} + +/// One row the content pass has yet to extract. +#[derive(Debug)] +pub struct PendingRow { + pub file_id: i64, + pub path: RowPath, + pub mime: Option, +} + /// One page of rows still awaiting content extraction under `cursor`'s range, -/// ordered by id, as `(id, name, path, mime)` tuples. Keyset paging: a row is -/// served exactly once even though the writer is concurrently flipping -/// `content_state` behind the reader. -#[allow(clippy::type_complexity)] +/// ordered by id. Keyset paging: a row is served exactly once even though the +/// writer is concurrently flipping `content_state` behind the reader. pub fn pending_content_page( conn: &Connection, cursor: &crate::file_handling::ExtractCursor, max_size: i64, limit: i64, -) -> Result)>, String> { +) -> Result, String> { let mut stmt = conn .prepare_cached( // `INDEXED BY`: left to itself the planner takes @@ -476,15 +548,16 @@ pub fn pending_content_page( .query_map( params![max_size, cursor.last_id, cursor.lo, cursor.hi, limit], |row| { - let parent: String = row.get(1)?; - let name: String = row.get(2)?; - let path = format!("{}{}", parent, name); - Ok(( - row.get::<_, i64>(0)?, - name, - path, - row.get::<_, Option>(3)?, - )) + // The parent is grown into the path in place rather than + // `format!`ed with the name into a third buffer. + let mut path: String = row.get(1)?; + let name_at = path.len(); + path.push_str(row.get_ref(2)?.as_str()?); + Ok(PendingRow { + file_id: row.get(0)?, + path: RowPath { path, name_at }, + mime: row.get(3)?, + }) }, ) .map_err(|e| format!("query pending content: {}", e))?; @@ -757,19 +830,56 @@ pub(super) fn pragma_number(conn: &Connection, pragma: &str) -> Result Option { + let mut stmt = conn + .prepare("SELECT stat FROM sqlite_stat1 WHERE tbl = 'files'") + .ok()?; + let rows = stmt + .query_map([], |r| r.get::<_, String>(0)) + .ok()? + .filter_map(Result::ok) + .filter_map(|stat| stat.split_whitespace().next()?.parse::().ok()); + rows.max() +} + /// Checkpoint → VACUUM → `PRAGMA optimize` → checkpoint — the trailing /// checkpoint matters because VACUUM's copy-back and `optimize` refill the /// log. Returns whether it vacuumed. /// /// Run on a connection from [`crate::db::open::open_maintenance`], never the -/// indexer's. `db_dir` is where the temporary database goes and must be the -/// index's own directory — default temp resolution can land on a RAM-backed +/// indexer's. VACUUM's temporary database goes in the index's own directory, +/// taken from `db_path` — default temp resolution can land on a RAM-backed /// `/tmp`. Peak transient space is roughly three times the index. -pub fn maintain(conn: &Connection, db_dir: &str) -> Result { +/// +/// The path rather than the directory, so the caller cannot pass one that is +/// not the index's, and so the readings below can see the log. +pub fn maintain(conn: &Connection, db_path: &str) -> Result { + let db_dir = std::path::Path::new(db_path) + .parent() + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default(); + let db_dir = db_dir.as_str(); + + note_log_on_entry(db_dir, wal_bytes(db_path)); + #[cfg(feature = "probe")] + let probe = MaintainProbe::start(db_path); + // Best-effort: compaction does not need the log empty to start. if let Err(e) = checkpoint_truncate(conn) { crate::log_warn!("{}", e); } + #[cfg(feature = "probe")] + probe.step("maintain checkpoint"); let page_count = pragma_number(conn, "page_count")?; let freelist = pragma_number(conn, "freelist_count")?; @@ -806,15 +916,54 @@ pub fn maintain(conn: &Connection, db_dir: &str) -> Result { let _ = conn.execute_batch("PRAGMA temp_store_directory = '';"); outcome?; } + #[cfg(feature = "probe")] + probe.step(if vacuumed { "VACUUM" } else { "VACUUM (skipped)" }); conn.execute_batch("PRAGMA optimize;") .map_err(|e| format!("optimize: {}", e))?; note_optimized(db_dir); + #[cfg(feature = "probe")] + probe.step("optimize"); checkpoint_truncate(conn)?; + #[cfg(feature = "probe")] + probe.step("maintain checkpoint"); Ok(vacuumed) } +/// Per-step log size and elapsed time through [`maintain`], for `probe` builds. +/// +/// The pass runs on its own connection after the indexer's has gone, so its +/// cost is invisible from the run's own instrumentation — and VACUUM's +/// copy-back is the single largest thing that writes to the log in a whole +/// run. Matches `indexing::pipeline`'s `tail` lines, which cover the half +/// before this one. +#[cfg(feature = "probe")] +struct MaintainProbe { + db_path: String, + started: std::time::Instant, +} + +#[cfg(feature = "probe")] +impl MaintainProbe { + fn start(db_path: &str) -> MaintainProbe { + MaintainProbe { + db_path: db_path.to_string(), + started: std::time::Instant::now(), + } + } + + fn step(&self, what: &str) { + crate::log_info!( + "tail t={:.1}s wal {} after {}", + self.started.elapsed().as_secs_f64(), + crate::testutil::mib(wal_bytes(&self.db_path)), + what + ); + } + +} + /// `PRAGMA optimize` acceptances, per index directory — per directory so /// concurrent tests against separate scratch indexes cannot satisfy each /// other's assertions. @@ -834,6 +983,34 @@ pub fn optimize_count(db_dir: &str) -> u64 { crate::lock_ok(&OPTIMIZED).get(db_dir).copied().unwrap_or(0) } +/// Log bytes [`maintain`] was last handed, per index directory. +static LOG_ON_ENTRY: std::sync::LazyLock< + std::sync::Mutex>, +> = std::sync::LazyLock::new(|| std::sync::Mutex::new(std::collections::HashMap::new())); + +fn note_log_on_entry(db_dir: &str, bytes: u64) { + crate::lock_ok(&LOG_ON_ENTRY).insert(db_dir.to_string(), bytes); +} + +/// How large the log was when [`maintain`] last started on the index in +/// `db_dir`; `None` if it has not run there in this process. +/// +/// A latch in the same shape as [`optimize_count`], and for the same reason: +/// the value is gone by the time a test could sample it. It exists to pin the +/// invariant the indexer's tail checkpoints establish — `maintain` is handed +/// an *empty* log, so its VACUUM's copy-back is the only thing in it rather +/// than a second layer over a whole run's writing. +pub fn log_on_entry_to_maintain(db_dir: &str) -> Option { + crate::lock_ok(&LOG_ON_ENTRY).get(db_dir).copied() +} + +/// The `-wal` beside `db_path`, in bytes; 0 when there is none. +fn wal_bytes(db_path: &str) -> u64 { + std::fs::metadata(format!("{}-wal", db_path)) + .map(|m| m.len()) + .unwrap_or(0) +} + fn get_info(conn: &Connection, key: &str) -> Option { conn.query_row( "SELECT value FROM schema_info WHERE key = ?1", diff --git a/crates/quicksearch-core/src/db/repo_tests.rs b/crates/quicksearch-core/src/db/repo_tests.rs index 3e75a90..09c6a60 100644 --- a/crates/quicksearch-core/src/db/repo_tests.rs +++ b/crates/quicksearch-core/src/db/repo_tests.rs @@ -108,6 +108,63 @@ fn a_compressed_body_carries_its_uncompressed_length() { } } +/// Several rows into one arena, which is the shape the writer uses. +/// +/// The trap this pins: `zstd`'s `WriteBuf` for `Vec` writes from **offset +/// zero** and sets the length, so compressing straight into a shared arena +/// silently overwrites the previous row and leaves every returned range +/// pointing past the end. Each body must come back byte-identical to what +/// the one-shot encoder produces, and out of its own range. +#[test] +fn an_arena_keeps_every_row_it_is_given() { + let mut enc = DocEncoder::new().unwrap(); + let bodies = [ + "the first document", + "", + "a considerably longer second document ".repeat(512).as_str(), + "third", + ] + .map(str::to_string); + + let mut arena = Vec::new(); + let mut ranges = Vec::new(); + for text in &bodies { + ranges.push(enc.encode_into(text, &mut arena).unwrap()); + } + + for (text, at) in bodies.iter().zip(&ranges) { + let blob = &arena[at.clone()]; + assert_eq!( + raw_text_len(blob), + Some(text.len() as u64), + "a row's frame does not describe its own body" + ); + assert_eq!( + DocDecoder::new().unwrap().decode(blob), + Some(text.as_str()), + "a row did not survive sharing the arena" + ); + } + + // The ranges tile the arena in order and account for all of it: a gap or + // an overlap means one row landed on another. + let mut next = 0; + for at in &ranges { + assert_eq!(at.start, next, "rows must be contiguous"); + next = at.end; + } + assert_eq!(next, arena.len(), "the arena holds exactly the four bodies"); + + // Reuse: a second batch must not read the first one's bytes. + arena.clear(); + let at = enc.encode_into("a fresh batch", &mut arena).unwrap(); + assert_eq!(at.start, 0); + assert_eq!( + DocDecoder::new().unwrap().decode(&arena[at]), + Some("a fresh batch") + ); +} + #[test] fn insert_update_delete_round_trip() { let (_dir, p) = tmp_path(); @@ -812,6 +869,80 @@ fn checkpoint_truncate_reports_an_incomplete_checkpoint() { assert!(wal_bytes(&p) > 0, "and the log is still there"); } +/// The floor under the case above. A TRUNCATE that cannot take the reset lock +/// leaves the file at its high-water mark, and without `journal_size_limit` +/// that mark is where it stays — one bad run leaves a multi-gigabyte log +/// behind for every later reader to page around. +/// +/// The limit is not a checkpoint: `sqlite3WalFrames` applies it at the **first +/// commit after the log restarts**, which is the next write once a checkpoint +/// has copied every frame out. So the space comes back on its own, from +/// whichever writer touches the index next, with no successful TRUNCATE +/// anywhere in the story. That is the property worth having — the run that +/// bloated the log is exactly the one whose checkpoint is most likely to lose +/// its lock race. +/// +/// The writing pragma profiles carry it; `db::schema`'s +/// `every_writing_profile_bounds_the_log` is what keeps them in step with +/// [`crate::config::MINIMUM_WAL_SIZE`], and this is what shows it works. +#[test] +fn journal_size_limit_gives_the_space_back_after_a_blocked_truncate() { + let (_dir, p) = tmp_path(); + let mut writer = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + seed_rows(&mut writer, 0..10); + writer + .busy_timeout(std::time::Duration::from_millis(100)) + .unwrap(); + + let peak = { + // Held across the seeding, which is what lets the log grow past the + // limit at all: pinned to an early frame, no checkpoint of any kind + // can reset it. This is the run's own shape — a reader per root, live + // from start to finish. + let reader = crate::db::open_existing(p.to_str().unwrap(), false).unwrap(); + let mut stmt = reader.prepare("SELECT id FROM files").unwrap(); + let mut rows = stmt.query([]).unwrap(); + rows.next().unwrap().expect("a row to hold the snapshot on"); + + // 6k rows measured 12.5 MiB of log; this clears 16 MiB with room. + seed_rows(&mut writer, 10..10_000); + let peak = wal_bytes(&p); + assert!( + peak > crate::config::MINIMUM_WAL_SIZE, + "the fixture left {} bytes of log, under the limit it must exceed", + peak + ); + + checkpoint_truncate(&writer).expect_err("a reader holds the log open"); + assert_eq!(wal_bytes(&p), peak, "a blocked TRUNCATE trims nothing"); + peak + }; + + // The reader is gone, so an ordinary PASSIVE checkpoint copies every + // frame out — but on its own it trims nothing, because the log has not + // restarted yet. + writer + .execute_batch("PRAGMA wal_checkpoint(PASSIVE);") + .unwrap(); + assert_eq!( + wal_bytes(&p), + peak, + "a checkpoint that does not restart the log cannot trim it" + ); + + // The next write restarts it, and *that* commit honours the limit. No + // TRUNCATE was ever accepted. + seed_rows(&mut writer, 10_000..10_001); + let after = wal_bytes(&p); + assert!( + after <= crate::config::MINIMUM_WAL_SIZE, + "the log went {} -> {} bytes against a {} byte limit", + peak, + after, + crate::config::MINIMUM_WAL_SIZE + ); +} + /// Autocheckpoint tries the reset lock exactly once, with no retry, so a /// reader querying back to back keeps the log growing for the whole run. An /// explicit checkpoint retries the same lock under `busy_timeout` and gets it. @@ -896,6 +1027,44 @@ fn a_busy_reader_defeats_the_autocheckpoint_but_not_a_forced_one() { ); } +/// Sizing the search cache reads the row count from `sqlite_stat1` rather than +/// counting, so it has to survive the two states that table is really in. +#[test] +fn the_analyzed_file_count_ignores_the_partial_index_and_missing_stats() { + let (_dir, p) = tmp_path(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + assert_eq!( + analyzed_file_count(&conn), + None, + "a never-analyzed index must say so, not report zero files" + ); + + seed_rows(&mut conn, 0..2000); + // All 2000 rows land content_state = 0, so `idx_files_content_pending` + // covers every one of them; mark most done to make the partial index + // genuinely smaller than the table, which is the trap being tested. + conn.execute("UPDATE files SET content_state = 1 WHERE id % 100 != 0", []) + .unwrap(); + conn.execute_batch("ANALYZE;").unwrap(); + + let pending: i64 = conn + .query_row( + "SELECT COUNT(*) FROM files WHERE content_state = 0", + [], + |r| r.get(0), + ) + .unwrap(); + assert!( + pending < 2000, + "the partial index must be smaller than the table for this to test anything" + ); + assert_eq!( + analyzed_file_count(&conn), + Some(2000), + "the partial index's smaller count must not win" + ); +} + #[test] fn maintain_vacuums_when_slack_is_significant() { let (_dir, p) = tmp_path(); @@ -919,9 +1088,8 @@ fn maintain_vacuums_when_slack_is_significant() { .unwrap(); assert!(freelist > 0, "the deletions should have freed pages"); - let dir = p.parent().unwrap().to_string_lossy().into_owned(); assert!( - maintain(&conn, &dir).unwrap(), + maintain(&conn, p.to_str().unwrap()).unwrap(), "that much slack is worth a vacuum" ); assert_eq!( @@ -955,9 +1123,8 @@ fn maintain_skips_vacuum_on_a_tight_file() { drop(conn); let conn = crate::db::open::open_maintenance(p.to_str().unwrap()).unwrap(); - let dir = p.parent().unwrap().to_string_lossy().into_owned(); assert!( - !maintain(&conn, &dir).unwrap(), + !maintain(&conn, p.to_str().unwrap()).unwrap(), "a file with no slack is not worth rewriting" ); // The checkpoint is not conditional on the vacuum, though. diff --git a/crates/quicksearch-core/src/db/schema.rs b/crates/quicksearch-core/src/db/schema.rs index 04b8bfb..971501a 100644 --- a/crates/quicksearch-core/src/db/schema.rs +++ b/crates/quicksearch-core/src/db/schema.rs @@ -17,6 +17,23 @@ //! //! `PRAGMA mmap_size` is absent from all of them: it would make memory //! behaviour differ between protected and unprotected installs. +//! +//! `PRAGMA journal_size_limit` is on the three *writing* profiles and on none +//! of the readers, which cannot set it. Without it the `-wal` keeps its +//! high-water mark on disk after every checkpoint that is not a successful +//! TRUNCATE — and a TRUNCATE silently downgrades whenever it cannot take the +//! reset lock (see [`crate::db::repo::checkpoint_truncate`]), which is most +//! likely in exactly the case that grew the log. One bad run then leaves a +//! multi-gigabyte file for every later reader to page around. +//! +//! The limit is a backstop, not a second checkpoint: SQLite applies it at the +//! first commit *after* the log restarts, so the space comes back from +//! whichever writer touches the index next, with no successful TRUNCATE +//! anywhere in the story. `repo::journal_size_limit_gives_the_space_back_after_a_blocked_truncate` +//! is the demonstration. It is spelled out in each profile rather than shared, +//! because a pragma string cannot interpolate a constant; +//! `every_writing_profile_bounds_the_log` is what keeps the three in step with +//! [`crate::config::MINIMUM_WAL_SIZE`]. /// The bulk indexer's write connection: one per run, dies with it. /// `synchronous = NORMAL` under WAL risks only the last commit on power loss @@ -28,6 +45,7 @@ pub const PRAGMAS_FAST: &str = " PRAGMA cache_size = -8192; PRAGMA temp_store = MEMORY; PRAGMA foreign_keys = ON; + PRAGMA journal_size_limit = 16777216; "; /// [`PRAGMAS_FAST`] but with `temp_store = FILE`: SQLCipher is compiled @@ -41,6 +59,7 @@ pub const PRAGMAS_MAINTENANCE: &str = " PRAGMA cache_size = -8192; PRAGMA temp_store = FILE; PRAGMA foreign_keys = ON; + PRAGMA journal_size_limit = 16777216; "; /// The coordinator's long-lived write connection — whatever its cache @@ -52,18 +71,106 @@ pub const PRAGMAS_INCREMENTAL: &str = " PRAGMA cache_size = -4096; PRAGMA temp_store = MEMORY; PRAGMA foreign_keys = ON; + PRAGMA journal_size_limit = 16777216; "; -/// The search worker's connection, held across a typing session. The one -/// deliberately large profile: SQLCipher caches pages *decrypted*, so on an -/// encrypted index an undersized cache re-pays AES-CBC + HMAC per 4 KiB and -/// warm queries run ~2.5× slower (`benches/search_perf.rs` sweeps it). -pub const PRAGMAS_SEARCH: &str = " - PRAGMA busy_timeout = 5000; - PRAGMA cache_size = -32768; - PRAGMA temp_store = MEMORY; - PRAGMA foreign_keys = ON; -"; +/// Cache ceiling for an **unencrypted** index, whatever its size. +/// +/// `benches/search_perf.rs` swept 1 MiB → 256 MiB against corpora from 200k to +/// 1M files and found no knee at all on a plain index: warm search was flat +/// within 10% across the whole range, because a page-cache miss there is a +/// `memcpy` from the OS cache. Only SQLCipher makes a miss expensive — it +/// caches pages *decrypted*, so a miss re-pays the AES-CBC. +/// Spending more than this on a plain index buys nothing measurable. +pub const SEARCH_CACHE_PLAIN_MIB: i64 = 16; + +/// Floor and ceiling for the **derived** value. The floor is the smallest the +/// sweep ever found sufficient. +/// +/// The cap is a deliberate limit on resident memory, and it does bind: it is +/// reached at about 800k files, and beyond that an encrypted index sits under +/// its knee. Measured at 1M files, whose `files` table is 132 MiB: 205 ms per +/// keystroke at 128 MiB against 58 ms at 256. That is the trade — a third of a +/// gigabyte held for the life of a search session, or a 3.5x slower one — and +/// it is the user's to make, which is what [`SEARCH_CACHE_OVERRIDE_MAX_MIB`] +/// is for. +pub const SEARCH_CACHE_MIN_MIB: i64 = 16; +pub const SEARCH_CACHE_MAX_MIB: i64 = 128; + +/// Ceiling on an **explicit** `[search] cache_size_mib`, well above the +/// automatic cap: capping a manual override at the automatic limit would deny +/// it in exactly the case that needs it, a large encrypted index whose knee is +/// past 128 MiB. +pub const SEARCH_CACHE_OVERRIDE_MAX_MIB: i64 = 1024; + +/// Lowering the override ceiling to the automatic one would quietly re-cap the +/// escape hatch; this refuses to compile instead. +const _: () = assert!(SEARCH_CACHE_OVERRIDE_MAX_MIB > SEARCH_CACHE_MAX_MIB); +const _: () = assert!(SEARCH_CACHE_MIN_MIB <= SEARCH_CACHE_MAX_MIB); + +/// Cache bytes to allow per indexed file, from `benches/search_perf.rs`. +/// +/// What every keystroke rescans is the `files` table — `search/cascade/passes` +/// answers filename queries with `WHERE f.name LIKE '%…%'`, a full table scan +/// with no FTS in it, and the fuzzy pass rescans with `WHERE 1=1`. So the +/// working set is that table, its size is linear in row count, and the knee in +/// the warm curve sits at the first ceiling that holds it. Below the knee an +/// encrypted index re-decrypts the table on every keystroke and runs 2.4–2.7x +/// slower; at it, keyed and plain are within 2% of each other. +/// +/// 139 bytes per row, times the 1.21x the sweep found sufficient. Both halves +/// were measured, and the product lands on the observed knee exactly: +/// +/// | corpus | `files` table | knee | ratio | this formula | +/// |---|---|---|---|---| +/// | 200k | 26.5 MiB | 32 MiB | 1.21x | 32 MiB | +/// | 600k | 79.5 MiB | 96 MiB | 1.21x | 96 MiB | +/// +/// The step is not subtle — keyed at 600k ran 121–130 ms at every ceiling up +/// to 64 MiB and 33.7 ms at 96. +/// +/// **Calibrated while [`HMAC_MODE`] was still HMAC-SHA512, and not re-swept +/// since.** A cache miss is now materially cheaper — `benches/cipher_hmac.rs` +/// measured warm search 1.78x faster with the authenticator gone — so the +/// knee this reaches for is shallower than it was, and the recommendation is +/// therefore *conservative*: it asks for at least as much cache as it needs, +/// never less. Erring that way is safe for latency and costs only resident +/// memory. Re-running `benches/search_perf.rs` would likely let both the +/// bytes-per-file figure and [`SEARCH_CACHE_MAX_MIB`] come down; until someone +/// does, do not quote the 2.4–2.7x above as current. +/// +/// **139 B/row assumes ordinary path lengths.** `parent` is stored per row, so +/// a tree far deeper than the measured one (`/seed/NNN/` plus five segments) +/// has wider rows and wants more; the narrow shape the older harnesses seeded +/// measured 69.5 B/row, half of this, and calibrating against it would have +/// under-sized every index by two. That is what `[search] cache_size_mib` +/// overrides, and why the GUI shows the recommendation beside it rather than +/// hiding the arithmetic. +pub const SEARCH_CACHE_BYTES_PER_FILE: i64 = 168; + +/// The cache ceiling this index wants, in MiB. +pub fn recommended_search_cache_mib(files: i64, keyed: bool) -> i64 { + if !keyed { + return SEARCH_CACHE_PLAIN_MIB; + } + let want = files.max(0).saturating_mul(SEARCH_CACHE_BYTES_PER_FILE) / (1024 * 1024); + want.clamp(SEARCH_CACHE_MIN_MIB, SEARCH_CACHE_MAX_MIB) +} + +/// The search worker's connection, held across a typing session, at an +/// explicit ceiling — see [`recommended_search_cache_mib`] for why it is not a +/// constant. The one deliberately large profile. +pub fn pragmas_search(cache_mib: i64) -> String { + format!( + "PRAGMA busy_timeout = 5000; + PRAGMA cache_size = -{}; + PRAGMA temp_store = MEMORY; + PRAGMA foreign_keys = ON;", + // The *override* ceiling: a caller may legitimately ask for more than + // the automatic cap, and only nonsense is refused here. + cache_mib.clamp(SEARCH_CACHE_MIN_MIB, SEARCH_CACHE_OVERRIDE_MAX_MIB) * 1024 + ) +} /// The *one-shot* readers. Pragmas safe on a read-only connection, where /// `journal_mode` and `synchronous` can't be changed on the file. @@ -166,6 +273,291 @@ CREATE TABLE config_validation ( ); "#; +/// The database page size, applied at creation and fixed for the file's life. +/// +/// 8192, not SQLite's and SQLCipher's default of 4096, because +/// `benches/page_geometry.rs` swept 1024→65536 on a keyed 200k-file index and +/// this is where the two opposing costs balance: +/// +/// | page | size | index rows/s | `files` scan | scattered rows | cold name | +/// |---|---|---|---|---|---| +/// | 1024 | 179.9 MiB | 1194 | 221 ms | 51 ms | 79.4 ms | +/// | 4096 | 154.8 MiB | 3289 | 100 ms | 43 ms | 44.4 ms | +/// | **8192** | **151.4 MiB** | **4955** | **55 ms** | **45 ms** | **35.7 ms** | +/// | 16384 | 151.4 MiB | 5592 | 37 ms | 58 ms | 34.2 ms | +/// | 65536 | 158.4 MiB | 7712 | 23 ms | 61 ms | 38.1 ms | +/// +/// The table above was swept while [`HMAC_MODE`] was still SQLCipher's +/// HMAC-SHA512, so the keyed columns overstate today's per-page cost. The +/// *shape* of the trade is unchanged — both arms of it are per-page work, so +/// removing the authenticator scales them together rather than moving the +/// balance — and the plain arm, which never had an HMAC, picked 8192 too. +/// +/// A scattered row fetch decrypts a whole page to read one ~130-byte row out +/// of it, so it wants a *small* page and degrades past 8192 (43 ms → 73 ms by +/// 32768). The `files` scan behind every filename query is sequential and +/// wants a *large* one, improving all the way — because the bytes decrypted +/// stay the same (~9 MiB either way) while the number of per-page codec and +/// pager operations falls. 8192 keeps scattered reads within +/// 3% of their optimum, halves the scan, is tied for the smallest file, and +/// indexes 1.5x faster than 4096. +/// +/// **Changing this is not a free edit** — see [`PROFILES_PREVIOUS`]. +pub const PAGE_SIZE: i64 = 8192; + +/// Which authenticator SQLCipher applies to every page — and so how much of +/// every page it spends on one. +/// +/// **This is a deliberately weakened setting; see [`HMAC_MODE`].** The +/// encryption itself is not a choice: SQLCipher 4 dropped `PRAGMA cipher` and +/// the provider hard-codes AES-256-CBC, so the HMAC is the only lever the +/// build has. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HmacMode { + /// No per-page authenticator. The reserve is the IV alone. + Off, + Sha256, + /// SQLCipher's own default. + Sha512, +} + +impl HmacMode { + /// Bytes taken off the end of every page: a 16-byte IV plus the digest, + /// rounded up to the 16-byte AES block. Both digests here are already a + /// multiple of it, so nothing rounds. + pub const fn reserve(self) -> i64 { + match self { + HmacMode::Off => 16, + HmacMode::Sha256 => 16 + 32, + HmacMode::Sha512 => 16 + 64, + } + } + + /// The pragmas selecting it, to be run **before** `PRAGMA key`. + /// + /// These are SQLCipher's process-*default* settings, not the + /// per-connection `cipher_use_hmac` / `cipher_hmac_algorithm`, and the + /// difference is not stylistic: the per-connection forms cannot lower a + /// reserve that `PRAGMA key` has already installed. See + /// `db::open::key_and_probe`, which owns the lock they need. + /// + /// Both statements are emitted for every mode, so the globals are fully + /// specified whatever the previous open left behind. + pub const fn default_pragmas(self) -> &'static str { + match self { + HmacMode::Off => { + "PRAGMA cipher_default_use_hmac = OFF; \ + PRAGMA cipher_default_hmac_algorithm = HMAC_SHA512;" + } + HmacMode::Sha256 => { + "PRAGMA cipher_default_use_hmac = ON; \ + PRAGMA cipher_default_hmac_algorithm = HMAC_SHA256;" + } + HmacMode::Sha512 => { + "PRAGMA cipher_default_use_hmac = ON; \ + PRAGMA cipher_default_hmac_algorithm = HMAC_SHA512;" + } + } + } + + /// For logs, and for the "Show database key" dialog, which has to tell + /// another SQLCipher tool what to expect. + pub const fn label(self) -> &'static str { + match self { + HmacMode::Off => "off", + HmacMode::Sha256 => "HMAC_SHA256", + HmacMode::Sha512 => "HMAC_SHA512", + } + } +} + +/// The per-page authenticator this build writes: **none**. +/// +/// # Why it is safe to drop +/// +/// A threat-model argument, not a benchmark alone. The index holds text read +/// out of files the same user can already read, so anything positioned to +/// *tamper* with the index could read the originals instead — what the HMAC +/// defends is nearly empty, and it is paid on every page read and every page +/// write. Nor does dropping it cost integrity relative to the product's own +/// baseline: an unprotected index has no per-page authentication either, and +/// never has, so this makes the two behave alike rather than putting the +/// protected one behind. Structural damage is still caught by SQLite's own +/// page-header and cell checks, in both key states, and the index is +/// re-derivable from disk regardless. +/// +/// The encryption is untouched. SQLCipher 4 removed `PRAGMA cipher` and the +/// provider hard-codes AES-256-CBC, so pages are as confidential as before; +/// only the authenticator is gone. +/// +/// # What it buys +/// +/// `benches/cipher_hmac.rs`, 200k files, all four arms in one process. Warm +/// total is six cascade shapes summed — the steady state of a typing session: +/// +/// | arm | seed | warm total | duplicates | size | +/// |---|---|---|---|---| +/// | plain (no password) | 30.9 s | 125.6 ms | 206 ms | 151.5 MiB | +/// | **keyed, HMAC off** | **37.7 s** | **146.2 ms** | **221 ms** | **151.5 MiB** | +/// | keyed, HMAC_SHA256 | 40.0 s | 209.0 ms | 250 ms | 150.6 MiB | +/// | keyed, HMAC_SHA512 | 41.3 s | 259.5 ms | 262 ms | 151.4 MiB | +/// +/// Against SQLCipher's default that is **1.78x on warm search**, and it takes +/// encrypted-over-plain from 2.07x to 1.16x — 84% of the penalty for having a +/// password at all. Per shape it is widest where it matters most: a filename +/// query went 32.6 ms → 13.8 ms and a rare body term 39.8 → 18.6. +/// +/// **SHA-256 is the arm to understand, because it is the one that +/// disappoints.** It halves the digest and gives back 32 bytes of every page, +/// yet recovers only 1.24x of the 2.07x. The reason is that the digest is not +/// where the money goes: `sqlcipher_openssl_hmac` calls +/// `EVP_MAC_fetch(NULL, "HMAC", NULL)`, `EVP_MAC_CTX_new` and an +/// `EVP_MAC_init` that fetches the digest *by name* — two OpenSSL 3 provider +/// lookups per page, paid whichever digest is chosen. Only `Off` removes them, +/// which is why the middle ground is worth so much less than it looks. +/// +/// Size is unmoved either way (151.5 against 151.4 MiB): `fts_pgsz_for` +/// re-derives the record size from the new reserve, so the 64 bytes handed +/// back per page go into leaves rather than into the file. +/// +/// # Changing it is not a free edit +/// +/// The reserve is part of the on-disk format, so a file written under another +/// mode does not decrypt at all. See [`PROFILES_PREVIOUS`]. +pub const HMAC_MODE: HmacMode = HmacMode::Off; + +/// A page size and an HMAC mode: everything about a keyed file's layout that +/// has to be known *before* it can be read, because its header is ciphertext +/// until SQLCipher has been told both. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Profile { + pub page_size: i64, + pub hmac: HmacMode, +} + +impl Profile { + /// Bytes reserved on every page. Zero unencrypted: a plain file has + /// neither an IV nor an authenticator. + pub const fn reserve(self, keyed: bool) -> i64 { + if keyed { + self.hmac.reserve() + } else { + 0 + } + } +} + +impl std::fmt::Display for Profile { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}-byte pages, HMAC {}", + self.page_size, + self.hmac.label() + ) + } +} + +/// The layout this build creates files at. +pub const PROFILE: Profile = Profile { + page_size: PAGE_SIZE, + hmac: HMAC_MODE, +}; + +/// Layouts earlier versions created files at, newest first. +/// +/// A keyed file under another profile does not decrypt *at all*: the header +/// comes back as noise, and without this list `open::key_and_probe` would +/// report it as `KEY_MISMATCH: wrong-password`. No schema-version bump can +/// rescue that, because the version cannot be read either. So `db::open` +/// reopens under each of these before declaring a mismatch, and a file that +/// answers to one is treated as ordinary schema drift: wiped and rebuilt at +/// [`PROFILE`]. +/// +/// Unencrypted files never needed it — `PRAGMA page_size` against an existing +/// file is silently ignored and they have no reserve to get wrong, so they +/// open at whatever they were built with and the version check does the rest — +/// but they take the same path for free. +/// +/// Anything appended here is a layout some user's index is still sitting at. +/// Entries can only be dropped when it is acceptable for those indexes to read +/// as a wrong password. +pub const PROFILES_PREVIOUS: &[Profile] = &[ + // Every protected index in the field is here: the shipped page size, under + // SQLCipher's own authenticator, before [`HMAC_MODE`] became `Off`. It is + // listed first because it is overwhelmingly the common case, and the probe + // stops at the first profile that answers. + Profile { + page_size: PAGE_SIZE, + hmac: HmacMode::Sha512, + }, + Profile { + page_size: 4096, + hmac: HmacMode::Sha512, + }, +]; + +/// A profile may not be listed as previous *and* current — the probe would +/// then retry the layout it just failed on, and `ProfileMatch::Previous` would +/// mean nothing. +const _: () = { + let mut i = 0; + while i < PROFILES_PREVIOUS.len() { + assert!( + !(PROFILES_PREVIOUS[i].page_size == PROFILE.page_size + && PROFILES_PREVIOUS[i].hmac as u8 == PROFILE.hmac as u8), + "PROFILES_PREVIOUS repeats the current profile" + ); + i += 1; + } +}; + +/// FTS5's `pgsz` for one profile, applied once at creation and then persisted +/// in `searchabletext_config`. +/// +/// The arithmetic, in one line: a table leaf holds +/// `page − reserve − 35` bytes inline, and an FTS5 record runs to `pgsz + 2`, +/// so `pgsz ≤ page − reserve − 37`. `MARGIN` keeps a record that overruns by a +/// byte or two from falling off the cliff. +/// +/// The cliff is worth stating because it is expensive and silent. FTS5's own +/// default is 4050 — a number SQLite chose so a full leaf fits inline in a +/// *plain* 4096-byte page. Keyed under SQLCipher's own HMAC-SHA512 the reserve +/// drops the limit to 3981 and that default misses it by 71 bytes, sending +/// **every** full leaf to an overflow page: a second fetch and decrypt on +/// every read of it. On the 60k-file corpus in `tests/encrypted_perf.rs` that +/// was 10954 of 12354 leaves and 70.7 MiB against the plain index's 65.2; +/// fitting them inline brought it to 65.5. +/// +/// It cuts the other way too: at a page size of 8192 a 4050-byte record leaves +/// half of every page empty, because a second one will not fit. So this is +/// derived from the profile rather than pinned — and it has to follow +/// [`HmacMode`] as well as the page size, because the reserve moves with both. +pub fn fts_pgsz_for(profile: Profile, keyed: bool) -> i64 { + /// Slack under the inline limit, in bytes. + const MARGIN: i64 = 9; + profile.page_size - profile.reserve(keyed) - 37 - MARGIN +} + +/// Set [`fts_pgsz_for`] on a freshly created `searchabletext`. Written +/// unconditionally, including when it lands on FTS5's own default: the value +/// is the same either way, and one code path is worth more than a `pgsz` row +/// saved. +pub fn fts_set_pgsz( + conn: &rusqlite::Connection, + profile: Profile, + keyed: bool, +) -> rusqlite::Result<()> { + conn.execute( + "INSERT INTO searchabletext(searchabletext, rank) VALUES('pgsz', ?1)", + [fts_pgsz_for(profile, keyed)], + ) + .map(|_| ()) +} + +/// FTS5's own default `pgsz`, from `FTS5_DEFAULT_PAGE_SIZE` in the amalgamation. +pub const FTS5_DEFAULT_PGSZ: i64 = 4050; + /// FTS5 virtual table DDL. Separate because the tokenizer is config-driven. /// /// *Contentless* FTS5 (`content=''`): column values are not stored. @@ -205,6 +597,124 @@ pub fn effective_tokenizer(tokenizer: &str) -> String { mod tests { use super::*; + /// Every profile that can write has to bound the log, and all three have to + /// bound it at the same figure — which is [`crate::config::MINIMUM_WAL_SIZE`], + /// spelled as a literal in each because a pragma string cannot interpolate. + /// The read-only profiles must *not* carry it: `journal_size_limit` is a + /// write to the file, and a read-only connection cannot make one. + #[test] + fn every_writing_profile_bounds_the_log() { + let want = format!( + "PRAGMA journal_size_limit = {};", + crate::config::MINIMUM_WAL_SIZE + ); + for (name, pragmas) in [ + ("PRAGMAS_FAST", PRAGMAS_FAST), + ("PRAGMAS_MAINTENANCE", PRAGMAS_MAINTENANCE), + ("PRAGMAS_INCREMENTAL", PRAGMAS_INCREMENTAL), + ] { + assert!(pragmas.contains(&want), "{} is missing {}", name, want); + } + for (name, pragmas) in [ + ("PRAGMAS_READONLY", PRAGMAS_READONLY.to_string()), + ("PRAGMAS_WALK_READER", PRAGMAS_WALK_READER.to_string()), + ("PRAGMAS_SEARCH", pragmas_search(32)), + ] { + assert!( + !pragmas.contains("journal_size_limit"), + "{} is read-only and cannot set journal_size_limit", + name + ); + } + } + + /// The sweep found no knee on a plain index at any corpus size, so this + /// must not grow with one — the whole reason it is a separate constant. + #[test] + fn a_plain_index_gets_the_same_ceiling_at_every_size() { + for files in [0, 1_000, 200_000, 1_000_000, 50_000_000] { + assert_eq!( + recommended_search_cache_mib(files, false), + SEARCH_CACHE_PLAIN_MIB, + "{} files", + files + ); + } + } + + #[test] + fn a_keyed_index_is_clamped_at_both_ends() { + assert_eq!(recommended_search_cache_mib(0, true), SEARCH_CACHE_MIN_MIB); + assert_eq!( + recommended_search_cache_mib(-1, true), + SEARCH_CACHE_MIN_MIB, + "a negative count is nonsense, not a reason to panic" + ); + assert_eq!( + recommended_search_cache_mib(i64::MAX, true), + SEARCH_CACHE_MAX_MIB, + "and an absurd one must not overflow into a small ceiling" + ); + } + + /// Between the clamps it has to actually track the corpus; a constant + /// would satisfy every other test here. + #[test] + fn a_keyed_index_grows_with_the_corpus_between_the_clamps() { + let small = recommended_search_cache_mib(300_000, true); + let large = recommended_search_cache_mib(700_000, true); + assert!( + small < large, + "300k wants {} MiB and 700k wants {} MiB", + small, + large + ); + assert!(large <= SEARCH_CACHE_MAX_MIB); + } + + /// The three corpora `benches/search_perf.rs` measured, against the knee + /// it found for each. Under the knee is the 2.4-4x regime this exists to + /// avoid, so the recommendation has to reach it. + #[test] + fn the_recommendation_clears_every_measured_knee() { + for (files, knee_mib) in [(200_000, 32), (600_000, 96), (1_000_000, 128)] { + let got = recommended_search_cache_mib(files, true); + assert!( + got >= knee_mib.min(SEARCH_CACHE_MAX_MIB), + "{} files: recommending {} MiB, under the measured knee of {} MiB", + files, + got, + knee_mib + ); + } + } + + #[test] + fn pragmas_search_writes_a_kib_ceiling_and_clamps_it() { + assert!(pragmas_search(64).contains("cache_size = -65536")); + assert!( + pragmas_search(0).contains(&format!("-{}", SEARCH_CACHE_MIN_MIB * 1024)), + "a zero must not reach SQLite, where it means its own default" + ); + assert!( + pragmas_search(999_999).contains(&format!("-{}", SEARCH_CACHE_OVERRIDE_MAX_MIB * 1024)), + "nor must an absurd one" + ); + } + + /// An explicit setting has to be able to exceed the automatic cap: past + /// ~800k files the derived value is capped *below* the measured knee, and + /// the override is the only way to reach it. + #[test] + fn an_explicit_ceiling_may_exceed_the_automatic_cap() { + let asked = SEARCH_CACHE_MAX_MIB * 2; + assert!( + pragmas_search(asked).contains(&format!("-{}", asked * 1024)), + "{} MiB was asked for and must be applied verbatim", + asked + ); + } + #[test] fn plain_trigram_gets_accent_stripping() { assert_eq!( diff --git a/crates/quicksearch-core/src/extract/audio.rs b/crates/quicksearch-core/src/extract/audio.rs index ae803fa..3baebb8 100644 --- a/crates/quicksearch-core/src/extract/audio.rs +++ b/crates/quicksearch-core/src/extract/audio.rs @@ -9,7 +9,7 @@ use lofty::{ tag::{Accessor, ItemKey}, }; -use super::{ExtractError, Extractor}; +use super::{ExtractError, Extractor, Scratch}; pub struct AudioExtractor; @@ -18,43 +18,53 @@ impl Extractor for AudioExtractor { mime.starts_with("audio/") } - fn extract(&self, path: &Path) -> Result { + fn extract( + &self, + path: &Path, + out: &mut String, + _scratch: &mut Scratch, + ) -> Result<(), ExtractError> { let tagged = Probe::open(path) .map_err(|e| format!("lofty probe {}: {}", path.display(), e))? .read() .map_err(|e| format!("lofty read {}: {}", path.display(), e))?; - let mut pieces: Vec = Vec::new(); + // Straight into the caller's buffer: the fields are short and few, + // and a `Vec` then `join` allocated every piece twice over. if let Some(tag) = tagged.primary_tag().or_else(|| tagged.first_tag()) { - let mut push = |value: Option| { - if let Some(v) = value.filter(|v: &String| !v.is_empty()) { - pieces.push(v); + // The `Accessor` shortcuts hand back a `Cow`, so their fallbacks + // are bound here rather than inside an `or_else` that would let + // the temporary die before it is read. + let (title, artist, album) = (tag.title(), tag.artist(), tag.album()); + let mut push = |value: Option<&str>| { + if let Some(v) = value.filter(|v: &&str| !v.is_empty()) { + if !out.is_empty() { + out.push(' '); + } + out.push_str(v); } }; // A tag can carry a value under `ItemKey` or the `Accessor` shortcut. push( tag.get_string(&ItemKey::TrackTitle) .filter(|v| !v.is_empty()) - .map(str::to_string) - .or_else(|| tag.title().map(|t| t.to_string())), + .or(title.as_deref()), ); push( tag.get_string(&ItemKey::TrackArtist) .filter(|v| !v.is_empty()) - .map(str::to_string) - .or_else(|| tag.artist().map(|a| a.to_string())), + .or(artist.as_deref()), ); push( tag.get_string(&ItemKey::AlbumTitle) .filter(|v| !v.is_empty()) - .map(str::to_string) - .or_else(|| tag.album().map(|a| a.to_string())), + .or(album.as_deref()), ); - push(tag.get_string(&ItemKey::Genre).map(str::to_string)); - push(tag.get_string(&ItemKey::Comment).map(str::to_string)); + push(tag.get_string(&ItemKey::Genre)); + push(tag.get_string(&ItemKey::Comment)); } - Ok(pieces.join(" ")) + Ok(()) } } @@ -62,6 +72,13 @@ impl Extractor for AudioExtractor { mod tests { use super::*; + /// The one-file form: these assert on text, not on buffer reuse. + fn extract(path: &std::path::Path) -> Result { + let mut out = String::new(); + let mut scratch = Scratch::new(&crate::config::Config::default()); + AudioExtractor.extract(path, &mut out, &mut scratch).map(|()| out) + } + /// An ID3v2.3 tag carrying `frames`, then silent MPEG frames so the probe /// recognizes the format from content. fn write_mp3(tag: &str, frames: &[(&str, &str)]) -> std::path::PathBuf { @@ -112,7 +129,7 @@ mod tests { ("TCON", "Synthpop"), ], ); - let out = AudioExtractor.extract(&path).expect("extract"); + let out = extract(&path).expect("extract"); for expected in ["Blue Monday", "New Order", "Power Corruption", "Synthpop"] { assert!( out.contains(expected), @@ -127,7 +144,7 @@ mod tests { #[test] fn an_untagged_file_yields_empty_text() { let path = write_mp3("audio-untagged", &[]); - let out = AudioExtractor.extract(&path).expect("extract"); + let out = extract(&path).expect("extract"); assert!(out.is_empty(), "unexpected text {:?}", out); } diff --git a/crates/quicksearch-core/src/extract/mod.rs b/crates/quicksearch-core/src/extract/mod.rs index 4e63290..7142a51 100644 --- a/crates/quicksearch-core/src/extract/mod.rs +++ b/crates/quicksearch-core/src/extract/mod.rs @@ -18,6 +18,134 @@ pub mod rtf; /// human-readable messages. pub type ExtractError = String; +/// The ceilings one extraction works under, read from the config once per +/// worker instead of being hardcoded per format. +/// +/// Every extractor used to carry its own 64 MiB constants, chosen +/// independently of the settings that actually bound the work: the content +/// pass never offers a file above `maximum_text_file_size` (2 MiB by +/// default), and everything an extractor produces past `maximum_text_size` +/// (256 KiB) is discarded by the caller moments later. A ceiling 256× above +/// the largest result that can be kept is not a safety margin, it is the +/// worst case a worker can reach — multiplied by the pool size. +#[derive(Debug, Clone, Copy)] +pub struct Limits { + /// Largest file body to read. A backstop rather than a policy: the pass + /// already filtered on it, so this catches a file that grew since the + /// walk sized it and a node whose `fstat` lies (procfs reports zero). + pub read: usize, + /// Largest text an extraction may produce. Past this the caller + /// truncates, so producing more is work and memory spent to be thrown + /// away — extractors stop here instead. + pub text: usize, + /// Largest a single container member may inflate to. A zip declares its + /// sizes but the deflate stream is what gets read, so this is the only + /// bound on a crafted archive. + pub inflate: usize, +} + +/// Headroom between the text kept and the markup carrying it. A document +/// whose *extracted text* is `maximum_text_size` arrives as several times +/// that in XML — runs, properties and namespaces — but not sixteen times, +/// and the early stop at [`Limits::text`] means the whole budget is reached +/// only by an archive built to reach it. +const INFLATE_FACTOR: usize = 16; + +/// Absolute ceiling on the inflation budget, whatever `maximum_text_size` is +/// set to. The budget is held **per worker** and the pools multiply it (up +/// to 64 workers, one pool per root), so it cannot be allowed to scale with +/// a setting freely. This is the old hardcoded per-member cap, kept as the +/// backstop it was always meant to be rather than the everyday value. +const MAX_INFLATE: usize = 64 * 1024 * 1024; + +impl Limits { + pub fn for_config(config: &crate::config::Config) -> Limits { + let text = config.processing.maximum_text_size.max(1); + Limits { + read: usize::try_from(config.processing.maximum_text_file_size).unwrap_or(usize::MAX), + text, + inflate: text.saturating_mul(INFLATE_FACTOR).min(MAX_INFLATE), + } + } +} + +/// Buffers one worker reuses for every file it handles. +/// +/// A pool member creates one of these before its loop and hands it to each +/// extraction; the buffers keep whatever capacity the largest file so far +/// needed, so per-file allocator traffic for the *intermediates* falls to +/// zero. What it deliberately does not hold is the extracted text: that +/// crosses a channel to the writer, so it is the payload rather than +/// scratch. +/// +/// Sizing is lazy on purpose. `maximum_text_file_size` clamps at 4 GiB +/// ([`crate::config::Config::clamp_out_of_range`]), so reserving it eagerly +/// would let a configured ceiling nobody reaches allocate per worker. +pub struct Scratch { + /// The head bytes a walk worker hashes and sniffs each file from. + head: Vec, + /// One container member or one whole file, for a parser that will not + /// take a reader. Grows to the largest member the worker has met and + /// stays there, bounded by [`Limits::inflate`]. + bytes: Vec, + /// quick-xml's per-event buffer. Separate from `bytes` because a + /// container walk holds both at once. + events: Vec, + /// XLSX's shared-string table. Cleared between files; the entries keep + /// their capacity, which is most of what a table costs. + strings: Vec, + limits: Limits, +} + +impl Scratch { + pub fn new(config: &crate::config::Config) -> Scratch { + Scratch { + head: Vec::new(), + bytes: Vec::new(), + events: Vec::new(), + strings: Vec::new(), + limits: Limits::for_config(config), + } + } + + pub fn limits(&self) -> Limits { + self.limits + } + + /// The head buffer, to be read into. Reused across files: the walk + /// hashes the head of every new or changed file, and a fresh + /// `hash_length` buffer apiece was one allocation per file. The caller + /// sizes it — [`crate::file_handling::get_file_hash`] does. + pub(crate) fn head_buffer(&mut self) -> &mut Vec { + &mut self.head + } + + pub(crate) fn head(&self) -> &[u8] { + &self.head + } + + /// The raw-bytes buffer: one container member, one OLE stream, or one + /// whole file for a format whose parser will not take a reader. The + /// caller clears it before filling. + pub(crate) fn bytes_mut(&mut self) -> &mut Vec { + &mut self.bytes + } + + /// The member and event buffers together, which a container walk holds + /// at once. Two `&mut self` accessors could not be. + pub(crate) fn container_bufs(&mut self) -> (&mut Vec, &mut Vec) { + (&mut self.bytes, &mut self.events) + } + + /// The same pair plus the shared-string table — XLSX needs all three. + /// The table is cleared but its entries keep their capacity, which is + /// most of what a shared-string table costs to rebuild. + pub(crate) fn xlsx_bufs(&mut self) -> (&mut Vec, &mut Vec, &mut Vec) { + self.strings.clear(); + (&mut self.bytes, &mut self.events, &mut self.strings) + } +} + /// Run `f`, turning a panic into an [`ExtractError`] naming the file. The /// extractors drive third-party parsers over bytes chosen by whoever wrote /// the file, and several are documented to panic on malformed input. @@ -27,14 +155,29 @@ fn contain_panic(path: &Path, f: impl FnOnce() -> T) -> Result bool; - /// Extracted text for the FTS5 `text` column. An [`ExtractError`] marks - /// the file's content state failed (so it is not retried every run). - /// Empty text is fine — filename search still works. - fn extract(&self, path: &Path) -> Result; + /// Extract this file's searchable text into `out`. An [`ExtractError`] + /// marks the file's content state failed (so it is not retried every + /// run). Empty text is fine — filename search still works. + /// + /// An extractor should stop once `out` reaches `scratch.limits().text`: + /// the caller truncates there, so anything beyond is produced to be + /// discarded. + fn extract( + &self, + path: &Path, + out: &mut String, + scratch: &mut Scratch, + ) -> Result<(), ExtractError>; /// Extract from bytes that are the file's *entire* contents, already in /// memory at walk time; keeps the text consistent with the size, mtime @@ -43,11 +186,15 @@ pub trait Extractor: Send + Sync { /// The default `None` means "I need the file on disk" — formats that seek /// or read a trailer must keep it; `Some(Err(_))` is a real failure. /// `path` is only so failures name the file — nothing here may open it. + /// + /// No `Scratch`: `head` is itself the walk worker's reused buffer, so an + /// extractor taking both would be holding two borrows of the same thing. fn extract_from_head( &self, _path: &Path, _head: &[u8], - ) -> Option> { + _out: &mut String, + ) -> Option> { None } } @@ -64,11 +211,15 @@ impl Registry { } } + /// The lowercasing is a stack copy, not a heap one: `find` runs two or + /// three times per indexed file (`content_extractable` alone calls it + /// twice), and a `String` apiece was pure per-file allocator traffic. fn find(&self, mime: &str) -> Option<&dyn Extractor> { - let lower = mime.to_ascii_lowercase(); + let lower = crate::mime::LowerMime::new(mime); + let lower = lower.as_str(mime); self.extractors .iter() - .find(|e| e.supports(&lower)) + .find(|e| e.supports(lower)) .map(|e| &**e) } @@ -78,8 +229,9 @@ impl Registry { self.find(mime).is_some() } - /// Run the handler for `mime` against `path`; `Ok(None)` if no extractor - /// claims the MIME. + /// Run the handler for `mime` against `path`, appending its text to + /// `out`. `Ok(false)` means no extractor claims the MIME and `out` was + /// not touched. /// /// A panicking parser becomes an `Err` here, at the boundary: every /// caller has more than one file to lose (a walk worker's panic costs the @@ -88,13 +240,19 @@ impl Registry { /// caller cannot forget. This cannot help with a stack overflow, which /// aborts rather than unwinding — see `vendor/pdf-extract`, which bounds /// the recursion that made that reachable. - pub fn extract(&self, path: &Path, mime: &str) -> Result, ExtractError> { + pub fn extract( + &self, + path: &Path, + mime: &str, + out: &mut String, + scratch: &mut Scratch, + ) -> Result { let Some(extractor) = self.find(mime) else { - return Ok(None); + return Ok(false); }; - contain_panic(path, || extractor.extract(path)) + contain_panic(path, || extractor.extract(path, out, scratch)) .and_then(|r| r) - .map(Some) + .map(|()| true) } /// [`Registry::extract`] for a file whose complete contents the caller @@ -105,17 +263,55 @@ impl Registry { path: &Path, mime: &str, head: &[u8], - ) -> Option> { + out: &mut String, + ) -> Option> { let extractor = self.find(mime)?; // The guard wraps the whole `Option` so a panic becomes // `Some(Err(..))` — a failure this file is charged with, not a // deferral to the content pass that would meet the same panic. - match contain_panic(path, || extractor.extract_from_head(path, head)) { + match contain_panic(path, || extractor.extract_from_head(path, head, out)) { Ok(outcome) => outcome, Err(e) => Some(Err(e)), } } + /// [`Registry::extract`] into a `String` of its own, under default + /// limits. + /// + /// For callers holding **one** file — probes, tests, a CLI invocation — + /// where there is no loop for a reused buffer to amortize over. Anything + /// in a pool should own a [`Scratch`] and call [`Registry::extract`], or + /// it pays the per-file allocations this exists to avoid. + pub fn extract_to_string( + &self, + path: &Path, + mime: &str, + config: &crate::config::Config, + ) -> Result, ExtractError> { + let mut out = String::new(); + let mut scratch = Scratch::new(config); + match self.extract(path, mime, &mut out, &mut scratch)? { + true => Ok(Some(out)), + false => Ok(None), + } + } + + /// [`Registry::extract_complete_head`] into a `String` of its own; see + /// [`Registry::extract_to_string`] for when to reach for it. + pub fn extract_head_to_string( + &self, + path: &Path, + mime: &str, + head: &[u8], + ) -> Option> { + let mut out = String::new(); + match self.extract_complete_head(path, mime, head, &mut out) { + Some(Ok(())) => Some(Ok(out)), + Some(Err(e)) => Some(Err(e)), + None => None, + } + } + /// The default set. Order matters: RTF precedes plaintext, which claims /// every `text/*` and would swallow `text/rtf` as raw control words; /// plaintext precedes audio because it deliberately claims playlist and @@ -145,12 +341,28 @@ impl Default for Registry { mod tests { use super::*; + fn cfg() -> crate::config::Config { + crate::config::Config::default() + } + + /// The one-file form; the pool's buffer reuse is not what these assert. + fn extract(r: &Registry, path: &Path, mime: &str) -> Result, ExtractError> { + r.extract_to_string(path, mime, &cfg()) + } + + fn extract_complete_head( + r: &Registry, + path: &Path, + mime: &str, + head: &[u8], + ) -> Option> { + r.extract_head_to_string(path, mime, head) + } + #[test] fn empty_registry_returns_none() { let r = Registry::new(); - let out = r - .extract(Path::new("/tmp/x"), "text/plain") - .expect("no error"); + let out = extract(&r, Path::new("/tmp/x"), "text/plain").expect("no error"); assert!(out.is_none()); } @@ -159,20 +371,14 @@ mod tests { let r = Registry::default_set(); let p = Path::new("/tmp/whatever"); - let out = r.extract_complete_head(p, "text/plain", b"hello"); + let out = extract_complete_head(&r, p, "text/plain", b"hello"); assert!(matches!(out, Some(Ok(ref c)) if c == "hello")); // A format that seeks or reads a trailer must not be handed a buffer. - assert!(r - .extract_complete_head(p, "application/pdf", b"%PDF-1.4") - .is_none()); - assert!(r - .extract_complete_head(p, "image/png", b"\x89PNG") - .is_none()); + assert!(extract_complete_head(&r, p, "application/pdf", b"%PDF-1.4").is_none()); + assert!(extract_complete_head(&r, p, "image/png", b"\x89PNG").is_none()); - assert!(r - .extract_complete_head(p, "application/x-nonesuch", b"..") - .is_none()); + assert!(extract_complete_head(&r, p, "application/x-nonesuch", b"..").is_none()); } #[test] @@ -187,14 +393,14 @@ mod tests { "application/x-sql", ] { assert!( - r.extract_complete_head(p, mime, b"x").is_some(), + extract_complete_head(&r, p, mime, b"x").is_some(), "{} should extract from a head", mime ); } for mime in ["application/rtf", "text/rtf"] { assert!( - r.extract_complete_head(p, mime, br"{\rtf1 x}").is_some(), + extract_complete_head(&r, p, mime, br"{\rtf1 x}").is_some(), "{} should extract from a head", mime ); @@ -205,8 +411,7 @@ mod tests { fn text_rtf_reaches_the_rtf_extractor_not_plaintext() { let r = Registry::default_set(); let p = Path::new("/tmp/whatever.rtf"); - let out = r - .extract_complete_head(p, "text/rtf", br"{\rtf1\ansi Hello {\b World}}") + let out = extract_complete_head(&r, p, "text/rtf", br"{\rtf1\ansi Hello {\b World}}") .expect("claimed") .expect("parsed"); assert_eq!(out, "Hello World"); @@ -236,7 +441,7 @@ mod tests { "application/octet-stream", "", ] { - let claimed = !matches!(r.extract(missing, mime), Ok(None)); + let claimed = !matches!(extract(&r, missing, mime), Ok(None)); assert_eq!( r.supports(mime), claimed, diff --git a/crates/quicksearch-core/src/extract/office.rs b/crates/quicksearch-core/src/extract/office.rs index 7d397ea..8e1c323 100644 --- a/crates/quicksearch-core/src/extract/office.rs +++ b/crates/quicksearch-core/src/extract/office.rs @@ -7,14 +7,14 @@ use std::error::Error; use std::fs::File; -use std::io::{BufReader, Read, Seek}; +use std::io::{BufRead, BufReader, Cursor, Read, Seek}; use std::path::Path; use quick_xml::events::Event; use quick_xml::Reader; use zip::ZipArchive; -use super::{ExtractError, Extractor}; +use super::{ExtractError, Extractor, Scratch}; pub struct OfficeExtractor; @@ -72,43 +72,72 @@ const ODF_SHEET: TextSpec = TextSpec { separator: Some(' '), }; -/// The text an `&entity;` or `Ӓ` reference stands for. quick-xml 0.41 -/// reports a reference as its own event, so a reader that ignores it silently -/// drops every `&` from the document. Only the five predefined entities -/// and numeric references are resolvable without a DTD. -fn entity_text(raw: &str) -> Option { +/// Append what an `&entity;` or `Ӓ` reference stands for to `out`, +/// reporting whether it resolved. quick-xml 0.41 reports a reference as its +/// own event, so a reader that ignores it silently drops every `&` from +/// the document. Only the five predefined entities and numeric references are +/// resolvable without a DTD. +/// +/// Pushed rather than returned: a document is mostly `&`s and `’`s, +/// and a `String` per reference was an allocation per *character* of output. +#[must_use] +fn push_entity_text(raw: &str, out: &mut String) -> bool { if let Some(digits) = raw.strip_prefix('#') { let code = match digits.strip_prefix(['x', 'X']) { - Some(hex) => u32::from_str_radix(hex, 16).ok()?, - None => digits.parse::().ok()?, + Some(hex) => u32::from_str_radix(hex, 16).ok(), + None => digits.parse::().ok(), + }; + let Some(c) = code.and_then(char::from_u32) else { + return false; }; - let c = char::from_u32(code)?; // `char::from_u32` accepts more than XML's character production does: - // `�` would put a literal NUL into an FTS5 column. `None` becomes + // `�` would put a literal NUL into an FTS5 column. `false` becomes // the same visible "unknown entity" error an unexpandable name gets. - let legal = !c.is_control() || matches!(c, '\t' | '\n' | '\r'); - return legal.then(|| String::from(c)); + if c.is_control() && !matches!(c, '\t' | '\n' | '\r') { + return false; + } + out.push(c); + return true; + } + match quick_xml::escape::resolve_predefined_entity(raw) { + Some(text) => { + out.push_str(text); + true + } + None => false, } - quick_xml::escape::resolve_predefined_entity(raw).map(String::from) } /// Append the text `spec` selects out of `xml` to `out`. Text-bearing /// elements are counted, not flagged: ODF nests them, and a flag made a /// span's close end the run, dropping everything up to the paragraph's /// close. The separator belongs after a *run* — several events since 0.41. -fn collect_xml_text(xml: &str, spec: &TextSpec, out: &mut String) -> Result<(), Box> { - let mut reader = Reader::from_str(xml); +/// +/// Reads from a stream and stops at `limit`: the member is never held whole, +/// and a document with more text than the caller will keep is abandoned at +/// the point the surplus begins rather than parsed to the end and truncated. +fn collect_xml_text( + xml: R, + spec: &TextSpec, + out: &mut String, + limit: usize, + buf: &mut Vec, +) -> Result<(), Box> { + let mut reader = Reader::from_reader(xml); // No `trim_text`: it trims each *event*, and since 0.41 an entity // reference splits the character data into separate events — `Jack & // Jill` would come back as `Jack&Jill`. Whitespace inside a text-bearing // element is content; between elements it is ignored anyway. - let mut buf = Vec::new(); + buf.clear(); // Open text-bearing elements; the run ends at zero, not on the innermost // close. let mut depth = 0usize; loop { - match reader.read_event_into(&mut buf) { + if out.len() >= limit { + return Ok(()); + } + match reader.read_event_into(buf) { Ok(Event::Start(ref e)) => { if spec.text.contains(&e.name().as_ref()) { depth += 1; @@ -122,9 +151,9 @@ fn collect_xml_text(xml: &str, spec: &TextSpec, out: &mut String) -> Result<(), let raw = e.decode()?; // An unexpandable entity is an error: dropping it takes // characters out of the indexed text silently. - let text = entity_text(&raw) - .ok_or_else(|| format!("Error parsing XML: unknown entity &{};", raw))?; - out.push_str(&text); + if !push_entity_text(&raw, out) { + return Err(format!("Error parsing XML: unknown entity &{};", raw).into()); + } } Ok(Event::End(ref e)) => { let name = e.name(); @@ -172,41 +201,38 @@ fn open_container(path: &Path) -> Result> { Ok(ZipArchive::new(BufReader::new(File::open(path)?))?) } -/// Cap on one decompressed member, mirroring `ole::MAX_TEXT_BYTES`: the zip -/// header declares sizes, but the deflate stream is what we actually read, so -/// a tiny archive can inflate without bound. -const MAX_XML_BYTES: usize = 64 * 1024 * 1024; - -/// Cap on the text taken from one *container*. [`MAX_XML_BYTES`] bounds each -/// member on its own, and a small archive can carry dozens that each inflate -/// to that cap: without a running total the peak is members × 64 MiB per -/// worker, and an allocation failure aborts rather than unwinding. -const MAX_TEXT_BYTES: usize = 64 * 1024 * 1024; - -/// One member's bytes as a string. An over-cap member keeps its prefix. -fn member_text( +/// Inflate one member into `buf`, which is the **worker's** buffer, reused +/// member after member and file after file: after the first document a +/// container costs no allocation for its members at all. +/// +/// A zip declares its sizes but the deflate stream is what actually gets +/// read, so `limit` — [`super::Limits::inflate`], derived from the config — +/// is the only real bound on what a crafted archive can expand to. It used +/// to be a hardcoded 64 MiB per member *and* another 64 MiB per container. +/// +/// The buffer keeps whatever capacity the largest member so far needed and +/// does not shrink, so one hostile document leaves that worker holding up to +/// `limit` for the rest of the pass. That is the trade for never allocating +/// in the common case, and it is bounded where it used to be 16× larger. +fn member_bytes( archive: &mut ZipArchive, name: &str, -) -> Result> { - let mut body = Vec::new(); + limit: usize, + buf: &mut Vec, +) -> Result<(), Box> { + buf.clear(); archive .by_name(name)? - .take(MAX_XML_BYTES as u64 + 1) - .read_to_end(&mut body)?; - let truncated = body.len() > MAX_XML_BYTES; - body.truncate(MAX_XML_BYTES); - match String::from_utf8(body) { - Ok(text) => Ok(text), - // Only a cut at the cap may split a character; invalid UTF-8 anywhere - // else still fails the extraction, as `read_to_string` always did. - Err(e) if truncated && e.utf8_error().valid_up_to() >= MAX_XML_BYTES - 3 => { - let valid = e.utf8_error().valid_up_to(); - let mut bytes = e.into_bytes(); - bytes.truncate(valid); - Ok(String::from_utf8(bytes)?) - } - Err(e) => Err(e.into()), - } + .take(limit as u64) + .read_to_end(buf)?; + Ok(()) +} + +/// A reader over bytes already in hand — what the XML parsers are driven +/// from, so quick-xml streams events out of the worker's buffer rather than +/// a copy of it. +fn xml_over(buf: &[u8]) -> Cursor<&[u8]> { + Cursor::new(buf) } /// Names of the `.xml` members under `prefix`, in archive order — not @@ -227,61 +253,87 @@ fn xml_members_under( } /// A format whose whole text lives in one member under one spec. -fn single_member(path: &Path, member: &str, spec: &TextSpec) -> Result> { +fn single_member( + path: &Path, + member: &str, + spec: &TextSpec, + out: &mut String, + scratch: &mut Scratch, +) -> Result<(), Box> { + let limits = scratch.limits(); let mut archive = open_container(path)?; - let xml = member_text(&mut archive, member)?; - let mut out = String::new(); - collect_xml_text(&xml, spec, &mut out)?; - Ok(out) + let (bytes, events) = scratch.container_bufs(); + member_bytes(&mut archive, member, limits.inflate, bytes)?; + collect_xml_text(xml_over(bytes), spec, out, limits.text, events) } /// Concatenate what `collect` gets out of each `.xml` member under `prefix`, -/// in archive order, bounded by [`MAX_TEXT_BYTES`]; whole members are kept or -/// dropped, never cut mid-way. +/// in archive order, stopping once the text reaches [`super::Limits::text`] +/// — the point past which the caller would discard it anyway. fn collect_members( archive: &mut Archive, prefix: &str, - mut collect: impl FnMut(&str, &mut String) -> Result<(), Box>, -) -> Result> { - let mut out = String::new(); + out: &mut String, + limit: usize, + mut collect: impl FnMut(&mut Archive, &str, &mut String) -> Result<(), Box>, +) -> Result<(), Box> { for name in xml_members_under(archive, prefix)? { - if out.len() >= MAX_TEXT_BYTES { + if out.len() >= limit { break; } - let xml = member_text(archive, &name)?; - collect(&xml, &mut out)?; + collect(archive, &name, out)?; } - Ok(out) + Ok(()) } -fn extract_pptx(path: &Path) -> Result> { +fn extract_pptx(path: &Path, out: &mut String, scratch: &mut Scratch) -> Result<(), Box> { + let limits = scratch.limits(); let mut archive = open_container(path)?; - collect_members(&mut archive, "ppt/slides/slide", |xml, out| { - collect_xml_text(xml, &PPTX, out)?; - out.push_str("\n--- New Slide ---\n"); - Ok(()) - }) + let (bytes, events) = scratch.container_bufs(); + collect_members( + &mut archive, + "ppt/slides/slide", + out, + limits.text, + |archive, name, out| { + member_bytes(archive, name, limits.inflate, bytes)?; + collect_xml_text(xml_over(bytes), &PPTX, out, limits.text, events)?; + out.push_str("\n--- New Slide ---\n"); + Ok(()) + }, + ) } // XLSX: shared strings plus cells -/// The workbook's shared-string table, in index order. Absent or unreadable -/// is not an error: a sheet of nothing but numbers has no table at all. -fn shared_strings(archive: &mut ZipArchive) -> Vec { - let Ok(xml) = member_text(archive, "xl/sharedStrings.xml") else { - return Vec::new(); - }; - let mut reader = Reader::from_str(&xml); +/// The workbook's shared-string table, in index order, into `strings`. +/// Absent or unreadable is not an error: a sheet of nothing but numbers has +/// no table at all. +/// +/// **This one member is read whole**, unlike every other: a `t="s"` cell +/// holds an *index* into the table, so a table cut short does not lose the +/// tail — it renders the wrong string for every cell past the cut, silently. +/// It is bounded by the same inflation budget and by nothing else. +fn shared_strings( + archive: &mut ZipArchive, + limit: usize, + bytes: &mut Vec, + events: &mut Vec, + strings: &mut Vec, +) { + if member_bytes(archive, "xl/sharedStrings.xml", limit, bytes).is_err() { + return; + } + let mut reader = Reader::from_reader(xml_over(bytes)); // No `trim_text`; see `collect_xml_text`. - let mut buf = Vec::new(); - let mut strings = Vec::new(); + events.clear(); let mut in_text = false; // One `` is one shared string but not one event (an entity reference // splits it); accumulated and pushed on the closing tag, or a cell with // `&` would become three table entries. let mut current = String::new(); loop { - match reader.read_event_into(&mut buf) { + match reader.read_event_into(events) { Ok(Event::Start(ref e)) if e.name().as_ref() == b"t" => { in_text = true; current.clear(); @@ -295,14 +347,12 @@ fn shared_strings(archive: &mut ZipArchive) -> Vec { } Ok(Event::Text(e)) if in_text => match e.decode() { Ok(s) => current.push_str(&s), - Err(_) => return strings, + Err(_) => return, }, Ok(Event::GeneralRef(e)) if in_text => { // This reader cannot fail; an unexpandable entity is left out. if let Ok(raw) = e.decode() { - if let Some(text) = entity_text(&raw) { - current.push_str(&text); - } + let _ = push_entity_text(&raw, &mut current); } } Ok(Event::End(ref e)) if e.name().as_ref() == b"t" => { @@ -312,29 +362,37 @@ fn shared_strings(archive: &mut ZipArchive) -> Vec { Ok(Event::Eof) | Err(_) => break, _ => {} } - buf.clear(); + events.clear(); } - strings } /// One worksheet's cells. A `t="s"` cell holds an index into `strings` /// rather than text of its own; every other type holds its value inline. -fn collect_sheet(xml: &str, strings: &[String], out: &mut String) -> Result<(), Box> { - let mut reader = Reader::from_str(xml); +fn collect_sheet( + xml: R, + strings: &[String], + out: &mut String, + limit: usize, + buf: &mut Vec, +) -> Result<(), Box> { + let mut reader = Reader::from_reader(xml); // No `trim_text`; see `collect_xml_text`. - let mut buf = Vec::new(); + buf.clear(); let mut in_cell = false; let mut cell_type = String::new(); loop { - match reader.read_event_into(&mut buf) { + if out.len() >= limit { + return Ok(()); + } + match reader.read_event_into(buf) { Ok(Event::Start(ref e)) if e.name().as_ref() == b"c" => { in_cell = true; cell_type.clear(); // `with_checks(false)`: the duplicate-attribute-name check is // quadratic with no bound but the tag's size - // (RUSTSEC-2026-0194), so one crafted `` in 64 MiB of - // inflated XML could hold this worker for hours, + // (RUSTSEC-2026-0194), so one crafted `` in the inflation + // budget's worth of XML could hold this worker for hours, // uncancellably. This extractor wants one attribute anyway. for attr in e.attributes().with_checks(false) { let attr = attr?; @@ -372,9 +430,9 @@ fn collect_sheet(xml: &str, strings: &[String], out: &mut String) -> Result<(), // Only inline values can carry one; a `t="s"` cell's is an index. Ok(Event::GeneralRef(e)) if in_cell && cell_type != "s" => { let raw = e.decode()?; - let text = entity_text(&raw) - .ok_or_else(|| format!("Error parsing XML: unknown entity &{};", raw))?; - out.push_str(&text); + if !push_entity_text(&raw, out) { + return Err(format!("Error parsing XML: unknown entity &{};", raw).into()); + } } Ok(Event::End(ref e)) => { let name = e.name(); @@ -393,12 +451,23 @@ fn collect_sheet(xml: &str, strings: &[String], out: &mut String) -> Result<(), Ok(()) } -fn extract_xlsx(path: &Path) -> Result> { +fn extract_xlsx(path: &Path, out: &mut String, scratch: &mut Scratch) -> Result<(), Box> { + let limits = scratch.limits(); let mut archive = open_container(path)?; - let strings = shared_strings(&mut archive); - collect_members(&mut archive, "xl/worksheets/sheet", |xml, out| { - collect_sheet(xml, &strings, out) - }) + // All three at once: the sheets are read while the table is live, and + // separate `&mut scratch` borrows cannot overlap. + let (bytes, events, strings) = scratch.xlsx_bufs(); + shared_strings(&mut archive, limits.inflate, bytes, events, strings); + collect_members( + &mut archive, + "xl/worksheets/sheet", + out, + limits.text, + |archive, name, out| { + member_bytes(archive, name, limits.inflate, bytes)?; + collect_sheet(xml_over(bytes), strings, out, limits.text, events) + }, + ) } // Dispatch @@ -406,16 +475,21 @@ fn extract_xlsx(path: &Path) -> Result> { /// Extract text from an office document, chosen by lowercase extension. An /// unhandled extension yields empty text: the MIME was claimed, so the file /// was simply named unlike its type. -fn extract_document_text(path: &Path, extension: &str) -> Result> { +fn extract_document_text( + path: &Path, + extension: &str, + out: &mut String, + scratch: &mut Scratch, +) -> Result<(), Box> { match extension { - "docx" => single_member(path, "word/document.xml", &DOCX), - "xlsx" => extract_xlsx(path), - "pptx" => extract_pptx(path), - "odt" | "odp" => single_member(path, "content.xml", &ODF_TEXT), - "ods" => single_member(path, "content.xml", &ODF_SHEET), + "docx" => single_member(path, "word/document.xml", &DOCX, out, scratch), + "xlsx" => extract_xlsx(path, out, scratch), + "pptx" => extract_pptx(path, out, scratch), + "odt" | "odp" => single_member(path, "content.xml", &ODF_TEXT, out, scratch), + "ods" => single_member(path, "content.xml", &ODF_SHEET, out, scratch), // Pre-2007 binary formats: a different container entirely. - "doc" | "xls" | "ppt" => super::ole::extract_ole_text(path, extension), - _ => Ok(String::new()), + "doc" | "xls" | "ppt" => super::ole::extract_ole_text(path, extension, out, scratch), + _ => Ok(()), } } @@ -424,16 +498,20 @@ impl Extractor for OfficeExtractor { mime_to_ext(mime).is_some() } - fn extract(&self, path: &Path) -> Result { + fn extract( + &self, + path: &Path, + out: &mut String, + scratch: &mut Scratch, + ) -> Result<(), ExtractError> { // From the path, not the MIME: `.docm` and `.docx` share a MIME. let ext = path .extension() .and_then(|s| s.to_str()) .map(|s| s.to_ascii_lowercase()) .unwrap_or_default(); - let text = extract_document_text(path, &ext) - .map_err(|e| format!("office extractor {}: {}", path.display(), e))?; - Ok(text) + extract_document_text(path, &ext, out, scratch) + .map_err(|e| format!("office extractor {}: {}", path.display(), e)) } } @@ -442,6 +520,22 @@ mod tests { use super::*; use std::io::Write; + fn scratch() -> Scratch { + Scratch::new(&crate::config::Config::default()) + } + + /// The one-file forms: these assert on extracted text, not on the buffer + /// reuse a pool worker gets. + fn extract_document_text(path: &Path, extension: &str) -> Result> { + let mut out = String::new(); + super::extract_document_text(path, extension, &mut out, &mut scratch()).map(|()| out) + } + + fn office_extract(path: &Path) -> Result { + let mut out = String::new(); + OfficeExtractor.extract(path, &mut out, &mut scratch()).map(|()| out) + } + /// A reader that only handles `Event::Text` loses `&` with no error; /// this fails by producing "Blake Co". #[test] @@ -450,7 +544,7 @@ mod tests { Blake & Co <tags> ’24 ’25\ "; let path = container("docx-entities", "docx", &[("word/document.xml", body)]); - let out = OfficeExtractor.extract(&path).expect("extract"); + let out = office_extract(&path).expect("extract"); assert!( out.contains("Blake & Co"), "predefined entity lost: {:?}", @@ -483,7 +577,7 @@ mod tests { ("xl/worksheets/sheet1.xml", sheet), ], ); - let out = OfficeExtractor.extract(&path).expect("extract"); + let out = office_extract(&path).expect("extract"); assert!( out.contains("Jack & Jill"), "entity lost through the shared-string table: {:?}", @@ -509,7 +603,7 @@ mod tests { ("xl/worksheets/sheet1.xml", sheet), ], ); - let out = OfficeExtractor.extract(&path).expect("extract"); + let out = office_extract(&path).expect("extract"); assert!( out.contains("Marmalade"), "the shared string was dropped by an indented index: {:?}", @@ -556,6 +650,78 @@ mod tests { path } + /// A docx whose text is far larger than any limit under test, and whose + /// XML compresses to almost nothing — the shape a hostile archive has. + fn oversized_docx(tag: &str, runs: usize) -> std::path::PathBuf { + let mut body = String::from(""); + for i in 0..runs { + body.push_str(""); + // Distinguishable, so a truncated result can be located. + body.push_str(&format!("paragraph{:08} ", i)); + body.push_str(""); + } + body.push_str(""); + container(tag, "docx", &[("word/document.xml", &body)]) + } + + fn limited(text: usize) -> Scratch { + let mut config = crate::config::Config::default(); + config.processing.maximum_text_size = text; + Scratch::new(&config) + } + + /// Extraction **stops** at `maximum_text_size` instead of running the + /// document to its end for the caller to truncate. The margin is what + /// makes this a real assertion: the old code produced every byte, so a + /// result the size of the document would pass a "≥ limit" check. + #[test] + fn a_document_larger_than_the_limit_stops_at_it() { + // ~2 MiB of text; the limit is 4 KiB, so 99.8% must never be built. + let path = oversized_docx("docx-oversize", 100_000); + let mut out = String::new(); + let mut scratch = limited(4096); + OfficeExtractor + .extract(&path, &mut out, &mut scratch) + .expect("extract"); + + assert!( + out.len() >= 4096, + "stopped short of the limit: {} bytes", + out.len() + ); + // One paragraph of overshoot is the documented allowance — the check + // is per event, not per byte. + assert!( + out.len() < 4096 * 2, + "ran past the limit rather than stopping at it: {} bytes", + out.len() + ); + assert!( + out.starts_with("paragraph00000000"), + "the kept text is the document's start: {:?}", + &out[..out.len().min(40)] + ); + } + + /// The same document with the shipped limits: still bounded, and still + /// the document's beginning rather than an arbitrary window. + #[test] + fn the_default_limits_bound_an_oversized_document() { + let path = oversized_docx("docx-oversize-default", 100_000); + let config = crate::config::Config::default(); + let mut out = String::new(); + let mut scratch = Scratch::new(&config); + OfficeExtractor + .extract(&path, &mut out, &mut scratch) + .expect("extract"); + assert!( + out.len() < config.processing.maximum_text_size * 2, + "{} bytes for a {}-byte limit", + out.len(), + config.processing.maximum_text_size + ); + } + const DOCX_BODY: &str = "\ Helloworld\ Second\ diff --git a/crates/quicksearch-core/src/extract/ole.rs b/crates/quicksearch-core/src/extract/ole.rs index 30ad652..47ba451 100644 --- a/crates/quicksearch-core/src/extract/ole.rs +++ b/crates/quicksearch-core/src/extract/ole.rs @@ -10,20 +10,34 @@ use std::fs::File; use std::io::Read; use std::path::Path; -/// Ceiling on extracted text from one legacy document: the formats can -/// declare far more text than they contain, and this earlier, cruder bound -/// keeps a hostile header from turning into an allocation. -const MAX_TEXT_BYTES: usize = 64 * 1024 * 1024; +use super::Scratch; -pub fn extract_ole_text(path: &Path, extension: &str) -> Result> { +/// Ceiling on extracted text from one legacy document, from the caller's +/// [`Limits::text`](super::Limits::text): the formats can declare far more +/// text than they contain, and this bound keeps a hostile header from turning +/// into an allocation. It used to be a hardcoded 64 MiB — 256× the largest +/// result the caller keeps. +pub fn extract_ole_text( + path: &Path, + extension: &str, + out: &mut String, + scratch: &mut Scratch, +) -> Result<(), Box> { + let budget = scratch.limits().text; let mut cfb = cfb::CompoundFile::open(File::open(path)?) .map_err(|e| format!("not a readable OLE2 compound file: {}", e))?; - match extension { - "doc" => doc::extract(&mut cfb), - "xls" => xls::extract(&mut cfb), - "ppt" => ppt::extract(&mut cfb), + let text = match extension { + "doc" => doc::extract(&mut cfb, budget), + "xls" => xls::extract(&mut cfb, budget), + "ppt" => ppt::extract(&mut cfb, budget), other => Err(format!("no OLE2 parser for .{}", other).into()), + }?; + if out.is_empty() { + *out = text; + } else { + out.push_str(&text); } + Ok(()) } fn stream(cfb: &mut cfb::CompoundFile, name: &str) -> Option> { @@ -118,6 +132,7 @@ mod doc { pub fn extract( cfb: &mut cfb::CompoundFile, + budget: usize, ) -> Result> { let doc = stream(cfb, "WordDocument").ok_or("no WordDocument stream")?; let flags = u16_at(&doc, FIB_FLAGS).ok_or("truncated FIB")?; @@ -137,7 +152,7 @@ mod doc { .ok_or("CLX runs past the end of the table stream")?; let pieces = piece_table(clx)?; - let out = decode_pieces(&doc, &pieces, MAX_TEXT_BYTES); + let out = decode_pieces(&doc, &pieces, budget); if out.trim().is_empty() { return Err("no text found in the piece table".into()); } @@ -298,12 +313,13 @@ mod xls { pub fn extract( cfb: &mut cfb::CompoundFile, + budget: usize, ) -> Result> { // BIFF8 names the stream "Workbook"; BIFF5 and earlier used "Book". let book = stream(cfb, "Workbook") .or_else(|| stream(cfb, "Book")) .ok_or("no Workbook stream")?; - extract_from_book(&book, MAX_TEXT_BYTES) + extract_from_book(&book, budget) } pub(super) fn extract_from_book(book: &[u8], budget: usize) -> Result> { @@ -566,18 +582,19 @@ mod ppt { pub fn extract( cfb: &mut cfb::CompoundFile, + budget: usize, ) -> Result> { let doc = stream(cfb, "PowerPoint Document").ok_or("no PowerPoint Document stream")?; let mut out = String::new(); - walk(&doc, 0, &mut out); + walk(&doc, 0, &mut out, budget); if out.trim().is_empty() { return Err("no text atoms found in the presentation".into()); } Ok(out) } - fn walk(body: &[u8], depth: u32, out: &mut String) { - if depth > MAX_DEPTH || out.len() >= MAX_TEXT_BYTES { + fn walk(body: &[u8], depth: u32, out: &mut String, budget: usize) { + if depth > MAX_DEPTH || out.len() >= budget { return; } let mut i = 0usize; @@ -590,7 +607,7 @@ mod ppt { return; }; if version & 0x000F == VERSION_CONTAINER { - walk(payload, depth + 1, out); + walk(payload, depth + 1, out, budget); } else { match rec_type { TEXT_BYTES_ATOM | CSTRING_ATOM if rec_type == CSTRING_ATOM => { diff --git a/crates/quicksearch-core/src/extract/ole_tests.rs b/crates/quicksearch-core/src/extract/ole_tests.rs index bbcbef6..ff07439 100644 --- a/crates/quicksearch-core/src/extract/ole_tests.rs +++ b/crates/quicksearch-core/src/extract/ole_tests.rs @@ -1,6 +1,14 @@ use super::*; use std::io::{Cursor, Write}; +/// The one-file form: these assert on extracted text, not on the buffer +/// reuse a pool worker gets. +fn extract_ole_text(path: &Path, extension: &str) -> Result> { + let mut out = String::new(); + let mut scratch = Scratch::new(&crate::config::Config::default()); + super::extract_ole_text(path, extension, &mut out, &mut scratch).map(|()| out) +} + fn container(tag: &str, ext: &str, streams: &[(&str, Vec)]) -> std::path::PathBuf { let path = crate::testutil::scratch_dir(tag).join(format!("doc.{ext}")); let mut cfb = cfb::CompoundFile::create(Cursor::new(Vec::new())).unwrap(); @@ -480,7 +488,7 @@ fn doc_pieces_within_the_budget_are_all_decoded() { /// emitted-text brake never advances however many of them there are. #[test] fn xls_control_character_cells_stop_at_the_budget() { - let control: String = std::iter::repeat('\u{1}').take(4096).collect(); + let control: String = std::iter::repeat_n('\u{1}', 4096).collect(); let mut sst = Vec::new(); sst.extend_from_slice(&le32(2)); // total diff --git a/crates/quicksearch-core/src/extract/pdf.rs b/crates/quicksearch-core/src/extract/pdf.rs index fb0767e..c3966d6 100644 --- a/crates/quicksearch-core/src/extract/pdf.rs +++ b/crates/quicksearch-core/src/extract/pdf.rs @@ -8,7 +8,7 @@ use std::sync::OnceLock; use pdf_extract::{Document, PlainTextOutput}; -use super::{ExtractError, Extractor}; +use super::{ExtractError, Extractor, Scratch}; thread_local! { /// True while this thread is inside a contained `pdf_extract` call. @@ -46,14 +46,32 @@ impl Extractor for PdfExtractor { mime == "application/pdf" } - fn extract(&self, path: &Path) -> Result { + /// The one format with no streaming option: `Document::load` builds the + /// whole object graph before a byte of text comes out, and a 2 MiB file + /// has been measured holding tens of megabytes. Its *input* is bounded by + /// `maximum_text_file_size` and its output by `maximum_text_size`, but the + /// middle is `pdf_extract`'s and there is no scratch to reuse — so peak + /// for PDFs alone is `workers × amplification`, one pool per root. + fn extract( + &self, + path: &Path, + out: &mut String, + _scratch: &mut Scratch, + ) -> Result<(), ExtractError> { // Loading is inside the guard too — a panic outside it takes the thread. install_quiet_panic_hook(); let path_buf = path.to_path_buf(); SUPPRESS_PANIC_PRINT.with(|flag| flag.set(true)); let result = std::panic::catch_unwind(move || extract_one_pass(&path_buf)); SUPPRESS_PANIC_PRINT.with(|flag| flag.set(false)); - result.map_err(|panic| format!("pdf_extract panicked: {}", panic_message(&*panic)))? + let text = result + .map_err(|panic| format!("pdf_extract panicked: {}", panic_message(&*panic)))??; + if out.is_empty() { + *out = text; + } else { + out.push_str(&text); + } + Ok(()) } } @@ -77,6 +95,12 @@ fn extract_one_pass(path: &Path) -> Result { mod tests { use super::*; + fn extract(path: &Path) -> Result { + let mut out = String::new(); + let mut scratch = Scratch::new(&crate::config::Config::default()); + PdfExtractor.extract(path, &mut out, &mut scratch).map(|()| out) + } + #[test] fn contained_panics_are_caught_quietly_with_reason() { install_quiet_panic_hook(); @@ -163,7 +187,7 @@ mod tests { }), ); - let out = PdfExtractor.extract(&path).expect("extract"); + let out = extract(&path).expect("extract"); assert!( out.contains("Hello QuickSearch"), "drawn text missing from {:?}", @@ -174,7 +198,7 @@ mod tests { #[test] fn missing_info_dictionary_still_yields_text() { let path = write_pdf("pdf-noinfo", "Body Only", None); - let out = PdfExtractor.extract(&path).expect("extract"); + let out = extract(&path).expect("extract"); assert!(out.contains("Body Only")); } @@ -183,8 +207,7 @@ mod tests { let path = crate::testutil::scratch_dir("pdf-malformed").join("broken.pdf"); std::fs::write(&path, b"%PDF-1.4\n\x00\x01\x02 not a pdf at all \xff\xfe").unwrap(); - let err = PdfExtractor - .extract(&path) + let err = extract(&path) .expect_err("malformed pdf must fail"); assert!( err.starts_with("pdf_extract"), @@ -226,7 +249,7 @@ mod tests { doc.save(&path).expect("write fixture pdf"); // The verdict that matters is that we reach this line at all. - let _ = PdfExtractor.extract(&path); + let _ = extract(&path); } /// A Form XObject drawing itself must be skipped — the second unbounded @@ -280,6 +303,6 @@ mod tests { let path = crate::testutil::scratch_dir("pdf-xobject-cycle").join("cycle.pdf"); doc.save(&path).expect("write fixture pdf"); - let _ = PdfExtractor.extract(&path); + let _ = extract(&path); } } diff --git a/crates/quicksearch-core/src/extract/plaintext.rs b/crates/quicksearch-core/src/extract/plaintext.rs index 4a83238..6bfca50 100644 --- a/crates/quicksearch-core/src/extract/plaintext.rs +++ b/crates/quicksearch-core/src/extract/plaintext.rs @@ -5,7 +5,7 @@ use std::fs::File; use std::io::Read; use std::path::Path; -use super::{ExtractError, Extractor}; +use super::{ExtractError, Extractor, Scratch}; /// Non-`text/*` MIMEs the plaintext extractor claims. Every entry must be /// emitted by some MIME source and map to a [`crate::mime::FileType`] @@ -38,16 +38,26 @@ pub(crate) const EXTRA_TEXT_MIMES: &[&str] = &[ "message/rfc822", ]; -fn decode(bytes: Vec, path: &Path) -> Result { - crate::textenc::decode_text(bytes, path) +/// Decode into `out`, moving rather than copying where the class allows it. +/// +/// This extractor is the one place a read buffer *becomes* the answer: a +/// UTF-8 file's bytes are its text, so consuming the buffer turns the read +/// into the output with no copy at all. That is why plaintext reads into a +/// fresh buffer instead of the worker's scratch — a reused buffer here would +/// trade one allocation for one full-length `memcpy` per file, on the format +/// that dominates every corpus. +fn decode_into(bytes: Vec, path: &Path, out: &mut String) -> Result<(), ExtractError> { + let text = crate::textenc::decode_text(bytes, path)?; + if out.is_empty() { + *out = text; + } else { + out.push_str(&text); + } + Ok(()) } pub struct PlaintextExtractor; -/// Ceiling on a single read, whatever the file claims — backstop for files -/// that grew since the walk sized them and for nodes whose `fstat` lies. -const MAX_READ: usize = 64 * 1024 * 1024; - /// Read `size` bytes from `f`, never more than `cap`. A short read is not an /// error: a file that shrank keeps its prefix. fn read_sized(f: &mut File, size: usize, cap: usize, path: &Path) -> Result, ExtractError> { @@ -73,7 +83,16 @@ impl Extractor for PlaintextExtractor { /// A file that shrank since the `fstat` keeps its prefix; one that grew is /// read to the sized length — its mtime moved, so the next run re-extracts. - fn extract(&self, path: &Path) -> Result { + fn extract( + &self, + path: &Path, + out: &mut String, + scratch: &mut Scratch, + ) -> Result<(), ExtractError> { + // `read`, not a constant of this module's own: the content pass never + // offers a file above it, so the only readers that reach the cap are + // the two below — a file that grew, and one whose size is a lie. + let cap = scratch.limits().read; let mut f = File::open(path).map_err(|e| format!("plaintext read {}: {}", path.display(), e))?; let size = f @@ -85,17 +104,32 @@ impl Extractor for PlaintextExtractor { // only they pay the read-to-EOF probe, capped against endless streams. if size == 0 { let mut buf = Vec::new(); - f.take(MAX_READ as u64) + f.take(cap as u64) .read_to_end(&mut buf) .map_err(|e| format!("plaintext read {}: {}", path.display(), e))?; - return decode(buf, path); + return decode_into(buf, path, out); } - decode(read_sized(&mut f, size, MAX_READ, path)?, path) + decode_into(read_sized(&mut f, size, cap, path)?, path, out) } - fn extract_from_head(&self, path: &Path, head: &[u8]) -> Option> { - Some(decode(head.to_vec(), path)) + /// The head belongs to the walk worker and is reused for the next file, + /// so it is decoded borrowed rather than copied to be given away. + fn extract_from_head( + &self, + path: &Path, + head: &[u8], + out: &mut String, + ) -> Option> { + Some( + crate::textenc::decode_borrowed_text(head, path).map(|text| { + if out.is_empty() { + *out = text; + } else { + out.push_str(&text); + } + }), + ) } } @@ -103,6 +137,31 @@ impl Extractor for PlaintextExtractor { mod tests { use super::*; + fn scratch() -> Scratch { + Scratch::new(&crate::config::Config::default()) + } + + /// The default `maximum_text_file_size`, which is what bounds a read now. + fn max_read() -> usize { + scratch().limits().read + } + + fn extract(path: &Path) -> Result { + let mut out = String::new(); + PlaintextExtractor + .extract(path, &mut out, &mut scratch()) + .map(|()| out) + } + + fn extract_from_head(path: &Path, head: &[u8]) -> Option> { + let mut out = String::new(); + match PlaintextExtractor.extract_from_head(path, head, &mut out) { + Some(Ok(())) => Some(Ok(out)), + Some(Err(e)) => Some(Err(e)), + None => None, + } + } + fn tmp(tag: &str, body: &[u8]) -> std::path::PathBuf { let p = crate::testutil::scratch_dir(tag).join("sample.txt"); crate::testutil::touch(&p, body); @@ -112,7 +171,7 @@ mod tests { #[test] fn reads_utf8_file() { let p = tmp("basic", b"hello world"); - let c = PlaintextExtractor.extract(&p).unwrap(); + let c = extract(&p).unwrap(); assert_eq!(c, "hello world"); std::fs::remove_file(&p).ok(); } @@ -123,10 +182,9 @@ mod tests { "agree", b"shared body with unicode: caf\xc3\xa9 \xe2\x9c\x93", ); - let from_disk = PlaintextExtractor.extract(&p).unwrap(); + let from_disk = extract(&p).unwrap(); let bytes = std::fs::read(&p).unwrap(); - let from_head = PlaintextExtractor - .extract_from_head(&p, &bytes) + let from_head = extract_from_head(&p, &bytes) .unwrap() .unwrap(); assert_eq!(from_disk, from_head); @@ -138,9 +196,8 @@ mod tests { // A NUL keeps this undecodable now that legacy charsets decode. let body = [0x68, 0x69, 0x00, 0xff]; let p = tmp("binary", &body); - let disk_err = PlaintextExtractor.extract(&p).unwrap_err(); - let head_err = PlaintextExtractor - .extract_from_head(&p, &body) + let disk_err = extract(&p).unwrap_err(); + let head_err = extract_from_head(&p, &body) .unwrap() .unwrap_err(); assert_eq!(disk_err, head_err, "one decode path, one message"); @@ -156,9 +213,8 @@ mod tests { fn latin1_decodes_via_both_paths() { let body = b"une journ\xe9e agr\xe9able pr\xe8s de la rivi\xe8re"; let p = tmp("latin1", body); - let from_disk = PlaintextExtractor.extract(&p).unwrap(); - let from_head = PlaintextExtractor - .extract_from_head(&p, body) + let from_disk = extract(&p).unwrap(); + let from_head = extract_from_head(&p, body) .unwrap() .unwrap(); assert_eq!(from_disk, from_head); @@ -172,9 +228,8 @@ mod tests { let mut body = vec![0xFF, 0xFE]; body.extend(src.encode_utf16().flat_map(|u| u.to_le_bytes())); let p = tmp("utf16", &body); - let from_disk = PlaintextExtractor.extract(&p).unwrap(); - let from_head = PlaintextExtractor - .extract_from_head(&p, &body) + let from_disk = extract(&p).unwrap(); + let from_head = extract_from_head(&p, &body) .unwrap() .unwrap(); assert_eq!(from_disk, from_head); @@ -189,7 +244,7 @@ mod tests { fn reads_a_file_larger_than_one_buffer_completely() { let body = "abcdefgh".repeat(200 * 1024 / 8); let p = tmp("large", body.as_bytes()); - let c = PlaintextExtractor.extract(&p).unwrap(); + let c = extract(&p).unwrap(); assert_eq!(c.len(), body.len()); assert_eq!(c, body); std::fs::remove_file(&p).ok(); @@ -198,14 +253,8 @@ mod tests { #[test] fn an_empty_file_extracts_to_empty_text() { let p = tmp("empty", b""); - assert_eq!(PlaintextExtractor.extract(&p).unwrap(), ""); - assert_eq!( - PlaintextExtractor - .extract_from_head(&p, &[]) - .unwrap() - .unwrap(), - "" - ); + assert_eq!(extract(&p).unwrap(), ""); + assert_eq!(extract_from_head(&p, &[]).unwrap().unwrap(), ""); std::fs::remove_file(&p).ok(); } @@ -220,7 +269,7 @@ mod tests { 0, "precondition: procfs reports zero size" ); - let c = PlaintextExtractor.extract(p).unwrap(); + let c = extract(p).unwrap(); assert!( c.contains("Name:"), "content must survive a zero st_size, got {} bytes", @@ -235,7 +284,7 @@ mod tests { // A mid-extract truncate is not reproducible; assert the property directly. f.set_len(10).unwrap(); drop(f); - let c = PlaintextExtractor.extract(&p).unwrap(); + let c = extract(&p).unwrap(); assert_eq!(c, "xxxxxxxxxx", "a shrunk file reads short, not fatal"); std::fs::remove_file(&p).ok(); } @@ -272,7 +321,7 @@ mod tests { fn a_short_read_keeps_what_was_there() { let p = tmp("short", b"only ten!!"); let mut f = File::open(&p).unwrap(); - let out = read_sized(&mut f, 1_000_000, MAX_READ, &p).unwrap(); + let out = read_sized(&mut f, 1_000_000, max_read(), &p).unwrap(); assert_eq!(out, b"only ten!!"); std::fs::remove_file(&p).ok(); } @@ -281,7 +330,7 @@ mod tests { fn a_file_under_the_cap_is_read_whole() { let body = vec![b'y'; 4096]; let p = tmp("uncapped", &body); - let out = PlaintextExtractor.extract(&p).unwrap(); + let out = extract(&p).unwrap(); assert_eq!(out.len(), 4096); std::fs::remove_file(&p).ok(); } diff --git a/crates/quicksearch-core/src/extract/rtf.rs b/crates/quicksearch-core/src/extract/rtf.rs index 6d0fe91..e6525f5 100644 --- a/crates/quicksearch-core/src/extract/rtf.rs +++ b/crates/quicksearch-core/src/extract/rtf.rs @@ -8,32 +8,43 @@ use std::path::Path; use rtf_parser::document::RtfDocument; -use super::{ExtractError, Extractor}; - -/// Ceiling on a single read; see [`super::plaintext`], same reasoning. -const MAX_READ: usize = 64 * 1024 * 1024; +use super::{ExtractError, Extractor, Scratch}; /// RTF is 7-bit ASCII by design — non-ASCII travels as `\'hh` and `\uN` /// escapes — so the lossy UTF-8 view loses nothing from a well-formed file. -fn parse(bytes: Vec, path: &Path) -> Result { - let source = String::from_utf8_lossy(&bytes); +/// +/// Borrowed: `from_utf8_lossy` borrows an already-valid buffer, so an owned +/// argument bought nothing and cost the head path a full copy. +fn parse(bytes: &[u8], path: &Path, out: &mut String) -> Result<(), ExtractError> { + let source = String::from_utf8_lossy(bytes); match RtfDocument::try_from(source.as_ref()) { - Ok(doc) => Ok(doc.get_text()), + Ok(doc) => { + // `get_text` builds its own string; taking it whole is one move + // when `out` is empty, which it is for every caller today. + let text = doc.get_text(); + if out.is_empty() { + *out = text; + } else { + out.push_str(&text); + } + Ok(()) + } Err(e) => Err(format!("rtf parse {}: {}", path.display(), e)), } } pub struct RtfExtractor; -/// Read at most `cap` bytes of `path`; `rtf-parser` amplifies its input -/// several-fold in heap, so the read stays bounded whatever the walk recorded. -fn read_capped(path: &Path, cap: u64) -> Result, ExtractError> { +/// Read at most `cap` bytes of `path` into `buf`; `rtf-parser` amplifies its +/// input several-fold in heap, so the read stays bounded whatever the walk +/// recorded. `buf` is the worker's, reused file after file. +fn read_capped(path: &Path, cap: u64, buf: &mut Vec) -> Result<(), ExtractError> { let file = File::open(path).map_err(|e| format!("rtf read {}: {}", path.display(), e))?; - let mut bytes = Vec::new(); + buf.clear(); file.take(cap) - .read_to_end(&mut bytes) + .read_to_end(buf) .map_err(|e| format!("rtf read {}: {}", path.display(), e))?; - Ok(bytes) + Ok(()) } impl Extractor for RtfExtractor { @@ -41,13 +52,25 @@ impl Extractor for RtfExtractor { mime == "application/rtf" || mime == "text/rtf" } - fn extract(&self, path: &Path) -> Result { - parse(read_capped(path, MAX_READ as u64)?, path) + fn extract( + &self, + path: &Path, + out: &mut String, + scratch: &mut Scratch, + ) -> Result<(), ExtractError> { + let cap = scratch.limits().read as u64; + read_capped(path, cap, scratch.bytes_mut())?; + parse(scratch.bytes_mut(), path, out) } /// RTF has no trailer and needs no seeking; a complete head parses like disk. - fn extract_from_head(&self, path: &Path, head: &[u8]) -> Option> { - Some(parse(head.to_vec(), path)) + fn extract_from_head( + &self, + path: &Path, + head: &[u8], + out: &mut String, + ) -> Option> { + Some(parse(head, path, out)) } } @@ -55,6 +78,34 @@ impl Extractor for RtfExtractor { mod tests { use super::*; + fn scratch() -> Scratch { + Scratch::new(&crate::config::Config::default()) + } + + /// The default `maximum_text_file_size`, which is what bounds a read now. + fn max_read() -> u64 { + scratch().limits().read as u64 + } + + fn read_capped_to_vec(path: &Path, cap: u64) -> Result, ExtractError> { + let mut buf = Vec::new(); + read_capped(path, cap, &mut buf).map(|()| buf) + } + + fn extract(path: &Path) -> Result { + let mut out = String::new(); + RtfExtractor.extract(path, &mut out, &mut scratch()).map(|()| out) + } + + fn extract_from_head(path: &Path, head: &[u8]) -> Option> { + let mut out = String::new(); + match RtfExtractor.extract_from_head(path, head, &mut out) { + Some(Ok(())) => Some(Ok(out)), + Some(Err(e)) => Some(Err(e)), + None => None, + } + } + fn tmp(tag: &str, body: &[u8]) -> std::path::PathBuf { let p = crate::testutil::scratch_dir(tag).join("sample.rtf"); crate::testutil::touch(&p, body); @@ -65,7 +116,7 @@ mod tests { fn extracts_text_without_control_words() { let body = br"{\rtf1\ansi Hello {\b World}!}"; let p = tmp("basic", body); - let c = RtfExtractor.extract(&p).unwrap(); + let c = extract(&p).unwrap(); assert_eq!(c, "Hello World!"); std::fs::remove_file(&p).ok(); } @@ -75,8 +126,8 @@ mod tests { // `\'e9` is the RTF hex escape for an e-acute: the literal stays 7-bit ASCII. let body = br"{\rtf1\ansi caf\'e9 at noon}"; let p = tmp("agree", body); - let from_disk = RtfExtractor.extract(&p).unwrap(); - let from_head = RtfExtractor.extract_from_head(&p, body).unwrap().unwrap(); + let from_disk = extract(&p).unwrap(); + let from_head = extract_from_head(&p, body).unwrap().unwrap(); assert_eq!(from_disk, from_head); assert!(from_disk.contains("café"), "{:?}", from_disk); std::fs::remove_file(&p).ok(); @@ -101,6 +152,7 @@ mod tests { Some("application/rtf"), &crate::extract::Registry::default_set(), &crate::config::Config::default(), + &mut scratch(), ); let text = match &outcome { crate::file_handling::ContentOutcome::Done { text } => text.clone(), @@ -116,7 +168,7 @@ mod tests { ); // The head path, through the registry, where containment for other panics lives. - let head = crate::extract::Registry::default_set().extract_complete_head( + let head = crate::extract::Registry::default_set().extract_head_to_string( &p, "application/rtf", body, @@ -137,7 +189,7 @@ mod tests { fn paragraph_breaks_reach_the_text() { let body = br"{\rtf1\ansi First paragraph.\par Second paragraph.\par}"; let p = tmp("par", body); - let text = RtfExtractor.extract(&p).unwrap(); + let text = extract(&p).unwrap(); assert!( text.contains("First paragraph.\nSecond paragraph."), "paragraphs ran together: {text:?}" @@ -148,7 +200,7 @@ mod tests { #[test] fn malformed_input_errors_and_names_the_file() { let p = tmp("broken", br"{\rtf1 truncated"); - let err = RtfExtractor.extract(&p).unwrap_err(); + let err = extract(&p).unwrap_err(); assert!( err.contains(&p.display().to_string()), "must name the file: {}", @@ -171,12 +223,12 @@ mod tests { let body = vec![b'x'; 4096]; let p = tmp("cap", &body); assert_eq!( - read_capped(&p, 100).unwrap().len(), + read_capped_to_vec(&p, 100).unwrap().len(), 100, "read past the cap" ); assert_eq!( - read_capped(&p, MAX_READ as u64).unwrap().len(), + read_capped_to_vec(&p, max_read()).unwrap().len(), 4096, "a file under the cap must be read whole" ); @@ -186,7 +238,7 @@ mod tests { #[test] fn a_missing_file_is_an_error_naming_it() { let p = crate::testutil::scratch_dir("rtf-missing").join("nope.rtf"); - let err = read_capped(&p, MAX_READ as u64).unwrap_err(); + let err = read_capped_to_vec(&p, max_read()).unwrap_err(); assert!(err.contains(&p.display().to_string()), "{err}"); } } diff --git a/crates/quicksearch-core/src/file_handling/batch.rs b/crates/quicksearch-core/src/file_handling/batch.rs index 7d99990..6428d59 100644 --- a/crates/quicksearch-core/src/file_handling/batch.rs +++ b/crates/quicksearch-core/src/file_handling/batch.rs @@ -10,33 +10,60 @@ use super::*; use crate::config::Config; use crate::db::repo::{self}; -/// The compressed sidecar for one row, or `None` where there is none to -/// write. `Err` is kept per row rather than failing the batch. -type Body = Result>, String>; +/// One writer's compressed sidecars: a batch's blobs end to end in `arena`, +/// with `slots[i]` saying where row `i`'s is — or that it has none, or that +/// its compression failed (kept per row rather than failing the batch). +/// +/// Everything here is reused across chunks. The encoder because building a +/// zstd context per chunk is wasted CPU (`benches/index.rs`, `zstd_encode`); +/// the arena because a `Vec` per row was one allocation per indexed +/// document, and the writer sees every one of them. +struct Bodies { + enc: repo::DocEncoder, + arena: Vec, + slots: Vec>, String>>, +} -/// Compress a batch's bodies through one context, before the caller takes -/// the connection — the lock covers only the SQL, and one reused -/// [`repo::DocEncoder`] cuts compression ~4.7x (`benches/index.rs`). -fn compress_bodies<'a>( - texts: impl Iterator>, - config: &Config, -) -> Result, String> { - let mut enc = repo::DocEncoder::new()?; - Ok(texts - .map(|text| match text { - Some(t) if config.processing.store_text_for_snippets && !t.is_empty() => { - enc.encode(t).map(Some) - } - _ => Ok(None), +impl Bodies { + fn new() -> Result { + Ok(Bodies { + enc: repo::DocEncoder::new()?, + arena: Vec::new(), + slots: Vec::new(), }) - .collect()) + } + + /// Compress one chunk's bodies, **before the caller takes the + /// connection**: the lock covers only the SQL. + fn fill<'a>(&mut self, texts: impl Iterator>, config: &Config) { + self.arena.clear(); + self.slots.clear(); + for text in texts { + let slot = match text { + Some(t) if config.processing.store_text_for_snippets && !t.is_empty() => { + self.enc.encode_into(t, &mut self.arena).map(Some) + } + _ => Ok(None), + }; + self.slots.push(slot); + } + } + + /// Row `i`'s blob, or why there is none. + fn get(&self, i: usize) -> Result, &str> { + match &self.slots[i] { + Ok(Some(at)) => Ok(Some(&self.arena[at.clone()])), + Ok(None) => Ok(None), + Err(e) => Err(e), + } + } } /// The sidecar blob for row `i`, or a logged skip if its compression failed. macro_rules! body_or_skip { ($bodies:expr, $i:expr, $what:expr) => { - match &$bodies[$i] { - Ok(b) => b.as_deref(), + match $bodies.get($i) { + Ok(b) => b, Err(e) => { crate::log_warn!("compress text for {}: {}", $what, e); continue; @@ -60,13 +87,15 @@ fn write_prepared_records( chunk_size: usize, write_row: impl Fn(&rusqlite::Transaction<'_>, &OwnedNewFile) -> Result, String>, ) -> Result<(), String> { + // One set of buffers for every chunk this call writes. + let mut bodies = Bodies::new()?; for batch in records.chunks(chunk_size) { if stop_flag.load(Ordering::Relaxed) { return Ok(()); } - // Outside the lock — see `compress_bodies`. - let bodies = compress_bodies(batch.iter().map(|r| r.inline_text.as_deref()), config)?; + // Outside the lock — see `Bodies::fill`. + bodies.fill(batch.iter().map(|r| r.inline_text.as_deref()), config); let conn = crate::lock_ok(conn_mutex); let tx = conn .unchecked_transaction() @@ -336,17 +365,19 @@ pub fn store_extracted( deadline: std::time::Instant, ) -> Result { let mut done = Stored::default(); + // One set of buffers for every chunk this turn writes. + let mut bodies = Bodies::new()?; for chunk in rows.chunks(STORE_CHUNK) { if stop_flag.load(Ordering::Relaxed) { break; } - // Outside the lock — see `compress_bodies`. - let bodies = compress_bodies( + // Outside the lock — see `Bodies::fill`. + bodies.fill( chunk .iter() .map(|r| crate::file_handling::outcome_body(&r.outcome)), config, - )?; + ); let conn = crate::lock_ok(conn_mutex); let tx = conn .unchecked_transaction() @@ -355,14 +386,12 @@ pub fn store_extracted( for (i, row) in chunk.iter().enumerate() { // Counted before anything can skip it: a failed row still leaves. done.consumed += 1; - match &bodies[i] { - Err(e) => crate::log_warn!("compress text for {}: {}", row.name, e), - Ok(zstd) => { - match store_content_outcome(&tx, row.file_id, &row.outcome, zstd.as_deref()) { - Ok(()) => done.written += 1, - Err(e) => crate::log_warn!("content indexing for {}: {}", row.name, e), - } - } + match bodies.get(i) { + Err(e) => crate::log_warn!("compress text for {}: {}", row.name(), e), + Ok(zstd) => match store_content_outcome(&tx, row.file_id, &row.outcome, zstd) { + Ok(()) => done.written += 1, + Err(e) => crate::log_warn!("content indexing for {}: {}", row.name(), e), + }, } if stop_flag.load(Ordering::Relaxed) || std::time::Instant::now() >= deadline { cut = true; diff --git a/crates/quicksearch-core/src/file_handling/count_and_extract_tests.rs b/crates/quicksearch-core/src/file_handling/count_and_extract_tests.rs index a773a4b..81f8cdb 100644 --- a/crates/quicksearch-core/src/file_handling/count_and_extract_tests.rs +++ b/crates/quicksearch-core/src/file_handling/count_and_extract_tests.rs @@ -1,7 +1,7 @@ use std::sync::{Arc, Mutex}; use crate::config::Config; -use crate::extract::Registry; +use crate::extract::{Registry, Scratch}; use crate::mime::guess_mime_from_head; use super::*; @@ -137,8 +137,8 @@ fn content_extractable_is_decide_contents_not_applicable() { let p = root.join(name); let path = p.to_str().unwrap(); let mime = guess_mime_from_head(&p, body); - let claimed = content_extractable(&p, mime.as_deref(), &cfg, ®istry); - let outcome = decide_content(path, mime.as_deref(), ®istry, &cfg); + let claimed = content_extractable(&p, mime, &cfg, ®istry); + let outcome = decide_content(path, mime, ®istry, &cfg, &mut Scratch::new(&cfg)); assert_eq!( claimed, outcome != ContentOutcome::NotApplicable, @@ -163,7 +163,7 @@ fn prepare_file_record_marks_only_claimable_files() { let needs = |cfg: &Config, name: &str| -> bool { let p = root.join(name); let meta = std::fs::metadata(&p).unwrap(); - prepare_file_record(p.to_str().unwrap(), &meta, cfg, ®istry) + prepare_file_record(p.to_str().unwrap(), &meta, cfg, ®istry, &mut Scratch::new(&Config::default())) .expect("regular file") .needs_content }; @@ -231,7 +231,7 @@ fn extract_scope_counts_only_files_an_extractor_claims() { .map(|name| { let p = root.join(name); let meta = std::fs::metadata(&p).unwrap(); - prepare_file_record(p.to_str().unwrap(), &meta, &config, ®istry) + prepare_file_record(p.to_str().unwrap(), &meta, &config, ®istry, &mut Scratch::new(&Config::default())) .expect("regular file") }) .collect(); diff --git a/crates/quicksearch-core/src/file_handling/records.rs b/crates/quicksearch-core/src/file_handling/records.rs index 6d04cda..7805d95 100644 --- a/crates/quicksearch-core/src/file_handling/records.rs +++ b/crates/quicksearch-core/src/file_handling/records.rs @@ -13,7 +13,7 @@ use sha2::{Digest, Sha256}; use super::*; use crate::config::Config; use crate::db::repo::{self, NewFile}; -use crate::extract::Registry; +use crate::extract::{Registry, Scratch}; use crate::mime::{guess_mime_from_head, mime_to_type, FileType}; /// One directory's indexed files, as `name -> mtime`. @@ -43,9 +43,13 @@ pub fn classify_by_mtime(stored: Option, mtime: u64) -> FileIndexAction { } /// Truncate to at most `max_bytes`, backing up to a UTF-8 char boundary. -fn safe_truncate_string(s: &str, max_bytes: usize) -> String { +/// +/// In place: this runs on documents up to `maximum_text_size`, and building +/// the prefix as a second `String` allocated and copied the whole thing only +/// to drop the original a line later. +pub(crate) fn safe_truncate(s: &mut String, max_bytes: usize) { if s.len() <= max_bytes { - return s.to_string(); + return; } let mut end = max_bytes; @@ -53,29 +57,40 @@ fn safe_truncate_string(s: &str, max_bytes: usize) -> String { end -= 1; } - s[..end].to_string() + s.truncate(end); } -/// Identify a file as `sha256(size || first hash_length bytes)`, returning -/// the head bytes alongside the digest for MIME sniffing. The cost is a -/// collision class — same-size files with identical heads read as duplicates +/// A SHA-256 digest. An array, not a `Vec`: it is a fixed 32 bytes, it is +/// produced once per new or changed file, and a heap allocation apiece is +/// per-file allocator traffic for something that fits in a register pair's +/// worth of stack. +pub type FileHash = [u8; 32]; + +/// Identify a file as `sha256(size || first hash_length bytes)`, leaving the +/// head bytes in `head` for MIME sniffing. The cost is a collision class — +/// same-size files with identical heads read as duplicates /// (`examples/hashprobe.rs` has the study); `crate::verify` is the way out. +/// +/// `head` is the caller's buffer, reused file after file; it is resized to +/// the bytes actually read and its previous contents are discarded. pub fn get_file_hash( size: u64, path: &Path, hash_length: usize, -) -> Result<(Vec, Vec), std::io::Error> { + head: &mut Vec, +) -> Result { // The caller's `is_file()` came from a `stat` taken before this open; a // FIFO renamed over the name in between would block this walk worker // forever and park the pool behind it. See `platform::open_regular_file`. let mut f: File = crate::platform::open_regular_file(path)?; - let mut head = vec![0u8; size.min(hash_length as u64) as usize]; - f.read_exact(&mut head)?; + head.clear(); + head.resize(size.min(hash_length as u64) as usize, 0); + f.read_exact(head)?; let mut hasher = Sha256::new(); hasher.update(size.to_le_bytes()); - hasher.update(&head); - Ok((hasher.finalize().to_vec(), head)) + hasher.update(&*head); + Ok(hasher.finalize().into()) } /// FTS5's automerge threshold (2..=16). 16 measured ~19% faster cold indexing @@ -102,8 +117,12 @@ pub fn fts_set_automerge(conn: &Connection, segments: u8) { const WRITE_CRISISMERGE: u8 = 32; /// Apply the write-side FTS5 settings, before a run starts writing. -/// `pgsz` was swept here too and **rejected**: more bytes written for an -/// index the same size. +/// +/// `pgsz` is deliberately absent: it is not a per-run setting. Sweeping it +/// *unencrypted* only ever wrote more bytes for an index the same size, so the +/// default 4050 stands there. Keyed is the opposite — SQLCipher's page reserve +/// makes 4050 a cliff — and that case is handled once at schema creation; see +/// [`crate::db::schema::FTS_PGSZ_ENCRYPTED`]. pub fn fts_begin_bulk_write(conn: &Connection) { fts_set_automerge(conn, WRITE_AUTOMERGE); if let Err(e) = conn.execute( @@ -114,16 +133,42 @@ pub fn fts_begin_bulk_write(conn: &Connection) { } } +/// Output leaf pages [`fts_finalize_after_text_indexing`] may write. +/// +/// FTS5's own `FTS5_OPT_WORK_UNIT`, which is the budget it gives one step of a +/// real `'optimize'`. **Unswept** — the other two constants here carry measured +/// tables and this one does not yet; it is a starting point chosen to match +/// SQLite's own unit of merge work, not a tuned figure. +/// +/// The budget is not a hard ceiling. `fts5IndexMergeLevel` only tests it at a +/// term boundary, so one term with a very long doclist writes past it; that is +/// the shape to expect if this ever needs raising or lowering. +const FINALIZE_MERGE_PAGES: i64 = 1000; + /// Merge FTS5 segments once a run has finished writing — this is what /// reclaims the tombstones a `contentless_delete` table accumulates. -/// Deliberately **not** `'optimize'`: measured, far more time and writes for -/// no search gain. Best-effort; an unconsolidated index is still correct. +/// Best-effort; an unconsolidated index is still correct. +/// +/// **The sign of the argument picks a different algorithm**, which is worth +/// spelling out because reading it as a plain page budget cost a release. From +/// `sqlite3Fts5IndexMerge` in the amalgamation: +/// +/// - **Positive** `N`: ordinary consolidation. Merges any level holding at +/// least `usermerge` segments (FTS5's default 4 — we set `automerge` and +/// `crisismerge`, never `usermerge`), for up to `N` output leaf pages. +/// - **Negative** `N`: `fts5IndexOptimizeStruct` with `nMin` forced to 1 — +/// that is `'optimize'`, hoisting every segment in the table into a single +/// level, merely rate-limited to `|N|` pages per call. It is built to be +/// called in a loop until `changes() == 0`. Called *once*, it leaves the +/// structure permanently mid-optimize, and that persists in `%_data`. +/// +/// So this takes the positive form. Tombstone reclamation survives the switch: +/// `fts5IndexMerge` falls through to `fts5IndexFindDeleteMerge` when no level +/// has `nMin` segments, and that path keys off `deletemerge` whatever the sign. pub fn fts_finalize_after_text_indexing(conn: &Connection) { - // A negative page budget means "keep merging until nothing is left worth - // merging". if let Err(e) = conn.execute( - "INSERT INTO searchabletext(searchabletext, rank) VALUES('merge', -16)", - [], + "INSERT INTO searchabletext(searchabletext, rank) VALUES('merge', ?1)", + [FINALIZE_MERGE_PAGES], ) { crate::log_warn!("FTS merge failed (non-fatal): {}", e); } @@ -137,12 +182,14 @@ pub struct OwnedNewFile { pub parent: String, pub size: u64, pub mtime: u64, - pub mime: Option, + /// Borrowed from the static tables [`guess_mime_from_head`] answers out + /// of; nothing here ever owns a MIME string. + pub mime: Option<&'static str>, pub ftype: FileType, /// `None` only for a dehydrated cloud placeholder. Stored as SQL NULL, /// which keeps such files out of duplicate detection: an empty or zero /// hash would make every one of them look identical. - pub hash: Option>, + pub hash: Option, /// Text extracted from the head bytes during the walk, for files small /// enough that the head *was* the whole file. `Some` means the content /// pass never has to open this file; `None` leaves it pending. @@ -162,9 +209,9 @@ impl OwnedNewFile { parent: &self.parent, size: self.size, mtime: self.mtime, - mime: self.mime.as_deref(), + mime: self.mime, ftype: self.ftype, - hash: self.hash.as_deref(), + hash: self.hash.as_ref().map(|h| &h[..]), needs_content: self.needs_content, } } @@ -192,11 +239,14 @@ pub fn hash_failure_counts() -> (u64, u64) { /// that only survived `to_string_lossy` does not qualify: the lossy spelling /// of one name is the real name of another, so it would hash and index the /// wrong file. +/// `scratch` holds the head buffer this reads into, reused across every file +/// a walk worker handles. pub fn prepare_file_record( path: &str, meta: &std::fs::Metadata, config: &Config, registry: &Registry, + scratch: &mut Scratch, ) -> Option { if !meta.is_file() { return None; @@ -213,11 +263,19 @@ pub fn prepare_file_record( // even the first byte would block on downloading the whole file. let dehydrated = crate::platform::is_cloud_placeholder(meta); - let (hash, head) = if dehydrated { - (None, Vec::new()) + let hash = if dehydrated { + // No head either: an empty buffer sniffs to the extension's answer, + // which is all a placeholder can be classified by. + scratch.head_buffer().clear(); + None } else { - match get_file_hash(size, Path::new(path), config.processing.hash_length) { - Ok((hash, head)) => (Some(hash), head), + match get_file_hash( + size, + Path::new(path), + config.processing.hash_length, + scratch.head_buffer(), + ) { + Ok(hash) => Some(hash), Err(e) => { // Throttled: on Windows a file another process holds open // fails here as a matter of course. @@ -231,29 +289,29 @@ pub fn prepare_file_record( let (parent, name) = split_db_path(path)?; let (parent, name) = (parent.to_string(), name.to_string()); - let mime = guess_mime_from_head(Path::new(path), &head); - let ftype = mime.as_deref().map(mime_to_type).unwrap_or(FileType::EMPTY); + let head = scratch.head(); + let mime = guess_mime_from_head(Path::new(path), head); + let ftype = mime.map(mime_to_type).unwrap_or(FileType::EMPTY); let needs_content = !dehydrated && size <= config.processing.maximum_text_file_size - && content_extractable(Path::new(path), mime.as_deref(), config, registry); + && content_extractable(Path::new(path), mime, config, registry); // When the head is the whole file, an extractor that works from bytes can // finish the job now; otherwise the file stays pending. - let inline_text = mime.as_deref().filter(|_| needs_content).and_then(|m| { + let inline_text = mime.filter(|_| needs_content).and_then(|m| { // Size 0 is excluded: procfs, sysfs and some FUSE mounts report it // for files that do have content, and inlining would store empty // text for them. if size == 0 || size > config.processing.hash_length as u64 { return None; } + let mut text = String::new(); // A panicking parser arrives here as `Some(Err(..))` — contained by // the registry, which is what keeps a walk worker alive. - match registry.extract_complete_head(Path::new(path), m, &head) { - Some(Ok(mut text)) => { - if text.len() > config.processing.maximum_text_size { - text = safe_truncate_string(&text, config.processing.maximum_text_size); - } + match registry.extract_complete_head(Path::new(path), m, head, &mut text) { + Some(Ok(())) => { + safe_truncate(&mut text, config.processing.maximum_text_size); Some(text) } // Recording a failure needs a file id the walk does not have; @@ -276,6 +334,7 @@ pub fn prepare_file_record( } /// [`prepare_file_record`] for a path not yet resolved — the watcher path. +/// One file at a time, so it owns the scratch rather than being handed one. pub fn prepare_file_record_from_path( path: &Path, config: &Config, @@ -287,7 +346,8 @@ pub fn prepare_file_record_from_path( } let db_path = path_to_db_string(&canonical); let meta = std::fs::metadata(&canonical).ok()?; - prepare_file_record(&db_path, &meta, config, registry) + let mut scratch = Scratch::new(config); + prepare_file_record(&db_path, &meta, config, registry, &mut scratch) } /// Extract content for one file and record the outcome on its row. `mime` is @@ -301,7 +361,10 @@ pub fn extract_and_store( registry: &Registry, config: &Config, ) -> Result<(), String> { - let outcome = decide_content(path, mime, registry, config); + // One file, called from the watcher and the CLI: a scratch per call is + // the right scope — there is no loop for a reused one to amortize over. + let mut scratch = Scratch::new(config); + let outcome = decide_content(path, mime, registry, config, &mut scratch); let zstd = match outcome_body(&outcome) { Some(text) => repo::encode_one(text, config.processing.store_text_for_snippets)?, None => None, @@ -336,30 +399,35 @@ pub fn content_extractable( /// Read `path` and decide what its content row should say. No database /// access, no locks held — this is the expensive half. +/// +/// `scratch` is the calling worker's, reused for every file it handles; the +/// text is a fresh `String` because it goes on to cross a channel. pub fn decide_content( path: &str, mime: Option<&str>, registry: &Registry, config: &Config, + scratch: &mut Scratch, ) -> ContentOutcome { let p = Path::new(path); if !content_extractable(p, mime, config, registry) { return ContentOutcome::NotApplicable; } + let mut text = String::new(); // A panicking parser is contained by the registry and arrives as `Err`, // which becomes this row's recorded failure reason, not a dead worker. let result = match mime { - Some(m) => registry.extract(p, m), - None => Ok(None), + Some(m) => registry.extract(p, m, &mut text, scratch), + None => Ok(false), }; match result { - Ok(Some(mut text)) => { - if text.len() > config.processing.maximum_text_size { - text = safe_truncate_string(&text, config.processing.maximum_text_size); - } + Ok(true) => { + // Extractors stop at the limit themselves; this is the backstop + // for the ones that can overshoot by a run or a slide. + safe_truncate(&mut text, config.processing.maximum_text_size); ContentOutcome::Done { text } } - Ok(None) => ContentOutcome::NotApplicable, + Ok(false) => ContentOutcome::NotApplicable, Err(reason) => ContentOutcome::Failed(reason), } } diff --git a/crates/quicksearch-core/src/file_handling/tests.rs b/crates/quicksearch-core/src/file_handling/tests.rs index f7ec6de..8d2b500 100644 --- a/crates/quicksearch-core/src/file_handling/tests.rs +++ b/crates/quicksearch-core/src/file_handling/tests.rs @@ -386,9 +386,8 @@ fn hash_covers_size_and_head_only() { std::fs::write(&c, [b"DIFF".as_slice(), &[0u8; 64], b"AAAA"].concat()).unwrap(); let h = |p: &Path| { - get_file_hash(std::fs::metadata(p).unwrap().len(), p, 8) - .unwrap() - .0 + let mut head = Vec::new(); + get_file_hash(std::fs::metadata(p).unwrap().len(), p, 8, &mut head).unwrap() }; assert_eq!(h(&a), h(&b), "tail differences are invisible by design"); assert_ne!(h(&a), h(&c), "head differences are caught"); @@ -398,10 +397,11 @@ fn hash_covers_size_and_head_only() { std::fs::write(&short, b"HEAD").unwrap(); assert_ne!(h(&a), h(&short)); - let (_, head) = get_file_hash(72, &a, 8).unwrap(); + let mut head = Vec::new(); + get_file_hash(72, &a, 8, &mut head).unwrap(); assert_eq!(head, b"HEAD\0\0\0\0", "exactly hash_length bytes"); - let (_, head) = get_file_hash(4, &short, 8).unwrap(); - assert_eq!(head, b"HEAD", "a short file hashes whole"); + get_file_hash(4, &short, 8, &mut head).unwrap(); + assert_eq!(head, b"HEAD", "a short file hashes whole, and the buffer is reused"); std::fs::remove_dir_all(&root).ok(); } diff --git a/crates/quicksearch-core/src/incremental.rs b/crates/quicksearch-core/src/incremental.rs index 56f919a..b7c11a9 100644 --- a/crates/quicksearch-core/src/incremental.rs +++ b/crates/quicksearch-core/src/incremental.rs @@ -185,7 +185,7 @@ fn upsert_file( &tx, file_id, &rec.path(), - rec.mime.as_deref(), + rec.mime, registry, config, )?; diff --git a/crates/quicksearch-core/src/indexing/mod.rs b/crates/quicksearch-core/src/indexing/mod.rs index 21e9e89..84d35b9 100644 --- a/crates/quicksearch-core/src/indexing/mod.rs +++ b/crates/quicksearch-core/src/indexing/mod.rs @@ -18,7 +18,9 @@ mod progress; #[cfg(test)] mod tests; -pub use progress::{overall_progress, OverallProgress, ReconcileProgress, RootPhase, RootProgress}; +pub use progress::{ + overall_progress, MaintenanceStep, OverallProgress, ReconcileProgress, RootPhase, RootProgress, +}; /// What a run is doing before its first file is walked. #[derive(Debug, Clone, PartialEq, Eq)] @@ -29,6 +31,9 @@ pub enum PrepStep { OpeningIndex, /// Re-testing stored rows against a configuration that changed since the last run. Reconciling(ReconcileProgress), + /// The prologue's remaining database work, once any reconcile has ended: + /// stamping the config, retrying failed files, reading the stored counts. + Starting, } #[derive(Debug, Clone)] @@ -43,6 +48,10 @@ pub enum IndexingStatus { Running { start_time: Instant, roots: Vec, + /// The index upkeep the writer is inside, if any. Run-wide, not + /// per-root: while it is set no file is moving and every counter in + /// `roots` is the last one published before the step began. + maintenance: Option, }, Stopping, /// Compacting and re-analysing the index after a run. Holds the database: @@ -409,13 +418,7 @@ impl IndexingService { } }; let armed = db::InterruptGuard::arm(interrupt, &conn); - - let dir = std::path::Path::new(db_path) - .parent() - .map(|p| p.to_string_lossy().into_owned()) - .unwrap_or_default(); - let outcome = crate::db::repo::maintain(&conn, &dir); - + let outcome = crate::db::repo::maintain(&conn, db_path); drop(armed); match outcome { Ok(true) => crate::log_info!("optimized the index and reclaimed unused space"), diff --git a/crates/quicksearch-core/src/indexing/pipeline.rs b/crates/quicksearch-core/src/indexing/pipeline.rs index 24a0749..27044e9 100644 --- a/crates/quicksearch-core/src/indexing/pipeline.rs +++ b/crates/quicksearch-core/src/indexing/pipeline.rs @@ -197,6 +197,22 @@ mod census { true } + /// One tail step's cost, in the two numbers the end-of-run WAL bug was + /// about: how long it held the writer and what it left in the log. + /// + /// Autocheckpoint is off across the tail, so a per-step reading is the + /// only way to say which step is responsible for the log's peak. + /// `db::repo::maintain` emits the same shape for the half that runs after + /// this connection has gone. + pub(super) fn tail(step: &str, db_path: &str, started: Instant) { + crate::log_info!( + "tail t={:.1}s wal {} after {}", + started.elapsed().as_secs_f64(), + mib(super::wal_len(&format!("{}-wal", db_path))), + step + ); + } + /// One line per structure group: the log collapses embedded newlines, so /// a multi-line report would arrive as one unreadable line. pub(super) fn report(cx: &RunCx<'_>, pipelines: &[RootPipeline], started: Instant) { @@ -231,7 +247,7 @@ mod census { .iter() .map(|r| { (crate::file_handling::outcome_body(&r.outcome).map_or(0, str::len) - + r.name.len()) as u64 + + r.name().len()) as u64 }) .sum(); crate::log_info!( @@ -456,6 +472,7 @@ impl RootPipeline { // Counting the range is the pass's job, on its own connection — // on the writer it is seconds of every other walk standing still. { + let _maintaining = cx.maintaining(MaintenanceStep::SizeLimit); let conn = crate::lock_ok(&cx.conn_mutex); mark_oversize_pending_na(&conn, &cursor, cx.config)?; } @@ -513,7 +530,7 @@ impl RootPipeline { let stored = store_extracted(&cx.conn_mutex, ready, cx.stop_flag, cx.config, deadline)?; if stored.consumed > 0 { // The last row *written*, not the last fetched. - *current_file = Some(ready[stored.consumed - 1].name.clone()); + *current_file = Some(ready[stored.consumed - 1].name().to_string()); } ready.drain(..stored.consumed); *written += stored.written; @@ -549,6 +566,8 @@ pub(super) struct RunCx<'a> { pub(super) config: &'a Config, pub(super) db_path: &'a str, pub(super) stop_flag: &'a Arc, + /// Where an upkeep step announces itself; see [`RunCx::maintaining`]. + pub(super) status: Arc>, /// Walk workers use it to finish small text files without the content pass. pub(super) registry: Arc, pub(super) quantum: usize, @@ -568,12 +587,14 @@ impl<'a> RunCx<'a> { config: &'a Config, db_path: &'a str, stop_flag: &'a Arc, + status: Arc>, ) -> RunCx<'a> { RunCx { conn_mutex, config, db_path, stop_flag, + status, registry: Arc::new(Registry::default_set()), quantum: config.processing.batch_size.max(1), slice: Duration::from_millis(config.processing.writer_turn_slice_ms), @@ -582,10 +603,45 @@ impl<'a> RunCx<'a> { stale_cleanup_ok: true, } } + + /// Mark the run as inside `step` for as long as the returned guard lives: + /// the writer is doing index upkeep, not file work, and the last per-file + /// snapshot would otherwise sit frozen and read as a hang. + /// + /// An annotation on the snapshot already published, so the counters keep + /// their last true values, `Stopping` is left alone, and the guard borrows + /// nothing from `cx` — every caller holds it mutably for the wrapped work. + pub(super) fn maintaining(&self, step: MaintenanceStep) -> MaintenanceGuard { + set_maintenance(&self.status, Some(step)); + MaintenanceGuard { + status: self.status.clone(), + } + } +} + +/// Clears the step its [`RunCx::maintaining`] set. +pub(super) struct MaintenanceGuard { + status: Arc>, +} + +impl Drop for MaintenanceGuard { + fn drop(&mut self) { + set_maintenance(&self.status, None); + } +} + +/// Annotate the published run, if there still is one: a status that has moved +/// on to `Stopping` is the command thread's, and a step is not news worth +/// resurrecting a run for. +fn set_maintenance(status: &Arc>, step: Option) { + if let IndexingStatus::Running { maintenance, .. } = &mut *crate::lock_ok(status) { + *maintenance = step; + } } /// Publish a status snapshot. Never clobbers Stopping — the command thread -/// owns that transition. +/// owns that transition. Always clears any upkeep step: fresh per-file +/// figures mean the writer is back on files. fn publish_status( status: &Arc>, run_start: Instant, @@ -597,6 +653,7 @@ fn publish_status( *g = IndexingStatus::Running { start_time: run_start, roots, + maintenance: None, }; } } @@ -674,11 +731,15 @@ fn build_pipeline( /// Reconcile deletions once every walk has ended — globally, because a file /// may be reachable through more than one root's symlinks. A no-op for a /// stopped or abnormally terminated run. -fn cleanup_stale(pipelines: &mut [RootPipeline], cx: &mut RunCx<'_>) -> Result<(), String> { +fn cleanup_stale(pipelines: &[RootPipeline], cx: &mut RunCx<'_>) -> Result<(), String> { let stopped = cx.stop_flag.load(Ordering::Relaxed); if !cx.stale_cleanup_ok || stopped { return Ok(()); } + // The whole pass, not just the deleting: the sweep below reads every + // stored parent under every root, and the merge that ends the deletion is + // minutes of writer time on a big index. + let _maintaining = cx.maintaining(MaintenanceStep::RemovingStale); for p in pipelines.iter() { sweep_unvisited_parents( &cx.conn_mutex, @@ -715,9 +776,6 @@ fn cleanup_stale(pipelines: &mut [RootPipeline], cx: &mut RunCx<'_>) -> Result<( } } if !stale_paths.is_empty() { - if let Some(first) = pipelines.first_mut() { - first.current_file = Some("Removing stale index entries…".to_string()); - } let started = Instant::now(); let stale_deleted = cleanup_stale_index_entries( &cx.conn_mutex, @@ -778,6 +836,9 @@ impl IndexingService { { return Ok(()); } + // Everything from here to the first walk is database work of its own; + // leaving the step on `Reconciling` reads as a reconcile that hung. + Self::set_prep_step(status, PrepStep::Starting); Self::update_config(&conn, config, &roots)?; // Failed files are retried once per run; only a retry can tell. @@ -800,8 +861,14 @@ impl IndexingService { // Autocheckpoint off for the run: it can never reset the log while a // reader per root is live, so it copies pages back perpetually at full // price. Safe here and nowhere else — this writer bounds its own log - // (`wal_cap_for_volume`, the forced checkpoint below, the optimize - // pass); a writer without all three must keep the automatic one. + // (`wal_cap_for_volume`, the forced checkpoint below, and the pair + // bracketing the tail once the readers are dropped); a writer without + // all of them must keep the automatic one. + // + // The tail pair is not optional and was once missing. Deferring to "the + // optimize pass checkpoints at the end" left everything after the loop — + // the FTS merge above all — piling onto the log unbounded, and handed + // `repo::maintain` a full one to run a VACUUM on top of. if let Err(e) = conn.execute_batch("PRAGMA wal_autocheckpoint = 0;") { crate::log_warn!("could not disable autocheckpoint (non-fatal): {}", e); } @@ -815,7 +882,7 @@ impl IndexingService { let count_cancel = Arc::new(AtomicBool::new(false)); let _count_guard = CancelOnDrop(count_cancel.clone()); - let mut cx = RunCx::new(conn_mutex, config, db_path, stop_flag); + let mut cx = RunCx::new(conn_mutex, config, db_path, stop_flag, status.clone()); let stored_counts: Vec> = { let conn = crate::lock_ok(&cx.conn_mutex); @@ -887,7 +954,7 @@ impl IndexingService { if !cleanup_done && pipelines.iter().all(|p| p.phase != RootPhase::Walking) { cleanup_done = true; - cleanup_stale(&mut pipelines, &mut cx)?; + cleanup_stale(&pipelines, &mut cx)?; progressed = true; } @@ -907,6 +974,7 @@ impl IndexingService { && wal_len(&wal_path) >= checkpoint_at { { + let _maintaining = cx.maintaining(MaintenanceStep::Checkpoint); let conn = crate::lock_ok(&cx.conn_mutex); if let Err(e) = crate::db::repo::checkpoint_truncate(&conn) { crate::log_warn!("{}", e); @@ -957,6 +1025,28 @@ impl IndexingService { } } + // The tail's readers, released before any of its writing. Every + // per-root walk prefetcher and content feeder lives in `pipelines`, and + // a read mark held by any one of them turns a TRUNCATE checkpoint into + // a silent PASSIVE one that truncates nothing — see + // [`repo::checkpoint_truncate`]. Nothing below reads `pipelines`: the + // stale cleanup and every status publish are inside the loop, and the + // counts iterate `roots`. + #[cfg(feature = "probe")] + let tail_started = Instant::now(); + drop(pipelines); + #[cfg(feature = "probe")] + census::tail("dropping the readers", db_path, tail_started); + + // First half of the pair that bounds the tail. It lands the run's own + // writing, so whatever the log holds from here is the tail's alone — + // which is what makes the FTS merge's cost legible rather than mixed + // in with a run's worth of log. On the stopped path too: that is + // exactly when the log is largest. + checkpoint_tail(&cx, interrupt); + #[cfg(feature = "probe")] + census::tail("tail checkpoint", db_path, tail_started); + if aborted { // Nothing is landed on the way out — "a stopped run promises // nothing"; the next run finds it all again. No stale cleanup @@ -978,33 +1068,65 @@ impl IndexingService { ); { + let _maintaining = cx.maintaining(MaintenanceStep::MergingText); let conn = crate::lock_ok(&cx.conn_mutex); fts_finalize_after_text_indexing(&conn); } + #[cfg(feature = "probe")] + census::tail("the FTS merge", db_path, tail_started); - // An absent stamp reads as "never indexed" and `periodic_due` starts - // another full run on the very next tick. - let now = crate::log::now_unix(); - let conn = crate::lock_ok(&cx.conn_mutex); - if let Err(e) = crate::db::repo::set_last_full_index(&conn, now) { - crate::log_warn!("{}", e); - } + { + // An absent stamp reads as "never indexed" and `periodic_due` + // starts another full run on the very next tick. + let now = crate::log::now_unix(); + let conn = crate::lock_ok(&cx.conn_mutex); + if let Err(e) = crate::db::repo::set_last_full_index(&conn, now) { + crate::log_warn!("{}", e); + } - // Per-root figures, while the pages are warm; under the interrupt - // guard because quitting should not wait out a per-root scan. - let _guard = db::InterruptGuard::arm(interrupt, &conn); - for root in &roots { - let range = ExtractCursor::for_root(root); - match repo::count_root(&conn, &range.lo, &range.hi) { - Ok(counts) => { - if let Err(e) = repo::set_root_counts(&conn, root, counts) { - crate::log_warn!("{}", e); + // Per-root figures, while the pages are warm; under the interrupt + // guard because quitting should not wait out a per-root scan. + let _maintaining = cx.maintaining(MaintenanceStep::RootCounts); + let _guard = db::InterruptGuard::arm(interrupt, &conn); + for root in &roots { + let range = ExtractCursor::for_root(root); + match repo::count_root(&conn, &range.lo, &range.hi) { + Ok(counts) => { + if let Err(e) = repo::set_root_counts(&conn, root, counts) { + crate::log_warn!("{}", e); + } } + Err(e) => crate::log_warn!("counts for {} unavailable: {}", root, e), } - Err(e) => crate::log_warn!("counts for {} unavailable: {}", root, e), } } + #[cfg(feature = "probe")] + census::tail("the per-root counts", db_path, tail_started); + + // Second half of the pair. `repo::maintain` runs next on its own + // connection and VACUUMs, whose copy-back pushes the whole database + // through the log — so it has to start from an empty one. Its own + // leading checkpoint cannot be relied on for that: it is best-effort + // and swallows the failure. + checkpoint_tail(&cx, interrupt); + #[cfg(feature = "probe")] + census::tail("tail checkpoint", db_path, tail_started); Ok(()) } } + +/// Land the log during the tail. Autocheckpoint is off for this connection +/// (see `run_indexing`), so between the writer loop and `repo::maintain` +/// nothing else will. +/// +/// Under the interrupt guard: a quit must not start waiting on a checkpoint's +/// lock, and an abandoned log is safe — the next run lands it. +fn checkpoint_tail(cx: &RunCx<'_>, interrupt: &db::InterruptSlot) { + let _maintaining = cx.maintaining(MaintenanceStep::Checkpoint); + let conn = crate::lock_ok(&cx.conn_mutex); + let _guard = db::InterruptGuard::arm(interrupt, &conn); + if let Err(e) = crate::db::repo::checkpoint_truncate(&conn) { + crate::log_warn!("{}", e); + } +} diff --git a/crates/quicksearch-core/src/indexing/progress.rs b/crates/quicksearch-core/src/indexing/progress.rs index 3001ce6..adf2011 100644 --- a/crates/quicksearch-core/src/indexing/progress.rs +++ b/crates/quicksearch-core/src/indexing/progress.rs @@ -11,6 +11,22 @@ pub enum RootPhase { Done, } +/// Index upkeep the writer stops file work to do. Every one of these blocks +/// the writer, so the per-root counters are frozen for its whole life. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MaintenanceStep { + /// Folding the write-ahead log back into the index file. + Checkpoint, + /// Deleting the rows of files that are no longer on disk. + RemovingStale, + /// Merging the full-text index's segments. + MergingText, + /// Re-reading each root's stored totals. + RootCounts, + /// Marking files above the size limit as having no text. + SizeLimit, +} + /// Progress for one indexing root; the GUI shows one row per root. #[derive(Debug, Clone)] pub struct RootProgress { diff --git a/crates/quicksearch-core/src/indexing/tests.rs b/crates/quicksearch-core/src/indexing/tests.rs index 52d1668..c8a043a 100644 --- a/crates/quicksearch-core/src/indexing/tests.rs +++ b/crates/quicksearch-core/src/indexing/tests.rs @@ -171,13 +171,16 @@ fn an_extracting_turn_lands_its_leftovers_one_slice_at_a_time() { ) .unwrap() .expect("unique path"); - ready.push(ExtractedRow { + ready.push(ExtractedRow::new( file_id, - name: format!("f{}.txt", i), - outcome: ContentOutcome::Done { + crate::db::repo::RowPath::new( + &crate::file_handling::dir_to_db_parent(&tree), + &format!("f{}.txt", i), + ), + ContentOutcome::Done { text: format!("sphinx of black quartz {}", i), }, - }); + )); } tx.commit().unwrap(); } @@ -224,7 +227,13 @@ fn an_extracting_turn_lands_its_leftovers_one_slice_at_a_time() { totals: None, current_file: None, }; - let mut cx = RunCx::new(conn_mutex.clone(), &config, &db_path, &stop); + let mut cx = RunCx::new( + conn_mutex.clone(), + &config, + &db_path, + &stop, + Arc::new(Mutex::new(IndexingStatus::Idle)), + ); cx.slice = Duration::ZERO; let mut turns = 0; @@ -276,8 +285,23 @@ fn an_extracting_turn_lands_its_leftovers_one_slice_at_a_time() { } fn run_with(config: &Config, db_path: &str, stop: &Arc) -> Result<(), String> { - IndexingService::run_indexing( + run_indexing_with( + config, + db_path, + stop, &Arc::new(Mutex::new(IndexingStatus::Idle)), + ) +} + +/// [`run_with`] for the tests that read the status the run leaves behind. +fn run_indexing_with( + config: &Config, + db_path: &str, + stop: &Arc, + status: &Arc>, +) -> Result<(), String> { + IndexingService::run_indexing( + status, &config.paths.indexing_paths, db_path, stop, @@ -472,6 +496,102 @@ fn an_interrupted_reconcile_records_nothing() { std::fs::remove_dir_all(&dir).ok(); } +/// A context with no run behind it: `maintaining` only ever touches the +/// status, never the database. +fn cx_for<'a>( + config: &'a Config, + stop: &'a Arc, + status: &Arc>, +) -> pipeline::RunCx<'a> { + let conn = rusqlite::Connection::open_in_memory().expect("in-memory database"); + pipeline::RunCx::new( + Arc::new(Mutex::new(conn)), + config, + "/nowhere", + stop, + status.clone(), + ) +} + +fn running_status(maintenance: Option) -> Arc> { + Arc::new(Mutex::new(IndexingStatus::Running { + start_time: Instant::now(), + roots: vec![progress(RootPhase::Extracting, 100, None)], + maintenance, + })) +} + +/// The counters freeze for the length of the step either way; the only +/// question is whether the status says so. +#[test] +fn an_upkeep_step_is_published_for_exactly_as_long_as_it_runs() { + let config = Config::default(); + let stop = Arc::new(AtomicBool::new(false)); + let status = running_status(None); + let cx = cx_for(&config, &stop, &status); + + let guard = cx.maintaining(MaintenanceStep::Checkpoint); + match &*crate::lock_ok(&status) { + IndexingStatus::Running { + roots, maintenance, .. + } => { + assert_eq!(*maintenance, Some(MaintenanceStep::Checkpoint)); + assert_eq!(roots.len(), 1, "the published snapshot was replaced"); + assert_eq!(roots[0].walked, 100, "the counters were rewritten"); + } + other => panic!("the run went missing: {:?}", other), + } + + drop(guard); + match &*crate::lock_ok(&status) { + IndexingStatus::Running { + roots, maintenance, .. + } => { + assert_eq!(*maintenance, None, "the step outlived its work"); + assert_eq!( + roots[0].walked, 100, + "the roots went missing on the way out" + ); + } + other => panic!("the run went missing: {:?}", other), + }; +} + +/// The command thread owns the `Stopping` transition — the rule +/// `publish_status` has always kept, and an upkeep step is no exception: +/// neither end of it may resurrect a run that has been told to stop. +#[test] +fn an_upkeep_step_never_clobbers_a_stop() { + let config = Config::default(); + let stop = Arc::new(AtomicBool::new(false)); + let status = Arc::new(Mutex::new(IndexingStatus::Stopping)); + let cx = cx_for(&config, &stop, &status); + + let guard = cx.maintaining(MaintenanceStep::MergingText); + assert!(matches!(*crate::lock_ok(&status), IndexingStatus::Stopping)); + drop(guard); + assert!(matches!(*crate::lock_ok(&status), IndexingStatus::Stopping)); +} + +/// A fresh snapshot means the writer is back on files; a step that ended +/// while the round was mid-flight must not linger on it. +#[test] +fn a_status_publish_clears_the_step() { + let dir = tmp_dir("publish-clears-step"); + let db_path = dir.join("index.db").to_string_lossy().into_owned(); + let config = config_with(vec![dir.to_string_lossy().into_owned()], &[]); + let stop = Arc::new(AtomicBool::new(false)); + let status = running_status(Some(MaintenanceStep::RootCounts)); + + run_indexing_with(&config, &db_path, &stop, &status).expect("indexed"); + match &*crate::lock_ok(&status) { + IndexingStatus::Running { maintenance, .. } => assert_eq!(*maintenance, None), + other => panic!("the run went missing: {:?}", other), + }; + + std::fs::remove_dir_all(&dir).ok(); +} + fn progress(phase: RootPhase, walked: usize, walk_total: Option) -> RootProgress { RootProgress { root: "/r".to_string(), diff --git a/crates/quicksearch-core/src/live.rs b/crates/quicksearch-core/src/live.rs index 5362b9b..80ed56f 100644 --- a/crates/quicksearch-core/src/live.rs +++ b/crates/quicksearch-core/src/live.rs @@ -477,8 +477,16 @@ impl Loop { return WindowUpdate::Unchanged; }; let mime = crate::mime::guess_mime_from_head(file, &head); - let outcome = - crate::file_handling::decide_content(path, mime.as_deref(), &self.registry, config); + // One file per call, on the UI's refresh path: no loop to amortize a + // longer-lived scratch over. + let mut scratch = crate::extract::Scratch::new(config); + let outcome = crate::file_handling::decide_content( + path, + mime, + &self.registry, + config, + &mut scratch, + ); let Some(text) = crate::file_handling::outcome_body(&outcome) else { return WindowUpdate::Unchanged; }; diff --git a/crates/quicksearch-core/src/mime.rs b/crates/quicksearch-core/src/mime.rs index ec72516..fccbe4a 100644 --- a/crates/quicksearch-core/src/mime.rs +++ b/crates/quicksearch-core/src/mime.rs @@ -80,20 +80,31 @@ const EXTENSION_OVERRIDES: &[(&str, &str)] = &[ const AMBIGUOUS_EXTENSIONS: &[&str] = &["mod", "mts", "org", "pot", "scm", "ts", "vhd"]; fn extension_is_ambiguous(path: &Path) -> bool { - path.extension() - .and_then(|e| e.to_str()) - .map(|e| e.to_ascii_lowercase()) - .is_some_and(|e| AMBIGUOUS_EXTENSIONS.contains(&e.as_str())) + path.extension().and_then(|e| e.to_str()).is_some_and(|e| { + AMBIGUOUS_EXTENSIONS + .iter() + .any(|a| a.eq_ignore_ascii_case(e)) + }) } fn extension_override(path: &Path) -> Option<&'static str> { - let ext = path.extension()?.to_str()?.to_ascii_lowercase(); + let ext = path.extension()?.to_str()?; EXTENSION_OVERRIDES .iter() - .find(|(e, _)| *e == ext) + .find(|(e, _)| e.eq_ignore_ascii_case(ext)) .map(|(_, mime)| *mime) } +/// The essence of a raw MIME — everything before any `;` parameter — as a +/// borrow of the same static. `Mime::essence_str` would do this too, but only +/// off an owned `Mime`, which is why the raw form is what gets asked for. +fn essence(raw: &'static str) -> &'static str { + match raw.split_once(';') { + Some((essence, _)) => essence.trim_end(), + None => raw, + } +} + /// Infer a MIME type from a path plus the file's leading bytes. /// /// `head` bounds both content checks: under 262 bytes (`infer`'s longest @@ -101,13 +112,20 @@ fn extension_override(path: &Path) -> Option<&'static str> { /// /// A `None` result is a real answer, not a "don't know": the content pass /// stores it and never re-derives it. -pub fn guess_mime_from_head(path: &Path, head: &[u8]) -> Option { +/// +/// `&'static str` rather than `String`: every answer comes from one of three +/// static tables ([`EXTENSION_OVERRIDES`], `mime_guess`'s, `infer`'s) or is a +/// literal, and this runs once per indexed file — an owned copy here was a +/// heap allocation per file for a string nobody mutates. +pub fn guess_mime_from_head(path: &Path, head: &[u8]) -> Option<&'static str> { if let Some(m) = extension_override(path) { - return Some(m.to_string()); + return Some(m); } - let by_extension = mime_guess::from_path(path).first().and_then(|g| { - let s = g.essence_str(); - (!s.is_empty() && s != "application/octet-stream").then(|| s.to_string()) + // `first_raw`, not `first`: the owned `Mime` exists only to be borrowed + // from, and its `essence_str` cannot outlive it. + let by_extension = mime_guess::from_path(path).first_raw().and_then(|raw| { + let s = essence(raw); + (!s.is_empty() && s != "application/octet-stream").then_some(s) }); if !extension_is_ambiguous(path) && by_extension.is_some() { return by_extension; @@ -120,19 +138,65 @@ pub fn guess_mime_from_head(path: &Path, head: &[u8]) -> Option { if magic == "application/x-ole-storage" && by_extension.is_some() { return by_extension; } - return Some(magic.to_string()); + return Some(magic); } if crate::textenc::looks_like_text(head) { - return Some("text/plain".to_string()); + return Some("text/plain"); } // Only an ambiguous extension still has an answer left to fall back on. by_extension } +/// The longest MIME any table here holds is 73 bytes; 128 leaves room and +/// keeps [`LowerMime`] a stack value. Anything longer names no format this +/// classifies, so it is matched as it came rather than growing a heap copy. +const MAX_MIME_LEN: usize = 128; + +/// A MIME lowercased without allocating. +/// +/// Nearly every MIME reaching the classifiers is already lowercase — +/// [`guess_mime_from_head`] answers from static tables — so the common path +/// borrows and only a genuinely mixed-case string is copied into the buffer. +/// This runs a few times per indexed file; `to_ascii_lowercase` there was a +/// heap allocation apiece. +pub(crate) struct LowerMime { + buf: [u8; MAX_MIME_LEN], + len: usize, + /// Set when the input was already lowercase (or too long to copy), in + /// which case [`LowerMime::as_str`] hands the original straight back. + borrowed: bool, +} + +impl LowerMime { + pub(crate) fn new(mime: &str) -> LowerMime { + let mut lower = LowerMime { + buf: [0; MAX_MIME_LEN], + len: mime.len(), + borrowed: true, + }; + if mime.len() <= MAX_MIME_LEN && mime.bytes().any(|b| b.is_ascii_uppercase()) { + lower.buf[..mime.len()].copy_from_slice(mime.as_bytes()); + // ASCII-only folding: a multi-byte sequence is left untouched, so + // what comes out is still the UTF-8 that went in. + lower.buf[..mime.len()].make_ascii_lowercase(); + lower.borrowed = false; + } + lower + } + + pub(crate) fn as_str<'a>(&'a self, original: &'a str) -> &'a str { + if self.borrowed { + return original; + } + std::str::from_utf8(&self.buf[..self.len]).unwrap_or(original) + } +} + /// Map a MIME string to a [`FileType`] bitmask. Ported from Baloo's /// `basicindexingjob.cpp:typesForMimeType`. pub fn mime_to_type(mime: &str) -> FileType { - let lower = mime.to_ascii_lowercase(); + let lower = LowerMime::new(mime); + let lower = lower.as_str(mime); let (top, sub) = match lower.split_once('/') { Some(pair) => pair, None => return FileType::EMPTY, @@ -317,7 +381,7 @@ mod tests { let mime = guess_mime_from_head(&PathBuf::from(name), b"") .unwrap_or_else(|| panic!("{} has no MIME", name)); assert!( - PlaintextExtractor.supports(&mime), + PlaintextExtractor.supports(mime), "{} -> {} is not extractable as text", name, mime @@ -329,11 +393,11 @@ mod tests { fn extension_overrides_are_case_insensitive() { use std::path::PathBuf; assert_eq!( - guess_mime_from_head(&PathBuf::from("DEPLOY.PS1"), b"").as_deref(), + guess_mime_from_head(&PathBuf::from("DEPLOY.PS1"), b""), Some("text/plain") ); assert_eq!( - guess_mime_from_head(&PathBuf::from("Build.Bat"), b"").as_deref(), + guess_mime_from_head(&PathBuf::from("Build.Bat"), b""), Some("text/plain") ); } @@ -343,11 +407,11 @@ mod tests { fn extension_overrides_beat_magic_bytes() { use std::path::PathBuf; assert_eq!( - guess_mime_from_head(&PathBuf::from("a.ps1"), b"Write-Host hi").as_deref(), + guess_mime_from_head(&PathBuf::from("a.ps1"), b"Write-Host hi"), Some("text/plain") ); assert_eq!( - guess_mime_from_head(&PathBuf::from("a.ps1"), b"%PDF-1.7").as_deref(), + guess_mime_from_head(&PathBuf::from("a.ps1"), b"%PDF-1.7"), Some("text/plain") ); } @@ -357,7 +421,7 @@ mod tests { use crate::extract::{plaintext::PlaintextExtractor, Extractor}; use std::path::PathBuf; let mime = guess_mime_from_head(&PathBuf::from("schema.sql"), b"").unwrap(); - assert!(PlaintextExtractor.supports(&mime), "{}", mime); + assert!(PlaintextExtractor.supports(mime), "{}", mime); } /// The content pass trusts the stored MIME and never reopens the file — @@ -385,7 +449,7 @@ mod tests { let mut body = magic.to_vec(); body.resize(head_bytes, 0); assert_eq!( - guess_mime_from_head(&path, &body).as_deref(), + guess_mime_from_head(&path, &body), Some(*expected), "{} must be detectable from a default-sized head", tag @@ -399,12 +463,11 @@ mod tests { fn a_head_shorter_than_the_signature_declines_rather_than_guessing() { use std::path::PathBuf; let path = PathBuf::from("/tmp/qs-sniff-truncated"); - assert_eq!(guess_mime_from_head(&path, b"").as_deref(), None); + assert_eq!(guess_mime_from_head(&path, b""), None); // A PNG magic truncated to two bytes: no magic match, no text guess. - assert_eq!(guess_mime_from_head(&path, &[0x89, 0x00]).as_deref(), None); + assert_eq!(guess_mime_from_head(&path, &[0x89, 0x00]), None); assert_eq!( - guess_mime_from_head(&path, &[0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a]) - .as_deref(), + guess_mime_from_head(&path, &[0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a]), Some("image/png") ); } @@ -452,7 +515,7 @@ mod tests { let mime = guess_mime_from_head(&path, head).unwrap_or_else(|| panic!("{} has no MIME", name)); let extracted = registry - .extract_complete_head(&path, &mime, head) + .extract_head_to_string(&path, mime, head) .unwrap_or_else(|| { panic!( "{} -> {} not claimed by a head-capable extractor", @@ -474,17 +537,17 @@ mod tests { use std::path::PathBuf; let readme = PathBuf::from("README"); assert_eq!( - guess_mime_from_head(&readme, b"QuickSearch indexes your files.\n").as_deref(), + guess_mime_from_head(&readme, b"QuickSearch indexes your files.\n"), Some("text/plain") ); let makefile = PathBuf::from("Makefile"); assert_eq!( - guess_mime_from_head(&makefile, b"all:\n\tcargo build\n").as_deref(), + guess_mime_from_head(&makefile, b"all:\n\tcargo build\n"), Some("text/plain") ); let blob = PathBuf::from("blob"); assert_eq!( - guess_mime_from_head(&blob, &[0x00, 0x01, 0x02, 0xFF]).as_deref(), + guess_mime_from_head(&blob, &[0x00, 0x01, 0x02, 0xFF]), None ); } @@ -505,13 +568,13 @@ mod tests { // legacy-encoded documents still type as text. let latin1 = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9."; assert_eq!( - guess_mime_from_head(&PathBuf::from("notes.txt"), latin1).as_deref(), + guess_mime_from_head(&PathBuf::from("notes.txt"), latin1), Some("text/plain") ); // A `.pb` that really is UTF-8 text still indexes. assert_eq!( - guess_mime_from_head(&PathBuf::from("notes.pb"), b"just some words\n").as_deref(), + guess_mime_from_head(&PathBuf::from("notes.pb"), b"just some words\n"), Some("text/plain") ); } @@ -522,11 +585,11 @@ mod tests { let ts_source = b"export function hi(): string { return 'hi'; }\n"; assert_eq!( - guess_mime_from_head(&PathBuf::from("app.ts"), ts_source).as_deref(), + guess_mime_from_head(&PathBuf::from("app.ts"), ts_source), Some("text/plain") ); assert_eq!( - guess_mime_from_head(&PathBuf::from("APP.TS"), ts_source).as_deref(), + guess_mime_from_head(&PathBuf::from("APP.TS"), ts_source), Some("text/plain") ); // An MPEG transport stream: no magic matcher, fails the text sniff, @@ -535,7 +598,7 @@ mod tests { ts_video[0] = 0x47; ts_video[188] = 0x47; assert_eq!( - guess_mime_from_head(&PathBuf::from("clip.ts"), &ts_video).as_deref(), + guess_mime_from_head(&PathBuf::from("clip.ts"), &ts_video), Some("video/vnd.dlna.mpeg-tts") ); @@ -543,30 +606,27 @@ mod tests { guess_mime_from_head( &PathBuf::from("go.mod"), b"module example.com/x\n\ngo 1.22\n" - ) - .as_deref(), + ), Some("text/plain") ); // gettext template vs PowerPoint template. assert_eq!( - guess_mime_from_head(&PathBuf::from("app.pot"), b"msgid \"hello\"\nmsgstr \"\"\n") - .as_deref(), + guess_mime_from_head(&PathBuf::from("app.pot"), b"msgid \"hello\"\nmsgstr \"\"\n"), Some("text/plain") ); let ole = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1, 0x00, 0x00]; assert_eq!( - guess_mime_from_head(&PathBuf::from("slides.pot"), &ole).as_deref(), + guess_mime_from_head(&PathBuf::from("slides.pot"), &ole), Some("application/vnd.ms-powerpoint") ); assert_eq!( - guess_mime_from_head(&PathBuf::from("cpu.vhd"), b"entity cpu is\nend cpu;\n") - .as_deref(), + guess_mime_from_head(&PathBuf::from("cpu.vhd"), b"entity cpu is\nend cpu;\n"), Some("text/plain") ); assert_eq!( - guess_mime_from_head(&PathBuf::from("disk.vhd"), &[0x00, 0x01, 0x02, 0x03]).as_deref(), + guess_mime_from_head(&PathBuf::from("disk.vhd"), &[0x00, 0x01, 0x02, 0x03]), Some("application/x-virtualbox-vhd") ); } diff --git a/crates/quicksearch-core/src/platform.rs b/crates/quicksearch-core/src/platform.rs index bd5bd25..36b8155 100644 --- a/crates/quicksearch-core/src/platform.rs +++ b/crates/quicksearch-core/src/platform.rs @@ -365,17 +365,49 @@ where .expect("spawn worker thread") } -/// Return free heap pages to the kernel. glibc's `free` keeps chunks on arena -/// free lists, so a transient peak stays in RSS for the life of the process; -/// `malloc_trim(0)` walks *every* arena, reclaiming other threads' leavings -/// too, and costs milliseconds — it must not go anywhere hot. Idempotent. +/// The allocator every binary in this workspace installs. +/// +/// Re-exported because `#[global_allocator]` only takes effect in the crate +/// that *declares* it — a library cannot choose one for its dependents. So +/// each binary names this type, and they must all name the same one: a +/// measurement harness left on the system allocator would report numbers for +/// a build nobody ships. +/// +/// ```ignore +/// #[global_allocator] +/// static GLOBAL: quicksearch_core::platform::Allocator = +/// quicksearch_core::platform::Allocator; +/// ``` +/// +/// Why not glibc: it gives each thread a 64 MiB arena and never shrinks one +/// below its high-water mark. A multi-million-file run settled at 985 MB RSS, +/// 871 MB of it anonymous slack that `malloc_trim` could not coalesce — +/// it only returns pages that are *wholly* free, and one live chunk pins +/// 4 KiB. Capping arenas at 2 cut the floor to 146 MB but made indexing +/// dramatically slower, because two arenas serialise every worker. mimalloc +/// has per-thread heaps with no lock on the fast path and decommits freed +/// segments, so it gives both. +pub use mimalloc::MiMalloc as Allocator; + +/// Return free memory to the kernel; idempotent, and costs milliseconds, so +/// it must not go anywhere hot. +/// +/// A run's peak is not its steady state — extraction buffers, path strings +/// and compression scratch are all freed by the end — but freed is not +/// returned. This is the point where a finished run gives it back. pub fn release_free_heap() { - #[cfg(all(target_os = "linux", target_env = "gnu"))] - { - // SAFETY: callable from any thread; glibc takes the arena locks itself. - unsafe { libc::malloc_trim(0) }; + // `libmimalloc-sys` binds only the allocation entry points, so this one + // is declared here. The symbol is in the static library that crate + // already links; depending on it is what puts it there. + extern "C" { + /// `void mi_collect(bool force)`. C `_Bool` and Rust `bool` are the + /// same one byte. + fn mi_collect(force: bool); } - // Elsewhere: `malloc_trim` is a glibc extension; musl frees to the kernel. + // SAFETY: no arguments of ours, no state of ours, callable from any + // thread. `true` asks it to return memory to the OS rather than merely to + // mimalloc's own free lists — the whole point of the call. + unsafe { mi_collect(true) }; } /// Create `dir` and its parents, readable only by their owner: diff --git a/crates/quicksearch-core/src/scope.rs b/crates/quicksearch-core/src/scope.rs index ab01b95..3eab671 100644 --- a/crates/quicksearch-core/src/scope.rs +++ b/crates/quicksearch-core/src/scope.rs @@ -309,8 +309,8 @@ pub fn advance( } } - // Deletions leave FTS tombstones; automerge collapses them. Skipping it - // costs only tidiness — the next run's automerge does the same. + // Deletions leave FTS tombstones; a merge collapses them. Skipping it + // costs only tidiness — the next run's merge does the same. if cancelled(cancel) { return Ok(()); } diff --git a/crates/quicksearch-core/src/search/mod.rs b/crates/quicksearch-core/src/search/mod.rs index cc79d40..53188a8 100644 --- a/crates/quicksearch-core/src/search/mod.rs +++ b/crates/quicksearch-core/src/search/mod.rs @@ -31,8 +31,11 @@ pub use duplicates::{find_duplicate_groups, DuplicateGroup}; /// decrypted pages and the next keystroke pays to refill them: on a 200k-file /// index the first query after a release costs 129 ms unencrypted and 192 ms /// encrypted, against ~10 ms warm. Encryption is why the gap widens — a -/// refill is an AES decrypt plus an HMAC verify per page rather than a -/// `memcpy`. What the release buys back is the ~42 MiB the trim in +/// refill is an AES decrypt per page rather than a `memcpy`. (The 192 ms was +/// measured while `db::schema::HMAC_MODE` was still HMAC-SHA512 and there was +/// a per-page verify to pay as well, so the gap is narrower now; the argument +/// for a long idle timeout only gets stronger as the two converge.) What the +/// release buys back is the ~42 MiB the trim in /// [`Worker::run`] returns, so this trades an idle process floor against /// stalling the one keystroke a user is most likely to notice. /// diff --git a/crates/quicksearch-core/src/testutil.rs b/crates/quicksearch-core/src/testutil.rs index 4d794d7..53ec207 100644 --- a/crates/quicksearch-core/src/testutil.rs +++ b/crates/quicksearch-core/src/testutil.rs @@ -193,13 +193,15 @@ impl Lcg { Lcg(seed) } - pub fn next(&mut self) -> u64 { + /// Not `next`: an inherent method by that name reads as `Iterator`'s, and + /// this one is infinite and returns a bare `u64` rather than an `Option`. + pub fn next_u64(&mut self) -> u64 { self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1); self.0 >> 33 } pub fn pick<'a, T>(&mut self, from: &'a [T]) -> &'a T { - &from[self.next() as usize % from.len()] + &from[self.next_u64() as usize % from.len()] } } @@ -231,6 +233,7 @@ pub const WORDS: &[&str] = &[ ]; /// What [`seed_index`] should build. +#[derive(Clone, Copy)] pub struct SeedSpec { pub files: usize, /// One file in every `content_every` gets extracted text. @@ -239,6 +242,12 @@ pub struct SeedSpec { pub body_words: usize, /// Directories to spread the rows across. pub dirs: usize, + /// Path segments in each stored `parent`. `1` is `/seed/NNN/` — short, and + /// what the search harnesses have always used. A real tree is nested, and + /// `parent` is stored per row, so this is most of what decides the width + /// of a `files` row and therefore how much cache a scan of it needs. Raise + /// it when calibrating anything against real-world row size. + pub dir_depth: usize, /// File names carrying [`NEEDLE`]. Kept far below any sane display limit /// — an early-exiting query measures how fast the cascade gives up. pub needle_names: usize, @@ -252,6 +261,31 @@ pub struct SeedSpec { /// `0` leaves every hash NULL — the shape the search harnesses seed, and /// the one whose row width their numbers were taken against. pub dup_every: usize, + /// Commit every N files instead of wrapping the whole seed in one + /// transaction (`0`). Each commit flushes FTS5's in-memory hash to its own + /// segment, so this is what gives a later `merge` real work — the shape a + /// production run has, where the writer commits in slices. Harnesses that + /// only want rows as fast as possible leave it at `0`. + pub commit_every: usize, + /// Build the index at this database page size instead of + /// [`crate::db::schema::PAGE_SIZE`]. Installed as a process-global + /// override for the whole seed *and left installed*, because a keyed file + /// cannot be reopened without it — see + /// [`crate::db::set_page_size_override`]. + pub page_size: Option, + /// Override FTS5's `pgsz` before a single row is written. `None` keeps + /// whatever the schema chose for this key state, which is what every + /// harness measuring the *product* wants. It exists so a benchmark can + /// pin FTS5's default 4050 on a keyed index and price + /// [`crate::db::schema::fts_pgsz_for`] against it in one process on + /// one corpus. + pub pgsz: Option, + /// Build the index under this per-page authenticator instead of + /// [`crate::db::schema::HMAC_MODE`]. A process-global override for the + /// same reason `page_size` is one — it sets the page reserve, so a keyed + /// file cannot be reopened without it. Ignored on a plain arm, which has + /// no reserve. + pub hmac: Option, } impl Default for SeedSpec { @@ -263,10 +297,15 @@ impl Default for SeedSpec { // full-text pass look free when it is the cascade's most expensive. body_words: 300, dirs: 500, + dir_depth: 1, needle_names: 50, needle_docs: 50, body_term_docs: 500, dup_every: 0, + commit_every: 0, + page_size: None, + pgsz: None, + hmac: None, } } } @@ -277,14 +316,33 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) { use crate::db::repo::{insert_file, set_content_done, NewFile}; use crate::mime::FileType; - let mut conn = crate::db::open_or_recreate(path.to_str().unwrap(), "trigram").unwrap(); + // Before the open, not after: the profile decides how the file is + // *created*, and on a keyed file it decides whether it can be read at all. + if let Some(page_size) = spec.page_size { + crate::db::set_page_size_override(page_size); + } + if let Some(hmac) = spec.hmac { + crate::db::set_hmac_mode_override(hmac); + } + let conn = crate::db::open_or_recreate(path.to_str().unwrap(), "trigram").unwrap(); + // Before the first row: `pgsz` decides how leaves are built, so setting it + // afterwards would only affect segments merged later. + if let Some(pgsz) = spec.pgsz { + conn.execute( + "INSERT INTO searchabletext(searchabletext, rank) VALUES('pgsz', ?1)", + [pgsz], + ) + .unwrap(); + } let mut rng = Lcg::new(0x5eed); // Spacing, not a random draw: a cluster at the front would let a pass // stop early and report a fraction of the work a real rare query costs. let name_stride = spec.files / spec.needle_names.max(1); let doc_stride = spec.files / spec.needle_docs.max(1); let body_stride = spec.files / spec.body_term_docs.max(1); - let tx = conn.transaction().unwrap(); + // `unchecked_transaction` borrows the connection shared, which is what + // lets `commit_every` end one and start the next inside the loop. + let mut tx = conn.unchecked_transaction().unwrap(); for i in 0..spec.files { let w1 = rng.pick(WORDS); let w2 = rng.pick(WORDS); @@ -294,7 +352,15 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) { format!("{}-{}-{:07}.txt", w1, w2, i) }; // Stored parents always end in a separator; see `dir_to_db_parent`. - let dir = format!("/seed/{:03}/", i % spec.dirs.max(1)); + // Deeper segments are derived from the directory index, not the file + // index, so files continue to share parents the way a real tree does. + let d = i % spec.dirs.max(1); + let mut dir = format!("/seed/{:03}", d); + for segment in 1..spec.dir_depth.max(1) { + dir.push('/'); + dir.push_str(WORDS[(d * 7 + segment * 13) % WORDS.len()]); + } + dir.push('/'); // Every `dup_every`-th row takes the hash of the one before it, so the // groups are pairs of equal-sized rows — the shape `find_duplicate_groups` // prices, since a hash covers the size. @@ -338,11 +404,232 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) { let body = body.join(" "); set_content_done(&tx, id, &body, zstd_of(&body).as_deref()).unwrap(); } + if spec.commit_every > 0 && (i + 1) % spec.commit_every == 0 { + tx.commit().unwrap(); + tx = conn.unchecked_transaction().unwrap(); + } } tx.commit().unwrap(); conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);").ok(); } +/// A raw 32-byte key for the measurement harnesses, deliberately **not** an +/// Argon2id derivation: the KDF costs half a second in release and minutes in +/// debug, and proves nothing about page work. It reaches SQLCipher as raw hex +/// either way (see `db::open::key_and_probe`), so a keyed arm measures what a +/// real unlocked index does. +pub const MEASUREMENT_KEY_HEX: &str = + "a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90"; + +pub fn measurement_key() -> crate::security::IndexKey { + crate::security::IndexKey::from_hex(MEASUREMENT_KEY_HEX).expect("a 64-hex-digit key") +} + +/// `(hits, misses)` in this connection's page cache since it was opened. +/// +/// A *miss* is the unit that costs money on a keyed index: the page has to be +/// read and AES-CBC decrypted before a single row can be read out of it, where +/// a hit is a pointer into memory SQLite already holds. So +/// counting misses per query shape attributes cost to the table that caused +/// it, which timing alone cannot do. +/// +/// `sqlite3_db_status` has no safe wrapper in rusqlite; the raw binding and +/// `Connection::handle` are both public, and neither the pointer nor the +/// out-params outlive this call. +pub fn cache_stats(conn: &rusqlite::Connection) -> (i64, i64) { + use rusqlite::ffi; + let mut hits = (0i32, 0i32); + let mut misses = (0i32, 0i32); + unsafe { + let handle = conn.handle(); + ffi::sqlite3_db_status( + handle, + ffi::SQLITE_DBSTATUS_CACHE_HIT, + &mut hits.0, + &mut hits.1, + 0, + ); + ffi::sqlite3_db_status( + handle, + ffi::SQLITE_DBSTATUS_CACHE_MISS, + &mut misses.0, + &mut misses.1, + 0, + ); + } + (hits.0 as i64, misses.0 as i64) +} + +/// FTS5's own default page size, which a keyed index used to inherit. Pinned +/// explicitly on the "before" arms of [`seed_arms`] so the cost of that +/// inheritance is priced in the same run as the fix, not remembered from +/// another one. +pub use crate::db::schema::FTS5_DEFAULT_PGSZ; + +/// Indices into [`seed_arms`]'s fixed order. The two `_4050` arms exist only +/// to price [`crate::db::schema::FTS_PGSZ_ENCRYPTED`] against what came +/// before; the other two are the shipped product. +pub const ARM_PLAIN_4050: usize = 0; +pub const ARM_PLAIN: usize = 1; +pub const ARM_KEYED_4050: usize = 2; +pub const ARM_KEYED: usize = 3; + +/// `(label, keyed, pgsz, path suffix)`, in [`seed_arms`] order. +const ARM_SHAPES: [(&str, bool, Option, &str); 4] = [ + ( + "plain, pgsz 4050", + false, + Some(FTS5_DEFAULT_PGSZ), + "plain-4050", + ), + ("plain, as shipped", false, None, "plain"), + ( + "keyed, pgsz 4050", + true, + Some(FTS5_DEFAULT_PGSZ), + "keyed-4050", + ), + ("keyed, as shipped", true, None, "keyed"), +]; + +/// One seeded index in a plain-vs-keyed comparison: `tests/encrypted_perf.rs` +/// gates four of them on size, `benches/page_geometry.rs` sweeps page sizes +/// across them. Defined here, once, so the harnesses report on the same shape. +pub struct Arm { + pub what: String, + pub keyed: bool, + /// `None` takes whatever the schema chose for this key state — the + /// shipped behaviour. `Some` pins a value, only ever used to reproduce + /// the old geometry. + pub pgsz: Option, + /// The database page size this arm was built at, and the one every open + /// of it must re-install: a keyed file's header is ciphertext, so it + /// cannot be read back off the file. + pub page_size: Option, + /// The per-page authenticator this arm was built under, re-installed on + /// every open for the same reason `page_size` is: it sets the page + /// reserve, which the header cannot be read without. + pub hmac: Option, + pub path: PathBuf, + /// How long seeding spent writing it: the database-write half of indexing + /// (rows, zstd bodies, FTS postings), which is the half a page geometry + /// can change. The walk and the extractors are not in it. + pub seeded_in: std::time::Duration, +} + +impl Arm { + /// Seed one arm from `spec` and time the write. `spec.page_size`, + /// `spec.hmac` and `spec.pgsz` define the geometry; `tag` names its + /// scratch directory. + pub fn seed(what: impl Into, tag: &str, keyed: bool, spec: &SeedSpec) -> Arm { + let arm = Arm { + what: what.into(), + keyed, + pgsz: spec.pgsz, + page_size: spec.page_size, + hmac: spec.hmac, + path: scratch_db(tag), + seeded_in: std::time::Duration::ZERO, + }; + let path = arm.path.clone(); + let spec = *spec; + let start = std::time::Instant::now(); + arm.with_key(|| seed_index(&path, &spec)); + Arm { + seeded_in: start.elapsed(), + ..arm + } + } + + /// Run `f` with this arm's key *and profile* installed process-wide, then + /// restore the shipped ones. Every open has to be wrapped: all three are + /// process-globals, and an index seeded under them and opened without them + /// fails as a wrong-password error rather than quietly. + pub fn with_key(&self, f: impl FnOnce() -> T) -> T { + crate::db::set_process_key(self.keyed.then(measurement_key)); + crate::db::set_page_size_override(self.page_size.unwrap_or(crate::db::schema::PAGE_SIZE)); + crate::db::set_hmac_mode_override(self.hmac.unwrap_or(crate::db::schema::HMAC_MODE)); + let out = f(); + crate::db::set_process_key(None); + crate::db::set_page_size_override(crate::db::schema::PAGE_SIZE); + crate::db::set_hmac_mode_override(crate::db::schema::HMAC_MODE); + out + } + + /// Delete this arm's scratch directory. The sweep seeds a lot of large + /// indexes; dropping each once measured keeps one resident at a time. + pub fn discard(self) { + if let Some(dir) = self.path.parent() { + std::fs::remove_dir_all(dir).ok(); + } + } + + /// A search connection on this arm, at the production pragma profile. + pub fn open_search(&self) -> rusqlite::Connection { + self.with_key(|| { + crate::db::open::open_search_reader(&self.path.to_string_lossy()).expect("open arm") + }) + } + + pub fn size_bytes(&self) -> u64 { + std::fs::metadata(&self.path).map(|m| m.len()).unwrap_or(0) + } + + /// Bytes `dbstat` attributes to one table. The number to compare a cache + /// ceiling against: `files` is what every keystroke rescans, so whether it + /// fits is what decides if a typing session stays warm. + pub fn table_bytes(&self, table: &str) -> u64 { + let conn = self.open_search(); + conn.query_row( + "SELECT COALESCE(SUM(pgsize), 0) FROM dbstat WHERE name = ?1", + [table], + |r| r.get::<_, i64>(0), + ) + .unwrap_or(0) as u64 + } + + /// `(leaf, overflow)` pages in `searchabletext_data`. The overflow count + /// is the whole diagnosis: SQLCipher's page reserve drops the inline + /// payload limit below what a leaf built for another profile assumes, and + /// each miss costs a second page — a second fetch and decrypt on every + /// read of it. See `db::schema::fts_pgsz_for`, which is what keeps the + /// count at zero. + pub fn fts_pages(&self) -> (i64, i64) { + let conn = self.open_search(); + let count = |pagetype: &str| -> i64 { + conn.query_row( + "SELECT COUNT(*) FROM dbstat \ + WHERE name = 'searchabletext_data' AND pagetype = ?1", + [pagetype], + |r| r.get(0), + ) + .expect("dbstat") + }; + (count("leaf"), count("overflow")) + } +} + +/// Seed the same corpus four times: plain and keyed, each on FTS5's default +/// page size and on whatever the schema picks. Identical content and identical +/// insertion order throughout, so arms differ *only* in those two variables — +/// which is what lets a display-limited query be compared at all (the cascade +/// stops when the limit fills, so a different rowid order would decide the +/// answer rather than the encryption). +/// +/// `spec.pgsz` is overridden per arm; everything else is the caller's. +pub fn seed_arms(tag: &str, spec: &SeedSpec) -> Vec { + ARM_SHAPES + .iter() + .map(|(what, keyed, pgsz, suffix)| { + let spec = SeedSpec { + pgsz: *pgsz, + ..*spec + }; + Arm::seed(*what, &format!("{}-{}", tag, suffix), *keyed, &spec) + }) + .collect() +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/quicksearch-core/src/textenc.rs b/crates/quicksearch-core/src/textenc.rs index c36eef7..8e03052 100644 --- a/crates/quicksearch-core/src/textenc.rs +++ b/crates/quicksearch-core/src/textenc.rs @@ -73,6 +73,11 @@ pub fn looks_like_text(head: &[u8]) -> bool { /// besides what [`looks_like_text`] does: by the time this runs, something — /// usually the extension — has already decided the file is text, so a /// windows-1252 `.txt` or Shift-JIS `.csv` still decodes. +/// +/// Takes the buffer **by value** so a UTF-8 file — the overwhelming majority +/// — becomes its `String` with no copy at all. Callers that only have a +/// borrow, and would otherwise clone one just to hand it over, want +/// [`decode_borrowed_text`]. pub fn decode_text(bytes: Vec, path: &Path) -> Result { if bytes.is_empty() { return Ok(String::new()); @@ -80,9 +85,28 @@ pub fn decode_text(bytes: Vec, path: &Path) -> Result { match classify(&bytes, false) { // Cannot fail: classify ran strict validation with truncated=false. TextClass::Utf8 => Ok(String::from_utf8(bytes).expect("classified as UTF-8")), + _ => decode_borrowed_text(&bytes, path), + } +} + +/// [`decode_text`] for bytes the caller does not own — the walk's head +/// buffer, which is reused for the next file and so cannot be given away. +/// +/// The one difference is the UTF-8 case, which must copy here; every other +/// class allocates its output either way. `head.to_vec()` at the call site +/// was that same copy plus a second one for the bytes. +pub fn decode_borrowed_text(bytes: &[u8], path: &Path) -> Result { + if bytes.is_empty() { + return Ok(String::new()); + } + match classify(bytes, false) { + // Cannot fail: classify ran strict validation with truncated=false. + TextClass::Utf8 => Ok(std::str::from_utf8(bytes) + .expect("classified as UTF-8") + .to_string()), TextClass::Bom(enc) => { // Strips the BOM, replaces malformed sequences with U+FFFD. - let (text, _, _) = enc.decode(&bytes); + let (text, _, _) = enc.decode(bytes); Ok(text.into_owned()) } TextClass::Legacy => { @@ -96,7 +120,7 @@ pub fn decode_text(bytes: Vec, path: &Path) -> Result { // Deny UTF-8: strict UTF-8 was already ruled out, so a UTF-8 // guess could only mean malformed UTF-8. let enc = det.guess(None, chardetng::Utf8Detection::Deny); - let (text, _, _) = enc.decode(&bytes); + let (text, _, _) = enc.decode(bytes); Ok(text.into_owned()) } TextClass::Binary => Err(format!("plaintext read {}: binary content", path.display())), diff --git a/crates/quicksearch-core/src/walk.rs b/crates/quicksearch-core/src/walk.rs index 1c3c3cc..e25ac9c 100644 --- a/crates/quicksearch-core/src/walk.rs +++ b/crates/quicksearch-core/src/walk.rs @@ -342,7 +342,15 @@ enum Known<'a> { /// At most one `stat`, then classify; only files that will be written get /// opened, and small text files are finished outright. "At most": on Windows /// [`PendingFile::cached`] may already hold the answer. -fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile { +/// +/// `scratch` is this worker's, carrying the head buffer every hash and MIME +/// sniff reads into. +fn prepare( + file: PendingFile, + known: Known<'_>, + ctx: &Ctx, + scratch: &mut crate::extract::Scratch, +) -> WalkedFile { let PendingFile { path, cached } = file; // Every route here has already screened the path for UTF-8: // `path_to_db_string` is lossy, and a lossy string would key another @@ -386,7 +394,7 @@ fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile { FileIndexAction::Skip => None, // `prepare_file_record` gates on `is_file()`, which keeps us from // opening a FIFO — an uninterruptible forever-block. - _ => prepare_file_record(&db_path, &meta, &ctx.config, &ctx.registry), + _ => prepare_file_record(&db_path, &meta, &ctx.config, &ctx.registry, scratch), }; WalkedFile { @@ -398,6 +406,10 @@ fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile { } fn worker(shared: &Shared, ctx: &Ctx, tx: &mpsc::SyncSender) { + // One per worker, for the whole walk: the head buffer inside it is what + // every file's hash and MIME sniff reads into, and a fresh one per file + // was an allocation per file. + let mut scratch = crate::extract::Scratch::new(&ctx.config); while let Some((job, slot)) = shared.take() { let _busy = shared.stats.enter(); if ctx.stop_flag.load(Ordering::Relaxed) { @@ -417,7 +429,12 @@ fn worker(shared: &Shared, ctx: &Ctx, tx: &mpsc::SyncSender) { slot.finish(found); let file = PendingFile::uncached(path); if tx - .send(WalkEvent::File(prepare(file, Known::Exact(stored), ctx))) + .send(WalkEvent::File(prepare( + file, + Known::Exact(stored), + ctx, + &mut scratch, + ))) .is_err() { shared.shutdown(); @@ -442,7 +459,12 @@ fn worker(shared: &Shared, ctx: &Ctx, tx: &mpsc::SyncSender) { return; } if tx - .send(WalkEvent::File(prepare(file, Known::InDir(&rows), ctx))) + .send(WalkEvent::File(prepare( + file, + Known::InDir(&rows), + ctx, + &mut scratch, + ))) .is_err() { // Receiver gone: the run was stopped or failed. Not an error. diff --git a/crates/quicksearch-core/tests/encrypted.rs b/crates/quicksearch-core/tests/encrypted.rs index 853babf..e5e150e 100644 --- a/crates/quicksearch-core/tests/encrypted.rs +++ b/crates/quicksearch-core/tests/encrypted.rs @@ -91,9 +91,8 @@ fn encrypted_index_lifecycle() { drop(conn); let conn = db::open::open_maintenance(&db_path.to_string_lossy()).unwrap(); - let dir = data.to_string_lossy().into_owned(); assert!( - quicksearch_core::db::repo::maintain(&conn, &dir).unwrap(), + quicksearch_core::db::repo::maintain(&conn, &db_path.to_string_lossy()).unwrap(), "that much slack should have been reclaimed" ); drop(conn); diff --git a/crates/quicksearch-core/tests/encrypted_perf.rs b/crates/quicksearch-core/tests/encrypted_perf.rs index 386388b..8789ff8 100644 --- a/crates/quicksearch-core/tests/encrypted_perf.rs +++ b/crates/quicksearch-core/tests/encrypted_perf.rs @@ -1,8 +1,8 @@ //! Encryption must cost a constant factor, not a different algorithm. //! -//! SQLCipher decrypts and HMAC-verifies every 4 KiB page it reads, so a keyed -//! index is intrinsically slower than a plain one — that part is not a bug and -//! this file does not try to gate it. What it gates is *amplification*: a query +//! SQLCipher AES-decrypts every page it reads, so a keyed index is +//! intrinsically slower than a plain one — that part is not a bug and this +//! file does not try to gate it. What it gates is *amplification*: a query //! whose cost is one page fetch per row is fine unencrypted (the page cache //! makes it nearly free) and disastrous keyed. `find_duplicate_groups` was //! exactly that until it was rewritten to stay inside `idx_files_hash`: @@ -14,40 +14,87 @@ //! //! Measured on 400k rows, so the ceiling below sits between those two: the old //! shape fails it, the current one passes with room. The ratio is what makes -//! this a *test* rather than a benchmark — both arms run the same workload on +//! this a *test* rather than a benchmark — every arm runs the same workload on //! the same machine in the same process, so host speed, CPU governor and CI //! contention divide out. Absolute times are printed but never asserted. //! +//! Since `db::schema::HMAC_MODE` became `Off` the constant factor is much +//! smaller — every shape here now runs 1.03–1.20x, where the same shapes were +//! up to 1.3x with a per-page HMAC-SHA512 to pay as well. +//! +//! # Size +//! +//! Four arms, because the second variable is FTS5's *record* size. A table +//! leaf holds `page − reserve − 35` bytes inline, where a plain file's reserve +//! is 0 and a keyed one's is `HMAC_MODE.reserve()`. FTS5's own default record +//! of 4050 was chosen to fit a plain 4096 page; `db::schema::fts_pgsz_for` +//! derives it from the profile instead. Measured at 120k files, +//! `schema::PAGE_SIZE` = 8192: +//! +//! | arm | size | fts leaves | overflow | +//! |---|---|---|---| +//! | plain, pgsz 4050 | 131.3 MiB | 10986 | 0 | +//! | plain, derived | 130.8 MiB | 10922 | 0 | +//! | keyed, pgsz 4050 | 131.2 MiB | 10986 | 0 | +//! | keyed, derived | 130.9 MiB | 10942 | 0 | +//! +//! Encrypted over plain on disk: **1.001x**. +//! +//! **The `_4050` arms no longer demonstrate much, and that is the change +//! rather than a defect in them.** They existed because a keyed page used to +//! give up 80 bytes, which left a keyed 8192 page holding only *one* +//! 4052-byte record — two would not fit under the 8077-byte limit — so half of +//! every page went empty and the index came out at 221.0 MiB, 1.688x plain. +//! At a 16-byte reserve the limit is 8141 and two fit with room, so FTS5's +//! fixed default happens to be fine here. It is still wrong at other page +//! sizes, which is why the derivation stays and why these arms still assert +//! `derived <= pinned` — just with a much smaller margin than they used to. +//! +//! The query times are unmoved by leaf geometry, within this seed's noise: the +//! working set is served from the search cache either way, so it shows up on +//! disk long before it shows up here. `benches/page_geometry.rs` is where it +//! is timed, on corpora that do not fit, and `benches/cipher_hmac.rs` is where +//! the authenticator itself was priced. +//! //! Its own integration binary because it installs a process-global key, the //! same reason `tests/encrypted.rs` gives. -use std::path::PathBuf; use std::sync::atomic::AtomicU64; use std::time::{Duration, Instant}; use quicksearch_core::db; use quicksearch_core::query::split::split_for_cascade; use quicksearch_core::search::{cascade, find_duplicate_groups, SearchHit, SearchOptions}; -use quicksearch_core::security::IndexKey; -use quicksearch_core::testutil::{scratch_db, seed_index, SeedSpec, BODY_TERM, NEEDLE}; +use quicksearch_core::testutil::{ + measurement_key, seed_arms, Arm, SeedSpec, ARM_KEYED, ARM_KEYED_4050, ARM_PLAIN, + ARM_PLAIN_4050, BODY_TERM, NEEDLE, +}; -/// A raw 32-byte key, not an Argon2id derivation: the KDF costs half a second -/// in release and minutes in debug, and proves nothing about page work. It -/// reaches SQLCipher as raw hex either way (see `db::open::key_and_probe`), so -/// what is measured below is identical to a real unlocked index. -const KEY_HEX: &str = "a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90"; +/// Ceiling on encrypted/plain for one workload. It still has to sit under the +/// 3.9x the old duplicate query cost — that is the regression this gate is +/// for — but it no longer has to leave room for a per-page HMAC: with +/// `HMAC_MODE` off the worst shape measures 1.20x, so 2.0 is 66% of headroom +/// over the worst observed and still fails the amplified shape outright. +/// Raising this without a measurement in the table above defeats it. +const MAX_RATIO: f64 = 2.0; -/// Ceiling on encrypted/plain for one workload. Between the 3.9x the old -/// duplicate query cost and the 1.3x the current one costs; see the table -/// above. Raising this without a measurement in the same table defeats it. -const MAX_RATIO: f64 = 3.0; +/// Ceiling on the encrypted index's *size* relative to the plain one, both as +/// shipped. Measured at 1.001x: `fts_pgsz_for` hands the reserve back to the +/// leaves, so a protected index is now the same size as an unprotected one. +/// The ceiling keeps room for a corpus whose table mix differs. +const MAX_SIZE_RATIO: f64 = 1.03; /// Enough rows that neither index fits in `PRAGMAS_SEARCH`'s 32 MiB page /// cache — the only regime where a per-page decrypt is visible at all. Below /// that both arms are served from cache, every ratio is 1.0, and the gate -/// silently stops testing anything. `index_is_larger_than_the_search_cache` -/// pins that this seed still clears it. -const FILES: usize = 60_000; +/// silently stops testing anything. The assertion below pins that this seed +/// still clears it. +/// +/// Raised from 60k when `schema::PAGE_SIZE` became 8192: the same queries got +/// fast enough that `cascade literal name` and `cascade wildcard` fell under +/// [`MIN_MEASURABLE`], which is that guard working, not failing. The seed has +/// to grow when the code outruns it. +const FILES: usize = 120_000; const CONTENT_EVERY: usize = 5; /// The cache the search connection actually opens with, from @@ -75,31 +122,8 @@ fn spec() -> SeedSpec { } } -fn key() -> IndexKey { - IndexKey::from_hex(KEY_HEX).expect("a 64-hex-digit key") -} - -/// Seed the same corpus twice, once plain and once keyed. Identical content -/// and identical insertion order, so the two indexes differ *only* by -/// encryption — which is what lets a display-limited query be compared at all -/// (the cascade stops when the limit fills, so a different rowid order would -/// decide the answer rather than the encryption). -fn seed_both() -> (PathBuf, PathBuf) { - let plain = scratch_db("encperf-plain"); - let keyed = scratch_db("encperf-keyed"); - - db::set_process_key(None); - seed_index(&plain, &spec()); - - db::set_process_key(Some(key())); - seed_index(&keyed, &spec()); - db::set_process_key(None); - - (plain, keyed) -} - -fn mib(path: &PathBuf) -> f64 { - std::fs::metadata(path).map(|m| m.len()).unwrap_or(0) as f64 / (1024.0 * 1024.0) +fn mib(bytes: u64) -> f64 { + bytes as f64 / (1024.0 * 1024.0) } /// Run `f` `RUNS` times, keeping the fastest. @@ -113,39 +137,44 @@ fn best_of(mut f: impl FnMut()) -> Duration { best } -/// One workload's verdict. Collected rather than asserted inline so a run -/// reports *every* ratio, not just the first one that failed. +/// One workload timed on every arm, in `seed_arms` order. Collected rather +/// than asserted inline so a run reports *every* ratio, not just the first one +/// that failed. struct Measured { what: &'static str, - plain: Duration, - keyed: Duration, + per_arm: Vec, } impl Measured { + /// Encrypted over plain, both as shipped — the ratio this file exists to + /// gate. fn ratio(&self) -> f64 { - self.keyed.as_secs_f64() / self.plain.as_secs_f64() + self.per_arm[SHIPPED_KEYED].as_secs_f64() / self.per_arm[SHIPPED_PLAIN].as_secs_f64() } fn line(&self) -> String { - format!( - "{:<28} plain {:>9.2?} encrypted {:>9.2?} ratio {:>5.2}x", - self.what, - self.plain, - self.keyed, - self.ratio() - ) + let times: String = self + .per_arm + .iter() + .map(|d| format!("{:>18.2?}", d)) + .collect::>() + .join(" "); + format!("{:<28}{} ratio {:>5.2}x", self.what, times, self.ratio()) } } /// Time `find_duplicate_groups`, which opens its own connection — so the /// process key has to be right at call time, not at open time. -fn time_duplicates(path: &PathBuf, keyed: bool) -> Duration { - let db_path = path.to_string_lossy().into_owned(); - best_of(|| { - db::set_process_key(keyed.then(key)); +fn time_duplicates(arm: &Arm) -> Duration { + let db_path = arm.path.to_string_lossy().into_owned(); + let keyed = arm.keyed; + let out = best_of(|| { + db::set_process_key(keyed.then(measurement_key)); let groups = find_duplicate_groups(&db_path, 200).expect("duplicate scan"); assert!(!groups.is_empty(), "the seed must contain duplicate groups"); - }) + }); + db::set_process_key(None); + out } /// Time one cascade query on a connection opened while its key state was @@ -166,27 +195,42 @@ fn time_query(conn: &rusqlite::Connection, query: &str, fuzzy: bool) -> Duration }) } +/// Aliases for `testutil`'s arm order, naming the pair that is the shipped +/// product; the other two exist only to price the change against. +const SHIPPED_PLAIN: usize = ARM_PLAIN; +const SHIPPED_KEYED: usize = ARM_KEYED; + #[test] fn encryption_costs_a_constant_factor_not_a_different_algorithm() { - let (plain, keyed) = seed_both(); + let arms = seed_arms("encperf", &spec()); - // Both connections are opened up front, each under its own key state. - db::set_process_key(None); - let plain_conn = db::open::open_search_reader(&plain.to_string_lossy()).expect("open plain"); - db::set_process_key(Some(key())); - let keyed_conn = db::open::open_search_reader(&keyed.to_string_lossy()).expect("open keyed"); - db::set_process_key(None); + // Every connection is opened up front, each under its own key state. + let conns: Vec = arms.iter().map(Arm::open_search).collect(); println!( - "seeded {} files ({} with content): plain {:.1} MiB, encrypted {:.1} MiB", + "seeded {} files ({} with content) per arm\n", FILES, FILES / CONTENT_EVERY, - mib(&plain), - mib(&keyed), ); + println!( + "{:<28}{:>10}{:>12}{:>12}", + "arm", "size", "fts leaves", "overflow" + ); + for arm in &arms { + let (leaf, overflow) = arm.fts_pages(); + println!( + "{:<28}{:>7.1} MiB{:>12}{:>12}", + arm.what, + mib(arm.size_bytes()), + leaf, + overflow + ); + } + println!(); + assert!( - (mib(&plain) * 1024.0 * 1024.0) as u64 > SEARCH_CACHE_BYTES, - "seed is smaller than the {} MiB search cache, so both arms would be \ + arms[SHIPPED_PLAIN].size_bytes() > SEARCH_CACHE_BYTES, + "seed is smaller than the {} MiB search cache, so every arm would be \ served entirely from memory and every ratio below would be a \ meaningless 1.0 — raise FILES", SEARCH_CACHE_BYTES / (1024 * 1024) @@ -195,12 +239,11 @@ fn encryption_costs_a_constant_factor_not_a_different_algorithm() { // Duplicate finding first: it is the shape this gate exists for. let mut measured = vec![Measured { what: "find_duplicate_groups", - plain: time_duplicates(&plain, false), - keyed: time_duplicates(&keyed, true), + per_arm: arms.iter().map(time_duplicates).collect(), }]; // The cascade's four shapes. Arms alternate per workload so a machine that - // slows down partway through moves both sides, not one. + // slows down partway through moves all of them, not one. for (what, query, fuzzy) in [ ("cascade literal name", NEEDLE, false), ("cascade literal body", BODY_TERM, false), @@ -210,18 +253,62 @@ fn encryption_costs_a_constant_factor_not_a_different_algorithm() { ] { measured.push(Measured { what, - plain: time_query(&plain_conn, query, fuzzy), - keyed: time_query(&keyed_conn, query, fuzzy), + per_arm: conns + .iter() + .map(|conn| time_query(conn, query, fuzzy)) + .collect(), }); } + println!( + "\n{:<28}{}", + "workload", + arms.iter() + .map(|a| format!("{:>18}", a.what)) + .collect::>() + .join(" ") + ); for m in &measured { println!("{}", m.line()); } + // Deriving the record size from the page size has to beat pinning FTS5's + // own 4050 — for *both* key states. It used to be a keyed-only concern, + // when the page size was the 4096 that 4050 was chosen for; at + // `schema::PAGE_SIZE` neither key state gets a fitting leaf by accident. + for (pinned, derived, what) in [ + (ARM_PLAIN_4050, SHIPPED_PLAIN, "plain"), + (ARM_KEYED_4050, SHIPPED_KEYED, "keyed"), + ] { + let (before, after) = (arms[pinned].size_bytes(), arms[derived].size_bytes()); + assert!( + after <= before, + "the derived pgsz costs the {} index space: {:.1} MiB against \ + {:.1} MiB on FTS5's fixed 4050", + what, + mib(after), + mib(before) + ); + } + let keyed_after = arms[SHIPPED_KEYED].size_bytes(); + let size_ratio = keyed_after as f64 / arms[SHIPPED_PLAIN].size_bytes() as f64; + println!("\nencrypted/plain on disk: {:.3}x", size_ratio); + assert!( + size_ratio <= MAX_SIZE_RATIO, + "an encrypted index is {:.3}x the plain one on disk, over the {:.2}x \ + ceiling — the usual cause is FTS5 leaves that no longer fit inside \ + SQLCipher's reduced usable page", + size_ratio, + MAX_SIZE_RATIO + ); + + // Only the shipped pair: nothing is asserted about the two `pgsz 4050` + // arms, so their timings being at the noise floor costs a reader nothing. let too_short: Vec<&Measured> = measured .iter() - .filter(|m| m.plain < MIN_MEASURABLE || m.keyed < MIN_MEASURABLE) + .filter(|m| { + m.per_arm[SHIPPED_PLAIN] < MIN_MEASURABLE || m.per_arm[SHIPPED_KEYED] < MIN_MEASURABLE + }) .collect(); assert!( too_short.is_empty(), diff --git a/crates/quicksearch-core/tests/extraction_corpus.rs b/crates/quicksearch-core/tests/extraction_corpus.rs index 5ff3b05..d7dfeb8 100644 --- a/crates/quicksearch-core/tests/extraction_corpus.rs +++ b/crates/quicksearch-core/tests/extraction_corpus.rs @@ -59,13 +59,13 @@ fn every_format_extracts_its_planted_text() { let mime = mime::guess_mime_from_head(&sample.path, &head) .unwrap_or_else(|| panic!("{} no MIME resolved", ctx(sample))); assert!( - registry.supports(&mime), + registry.supports(mime), "{} MIME {mime:?} is claimed by no extractor", ctx(sample) ); let content = registry - .extract(&sample.path, &mime) + .extract_to_string(&sample.path, mime, &Config::default()) .unwrap_or_else(|e| panic!("{} extraction failed: {e}", ctx(sample))) .unwrap_or_else(|| panic!("{} MIME {mime:?} dispatched nowhere", ctx(sample))); @@ -92,7 +92,7 @@ fn head_extraction_agrees_with_reading_the_file() { let head = head_of(&sample.path); let mime = mime::guess_mime_from_head(&sample.path, &head).expect("MIME"); let whole = std::fs::read(&sample.path).expect("read whole file"); - let from_head = registry.extract_complete_head(&sample.path, &mime, &whole); + let from_head = registry.extract_head_to_string(&sample.path, mime, &whole); if !sample.head_path { // A format that seeks or reads a trailer must never be handed a @@ -110,7 +110,7 @@ fn head_extraction_agrees_with_reading_the_file() { .unwrap_or_else(|| panic!("{} declined the head path", ctx(sample))) .unwrap_or_else(|e| panic!("{} head extraction failed: {e}", ctx(sample))); let from_disk = registry - .extract(&sample.path, &mime) + .extract_to_string(&sample.path, mime, &Config::default()) .expect("on-disk extraction") .expect("claimed"); @@ -214,13 +214,14 @@ fn search(conn: &rusqlite::Connection, term: &str) -> Vec { #[test] fn rtf_unicode_escapes_survive_extraction() { let dir = quicksearch_core::testutil::scratch_dir("rtf-escapes"); + let registry = Registry::default_set(); let extract = |name: &str, body: &str| { - use quicksearch_core::extract::Extractor; let path = dir.join(name); std::fs::write(&path, body).unwrap(); - quicksearch_core::extract::rtf::RtfExtractor - .extract(&path) + registry + .extract_to_string(&path, "application/rtf", &Config::default()) .unwrap_or_else(|e| panic!("{name}: {e}")) + .unwrap_or_else(|| panic!("{name}: rtf dispatched nowhere")) }; // Both halves matter: the escape survives, and so does the word. diff --git a/crates/quicksearch-core/tests/full_index.rs b/crates/quicksearch-core/tests/full_index.rs index f9c2778..772aef9 100644 --- a/crates/quicksearch-core/tests/full_index.rs +++ b/crates/quicksearch-core/tests/full_index.rs @@ -1683,3 +1683,196 @@ fn high_byte_binaries_are_listed_but_not_text_extracted() { assert_eq!(probe("notes.md"), (1, 1, 0), "ordinary UTF-8 is unaffected"); } + +/// A helper for the two tail tests below: run to completion and to `Idle`, +/// which is the *end* of the post-run maintenance pass, sampling the log +/// throughout. Returns its peak. +/// +/// `Idle` and not the completion marker: the marker lands inside +/// `run_indexing`, before the FTS merge, the tail checkpoints and the whole of +/// `repo::maintain` — which is precisely the window under test. +fn reindex_sampling_the_log(root: &Path, db: &Path, config: &Config) -> u64 { + let wal = db.with_file_name(format!( + "{}-wal", + db.file_name().and_then(|s| s.to_str()).unwrap() + )); + let service = IndexingService::new(); + service + .start_indexing( + vec![root.to_string_lossy().into_owned()], + db.to_string_lossy().into_owned(), + config.clone(), + ) + .unwrap(); + + let mut peak = 0u64; + let deadline = Instant::now() + Duration::from_secs(180); + loop { + peak = peak.max(std::fs::metadata(&wal).map(|m| m.len()).unwrap_or(0)); + match service.get_status() { + IndexingStatus::Idle => break, + IndexingStatus::Error(e) => panic!("indexing failed: {}", e), + _ => {} + } + assert!(Instant::now() < deadline, "the run never reached Idle"); + std::thread::sleep(Duration::from_millis(1)); + } + drop(service); + peak +} + +/// A text-heavy tree, for the two tail tests. Big enough that the FTS index +/// dominates the database, which is what makes a log measured against the +/// database size mean anything. +fn seed_text_tree(tag: &str) -> Scratch { + let root = Scratch::dir(tag); + let body: Vec = "sphinx of black quartz judge my vow " + .repeat(200) + .into_bytes(); + for i in 0..4000 { + touch(&root.join(format!("d{}/f{:05}.txt", i % 40, i)), &body); + } + root +} + +/// The window [`the_wal_stays_bounded_during_a_run`] explicitly declines to +/// cover — "a reading taken afterwards proves nothing" — and the one a +/// released bug lived in. +/// +/// Everything after the writer loop is database work with no checkpoint of its +/// own: the FTS merge, the completion stamp, the per-root counts, then +/// `repo::maintain`'s VACUUM, whose copy-back pushes the whole database +/// through the log. With autocheckpoint off for the run and every per-root +/// reader still holding a read mark, that all piled onto one log — a warm +/// reindex, which writes almost nothing during the loop and so never trips the +/// in-loop checkpoint, left a `-wal` several times the size of the index. +/// +/// **A reader is held across both runs, and the test is vacuous without it.** +/// The application always has one — the search worker keeps its connection for +/// `IDLE_RELEASE`, half an hour. A test that does not leaves the indexer's +/// connection as the last handle on the file, and SQLite checkpoints and +/// *deletes* the log when the last one closes, papering over anything the run +/// failed to land. +/// +/// Two assertions, doing different jobs. The **mechanism** is that the tail's +/// checkpoints hand `repo::maintain` an empty log — read back through +/// [`repo::log_on_entry_to_maintain`], a latch, because the value is gone by +/// the time a test could sample it. That is what fails without the fix. The +/// **peak** is the guard on the reported symptom, and it is honest about its +/// limits: a fixture this size cannot build a tail large enough to breach the +/// ceiling on its own, so it protects the released behaviour rather than +/// reproducing the bug. +#[test] +fn the_wal_stays_bounded_through_the_tail_of_a_warm_reindex() { + let root = seed_text_tree("wal-tail"); + let db_dir = Scratch::dir("wal-tail-db"); + let db = db_dir.join("index.sqlite"); + let dir_key = db_dir.to_string_lossy().into_owned(); + let config = Config::default(); + + // An empty root first, purely to bring the index into existence so the + // reader below can be opened before the run that matters. + let empty = Scratch::dir("wal-tail-empty"); + reindex_sampling_the_log(&empty, &db, &config); + + let reader = rusqlite::Connection::open(&db).unwrap(); + // Lazily attached: without a statement there is no handle on the file yet, + // and the point of this connection is to be one. + reader + .query_row("SELECT COUNT(*) FROM files", [], |r| r.get::<_, i64>(0)) + .unwrap(); + + // The cold run is where the mechanism is visible: it fills the log, and + // 30-odd MiB against a 512 MiB default cap means the in-loop checkpoint + // never fires, so the tail's is the only one there is. + reindex_sampling_the_log(&root, &db, &config); + let indexed = std::fs::metadata(&db).unwrap().len(); + assert!( + indexed > 8 * 1024 * 1024, + "the fixture built a {} byte index; too small to measure a log against", + indexed + ); + assert_eq!( + quicksearch_core::db::repo::log_on_entry_to_maintain(&dir_key), + Some(0), + "the tail must land its log before the pass that VACUUMs through it" + ); + + // Nothing on disk has changed, so every byte of log below is the tail's. + let peak = reindex_sampling_the_log(&root, &db, &config); + + assert_eq!( + quicksearch_core::db::repo::log_on_entry_to_maintain(&dir_key), + Some(0), + "and a warm reindex's tail must land its own" + ); + drop(reader); + + // One VACUUM's copy-back is the largest thing the tail may legitimately + // write, plus the log's own 16 MiB floor. The bug cleared twice the index. + let ceiling = indexed + 16 * 1024 * 1024; + assert!( + peak < ceiling, + "the tail peaked at {} bytes of log against a {} byte index", + peak, + indexed + ); + assert_eq!( + wal_path(&db).metadata().map(|m| m.len()).unwrap_or(0), + 0, + "and the tail leaves no log behind" + ); +} + +fn wal_path(db: &Path) -> std::path::PathBuf { + db.with_file_name(format!( + "{}-wal", + db.file_name().and_then(|s| s.to_str()).unwrap() + )) +} + +/// A reindex that finds nothing to do must not grow the FTS index. +/// +/// The end-of-run merge takes a **positive** page budget for a reason. With a +/// negative one SQLite routes through `fts5IndexOptimizeStruct` — that is +/// `optimize`, merely rate-limited — hoisting every segment into a single +/// level and leaving the structure mid-merge in `%_data`. Called once per run +/// rather than looped to completion, as it was, each run restarted that and +/// churned pages for a corpus that had not changed. +#[test] +fn repeated_warm_reindexes_do_not_grow_the_fts_index() { + let root = seed_text_tree("fts-churn"); + let db_dir = Scratch::dir("fts-churn-db"); + let db = db_dir.join("index.sqlite"); + let config = Config::default(); + + let fts_pages = |db: &Path| -> i64 { + let conn = rusqlite::Connection::open(db).unwrap(); + conn.query_row( + "SELECT COUNT(*) FROM dbstat WHERE name = 'searchabletext_data'", + [], + |r| r.get(0), + ) + .unwrap() + }; + + reindex_sampling_the_log(&root, &db, &config); + let first = fts_pages(&db); + assert!(first > 0, "the fixture built no FTS index"); + + let mut sizes = vec![first]; + for _ in 0..3 { + reindex_sampling_the_log(&root, &db, &config); + sizes.push(fts_pages(&db)); + } + + // Not equality: `PRAGMA optimize` and a merge that consolidates real + // segments may move the figure either way once. Monotonic growth over + // three no-op runs is the signature of a structure that never settles. + let last = *sizes.last().unwrap(); + assert!( + last <= first, + "searchabletext_data grew across no-op reindexes: {:?} pages", + sizes + ); +} diff --git a/crates/quicksearch-core/tests/page_size.rs b/crates/quicksearch-core/tests/page_size.rs new file mode 100644 index 0000000..99d00d1 --- /dev/null +++ b/crates/quicksearch-core/tests/page_size.rs @@ -0,0 +1,387 @@ +//! Every layout the benches sweep has to survive a round trip through the real +//! open path, keyed and plain. +//! +//! Its own integration binary, and one `#[test]`, for the reason +//! `tests/encrypted.rs` gives: it mutates process-global state — the key, +//! `db::set_page_size_override` and `db::set_hmac_mode_override` — which unit +//! tests must never do, because the lib test binary runs them in parallel +//! against the same globals. +//! +//! The keyed half is the one that matters. A keyed file's header is +//! ciphertext, so SQLCipher cannot discover the layout by reading it: told the +//! wrong page size *or* the wrong HMAC mode — which sets the page reserve — +//! the header decrypts to noise and `key_and_probe` reports +//! `KEY_MISMATCH: wrong-password`. This pins that what we write with is what +//! we read back with, so that failure mode stays reachable only by actually +//! changing `schema::PROFILE` — which its doc comments spell out. + +use quicksearch_core::db; +use quicksearch_core::db::schema::{HmacMode, Profile}; +use quicksearch_core::testutil::{measurement_key, scratch_db, seed_index, SeedSpec}; + +/// The sweep, plus 16384 to keep one size above anything considered. +const SWEPT: [i64; 5] = [1024, 2048, 4096, 8192, 16384]; + +/// Every authenticator `benches/cipher_hmac.rs` prices. A page size is only +/// half the layout; the reserve is the other half and it is just as fatal to +/// get wrong. +const MODES: [HmacMode; 3] = [HmacMode::Off, HmacMode::Sha256, HmacMode::Sha512]; + +/// Enough documents to fill real FTS5 leaves at the largest page size here; +/// below that, nothing would ever reach the inline limit and the overflow +/// assertion would pass on a broken derivation. +const FILES: usize = 4_000; + +fn spec(profile: Profile) -> SeedSpec { + SeedSpec { + files: FILES, + content_every: 2, + page_size: Some(profile.page_size), + hmac: Some(profile.hmac), + ..SeedSpec::default() + } +} + +/// Put the process globals back where a fresh run would have them. +fn restore_shipped_profile() { + db::set_process_key(None); + db::set_page_size_override(db::schema::PAGE_SIZE); + db::set_hmac_mode_override(db::schema::HMAC_MODE); +} + +/// `PRAGMA page_size` answers as TEXT on a keyed connection and INTEGER +/// otherwise — the same quirk `db::repo::pragma_number` exists for, which is +/// crate-private. +fn page_size_of(conn: &rusqlite::Connection) -> i64 { + conn.query_row("PRAGMA page_size", [], |r| { + Ok(match r.get_ref(0)? { + rusqlite::types::ValueRef::Integer(n) => n, + rusqlite::types::ValueRef::Text(t) => { + std::str::from_utf8(t).unwrap().trim().parse().unwrap() + } + other => panic!("page_size answered {:?}", other.data_type()), + }) + }) + .unwrap() +} + +/// One `#[test]`, two phases, for the reason the header gives: both phases +/// drive the same process globals, so running them concurrently would have +/// each one moving the other's page size out from under it. +#[test] +fn page_geometry_round_trips_and_older_files_rebuild() { + every_swept_page_size_round_trips_keyed_and_plain(); + every_hmac_mode_round_trips(); + an_index_under_a_previous_profile_is_rebuilt_not_called_a_wrong_password(); +} + +fn every_swept_page_size_round_trips_keyed_and_plain() { + for page_size in SWEPT { + for keyed in [false, true] { + let profile = Profile { + page_size, + hmac: db::schema::HMAC_MODE, + }; + let path = scratch_db(&format!("pagesize-{}-{}", page_size, keyed)); + db::set_process_key(keyed.then(measurement_key)); + seed_index(&path, &spec(profile)); + + // A *fresh* open, which is where a keyed file at an unexpected + // page size would fail outright. + let conn = db::open_existing(&path.to_string_lossy(), false).unwrap_or_else(|e| { + panic!("reopen page_size={} keyed={}: {}", page_size, keyed, e) + }); + + assert_eq!( + page_size_of(&conn), + page_size, + "page_size={} keyed={}: the file came back at another size", + page_size, + keyed + ); + + let rows: i64 = conn + .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) + .unwrap(); + assert_eq!( + rows, FILES as i64, + "reopen must find the corpus, not wipe it" + ); + + // The derived pgsz has to keep FTS5 leaves inline at every size, + // which is the whole reason it is derived rather than pinned. + let overflow: i64 = conn + .query_row( + "SELECT COUNT(*) FROM dbstat \ + WHERE name = 'searchabletext_data' AND pagetype = 'overflow'", + [], + |r| r.get(0), + ) + .unwrap(); + let leaves: i64 = conn + .query_row( + "SELECT COUNT(*) FROM dbstat \ + WHERE name = 'searchabletext_data' AND pagetype = 'leaf'", + [], + |r| r.get(0), + ) + .unwrap(); + assert!( + leaves > 10, + "page_size={} keyed={}: {} leaves is too few to have filled any", + page_size, + keyed, + leaves + ); + assert_eq!( + overflow, 0, + "page_size={} keyed={}: {} of {} leaves overflowed — the \ + derived pgsz missed the inline limit", + page_size, keyed, overflow, leaves + ); + + drop(conn); + std::fs::remove_dir_all(path.parent().unwrap()).ok(); + } + } + + restore_shipped_profile(); +} + +/// The same round trip across the other half of the layout. A keyed file +/// written under one authenticator and read under another does not decrypt at +/// all, so `benches/cipher_hmac.rs` can only compare modes if each one +/// survives its own open — and the FTS5 derivation has to follow the reserve +/// or the arm being measured is one full of overflow pages. +fn every_hmac_mode_round_trips() { + for hmac in MODES { + for keyed in [false, true] { + let profile = Profile { + page_size: db::schema::PAGE_SIZE, + hmac, + }; + let path = scratch_db(&format!("hmac-{}-{}", hmac.label(), keyed)); + db::set_process_key(keyed.then(measurement_key)); + db::set_hmac_mode_override(hmac); + seed_index(&path, &spec(profile)); + + let conn = db::open_existing(&path.to_string_lossy(), false) + .unwrap_or_else(|e| panic!("reopen hmac={:?} keyed={}: {}", hmac, keyed, e)); + + let rows: i64 = conn + .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) + .unwrap(); + assert_eq!( + rows, FILES as i64, + "hmac={:?} keyed={}: reopen must find the corpus, not wipe it", + hmac, keyed + ); + + let overflow: i64 = conn + .query_row( + "SELECT COUNT(*) FROM dbstat \ + WHERE name = 'searchabletext_data' AND pagetype = 'overflow'", + [], + |r| r.get(0), + ) + .unwrap(); + let leaves: i64 = conn + .query_row( + "SELECT COUNT(*) FROM dbstat \ + WHERE name = 'searchabletext_data' AND pagetype = 'leaf'", + [], + |r| r.get(0), + ) + .unwrap(); + assert!( + leaves > 10, + "hmac={:?} keyed={}: {} leaves is too few to have filled any", + hmac, + keyed, + leaves + ); + assert_eq!( + overflow, 0, + "hmac={:?} keyed={}: {} of {} leaves overflowed — the derived \ + pgsz did not follow the reserve this mode sets", + hmac, keyed, overflow, leaves + ); + + // A protected arm has to actually be encrypted whatever the + // authenticator: `cipher_use_hmac = OFF` weakens the file, it does + // not turn the cipher off. + drop(conn); + if keyed { + let head = std::fs::read(&path).unwrap(); + assert_ne!( + &head[..16], + b"SQLite format 3\0", + "hmac={:?}: the file is plaintext, not merely unauthenticated", + hmac + ); + } + std::fs::remove_dir_all(path.parent().unwrap()).ok(); + } + } + + restore_shipped_profile(); +} + +/// The migration this build has to survive: indexes already on disk were built +/// under a `PROFILES_PREVIOUS` layout, and a keyed one read under the wrong +/// page size *or* the wrong page reserve decrypts to noise. Without the +/// reopen-under-the-old-profile retry every such index would come back as +/// `KEY_MISMATCH: wrong-password` — an accusation the user cannot act on, +/// against a password that is perfectly correct. +/// +/// Each entry is checked twice over: once with the schema version rolled back, +/// as a page-size change always came with, and once left at the current +/// version. The second case is the one an HMAC change introduces — the file +/// reads back perfectly under its old profile, so nothing but the profile +/// itself says it is stale. +fn an_index_under_a_previous_profile_is_rebuilt_not_called_a_wrong_password() { + // This test iterates the list, so an empty one would make it vacuous + // rather than failing — and an empty one is exactly the regression it + // exists to catch. Entries may only be dropped when it is acceptable for + // indexes under that layout to read as a wrong password. + assert!( + !db::schema::PROFILES_PREVIOUS.is_empty(), + "PROFILES_PREVIOUS is empty: every index built under an earlier \ + layout now reports KEY_MISMATCH instead of rebuilding, and this test \ + would have said nothing about it" + ); + let mut checked = 0; + for keyed in [false, true] { + for previous in db::schema::PROFILES_PREVIOUS { + for stale_version in [true, false] { + checked += 1; + let tag = format!( + "prevprofile-{}-{}-{}-{}", + previous.page_size, + previous.hmac.label(), + keyed, + stale_version + ); + let path = scratch_db(&tag); + let db_path = path.to_string_lossy().into_owned(); + let what = format!( + "profile=({}) keyed={} stale_version={}", + previous, keyed, stale_version + ); + + // Build one the way the old version would have. + db::set_process_key(keyed.then(measurement_key)); + seed_index(&path, &spec(*previous)); + { + let conn = db::open_existing(&db_path, true).unwrap(); + assert_eq!( + page_size_of(&conn), + previous.page_size, + "{}: the fixture must actually be at {} bytes", + what, + previous.page_size + ); + if stale_version { + conn.execute( + "UPDATE schema_info SET value = ?1 WHERE key = 'version'", + [(db::CURRENT_SCHEMA_VERSION - 1).to_string()], + ) + .unwrap(); + } + } + + // Back to the shipped layout, as a new build would be. + db::set_page_size_override(db::schema::PAGE_SIZE); + db::set_hmac_mode_override(db::schema::HMAC_MODE); + + // An unencrypted file is the one case where the profile is not + // a staleness signal, and that is by design: it has no + // reserve, so the HMAC half never applied to it, and SQLite + // ignores `PRAGMA page_size` on a file that exists, so it just + // opens at whatever it was built with. Only the schema version + // can condemn it — and the version is what a page-size change + // has always moved alongside. Pin the *positive* behaviour so + // this stays a decision rather than a gap. + if !keyed && !stale_version { + db::open_existing(&db_path, false).unwrap_or_else(|e| { + panic!( + "{}: a plain index at the current schema version \ + must stay usable whatever its page size: {}", + what, e + ) + }); + assert!( + !db::index_needs_rebuild(&db_path), + "{}: and it must not be condemned to a rebuild either", + what + ); + db::set_process_key(None); + std::fs::remove_dir_all(path.parent().unwrap()).ok(); + continue; + } + + // The key still verifies: it is the right key, on a file whose + // layout this build no longer writes. + db::verify_process_key(&db_path) + .unwrap_or_else(|e| panic!("{}: a correct password was refused: {}", what, e)); + + // A consumer is told to re-index rather than that the file is + // unreadable or the password wrong. + let refusal = db::open_existing(&db_path, false).unwrap_err(); + assert!( + !refusal.starts_with(db::KEY_MISMATCH_PREFIX), + "{}: consumers must not see a key error: {}", + what, + refusal + ); + assert!( + refusal.contains("Re-index"), + "{}: the refusal must say what to do: {}", + what, + refusal + ); + + // And the indexer announces the rebuild before doing it. + assert!( + db::index_needs_rebuild(&db_path), + "{}: the rebuild must be announced", + what + ); + + // The rebuild lands at the current layout, encrypted if it was. + let conn = db::open_or_recreate(&db_path, "trigram") + .unwrap_or_else(|e| panic!("{}: rebuild failed: {}", what, e)); + assert_eq!( + page_size_of(&conn), + db::schema::PAGE_SIZE, + "{}: the rebuilt file must adopt the current page size", + what + ); + let rows: i64 = conn + .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) + .unwrap(); + assert_eq!(rows, 0, "a rebuild starts empty; the walk refills it"); + drop(conn); + + if keyed { + let head = std::fs::read(&path).unwrap(); + assert_ne!( + &head[..16], + b"SQLite format 3\0", + "a rebuilt protected index must come back encrypted" + ); + } + + db::set_process_key(None); + std::fs::remove_dir_all(path.parent().unwrap()).ok(); + } + } + } + assert_eq!( + checked, + 4 * db::schema::PROFILES_PREVIOUS.len(), + "every previous profile has to be checked keyed and plain, at a stale \ + schema version and at the current one" + ); + restore_shipped_profile(); +} diff --git a/crates/quicksearch-core/tests/prefilter_fuzz.rs b/crates/quicksearch-core/tests/prefilter_fuzz.rs index 2b88b25..8c26bc8 100644 --- a/crates/quicksearch-core/tests/prefilter_fuzz.rs +++ b/crates/quicksearch-core/tests/prefilter_fuzz.rs @@ -80,10 +80,10 @@ fn bodies() -> Vec { "planning", ]; for d in 0..DOCS { - let n = 20 + (lcg.next() as usize % 40); + let n = 20 + (lcg.next_u64() as usize % 40); let mut body = String::new(); for _ in 0..n { - body.push_str(words[lcg.next() as usize % words.len()]); + body.push_str(words[lcg.next_u64() as usize % words.len()]); body.push(' '); } // Fixed shapes beside the random ones: all-one-character, and empty. @@ -129,13 +129,13 @@ fn oracle_distance(pattern: &[u8], hay: &[u8]) -> usize { fn corrupt(chars: &mut Vec, lcg: &mut Lcg) { // Includes multi-byte replacements: one character edit, several bytes. const REPLACEMENTS: [char; 6] = ['x', 'Q', '7', 'é', '語', '🙂']; - let pick = REPLACEMENTS[lcg.next() as usize % REPLACEMENTS.len()]; + let pick = REPLACEMENTS[lcg.next_u64() as usize % REPLACEMENTS.len()]; if chars.is_empty() { chars.push(pick); return; } - let at = lcg.next() as usize % chars.len(); - match lcg.next() % 3 { + let at = lcg.next_u64() as usize % chars.len(); + match lcg.next_u64() % 3 { 0 => chars[at] = pick, // substitution 1 => chars.insert(at, pick), // insertion _ => { @@ -154,7 +154,7 @@ fn substring_of<'a>(body: &'a str, len: usize, lcg: &mut Lcg) -> Option<&'a str> if chars < len || len == 0 { return None; } - let start = lcg.next() as usize % (chars - len + 1); + let start = lcg.next_u64() as usize % (chars - len + 1); Some(&body[bounds[start]..bounds[start + len]]) } @@ -175,7 +175,7 @@ fn a_surviving_chunk_always_remains_after_k_edits() { for &cap in &CAPS { for len in sweep_lengths() { for _ in 0..ITERS_PER_LEN { - let body = &bodies[lcg.next() as usize % bodies.len()]; + let body = &bodies[lcg.next_u64() as usize % bodies.len()]; let Some(original) = substring_of(body, len, &mut lcg) else { continue; }; @@ -183,7 +183,7 @@ fn a_surviving_chunk_always_remains_after_k_edits() { continue; }; let mut chars: Vec = original.chars().collect(); - let edits = lcg.next() as usize % (k + 1); + let edits = lcg.next_u64() as usize % (k + 1); for _ in 0..edits { corrupt(&mut chars, &mut lcg); } @@ -342,7 +342,7 @@ fn a_corrupted_substring_still_finds_the_document_it_came_from() { for &cap in caps() { for len in sweep_lengths() { for _ in 0..iters_per_len() { - let doc = lcg.next() as usize % bodies.len(); + let doc = lcg.next_u64() as usize % bodies.len(); let body = &bodies[doc]; let Some(original) = substring_of(body, len, &mut lcg) else { continue; @@ -353,7 +353,7 @@ fn a_corrupted_substring_still_finds_the_document_it_came_from() { let edits = if planned == 0 { 0 } else { - lcg.next() as usize % (planned + 1) + lcg.next_u64() as usize % (planned + 1) }; for _ in 0..edits { corrupt(&mut chars, &mut lcg); @@ -451,7 +451,7 @@ fn every_document_within_the_budget_is_found_and_nothing_outside_it_is() { for &cap in caps() { for len in sweep_lengths() { for _ in 0..ITERS { - let doc = lcg.next() as usize % bodies.len(); + let doc = lcg.next_u64() as usize % bodies.len(); let Some(original) = substring_of(&bodies[doc], len, &mut lcg) else { continue; }; @@ -460,7 +460,7 @@ fn every_document_within_the_budget_is_found_and_nothing_outside_it_is() { let edits = if planned == 0 { 0 } else { - lcg.next() as usize % (planned + 1) + lcg.next_u64() as usize % (planned + 1) }; for _ in 0..edits { corrupt(&mut chars, &mut lcg); @@ -551,7 +551,7 @@ fn regexify(sub: &str, lcg: &mut Lcg) -> String { let mut out = String::new(); for c in sub.chars() { // Most characters stay literal, or every literal set goes empty. - match lcg.next() % 10 { + match lcg.next_u64() % 10 { 0 if c != '\n' => out.push('.'), 1 => out.push_str(&format!("[{}z]", esc(c))), 2 => out.push_str(&format!("(?:{}|zzq)", esc(c))), @@ -560,7 +560,7 @@ fn regexify(sub: &str, lcg: &mut Lcg) -> String { _ => out.push_str(&esc(c)), } } - match lcg.next() % 6 { + match lcg.next_u64() % 6 { 0 => format!(".*{out}"), 1 => format!("{out}.*"), _ => out, @@ -585,7 +585,7 @@ fn a_regex_finds_exactly_the_documents_it_matches() { for len in sweep_lengths() { for _ in 0..iters_per_len().min(40) { - let doc = lcg.next() as usize % bodies.len(); + let doc = lcg.next_u64() as usize % bodies.len(); let Some(sub) = substring_of(&bodies[doc], len, &mut lcg) else { continue; }; diff --git a/crates/quicksearch-core/tests/release_free_heap.rs b/crates/quicksearch-core/tests/release_free_heap.rs new file mode 100644 index 0000000..9a02862 --- /dev/null +++ b/crates/quicksearch-core/tests/release_free_heap.rs @@ -0,0 +1,74 @@ +//! `release_free_heap` has to actually return memory to the kernel. +//! +//! Its own integration binary because it needs a `#[global_allocator]`, which +//! only takes effect in the crate that declares it — the lib test harness +//! declares none, so a unit test here would measure the system allocator and +//! pass no matter what `release_free_heap` did. +//! +//! The failure this guards is silent. `mi_collect` is reached through an +//! `extern "C"` block that `libmimalloc-sys` does not provide a binding for, +//! so deleting the call, or the declaration drifting from +//! `void mi_collect(bool)`, leaves a build that compiles, runs, and quietly +//! keeps every run's peak resident for the life of the process. That is +//! exactly the bug this replaced: glibc settled a multi-million-file run at +//! 985 MB with 871 MB of unreturnable slack. + +#[global_allocator] +static GLOBAL: quicksearch_core::platform::Allocator = quicksearch_core::platform::Allocator; + +/// Blocks big enough to be worth returning and small enough to come from the +/// allocator's segments rather than a direct `mmap` — the mid-size churn that +/// fragments, not the large buffers that were always given back cleanly. +const BLOCK: usize = 100 * 1024; +const BLOCKS: usize = 4_000; + +#[cfg(target_os = "linux")] +fn rss_bytes() -> u64 { + let status = std::fs::read_to_string("/proc/self/status").expect("/proc/self/status"); + status + .lines() + .find_map(|l| l.strip_prefix("VmRSS:")) + .and_then(|rest| rest.split_whitespace().next()?.parse::().ok()) + .expect("VmRSS") + * 1024 +} + +#[cfg(target_os = "linux")] +#[test] +fn releasing_the_free_heap_returns_it_to_the_kernel() { + let mib = |bytes: u64| bytes as f64 / (1024.0 * 1024.0); + + let before = rss_bytes(); + // Written to, not just reserved: untouched pages are never resident, so + // an unwritten allocation would prove nothing about reclaiming one. + let mut held: Vec> = (0..BLOCKS) + .map(|i| vec![(i % 251) as u8; BLOCK]) + .collect(); + let peak = rss_bytes(); + assert!( + peak > before + (BLOCKS * BLOCK / 2) as u64, + "the corpus never became resident: {:.0} MiB to {:.0} MiB", + mib(before), + mib(peak) + ); + + held.clear(); + held.shrink_to_fit(); + let freed = rss_bytes(); + + quicksearch_core::platform::release_free_heap(); + let released = rss_bytes(); + + // Deliberately loose: the point is order-of-magnitude reclamation, not a + // figure that drifts with allocator versions. Measured 454 MiB peak, still + // 454 MiB after the frees, 5 MiB after the call. + assert!( + released < before + (BLOCKS * BLOCK / 4) as u64, + "release_free_heap kept {:.0} MiB resident (started {:.0}, peaked {:.0}, \ + {:.0} after freeing) — mi_collect is not reaching the allocator", + mib(released), + mib(before), + mib(peak), + mib(freed) + ); +} diff --git a/crates/quicksearch-core/tests/search_cache.rs b/crates/quicksearch-core/tests/search_cache.rs new file mode 100644 index 0000000..d0098d1 --- /dev/null +++ b/crates/quicksearch-core/tests/search_cache.rs @@ -0,0 +1,127 @@ +//! The search connection has to *open* at the ceiling it was sized for. +//! +//! `db::schema`'s unit tests cover the arithmetic; this covers the wiring, +//! which is where it can silently do nothing: the ceiling is resolved inside +//! `open_search_reader` from a `sqlite_stat1` read and a process-global, and a +//! break anywhere along that path leaves a connection quietly running on the +//! read-only profile's 4 MiB with every test still green. +//! +//! Its own integration binary, and one `#[test]`, for the reason +//! `tests/encrypted.rs` gives: it drives process-global state — the key and +//! the cache override — which unit tests must never do, because the lib test +//! binary runs them in parallel against the same globals. + +use quicksearch_core::db; +use quicksearch_core::db::schema::{ + recommended_search_cache_mib, SEARCH_CACHE_MAX_MIB, SEARCH_CACHE_PLAIN_MIB, +}; +use quicksearch_core::testutil::{measurement_key, scratch_db, seed_index, SeedSpec}; + +/// Enough rows that the recommendation clears the floor and is therefore +/// actually derived rather than clamped — at 168 B/file, 16 MiB is reached +/// around 100k. Seeded without content: the FTS write is the slow part and +/// this measures nothing about it. +const FILES: usize = 150_000; + +fn spec() -> SeedSpec { + SeedSpec { + files: FILES, + // No document bodies at all; `content_every` past `files` never fires. + content_every: FILES + 1, + commit_every: 25_000, + dup_every: 2, + dir_depth: 6, + ..SeedSpec::default() + } +} + +/// `PRAGMA cache_size` reads back as the negative KiB it was set to. +fn cache_mib(conn: &rusqlite::Connection) -> i64 { + let kib: i64 = conn + .query_row("PRAGMA cache_size", [], |r| r.get(0)) + .unwrap(); + assert!( + kib < 0, + "cache_size came back as {} — a positive value is a *page* count, \ + which would mean the KiB form never reached SQLite", + kib + ); + -kib / 1024 +} + +#[test] +fn the_search_connection_opens_at_the_ceiling_it_was_sized_for() { + let path = scratch_db("searchcache"); + let db_path = path.to_string_lossy().into_owned(); + + // --- keyed, automatic ------------------------------------------------- + db::set_process_key(Some(measurement_key())); + db::set_search_cache_override(None); + seed_index(&path, &spec()); + + // Without stats there is nothing to derive from, so the floor is correct + // and is what must come out. + { + let conn = db::open::open_search_reader(&db_path).unwrap(); + assert_eq!( + cache_mib(&conn), + recommended_search_cache_mib(0, true), + "a never-analyzed index must fall back, not guess" + ); + } + + // `PRAGMA optimize` is what a real indexing run leaves behind; from here + // the row count is readable and the ceiling should track it. + { + let conn = db::open_existing(&db_path, true).unwrap(); + conn.execute_batch("ANALYZE;").unwrap(); + } + let expected = recommended_search_cache_mib(FILES as i64, true); + assert!( + expected > recommended_search_cache_mib(0, true), + "the corpus must be big enough to clear the floor, or this proves nothing" + ); + { + let conn = db::open::open_search_reader(&db_path).unwrap(); + assert_eq!( + cache_mib(&conn), + expected, + "the derived ceiling did not reach the connection" + ); + } + + // --- an explicit override wins, including past the automatic cap ------ + let big = SEARCH_CACHE_MAX_MIB * 2; + db::set_search_cache_override(Some(big)); + { + let conn = db::open::open_search_reader(&db_path).unwrap(); + assert_eq!( + cache_mib(&conn), + big, + "an explicit ceiling must be applied verbatim, above the cap" + ); + } + db::set_search_cache_override(None); + + // --- plain, automatic ------------------------------------------------- + // Same corpus, no key: the sweep found no knee unencrypted, so this must + // be flat regardless of how large the index is. + let plain = scratch_db("searchcache-plain"); + db::set_process_key(None); + seed_index(&plain, &spec()); + { + let conn = db::open_existing(&plain.to_string_lossy(), true).unwrap(); + conn.execute_batch("ANALYZE;").unwrap(); + } + { + let conn = db::open::open_search_reader(&plain.to_string_lossy()).unwrap(); + assert_eq!( + cache_mib(&conn), + SEARCH_CACHE_PLAIN_MIB, + "an unencrypted index must not grow its cache with the corpus" + ); + } + + std::fs::remove_dir_all(path.parent().unwrap()).ok(); + std::fs::remove_dir_all(plain.parent().unwrap()).ok(); +} diff --git a/crates/quicksearch-gui/src/app.rs b/crates/quicksearch-gui/src/app.rs index 14c7034..2f3a121 100644 --- a/crates/quicksearch-gui/src/app.rs +++ b/crates/quicksearch-gui/src/app.rs @@ -9,7 +9,8 @@ use quicksearch_core::config::{diff_actions, nested_roots, Config, SecurityConfi use quicksearch_core::coordinator::{IndexMode, IndexerState, ReconcileState, WatcherStatus}; use quicksearch_core::db; use quicksearch_core::indexing::{ - overall_progress, ConfigChange, IndexingStatus, PrepStep, RootPhase, RootProgress, + overall_progress, ConfigChange, IndexingStatus, MaintenanceStep, PrepStep, RootPhase, + RootProgress, }; use quicksearch_core::platform::{IndexLock, LockError}; use quicksearch_core::search::SearchOptions; @@ -392,6 +393,17 @@ impl QuickSearchApp { search.set_db_path(new.resolved_database_path()); } } + // The ceiling is a property of the connection, so it only takes effect + // on the next open — release the one being held rather than leaving + // the setting to appear ignored until the next idle timeout. + if new.search.cache_size_mib != self.cfg.search.cache_size_mib { + quicksearch_core::db::set_search_cache_override( + (new.search.cache_size_mib != 0).then_some(new.search.cache_size_mib as i64), + ); + if let Some(search) = self.backend.search() { + search.release_connection(); + } + } // Only settings that leave the stored file unreadable need a rebuild. self.backend.coordinator.apply_config(new.clone()); if actions.requires_rebuild { @@ -580,8 +592,11 @@ pub(crate) fn pin_live_fields(new: &mut Config, live: &Config) { new.security = live.security.clone(); new.indexing.auto_index = live.indexing.auto_index; // The column picker writes straight to the live config; pinning stops a - // draft taken before a header-menu change from undoing it on Apply. + // draft taken before a header-menu change from undoing it on Apply. The + // advanced-settings toggle is written the same way and needs the same + // protection — Apply must not put the rows away again. new.search.columns = live.search.columns.clone(); + new.ui.show_advanced_settings = live.ui.show_advanced_settings; } fn clamp_scale(scale: f32) -> f32 { @@ -751,7 +766,8 @@ impl eframe::App for QuickSearchApp { } } Tab::Settings => { - let out = self.settings.ui(ui, &self.cfg); + let indexed_files = self.backend.coordinator.state().files; + let out = self.settings.ui(ui, &self.cfg, indexed_files); if let Some(new_cfg) = out.applied { self.apply_new_config(ctx, new_cfg); } @@ -765,6 +781,12 @@ impl eframe::App for QuickSearchApp { self.search.mark_sort_dirty(); self.save_cfg(); } + // Likewise: revealing a setting is not an edit to one, so it + // takes effect and is remembered without an Apply. + if let Some(show_advanced) = out.show_advanced { + self.cfg.ui.show_advanced_settings = show_advanced; + self.save_cfg(); + } } }); diff --git a/crates/quicksearch-gui/src/app/modals.rs b/crates/quicksearch-gui/src/app/modals.rs index 48723ab..ee21f4c 100644 --- a/crates/quicksearch-gui/src/app/modals.rs +++ b/crates/quicksearch-gui/src/app/modals.rs @@ -93,9 +93,16 @@ impl QuickSearchApp { if !self.stale_index_prompt { return; } - if stale_index_window(ctx, self.key_source) { + let command = stale_prompt_should_command( + self.backend.coordinator.state().mode, + self.backend.coordinator.is_indexing(), + ); + if stale_index_window(ctx, self.key_source, command) { self.stale_index_prompt = false; - self.backend.rebuild_index(); + if command { + self.backend.rebuild_index(); + } + // Either way the index is being replaced under the tab. self.dups.state = DupState::NotLoaded; } } @@ -431,7 +438,17 @@ fn reconcile_quit_modal(ctx: &egui::Context) -> Option { choice } -fn stale_index_window(ctx: &egui::Context, key_source: KeySource) -> bool { +/// Whether the stale-index button has to command the rebuild itself. +/// +/// In Auto the coordinator already does: the recreated index has no +/// `last_full_index`, so its first tick schedules a full run. Sending +/// `RebuildIndex` on top of that deletes the rebuild in progress and starts +/// it over from zero. Only manual mode needs the button to do anything. +pub(super) fn stale_prompt_should_command(mode: IndexMode, indexing: bool) -> bool { + mode != IndexMode::Auto && !indexing +} + +fn stale_index_window(ctx: &egui::Context, key_source: KeySource, command: bool) -> bool { centered_modal(ctx, "Index reset for this version", |ui| { ui.set_max_width(440.0); ui.label( @@ -460,7 +477,8 @@ fn stale_index_window(ctx: &egui::Context, key_source: KeySource) -> bool { until the rebuild finishes; progress is on the Manage Index tab.", )); ui.add_space(4.0); - ui.button("Rebuild now").clicked() + ui.button(if command { "Rebuild now" } else { "Continue" }) + .clicked() }) .unwrap_or(false) } @@ -477,10 +495,17 @@ fn display_value(value: &str) -> String { mod tests { use super::*; - fn frame(ctx: &egui::Context, source: KeySource, events: Vec) -> bool { + fn frame( + ctx: &egui::Context, + source: KeySource, + command: bool, + events: Vec, + ) -> bool { let input = crate::test_ui::raw_input(SCREEN, events); let mut clicked = false; - let _ = ctx.run(input, |ctx| clicked = stale_index_window(ctx, source)); + let _ = ctx.run(input, |ctx| { + clicked = stale_index_window(ctx, source, command) + }); clicked } @@ -495,27 +520,31 @@ mod tests { KeySource::Prompt, KeySource::Keychain, ] { - let ctx = crate::test_ui::ctx(); - assert!( - !frame(&ctx, source, Vec::new()), - "an untouched frame must not request a rebuild" - ); + // Both button labels: "Rebuild now" when the click starts the + // rebuild, "Continue" when one is already running. + for command in [true, false] { + let ctx = crate::test_ui::ctx(); + assert!( + !frame(&ctx, source, command, Vec::new()), + "an untouched frame must not request a rebuild" + ); - // The window's height depends on which sentence is shown; sweep. - let mut fired = None; - 'sweep: for y in (230..480).step_by(3) { - for x in (250..760).step_by(6) { - let pos = egui::pos2(x as f32, y as f32); - if frame(&ctx, source, click_at(pos)) { - fired = Some(pos); - break 'sweep; + // The window's height depends on which sentence is shown; sweep. + let mut fired = None; + 'sweep: for y in (230..480).step_by(3) { + for x in (250..760).step_by(6) { + let pos = egui::pos2(x as f32, y as f32); + if frame(&ctx, source, command, click_at(pos)) { + fired = Some(pos); + break 'sweep; + } } } + assert!( + fired.is_some(), + "no clickable button for {source:?} (command: {command})" + ); } - assert!( - fired.is_some(), - "no clickable Rebuild button for {source:?}" - ); } } diff --git a/crates/quicksearch-gui/src/app/security.rs b/crates/quicksearch-gui/src/app/security.rs index a89e7e0..4e7d444 100644 --- a/crates/quicksearch-gui/src/app/security.rs +++ b/crates/quicksearch-gui/src/app/security.rs @@ -408,7 +408,26 @@ fn confirm_key_modal(ctx: &egui::Context, pw: &mut String, wrong: bool) -> (bool .unwrap_or((false, false)) } +/// The page size other SQLCipher tools assume unless told otherwise, and so +/// the one this screen has to talk them out of. +const SQLCIPHER_DEFAULT_PAGE_SIZE: i64 = 4096; + +/// The pragma another tool needs for our page authenticator, spelled out +/// because the setting is not one anybody would guess: the reserve it decides +/// is part of the on-disk layout, so getting it wrong reads as a bad key +/// rather than as a failed integrity check. +fn hmac_pragma_hint() -> &'static str { + use quicksearch_core::db::schema::{HmacMode, HMAC_MODE}; + match HMAC_MODE { + HmacMode::Off => "PRAGMA cipher_use_hmac = OFF;", + HmacMode::Sha256 => "PRAGMA cipher_hmac_algorithm = HMAC_SHA256;", + HmacMode::Sha512 => "", + } +} + fn reveal_key_modal(ctx: &egui::Context, display: &str) -> (bool, bool) { + use quicksearch_core::db::schema::{HMAC_MODE, PAGE_SIZE}; + centered_modal(ctx, "Database key", |ui| { ui.set_max_width(420.0); ui.label( @@ -420,10 +439,23 @@ fn reveal_key_modal(ctx: &egui::Context, display: &str) -> (bool, bool) { ui.label(egui::RichText::new(display).monospace()); }); ui.add_space(6.0); - ui.label(hint( - "Other SQLCipher tools take the key in this form. A copy stays on the \ - clipboard until something else replaces it.", - )); + // The key on its own is not enough: under any other page size or + // page authenticator the file decrypts to noise, and every tool + // reports that as a wrong key. Both are shown as their own lines, not + // in the small print, because both have to be entered alongside the + // key. + ui.label(egui::RichText::new(format!("Page size: {}", PAGE_SIZE)).monospace()); + ui.label(egui::RichText::new(format!("Page HMAC: {}", HMAC_MODE.label())).monospace()); + ui.add_space(6.0); + ui.label(hint(format!( + "Other SQLCipher tools take the key in this form, but default to \ + {}-byte pages and HMAC_SHA512 — set both of the above as well or \ + the index will not open ({} {}). Copy puts the key alone on the \ + clipboard, where it stays until something else replaces it.", + SQLCIPHER_DEFAULT_PAGE_SIZE, + format_args!("PRAGMA cipher_page_size = {};", PAGE_SIZE), + hmac_pragma_hint(), + ))); ui.add_space(6.0); ui.horizontal(|ui| (ui.button("Copy").clicked(), ui.button("Close").clicked())) .inner diff --git a/crates/quicksearch-gui/src/app/security_tests.rs b/crates/quicksearch-gui/src/app/security_tests.rs index 4814a3a..5e6b1f9 100644 --- a/crates/quicksearch-gui/src/app/security_tests.rs +++ b/crates/quicksearch-gui/src/app/security_tests.rs @@ -142,6 +142,68 @@ fn the_reveal_shows_the_key_and_what_holding_it_means() { assert!(painted.contains(&"Close".to_string()), "{painted:?}"); } +/// The key alone opens nothing: a tool left on SQLCipher's defaults decrypts +/// this file to noise and calls the key wrong. The screen has to say both +/// halves of the layout, as the values the other tool needs typed in. +#[test] +fn the_reveal_shows_the_layout_the_index_was_built_under() { + use quicksearch_core::db::schema::{HMAC_MODE, PAGE_SIZE}; + + let ctx = crate::test_ui::ctx(); + let painted = painted_text(&frame(&ctx, &format!("0x{KEY}"), Vec::new()).0); + + assert!( + painted.contains(&format!("Page size: {PAGE_SIZE}")), + "the page size is not on screen: {painted:?}" + ); + assert!( + painted.contains(&format!("Page HMAC: {}", HMAC_MODE.label())), + "the page authenticator is not on screen: {painted:?}" + ); + assert!( + painted.iter().any(|t| t.contains("set both of the above")), + "nothing says the layout has to be entered too: {painted:?}" + ); + assert_ne!( + PAGE_SIZE, SQLCIPHER_DEFAULT_PAGE_SIZE, + "the advice only makes sense while the index is off the default" + ); +} + +/// A tool cannot be told "HMAC off" in prose — it needs the pragma. The hint +/// carries it whenever the index is off SQLCipher's default authenticator, and +/// omits it when there is nothing to say. +#[test] +fn the_reveal_spells_out_the_hmac_pragma_when_there_is_one() { + use quicksearch_core::db::schema::{HmacMode, HMAC_MODE, PAGE_SIZE}; + + let ctx = crate::test_ui::ctx(); + let painted = painted_text(&frame(&ctx, &format!("0x{KEY}"), Vec::new()).0); + let hint = painted + .iter() + .find(|t| t.contains("set both of the above")) + .unwrap_or_else(|| panic!("no layout hint painted: {painted:?}")); + + match HMAC_MODE { + HmacMode::Sha512 => assert!( + !hint.contains("cipher_use_hmac") && !hint.contains("cipher_hmac_algorithm"), + "the index is on SQLCipher's own default; there is no pragma to give: {hint}" + ), + HmacMode::Off => assert!( + hint.contains("PRAGMA cipher_use_hmac = OFF;"), + "the pragma that turns the authenticator off is missing: {hint}" + ), + HmacMode::Sha256 => assert!( + hint.contains("PRAGMA cipher_hmac_algorithm = HMAC_SHA256;"), + "the pragma that selects the authenticator is missing: {hint}" + ), + } + assert!( + hint.contains(&format!("PRAGMA cipher_page_size = {};", PAGE_SIZE)), + "the page-size pragma is missing: {hint}" + ); +} + #[test] fn both_of_the_reveal_buttons_report_their_click() { let display = format!("0x{KEY}"); diff --git a/crates/quicksearch-gui/src/app/status_bar.rs b/crates/quicksearch-gui/src/app/status_bar.rs index 1a81f83..98fb089 100644 --- a/crates/quicksearch-gui/src/app/status_bar.rs +++ b/crates/quicksearch-gui/src/app/status_bar.rs @@ -79,6 +79,7 @@ impl QuickSearchApp { ("Finishing the previous run…".to_string(), None) } PrepStep::OpeningIndex => ("Opening the index…".to_string(), None), + PrepStep::Starting => ("Getting the index ready…".to_string(), None), PrepStep::Reconciling(r) => ( format!( "Applying configuration change · {} entries", @@ -116,10 +117,12 @@ impl QuickSearchApp { IndexingStatus::Optimizing => { ui.label(egui::RichText::new("Optimizing index…").small()); } - IndexingStatus::Running { roots, .. } => { + IndexingStatus::Running { + roots, maintenance, .. + } => { let colors = palette(ui.visuals().dark_mode); let rate = self.manage.speed.files_per_sec(); - status_line(ui, &running_line(roots, rate, &colors)); + status_line(ui, &running_line(roots, *maintenance, rate, &colors)); progress_widget(ui, overall_progress(roots).fraction()); } } @@ -181,12 +184,18 @@ fn status_line(ui: &mut egui::Ui, spans: &[Span]) { ui.label(job); } -/// The bottom bar's line for a run in progress. -fn running_line(roots: &[RootProgress], rate: Option, colors: &Palette) -> Vec { - let phase = if roots.iter().any(|r| r.phase == RootPhase::Walking) { - colors.yellow - } else { - colors.green +/// The bottom bar's line for a run in progress. An upkeep step outranks the +/// per-root phases: while one runs it is the only thing moving. +fn running_line( + roots: &[RootProgress], + maintenance: Option, + rate: Option, + colors: &Palette, +) -> Vec { + let (word, phase) = match maintenance { + Some(_) => ("Maintenance", colors.orange), + None if roots.iter().any(|r| r.phase == RootPhase::Walking) => ("Indexing", colors.yellow), + None => ("Indexing", colors.green), }; let done = roots.iter().filter(|r| r.phase == RootPhase::Done).count(); let progress = overall_progress(roots); @@ -210,7 +219,15 @@ fn running_line(roots: &[RootProgress], rate: Option, colors: &Palette) -> if total_workers > 0 { rest.push_str(&format!(" · {}/{} workers", active, total_workers)); } - vec![("Indexing".to_string(), Some(phase)), (rest, None)] + if let Some(step) = maintenance { + // Last, and without the ellipsis the tab's own line carries: the bar + // is one sentence, not a heading. + rest.push_str(&format!( + " · {}", + crate::format::fmt_maintenance(step).trim_end_matches('…') + )); + } + vec![(word.to_string(), Some(phase)), (rest, None)] } fn idle_line(mode: IndexMode, files: i64, colors: &Palette) -> Vec { @@ -270,6 +287,7 @@ mod tests { line(&running_line( &[root(RootPhase::Walking, 100, Some(1000))], None, + None, &colors )), "Indexing 100 / 1,000 (10%) · 2/4 workers" @@ -279,6 +297,7 @@ mod tests { line(&running_line( &[root(RootPhase::Walking, 100, None)], None, + None, &colors )), "Indexing · 100 files · 2/4 workers" @@ -294,16 +313,44 @@ mod tests { done.active_workers = 0; done.total_workers = 0; assert_eq!( - line(&running_line(&[extracting, done], Some(120.0), &colors)), + line(&running_line( + &[extracting, done], + None, + Some(120.0), + &colors + )), "Indexing 2,200 / 2,800 (79%) · 1/2 roots done · 120 files/s · 3/4 workers" ); } + /// The counters are still the run's last true position; only the phase + /// word and the trailing clause say that nothing is moving. + #[test] + fn an_upkeep_step_renames_the_phase_and_names_itself() { + let colors = palette(true); + let spans = running_line( + &[root(RootPhase::Extracting, 100, Some(1000))], + Some(MaintenanceStep::Checkpoint), + None, + &colors, + ); + assert_eq!( + line(&spans), + "Maintenance 100 / 100 (100%) · 2/4 workers · Compacting the write-ahead log" + ); + assert_eq!(spans[0].1, Some(colors.orange)); + } + #[test] fn only_the_phase_word_of_the_running_line_is_hinted() { for dark in [true, false] { let colors = palette(dark); - let spans = running_line(&[root(RootPhase::Walking, 100, Some(1000))], None, &colors); + let spans = running_line( + &[root(RootPhase::Walking, 100, Some(1000))], + None, + None, + &colors, + ); assert_eq!(spans[0].0, "Indexing"); assert_eq!(spans[0].1, Some(colors.yellow), "dark_mode={}", dark); assert!( @@ -317,7 +364,7 @@ mod tests { #[test] fn the_running_hint_follows_the_least_advanced_root() { let colors = palette(true); - let hint = |roots: &[RootProgress]| running_line(roots, None, &colors)[0].1; + let hint = |roots: &[RootProgress]| running_line(roots, None, None, &colors)[0].1; assert_eq!( hint(&[ @@ -338,6 +385,21 @@ mod tests { hint(&[root(RootPhase::Done, 100, None)]), Some(colors.green) ); + // Upkeep outranks every phase: it is the only thing running. + for phase in [RootPhase::Walking, RootPhase::Extracting, RootPhase::Done] { + assert_eq!( + running_line( + &[root(phase, 100, Some(1000))], + Some(MaintenanceStep::MergingText), + None, + &colors + )[0] + .1, + Some(colors.orange), + "{:?}", + phase + ); + } } #[test] diff --git a/crates/quicksearch-gui/src/app/tests.rs b/crates/quicksearch-gui/src/app/tests.rs index a03b9b5..8ca6657 100644 --- a/crates/quicksearch-gui/src/app/tests.rs +++ b/crates/quicksearch-gui/src/app/tests.rs @@ -237,3 +237,18 @@ fn only_light_is_light() { ); } } + +/// In automatic mode the rebuild is already under way behind the modal; +/// commanding another one would delete its progress and start over. +#[test] +fn the_stale_index_prompt_never_restarts_a_rebuild_already_running() { + use super::modals::stale_prompt_should_command; + + assert!(!stale_prompt_should_command(IndexMode::Auto, true)); + // Auto before the first tick: the coordinator still gets there on its own. + assert!(!stale_prompt_should_command(IndexMode::Auto, false)); + assert!(!stale_prompt_should_command(IndexMode::ManualRunning, true)); + + // Manual and stopped: nothing else would ever start it. + assert!(stale_prompt_should_command(IndexMode::ManualStopped, false)); +} diff --git a/crates/quicksearch-gui/src/cli_main.rs b/crates/quicksearch-gui/src/cli_main.rs index aa3a732..1dfe992 100644 --- a/crates/quicksearch-gui/src/cli_main.rs +++ b/crates/quicksearch-gui/src/cli_main.rs @@ -4,6 +4,10 @@ //! shell waiting for the process all behave normally. `src/cli.rs` and //! `src/format.rs` are compiled into both binaries. +// Per-binary, like the GUI's; see `platform::Allocator`. +#[global_allocator] +static GLOBAL: quicksearch_core::platform::Allocator = quicksearch_core::platform::Allocator; + mod cli; #[allow(dead_code)] mod format; diff --git a/crates/quicksearch-gui/src/format.rs b/crates/quicksearch-gui/src/format.rs index 452bb9a..48ad09f 100644 --- a/crates/quicksearch-gui/src/format.rs +++ b/crates/quicksearch-gui/src/format.rs @@ -1,5 +1,7 @@ //! Small display formatters shared across tabs. +use quicksearch_core::indexing::MaintenanceStep; + /// Human-readable byte size: `999 B`, `1.2 KB`, `4.7 MB`, `1.3 GB`. pub fn human_size(bytes: u64) -> String { const UNITS: [&str; 5] = ["B", "KB", "MB", "GB", "TB"]; @@ -98,6 +100,15 @@ pub fn fmt_elapsed(d: std::time::Duration) -> String { } } +/// Search timing: time to the first result, then to the last pass. A search +/// that matched nothing has no first result, and reads as the total alone. +pub fn fmt_search_times(first: Option, total: std::time::Duration) -> String { + match first { + Some(first) => format!("{} / {}", fmt_elapsed(first), fmt_elapsed(total)), + None => fmt_elapsed(total), + } +} + /// A running clock: `0:07`, `4:32`, `1:04:12`; fixed-width seconds. pub fn fmt_duration_clock(d: std::time::Duration) -> String { let secs = d.as_secs(); @@ -109,6 +120,18 @@ pub fn fmt_duration_clock(d: std::time::Duration) -> String { } } +/// The index upkeep a run is inside, in the words a user reads. Every one of +/// these freezes the per-file counters for as long as it runs. +pub fn fmt_maintenance(step: MaintenanceStep) -> &'static str { + match step { + MaintenanceStep::Checkpoint => "Compacting the write-ahead log…", + MaintenanceStep::RemovingStale => "Removing entries for deleted files…", + MaintenanceStep::MergingText => "Merging the text index…", + MaintenanceStep::RootCounts => "Updating folder totals…", + MaintenanceStep::SizeLimit => "Applying the file-size limit…", + } +} + /// What a finished reconciliation did, in one line. A zero clause is left /// out, and a pass that changed nothing still reports that it ran. pub fn fmt_reconcile_summary(deleted: usize, recontented: usize) -> String { @@ -202,6 +225,22 @@ mod tests { assert_eq!(fmt_elapsed(Duration::from_millis(2340)), "2.3 s"); } + #[test] + fn search_times_pair_up() { + use std::time::Duration; + let ms = Duration::from_millis; + assert_eq!(fmt_search_times(Some(ms(12)), ms(340)), "12 ms / 340 ms"); + assert_eq!(fmt_search_times(Some(ms(5)), ms(5)), "5 ms / 5 ms"); + // Each side carries its own unit, so a pair may straddle the boundary. + assert_eq!(fmt_search_times(Some(ms(800)), ms(1400)), "800 ms / 1.4 s"); + assert_eq!( + fmt_search_times(Some(ms(1000)), ms(12_300)), + "1.0 s / 12.3 s" + ); + // Nothing matched: no first result to report. + assert_eq!(fmt_search_times(None, ms(340)), "340 ms"); + } + #[test] fn ago_buckets() { let now = quicksearch_core::log::now_unix(); diff --git a/crates/quicksearch-gui/src/main.rs b/crates/quicksearch-gui/src/main.rs index 7ec0119..1fb1757 100644 --- a/crates/quicksearch-gui/src/main.rs +++ b/crates/quicksearch-gui/src/main.rs @@ -4,6 +4,12 @@ //! `quicksearch-cli`. A query passed here still seeds the search box. #![cfg_attr(windows, windows_subsystem = "windows")] +// Only the binary that declares it gets it — a library cannot choose an +// allocator for its dependents — so this line is what actually puts the app +// on mimalloc. See `platform::Allocator` for why it is not glibc's. +#[global_allocator] +static GLOBAL: quicksearch_core::platform::Allocator = quicksearch_core::platform::Allocator; + mod app; mod backend; #[cfg(feature = "capture")] @@ -73,6 +79,11 @@ fn main() { Ok(c) => (c, None), Err(e) => (Config::default(), Some(e)), }; + // Before any search connection exists: the ceiling is applied at open, and + // `0` leaves it derived from the index. + quicksearch_core::db::set_search_cache_override( + (config.search.cache_size_mib != 0).then_some(config.search.cache_size_mib as i64), + ); let initial_query = seed_query(); // After the CLI early-exit, deliberately: the CLI only reads. Two diff --git a/crates/quicksearch-gui/src/manage_tab.rs b/crates/quicksearch-gui/src/manage_tab.rs index 7e914d9..52edda5 100644 --- a/crates/quicksearch-gui/src/manage_tab.rs +++ b/crates/quicksearch-gui/src/manage_tab.rs @@ -7,12 +7,12 @@ use std::time::{Duration, Instant}; use quicksearch_core::config::Config; use quicksearch_core::coordinator::{IndexMode, IndexerState, ReconcileState, WatcherStatus}; use quicksearch_core::indexing::{ - IndexingStatus, PrepStep, ReconcileProgress, RootPhase, RootProgress, + IndexingStatus, MaintenanceStep, PrepStep, ReconcileProgress, RootPhase, RootProgress, }; use crate::format::{ - fmt_duration_clock, fmt_interval, fmt_rate, fmt_reconcile_summary, group_thousands, human_size, - middle_truncate, + fmt_duration_clock, fmt_interval, fmt_maintenance, fmt_rate, fmt_reconcile_summary, + group_thousands, human_size, middle_truncate, }; use crate::tips::{self, Tipped}; use crate::tracker::SpeedTracker; @@ -56,15 +56,21 @@ impl ManageTab { pub fn observe(&mut self, status: &IndexingStatus) { match status { - IndexingStatus::Running { roots, .. } => { + IndexingStatus::Running { + maintenance: None, + roots, + .. + } => { let total: usize = roots.iter().map(|r| r.walked + r.extracted).sum(); self.speed.record(total); } - // Preparing included: a stale files/sec would read as progress. + // Preparing and a run's upkeep steps included: no file is moving + // in either, and a stale files/sec would read as progress. IndexingStatus::Idle | IndexingStatus::Error(_) | IndexingStatus::Optimizing - | IndexingStatus::Preparing { .. } => self.speed.reset(), + | IndexingStatus::Preparing { .. } + | IndexingStatus::Running { .. } => self.speed.reset(), _ => {} } } @@ -649,9 +655,17 @@ fn status_contents(ui: &mut egui::Ui, state: &IndexerState, speed: &SpeedTracker IndexingStatus::Optimizing => { ui.label("Optimizing index; reclaiming unused space…"); } - IndexingStatus::Running { roots, .. } => { + IndexingStatus::Running { + roots, maintenance, .. + } => { for root in roots { - root_row(ui, root); + root_row(ui, root, *maintenance); + } + // Run-wide, so said once rather than once per root. Purely + // additional: the per-root file hints stay put underneath it, so + // a step starting does not reflow the block. + if let Some(step) = maintenance { + ui.label(hint(fmt_maintenance(*step))); } if let Some(rate) = speed.files_per_sec() { ui.label( @@ -674,6 +688,7 @@ fn prep_row(ui: &mut egui::Ui, step: &PrepStep, elapsed: Duration) { PrepStep::PreviousRun => waiting_row(ui, "Finishing the previous run…", elapsed), PrepStep::OpeningIndex => waiting_row(ui, "Opening the index…", elapsed), PrepStep::Reconciling(r) => reconcile_row(ui, r, Some(elapsed)), + PrepStep::Starting => waiting_row(ui, "Getting the index ready…", elapsed), } } @@ -732,18 +747,28 @@ fn reconcile_row(ui: &mut egui::Ui, r: &ReconcileProgress, elapsed: Option) { let divider = |ui: &mut egui::Ui| { ui.label(egui::RichText::new("|").weak()); }; let phase = crate::color::palette(ui.visuals().dark_mode); + // The counters stay keyed on the root's phase; only the word changes. + let (word, color) = match (maintenance, r.phase) { + (Some(_), _) => ("maintenance", phase.orange), + (None, RootPhase::Walking) => ("indexing", phase.yellow), + (None, RootPhase::Extracting) => ("extracting text", phase.green), + (None, RootPhase::Done) => ("done", phase.blue), + }; ui.horizontal(|ui| { ui.monospace(middle_truncate(&r.root, 48)); divider(ui); + ui.label(egui::RichText::new(word).color(color)); + divider(ui); match r.phase { RootPhase::Walking => { - ui.label(egui::RichText::new("indexing").color(phase.yellow)); - divider(ui); let workers = format!("{}/{} workers", r.active_workers, r.total_workers); match r.walk_denominator() { Some(total) if total > 0 => { @@ -768,8 +793,6 @@ fn root_row(ui: &mut egui::Ui, r: &RootProgress) { } } RootPhase::Extracting => { - ui.label(egui::RichText::new("extracting text").color(phase.green)); - divider(ui); let workers = format!("{}/{} workers", r.active_workers, r.total_workers); match r.extract_total { Some(total) => { @@ -800,8 +823,6 @@ fn root_row(ui: &mut egui::Ui, r: &RootProgress) { } RootPhase::Done => { // Whole-root totals, not just this run's new work. - ui.label(egui::RichText::new("done").color(phase.blue)); - divider(ui); ui.label(format!( "indexed {}, extracted {}", group_thousands(r.walked as u64), @@ -811,6 +832,10 @@ fn root_row(ui: &mut egui::Ui, r: &RootProgress) { } } }); + // Drawn during upkeep too, where it names the last file *written* rather + // than one in flight. That moment of staleness is worth less than the row + // keeping its height: with a checkpoint every few seconds, a line + // vanishing and returning per root reflowed the whole block on a loop. if let Some(f) = &r.current_file { ui.label(hint(middle_truncate(f, 90))); } diff --git a/crates/quicksearch-gui/src/manage_tab/tests.rs b/crates/quicksearch-gui/src/manage_tab/tests.rs index 7164102..e0f44cb 100644 --- a/crates/quicksearch-gui/src/manage_tab/tests.rs +++ b/crates/quicksearch-gui/src/manage_tab/tests.rs @@ -72,10 +72,15 @@ fn running_state(roots: &[&str], current_file: Option<&str>) -> IndexerState { } fn state_with(roots: Vec) -> IndexerState { + maintaining_state(roots, None) +} + +fn maintaining_state(roots: Vec, step: Option) -> IndexerState { IndexerState { activity: IndexingStatus::Running { start_time: std::time::Instant::now(), roots, + maintenance: step, }, ..idle_state() } @@ -433,10 +438,89 @@ fn every_phase_word_is_painted_in_its_hint_color() { let spans = frame_spans(&ctx, &mut tab, &state); let hint = spans.iter().find(|(text, _)| text == word).map(|(_, c)| *c); assert_eq!(hint, Some(want), "{:?}: {:?} in {:?}", theme, word, spans); + + // Upkeep blocks the writer whatever the root was doing, so it + // replaces the phase word rather than sitting beside it. + let state = maintaining_state( + vec![root_progress(phase, 100, Some(1000))], + Some(MaintenanceStep::Checkpoint), + ); + let spans = frame_spans(&ctx, &mut tab, &state); + let painted = |want: &str| spans.iter().find(|(t, _)| t == want).map(|(_, c)| *c); + assert_eq!( + painted("maintenance"), + Some(colors.orange), + "{:?}: {:?}", + theme, + spans + ); + assert_eq!( + painted(word), + None, + "{:?}: {:?} survived: {:?}", + theme, + word, + spans + ); } } } +/// Run-wide state: said once, however many roots are on screen. And said +/// *alongside* the per-root file hints rather than instead of them. +/// +/// The hint count is the assertion that matters. A checkpoint fires every few +/// seconds on a large index, so a line that disappears per root and comes back +/// reflowed the whole block on a loop. Keeping it costs a moment of staleness +/// (it names the last file written, not one in flight) and buys a row that +/// holds its height. +#[test] +fn an_upkeep_step_names_itself_once_and_keeps_the_file_hints() { + let ctx = crate::test_ui::ctx(); + let mut tab = ManageTab::new(); + + let with_file = |root: &str| RootProgress { + root: root.to_string(), + current_file: Some("/data/a-file-that-already-landed.txt".to_string()), + ..root_progress(RootPhase::Extracting, 100, Some(1000)) + }; + let roots = vec![with_file("/data"), with_file("/media")]; + let hints = |drawn: &[String]| { + drawn + .iter() + .filter(|t| t.contains("a-file-that-already-landed.txt")) + .count() + }; + + let running = frame_text(&ctx, &mut tab, &state_with(roots.clone())); + assert_eq!( + hints(&running), + 2, + "the per-root file hint is drawn while files are moving: {:?}", + running + ); + + let text = frame_text( + &ctx, + &mut tab, + &maintaining_state(roots, Some(MaintenanceStep::RemovingStale)), + ); + assert_eq!( + text.iter() + .filter(|t| t.contains("Removing entries for deleted files")) + .count(), + 1, + "the step is run-wide, not per-root: {:?}", + text + ); + assert_eq!( + hints(&text), + 2, + "every root keeps its file hint through the step: {:?}", + text + ); +} + #[test] fn a_walking_root_without_a_count_shows_no_denominator() { let ctx = crate::test_ui::ctx(); @@ -459,6 +543,7 @@ fn each_prologue_step_says_what_it_is_waiting_on() { for (step, expected) in [ (PrepStep::PreviousRun, "Finishing the previous run…"), (PrepStep::OpeningIndex, "Opening the index…"), + (PrepStep::Starting, "Getting the index ready…"), ] { let text = frame_text(&ctx, &mut tab, &preparing_state(step)).join(" | "); assert!(text.contains(expected), "{}", text); diff --git a/crates/quicksearch-gui/src/search_tab.rs b/crates/quicksearch-gui/src/search_tab.rs index 1ece3c0..0efb71d 100644 --- a/crates/quicksearch-gui/src/search_tab.rs +++ b/crates/quicksearch-gui/src/search_tab.rs @@ -11,7 +11,7 @@ use quicksearch_core::search::{MatchField, SearchHit, SearchUpdate}; use quicksearch_core::snippet::Snippet; use crate::color::rank_tier_color; -use crate::format::{fmt_elapsed, fmt_mtime, human_size}; +use crate::format::{fmt_mtime, fmt_search_times, human_size}; use crate::platform; use crate::spotlight::{Spot, Spotlit}; @@ -26,8 +26,10 @@ use ignore_dialog::dir_ignore_pattern; pub use ignore_dialog::IgnoreDialog; use snippet_render::{centered_match_job, marked_field_job, path_cell_job, snippet_job}; -/// Sized for the longest `fmt_elapsed` output, so the query box never resizes. -const STATUS_SLOT_WIDTH: f32 = 52.0; +/// Sized for the longest `fmt_search_times` output, so the query box never +/// resizes. The widest pair lays out around 65 pt; the rest is slack, and +/// `the_duration_readout_fits_its_slot` is what holds this number honest. +const STATUS_SLOT_WIDTH: f32 = 72.0; /// Repeat-button gutter, held whether or not it shows so text never shifts. const REPEAT_SLOT_W: i8 = 20; @@ -40,6 +42,16 @@ const FUZZY_SLOT_WIDTH: f32 = 66.0; /// unconditionally, so no layout direction can flip them. const FUZZY_HINT: &str = "Also run fuzzy filename and full-text passes (slower)"; +/// What the pair of times means. The gap between them is the point: the +/// cascade runs its passes best-match-first, so a search can answer at once +/// and go on working for a while afterwards. +const TIMES_TIP: &str = "Time to the first result retrieved / time to search completion. The search runs in passes, \ + best matches first, so results keep arriving after the likely-most-useful ones."; + +/// The same readout when no result ever arrived: nothing matched, or the +/// search failed. Either way there is no first result to have timed. +const TOTAL_ONLY_TIP: &str = "Time to search completion"; + /// An em dash, not a hyphen: at body size `-` reads as a typo. const NO_CONTENT_MATCH: &str = "—"; @@ -341,6 +353,12 @@ pub struct SearchTab { pub selected: Option, pub running: bool, search_started: Option, + /// Time from the search starting to its first hit arriving. Taken when the + /// batch lands, not when it is painted: the swap waits on [`FADE_OUT_SECS`], + /// which would floor every reading at the same animation constant. + first_hit: Option, + /// Time from the search starting to the last pass finishing — which is also + /// when the display limit was hit, since the cascade stops there. elapsed: Option, pub limited: bool, pub error: Option, @@ -392,6 +410,7 @@ impl SearchTab { selected: None, running: false, search_started: None, + first_hit: None, elapsed: None, limited: false, error: None, @@ -448,6 +467,7 @@ impl SearchTab { self.swap_pending = true; self.running = true; self.search_started = Some(Instant::now()); + self.first_hit = None; self.elapsed = None; self.limited = false; self.error = None; @@ -477,6 +497,11 @@ impl SearchTab { match update { SearchUpdate::Started { .. } => {} SearchUpdate::Hits { hits, .. } => { + // An empty batch is not a result; the cascade never sends one, + // but the event is public and this is what it would mean. + if self.first_hit.is_none() && !hits.is_empty() { + self.first_hit = self.search_started.map(|t| t.elapsed()); + } if self.swap_pending { admit(&mut self.staging, hits, display_limit, &mut self.limited); } else { @@ -788,8 +813,13 @@ impl SearchTab { ui.add(egui::Spinner::new().size(16.0)); } else if show_elapsed { if let Some(elapsed) = self.elapsed { - ui.label(hint(fmt_elapsed(elapsed))) - .on_hover_text("Time to run all search passes"); + let tip = if self.first_hit.is_some() { + TIMES_TIP + } else { + TOTAL_ONLY_TIP + }; + ui.label(hint(fmt_search_times(self.first_hit, elapsed))) + .on_hover_text(tip); } } }, diff --git a/crates/quicksearch-gui/src/search_tab/tests.rs b/crates/quicksearch-gui/src/search_tab/tests.rs index 4c77029..6d23ed5 100644 --- a/crates/quicksearch-gui/src/search_tab/tests.rs +++ b/crates/quicksearch-gui/src/search_tab/tests.rs @@ -1892,6 +1892,233 @@ fn the_query_strip_reads_help_box_duration_fuzzy() { assert!(elapsed < fuzzy, "the duration is not left of Fuzzy"); } +/// The query strip's timing readout, found by its units. No other cell ends +/// this way — `human_size` writes " B"/" KB", and the count reads "3 results". +fn duration_readout(out: &egui::FullOutput) -> String { + let painted = painted_text(out); + painted + .iter() + .find(|t| t.ends_with(" ms") || t.ends_with(" s")) + .unwrap_or_else(|| panic!("no duration readout among {painted:?}")) + .clone() +} + +/// One number cannot separate a search that answered at once and then ground +/// through its late passes from one that was slow the whole way. +#[test] +fn the_readout_reports_the_first_result_and_the_last_pass() { + let ctx = crate::test_ui::ctx(); + let mut tab = tab_with_results(1); + tab.on_search_started(1); + tab.apply_update( + SearchUpdate::Hits { + generation: 1, + hits: vec![hit(1, "alpha_widget_0.txt", 3.0, 116)], + }, + 1000, + ); + let first = tab.first_hit.expect("no first-result time recorded"); + tab.apply_update( + SearchUpdate::Completed { + generation: 1, + total: 1, + limited: false, + }, + 1000, + ); + assert!( + tab.elapsed.expect("no completion time") >= first, + "completion came before the first result" + ); + + let readout = duration_readout(&run_frame(&ctx, &mut tab, vec![])); + assert_eq!( + readout, + crate::format::fmt_search_times(tab.first_hit, tab.elapsed.unwrap()), + "the strip is not showing both times" + ); + assert!(readout.contains(" / "), "only one time painted: {readout}"); +} + +/// A tab whose search finished, having found `hits` results. +fn timed_tab(ctx: &egui::Context, hits: usize) -> SearchTab { + let mut tab = tab_with_results(hits); + tab.on_search_started(1); + if hits > 0 { + tab.apply_update( + SearchUpdate::Hits { + generation: 1, + hits: (0..hits) + .map(|i| hit(i as i64, &format!("alpha_widget_{i}.txt"), 3.0, 116)) + .collect(), + }, + 1000, + ); + } + tab.apply_update( + SearchUpdate::Completed { + generation: 1, + total: hits, + limited: false, + }, + 1000, + ); + run_frame(ctx, &mut tab, vec![]); + tab +} + +/// The readout as painted, and a point inside it. Taken near its left edge: +/// the strip's widgets sit close together, and the centre of a short readout +/// is not reliably the readout's own hit-test. +fn readout_and_pointer(ctx: &egui::Context, tab: &mut SearchTab) -> (String, egui::Pos2) { + let out = run_frame(ctx, tab, vec![]); + let readout = duration_readout(&out); + let rect = crate::test_ui::painted(&out) + .into_iter() + .find(|(t, _)| *t == readout) + .map(|(_, r)| r) + .expect("the readout was not painted"); + (readout, egui::pos2(rect.left() + 2.0, rect.center().y)) +} + +/// Whether hovering `pos` brings up `tip`. The tooltip is its own area, so it +/// may land a frame or two behind the pointer. +fn hover_shows(ctx: &egui::Context, tab: &mut SearchTab, pos: egui::Pos2, tip: &str) -> bool { + let mut out = run_frame(ctx, tab, vec![egui::Event::PointerMoved(pos)]); + for _ in 0..3 { + if painted_text(&out).iter().any(|t| t == tip) { + return true; + } + out = run_frame(ctx, tab, vec![]); + } + false +} + +/// Two bare numbers separated by a slash explain nothing on their own. +#[test] +fn hovering_the_readout_says_what_the_times_are() { + let ctx = crate::test_ui::ctx(); + // Testing that the tooltip is wired up, not egui's hover timing. + ctx.style_mut(|s| { + s.interaction.tooltip_delay = 0.0; + s.interaction.show_tooltips_only_when_still = false; + }); + + let mut tab = timed_tab(&ctx, 1); + let (readout, pos) = readout_and_pointer(&ctx, &mut tab); + assert!( + hover_shows(&ctx, &mut tab, pos, TIMES_TIP), + "hovering {readout:?} explained nothing" + ); + + // Nothing matched: the readout is one number, and says why. + let mut tab = timed_tab(&ctx, 0); + let (readout, pos) = readout_and_pointer(&ctx, &mut tab); + assert!( + hover_shows(&ctx, &mut tab, pos, TOTAL_ONLY_TIP), + "hovering {readout:?} still promised a first result" + ); +} + +/// A batch that arrives after the first must not restart the clock, and a +/// stale generation's batch must not start it at all. +#[test] +fn the_first_result_time_is_the_first_one() { + let mut tab = tab_with_results(1); + tab.on_search_started(2); + + // From the search before this one. + tab.apply_update( + SearchUpdate::Hits { + generation: 1, + hits: vec![hit(1, "stale.txt", 3.0, 10)], + }, + 1000, + ); + assert_eq!(tab.first_hit, None, "a stale batch started the clock"); + + // An empty batch is not a result. + tab.apply_update( + SearchUpdate::Hits { + generation: 2, + hits: vec![], + }, + 1000, + ); + assert_eq!(tab.first_hit, None, "an empty batch counted as a result"); + + tab.apply_update( + SearchUpdate::Hits { + generation: 2, + hits: vec![hit(2, "first.txt", 3.0, 10)], + }, + 1000, + ); + let first = tab.first_hit.expect("no first-result time recorded"); + std::thread::sleep(std::time::Duration::from_millis(2)); + tab.apply_update( + SearchUpdate::Hits { + generation: 2, + hits: vec![hit(3, "second.txt", 3.0, 10)], + }, + 1000, + ); + assert_eq!(tab.first_hit, Some(first), "a later batch moved the clock"); + + // And the next search starts over. + tab.on_search_started(3); + assert_eq!(tab.first_hit, None); +} + +/// Nothing matched, so there was no first result to time. +#[test] +fn a_search_that_found_nothing_shows_one_time() { + let ctx = crate::test_ui::ctx(); + let mut tab = completed_tab(&ctx); + assert_eq!(tab.first_hit, None, "no batch was ever sent"); + let readout = duration_readout(&run_frame(&ctx, &mut tab, vec![])); + assert!( + !readout.contains('/'), + "a missing time was painted: {readout}" + ); +} + +/// The fixed slot is what keeps the query box from resizing when a search +/// finishes; a readout wider than it would shove the box sideways. +#[test] +fn the_duration_readout_fits_its_slot() { + let ctx = crate::test_ui::ctx(); + // egui has no fonts until it has run a frame. + run_frame(&ctx, &mut new_tab(), vec![]); + // Resolved outside the closure: `fonts` holds a lock `style` also wants. + let font = egui::TextStyle::Small.resolve(&ctx.style()); + let ms = std::time::Duration::from_millis; + // Every shape the pair takes: both in milliseconds, straddling the unit + // boundary, and a search slow enough to be worth complaining about. + let over: Vec<(String, f32)> = [ + (ms(999), ms(999)), + (ms(888), ms(12_300)), + (ms(12_300), ms(45_600)), + (ms(123_400), ms(456_700)), + ] + .into_iter() + .map(|(first, total)| { + let text = crate::format::fmt_search_times(Some(first), total); + let width = ctx.fonts(|f| { + f.layout_no_wrap(text.clone(), font.clone(), egui::Color32::WHITE) + .size() + .x + }); + (text, width) + }) + .filter(|(_, width)| *width > STATUS_SLOT_WIDTH) + .collect(); + assert!( + over.is_empty(), + "past the {STATUS_SLOT_WIDTH} pt slot: {over:?}" + ); +} + /// Splitting the widget must not silently lose a click target the combined /// `ui.checkbox` had. #[test] diff --git a/crates/quicksearch-gui/src/settings_tab.rs b/crates/quicksearch-gui/src/settings_tab.rs index a661dac..38efe20 100644 --- a/crates/quicksearch-gui/src/settings_tab.rs +++ b/crates/quicksearch-gui/src/settings_tab.rs @@ -6,16 +6,82 @@ use crate::tips::{self, tip_row, Tipped}; use crate::ui_util::hint; use quicksearch_core::config::{ColumnsConfig, Config}; -fn drag_row( - ui: &mut egui::Ui, - label: &str, - tip: &'static tips::Tip, - value: &mut N, - range: std::ops::RangeInclusive, -) { - tip_row(ui, label, tip, |ui| { - ui.add(egui::DragValue::new(value).range(range)) - }); +/// Who a row is for. +/// +/// [`Level::Advanced`] means one of two things, and usually both: a person who +/// indexed their home folder and nothing else will never need to change it, or +/// they could not tell what it does without already knowing how the indexer +/// works. A byte budget over the writer's batching is both. Password +/// protection is neither, however technical it sounds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Level { + Everyday, + Advanced, +} + +/// The settings form's render context: which rows are on screen. +/// +/// Every row goes through [`Form`], which is what keeps the two lists from +/// drifting — a setting cannot be added to the tab without saying who it is +/// for, and it cannot be shown without a tooltip either, because +/// [`tips::tip_row`] is the only way through. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Form { + pub advanced: bool, +} + +impl Form { + fn shows(self, level: Level) -> bool { + level == Level::Everyday || self.advanced + } + + fn row( + self, + level: Level, + ui: &mut egui::Ui, + label: impl Into, + tip: &'static tips::Tip, + widget: impl FnOnce(&mut egui::Ui) -> egui::Response, + ) { + if self.shows(level) { + tip_row(ui, label, tip, widget); + } + } + + fn drag( + self, + level: Level, + ui: &mut egui::Ui, + label: impl Into, + tip: &'static tips::Tip, + value: &mut N, + range: std::ops::RangeInclusive, + ) { + self.row(level, ui, label, tip, |ui| { + ui.add(egui::DragValue::new(value).range(range)) + }); + } +} + +/// A label ruled underneath in the palette's orange. +/// +/// For the advanced toggle, which sits in the same two-column form as the +/// settings but is not one of them — it decides which of them are on screen. +/// The rule marks that difference without a second type size or a box: the +/// text keeps the ordinary label color, so it reads as part of the form. +fn accented_label(ui: &egui::Ui, text: &str) -> egui::text::LayoutJob { + let mut job = egui::text::LayoutJob::default(); + job.append( + text, + 0.0, + egui::text::TextFormat { + font_id: egui::TextStyle::Body.resolve(ui.style()), + color: ui.visuals().text_color(), + underline: egui::Stroke::new(1.0, crate::color::palette(ui.visuals().dark_mode).orange), + ..Default::default() + }, + ); + job } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -41,6 +107,9 @@ pub struct SettingsOutput { pub security: Option, /// Like Security, edits the live config, so it takes effect without Apply. pub columns: Option, + /// Also live: a view preference should not need an Apply to look at, and + /// drafting it would make merely revealing a setting read as an edit. + pub show_advanced: Option, } pub struct SettingsTab { @@ -111,7 +180,15 @@ impl SettingsTab { self.keychain_active } - pub fn ui(&mut self, ui: &mut egui::Ui, current: &Config) -> SettingsOutput { + /// `indexed_files` is the coordinator's count, used only to show what the + /// automatic search cache works out to for *this* index; `None` while it + /// is not yet known. + pub fn ui( + &mut self, + ui: &mut egui::Ui, + current: &Config, + indexed_files: Option, + ) -> SettingsOutput { self.stage(current); let mut out = SettingsOutput::default(); let keychain_active = self.keychain_active(current); @@ -119,26 +196,65 @@ impl SettingsTab { let capturing = &mut self.capturing_hotkey; let draft = self.draft.as_mut().unwrap(); + // Live, like the columns below: read from the saved config, not the + // draft, so ticking it reveals the rows at once instead of after Apply. + let form = Form { + advanced: current.ui.show_advanced_settings, + }; + let scroll = egui::ScrollArea::vertical() .auto_shrink([false; 2]) .show(ui, |ui| { // A maximized window would stretch every hint into one line. ui.set_max_width(620.0); - ui.heading(egui::RichText::new("Paths").strong()); - egui::Grid::new("opt-paths").num_columns(2).show(ui, |ui| { - tip_row(ui, "Database file", &tips::DATABASE_PATH, |ui| { - ui.add( - egui::TextEdit::singleline(&mut draft.paths.database_path) - .desired_width(260.0), - ) + // The same two-column shape as every settings row — label + // left, control right — so it reads as part of the form; the + // orange rule is what says it governs the form rather than + // belonging to it. + let label = accented_label(ui, "Show advanced settings"); + egui::Grid::new("opt-advanced") + .num_columns(2) + .show(ui, |ui| { + tip_row(ui, label, &tips::SHOW_ADVANCED, |ui| { + let mut advanced = form.advanced; + let response = ui.checkbox(&mut advanced, ""); + if response.changed() { + out.show_advanced = Some(advanced); + } + response + }); }); - }); - ui.label(hint("Indexed folders are managed on the Manage Index tab.")); + ui.label(hint( + "Advanced settings control how the index is built, stored \ + and searched. The defaults suit almost every installation.", + )); ui.separator(); + // The whole section, heading and all: its only row is the + // database path. + if form.advanced { + ui.heading(egui::RichText::new("Paths").strong()); + egui::Grid::new("opt-paths").num_columns(2).show(ui, |ui| { + form.row( + Level::Advanced, + ui, + "Database file", + &tips::DATABASE_PATH, + |ui| { + ui.add( + egui::TextEdit::singleline(&mut draft.paths.database_path) + .desired_width(260.0), + ) + }, + ); + }); + ui.label(hint("Indexed folders are managed on the Manage Index tab.")); + ui.separator(); + } + ui.heading(egui::RichText::new("Indexing").strong()); - config_editor_ui(ui, draft, Section::Indexing); + config_editor_ui(ui, draft, Section::Indexing, indexed_files, form); ui.label(hint( "Automatic and manual indexing are switched on the \ Manage Index tab.", @@ -146,11 +262,11 @@ impl SettingsTab { ui.separator(); ui.heading(egui::RichText::new("Processing").strong()); - config_editor_ui(ui, draft, Section::Processing); + config_editor_ui(ui, draft, Section::Processing, indexed_files, form); ui.separator(); ui.heading(egui::RichText::new("Search").strong()); - config_editor_ui(ui, draft, Section::Search); + config_editor_ui(ui, draft, Section::Search, indexed_files, form); ui.add_space(6.0); // Live, not drafted — see `columns_ui`. out.columns = columns_ui(ui, ¤t.search.columns); @@ -158,19 +274,27 @@ impl SettingsTab { ui.heading(egui::RichText::new("Interface").strong()); egui::Grid::new("opt-ui").num_columns(2).show(ui, |ui| { - tip_row(ui, "UI scale", &tips::UI_SCALE, |ui| { + form.row(Level::Everyday, ui, "UI scale", &tips::UI_SCALE, |ui| { ui.add( egui::Slider::new(&mut draft.ui.scale, 0.5..=2.5) .step_by(0.05) .fixed_decimals(2), ) }); - tip_row(ui, "Search shortcut", &tips::SEARCH_HOTKEY, |ui| { - hotkey_edit(ui, &mut draft.ui.search_hotkey, capturing) - }); - tip_row(ui, "Color scheme", &tips::COLOR_SCHEME, |ui| { - color_scheme_edit(ui, &mut draft.ui.color_scheme) - }); + form.row( + Level::Everyday, + ui, + "Search shortcut", + &tips::SEARCH_HOTKEY, + |ui| hotkey_edit(ui, &mut draft.ui.search_hotkey, capturing), + ); + form.row( + Level::Everyday, + ui, + "Color scheme", + &tips::COLOR_SCHEME, + |ui| color_scheme_edit(ui, &mut draft.ui.color_scheme), + ); }); hotkey_note(ui, &draft.ui.search_hotkey, ¤t.ui.search_hotkey); ui.separator(); @@ -178,7 +302,7 @@ impl SettingsTab { // Security acts on the live config, not the draft; the KDF // salt is never shown anywhere in the GUI. ui.heading(egui::RichText::new("Security").strong()); - out.security = security_ui(ui, current, keychain_active); + out.security = security_ui(ui, current, keychain_active, form); ui.separator(); let p = crate::color::palette(ui.visuals().dark_mode); @@ -203,11 +327,16 @@ impl SettingsTab { } }); }); - ui.label(hint( + // The second sentence names two rows that are only on screen + // with advanced settings shown. + ui.label(hint(if form.advanced { "Narrowing a filter removes the entries it excludes; widening \ one reindexes to find what it now allows. Only the tokenizer \ - and hash length require a full rebuild.", - )); + and hash length require a full rebuild." + } else { + "Narrowing a filter removes the entries it excludes; widening \ + one reindexes to find what it now allows." + })); }); crate::ui_util::more_below_hint(ui, &scroll); @@ -381,6 +510,7 @@ fn security_ui( ui: &mut egui::Ui, current: &Config, keychain_active: bool, + form: Form, ) -> Option { let mut action = None; if current.security.password_protected { @@ -408,10 +538,13 @@ fn security_ui( action = Some(SecurityAction::Disable); } }); - if ui - .button("Show database key…") - .tip(&tips::SHOW_KEY) - .clicked() + // The raw key is for someone recovering the file by hand; the password + // controls above it are for everyone. + if form.advanced + && ui + .button("Show database key…") + .tip(&tips::SHOW_KEY) + .clicked() { action = Some(SecurityAction::ShowKey); } @@ -440,9 +573,57 @@ fn security_ui( action } -/// Every row goes through [`crate::tips::tip_row`], so a setting cannot -/// arrive here without a tooltip. -fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section) { +/// What the automatic search cache resolves to, as a sentence. `None` when the +/// file count is not known yet, in which case the row shows nothing rather +/// than a number that would be wrong. +fn search_cache_hint(config: &Config, indexed_files: Option) -> Option { + use quicksearch_core::db::schema::{ + recommended_search_cache_mib, SEARCH_CACHE_BYTES_PER_FILE, SEARCH_CACHE_MAX_MIB, + }; + if config.search.cache_size_mib != 0 { + return None; + } + let files = indexed_files?; + let keyed = config.security.password_protected; + let mib = recommended_search_cache_mib(files, keyed); + if !keyed { + return Some(format!( + "Automatic: {} MiB. An unencrypted index reads a cache miss \ + straight from the operating system, so a larger cache measures no \ + faster.", + mib + )); + } + let counted = crate::format::group_thousands(files.max(0) as u64); + // Past ~800k files the automatic value is capped below what the index + // wants. Saying so is the only way the override is discoverable in the one + // case that needs it. + let wanted = files.max(0).saturating_mul(SEARCH_CACHE_BYTES_PER_FILE) / (1024 * 1024); + if wanted > SEARCH_CACHE_MAX_MIB { + return Some(format!( + "Automatic: {} MiB, the most it will choose on its own. This \ + index's {} files want about {} MiB to search at full speed — set \ + that here if you would rather spend the memory than the time.", + mib, counted, wanted + )); + } + Some(format!( + "Automatic: {} MiB, sized to hold this index's {} file records — an \ + encrypted index re-decrypts them on every keystroke when they do not \ + fit.", + mib, counted + )) +} + +/// Every row goes through [`Form`], so a setting cannot arrive here without a +/// tooltip or without saying who it is for. +fn config_editor_ui( + ui: &mut egui::Ui, + config: &mut Config, + section: Section, + indexed_files: Option, + form: Form, +) { match section { Section::Indexing => { egui::Grid::new("cfg-indexing") @@ -450,31 +631,47 @@ fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section) { .show(ui, |ui| { // Automatic vs manual is absent: it is live state, and a // staged copy would fight the Manage Index buttons. - tip_row(ui, "Full reindex every", &tips::REINDEX_INTERVAL, |ui| { - ui.horizontal(|ui| { - ui.add( - egui::DragValue::new(&mut config.indexing.reindex_interval_minutes) + form.row( + Level::Advanced, + ui, + "Full reindex every", + &tips::REINDEX_INTERVAL, + |ui| { + ui.horizontal(|ui| { + ui.add( + egui::DragValue::new( + &mut config.indexing.reindex_interval_minutes, + ) .range(5..=60 * 24 * 30), - ); - ui.label("minutes"); - }) - .response - }); + ); + ui.label("minutes"); + }) + .response + }, + ); - tip_row(ui, "Follow symlinks", &tips::FOLLOW_SYMLINKS, |ui| { - ui.checkbox(&mut config.indexing.follow_symlinks, "") - }); + form.row( + Level::Advanced, + ui, + "Follow symlinks", + &tips::FOLLOW_SYMLINKS, + |ui| ui.checkbox(&mut config.indexing.follow_symlinks, ""), + ); - tip_row(ui, "Include hidden files", &tips::INCLUDE_HIDDEN, |ui| { - ui.checkbox(&mut config.indexing.include_hidden, "") - }); + form.row( + Level::Everyday, + ui, + "Include hidden files", + &tips::INCLUDE_HIDDEN, + |ui| ui.checkbox(&mut config.indexing.include_hidden, ""), + ); }); } Section::Processing => { egui::Grid::new("cfg-processing") .num_columns(2) .show(ui, |ui| { - tip_row(ui, "Tokenizer", &tips::TOKENIZER, |ui| { + form.row(Level::Advanced, ui, "Tokenizer", &tips::TOKENIZER, |ui| { egui::ComboBox::from_id_salt("cfg-tokenize") .selected_text(&config.processing.tokenize) .show_ui(ui, |ui| { @@ -489,14 +686,19 @@ fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section) { .response }); - ui.label(""); - ui.hyperlink_to( - "Tokenizer documentation", - "https://www.sqlite.org/fts5.html#tokenizers", - ); - ui.end_row(); + // Not a row, so it needs its own guard: it documents the + // tokenizer above and makes no sense without it. + if form.advanced { + ui.label(""); + ui.hyperlink_to( + "Tokenizer documentation", + "https://www.sqlite.org/fts5.html#tokenizers", + ); + ui.end_row(); + } - drag_row( + form.drag( + Level::Advanced, ui, "Hash sample size (bytes)", &tips::HASH_LENGTH, @@ -504,7 +706,8 @@ fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section) { 512..=1_048_576, ); - drag_row( + form.drag( + Level::Advanced, ui, "Max stored text (bytes)", &tips::MAX_STORED_TEXT, @@ -512,7 +715,8 @@ fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section) { 1024..=16_777_216, ); - drag_row( + form.drag( + Level::Advanced, ui, "Max text file size (bytes)", &tips::MAX_TEXT_FILE_SIZE, @@ -520,7 +724,8 @@ fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section) { 1024..=1_073_741_824, ); - drag_row( + form.drag( + Level::Advanced, ui, "Batch size", &tips::BATCH_SIZE, @@ -528,7 +733,8 @@ fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section) { 10..=100_000, ); - drag_row( + form.drag( + Level::Advanced, ui, "Max WAL size (bytes)", &tips::MAX_WAL_SIZE, @@ -536,21 +742,27 @@ fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section) { 0u64..=8_589_934_592u64, ); - tip_row(ui, "Store text for snippets", &tips::STORE_TEXT, |ui| { - ui.checkbox(&mut config.processing.store_text_for_snippets, "") - }); + form.row( + Level::Everyday, + ui, + "Store text for snippets", + &tips::STORE_TEXT, + |ui| ui.checkbox(&mut config.processing.store_text_for_snippets, ""), + ); }); } Section::Search => { egui::Grid::new("cfg-search").num_columns(2).show(ui, |ui| { - tip_row( + form.row( + Level::Everyday, ui, "Fuzzy search ON by default", &tips::FUZZY_DEFAULT, |ui| ui.checkbox(&mut config.search.fuzzy_default, ""), ); - drag_row( + form.drag( + Level::Advanced, ui, "Fuzzy edit distance", &tips::FUZZY_EDITS, @@ -558,7 +770,8 @@ fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section) { 0..=8, ); - drag_row( + form.drag( + Level::Advanced, ui, "Display limit", &tips::DISPLAY_LIMIT, @@ -566,7 +779,8 @@ fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section) { 50..=100_000, ); - drag_row( + form.drag( + Level::Advanced, ui, "Stream batch size", &tips::RESULTS_PER_PAGE, @@ -574,7 +788,8 @@ fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section) { 10..=10_000, ); - drag_row( + form.drag( + Level::Advanced, ui, "Debounce (ms)", &tips::DEBOUNCE, @@ -582,16 +797,37 @@ fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section) { 0..=2000, ); - tip_row(ui, "Live results", &tips::LIVE_RESULTS, |ui| { - ui.checkbox(&mut config.search.live_results, "") + form.row( + Level::Everyday, + ui, + "Live results", + &tips::LIVE_RESULTS, + |ui| ui.checkbox(&mut config.search.live_results, ""), + ); + + // 0 is "derive it from the index", which is why this is a + // plain box rather than a range starting at the floor. + form.drag( + Level::Advanced, + ui, + "Search cache MiB (0 = auto)", + &tips::SEARCH_CACHE, + &mut config.search.cache_size_mib, + 0..=quicksearch_core::db::schema::SEARCH_CACHE_OVERRIDE_MAX_MIB as usize, + ); + }); + // Both come and go as their values are edited, and both explain + // advanced rows — there is nothing to say when those are hidden. + if form.advanced { + crate::ui_util::stable_section(ui, |ui| { + if let Some(warning) = config.search.fuzzy_edits_warning() { + ui.colored_label(ui.visuals().warn_fg_color, warning); + } + if let Some(recommended) = search_cache_hint(config, indexed_files) { + ui.label(hint(&recommended)); + } }); - }); - // The warning comes and goes as the value is edited. - crate::ui_util::stable_section(ui, |ui| { - if let Some(warning) = config.search.fuzzy_edits_warning() { - ui.colored_label(ui.visuals().warn_fg_color, warning); - } - }); + } } } } diff --git a/crates/quicksearch-gui/src/settings_tab/tests.rs b/crates/quicksearch-gui/src/settings_tab/tests.rs index 9f4d641..616eb5d 100644 --- a/crates/quicksearch-gui/src/settings_tab/tests.rs +++ b/crates/quicksearch-gui/src/settings_tab/tests.rs @@ -239,77 +239,130 @@ fn an_unapplied_shortcut_says_it_is_not_in_force_yet() { assert_eq!(run("Ctrl+Shift+F", "Ctrl+Shift+F"), ""); } -/// Every row with the tip it must show, so a wrong tooltip is impossible. -const ROWS: &[(Section, &str, &tips::Tip)] = &[ +/// Every row with the tip it must show and who it is for, so a wrong tooltip +/// is impossible and the everyday/advanced split is written down once. +const ROWS: &[(Section, &str, &tips::Tip, Level)] = &[ ( Section::Indexing, "Full reindex every", &tips::REINDEX_INTERVAL, + Level::Advanced, + ), + ( + Section::Indexing, + "Follow symlinks", + &tips::FOLLOW_SYMLINKS, + Level::Advanced, ), - (Section::Indexing, "Follow symlinks", &tips::FOLLOW_SYMLINKS), ( Section::Indexing, "Include hidden files", &tips::INCLUDE_HIDDEN, + Level::Everyday, + ), + ( + Section::Processing, + "Tokenizer", + &tips::TOKENIZER, + Level::Advanced, ), - (Section::Processing, "Tokenizer", &tips::TOKENIZER), ( Section::Processing, "Hash sample size (bytes)", &tips::HASH_LENGTH, + Level::Advanced, ), ( Section::Processing, "Max stored text (bytes)", &tips::MAX_STORED_TEXT, + Level::Advanced, ), ( Section::Processing, "Max text file size (bytes)", &tips::MAX_TEXT_FILE_SIZE, + Level::Advanced, + ), + ( + Section::Processing, + "Batch size", + &tips::BATCH_SIZE, + Level::Advanced, ), - (Section::Processing, "Batch size", &tips::BATCH_SIZE), ( Section::Processing, "Max WAL size (bytes)", &tips::MAX_WAL_SIZE, + Level::Advanced, ), ( Section::Processing, "Store text for snippets", &tips::STORE_TEXT, + Level::Everyday, ), ( Section::Search, "Fuzzy search ON by default", &tips::FUZZY_DEFAULT, + Level::Everyday, + ), + ( + Section::Search, + "Fuzzy edit distance", + &tips::FUZZY_EDITS, + Level::Advanced, + ), + ( + Section::Search, + "Display limit", + &tips::DISPLAY_LIMIT, + Level::Advanced, ), - (Section::Search, "Fuzzy edit distance", &tips::FUZZY_EDITS), - (Section::Search, "Display limit", &tips::DISPLAY_LIMIT), ( Section::Search, "Stream batch size", &tips::RESULTS_PER_PAGE, + Level::Advanced, + ), + ( + Section::Search, + "Debounce (ms)", + &tips::DEBOUNCE, + Level::Advanced, + ), + ( + Section::Search, + "Live results", + &tips::LIVE_RESULTS, + Level::Everyday, + ), + ( + Section::Search, + "Search cache MiB (0 = auto)", + &tips::SEARCH_CACHE, + Level::Advanced, ), - (Section::Search, "Debounce (ms)", &tips::DEBOUNCE), - (Section::Search, "Live results", &tips::LIVE_RESULTS), ]; -/// Rendered without the tab's scroll area so nothing sits below the fold. +/// Rendered without the tab's scroll area so nothing sits below the fold, and +/// with advanced on so every row is present to be hovered. #[test] fn every_row_shows_its_own_tip() { - for (section, label, tip) in ROWS { + for (section, label, tip, _) in ROWS { let ctx = crate::test_ui::ctx(); ctx.style_mut(|s| { s.interaction.tooltip_delay = 0.0; s.interaction.show_tooltips_only_when_still = false; }); let mut cfg = Config::default(); + let form = Form { advanced: true }; let mut run = |events: Vec| { let input = crate::test_ui::raw_input(egui::vec2(600.0, 800.0), events); ctx.run(input, |ctx| { egui::CentralPanel::default() - .show(ctx, |ui| config_editor_ui(ui, &mut cfg, *section)); + .show(ctx, |ui| config_editor_ui(ui, &mut cfg, *section, None, form)); }) }; @@ -341,14 +394,21 @@ fn hovering_a_setting_label_explains_it() { s.interaction.tooltip_delay = 0.0; s.interaction.show_tooltips_only_when_still = false; }); - let cfg = Config::default(); + // Tokenizer is an advanced row, so the whole tab has to be showing them. + let cfg = Config { + ui: quicksearch_core::config::UiConfig { + show_advanced_settings: true, + ..Default::default() + }, + ..Config::default() + }; let mut w = SettingsTab::new(); let run = |w: &mut SettingsTab, events: Vec| { let input = crate::test_ui::raw_input(egui::vec2(1000.0, 900.0), events); ctx.run(input, |ctx| { egui::CentralPanel::default().show(ctx, |ui| { - w.ui(ui, &cfg); + w.ui(ui, &cfg, None); }); }) }; @@ -382,7 +442,7 @@ fn the_tab_renders_and_apply_reports_the_draft() { let input = crate::test_ui::raw_input(egui::vec2(1000.0, 900.0), events); let mut out = SettingsOutput::default(); let full = ctx.run(input, |ctx| { - egui::CentralPanel::default().show(ctx, |ui| out = w.ui(ui, &cfg)); + egui::CentralPanel::default().show(ctx, |ui| out = w.ui(ui, &cfg, None)); }); crate::test_ui::assert_no_tofu(&ctx, &full); (out, full) @@ -479,7 +539,7 @@ fn run_security( let mut action = None; let full = ctx.run(input, |ctx| { egui::CentralPanel::default().show(ctx, |ui| { - action = security_ui(ui, current, false); + action = security_ui(ui, current, false, Form { advanced: true }); }); }); crate::test_ui::assert_no_tofu(ctx, &full); @@ -551,3 +611,119 @@ fn a_stale_draft_cannot_revert_the_columns() { "applying the stale draft reverted the column" ); } + +/// The hint is the only place the automatic ceiling is visible, and the only +/// thing that makes the override discoverable when the cap bites. +#[test] +fn the_search_cache_hint_explains_the_automatic_value() { + let mut cfg = Config::default(); + + assert_eq!( + search_cache_hint(&cfg, None), + None, + "with no file count there is no honest number to show" + ); + + // An explicit setting is not automatic, so there is nothing to explain. + cfg.search.cache_size_mib = 64; + assert_eq!(search_cache_hint(&cfg, Some(200_000)), None); + cfg.search.cache_size_mib = 0; + + // Unencrypted: a fixed value, and the reason for it. + let plain = search_cache_hint(&cfg, Some(200_000)).expect("a hint"); + assert!(plain.contains("16 MiB"), "{}", plain); + assert!(plain.contains("unencrypted"), "{}", plain); + + // Encrypted and inside the cap: the derived value, and the file count it + // came from. + cfg.security.password_protected = true; + let keyed = search_cache_hint(&cfg, Some(200_000)).expect("a hint"); + assert!(keyed.contains("32 MiB"), "{}", keyed); + assert!(keyed.contains("200,000"), "{}", keyed); + + // Encrypted and past it: says so, and says what the index actually wants, + // or the override cannot be found by the people who need it. + let capped = search_cache_hint(&cfg, Some(2_000_000)).expect("a hint"); + assert!( + capped.contains("128 MiB") && capped.contains("320 MiB"), + "the capped hint must name both the cap and the want: {}", + capped + ); +} + +/// The whole point: with advanced off, only the everyday rows are on screen, +/// and with it on nothing has gone missing. `ROWS` is the categorisation, so +/// this fails the moment a row is added without deciding who it is for. +#[test] +fn advanced_rows_are_hidden_until_asked_for() { + let painted_labels = |advanced: bool| -> Vec<&'static str> { + let ctx = crate::test_ui::ctx(); + let mut cfg = Config::default(); + let form = Form { advanced }; + let mut shown = Vec::new(); + for section in [Section::Indexing, Section::Processing, Section::Search] { + let out = ctx.run( + crate::test_ui::raw_input(egui::vec2(600.0, 800.0), vec![]), + |ctx| { + egui::CentralPanel::default().show(ctx, |ui| { + config_editor_ui(ui, &mut cfg, section, None, form) + }); + }, + ); + let text = painted_text(&out).join("\n"); + for (row_section, label, _, _) in ROWS { + if *row_section == section && text.contains(*label) { + shown.push(*label); + } + } + } + shown + }; + + let everyday: Vec<&str> = ROWS + .iter() + .filter(|(_, _, _, level)| *level == Level::Everyday) + .map(|(_, label, _, _)| *label) + .collect(); + let all: Vec<&str> = ROWS.iter().map(|(_, label, _, _)| *label).collect(); + + assert!( + !everyday.is_empty() && everyday.len() < all.len(), + "a split with nothing on one side is not a split: {} of {}", + everyday.len(), + all.len() + ); + assert_eq!( + painted_labels(false), + everyday, + "the default view must show the everyday rows and only those" + ); + assert_eq!( + painted_labels(true), + all, + "turning advanced on must bring every row back" + ); +} + +/// Revealing a setting is not editing one: the toggle writes through +/// `SettingsOutput` and must never make the tab read as dirty, or looking at +/// an advanced setting would demand an Apply. +#[test] +fn showing_advanced_settings_is_not_an_unsaved_edit() { + let mut w = SettingsTab::new(); + let mut cfg = Config::default(); + assert!(!cfg.ui.show_advanced_settings, "hidden by default"); + w.stage(&cfg); + + // The checkbox writes straight to the live config, as the app does. + cfg.ui.show_advanced_settings = true; + assert!(!w.is_dirty(&cfg), "revealing rows read as an edit"); + + // And a draft staged while they were hidden must not put them away again. + let mut applied = w.draft_config().expect("a draft"); + crate::app::pin_live_fields(&mut applied, &cfg); + assert!( + applied.ui.show_advanced_settings, + "applying the stale draft hid the advanced settings again" + ); +} diff --git a/crates/quicksearch-gui/src/tips.rs b/crates/quicksearch-gui/src/tips.rs index dba1a6a..30e06ea 100644 --- a/crates/quicksearch-gui/src/tips.rs +++ b/crates/quicksearch-gui/src/tips.rs @@ -52,7 +52,7 @@ impl Tipped for egui::Response { /// One grid row; the label and the control share the tooltip. pub fn tip_row( ui: &mut egui::Ui, - label: &str, + label: impl Into, tip: &'static Tip, widget: impl FnOnce(&mut egui::Ui) -> egui::Response, ) { @@ -61,6 +61,26 @@ pub fn tip_row( ui.end_row(); } +// --- Settings: the advanced toggle ---------------------------------------- + +pub static SHOW_ADVANCED: Tip = Tip { + title: "Show advanced settings", + body: "Reveals the rest of the Settings tab: where the index file lives, \ + how text is broken into searchable pieces, how much of a file is \ + read, how much memory searching may use, and similar.\n\n\ + They are hidden by default because their defaults are right for \ + almost every installation, and because a wrong value can make \ + indexing slower, searching worse, or a rebuild necessary. Nothing \ + is lost by leaving this off — every setting behind it keeps working \ + at its default.", + examples: &[ + "on when you want the index kept on a different drive, or are tuning \ + a very large collection.", + "off for everyday use.", + ], + caution: None, +}; + // --- Settings: Paths ------------------------------------------------------ pub static DATABASE_PATH: Tip = Tip { @@ -215,13 +235,15 @@ pub static MAX_WAL_SIZE: Tip = Tip { body: "While indexing, changes are written to a companion file beside \ the index and folded in afterwards. That normally happens by \ itself, but during a long run with searches going on at the same \ - time the companion file keeps growing, sometimes past the size of \ - the index. This is the point at which QuickSearch pauses and folds \ - it in regardless.\n\n\ - Another speed setting; the default suits most machines.", + time the companion file keeps growing, often past the size of the \ + index itself: it records every version of every page the run \ + touches, where the index keeps only the last. This is the point at \ + which QuickSearch pauses and folds it in regardless.\n\n\ + Folding in less often makes indexing faster and makes searching \ + during it slower.", examples: &[ - "536870912, 512 MB, is the default.", - "67108864, 64 MB, when disk space is tight.", + "2147483648, 2 GB, is the default.", + "67108864, 64 MB, to favour searching while indexing runs.", "0 to never force it and let the database decide. Any other value below 16 MB \ is treated as 16 MB.", ], @@ -289,6 +311,28 @@ pub static DISPLAY_LIMIT: Tip = Tip { caution: None, }; +pub static SEARCH_CACHE: Tip = Tip { + title: "Search cache", + body: "The size of the memory-backed cache for the Indexing database.\n\n\ + When set to 0, QuickSearch automatically sizes it for your index, capped at 128 MiB. \ + The recommended value is shown below the setting. It matters most on an \ + encrypted index, which has to decrypt anything the cache does not \ + already hold. If this is smaller than the recommended value it will \ + make search results four times slower.\n\n\ + An unencrypted index doesn't need much cache so we give it \ + a small amount which doesn't change with index size.", + examples: &[ + "0 sizes it automatically, and is right unless your folders are \ + nested unusually deep.", + "a fixed value when you would rather cap what QuickSearch keeps \ + resident, at the cost of slower searching on a large index.", + ], + caution: Some( + "This memory is held for as long as the search stays open, and \ + released after a long idle.", + ), +}; + pub static RESULTS_PER_PAGE: Tip = Tip { title: "Stream batch size", body: "Results arrive in batches while a search runs, and this is how \