From 42f26b1e72675872b9997df1592ed651463304ee Mon Sep 17 00:00:00 2001 From: Jeremy Karst Date: Sun, 23 Aug 2026 20:01:16 -0400 Subject: [PATCH] Increase search db connection's timeout to 30m. Remove the indexer's duplicate-visit set, ensuring we index correctly without it. --- crates/quicksearch-core/Cargo.toml | 8 + crates/quicksearch-core/benches/index.rs | 20 +- .../quicksearch-core/examples/extractprobe.rs | 290 ++++++++++++++++++ crates/quicksearch-core/examples/hashprobe.rs | 1 - .../quicksearch-core/examples/indexprobe.rs | 57 +++- crates/quicksearch-core/examples/memprobe.rs | 42 ++- crates/quicksearch-core/src/coordinator.rs | 8 +- .../quicksearch-core/src/coordinator/inner.rs | 12 +- crates/quicksearch-core/src/db/repo_tests.rs | 33 +- crates/quicksearch-core/src/extract/mod.rs | 6 +- crates/quicksearch-core/src/extract/office.rs | 6 +- .../quicksearch-core/src/extract/plaintext.rs | 6 +- crates/quicksearch-core/src/extract/rtf.rs | 6 +- .../src/file_handling/records.rs | 4 +- crates/quicksearch-core/src/incremental.rs | 4 +- .../quicksearch-core/src/indexing/pipeline.rs | 104 ++++++- crates/quicksearch-core/src/indexing/tests.rs | 61 +++- crates/quicksearch-core/src/query/split.rs | 2 +- .../src/search/cascade/passes.rs | 33 +- crates/quicksearch-core/src/search/mod.rs | 21 +- crates/quicksearch-core/src/testutil.rs | 23 +- crates/quicksearch-core/src/verify.rs | 5 +- crates/quicksearch-core/src/walk.rs | 41 ++- crates/quicksearch-core/src/walk/tests.rs | 11 +- .../quicksearch-core/tests/encrypted_perf.rs | 250 +++++++++++++++ .../tests/extraction_corpus.rs | 3 +- crates/quicksearch-core/tests/full_index.rs | 60 ++++ crates/quicksearch-gui/src/app/security.rs | 13 +- crates/quicksearch-gui/src/app/tests.rs | 13 +- crates/quicksearch-gui/src/backend.rs | 11 +- crates/quicksearch-gui/src/query_highlight.rs | 2 +- .../quicksearch-gui/src/search_tab/tests.rs | 5 +- 32 files changed, 1010 insertions(+), 151 deletions(-) create mode 100644 crates/quicksearch-core/examples/extractprobe.rs create mode 100644 crates/quicksearch-core/tests/encrypted_perf.rs diff --git a/crates/quicksearch-core/Cargo.toml b/crates/quicksearch-core/Cargo.toml index 45f7b0f..e255ba6 100644 --- a/crates/quicksearch-core/Cargo.toml +++ b/crates/quicksearch-core/Cargo.toml @@ -11,6 +11,14 @@ description = "Indexing, storage and search engine behind QuickSearch." name = "quicksearch_core" path = "src/lib.rs" +# Off by default and never enabled by a shipped build. `probe` turns on the +# indexing run's structure census (`indexing::pipeline`), which attributes +# peak memory to a *structure*; `smaps` can only attribute it to a mapping, +# and every one of these lives in the same anonymous heap. Built for the +# `memprobe` example, which reads the census off stderr. +[features] +probe = [] + # Several dependencies below are named directly despite already being in the # lockfile transitively ("transitive already"): they compile nothing new. [dependencies] diff --git a/crates/quicksearch-core/benches/index.rs b/crates/quicksearch-core/benches/index.rs index 96c1705..582a134 100644 --- a/crates/quicksearch-core/benches/index.rs +++ b/crates/quicksearch-core/benches/index.rs @@ -10,7 +10,7 @@ mod corpus; use divan::Bencher; -use quicksearch_core::{mime, textenc, walk}; +use quicksearch_core::{mime, textenc}; fn main() { divan::main(); @@ -136,21 +136,3 @@ mod mime_sniff { } } -/// SHA-256 over the path string, per file, every run. Reference only: it -/// confirms the cost is small enough that the collision-resistance argument -/// stands. -mod path_digest { - use super::*; - - #[divan::bench] - fn per_path(bencher: Bencher) { - let rows = corpus::rows(); - bencher.bench(|| { - let mut acc = 0u128; - for row in divan::black_box(rows) { - acc ^= walk::path_digest(&row.path); - } - acc - }); - } -} diff --git a/crates/quicksearch-core/examples/extractprobe.rs b/crates/quicksearch-core/examples/extractprobe.rs new file mode 100644 index 0000000..8d6d6f6 --- /dev/null +++ b/crates/quicksearch-core/examples/extractprobe.rs @@ -0,0 +1,290 @@ +//! What one content extraction *costs in memory*, and what a pool of them +//! costs together. +//! +//! [`memprobe`](memprobe.rs) measures a whole run, where extraction is mixed +//! in with the walk, the writer and SQLite. This isolates the extractors: +//! the same [`decide_content`] the content pass calls, over a real corpus, +//! with a pool the size of a real one. +//! +//! ```text +//! cargo build -p quicksearch-core --example extractprobe --release +//! ./target/release/examples/extractprobe ~/Documents # 1 worker: per-file cost +//! ./target/release/examples/extractprobe ~/Documents 16 # a network root's pool +//! ./target/release/examples/extractprobe ~/Documents 16 4 # four such roots +//! ``` +//! +//! The question it answers: **does peak memory track the number of files or +//! the number of workers?** A run's file count is the user's; its worker +//! count is ours (`walk::thread_count_for`, one pool per root), so if the +//! peak follows the pool, the fix is a bound and not a rewrite. +//! +//! `peak live` is from a counting allocator and is exact; `VmHWM` is what the +//! machine sees, and the gap between them is glibc holding freed chunks on +//! its arena free lists. With one worker the per-file table is exact — the +//! largest entries are the files that would spike a real run. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + +mod common; + +// --------------------------------------------------------------------------- +// Allocation accounting +// --------------------------------------------------------------------------- + +/// `System`, counting — per binary, so the shipped `quicksearch` is +/// untouched. `PEAK_LIVE` is the high-water of live bytes: unlike RSS it +/// cannot be inflated by the allocator declining to return pages. +struct Counting; + +static LIVE: AtomicU64 = AtomicU64::new(0); +static PEAK_LIVE: AtomicU64 = AtomicU64::new(0); +/// High-water since the last [`take_mark`]; the per-file column. +static MARK: AtomicU64 = AtomicU64::new(0); + +#[inline] +fn note(live: u64) { + PEAK_LIVE.fetch_max(live, Ordering::Relaxed); + MARK.fetch_max(live, Ordering::Relaxed); +} + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, l: Layout) -> *mut u8 { + let p = unsafe { System.alloc(l) }; + if !p.is_null() { + note(LIVE.fetch_add(l.size() as u64, Ordering::Relaxed) + l.size() as u64); + } + p + } + unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 { + let p = unsafe { System.alloc_zeroed(l) }; + if !p.is_null() { + note(LIVE.fetch_add(l.size() as u64, Ordering::Relaxed) + l.size() as u64); + } + p + } + unsafe fn dealloc(&self, p: *mut u8, l: Layout) { + LIVE.fetch_sub(l.size() as u64, Ordering::Relaxed); + unsafe { System.dealloc(p, l) } + } + unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 { + let q = unsafe { System.realloc(p, l, new) }; + if !q.is_null() { + let (old, new) = (l.size() as u64, new as u64); + let live = if new >= old { + LIVE.fetch_add(new - old, Ordering::Relaxed) + (new - old) + } else { + LIVE.fetch_sub(old - new, Ordering::Relaxed) - (old - new) + }; + note(live); + } + q + } +} + +#[global_allocator] +static ALLOCATOR: Counting = Counting; + +/// The high-water since the last call, rebased to live-now. Meaningful only +/// while one thread is allocating — hence the per-file table's `workers == 1` +/// guard. +fn take_mark() -> u64 { + let live = LIVE.load(Ordering::Relaxed); + MARK.swap(live, Ordering::Relaxed).saturating_sub(live) +} + +use quicksearch_core::config::Config; +use quicksearch_core::extract::Registry; +use quicksearch_core::file_handling::decide_content; +use quicksearch_core::testutil::mib; + +/// Head bytes read for the MIME sniff — the same window the walk uses, so +/// this probe classifies files exactly as a run would. +fn sniff(path: &Path, hash_length: usize) -> Option { + use std::io::Read; + let mut f = std::fs::File::open(path).ok()?; + let mut head = vec![0u8; hash_length]; + let n = f.read(&mut head).ok()?; + head.truncate(n); + quicksearch_core::mime::guess_mime_from_head(path, &head) +} + +struct Candidate { + path: String, + mime: String, + size: u64, +} + +/// Every file under `dir` an extractor claims and a run would not reject as +/// oversize: exactly the set the content pass would be handed. +fn candidates(dir: &Path, config: &Config, registry: &Registry) -> Vec { + let mut out = Vec::new(); + for entry in walkdir::WalkDir::new(dir) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + { + let Ok(meta) = entry.metadata() else { continue }; + if meta.len() > config.processing.maximum_text_file_size { + continue; + } + let Some(path) = entry.path().to_str().map(str::to_string) else { + continue; + }; + let Some(mime) = sniff(entry.path(), config.processing.hash_length) else { + continue; + }; + if !registry.supports(&mime) { + continue; + } + out.push(Candidate { + path, + mime, + size: meta.len(), + }); + } + out +} + +fn main() { + let mut args = std::env::args().skip(1); + let Some(dir) = args.next().map(PathBuf::from) else { + eprintln!("usage: extractprobe [workers] [replicas]"); + std::process::exit(2); + }; + let workers: usize = args + .next() + .map(|v| v.parse().expect("workers must be a number")) + .unwrap_or(1) + .max(1); + let replicas: usize = args + .next() + .map(|v| v.parse().expect("replicas must be a number")) + .unwrap_or(1) + .max(1); + + let config = Config::default(); + let registry = Arc::new(Registry::default_set()); + + let found = candidates(&dir, &config, ®istry); + if found.is_empty() { + eprintln!( + "extractprobe: nothing under {} is extractable", + dir.display() + ); + std::process::exit(1); + } + eprintln!( + "extractprobe {}: {} extractable file(s), {} workers, {} replica(s)", + dir.display(), + found.len(), + workers, + replicas + ); + + // Baseline *after* the scan: the candidate list is the probe's own cost, + // not extraction's. + let baseline_rss = rss(); + let baseline_live = LIVE.load(Ordering::Relaxed); + PEAK_LIVE.store(baseline_live, Ordering::Relaxed); + + // Replicas share one queue, so every worker stays busy to the end + // instead of the pool draining down to one straggler. + let queue: Vec<&Candidate> = (0..replicas).flat_map(|_| found.iter()).collect(); + let next = AtomicUsize::new(0); + let worst: Mutex> = Mutex::new(Vec::new()); + let per_file = workers == 1; + + let start = Instant::now(); + std::thread::scope(|s| { + for _ in 0..workers { + let (queue, next, worst) = (&queue, &next, &worst); + let (registry, config) = (registry.clone(), config.clone()); + s.spawn(move || loop { + let i = next.fetch_add(1, Ordering::Relaxed); + let Some(c) = queue.get(i) else { return }; + if per_file { + take_mark(); + } + let outcome = decide_content(&c.path, Some(&c.mime), ®istry, &config); + if per_file { + let cost = take_mark(); + let text = + quicksearch_core::file_handling::outcome_body(&outcome).map_or(0, str::len); + let mut w = worst + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + w.push((cost, c.path.clone(), c.size, c.mime.clone())); + // Kept small so the table itself is not the peak. + w.sort_by_key(|(cost, ..)| std::cmp::Reverse(*cost)); + w.truncate(12); + let _ = text; + } + }); + } + }); + let elapsed = start.elapsed(); + + let peak_live = PEAK_LIVE + .load(Ordering::Relaxed) + .saturating_sub(baseline_live); + let hwm = common::vm_hwm().unwrap_or(0); + eprintln!( + "\n {} file(s) in {:.1}s ({:.0}/s)", + queue.len(), + elapsed.as_secs_f64(), + queue.len() as f64 / elapsed.as_secs_f64().max(0.001), + ); + eprintln!( + " peak live {} above a {} baseline — what extraction really holds", + mib(peak_live), + mib(baseline_live), + ); + eprintln!( + " peak RSS {} (VmHWM), now {} — the gap is glibc holding freed chunks", + mib(hwm), + mib(rss()), + ); + eprintln!( + " baseline {} RSS before the first extraction", + mib(baseline_rss) + ); + eprintln!( + " per worker {} of peak live, at {} workers", + mib(peak_live / workers as u64), + workers, + ); + + let w = worst + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if !w.is_empty() { + eprintln!("\n most expensive files (live bytes held while extracting):"); + for (cost, path, size, mime) in w.iter() { + eprintln!( + " {:>10} from a {:>9} {} {}", + mib(*cost), + mib(*size), + mime, + path + ); + } + eprintln!( + "\n A pool of N runs N of these at once. `maximum_text_file_size` bounds\n \ + the input ({}), not the working set above.", + mib(config.processing.maximum_text_file_size), + ); + } +} + +/// Resident set size now, from `/proc/self/statm` field 2 (resident pages). +fn rss() -> u64 { + std::fs::read_to_string("/proc/self/statm") + .ok() + .and_then(|s| s.split_whitespace().nth(1)?.parse::().ok()) + .map(|pages| pages * 4096) + .unwrap_or(0) +} diff --git a/crates/quicksearch-core/examples/hashprobe.rs b/crates/quicksearch-core/examples/hashprobe.rs index 3dca6b2..d9e7934 100644 --- a/crates/quicksearch-core/examples/hashprobe.rs +++ b/crates/quicksearch-core/examples/hashprobe.rs @@ -835,5 +835,4 @@ mod tests { assert_eq!(stat.reported_pairs, 10_000 * 9_999 / 2); assert_eq!(stat.fp_pairs, 0); } - } diff --git a/crates/quicksearch-core/examples/indexprobe.rs b/crates/quicksearch-core/examples/indexprobe.rs index bd93af7..c6b2665 100644 --- a/crates/quicksearch-core/examples/indexprobe.rs +++ b/crates/quicksearch-core/examples/indexprobe.rs @@ -23,6 +23,11 @@ //! //! `QSB_HASH_LENGTH` overrides `[processing] hash_length` for a run — how //! [`hashprobe`](hashprobe.rs) gets its end-to-end column. +//! +//! `QSB_KEY=<64 hex digits>` measures an encrypted index. It is not optional +//! dressing: without a key installed this probe polls the completion marker +//! through a plain open, which an encrypted index cannot answer, so the run +//! reports a three-hour hang instead of its actual time. mod common; @@ -340,17 +345,51 @@ fn hash_length_override() -> Option { .and_then(|v| v.parse().ok()) } +/// `QSB_KEY=<64 hex digits>` measures an *encrypted* index: the key is +/// installed process-wide before any connection exists, exactly as the GUI +/// does after an unlock. Raw hex rather than a password, so no Argon2id +/// derivation lands inside a timed run. +fn install_key() -> bool { + match std::env::var("QSB_KEY") { + Ok(hex) => { + let key = quicksearch_core::security::IndexKey::from_hex(hex.trim()) + .expect("QSB_KEY must be 64 hex digits"); + quicksearch_core::db::set_process_key(Some(key)); + true + } + Err(_) => false, + } +} + +/// Open the index the way this run's key demands. +/// +/// **A plain open cannot read an encrypted index**, and +/// [`get_last_full_index`](quicksearch_core::db::repo::get_last_full_index) +/// reports that failure as `None` — indistinguishable from "not finished +/// yet". Polling an encrypted run through a plain open therefore never +/// observes its own completion and sits here until the deadline, which reads +/// as an indexing slowdown of several orders of magnitude rather than as the +/// probe defect it is. +fn probe_open(db: &Path, keyed: bool) -> Option { + if keyed { + quicksearch_core::db::open_existing(&db.to_string_lossy(), false).ok() + } else { + rusqlite::Connection::open(db).ok() + } +} + fn run(mode: &str, tree: &Path, db: &Path) { let mut config = Config::default(); if let Some(n) = hash_length_override() { config.processing.hash_length = n; } let hash_length = config.processing.hash_length; + let keyed = install_key(); // The marker is the one unambiguous completion signal; polling the // status enum races on a small tree. if db.exists() { - let conn = rusqlite::Connection::open(db).expect("open db"); + let conn = probe_open(db, keyed).expect("open db"); conn.execute("DELETE FROM schema_info WHERE key = 'last_full_index'", []) .expect("clear marker"); } @@ -374,14 +413,17 @@ fn run(mode: &str, tree: &Path, db: &Path) { ) .expect("start indexing"); - let deadline = Instant::now() + Duration::from_secs(600); + // Generous, like `memprobe`'s: the scale sweep this probe exists for runs + // hundreds of thousands of files, and a run that times out reports + // nothing. Past this is a hang, not a slow disk. + let deadline = Instant::now() + Duration::from_secs(3 * 3600); let mut done = false; while Instant::now() < deadline { if let IndexingStatus::Error(e) = service.get_status() { panic!("indexing failed: {}", e); } if db.exists() { - if let Ok(conn) = rusqlite::Connection::open(db) { + if let Some(conn) = probe_open(db, keyed) { if quicksearch_core::db::repo::get_last_full_index(&conn).is_some() { done = true; break; @@ -391,7 +433,11 @@ fn run(mode: &str, tree: &Path, db: &Path) { std::thread::sleep(Duration::from_millis(5)); } let elapsed = start.elapsed(); - assert!(done, "indexing did not finish within the timeout"); + assert!( + done, + "indexing did not finish within the timeout (keyed = {})", + keyed + ); // The run's last checkpoint happens inside here, so the sampler outlives it. service.stop_indexing().expect("stop"); wal_stop.store(true, Ordering::Relaxed); @@ -399,8 +445,7 @@ fn run(mode: &str, tree: &Path, db: &Path) { // Count what was actually indexed rather than assuming `gen`'s tree — // pointing the probe elsewhere made the rate a fiction. - let total = rusqlite::Connection::open(db) - .ok() + let total = probe_open(db, keyed) .and_then(|c| quicksearch_core::db::repo::row_count(&c).ok()) .unwrap_or(0); diff --git a/crates/quicksearch-core/examples/memprobe.rs b/crates/quicksearch-core/examples/memprobe.rs index b840b17..8099d23 100644 --- a/crates/quicksearch-core/examples/memprobe.rs +++ b/crates/quicksearch-core/examples/memprobe.rs @@ -11,6 +11,13 @@ //! ./target/release/examples/memprobe cold ~ /var/tmp/qs-mem/index.db 250 probe.toml //! ``` //! +//! **Roots are comma-separated**, because per-root state is what multiplies: +//! a one-root run cannot show the buffers that exist once per pipeline. +//! +//! ```text +//! ./target/release/examples/memprobe cold /media/shared,/home/me,/usr /var/tmp/qs-mem/index.db +//! ``` +//! //! Trailing arguments: sampling interval in ms (default 100) and a config //! file. `cold` deletes the database first; `warm` re-runs against the //! finished one. Reading the report: **VmHWM** cannot miss a spike — quote @@ -18,6 +25,10 @@ //! a per-file cost. Nothing reported is evictable page cache. `settled RSS` //! is what a long-lived process keeps — glibc frees to its arena, so //! without `release_free_heap` the peak becomes the floor. +//! +//! Built with `--features probe` the indexer also prints a `census` line +//! naming what its run-scoped structures hold, which is the half `smaps` +//! cannot answer: a mapping is "heap", never "the stale-candidate list". use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; @@ -66,12 +77,22 @@ struct Sample { fn main() { let mut args = std::env::args().skip(1); let mode = args.next().unwrap_or_default(); - let (Some(root), Some(db)) = (args.next(), args.next()) else { - eprintln!("usage: memprobe [sample_ms] [config.toml]"); + let (Some(roots), Some(db)) = (args.next(), args.next()) else { + eprintln!("usage: memprobe [sample_ms] [config.toml]"); std::process::exit(2); }; if mode != "cold" && mode != "warm" { - eprintln!("usage: memprobe [sample_ms] [config.toml]"); + eprintln!("usage: memprobe [sample_ms] [config.toml]"); + std::process::exit(2); + } + let roots: Vec = roots + .split(',') + .map(str::trim) + .filter(|r| !r.is_empty()) + .map(str::to_string) + .collect(); + if roots.is_empty() { + eprintln!("memprobe: no roots given"); std::process::exit(2); } let interval = Duration::from_millis( @@ -92,10 +113,10 @@ fn main() { } } - run(&mode, &root, &db, interval, config_path.as_deref()); + run(&mode, &roots, &db, interval, config_path.as_deref()); } -fn run(mode: &str, root: &str, db: &Path, interval: Duration, config_path: Option<&Path>) { +fn run(mode: &str, roots: &[String], db: &Path, interval: Duration, config_path: Option<&Path>) { // A config file only supplies the knobs that change *what* indexing // does; the defaults keep runs comparable. let config = match config_path { @@ -112,9 +133,10 @@ fn run(mode: &str, root: &str, db: &Path, interval: Duration, config_path: Optio let baseline = rss().expect("read /proc/self/statm"); eprintln!( - "memprobe {}: root={} db={}\n baseline RSS {} (process before indexing starts)", + "memprobe {}: {} root(s)={} db={}\n baseline RSS {} (process before indexing starts)", mode, - root, + roots.len(), + roots.join(" "), db.display(), mib(baseline) ); @@ -122,11 +144,7 @@ fn run(mode: &str, root: &str, db: &Path, interval: Duration, config_path: Optio let service = IndexingService::new(); let start = Instant::now(); service - .start_indexing( - vec![root.to_string()], - db.to_string_lossy().into_owned(), - config, - ) + .start_indexing(roots.to_vec(), db.to_string_lossy().into_owned(), config) .expect("start indexing"); let mut samples: Vec = Vec::new(); diff --git a/crates/quicksearch-core/src/coordinator.rs b/crates/quicksearch-core/src/coordinator.rs index 10bb68c..a2e16b0 100644 --- a/crates/quicksearch-core/src/coordinator.rs +++ b/crates/quicksearch-core/src/coordinator.rs @@ -59,9 +59,13 @@ pub enum WatcherStatus { /// Registration in flight — it walks every root, so this can last /// minutes on large or networked trees. Starting, - Active { dirs: usize }, + Active { + dirs: usize, + }, /// Live updates unavailable; the periodic reindex is the only refresh. - Disabled { reason: WatchError }, + Disabled { + reason: WatchError, + }, } /// A config reconciliation the coordinator applies between runs. diff --git a/crates/quicksearch-core/src/coordinator/inner.rs b/crates/quicksearch-core/src/coordinator/inner.rs index 4b6197f..8f82167 100644 --- a/crates/quicksearch-core/src/coordinator/inner.rs +++ b/crates/quicksearch-core/src/coordinator/inner.rs @@ -442,7 +442,11 @@ impl Inner { fn apply_queue(&mut self, conn: &mut Connection, targeted: bool) { let deadline = Instant::now() + APPLY_BUDGET; let chunk = self.config.processing.batch_size.max(1); - let queue = if targeted { &self.targeted } else { &self.pending }; + let queue = if targeted { + &self.targeted + } else { + &self.pending + }; let removals: Vec = queue .iter() @@ -474,7 +478,11 @@ impl Inner { } if Instant::now() < deadline { - let queue = if targeted { &self.targeted } else { &self.pending }; + let queue = if targeted { + &self.targeted + } else { + &self.pending + }; let upserts: Vec = queue .iter() .filter(|(_, ev)| !is_removal(ev)) diff --git a/crates/quicksearch-core/src/db/repo_tests.rs b/crates/quicksearch-core/src/db/repo_tests.rs index e311af3..3e75a90 100644 --- a/crates/quicksearch-core/src/db/repo_tests.rs +++ b/crates/quicksearch-core/src/db/repo_tests.rs @@ -1182,9 +1182,8 @@ fn deleting_a_file_row_cascades_the_fk_tables() { let (_dir, path) = tmp_path(); let mut conn = open_or_recreate(path.to_str().unwrap(), "trigram").unwrap(); let ids = seeded(&mut conn, &["/casc/a.txt", "/casc/b.txt"]); - let count = |conn: &Connection, sql: &str| -> i64 { - conn.query_row(sql, [], |r| r.get(0)).unwrap() - }; + let count = + |conn: &Connection, sql: &str| -> i64 { conn.query_row(sql, [], |r| r.get(0)).unwrap() }; { let tx = conn.transaction().unwrap(); set_content_failed(&tx, ids["/casc/b.txt"], "boom").unwrap(); @@ -1229,15 +1228,17 @@ fn retry_failed_files_resets_state_and_clears_records() { tx.commit().unwrap(); let state = |id: i64| -> i64 { - conn.query_row( - "SELECT content_state FROM files WHERE id = ?1", - [id], - |r| r.get(0), - ) + conn.query_row("SELECT content_state FROM files WHERE id = ?1", [id], |r| { + r.get(0) + }) .unwrap() }; assert_eq!(state(ids["/retry/bad.txt"]), STATE_PENDING); - assert_eq!(state(ids["/retry/good.txt"]), STATE_DONE, "done rows untouched"); + assert_eq!( + state(ids["/retry/good.txt"]), + STATE_DONE, + "done rows untouched" + ); let failures: i64 = conn .query_row("SELECT COUNT(*) FROM failed_files", [], |r| r.get(0)) .unwrap(); @@ -1267,7 +1268,11 @@ fn a_changed_file_clears_its_failure_record() { parent: "/chg/", size: 2, mtime: 9, - mime: if needs_content { Some("text/plain") } else { None }, + mime: if needs_content { + Some("text/plain") + } else { + None + }, ftype: FileType::TEXT, hash: None, needs_content, @@ -1284,11 +1289,9 @@ fn a_changed_file_clears_its_failure_record() { let b = update(&mut conn, "b.txt", true); let state = |id: i64| -> i64 { - conn.query_row( - "SELECT content_state FROM files WHERE id = ?1", - [id], - |r| r.get(0), - ) + conn.query_row("SELECT content_state FROM files WHERE id = ?1", [id], |r| { + r.get(0) + }) .unwrap() }; assert_eq!(state(a), STATE_NA); diff --git a/crates/quicksearch-core/src/extract/mod.rs b/crates/quicksearch-core/src/extract/mod.rs index 64bc5fe..4e63290 100644 --- a/crates/quicksearch-core/src/extract/mod.rs +++ b/crates/quicksearch-core/src/extract/mod.rs @@ -88,11 +88,7 @@ impl Registry { /// caller cannot forget. This cannot help with a stack overflow, which /// aborts rather than unwinding — see `vendor/pdf-extract`, which bounds /// the recursion that made that reachable. - pub fn extract( - &self, - path: &Path, - mime: &str, - ) -> Result, ExtractError> { + pub fn extract(&self, path: &Path, mime: &str) -> Result, ExtractError> { let Some(extractor) = self.find(mime) else { return Ok(None); }; diff --git a/crates/quicksearch-core/src/extract/office.rs b/crates/quicksearch-core/src/extract/office.rs index c1b643c..7d397ea 100644 --- a/crates/quicksearch-core/src/extract/office.rs +++ b/crates/quicksearch-core/src/extract/office.rs @@ -461,11 +461,7 @@ mod tests { "angle-bracket entities lost: {:?}", out ); - assert!( - out.contains('\u{2019}'), - "numeric entities lost: {:?}", - out - ); + assert!(out.contains('\u{2019}'), "numeric entities lost: {:?}", out); assert!( !out.contains("&") && !out.contains("&#"), "entities left unresolved: {:?}", diff --git a/crates/quicksearch-core/src/extract/plaintext.rs b/crates/quicksearch-core/src/extract/plaintext.rs index 6b32f99..4a83238 100644 --- a/crates/quicksearch-core/src/extract/plaintext.rs +++ b/crates/quicksearch-core/src/extract/plaintext.rs @@ -94,11 +94,7 @@ impl Extractor for PlaintextExtractor { decode(read_sized(&mut f, size, MAX_READ, path)?, path) } - fn extract_from_head( - &self, - path: &Path, - head: &[u8], - ) -> Option> { + fn extract_from_head(&self, path: &Path, head: &[u8]) -> Option> { Some(decode(head.to_vec(), path)) } } diff --git a/crates/quicksearch-core/src/extract/rtf.rs b/crates/quicksearch-core/src/extract/rtf.rs index 6d9acc3..6d0fe91 100644 --- a/crates/quicksearch-core/src/extract/rtf.rs +++ b/crates/quicksearch-core/src/extract/rtf.rs @@ -46,11 +46,7 @@ impl Extractor for RtfExtractor { } /// RTF has no trailer and needs no seeking; a complete head parses like disk. - fn extract_from_head( - &self, - path: &Path, - head: &[u8], - ) -> Option> { + fn extract_from_head(&self, path: &Path, head: &[u8]) -> Option> { Some(parse(head.to_vec(), path)) } } diff --git a/crates/quicksearch-core/src/file_handling/records.rs b/crates/quicksearch-core/src/file_handling/records.rs index 9fac203..6d04cda 100644 --- a/crates/quicksearch-core/src/file_handling/records.rs +++ b/crates/quicksearch-core/src/file_handling/records.rs @@ -314,7 +314,9 @@ pub fn extract_and_store( #[derive(Debug, Clone, PartialEq, Eq)] pub enum ContentOutcome { /// Text, already truncated to `maximum_text_size`. - Done { text: String }, + Done { + text: String, + }, NotApplicable, Failed(String), } diff --git a/crates/quicksearch-core/src/incremental.rs b/crates/quicksearch-core/src/incremental.rs index 1ea3c8f..56f919a 100644 --- a/crates/quicksearch-core/src/incremental.rs +++ b/crates/quicksearch-core/src/incremental.rs @@ -42,7 +42,9 @@ pub enum Applied { Done, /// Budget spent. What was written is committed; the caller should re-queue /// the same event with `resume_from` set to `done`. - Unfinished { done: usize }, + Unfinished { + done: usize, + }, } /// Apply one filesystem event to the index. Missing files are treated as diff --git a/crates/quicksearch-core/src/indexing/pipeline.rs b/crates/quicksearch-core/src/indexing/pipeline.rs index 6756a72..24a0749 100644 --- a/crates/quicksearch-core/src/indexing/pipeline.rs +++ b/crates/quicksearch-core/src/indexing/pipeline.rs @@ -166,6 +166,91 @@ fn per_second(n: usize, elapsed: Duration) -> Option { /// producer wakes it at once. const IDLE_BACKOFF: Duration = Duration::from_millis(2); +/// What each run-scoped structure holds, for the `probe` builds only. +/// +/// Peak RSS is one anonymous heap: `smaps` can say "the heap grew", never +/// "the stale-candidate list grew". This says which. Sizes are estimates of +/// the *heap* each structure owns — the point is which one dominates and +/// whether it tracks the tree, not a byte-exact total. See `examples/memprobe.rs`. +#[cfg(feature = "probe")] +mod census { + use super::{RootPipeline, RunCx}; + use crate::testutil::mib; + use std::time::{Duration, Instant}; + + /// Rare next to a status publish: each line walks every string in the + /// run-scoped sets, which is O(tree) work that must not shape what it + /// measures. + const INTERVAL: Duration = Duration::from_secs(2); + + /// Owned string bytes plus the `String` headers a collection holds. + fn strings_bytes<'a>(it: impl Iterator, len: usize) -> u64 { + let body: usize = it.map(String::len).sum(); + body as u64 + (len * std::mem::size_of::()) as u64 + } + + pub(super) fn due(last: &mut Instant) -> bool { + if last.elapsed() < INTERVAL { + return false; + } + *last = Instant::now(); + true + } + + /// One line per structure group: the log collapses embedded newlines, so + /// a multi-line report would arrive as one unreadable line. + pub(super) fn report(cx: &RunCx<'_>, pipelines: &[RootPipeline], started: Instant) { + let at = started.elapsed().as_secs_f64(); + crate::log_info!( + "census t={:.1}s stale {} ({}) aliased {} ({})", + at, + cx.stale_candidates.len(), + mib(strings_bytes( + cx.stale_candidates.iter(), + cx.stale_candidates.len() + )), + cx.aliased_paths.len(), + mib(strings_bytes( + cx.aliased_paths.iter(), + cx.aliased_paths.len() + )), + ); + for p in pipelines { + let (dirs, dir_bytes) = p.walk.seen_dirs_footprint(); + let inline = |rows: &[crate::file_handling::OwnedNewFile]| -> u64 { + rows.iter() + .map(|r| { + (r.inline_text.as_ref().map_or(0, String::len) + + r.name.len() + + r.parent.len()) as u64 + }) + .sum() + }; + let ready: u64 = p + .ready + .iter() + .map(|r| { + (crate::file_handling::outcome_body(&r.outcome).map_or(0, str::len) + + r.name.len()) as u64 + }) + .sum(); + crate::log_info!( + "census t={:.1}s {} [{:?}] dirs {} ({}) pending {}+{} ({}) ready {} ({})", + at, + p.root, + p.phase, + dirs, + mib(dir_bytes), + p.pending_inserts.len(), + p.pending_updates.len(), + mib(inline(&p.pending_inserts) + inline(&p.pending_updates)), + p.ready.len(), + mib(ready), + ); + } + } +} + fn report_run_warnings() { let (failed, suppressed) = crate::file_handling::hash_failure_counts(); if failed > 0 { @@ -279,9 +364,6 @@ impl RootPipeline { // parent's absence as proof the file is gone. cx.aliased_paths.insert(file.path.clone()); } - if !cx.seen_paths.insert(file.digest) { - continue; - } let Some(rec) = file.record else { continue }; if file.action == FileIndexAction::Update { self.pending_updates.push(rec); @@ -473,9 +555,6 @@ pub(super) struct RunCx<'a> { /// Writer time one root's turn may take before the round moves on /// ([`crate::config::ProcessingConfig::writer_turn_slice_ms`]). pub(super) slice: Duration, - /// 128-bit path digests, not paths: measured, owning every path string - /// again was the single largest allocation in a run. - pub(super) seen_paths: HashSet, /// Rows with no file behind them, per-directory plus the vanished sweep. pub(super) stale_candidates: Vec, /// Paths reached via symlink; their rows may live outside every root. @@ -498,7 +577,6 @@ impl<'a> RunCx<'a> { registry: Arc::new(Registry::default_set()), quantum: config.processing.batch_size.max(1), slice: Duration::from_millis(config.processing.writer_turn_slice_ms), - seen_paths: HashSet::new(), stale_candidates: Vec::new(), aliased_paths: HashSet::new(), stale_cleanup_ok: true, @@ -673,8 +751,9 @@ impl IndexingService { crate::walk::reset_run_warnings(); crate::file_handling::reset_run_warnings(); - // Canonicalized first so spelling variants collapse to one walk; - // nested-root dedup is the `seen_paths` set's job. + // Canonicalized first so spelling variants collapse to one walk. + // Nested roots need no handling here: the coordinator refuses to + // start a run with any (`config::nested_roots`). let mut seen_roots = HashSet::new(); let roots: Vec = paths .iter() @@ -763,6 +842,8 @@ impl IndexingService { let aborted; let mut cleanup_done = false; let mut rr = 0usize; + #[cfg(feature = "probe")] + let mut last_census = Instant::now(); let wal_path = format!("{}-wal", db_path); let configured_cap = match config.processing.maximum_wal_size { 0 => 0, @@ -812,6 +893,11 @@ impl IndexingService { publish_status(status, run_start, &pipelines); + #[cfg(feature = "probe")] + if census::due(&mut last_census) { + census::report(&cx, &pipelines, run_started); + } + // After `publish_status`: the checkpoint may block for // `busy_timeout`, and it must not sit in front of `stop_indexing` // — hence the stop-flag check. diff --git a/crates/quicksearch-core/src/indexing/tests.rs b/crates/quicksearch-core/src/indexing/tests.rs index 2f98f33..52d1668 100644 --- a/crates/quicksearch-core/src/indexing/tests.rs +++ b/crates/quicksearch-core/src/indexing/tests.rs @@ -494,11 +494,41 @@ fn the_walk_denominator_prefers_the_best_available_count() { for (label, phase, walked, estimate, want) in [ ("estimate", RootPhase::Walking, 100, Some(1000), Some(1000)), ("no count yet", RootPhase::Walking, 100, None, None), - ("overtaken", RootPhase::Walking, 1500, Some(1000), Some(1500)), - ("exact", RootPhase::Extracting, 261_088, Some(6_677_062), Some(261_088)), - ("exact, no estimate", RootPhase::Extracting, 261_088, None, Some(261_088)), - ("done", RootPhase::Done, 261_088, Some(6_677_062), Some(261_088)), - ("done, no estimate", RootPhase::Done, 261_088, None, Some(261_088)), + ( + "overtaken", + RootPhase::Walking, + 1500, + Some(1000), + Some(1500), + ), + ( + "exact", + RootPhase::Extracting, + 261_088, + Some(6_677_062), + Some(261_088), + ), + ( + "exact, no estimate", + RootPhase::Extracting, + 261_088, + None, + Some(261_088), + ), + ( + "done", + RootPhase::Done, + 261_088, + Some(6_677_062), + Some(261_088), + ), + ( + "done, no estimate", + RootPhase::Done, + 261_088, + None, + Some(261_088), + ), ] { assert_eq!( progress(phase, walked, estimate).walk_denominator(), @@ -612,13 +642,28 @@ fn the_wal_cap_is_bounded_by_the_volume() { for (label, cfg, free, want) in [ ("roomy", configured, 500 * 1024 * 1024 * 1024, configured), // Exactly enough: floor plus four times the log. - ("just enough", configured, 128 * 1024 * 1024 + configured * 4, configured), + ( + "just enough", + configured, + 128 * 1024 * 1024 + configured * 4, + configured, + ), // 1 GiB free: 896 MiB above the floor, a quarter of which is 224 MiB. ("tight", configured, 1024 * 1024 * 1024, 224 * 1024 * 1024), ("disabled, tight", 0, 1024 * 1024 * 1024, 224 * 1024 * 1024), ("full", configured, 0, crate::config::MINIMUM_WAL_SIZE), - ("nearly full", configured, 1024, crate::config::MINIMUM_WAL_SIZE), - ("at the floor", configured, 127 * 1024 * 1024, crate::config::MINIMUM_WAL_SIZE), + ( + "nearly full", + configured, + 1024, + crate::config::MINIMUM_WAL_SIZE, + ), + ( + "at the floor", + configured, + 127 * 1024 * 1024, + crate::config::MINIMUM_WAL_SIZE, + ), ] { assert_eq!(wal_cap_for_free(cfg, free), want, "{label}"); } diff --git a/crates/quicksearch-core/src/query/split.rs b/crates/quicksearch-core/src/query/split.rs index 2c08d55..a9d2247 100644 --- a/crates/quicksearch-core/src/query/split.rs +++ b/crates/quicksearch-core/src/query/split.rs @@ -8,10 +8,10 @@ //! unrecognized `key:value` (`12:30`) reassembles verbatim; `AND`/`OR` pass //! through as words, parens are dropped. -use super::Op; use super::lexer::{tokenize, Token}; use super::pattern::{RegexQuery, TermPart, TermPattern}; use super::translator::{build_filter, is_filter_key, TranslateError}; +use super::Op; /// The cascade's parsed input: one term string plus composable filter SQL. #[derive(Debug, Clone, Default)] diff --git a/crates/quicksearch-core/src/search/cascade/passes.rs b/crates/quicksearch-core/src/search/cascade/passes.rs index 7f9889b..446c181 100644 --- a/crates/quicksearch-core/src/search/cascade/passes.rs +++ b/crates/quicksearch-core/src/search/cascade/passes.rs @@ -283,7 +283,16 @@ impl<'a> Cx<'a> { // A name/path hit's snippet is the field itself, span marked. let snip = snippet::whole_field(if is_path_tier { path } else { name }, match_range); - row_hit(row, file_id, path, name, rank, rank as u8, Some(snip), is_path_tier) + row_hit( + row, + file_id, + path, + name, + rank, + rank as u8, + Some(snip), + is_path_tier, + ) }, ) } @@ -535,7 +544,16 @@ impl<'a> Cx<'a> { }; let snip = snippet::whole_field(if is_path_tier { path } else { name }, match_range); - row_hit(row, file_id, path, name, rank, rank as u8, Some(snip), is_path_tier) + row_hit( + row, + file_id, + path, + name, + rank, + rank as u8, + Some(snip), + is_path_tier, + ) }, ) } @@ -575,7 +593,16 @@ impl<'a> Cx<'a> { let r = clamp_match_range(text, r, SNIPPET_WINDOW_CHARS); snippet::window_around(text, (r.start, r.end), &snippet_opts) }); - row_hit(row, file_id, path, name, 6.0 + count_frac(count), 6, snip, false) + row_hit( + row, + file_id, + path, + name, + 6.0 + count_frac(count), + 6, + snip, + false, + ) }) } } diff --git a/crates/quicksearch-core/src/search/mod.rs b/crates/quicksearch-core/src/search/mod.rs index b96b5f2..cc79d40 100644 --- a/crates/quicksearch-core/src/search/mod.rs +++ b/crates/quicksearch-core/src/search/mod.rs @@ -25,9 +25,24 @@ use crate::snippet::Snippet; pub use cascade::Outcome; pub use duplicates::{find_duplicate_groups, DuplicateGroup}; -/// Idle window before the connection is released: an open reader stops -/// SQLite resetting the WAL and pins a deleted index's blocks. -const IDLE_RELEASE: Duration = Duration::from_secs(30); +/// Idle window before the connection is released. +/// +/// Long, because releasing throws away [`PRAGMAS_SEARCH`]'s 32 MiB of +/// decrypted pages and the next keystroke pays to refill them: on a 200k-file +/// index the first query after a release costs 129 ms unencrypted and 192 ms +/// encrypted, against ~10 ms warm. Encryption is why the gap widens — a +/// refill is an AES decrypt plus an HMAC verify per page rather than a +/// `memcpy`. What the release buys back is the ~42 MiB the trim in +/// [`Worker::run`] returns, so this trades an idle process floor against +/// stalling the one keystroke a user is most likely to notice. +/// +/// It is *not* what lets a rebuild delete the file: `Backend::rebuild_index` +/// and `clear_index` send [`WorkerMsg::ReleaseConnection`] first, because +/// Windows fails the delete while any handle is open. Nor does holding a +/// connection stop the WAL truncating — a read mark lives for the length of a +/// statement, not of the connection (`db::repo_tests` pins both halves of +/// that: a checkpoint loses to a reader *mid-query*, not to an idle one). +const IDLE_RELEASE: Duration = Duration::from_secs(30 * 60); /// One search result. `rank` is the sort key (lower = better): integer part /// = cascade stage (1–11), fraction = tiebreak. Batches arrive rank-ordered diff --git a/crates/quicksearch-core/src/testutil.rs b/crates/quicksearch-core/src/testutil.rs index d7812fd..4d794d7 100644 --- a/crates/quicksearch-core/src/testutil.rs +++ b/crates/quicksearch-core/src/testutil.rs @@ -247,6 +247,11 @@ pub struct SeedSpec { /// Document bodies carrying [`BODY_TERM`]; sized by the caller to stay /// under the display limit. pub body_term_docs: usize, + /// One row in every `dup_every` repeats its predecessor's content hash, + /// giving [`crate::search::find_duplicate_groups`] real groups to rank. + /// `0` leaves every hash NULL — the shape the search harnesses seed, and + /// the one whose row width their numbers were taken against. + pub dup_every: usize, } impl Default for SeedSpec { @@ -261,6 +266,7 @@ impl Default for SeedSpec { needle_names: 50, needle_docs: 50, body_term_docs: 500, + dup_every: 0, } } } @@ -289,6 +295,21 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) { }; // Stored parents always end in a separator; see `dir_to_db_parent`. let dir = format!("/seed/{:03}/", i % spec.dirs.max(1)); + // Every `dup_every`-th row takes the hash of the one before it, so the + // groups are pairs of equal-sized rows — the shape `find_duplicate_groups` + // prices, since a hash covers the size. + let hash = (spec.dup_every > 0).then(|| { + let group = if i % spec.dup_every == spec.dup_every - 1 { + i.saturating_sub(1) + } else { + i + }; + let mut bytes = [0u8; 32]; + for (b, slot) in bytes.iter_mut().enumerate() { + *slot = ((group as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15) >> (b % 8 * 8)) as u8; + } + bytes + }); let id = insert_file( &tx, &NewFile { @@ -298,7 +319,7 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) { mtime: 1_700_000_000 + i as u64, mime: Some("text/plain"), ftype: FileType::TEXT, - hash: None, + hash: hash.as_ref().map(|h| h.as_slice()), needs_content: i % spec.content_every.max(1) == 0, }, ) diff --git a/crates/quicksearch-core/src/verify.rs b/crates/quicksearch-core/src/verify.rs index 724fd31..fb4be4b 100644 --- a/crates/quicksearch-core/src/verify.rs +++ b/crates/quicksearch-core/src/verify.rs @@ -27,7 +27,10 @@ pub enum MemberVerdict { DiffersAt(u64), /// Lengths disagree, so nothing was read. Within a duplicate group this /// can only mean a stale index — the hash covers the size. - LengthDiffers { len: u64, reference_len: u64 }, + LengthDiffers { + len: u64, + reference_len: u64, + }, Unreadable(String), } diff --git a/crates/quicksearch-core/src/walk.rs b/crates/quicksearch-core/src/walk.rs index 234df5e..1c3c3cc 100644 --- a/crates/quicksearch-core/src/walk.rs +++ b/crates/quicksearch-core/src/walk.rs @@ -14,8 +14,6 @@ use std::sync::{mpsc, Arc, Condvar, Mutex}; use std::thread::JoinHandle; use std::time::UNIX_EPOCH; -use sha2::{Digest, Sha256}; - use crate::config::{Config, IgnoreSet}; use crate::extract::Registry; use crate::file_handling::{ @@ -47,8 +45,6 @@ pub struct WalkedFile { pub path: String, pub action: FileIndexAction, pub record: Option, - /// 128-bit truncated SHA-256 of the path, for the duplicate-visit set. - pub digest: u128, /// True when this file was reached by resolving a symlink. Its row is /// invisible to its real parent's reconciliation, so the caller must /// exempt it from the vanished-directory sweep. @@ -57,12 +53,11 @@ pub struct WalkedFile { impl WalkedFile { /// Seen, but with nothing to write: the row stays. - fn skipped(path: String, digest: u128, aliased: bool) -> Self { + fn skipped(path: String, aliased: bool) -> Self { WalkedFile { path, action: FileIndexAction::Skip, record: None, - digest, aliased, } } @@ -344,17 +339,6 @@ enum Known<'a> { Exact(Option), } -/// 128-bit truncated SHA-256 of a path, for the writer's duplicate-visit set. -/// 16 bytes is ~4e-26 collision probability at 7M paths (8 would be ~1e-6), -/// and a collision silently drops a real file. Cryptographic because shared -/// filenames are attacker-supplied: a chosen pair could hide one file. -pub fn path_digest(path: &str) -> u128 { - let digest = Sha256::digest(path.as_bytes()); - let mut bytes = [0u8; 16]; - bytes.copy_from_slice(&digest[..16]); - u128::from_be_bytes(bytes) -} - /// At most one `stat`, then classify; only files that will be written get /// opened, and small text files are finished outright. "At most": on Windows /// [`PendingFile::cached`] may already hold the answer. @@ -362,20 +346,19 @@ fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile { let PendingFile { path, cached } = file; // Every route here has already screened the path for UTF-8: // `path_to_db_string` is lossy, and a lossy string would key another - // file's row and could consume its digest. + // file's row. debug_assert!( path.to_str().is_some(), "an unrepresentable path reached prepare(): {:?}", path ); let db_path = path_to_db_string(&path); - let digest = path_digest(&db_path); let aliased = matches!(known, Known::Exact(_)); let Ok(meta) = crate::platform::metadata_or_stat(&path, cached) else { // Seen but unreadable: a transient stat failure must not read as // "deleted". - return WalkedFile::skipped(db_path, digest, aliased); + return WalkedFile::skipped(db_path, aliased); }; let Some(mtime) = meta .modified() @@ -383,7 +366,7 @@ fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile { .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) .map(|d| d.as_secs()) else { - return WalkedFile::skipped(db_path, digest, aliased); + return WalkedFile::skipped(db_path, aliased); }; let action = match known { @@ -410,7 +393,6 @@ fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile { path: db_path, action, record, - digest, aliased, } } @@ -546,6 +528,21 @@ impl ParallelWalk { self.shared.stats.clone() } + /// How many directories the walk has queued and roughly what their paths + /// own on the heap — the set is live for the whole run, so it is one of + /// the few structures that tracks the tree. Locks the queue and walks it, + /// so it belongs to the census and nothing else. + #[cfg(feature = "probe")] + pub fn seen_dirs_footprint(&self) -> (usize, u64) { + let q = crate::lock_ok(&self.shared.queue); + let bytes: u64 = q + .seen_dirs + .iter() + .map(|d| (d.as_os_str().len() + std::mem::size_of::()) as u64) + .sum(); + (q.seen_dirs.len(), bytes) + } + /// Join the workers and report whether every one finished cleanly. A dead /// worker and a finished one look identical from the receiving end, and /// treating a panicked walk as complete would hand stale cleanup a diff --git a/crates/quicksearch-core/src/walk/tests.rs b/crates/quicksearch-core/src/walk/tests.rs index 815551f..5164ce7 100644 --- a/crates/quicksearch-core/src/walk/tests.rs +++ b/crates/quicksearch-core/src/walk/tests.rs @@ -231,9 +231,9 @@ fn a_root_that_resolves_onto_an_unrepresentable_name_is_not_walked() { ); } -/// The pipeline hashes a `WalkedFile`'s path into `seen_paths` before it looks -/// for a record, so a bad name yielded under its lossy spelling made whichever -/// of the two files the walk reached second fall out of the index. +/// A `WalkedFile`'s path is the key its row is written under, so a bad name +/// yielded under its lossy spelling would claim the twin's row: whichever of +/// the two files the walk reached second fell out of the index. #[test] fn a_bad_name_cannot_stand_in_for_its_lossy_twin() { let root = tmp_tree("nonutf8-twin"); @@ -489,8 +489,9 @@ fn symlink_loop_terminates() { #[test] #[cfg(unix)] fn symlinked_file_resolves_to_its_target_path() { - // The walker dedupes *directories*; the caller's `seen_paths` dedupes - // files, so both routes must agree on the canonical path. + // The walker dedupes *directories*, not files: a file reached both ways + // is yielded twice, so both routes must agree on the canonical path or + // the writer would key two different rows for one file. let root = tmp_tree("symlink-file"); touch(&root.join("real/target.txt")); fs::create_dir_all(root.join("links")).unwrap(); diff --git a/crates/quicksearch-core/tests/encrypted_perf.rs b/crates/quicksearch-core/tests/encrypted_perf.rs new file mode 100644 index 0000000..386388b --- /dev/null +++ b/crates/quicksearch-core/tests/encrypted_perf.rs @@ -0,0 +1,250 @@ +//! Encryption must cost a constant factor, not a different algorithm. +//! +//! SQLCipher decrypts and HMAC-verifies every 4 KiB page it reads, so a keyed +//! index is intrinsically slower than a plain one — that part is not a bug and +//! this file does not try to gate it. What it gates is *amplification*: a query +//! whose cost is one page fetch per row is fine unencrypted (the page cache +//! makes it nearly free) and disastrous keyed. `find_duplicate_groups` was +//! exactly that until it was rewritten to stay inside `idx_files_hash`: +//! +//! | shape | plain | encrypted | ratio | +//! |---|---|---|---| +//! | row fetch per file (pre-`8a7810d`) | 0.59 s | 2.28 s | **3.9x** | +//! | covering index scan (current) | 0.34 s | 0.44 s | 1.3x | +//! +//! Measured on 400k rows, so the ceiling below sits between those two: the old +//! shape fails it, the current one passes with room. The ratio is what makes +//! this a *test* rather than a benchmark — both arms run the same workload on +//! the same machine in the same process, so host speed, CPU governor and CI +//! contention divide out. Absolute times are printed but never asserted. +//! +//! Its own integration binary because it installs a process-global key, the +//! same reason `tests/encrypted.rs` gives. + +use std::path::PathBuf; +use std::sync::atomic::AtomicU64; +use std::time::{Duration, Instant}; + +use quicksearch_core::db; +use quicksearch_core::query::split::split_for_cascade; +use quicksearch_core::search::{cascade, find_duplicate_groups, SearchHit, SearchOptions}; +use quicksearch_core::security::IndexKey; +use quicksearch_core::testutil::{scratch_db, seed_index, SeedSpec, BODY_TERM, NEEDLE}; + +/// A raw 32-byte key, not an Argon2id derivation: the KDF costs half a second +/// in release and minutes in debug, and proves nothing about page work. It +/// reaches SQLCipher as raw hex either way (see `db::open::key_and_probe`), so +/// what is measured below is identical to a real unlocked index. +const KEY_HEX: &str = "a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90"; + +/// Ceiling on encrypted/plain for one workload. Between the 3.9x the old +/// duplicate query cost and the 1.3x the current one costs; see the table +/// above. Raising this without a measurement in the same table defeats it. +const MAX_RATIO: f64 = 3.0; + +/// Enough rows that neither index fits in `PRAGMAS_SEARCH`'s 32 MiB page +/// cache — the only regime where a per-page decrypt is visible at all. Below +/// that both arms are served from cache, every ratio is 1.0, and the gate +/// silently stops testing anything. `index_is_larger_than_the_search_cache` +/// pins that this seed still clears it. +const FILES: usize = 60_000; +const CONTENT_EVERY: usize = 5; + +/// The cache the search connection actually opens with, from +/// `db::schema::PRAGMAS_SEARCH`. +const SEARCH_CACHE_BYTES: u64 = 32 * 1024 * 1024; + +/// Best-of-N. The minimum is the run least disturbed by everything else on +/// the box, which is the honest figure for a comparison — a mean would +/// measure the CI runner's other tenants. +const RUNS: u32 = 5; + +/// Below this, a ratio is noise over noise: two sub-millisecond timings +/// divide into anything. Every workload here is far above it; the guard is +/// for the day someone shrinks the seed. +const MIN_MEASURABLE: Duration = Duration::from_millis(3); + +fn spec() -> SeedSpec { + SeedSpec { + files: FILES, + content_every: CONTENT_EVERY, + // One row in five pairs up: enough groups that ranking them is real + // work, not so many that the whole table is one giant group. + dup_every: 5, + ..SeedSpec::default() + } +} + +fn key() -> IndexKey { + IndexKey::from_hex(KEY_HEX).expect("a 64-hex-digit key") +} + +/// Seed the same corpus twice, once plain and once keyed. Identical content +/// and identical insertion order, so the two indexes differ *only* by +/// encryption — which is what lets a display-limited query be compared at all +/// (the cascade stops when the limit fills, so a different rowid order would +/// decide the answer rather than the encryption). +fn seed_both() -> (PathBuf, PathBuf) { + let plain = scratch_db("encperf-plain"); + let keyed = scratch_db("encperf-keyed"); + + db::set_process_key(None); + seed_index(&plain, &spec()); + + db::set_process_key(Some(key())); + seed_index(&keyed, &spec()); + db::set_process_key(None); + + (plain, keyed) +} + +fn mib(path: &PathBuf) -> f64 { + std::fs::metadata(path).map(|m| m.len()).unwrap_or(0) as f64 / (1024.0 * 1024.0) +} + +/// Run `f` `RUNS` times, keeping the fastest. +fn best_of(mut f: impl FnMut()) -> Duration { + let mut best = Duration::MAX; + for _ in 0..RUNS { + let start = Instant::now(); + f(); + best = best.min(start.elapsed()); + } + best +} + +/// One workload's verdict. Collected rather than asserted inline so a run +/// reports *every* ratio, not just the first one that failed. +struct Measured { + what: &'static str, + plain: Duration, + keyed: Duration, +} + +impl Measured { + fn ratio(&self) -> f64 { + self.keyed.as_secs_f64() / self.plain.as_secs_f64() + } + + fn line(&self) -> String { + format!( + "{:<28} plain {:>9.2?} encrypted {:>9.2?} ratio {:>5.2}x", + self.what, + self.plain, + self.keyed, + self.ratio() + ) + } +} + +/// Time `find_duplicate_groups`, which opens its own connection — so the +/// process key has to be right at call time, not at open time. +fn time_duplicates(path: &PathBuf, keyed: bool) -> Duration { + let db_path = path.to_string_lossy().into_owned(); + best_of(|| { + db::set_process_key(keyed.then(key)); + let groups = find_duplicate_groups(&db_path, 200).expect("duplicate scan"); + assert!(!groups.is_empty(), "the seed must contain duplicate groups"); + }) +} + +/// Time one cascade query on a connection opened while its key state was +/// installed. Connections keep their own codec, so no toggling is needed once +/// they are open. +fn time_query(conn: &rusqlite::Connection, query: &str, fuzzy: bool) -> Duration { + let split = split_for_cascade(query).expect("query parses"); + let options = SearchOptions { + fuzzy, + ..SearchOptions::default() + }; + best_of(|| { + let latest = AtomicU64::new(1); + let mut hits = 0usize; + let mut sink = |batch: Vec| hits += batch.len(); + cascade::run(conn, &split, &options, 1, &latest, &mut sink).expect("cascade runs"); + assert!(hits > 0, "'{}' must match something to be timed", query); + }) +} + +#[test] +fn encryption_costs_a_constant_factor_not_a_different_algorithm() { + let (plain, keyed) = seed_both(); + + // Both connections are opened up front, each under its own key state. + db::set_process_key(None); + let plain_conn = db::open::open_search_reader(&plain.to_string_lossy()).expect("open plain"); + db::set_process_key(Some(key())); + let keyed_conn = db::open::open_search_reader(&keyed.to_string_lossy()).expect("open keyed"); + db::set_process_key(None); + + println!( + "seeded {} files ({} with content): plain {:.1} MiB, encrypted {:.1} MiB", + FILES, + FILES / CONTENT_EVERY, + mib(&plain), + mib(&keyed), + ); + assert!( + (mib(&plain) * 1024.0 * 1024.0) as u64 > SEARCH_CACHE_BYTES, + "seed is smaller than the {} MiB search cache, so both arms would be \ + served entirely from memory and every ratio below would be a \ + meaningless 1.0 — raise FILES", + SEARCH_CACHE_BYTES / (1024 * 1024) + ); + + // Duplicate finding first: it is the shape this gate exists for. + let mut measured = vec![Measured { + what: "find_duplicate_groups", + plain: time_duplicates(&plain, false), + keyed: time_duplicates(&keyed, true), + }]; + + // The cascade's four shapes. Arms alternate per workload so a machine that + // slows down partway through moves both sides, not one. + for (what, query, fuzzy) in [ + ("cascade literal name", NEEDLE, false), + ("cascade literal body", BODY_TERM, false), + ("cascade fuzzy", "quartzlte", true), + ("cascade wildcard", "quart*", false), + ("cascade regex", "regex:quart[sz]ite", false), + ] { + measured.push(Measured { + what, + plain: time_query(&plain_conn, query, fuzzy), + keyed: time_query(&keyed_conn, query, fuzzy), + }); + } + + for m in &measured { + println!("{}", m.line()); + } + + let too_short: Vec<&Measured> = measured + .iter() + .filter(|m| m.plain < MIN_MEASURABLE || m.keyed < MIN_MEASURABLE) + .collect(); + assert!( + too_short.is_empty(), + "these workloads finished under {:?}, so their ratios are noise over \ + noise rather than a measurement:\n{}", + MIN_MEASURABLE, + too_short + .iter() + .map(|m| m.line()) + .collect::>() + .join("\n") + ); + + let amplified: Vec<&Measured> = measured.iter().filter(|m| m.ratio() > MAX_RATIO).collect(); + assert!( + amplified.is_empty(), + "encryption amplified these beyond {:.1}x, which means per-page work \ + scaling with rows rather than a constant factor:\n{}", + MAX_RATIO, + amplified + .iter() + .map(|m| m.line()) + .collect::>() + .join("\n") + ); +} diff --git a/crates/quicksearch-core/tests/extraction_corpus.rs b/crates/quicksearch-core/tests/extraction_corpus.rs index 4987550..5ff3b05 100644 --- a/crates/quicksearch-core/tests/extraction_corpus.rs +++ b/crates/quicksearch-core/tests/extraction_corpus.rs @@ -115,7 +115,8 @@ fn head_extraction_agrees_with_reading_the_file() { .expect("claimed"); assert_eq!( - from_head, from_disk, + from_head, + from_disk, "{} head and disk extraction disagree", ctx(sample) ); diff --git a/crates/quicksearch-core/tests/full_index.rs b/crates/quicksearch-core/tests/full_index.rs index 7c7c60a..f9c2778 100644 --- a/crates/quicksearch-core/tests/full_index.rs +++ b/crates/quicksearch-core/tests/full_index.rs @@ -483,6 +483,66 @@ fn a_symlink_target_in_an_unwalked_directory_survives_reindexing() { assert!(off[0].ends_with("normal.txt")); } +/// A target *inside* the root, so the walk reaches it twice — once by reading +/// its own directory, once by resolving the link. The other two symlink tests +/// use targets the walk never reaches directly, so this is the only one where +/// a duplicate visit actually happens. +/// +/// **This is what replaced the writer's duplicate-visit set.** That set held a +/// path digest for every walked file — the largest thing a run kept — to stop +/// a second visit reaching the writer. It was removed once measured: a repeat +/// visit is already collapsed by `insert_file` being `INSERT OR IGNORE` and +/// returning `None` for a row that exists, so it can produce neither a second +/// row nor a second FTS entry. This test is the guard on that claim; it is +/// expected to fail if `insert_file` ever stops ignoring conflicts. +#[test] +#[cfg(unix)] +fn a_file_reachable_both_directly_and_through_a_link_gets_one_row() { + let root = Scratch::dir("alias-dup-root"); + let db_dir = Scratch::dir("alias-dup-db"); + let mut config = Config::default(); + config.indexing.follow_symlinks = true; + + let target = root.join("real/file.txt"); + touch(&target, b"reachable two ways"); + std::os::unix::fs::symlink(&target, root.join("link.txt")).unwrap(); + + let db = db_dir.join("links-on.sqlite"); + index_once(&root, &db, &config); + let on = rows(&db); + assert_eq!( + on.len(), + 1, + "one canonical path, one row, however many ways it was reached: {:?}", + on + ); + assert!( + on[0].0.ends_with("real/file.txt"), + "stored under the target" + ); + + // The link is not a second document: a duplicated visit that reached the + // writer twice would tokenize the body twice into a contentless FTS + // table, where nothing would later collapse the two. + let conn = rusqlite::Connection::open(&db).unwrap(); + let hits: i64 = conn + .query_row( + "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH '\"reachable\"'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(hits, 1, "the body is indexed once"); + + // Links off: the link is never followed, so the same tree still yields + // exactly the one row — the run that skips the set entirely. + let db2 = db_dir.join("links-off.sqlite"); + index_once(&root, &db2, &Config::default()); + let off = rows(&db2); + assert_eq!(off.len(), 1, "only the real file: {:?}", off); + assert_eq!(off[0].0, on[0].0, "and the same path as with links on"); +} + #[test] #[cfg(unix)] fn a_modified_symlink_target_is_updated_not_silently_ignored() { diff --git a/crates/quicksearch-gui/src/app/security.rs b/crates/quicksearch-gui/src/app/security.rs index 62b8183..a89e7e0 100644 --- a/crates/quicksearch-gui/src/app/security.rs +++ b/crates/quicksearch-gui/src/app/security.rs @@ -38,16 +38,9 @@ impl Drop for SecurityPrompt { /// Confirm the password, re-derive from it, reveal the installed key. /// Nothing here can change the key or the config. pub(super) enum KeyPrompt { - Confirm { - pw: String, - wrong: bool, - }, - Deriving { - rx: mpsc::Receiver, - }, - Reveal { - display: String, - }, + Confirm { pw: String, wrong: bool }, + Deriving { rx: mpsc::Receiver }, + Reveal { display: String }, } impl Drop for KeyPrompt { diff --git a/crates/quicksearch-gui/src/app/tests.rs b/crates/quicksearch-gui/src/app/tests.rs index 6bc50db..a03b9b5 100644 --- a/crates/quicksearch-gui/src/app/tests.rs +++ b/crates/quicksearch-gui/src/app/tests.rs @@ -166,7 +166,13 @@ fn only_an_empty_duplicates_tab_scans_on_arrival() { ); } // No other tab scans, whatever the Duplicates tab is holding. - for tab in [Tab::Search, Tab::Manage, Tab::Logs, Tab::Help, Tab::Settings] { + for tab in [ + Tab::Search, + Tab::Manage, + Tab::Logs, + Tab::Help, + Tab::Settings, + ] { assert!(!arrival_starts_dup_scan(tab, &DupState::NotLoaded)); } } @@ -189,7 +195,10 @@ fn an_index_that_changed_invalidates_what_the_tab_holds() { ); // Out of sight: drop it, and let the next visit pay for the rescan. - assert_eq!(dup_invalidation(Tab::Search, &loaded()), DupInvalidation::Drop); + assert_eq!( + dup_invalidation(Tab::Search, &loaded()), + DupInvalidation::Drop + ); // Nothing to invalidate. assert_eq!( diff --git a/crates/quicksearch-gui/src/backend.rs b/crates/quicksearch-gui/src/backend.rs index 4a07406..9e216b0 100644 --- a/crates/quicksearch-gui/src/backend.rs +++ b/crates/quicksearch-gui/src/backend.rs @@ -43,9 +43,11 @@ pub struct Backend { impl Backend { /// Rebuild, after letting go of everything holding the index open: the - /// search worker keeps its connection warm for half a minute, and - /// without the release the delete fails on Windows and the rebuild - /// silently becomes an ordinary run against the old index. + /// search worker keeps its connection warm for half an hour + /// (`search::IDLE_RELEASE`), and without the release the delete fails on + /// Windows and the rebuild silently becomes an ordinary run against the + /// old index. The window is long enough that waiting one out is not a + /// fallback — this release is the only thing that makes the delete work. pub fn rebuild_index(&self) { if let Some(search) = &self.search { search.release_connection(); @@ -138,7 +140,8 @@ impl Backend { let (tx, rx) = mpsc::channel(); let db = config.resolved_database_path(); std::thread::spawn(move || { - let result = quicksearch_core::search::find_duplicate_groups(&db.to_string_lossy(), 500); + let result = + quicksearch_core::search::find_duplicate_groups(&db.to_string_lossy(), 500); let _ = tx.send(result); ctx.request_repaint(); }); diff --git a/crates/quicksearch-gui/src/query_highlight.rs b/crates/quicksearch-gui/src/query_highlight.rs index a89e550..b61b085 100644 --- a/crates/quicksearch-gui/src/query_highlight.rs +++ b/crates/quicksearch-gui/src/query_highlight.rs @@ -10,10 +10,10 @@ use std::sync::Arc; use egui::text::{LayoutJob, TextFormat}; use egui::{Color32, Galley, Stroke}; -use quicksearch_core::query::Op; use quicksearch_core::query::lexer::{tokenize_spanned, Token}; use quicksearch_core::query::pattern::RegexQuery; use quicksearch_core::query::translator::{build_filter, is_filter_key}; +use quicksearch_core::query::Op; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Class { diff --git a/crates/quicksearch-gui/src/search_tab/tests.rs b/crates/quicksearch-gui/src/search_tab/tests.rs index bb6f6a2..4c77029 100644 --- a/crates/quicksearch-gui/src/search_tab/tests.rs +++ b/crates/quicksearch-gui/src/search_tab/tests.rs @@ -69,7 +69,10 @@ fn render_perf() { for (label, mut tab) in [ ("empty (floor)", new_tab()), ("1000 rows, name only", tab_with_results(1000)), - ("1000 rows, content snippets", tab_with_content_snippets(1000)), + ( + "1000 rows, content snippets", + tab_with_content_snippets(1000), + ), ] { for _ in 0..10 { timed_frame(&ctx, &mut tab);