jeremy/quick_search
jeremy/quick_search
1
0
Fork 0
Code Issues Pull requests Projects Releases 12 Packages Wiki Activity Actions

Increase search db connection's timeout to 30m. Remove the indexer's duplicate-visit set, ensuring we index correctly without it.

Browse source
This commit is contained in:
Jeremy Karst 2026-08-23 20:01:16 -04:00
parent 3cb70a184f
commit 42f26b1e72
32 changed files with 1010 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
crates/quicksearch-core/Cargo.toml
View file

@ -11,6 +11,14 @@ description = "Indexing, storage and search engine behind QuickSearch."
name = "quicksearch_core"
path = "src/lib.rs"
# Off by default and never enabled by a shipped build. `probe` turns on the
# indexing run's structure census (`indexing::pipeline`), which attributes
# peak memory to a *structure*; `smaps` can only attribute it to a mapping,
# and every one of these lives in the same anonymous heap. Built for the
# `memprobe` example, which reads the census off stderr.
[features]
probe = []
# Several dependencies below are named directly despite already being in the
# lockfile transitively ("transitive already"): they compile nothing new.
[dependencies]

20
crates/quicksearch-core/benches/index.rs
View file

@ -10,7 +10,7 @@
mod corpus;
use divan::Bencher;
use quicksearch_core::{mime, textenc, walk};
use quicksearch_core::{mime, textenc};
fn main() {
divan::main();
@ -136,21 +136,3 @@ mod mime_sniff {
}
}
/// SHA-256 over the path string, per file, every run. Reference only: it
/// confirms the cost is small enough that the collision-resistance argument
/// stands.
mod path_digest {
use super::*;
#[divan::bench]
fn per_path(bencher: Bencher) {
let rows = corpus::rows();
bencher.bench(|| {
let mut acc = 0u128;
for row in divan::black_box(rows) {
acc ^= walk::path_digest(&row.path);
}
acc
});
}
}

290
crates/quicksearch-core/examples/extractprobe.rs Normal file
View file

@ -0,0 +1,290 @@
//! What one content extraction *costs in memory*, and what a pool of them
//! costs together.
//!
//! [`memprobe`](memprobe.rs) measures a whole run, where extraction is mixed
//! in with the walk, the writer and SQLite. This isolates the extractors:
//! the same [`decide_content`] the content pass calls, over a real corpus,
//! with a pool the size of a real one.
//!
//! ```text
//! cargo build -p quicksearch-core --example extractprobe --release
//! ./target/release/examples/extractprobe ~/Documents # 1 worker: per-file cost
//! ./target/release/examples/extractprobe ~/Documents 16 # a network root's pool
//! ./target/release/examples/extractprobe ~/Documents 16 4 # four such roots
//! ```
//!
//! The question it answers: **does peak memory track the number of files or
//! the number of workers?** A run's file count is the user's; its worker
//! count is ours (`walk::thread_count_for`, one pool per root), so if the
//! peak follows the pool, the fix is a bound and not a rewrite.
//!
//! `peak live` is from a counting allocator and is exact; `VmHWM` is what the
//! machine sees, and the gap between them is glibc holding freed chunks on
//! its arena free lists. With one worker the per-file table is exact — the
//! largest entries are the files that would spike a real run.
use std::alloc::{GlobalAlloc, Layout, System};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Instant;
mod common;
// ---------------------------------------------------------------------------
// Allocation accounting
// ---------------------------------------------------------------------------
/// `System`, counting — per binary, so the shipped `quicksearch` is
/// untouched. `PEAK_LIVE` is the high-water of live bytes: unlike RSS it
/// cannot be inflated by the allocator declining to return pages.
struct Counting;
static LIVE: AtomicU64 = AtomicU64::new(0);
static PEAK_LIVE: AtomicU64 = AtomicU64::new(0);
/// High-water since the last [`take_mark`]; the per-file column.
static MARK: AtomicU64 = AtomicU64::new(0);
#[inline]
fn note(live: u64) {
PEAK_LIVE.fetch_max(live, Ordering::Relaxed);
MARK.fetch_max(live, Ordering::Relaxed);
}
unsafe impl GlobalAlloc for Counting {
unsafe fn alloc(&self, l: Layout) -> *mut u8 {
let p = unsafe { System.alloc(l) };
if !p.is_null() {
note(LIVE.fetch_add(l.size() as u64, Ordering::Relaxed) + l.size() as u64);
}
p
}
unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 {
let p = unsafe { System.alloc_zeroed(l) };
if !p.is_null() {
note(LIVE.fetch_add(l.size() as u64, Ordering::Relaxed) + l.size() as u64);
}
p
}
unsafe fn dealloc(&self, p: *mut u8, l: Layout) {
LIVE.fetch_sub(l.size() as u64, Ordering::Relaxed);
unsafe { System.dealloc(p, l) }
}
unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 {
let q = unsafe { System.realloc(p, l, new) };
if !q.is_null() {
let (old, new) = (l.size() as u64, new as u64);
let live = if new >= old {
LIVE.fetch_add(new - old, Ordering::Relaxed) + (new - old)
} else {
LIVE.fetch_sub(old - new, Ordering::Relaxed) - (old - new)
};
note(live);
}
q
}
}
#[global_allocator]
static ALLOCATOR: Counting = Counting;
/// The high-water since the last call, rebased to live-now. Meaningful only
/// while one thread is allocating — hence the per-file table's `workers == 1`
/// guard.
fn take_mark() -> u64 {
let live = LIVE.load(Ordering::Relaxed);
MARK.swap(live, Ordering::Relaxed).saturating_sub(live)
}
use quicksearch_core::config::Config;
use quicksearch_core::extract::Registry;
use quicksearch_core::file_handling::decide_content;
use quicksearch_core::testutil::mib;
/// Head bytes read for the MIME sniff — the same window the walk uses, so
/// this probe classifies files exactly as a run would.
fn sniff(path: &Path, hash_length: usize) -> Option<String> {
use std::io::Read;
let mut f = std::fs::File::open(path).ok()?;
let mut head = vec![0u8; hash_length];
let n = f.read(&mut head).ok()?;
head.truncate(n);
quicksearch_core::mime::guess_mime_from_head(path, &head)
}
struct Candidate {
path: String,
mime: String,
size: u64,
}
/// Every file under `dir` an extractor claims and a run would not reject as
/// oversize: exactly the set the content pass would be handed.
fn candidates(dir: &Path, config: &Config, registry: &Registry) -> Vec<Candidate> {
let mut out = Vec::new();
for entry in walkdir::WalkDir::new(dir)
.into_iter()
.filter_map(Result::ok)
.filter(|e| e.file_type().is_file())
{
let Ok(meta) = entry.metadata() else { continue };
if meta.len() > config.processing.maximum_text_file_size {
continue;
}
let Some(path) = entry.path().to_str().map(str::to_string) else {
continue;
};
let Some(mime) = sniff(entry.path(), config.processing.hash_length) else {
continue;
};
if !registry.supports(&mime) {
continue;
}
out.push(Candidate {
path,
mime,
size: meta.len(),
});
}
out
}
fn main() {
let mut args = std::env::args().skip(1);
let Some(dir) = args.next().map(PathBuf::from) else {
eprintln!("usage: extractprobe <dir> [workers] [replicas]");
std::process::exit(2);
};
let workers: usize = args
.next()
.map(|v| v.parse().expect("workers must be a number"))
.unwrap_or(1)
.max(1);
let replicas: usize = args
.next()
.map(|v| v.parse().expect("replicas must be a number"))
.unwrap_or(1)
.max(1);
let config = Config::default();
let registry = Arc::new(Registry::default_set());
let found = candidates(&dir, &config, &registry);
if found.is_empty() {
eprintln!(
"extractprobe: nothing under {} is extractable",
dir.display()
);
std::process::exit(1);
}
eprintln!(
"extractprobe {}: {} extractable file(s), {} workers, {} replica(s)",
dir.display(),
found.len(),
workers,
replicas
);
// Baseline *after* the scan: the candidate list is the probe's own cost,
// not extraction's.
let baseline_rss = rss();
let baseline_live = LIVE.load(Ordering::Relaxed);
PEAK_LIVE.store(baseline_live, Ordering::Relaxed);
// Replicas share one queue, so every worker stays busy to the end
// instead of the pool draining down to one straggler.
let queue: Vec<&Candidate> = (0..replicas).flat_map(|_| found.iter()).collect();
let next = AtomicUsize::new(0);
let worst: Mutex<Vec<(u64, String, u64, String)>> = Mutex::new(Vec::new());
let per_file = workers == 1;
let start = Instant::now();
std::thread::scope(|s| {
for _ in 0..workers {
let (queue, next, worst) = (&queue, &next, &worst);
let (registry, config) = (registry.clone(), config.clone());
s.spawn(move || loop {
let i = next.fetch_add(1, Ordering::Relaxed);
let Some(c) = queue.get(i) else { return };
if per_file {
take_mark();
}
let outcome = decide_content(&c.path, Some(&c.mime), &registry, &config);
if per_file {
let cost = take_mark();
let text =
quicksearch_core::file_handling::outcome_body(&outcome).map_or(0, str::len);
let mut w = worst
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
w.push((cost, c.path.clone(), c.size, c.mime.clone()));
// Kept small so the table itself is not the peak.
w.sort_by_key(|(cost, ..)| std::cmp::Reverse(*cost));
w.truncate(12);
let _ = text;
}
});
}
});
let elapsed = start.elapsed();
let peak_live = PEAK_LIVE
.load(Ordering::Relaxed)
.saturating_sub(baseline_live);
let hwm = common::vm_hwm().unwrap_or(0);
eprintln!(
"\n {} file(s) in {:.1}s ({:.0}/s)",
queue.len(),
elapsed.as_secs_f64(),
queue.len() as f64 / elapsed.as_secs_f64().max(0.001),
);
eprintln!(
" peak live {} above a {} baseline — what extraction really holds",
mib(peak_live),
mib(baseline_live),
);
eprintln!(
" peak RSS {} (VmHWM), now {} — the gap is glibc holding freed chunks",
mib(hwm),
mib(rss()),
);
eprintln!(
" baseline {} RSS before the first extraction",
mib(baseline_rss)
);
eprintln!(
" per worker {} of peak live, at {} workers",
mib(peak_live / workers as u64),
workers,
);
let w = worst
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if !w.is_empty() {
eprintln!("\n most expensive files (live bytes held while extracting):");
for (cost, path, size, mime) in w.iter() {
eprintln!(
" {:>10} from a {:>9} {} {}",
mib(*cost),
mib(*size),
mime,
path
);
}
eprintln!(
"\n A pool of N runs N of these at once. `maximum_text_file_size` bounds\n \
the input ({}), not the working set above.",
mib(config.processing.maximum_text_file_size),
);
}
}
/// Resident set size now, from `/proc/self/statm` field 2 (resident pages).
fn rss() -> u64 {
std::fs::read_to_string("/proc/self/statm")
.ok()
.and_then(|s| s.split_whitespace().nth(1)?.parse::<u64>().ok())
.map(|pages| pages * 4096)
.unwrap_or(0)
}

1
crates/quicksearch-core/examples/hashprobe.rs
View file

@ -835,5 +835,4 @@ mod tests {
assert_eq!(stat.reported_pairs, 10_000 * 9_999 / 2);
assert_eq!(stat.fp_pairs, 0);
}
}

57
crates/quicksearch-core/examples/indexprobe.rs
View file

@ -23,6 +23,11 @@
//!
//! `QSB_HASH_LENGTH` overrides `[processing] hash_length` for a run — how
//! [`hashprobe`](hashprobe.rs) gets its end-to-end column.
//!
//! `QSB_KEY=<64 hex digits>` measures an encrypted index. It is not optional
//! dressing: without a key installed this probe polls the completion marker
//! through a plain open, which an encrypted index cannot answer, so the run
//! reports a three-hour hang instead of its actual time.
mod common;
@ -340,17 +345,51 @@ fn hash_length_override() -> Option<usize> {
.and_then(|v| v.parse().ok())
}
/// `QSB_KEY=<64 hex digits>` measures an *encrypted* index: the key is
/// installed process-wide before any connection exists, exactly as the GUI
/// does after an unlock. Raw hex rather than a password, so no Argon2id
/// derivation lands inside a timed run.
fn install_key() -> bool {
match std::env::var("QSB_KEY") {
Ok(hex) => {
let key = quicksearch_core::security::IndexKey::from_hex(hex.trim())
.expect("QSB_KEY must be 64 hex digits");
quicksearch_core::db::set_process_key(Some(key));
true
}
Err(_) => false,
}
}
/// Open the index the way this run's key demands.
///
/// **A plain open cannot read an encrypted index**, and
/// [`get_last_full_index`](quicksearch_core::db::repo::get_last_full_index)
/// reports that failure as `None` — indistinguishable from "not finished
/// yet". Polling an encrypted run through a plain open therefore never
/// observes its own completion and sits here until the deadline, which reads
/// as an indexing slowdown of several orders of magnitude rather than as the
/// probe defect it is.
fn probe_open(db: &Path, keyed: bool) -> Option<rusqlite::Connection> {
if keyed {
quicksearch_core::db::open_existing(&db.to_string_lossy(), false).ok()
} else {
rusqlite::Connection::open(db).ok()
}
}
fn run(mode: &str, tree: &Path, db: &Path) {
let mut config = Config::default();
if let Some(n) = hash_length_override() {
config.processing.hash_length = n;
}
let hash_length = config.processing.hash_length;
let keyed = install_key();
// The marker is the one unambiguous completion signal; polling the
// status enum races on a small tree.
if db.exists() {
let conn = rusqlite::Connection::open(db).expect("open db");
let conn = probe_open(db, keyed).expect("open db");
conn.execute("DELETE FROM schema_info WHERE key = 'last_full_index'", [])
.expect("clear marker");
}
@ -374,14 +413,17 @@ fn run(mode: &str, tree: &Path, db: &Path) {
)
.expect("start indexing");
let deadline = Instant::now() + Duration::from_secs(600);
// Generous, like `memprobe`'s: the scale sweep this probe exists for runs
// hundreds of thousands of files, and a run that times out reports
// nothing. Past this is a hang, not a slow disk.
let deadline = Instant::now() + Duration::from_secs(3 * 3600);
let mut done = false;
while Instant::now() < deadline {
if let IndexingStatus::Error(e) = service.get_status() {
panic!("indexing failed: {}", e);
}
if db.exists() {
if let Ok(conn) = rusqlite::Connection::open(db) {
if let Some(conn) = probe_open(db, keyed) {
if quicksearch_core::db::repo::get_last_full_index(&conn).is_some() {
done = true;
break;
@ -391,7 +433,11 @@ fn run(mode: &str, tree: &Path, db: &Path) {
std::thread::sleep(Duration::from_millis(5));
}
let elapsed = start.elapsed();
assert!(done, "indexing did not finish within the timeout");
assert!(
done,
"indexing did not finish within the timeout (keyed = {})",
keyed
);
// The run's last checkpoint happens inside here, so the sampler outlives it.
service.stop_indexing().expect("stop");
wal_stop.store(true, Ordering::Relaxed);
@ -399,8 +445,7 @@ fn run(mode: &str, tree: &Path, db: &Path) {
// Count what was actually indexed rather than assuming `gen`'s tree —
// pointing the probe elsewhere made the rate a fiction.
let total = rusqlite::Connection::open(db)
.ok()
let total = probe_open(db, keyed)
.and_then(|c| quicksearch_core::db::repo::row_count(&c).ok())
.unwrap_or(0);

42
crates/quicksearch-core/examples/memprobe.rs
View file

@ -11,6 +11,13 @@
//! ./target/release/examples/memprobe cold ~ /var/tmp/qs-mem/index.db 250 probe.toml
//! ```
//!
//! **Roots are comma-separated**, because per-root state is what multiplies:
//! a one-root run cannot show the buffers that exist once per pipeline.
//!
//! ```text
//! ./target/release/examples/memprobe cold /media/shared,/home/me,/usr /var/tmp/qs-mem/index.db
//! ```
//!
//! Trailing arguments: sampling interval in ms (default 100) and a config
//! file. `cold` deletes the database first; `warm` re-runs against the
//! finished one. Reading the report: **VmHWM** cannot miss a spike — quote
@ -18,6 +25,10 @@
//! a per-file cost. Nothing reported is evictable page cache. `settled RSS`
//! is what a long-lived process keeps — glibc frees to its arena, so
//! without `release_free_heap` the peak becomes the floor.
//!
//! Built with `--features probe` the indexer also prints a `census` line
//! naming what its run-scoped structures hold, which is the half `smaps`
//! cannot answer: a mapping is "heap", never "the stale-candidate list".
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
@ -66,12 +77,22 @@ struct Sample {
fn main() {
let mut args = std::env::args().skip(1);
let mode = args.next().unwrap_or_default();
let (Some(root), Some(db)) = (args.next(), args.next()) else {
eprintln!("usage: memprobe <cold|warm> <root> <db> [sample_ms] [config.toml]");
let (Some(roots), Some(db)) = (args.next(), args.next()) else {
eprintln!("usage: memprobe <cold|warm> <root[,root...]> <db> [sample_ms] [config.toml]");
std::process::exit(2);
};
if mode != "cold" && mode != "warm" {
eprintln!("usage: memprobe <cold|warm> <root> <db> [sample_ms] [config.toml]");
eprintln!("usage: memprobe <cold|warm> <root[,root...]> <db> [sample_ms] [config.toml]");
std::process::exit(2);
}
let roots: Vec<String> = roots
.split(',')
.map(str::trim)
.filter(|r| !r.is_empty())
.map(str::to_string)
.collect();
if roots.is_empty() {
eprintln!("memprobe: no roots given");
std::process::exit(2);
}
let interval = Duration::from_millis(
@ -92,10 +113,10 @@ fn main() {
}
}
run(&mode, &root, &db, interval, config_path.as_deref());
run(&mode, &roots, &db, interval, config_path.as_deref());
}
fn run(mode: &str, root: &str, db: &Path, interval: Duration, config_path: Option<&Path>) {
fn run(mode: &str, roots: &[String], db: &Path, interval: Duration, config_path: Option<&Path>) {
// A config file only supplies the knobs that change *what* indexing
// does; the defaults keep runs comparable.
let config = match config_path {
@ -112,9 +133,10 @@ fn run(mode: &str, root: &str, db: &Path, interval: Duration, config_path: Optio
let baseline = rss().expect("read /proc/self/statm");
eprintln!(
"memprobe {}: root={} db={}\n baseline RSS {} (process before indexing starts)",
"memprobe {}: {} root(s)={} db={}\n baseline RSS {} (process before indexing starts)",
mode,
root,
roots.len(),
roots.join(" "),
db.display(),
mib(baseline)
);
@ -122,11 +144,7 @@ fn run(mode: &str, root: &str, db: &Path, interval: Duration, config_path: Optio
let service = IndexingService::new();
let start = Instant::now();
service
.start_indexing(
vec![root.to_string()],
db.to_string_lossy().into_owned(),
config,
)
.start_indexing(roots.to_vec(), db.to_string_lossy().into_owned(), config)
.expect("start indexing");
let mut samples: Vec<Sample> = Vec::new();

8
crates/quicksearch-core/src/coordinator.rs
View file

@ -59,9 +59,13 @@ pub enum WatcherStatus {
/// Registration in flight — it walks every root, so this can last
/// minutes on large or networked trees.
Starting,
Active { dirs: usize },
Active {
dirs: usize,
},
/// Live updates unavailable; the periodic reindex is the only refresh.
Disabled { reason: WatchError },
Disabled {
reason: WatchError,
},
}
/// A config reconciliation the coordinator applies between runs.

12
crates/quicksearch-core/src/coordinator/inner.rs
View file

@ -442,7 +442,11 @@ impl Inner {
fn apply_queue(&mut self, conn: &mut Connection, targeted: bool) {
let deadline = Instant::now() + APPLY_BUDGET;
let chunk = self.config.processing.batch_size.max(1);
let queue = if targeted { &self.targeted } else { &self.pending };
let queue = if targeted {
&self.targeted
} else {
&self.pending
};
let removals: Vec<PathBuf> = queue
.iter()
@ -474,7 +478,11 @@ impl Inner {
}
if Instant::now() < deadline {
let queue = if targeted { &self.targeted } else { &self.pending };
let queue = if targeted {
&self.targeted
} else {
&self.pending
};
let upserts: Vec<PathBuf> = queue
.iter()
.filter(|(_, ev)| !is_removal(ev))

33
crates/quicksearch-core/src/db/repo_tests.rs
View file

@ -1182,9 +1182,8 @@ fn deleting_a_file_row_cascades_the_fk_tables() {
let (_dir, path) = tmp_path();
let mut conn = open_or_recreate(path.to_str().unwrap(), "trigram").unwrap();
let ids = seeded(&mut conn, &["/casc/a.txt", "/casc/b.txt"]);
let count = |conn: &Connection, sql: &str| -> i64 {
conn.query_row(sql, [], |r| r.get(0)).unwrap()
};
let count =
|conn: &Connection, sql: &str| -> i64 { conn.query_row(sql, [], |r| r.get(0)).unwrap() };
{
let tx = conn.transaction().unwrap();
set_content_failed(&tx, ids["/casc/b.txt"], "boom").unwrap();
@ -1229,15 +1228,17 @@ fn retry_failed_files_resets_state_and_clears_records() {
tx.commit().unwrap();
let state = |id: i64| -> i64 {
conn.query_row(
"SELECT content_state FROM files WHERE id = ?1",
[id],
|r| r.get(0),
)
conn.query_row("SELECT content_state FROM files WHERE id = ?1", [id], |r| {
r.get(0)
})
.unwrap()
};
assert_eq!(state(ids["/retry/bad.txt"]), STATE_PENDING);
assert_eq!(state(ids["/retry/good.txt"]), STATE_DONE, "done rows untouched");
assert_eq!(
state(ids["/retry/good.txt"]),
STATE_DONE,
"done rows untouched"
);
let failures: i64 = conn
.query_row("SELECT COUNT(*) FROM failed_files", [], |r| r.get(0))
.unwrap();
@ -1267,7 +1268,11 @@ fn a_changed_file_clears_its_failure_record() {
parent: "/chg/",
size: 2,
mtime: 9,
mime: if needs_content { Some("text/plain") } else { None },
mime: if needs_content {
Some("text/plain")
} else {
None
},
ftype: FileType::TEXT,
hash: None,
needs_content,
@ -1284,11 +1289,9 @@ fn a_changed_file_clears_its_failure_record() {
let b = update(&mut conn, "b.txt", true);
let state = |id: i64| -> i64 {
conn.query_row(
"SELECT content_state FROM files WHERE id = ?1",
[id],
|r| r.get(0),
)
conn.query_row("SELECT content_state FROM files WHERE id = ?1", [id], |r| {
r.get(0)
})
.unwrap()
};
assert_eq!(state(a), STATE_NA);

6
crates/quicksearch-core/src/extract/mod.rs
View file

@ -88,11 +88,7 @@ impl Registry {
/// caller cannot forget. This cannot help with a stack overflow, which
/// aborts rather than unwinding — see `vendor/pdf-extract`, which bounds
/// the recursion that made that reachable.
pub fn extract(
&self,
path: &Path,
mime: &str,
) -> Result<Option<String>, ExtractError> {
pub fn extract(&self, path: &Path, mime: &str) -> Result<Option<String>, ExtractError> {
let Some(extractor) = self.find(mime) else {
return Ok(None);
};

6
crates/quicksearch-core/src/extract/office.rs
View file

@ -461,11 +461,7 @@ mod tests {
"angle-bracket entities lost: {:?}",
out
);
assert!(
out.contains('\u{2019}'),
"numeric entities lost: {:?}",
out
);
assert!(out.contains('\u{2019}'), "numeric entities lost: {:?}", out);
assert!(
!out.contains("&amp;") && !out.contains("&#"),
"entities left unresolved: {:?}",

6
crates/quicksearch-core/src/extract/plaintext.rs
View file

@ -94,11 +94,7 @@ impl Extractor for PlaintextExtractor {
decode(read_sized(&mut f, size, MAX_READ, path)?, path)
}
fn extract_from_head(
&self,
path: &Path,
head: &[u8],
) -> Option<Result<String, ExtractError>> {
fn extract_from_head(&self, path: &Path, head: &[u8]) -> Option<Result<String, ExtractError>> {
Some(decode(head.to_vec(), path))
}
}

6
crates/quicksearch-core/src/extract/rtf.rs
View file

@ -46,11 +46,7 @@ impl Extractor for RtfExtractor {
}
/// RTF has no trailer and needs no seeking; a complete head parses like disk.
fn extract_from_head(
&self,
path: &Path,
head: &[u8],
) -> Option<Result<String, ExtractError>> {
fn extract_from_head(&self, path: &Path, head: &[u8]) -> Option<Result<String, ExtractError>> {
Some(parse(head.to_vec(), path))
}
}

4
crates/quicksearch-core/src/file_handling/records.rs
View file

@ -314,7 +314,9 @@ pub fn extract_and_store(
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ContentOutcome {
/// Text, already truncated to `maximum_text_size`.
Done { text: String },
Done {
text: String,
},
NotApplicable,
Failed(String),
}

4
crates/quicksearch-core/src/incremental.rs
View file

@ -42,7 +42,9 @@ pub enum Applied {
Done,
/// Budget spent. What was written is committed; the caller should re-queue
/// the same event with `resume_from` set to `done`.
Unfinished { done: usize },
Unfinished {
done: usize,
},
}
/// Apply one filesystem event to the index. Missing files are treated as

104
crates/quicksearch-core/src/indexing/pipeline.rs
View file

@ -166,6 +166,91 @@ fn per_second(n: usize, elapsed: Duration) -> Option<f64> {
/// producer wakes it at once.
const IDLE_BACKOFF: Duration = Duration::from_millis(2);
/// What each run-scoped structure holds, for the `probe` builds only.
///
/// Peak RSS is one anonymous heap: `smaps` can say "the heap grew", never
/// "the stale-candidate list grew". This says which. Sizes are estimates of
/// the *heap* each structure owns — the point is which one dominates and
/// whether it tracks the tree, not a byte-exact total. See `examples/memprobe.rs`.
#[cfg(feature = "probe")]
mod census {
use super::{RootPipeline, RunCx};
use crate::testutil::mib;
use std::time::{Duration, Instant};
/// Rare next to a status publish: each line walks every string in the
/// run-scoped sets, which is O(tree) work that must not shape what it
/// measures.
const INTERVAL: Duration = Duration::from_secs(2);
/// Owned string bytes plus the `String` headers a collection holds.
fn strings_bytes<'a>(it: impl Iterator<Item = &'a String>, len: usize) -> u64 {
let body: usize = it.map(String::len).sum();
body as u64 + (len * std::mem::size_of::<String>()) as u64
}
pub(super) fn due(last: &mut Instant) -> bool {
if last.elapsed() < INTERVAL {
return false;
}
*last = Instant::now();
true
}
/// One line per structure group: the log collapses embedded newlines, so
/// a multi-line report would arrive as one unreadable line.
pub(super) fn report(cx: &RunCx<'_>, pipelines: &[RootPipeline], started: Instant) {
let at = started.elapsed().as_secs_f64();
crate::log_info!(
"census t={:.1}s stale {} ({}) aliased {} ({})",
at,
cx.stale_candidates.len(),
mib(strings_bytes(
cx.stale_candidates.iter(),
cx.stale_candidates.len()
)),
cx.aliased_paths.len(),
mib(strings_bytes(
cx.aliased_paths.iter(),
cx.aliased_paths.len()
)),
);
for p in pipelines {
let (dirs, dir_bytes) = p.walk.seen_dirs_footprint();
let inline = |rows: &[crate::file_handling::OwnedNewFile]| -> u64 {
rows.iter()
.map(|r| {
(r.inline_text.as_ref().map_or(0, String::len)
+ r.name.len()
+ r.parent.len()) as u64
})
.sum()
};
let ready: u64 = p
.ready
.iter()
.map(|r| {
(crate::file_handling::outcome_body(&r.outcome).map_or(0, str::len)
+ r.name.len()) as u64
})
.sum();
crate::log_info!(
"census t={:.1}s {} [{:?}] dirs {} ({}) pending {}+{} ({}) ready {} ({})",
at,
p.root,
p.phase,
dirs,
mib(dir_bytes),
p.pending_inserts.len(),
p.pending_updates.len(),
mib(inline(&p.pending_inserts) + inline(&p.pending_updates)),
p.ready.len(),
mib(ready),
);
}
}
}
fn report_run_warnings() {
let (failed, suppressed) = crate::file_handling::hash_failure_counts();
if failed > 0 {
@ -279,9 +364,6 @@ impl RootPipeline {
// parent's absence as proof the file is gone.
cx.aliased_paths.insert(file.path.clone());
}
if !cx.seen_paths.insert(file.digest) {
continue;
}
let Some(rec) = file.record else { continue };
if file.action == FileIndexAction::Update {
self.pending_updates.push(rec);
@ -473,9 +555,6 @@ pub(super) struct RunCx<'a> {
/// Writer time one root's turn may take before the round moves on
/// ([`crate::config::ProcessingConfig::writer_turn_slice_ms`]).
pub(super) slice: Duration,
/// 128-bit path digests, not paths: measured, owning every path string
/// again was the single largest allocation in a run.
pub(super) seen_paths: HashSet<u128>,
/// Rows with no file behind them, per-directory plus the vanished sweep.
pub(super) stale_candidates: Vec<String>,
/// Paths reached via symlink; their rows may live outside every root.
@ -498,7 +577,6 @@ impl<'a> RunCx<'a> {
registry: Arc::new(Registry::default_set()),
quantum: config.processing.batch_size.max(1),
slice: Duration::from_millis(config.processing.writer_turn_slice_ms),
seen_paths: HashSet::new(),
stale_candidates: Vec::new(),
aliased_paths: HashSet::new(),
stale_cleanup_ok: true,
@ -673,8 +751,9 @@ impl IndexingService {
crate::walk::reset_run_warnings();
crate::file_handling::reset_run_warnings();
// Canonicalized first so spelling variants collapse to one walk;
// nested-root dedup is the `seen_paths` set's job.
// Canonicalized first so spelling variants collapse to one walk.
// Nested roots need no handling here: the coordinator refuses to
// start a run with any (`config::nested_roots`).
let mut seen_roots = HashSet::new();
let roots: Vec<String> = paths
.iter()
@ -763,6 +842,8 @@ impl IndexingService {
let aborted;
let mut cleanup_done = false;
let mut rr = 0usize;
#[cfg(feature = "probe")]
let mut last_census = Instant::now();
let wal_path = format!("{}-wal", db_path);
let configured_cap = match config.processing.maximum_wal_size {
0 => 0,
@ -812,6 +893,11 @@ impl IndexingService {
publish_status(status, run_start, &pipelines);
#[cfg(feature = "probe")]
if census::due(&mut last_census) {
census::report(&cx, &pipelines, run_started);
}
// After `publish_status`: the checkpoint may block for
// `busy_timeout`, and it must not sit in front of `stop_indexing`
// — hence the stop-flag check.

61
crates/quicksearch-core/src/indexing/tests.rs
View file

@ -494,11 +494,41 @@ fn the_walk_denominator_prefers_the_best_available_count() {
for (label, phase, walked, estimate, want) in [
("estimate", RootPhase::Walking, 100, Some(1000), Some(1000)),
("no count yet", RootPhase::Walking, 100, None, None),
("overtaken", RootPhase::Walking, 1500, Some(1000), Some(1500)),
("exact", RootPhase::Extracting, 261_088, Some(6_677_062), Some(261_088)),
("exact, no estimate", RootPhase::Extracting, 261_088, None, Some(261_088)),
("done", RootPhase::Done, 261_088, Some(6_677_062), Some(261_088)),
("done, no estimate", RootPhase::Done, 261_088, None, Some(261_088)),
(
"overtaken",
RootPhase::Walking,
1500,
Some(1000),
Some(1500),
),
(
"exact",
RootPhase::Extracting,
261_088,
Some(6_677_062),
Some(261_088),
),
(
"exact, no estimate",
RootPhase::Extracting,
261_088,
None,
Some(261_088),
),
(
"done",
RootPhase::Done,
261_088,
Some(6_677_062),
Some(261_088),
),
(
"done, no estimate",
RootPhase::Done,
261_088,
None,
Some(261_088),
),
] {
assert_eq!(
progress(phase, walked, estimate).walk_denominator(),
@ -612,13 +642,28 @@ fn the_wal_cap_is_bounded_by_the_volume() {
for (label, cfg, free, want) in [
("roomy", configured, 500 * 1024 * 1024 * 1024, configured),
// Exactly enough: floor plus four times the log.
("just enough", configured, 128 * 1024 * 1024 + configured * 4, configured),
(
"just enough",
configured,
128 * 1024 * 1024 + configured * 4,
configured,
),
// 1 GiB free: 896 MiB above the floor, a quarter of which is 224 MiB.
("tight", configured, 1024 * 1024 * 1024, 224 * 1024 * 1024),
("disabled, tight", 0, 1024 * 1024 * 1024, 224 * 1024 * 1024),
("full", configured, 0, crate::config::MINIMUM_WAL_SIZE),
("nearly full", configured, 1024, crate::config::MINIMUM_WAL_SIZE),
("at the floor", configured, 127 * 1024 * 1024, crate::config::MINIMUM_WAL_SIZE),
(
"nearly full",
configured,
1024,
crate::config::MINIMUM_WAL_SIZE,
),
(
"at the floor",
configured,
127 * 1024 * 1024,
crate::config::MINIMUM_WAL_SIZE,
),
] {
assert_eq!(wal_cap_for_free(cfg, free), want, "{label}");
}

2
crates/quicksearch-core/src/query/split.rs
View file

@ -8,10 +8,10 @@
//! unrecognized `key:value` (`12:30`) reassembles verbatim; `AND`/`OR` pass
//! through as words, parens are dropped.
use super::Op;
use super::lexer::{tokenize, Token};
use super::pattern::{RegexQuery, TermPart, TermPattern};
use super::translator::{build_filter, is_filter_key, TranslateError};
use super::Op;
/// The cascade's parsed input: one term string plus composable filter SQL.
#[derive(Debug, Clone, Default)]

33
crates/quicksearch-core/src/search/cascade/passes.rs
View file

@ -283,7 +283,16 @@ impl<'a> Cx<'a> {
// A name/path hit's snippet is the field itself, span marked.
let snip =
snippet::whole_field(if is_path_tier { path } else { name }, match_range);
row_hit(row, file_id, path, name, rank, rank as u8, Some(snip), is_path_tier)
row_hit(
row,
file_id,
path,
name,
rank,
rank as u8,
Some(snip),
is_path_tier,
)
},
)
}
@ -535,7 +544,16 @@ impl<'a> Cx<'a> {
};
let snip =
snippet::whole_field(if is_path_tier { path } else { name }, match_range);
row_hit(row, file_id, path, name, rank, rank as u8, Some(snip), is_path_tier)
row_hit(
row,
file_id,
path,
name,
rank,
rank as u8,
Some(snip),
is_path_tier,
)
},
)
}
@ -575,7 +593,16 @@ impl<'a> Cx<'a> {
let r = clamp_match_range(text, r, SNIPPET_WINDOW_CHARS);
snippet::window_around(text, (r.start, r.end), &snippet_opts)
});
row_hit(row, file_id, path, name, 6.0 + count_frac(count), 6, snip, false)
row_hit(
row,
file_id,
path,
name,
6.0 + count_frac(count),
6,
snip,
false,
)
})
}
}

21
crates/quicksearch-core/src/search/mod.rs
View file

@ -25,9 +25,24 @@ use crate::snippet::Snippet;
pub use cascade::Outcome;
pub use duplicates::{find_duplicate_groups, DuplicateGroup};
/// Idle window before the connection is released: an open reader stops
/// SQLite resetting the WAL and pins a deleted index's blocks.
const IDLE_RELEASE: Duration = Duration::from_secs(30);
/// Idle window before the connection is released.
///
/// Long, because releasing throws away [`PRAGMAS_SEARCH`]'s 32 MiB of
/// decrypted pages and the next keystroke pays to refill them: on a 200k-file
/// index the first query after a release costs 129 ms unencrypted and 192 ms
/// encrypted, against ~10 ms warm. Encryption is why the gap widens — a
/// refill is an AES decrypt plus an HMAC verify per page rather than a
/// `memcpy`. What the release buys back is the ~42 MiB the trim in
/// [`Worker::run`] returns, so this trades an idle process floor against
/// stalling the one keystroke a user is most likely to notice.
///
/// It is *not* what lets a rebuild delete the file: `Backend::rebuild_index`
/// and `clear_index` send [`WorkerMsg::ReleaseConnection`] first, because
/// Windows fails the delete while any handle is open. Nor does holding a
/// connection stop the WAL truncating — a read mark lives for the length of a
/// statement, not of the connection (`db::repo_tests` pins both halves of
/// that: a checkpoint loses to a reader *mid-query*, not to an idle one).
const IDLE_RELEASE: Duration = Duration::from_secs(30 * 60);
/// One search result. `rank` is the sort key (lower = better): integer part
/// = cascade stage (1–11), fraction = tiebreak. Batches arrive rank-ordered

23
crates/quicksearch-core/src/testutil.rs
View file

@ -247,6 +247,11 @@ pub struct SeedSpec {
/// Document bodies carrying [`BODY_TERM`]; sized by the caller to stay
/// under the display limit.
pub body_term_docs: usize,
/// One row in every `dup_every` repeats its predecessor's content hash,
/// giving [`crate::search::find_duplicate_groups`] real groups to rank.
/// `0` leaves every hash NULL — the shape the search harnesses seed, and
/// the one whose row width their numbers were taken against.
pub dup_every: usize,
}
impl Default for SeedSpec {
@ -261,6 +266,7 @@ impl Default for SeedSpec {
needle_names: 50,
needle_docs: 50,
body_term_docs: 500,
dup_every: 0,
}
}
}
@ -289,6 +295,21 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) {
};
// Stored parents always end in a separator; see `dir_to_db_parent`.
let dir = format!("/seed/{:03}/", i % spec.dirs.max(1));
// Every `dup_every`-th row takes the hash of the one before it, so the
// groups are pairs of equal-sized rows — the shape `find_duplicate_groups`
// prices, since a hash covers the size.
let hash = (spec.dup_every > 0).then(|| {
let group = if i % spec.dup_every == spec.dup_every - 1 {
i.saturating_sub(1)
} else {
i
};
let mut bytes = [0u8; 32];
for (b, slot) in bytes.iter_mut().enumerate() {
*slot = ((group as u64).wrapping_mul(0x9e37_79b9_7f4a_7c15) >> (b % 8 * 8)) as u8;
}
bytes
});
let id = insert_file(
&tx,
&NewFile {
@ -298,7 +319,7 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) {
mtime: 1_700_000_000 + i as u64,
mime: Some("text/plain"),
ftype: FileType::TEXT,
hash: None,
hash: hash.as_ref().map(|h| h.as_slice()),
needs_content: i % spec.content_every.max(1) == 0,
},
)

5
crates/quicksearch-core/src/verify.rs
View file

@ -27,7 +27,10 @@ pub enum MemberVerdict {
DiffersAt(u64),
/// Lengths disagree, so nothing was read. Within a duplicate group this
/// can only mean a stale index — the hash covers the size.
LengthDiffers { len: u64, reference_len: u64 },
LengthDiffers {
len: u64,
reference_len: u64,
},
Unreadable(String),
}

41
crates/quicksearch-core/src/walk.rs
View file

@ -14,8 +14,6 @@ use std::sync::{mpsc, Arc, Condvar, Mutex};
use std::thread::JoinHandle;
use std::time::UNIX_EPOCH;
use sha2::{Digest, Sha256};
use crate::config::{Config, IgnoreSet};
use crate::extract::Registry;
use crate::file_handling::{
@ -47,8 +45,6 @@ pub struct WalkedFile {
pub path: String,
pub action: FileIndexAction,
pub record: Option<OwnedNewFile>,
/// 128-bit truncated SHA-256 of the path, for the duplicate-visit set.
pub digest: u128,
/// True when this file was reached by resolving a symlink. Its row is
/// invisible to its real parent's reconciliation, so the caller must
/// exempt it from the vanished-directory sweep.
@ -57,12 +53,11 @@ pub struct WalkedFile {
impl WalkedFile {
/// Seen, but with nothing to write: the row stays.
fn skipped(path: String, digest: u128, aliased: bool) -> Self {
fn skipped(path: String, aliased: bool) -> Self {
WalkedFile {
path,
action: FileIndexAction::Skip,
record: None,
digest,
aliased,
}
}
@ -344,17 +339,6 @@ enum Known<'a> {
Exact(Option<u64>),
}
/// 128-bit truncated SHA-256 of a path, for the writer's duplicate-visit set.
/// 16 bytes is ~4e-26 collision probability at 7M paths (8 would be ~1e-6),
/// and a collision silently drops a real file. Cryptographic because shared
/// filenames are attacker-supplied: a chosen pair could hide one file.
pub fn path_digest(path: &str) -> u128 {
let digest = Sha256::digest(path.as_bytes());
let mut bytes = [0u8; 16];
bytes.copy_from_slice(&digest[..16]);
u128::from_be_bytes(bytes)
}
/// At most one `stat`, then classify; only files that will be written get
/// opened, and small text files are finished outright. "At most": on Windows
/// [`PendingFile::cached`] may already hold the answer.
@ -362,20 +346,19 @@ fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile {
let PendingFile { path, cached } = file;
// Every route here has already screened the path for UTF-8:
// `path_to_db_string` is lossy, and a lossy string would key another
// file's row and could consume its digest.
// file's row.
debug_assert!(
path.to_str().is_some(),
"an unrepresentable path reached prepare(): {:?}",
path
);
let db_path = path_to_db_string(&path);
let digest = path_digest(&db_path);
let aliased = matches!(known, Known::Exact(_));
let Ok(meta) = crate::platform::metadata_or_stat(&path, cached) else {
// Seen but unreadable: a transient stat failure must not read as
// "deleted".
return WalkedFile::skipped(db_path, digest, aliased);
return WalkedFile::skipped(db_path, aliased);
};
let Some(mtime) = meta
.modified()
@ -383,7 +366,7 @@ fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile {
.and_then(|t| t.duration_since(UNIX_EPOCH).ok())
.map(|d| d.as_secs())
else {
return WalkedFile::skipped(db_path, digest, aliased);
return WalkedFile::skipped(db_path, aliased);
};
let action = match known {
@ -410,7 +393,6 @@ fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile {
path: db_path,
action,
record,
digest,
aliased,
}
}
@ -546,6 +528,21 @@ impl ParallelWalk {
self.shared.stats.clone()
}
/// How many directories the walk has queued and roughly what their paths
/// own on the heap — the set is live for the whole run, so it is one of
/// the few structures that tracks the tree. Locks the queue and walks it,
/// so it belongs to the census and nothing else.
#[cfg(feature = "probe")]
pub fn seen_dirs_footprint(&self) -> (usize, u64) {
let q = crate::lock_ok(&self.shared.queue);
let bytes: u64 = q
.seen_dirs
.iter()
.map(|d| (d.as_os_str().len() + std::mem::size_of::<PathBuf>()) as u64)
.sum();
(q.seen_dirs.len(), bytes)
}
/// Join the workers and report whether every one finished cleanly. A dead
/// worker and a finished one look identical from the receiving end, and
/// treating a panicked walk as complete would hand stale cleanup a

11
crates/quicksearch-core/src/walk/tests.rs
View file

@ -231,9 +231,9 @@ fn a_root_that_resolves_onto_an_unrepresentable_name_is_not_walked() {
);
}
/// The pipeline hashes a `WalkedFile`'s path into `seen_paths` before it looks
/// for a record, so a bad name yielded under its lossy spelling made whichever
/// of the two files the walk reached second fall out of the index.
/// A `WalkedFile`'s path is the key its row is written under, so a bad name
/// yielded under its lossy spelling would claim the twin's row: whichever of
/// the two files the walk reached second fell out of the index.
#[test]
fn a_bad_name_cannot_stand_in_for_its_lossy_twin() {
let root = tmp_tree("nonutf8-twin");
@ -489,8 +489,9 @@ fn symlink_loop_terminates() {
#[test]
#[cfg(unix)]
fn symlinked_file_resolves_to_its_target_path() {
// The walker dedupes *directories*; the caller's `seen_paths` dedupes
// files, so both routes must agree on the canonical path.
// The walker dedupes *directories*, not files: a file reached both ways
// is yielded twice, so both routes must agree on the canonical path or
// the writer would key two different rows for one file.
let root = tmp_tree("symlink-file");
touch(&root.join("real/target.txt"));
fs::create_dir_all(root.join("links")).unwrap();

250
crates/quicksearch-core/tests/encrypted_perf.rs Normal file
View file

@ -0,0 +1,250 @@
//! Encryption must cost a constant factor, not a different algorithm.
//!
//! SQLCipher decrypts and HMAC-verifies every 4 KiB page it reads, so a keyed
//! index is intrinsically slower than a plain one — that part is not a bug and
//! this file does not try to gate it. What it gates is *amplification*: a query
//! whose cost is one page fetch per row is fine unencrypted (the page cache
//! makes it nearly free) and disastrous keyed. `find_duplicate_groups` was
//! exactly that until it was rewritten to stay inside `idx_files_hash`:
//!
//! | shape | plain | encrypted | ratio |
//! |---|---|---|---|
//! | row fetch per file (pre-`8a7810d`) | 0.59 s | 2.28 s | **3.9x** |
//! | covering index scan (current) | 0.34 s | 0.44 s | 1.3x |
//!
//! Measured on 400k rows, so the ceiling below sits between those two: the old
//! shape fails it, the current one passes with room. The ratio is what makes
//! this a *test* rather than a benchmark — both arms run the same workload on
//! the same machine in the same process, so host speed, CPU governor and CI
//! contention divide out. Absolute times are printed but never asserted.
//!
//! Its own integration binary because it installs a process-global key, the
//! same reason `tests/encrypted.rs` gives.
use std::path::PathBuf;
use std::sync::atomic::AtomicU64;
use std::time::{Duration, Instant};
use quicksearch_core::db;
use quicksearch_core::query::split::split_for_cascade;
use quicksearch_core::search::{cascade, find_duplicate_groups, SearchHit, SearchOptions};
use quicksearch_core::security::IndexKey;
use quicksearch_core::testutil::{scratch_db, seed_index, SeedSpec, BODY_TERM, NEEDLE};
/// A raw 32-byte key, not an Argon2id derivation: the KDF costs half a second
/// in release and minutes in debug, and proves nothing about page work. It
/// reaches SQLCipher as raw hex either way (see `db::open::key_and_probe`), so
/// what is measured below is identical to a real unlocked index.
const KEY_HEX: &str = "a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90";
/// Ceiling on encrypted/plain for one workload. Between the 3.9x the old
/// duplicate query cost and the 1.3x the current one costs; see the table
/// above. Raising this without a measurement in the same table defeats it.
const MAX_RATIO: f64 = 3.0;
/// Enough rows that neither index fits in `PRAGMAS_SEARCH`'s 32 MiB page
/// cache — the only regime where a per-page decrypt is visible at all. Below
/// that both arms are served from cache, every ratio is 1.0, and the gate
/// silently stops testing anything. `index_is_larger_than_the_search_cache`
/// pins that this seed still clears it.
const FILES: usize = 60_000;
const CONTENT_EVERY: usize = 5;
/// The cache the search connection actually opens with, from
/// `db::schema::PRAGMAS_SEARCH`.
const SEARCH_CACHE_BYTES: u64 = 32 * 1024 * 1024;
/// Best-of-N. The minimum is the run least disturbed by everything else on
/// the box, which is the honest figure for a comparison — a mean would
/// measure the CI runner's other tenants.
const RUNS: u32 = 5;
/// Below this, a ratio is noise over noise: two sub-millisecond timings
/// divide into anything. Every workload here is far above it; the guard is
/// for the day someone shrinks the seed.
const MIN_MEASURABLE: Duration = Duration::from_millis(3);
fn spec() -> SeedSpec {
SeedSpec {
files: FILES,
content_every: CONTENT_EVERY,
// One row in five pairs up: enough groups that ranking them is real
// work, not so many that the whole table is one giant group.
dup_every: 5,
..SeedSpec::default()
}
}
fn key() -> IndexKey {
IndexKey::from_hex(KEY_HEX).expect("a 64-hex-digit key")
}
/// Seed the same corpus twice, once plain and once keyed. Identical content
/// and identical insertion order, so the two indexes differ *only* by
/// encryption — which is what lets a display-limited query be compared at all
/// (the cascade stops when the limit fills, so a different rowid order would
/// decide the answer rather than the encryption).
fn seed_both() -> (PathBuf, PathBuf) {
let plain = scratch_db("encperf-plain");
let keyed = scratch_db("encperf-keyed");
db::set_process_key(None);
seed_index(&plain, &spec());
db::set_process_key(Some(key()));
seed_index(&keyed, &spec());
db::set_process_key(None);
(plain, keyed)
}
fn mib(path: &PathBuf) -> f64 {
std::fs::metadata(path).map(|m| m.len()).unwrap_or(0) as f64 / (1024.0 * 1024.0)
}
/// Run `f` `RUNS` times, keeping the fastest.
fn best_of(mut f: impl FnMut()) -> Duration {
let mut best = Duration::MAX;
for _ in 0..RUNS {
let start = Instant::now();
f();
best = best.min(start.elapsed());
}
best
}
/// One workload's verdict. Collected rather than asserted inline so a run
/// reports *every* ratio, not just the first one that failed.
struct Measured {
what: &'static str,
plain: Duration,
keyed: Duration,
}
impl Measured {
fn ratio(&self) -> f64 {
self.keyed.as_secs_f64() / self.plain.as_secs_f64()
}
fn line(&self) -> String {
format!(
"{:<28} plain {:>9.2?} encrypted {:>9.2?} ratio {:>5.2}x",
self.what,
self.plain,
self.keyed,
self.ratio()
)
}
}
/// Time `find_duplicate_groups`, which opens its own connection — so the
/// process key has to be right at call time, not at open time.
fn time_duplicates(path: &PathBuf, keyed: bool) -> Duration {
let db_path = path.to_string_lossy().into_owned();
best_of(|| {
db::set_process_key(keyed.then(key));
let groups = find_duplicate_groups(&db_path, 200).expect("duplicate scan");
assert!(!groups.is_empty(), "the seed must contain duplicate groups");
})
}
/// Time one cascade query on a connection opened while its key state was
/// installed. Connections keep their own codec, so no toggling is needed once
/// they are open.
fn time_query(conn: &rusqlite::Connection, query: &str, fuzzy: bool) -> Duration {
let split = split_for_cascade(query).expect("query parses");
let options = SearchOptions {
fuzzy,
..SearchOptions::default()
};
best_of(|| {
let latest = AtomicU64::new(1);
let mut hits = 0usize;
let mut sink = |batch: Vec<SearchHit>| hits += batch.len();
cascade::run(conn, &split, &options, 1, &latest, &mut sink).expect("cascade runs");
assert!(hits > 0, "'{}' must match something to be timed", query);
})
}
#[test]
fn encryption_costs_a_constant_factor_not_a_different_algorithm() {
let (plain, keyed) = seed_both();
// Both connections are opened up front, each under its own key state.
db::set_process_key(None);
let plain_conn = db::open::open_search_reader(&plain.to_string_lossy()).expect("open plain");
db::set_process_key(Some(key()));
let keyed_conn = db::open::open_search_reader(&keyed.to_string_lossy()).expect("open keyed");
db::set_process_key(None);
println!(
"seeded {} files ({} with content): plain {:.1} MiB, encrypted {:.1} MiB",
FILES,
FILES / CONTENT_EVERY,
mib(&plain),
mib(&keyed),
);
assert!(
(mib(&plain) * 1024.0 * 1024.0) as u64 > SEARCH_CACHE_BYTES,
"seed is smaller than the {} MiB search cache, so both arms would be \
served entirely from memory and every ratio below would be a \
meaningless 1.0 — raise FILES",
SEARCH_CACHE_BYTES / (1024 * 1024)
);
// Duplicate finding first: it is the shape this gate exists for.
let mut measured = vec![Measured {
what: "find_duplicate_groups",
plain: time_duplicates(&plain, false),
keyed: time_duplicates(&keyed, true),
}];
// The cascade's four shapes. Arms alternate per workload so a machine that
// slows down partway through moves both sides, not one.
for (what, query, fuzzy) in [
("cascade literal name", NEEDLE, false),
("cascade literal body", BODY_TERM, false),
("cascade fuzzy", "quartzlte", true),
("cascade wildcard", "quart*", false),
("cascade regex", "regex:quart[sz]ite", false),
] {
measured.push(Measured {
what,
plain: time_query(&plain_conn, query, fuzzy),
keyed: time_query(&keyed_conn, query, fuzzy),
});
}
for m in &measured {
println!("{}", m.line());
}
let too_short: Vec<&Measured> = measured
.iter()
.filter(|m| m.plain < MIN_MEASURABLE || m.keyed < MIN_MEASURABLE)
.collect();
assert!(
too_short.is_empty(),
"these workloads finished under {:?}, so their ratios are noise over \
noise rather than a measurement:\n{}",
MIN_MEASURABLE,
too_short
.iter()
.map(|m| m.line())
.collect::<Vec<_>>()
.join("\n")
);
let amplified: Vec<&Measured> = measured.iter().filter(|m| m.ratio() > MAX_RATIO).collect();
assert!(
amplified.is_empty(),
"encryption amplified these beyond {:.1}x, which means per-page work \
scaling with rows rather than a constant factor:\n{}",
MAX_RATIO,
amplified
.iter()
.map(|m| m.line())
.collect::<Vec<_>>()
.join("\n")
);
}

3
crates/quicksearch-core/tests/extraction_corpus.rs
View file

@ -115,7 +115,8 @@ fn head_extraction_agrees_with_reading_the_file() {
.expect("claimed");
assert_eq!(
from_head, from_disk,
from_head,
from_disk,
"{} head and disk extraction disagree",
ctx(sample)
);

60
crates/quicksearch-core/tests/full_index.rs
View file

@ -483,6 +483,66 @@ fn a_symlink_target_in_an_unwalked_directory_survives_reindexing() {
assert!(off[0].ends_with("normal.txt"));
}
/// A target *inside* the root, so the walk reaches it twice — once by reading
/// its own directory, once by resolving the link. The other two symlink tests
/// use targets the walk never reaches directly, so this is the only one where
/// a duplicate visit actually happens.
///
/// **This is what replaced the writer's duplicate-visit set.** That set held a
/// path digest for every walked file — the largest thing a run kept — to stop
/// a second visit reaching the writer. It was removed once measured: a repeat
/// visit is already collapsed by `insert_file` being `INSERT OR IGNORE` and
/// returning `None` for a row that exists, so it can produce neither a second
/// row nor a second FTS entry. This test is the guard on that claim; it is
/// expected to fail if `insert_file` ever stops ignoring conflicts.
#[test]
#[cfg(unix)]
fn a_file_reachable_both_directly_and_through_a_link_gets_one_row() {
let root = Scratch::dir("alias-dup-root");
let db_dir = Scratch::dir("alias-dup-db");
let mut config = Config::default();
config.indexing.follow_symlinks = true;
let target = root.join("real/file.txt");
touch(&target, b"reachable two ways");
std::os::unix::fs::symlink(&target, root.join("link.txt")).unwrap();
let db = db_dir.join("links-on.sqlite");
index_once(&root, &db, &config);
let on = rows(&db);
assert_eq!(
on.len(),
1,
"one canonical path, one row, however many ways it was reached: {:?}",
on
);
assert!(
on[0].0.ends_with("real/file.txt"),
"stored under the target"
);
// The link is not a second document: a duplicated visit that reached the
// writer twice would tokenize the body twice into a contentless FTS
// table, where nothing would later collapse the two.
let conn = rusqlite::Connection::open(&db).unwrap();
let hits: i64 = conn
.query_row(
"SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH '\"reachable\"'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(hits, 1, "the body is indexed once");
// Links off: the link is never followed, so the same tree still yields
// exactly the one row — the run that skips the set entirely.
let db2 = db_dir.join("links-off.sqlite");
index_once(&root, &db2, &Config::default());
let off = rows(&db2);
assert_eq!(off.len(), 1, "only the real file: {:?}", off);
assert_eq!(off[0].0, on[0].0, "and the same path as with links on");
}
#[test]
#[cfg(unix)]
fn a_modified_symlink_target_is_updated_not_silently_ignored() {

13
crates/quicksearch-gui/src/app/security.rs
View file

@ -38,16 +38,9 @@ impl Drop for SecurityPrompt {
/// Confirm the password, re-derive from it, reveal the installed key.
/// Nothing here can change the key or the config.
pub(super) enum KeyPrompt {
Confirm {
pw: String,
wrong: bool,
},
Deriving {
rx: mpsc::Receiver<IndexKey>,
},
Reveal {
display: String,
},
Confirm { pw: String, wrong: bool },
Deriving { rx: mpsc::Receiver<IndexKey> },
Reveal { display: String },
}
impl Drop for KeyPrompt {

13
crates/quicksearch-gui/src/app/tests.rs
View file

@ -166,7 +166,13 @@ fn only_an_empty_duplicates_tab_scans_on_arrival() {
);
}
// No other tab scans, whatever the Duplicates tab is holding.
for tab in [Tab::Search, Tab::Manage, Tab::Logs, Tab::Help, Tab::Settings] {
for tab in [
Tab::Search,
Tab::Manage,
Tab::Logs,
Tab::Help,
Tab::Settings,
] {
assert!(!arrival_starts_dup_scan(tab, &DupState::NotLoaded));
}
}
@ -189,7 +195,10 @@ fn an_index_that_changed_invalidates_what_the_tab_holds() {
);
// Out of sight: drop it, and let the next visit pay for the rescan.
assert_eq!(dup_invalidation(Tab::Search, &loaded()), DupInvalidation::Drop);
assert_eq!(
dup_invalidation(Tab::Search, &loaded()),
DupInvalidation::Drop
);
// Nothing to invalidate.
assert_eq!(

11
crates/quicksearch-gui/src/backend.rs
View file

@ -43,9 +43,11 @@ pub struct Backend {
impl Backend {
/// Rebuild, after letting go of everything holding the index open: the
/// search worker keeps its connection warm for half a minute, and
/// without the release the delete fails on Windows and the rebuild
/// silently becomes an ordinary run against the old index.
/// search worker keeps its connection warm for half an hour
/// (`search::IDLE_RELEASE`), and without the release the delete fails on
/// Windows and the rebuild silently becomes an ordinary run against the
/// old index. The window is long enough that waiting one out is not a
/// fallback — this release is the only thing that makes the delete work.
pub fn rebuild_index(&self) {
if let Some(search) = &self.search {
search.release_connection();
@ -138,7 +140,8 @@ impl Backend {
let (tx, rx) = mpsc::channel();
let db = config.resolved_database_path();
std::thread::spawn(move || {
let result = quicksearch_core::search::find_duplicate_groups(&db.to_string_lossy(), 500);
let result =
quicksearch_core::search::find_duplicate_groups(&db.to_string_lossy(), 500);
let _ = tx.send(result);
ctx.request_repaint();
});

2
crates/quicksearch-gui/src/query_highlight.rs
View file

@ -10,10 +10,10 @@ use std::sync::Arc;
use egui::text::{LayoutJob, TextFormat};
use egui::{Color32, Galley, Stroke};
use quicksearch_core::query::Op;
use quicksearch_core::query::lexer::{tokenize_spanned, Token};
use quicksearch_core::query::pattern::RegexQuery;
use quicksearch_core::query::translator::{build_filter, is_filter_key};
use quicksearch_core::query::Op;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Class {

5
crates/quicksearch-gui/src/search_tab/tests.rs
View file

@ -69,7 +69,10 @@ fn render_perf() {
for (label, mut tab) in [
("empty (floor)", new_tab()),
("1000 rows, name only", tab_with_results(1000)),
("1000 rows, content snippets", tab_with_content_snippets(1000)),
(
"1000 rows, content snippets",
tab_with_content_snippets(1000),
),
] {
for _ in 0..10 {
timed_frame(&ctx, &mut tab);