jeremy/quick_search
jeremy/quick_search
1
0
Fork 0
Code Issues Pull requests Projects Releases 15 Packages Wiki Activity Actions

Another performance pass: Batch writes and cut background work across indexing, search and the GUI
All checks were successful
CI / linux (push) Successful in 13m58s
Details
CI / windows-cross (push) Successful in 4m51s
Details
CI / release (push) Successful in 13s
Details

Browse source
This commit is contained in:
Jeremy Karst 2026-09-27 14:19:39 -04:00
parent 590ff9ad0b
commit 5bc2d53ad9
41 changed files with 1368 additions and 1356 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
crates/quicksearch-core/benches/index.rs
View file

@ -16,9 +16,8 @@ fn main() {
divan::main(); divan::main();
} }
/// Compressing an extracted document: pure CPU, done before the writer takes /// Compressing an extracted document: pure CPU, done by the writer as each
/// the connection. `batch_*` measures a whole chunk, as `Bodies::fill` /// row is stored. `batch_*` measures a whole walk batch of small bodies.
/// compresses one.
mod zstd_encode { mod zstd_encode {
use super::*; use super::*;
@ -38,9 +37,8 @@ mod zstd_encode {
bencher.bench_local(move || enc.compress(divan::black_box(text)).unwrap()); bencher.bench_local(move || enc.compress(divan::black_box(text)).unwrap());
} }
/// One writer chunk of typical documents. A contention figure, not a /// One walk batch of typical documents. All compression together is
/// throughput one: the lock serializes the indexer against itself, and /// under 1% of a cold run.
/// all compression together is under 1% of a cold run.
const BATCH: usize = 500; const BATCH: usize = 500;
#[divan::bench] #[divan::bench]
@ -235,11 +233,9 @@ mod update_narrowing {
} }
} }
/// A watcher event for a directory whose files are all indexed and unchanged: /// A watcher event for a directory of indexed files, applied in one
/// a re-sent `Create`, a `chmod -R`, the live view re-checking what it shows. /// transaction as the coordinator applies a slice of its queue.
/// Nothing is written, so the cost is whatever it reads to learn that; the mod directory_event {
/// full walk's answer is one `stat` per file.
mod unchanged_event {
use super::*; use super::*;
use quicksearch_core::config::{Config, IgnoreSet}; use quicksearch_core::config::{Config, IgnoreSet};
use quicksearch_core::extract::Registry; use quicksearch_core::extract::Registry;
@ -251,8 +247,11 @@ mod unchanged_event {
const FILES: usize = 500; const FILES: usize = 500;
/// Every file unchanged: a re-sent `Create`, a `chmod -R`, the live view
/// re-checking what it shows. Nothing is written, so the cost is whatever
/// it reads to learn that; the full walk's answer is one `stat` per file.
#[divan::bench] #[divan::bench]
fn directory_create(bencher: Bencher) { fn unchanged(bencher: Bencher) {
let (dir, db) = Scratch::db("bench-unchanged-event"); let (dir, db) = Scratch::db("bench-unchanged-event");
let tree = db.with_file_name("tree"); let tree = db.with_file_name("tree");
std::fs::create_dir_all(&tree).unwrap(); std::fs::create_dir_all(&tree).unwrap();
@ -277,11 +276,65 @@ mod unchanged_event {
cancel: &cancel, cancel: &cancel,
resume_from: 0, resume_from: 0,
}; };
apply_fs_event(&mut conn, &event, &config, &ignore, &registry, &budget).unwrap() // One transaction per event, as the coordinator's apply slice is.
let tx = conn.transaction().unwrap();
let applied =
apply_fs_event(&tx, &event, &config, &ignore, &registry, &budget).unwrap();
tx.commit().unwrap();
applied
}; };
// Indexed once; every timed run finds it unchanged. // Indexed once; every timed run finds it unchanged.
apply(); apply();
bencher.bench_local(apply); bencher.bench_local(apply);
drop(dir); drop(dir);
} }
/// The same directory with every file's mtime moved on before each run,
/// so each event rewrites all of its rows and their text: a checkout or
/// an unpack, where commit and FTS5 segment costs add up per file.
#[divan::bench]
fn changed(bencher: Bencher) {
let (dir, db) = Scratch::db("bench-changed-event");
let tree = db.with_file_name("tree");
std::fs::create_dir_all(&tree).unwrap();
let files: Vec<std::path::PathBuf> =
(0..FILES).map(|i| tree.join(format!("f{:04}.txt", i))).collect();
for file in &files {
std::fs::write(file, corpus::text(1 << 10, 4)).unwrap();
}
let mut conn =
quicksearch_core::db::open_or_recreate(db.to_str().unwrap(), &Default::default())
.unwrap();
let config = Config::default();
let ignore = IgnoreSet::compile(&config.indexing.ignore_patterns).unwrap();
let registry = Registry::default_set();
let event = FsEvent::Create(tree.clone());
let cancel = AtomicBool::new(false);
let mut apply = move || {
let budget = Budget {
deadline: Instant::now() + Duration::from_secs(3600),
cancel: &cancel,
resume_from: 0,
};
// One transaction per event, as the coordinator's apply slice is.
let tx = conn.transaction().unwrap();
let applied =
apply_fs_event(&tx, &event, &config, &ignore, &registry, &budget).unwrap();
tx.commit().unwrap();
applied
};
apply();
let mut tick = 0u64;
bencher
.with_inputs(|| {
tick += 1;
let at = std::time::UNIX_EPOCH + Duration::from_secs(1_700_000_000 + tick);
for file in &files {
let f = std::fs::File::options().write(true).open(file).unwrap();
f.set_modified(at).unwrap();
}
})
.bench_local_values(|()| apply());
drop(dir);
}
} }

26
crates/quicksearch-core/benches/index_alloc.rs
View file

@ -157,12 +157,12 @@ fn walk_traffic_per_file(samples: &[corpus::Sample], config: &Config, registry:
); );
} }
/// Rows per writer chunk: `batch::STORE_CHUNK`, the unit `Bodies::fill` is /// Rows per writer turn: `pipeline::READY_TOPUP`, the most one
/// handed. /// `store_extracted` call is handed.
const CHUNK: usize = 32; const CHUNK: usize = 64;
/// The writer's half that runs outside the connection lock: one output /// The writer's compression: one output `Vec<u8>` per row against the run's
/// `Vec<u8>` per row against one arena per chunk. /// one encoder and its reused buffer.
/// ///
/// The zstd context does not appear here: `zstd::bulk::Compressor::new` /// The zstd context does not appear here: `zstd::bulk::Compressor::new`
/// allocates through zstd's own C allocator, so its cost is CPU only /// allocates through zstd's own C allocator, so its cost is CPU only
@ -186,17 +186,15 @@ fn compression_traffic_per_chunk(config: &Config) {
.map(|_| repo::encode_one(&doc).expect("encode")) .map(|_| repo::encode_one(&doc).expect("encode"))
.collect::<Vec<_>>() .collect::<Vec<_>>()
}); });
// What `Bodies::fill` does: one encoder and one arena for the chunk. // What `store_extracted` does: one encoder, its buffer reused per row.
let arena = measure(|| { let reused = measure(|| {
let mut enc = DocEncoder::new().expect("encoder"); let mut enc = DocEncoder::new().expect("encoder");
let mut arena = Vec::new();
(0..CHUNK) (0..CHUNK)
.map(|_| enc.encode_into(&doc, &mut arena).expect("encode")) .map(|_| enc.body(&doc).expect("encode").map_or(0, <[u8]>::len))
.collect::<Vec<_>>() .sum::<usize>()
.len()
}); });
for (shape, usage) in [("a Vec per row", per_row), ("one arena, chunk", arena)] { for (shape, usage) in [("a Vec per row", per_row), ("one encoder, reused", reused)] {
println!( println!(
"{:<24} {:>10} {:>10} {:>12} {:>12}", "{:<24} {:>10} {:>10} {:>12} {:>12}",
shape, shape,
@ -207,8 +205,8 @@ fn compression_traffic_per_chunk(config: &Config) {
); );
} }
println!( println!(
"\n The arena is reused across every chunk a writer call handles, so after the\n \ "\n The encoder lives for the whole run, so after the first turn its growth is\n \
first its growth is zero too. The zstd context is invisible here — see above.\n" zero too. The zstd context is invisible here — see above.\n"
); );
} }

46
crates/quicksearch-core/benches/search.rs
View file

@ -165,39 +165,40 @@ mod substring {
bencher.bench(|| finder.find_iter(divan::black_box(text)).count()); bencher.bench(|| finder.find_iter(divan::black_box(text)).count());
} }
/// What `pass_fulltext` runs per row on the literal path: a /// What `pass_fulltext` runs per row on the literal path: one folded
/// case-sensitive count, then one folded extraction yielding both. /// extraction yielding the window and both counts.
#[divan::bench(args = corpus::SIZES)] #[divan::bench(args = corpus::SIZES)]
fn cascade_row_sweeps(bencher: Bencher, size: usize) { fn cascade_row_sweeps(bencher: Bencher, size: usize) {
let pattern = literal(corpus::NEEDLE);
let text = corpus::text_mixed(size, 4); let text = corpus::text_mixed(size, 4);
let folded = corpus::text_folded(size, 4); let folded = corpus::text_folded(size, 4);
let opts = snippet::Options { approx_bytes: 600 }; let term = corpus::NEEDLE.to_ascii_lowercase();
let budget = 600;
bencher.bench(|| { bencher.bench(|| {
let (s, b) = snippet::extract_folded( snippet::extract_folded(
divan::black_box(text), divan::black_box(text),
divan::black_box(folded), divan::black_box(folded),
&term,
corpus::NEEDLE, corpus::NEEDLE,
&opts, budget,
); )
let a = pattern.count(text); .map(|f| (f.count, f.exact, f.snippet.ranges.len()))
(a, b, s.ranges.len())
}); });
} }
/// The rejected shape: a separate folded count sweeps the document a /// The replaced shape: a separate case-sensitive count sweeps the
/// third time for a number the extraction already has. /// document a second time for a number the extraction now has.
#[divan::bench(args = corpus::SIZES)] #[divan::bench(args = corpus::SIZES)]
fn cascade_row_sweeps_separate_count(bencher: Bencher, size: usize) { fn cascade_row_sweeps_separate_count(bencher: Bencher, size: usize) {
let pattern = literal(corpus::NEEDLE);
let text = corpus::text_mixed(size, 4); let text = corpus::text_mixed(size, 4);
let folded = corpus::text_folded(size, 4); let folded = corpus::text_folded(size, 4);
let opts = snippet::Options { approx_bytes: 600 }; let term = corpus::NEEDLE.to_ascii_lowercase();
let budget = 600;
bencher.bench(|| { bencher.bench(|| {
let a = pattern.count(divan::black_box(text)); let exact =
let b = pattern.count_folded(divan::black_box(folded)); memchr::memmem::find_iter(divan::black_box(text).as_bytes(), corpus::NEEDLE)
let (s, _) = snippet::extract_folded(text, folded, corpus::NEEDLE, &opts); .count();
(a, b, s.ranges.len()) let f = snippet::extract_folded(text, folded, &term, corpus::NEEDLE, budget);
(exact, f.map(|f| (f.count, f.snippet.ranges.len())))
}); });
} }
} }
@ -211,13 +212,15 @@ mod snippet_extract {
fn extract_folded(bencher: Bencher, size: usize) { fn extract_folded(bencher: Bencher, size: usize) {
let text = corpus::text_mixed(size, 64); let text = corpus::text_mixed(size, 64);
let folded = corpus::text_folded(size, 64); let folded = corpus::text_folded(size, 64);
let opts = snippet::Options { approx_bytes: 600 }; let term = corpus::NEEDLE.to_ascii_lowercase();
let budget = 600;
bencher.bench(|| { bencher.bench(|| {
snippet::extract_folded( snippet::extract_folded(
divan::black_box(text), divan::black_box(text),
divan::black_box(folded), divan::black_box(folded),
&term,
corpus::NEEDLE, corpus::NEEDLE,
&opts, budget,
) )
}); });
} }
@ -249,7 +252,6 @@ mod filename_ladder {
#[divan::bench] #[divan::bench]
fn find_first_ci_scratch(bencher: Bencher) { fn find_first_ci_scratch(bencher: Bencher) {
let pattern = literal("quartzite");
let rows = corpus::rows(); let rows = corpus::rows();
let mut scratch = String::new(); let mut scratch = String::new();
bencher.bench_local(move || { bencher.bench_local(move || {
@ -258,14 +260,14 @@ mod filename_ladder {
scratch.clear(); scratch.clear();
scratch.push_str(&row.name); scratch.push_str(&row.name);
scratch.make_ascii_lowercase(); scratch.make_ascii_lowercase();
if pattern.find_first_folded(&scratch).is_some() { if memchr::memmem::find(scratch.as_bytes(), b"quartzite").is_some() {
found += 1; found += 1;
continue; continue;
} }
scratch.clear(); scratch.clear();
scratch.push_str(&row.path); scratch.push_str(&row.path);
scratch.make_ascii_lowercase(); scratch.make_ascii_lowercase();
if pattern.find_first_folded(&scratch).is_some() { if memchr::memmem::find(scratch.as_bytes(), b"quartzite").is_some() {
found += 1; found += 1;
} }
} }

29
crates/quicksearch-core/examples/walkprobe.rs
View file

@ -20,14 +20,11 @@
use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicBool;
use std::sync::Arc; use std::sync::Arc;
use std::time::{Instant, UNIX_EPOCH}; use std::time::Instant;
use quicksearch_core::config::{Config, IgnoreSet}; use quicksearch_core::config::{Config, IgnoreSet};
use quicksearch_core::extract::Registry; use quicksearch_core::extract::Registry;
use quicksearch_core::file_handling::{ use quicksearch_core::file_handling::{filtered_walk, prepare_file_record, UnreadableDirs};
classify_for_indexing, filtered_walk, prepare_file_record, DirRows, FileIndexAction,
UnreadableDirs,
};
use quicksearch_core::walk::{walk_indexable_files, WalkEvent}; use quicksearch_core::walk::{walk_indexable_files, WalkEvent};
fn main() { fn main() {
@ -42,11 +39,10 @@ fn main() {
let _ = std::fs::remove_file(&db); let _ = std::fs::remove_file(&db);
quicksearch_core::db::open_or_recreate(db.to_str().unwrap(), &config.processing) quicksearch_core::db::open_or_recreate(db.to_str().unwrap(), &config.processing)
.expect("scratch index"); .expect("scratch index");
let existing = DirRows::new();
let start = Instant::now(); let start = Instant::now();
let (seen, prepared) = match mode.as_str() { let (seen, prepared) = match mode.as_str() {
"serial" => serial(&root, &config, &existing), "serial" => serial(&root, &config),
_ => parallel(&root, &config, db.to_str().unwrap()), _ => parallel(&root, &config, db.to_str().unwrap()),
}; };
let _ = std::fs::remove_file(&db); let _ = std::fs::remove_file(&db);
@ -59,7 +55,8 @@ fn main() {
); );
} }
fn serial(root: &str, config: &Config, existing: &DirRows) -> (usize, usize) { /// Against the same empty index, so every file is new and gets prepared.
fn serial(root: &str, config: &Config) -> (usize, usize) {
let ignore = IgnoreSet::compile(&[]).unwrap(); let ignore = IgnoreSet::compile(&[]).unwrap();
let registry = Registry::default_set(); let registry = Registry::default_set();
// One for the whole walk, as a walk worker holds one. // One for the whole walk, as a walk worker holds one.
@ -74,21 +71,7 @@ fn serial(root: &str, config: &Config, existing: &DirRows) -> (usize, usize) {
let Ok(meta) = std::fs::metadata(entry.path()) else { let Ok(meta) = std::fs::metadata(entry.path()) else {
continue; continue;
}; };
let Some(mtime) = meta if prepare_file_record(&path, &meta, config, &registry, &mut scratch).is_some() {
.modified()
.ok()
.and_then(|t| t.duration_since(UNIX_EPOCH).ok())
.map(|d| d.as_secs())
else {
continue;
};
let name = std::path::Path::new(&path)
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_default();
if classify_for_indexing(&name, mtime, existing) != FileIndexAction::Skip
&& prepare_file_record(&path, &meta, config, &registry, &mut scratch).is_some()
{
prepared += 1; prepared += 1;
} }
} }

16
crates/quicksearch-core/src/content.rs
View file

@ -331,16 +331,14 @@ mod tests {
use crate::db::open_or_recreate; use crate::db::open_or_recreate;
use crate::db::repo::{self, insert_file, NewFile}; use crate::db::repo::{self, insert_file, NewFile};
use crate::file_handling::{ use crate::file_handling::{extract_scope_prepare, store_extracted, ExtractScope, Stored};
extract_scope_prepare, store_extracted, Bodies, ExtractScope, Stored,
};
use crate::mime::FileType; use crate::mime::FileType;
use crate::testutil::Scratch; use crate::testutil::Scratch;
use std::path::PathBuf; use std::path::PathBuf;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
fn bodies() -> Bodies { fn encoder() -> crate::db::repo::DocEncoder {
Bodies::new().unwrap() crate::db::repo::DocEncoder::new().unwrap()
} }
fn tree(s: &Scratch) -> PathBuf { fn tree(s: &Scratch) -> PathBuf {
@ -428,7 +426,7 @@ mod tests {
assert!(pass.finish()); assert!(pass.finish());
let stop = Arc::new(AtomicBool::new(false)); let stop = Arc::new(AtomicBool::new(false));
let far = Instant::now() + Duration::from_secs(60); let far = Instant::now() + Duration::from_secs(60);
store_extracted(conn, &rows, &stop, far, &mut bodies()).unwrap() store_extracted(conn, &rows, &stop, far, &mut encoder()).unwrap()
} }
#[test] #[test]
@ -552,7 +550,7 @@ mod tests {
// A deadline already gone by: one row, then out. // A deadline already gone by: one row, then out.
let past = Instant::now() - Duration::from_secs(1); let past = Instant::now() - Duration::from_secs(1);
assert_eq!( assert_eq!(
store_extracted(&conn_mutex, &rows, &stop, past, &mut bodies()).unwrap(), store_extracted(&conn_mutex, &rows, &stop, past, &mut encoder()).unwrap(),
Stored { Stored {
consumed: 1, consumed: 1,
written: 1 written: 1
@ -560,7 +558,7 @@ mod tests {
); );
let far = Instant::now() + Duration::from_secs(60); let far = Instant::now() + Duration::from_secs(60);
assert_eq!( assert_eq!(
store_extracted(&conn_mutex, &rows[1..], &stop, far, &mut bodies()).unwrap(), store_extracted(&conn_mutex, &rows[1..], &stop, far, &mut encoder()).unwrap(),
Stored { Stored {
consumed: 4, consumed: 4,
written: 4 written: 4
@ -580,7 +578,7 @@ mod tests {
// Stopped before it starts: nothing consumed. // Stopped before it starts: nothing consumed.
stop.store(true, std::sync::atomic::Ordering::Relaxed); stop.store(true, std::sync::atomic::Ordering::Relaxed);
assert_eq!( assert_eq!(
store_extracted(&conn_mutex, &rows, &stop, far, &mut bodies()).unwrap(), store_extracted(&conn_mutex, &rows, &stop, far, &mut encoder()).unwrap(),
Stored::default() Stored::default()
); );
} }

6
crates/quicksearch-core/src/coordinator.rs
View file

@ -43,8 +43,6 @@ use inner::Inner;
/// Pending-event ceiling; beyond this a full run is cheaper than replay. /// Pending-event ceiling; beyond this a full run is cheaper than replay.
const PENDING_OVERFLOW: usize = 100_000; const PENDING_OVERFLOW: usize = 100_000;
const FILE_COUNT_INTERVAL: Duration = Duration::from_secs(30);
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IndexMode { pub enum IndexMode {
Auto, Auto,
@ -211,7 +209,7 @@ impl IndexCoordinator {
watcher_rx: None, watcher_rx: None,
watcher_gen: 0, watcher_gen: 0,
pending: inner::Debounced::default(), pending: inner::Debounced::default(),
targeted: HashMap::new(), targeted: inner::Debounced::default(),
resume_from: HashMap::new(), resume_from: HashMap::new(),
needs_full_run: false, needs_full_run: false,
run_retry: inner::Backoff::default(), run_retry: inner::Backoff::default(),
@ -219,7 +217,7 @@ impl IndexCoordinator {
reconcile_done: None, reconcile_done: None,
reconcile_cut_short: false, reconcile_cut_short: false,
run_in_flight: false, run_in_flight: false,
files_at: None, files_stale: true,
was_busy: false, was_busy: false,
write_conn: None, write_conn: None,
ignore: Arc::new(IgnoreSet::compile(&[]).expect("empty ignore set")), ignore: Arc::new(IgnoreSet::compile(&[]).expect("empty ignore set")),

152
crates/quicksearch-core/src/coordinator/inner.rs
View file

@ -8,9 +8,10 @@ const RUN_RETRY_BASE: Duration = Duration::from_secs(30);
const RUN_RETRY_MAX: Duration = Duration::from_secs(300); const RUN_RETRY_MAX: Duration = Duration::from_secs(300);
/// The watcher's event queue and the two timers that decide when to apply /// An event queue and the two timers that decide when to apply it: once no
/// it: once no event has arrived for `pending_settle`, or once the oldest /// event has arrived for `pending_settle`, or once the oldest has waited
/// has waited `pending_max_defer`, so a trickle cannot defer it forever. /// `pending_max_defer`, so a trickle cannot defer it forever. One for the
/// watcher's events, one for the paths a frontend is displaying.
#[derive(Default)] #[derive(Default)]
pub(super) struct Debounced { pub(super) struct Debounced {
pub(super) events: HashMap<PathBuf, FsEvent>, pub(super) events: HashMap<PathBuf, FsEvent>,
@ -107,6 +108,17 @@ impl Backoff {
} }
} }
/// Whether `new` changes anything [`Inner::start_watcher`] registers with.
/// Every other edit (theme, shortcut, search limits) keeps the running
/// watcher: restarting it re-walks every directory under every root to
/// re-register its watch, and misses whatever changes in the gap.
pub(super) fn watch_inputs_differ(old: &Config, new: &Config) -> bool {
old.resolved_indexing_paths() != new.resolved_indexing_paths()
|| old.indexing.include_hidden != new.indexing.include_hidden
|| old.indexing.follow_symlinks != new.indexing.follow_symlinks
|| old.indexing.ignore_patterns != new.indexing.ignore_patterns
}
/// Log a failed apply of `what` by the queue's policy: a targeted failure /// Log a failed apply of `what` by the queue's policy: a targeted failure
/// only logs, a pending one also schedules a full run. `true` for the latter. /// only logs, a pending one also schedules a full run. `true` for the latter.
fn apply_failed(targeted: bool, what: &str, e: impl std::fmt::Display) -> bool { fn apply_failed(targeted: bool, what: &str, e: impl std::fmt::Display) -> bool {
@ -149,7 +161,7 @@ pub(super) struct Inner {
/// Paths a frontend asked for by name ([`IndexCoordinator::update_paths`]). /// Paths a frontend asked for by name ([`IndexCoordinator::update_paths`]).
/// Survives [`Inner::clear_pending`] and applies in manual mode: it keeps /// Survives [`Inner::clear_pending`] and applies in manual mode: it keeps
/// the rows a user is *reading* in step with the disk. /// the rows a user is *reading* in step with the disk.
pub(super) targeted: HashMap<PathBuf, FsEvent>, pub(super) targeted: Debounced,
/// How far a directory event got before its turn's budget ran out. /// How far a directory event got before its turn's budget ran out.
pub(super) resume_from: HashMap<PathBuf, usize>, pub(super) resume_from: HashMap<PathBuf, usize>,
pub(super) needs_full_run: bool, pub(super) needs_full_run: bool,
@ -163,7 +175,9 @@ pub(super) struct Inner {
/// Set while a full run holds the database; cleared by the tick that sees /// Set while a full run holds the database; cleared by the tick that sees
/// it end. /// it end.
pub(super) run_in_flight: bool, pub(super) run_in_flight: bool,
pub(super) files_at: Option<Instant>, /// The published row count may be behind the database: set by anything
/// that could have written, cleared by [`Inner::refresh_file_count`].
pub(super) files_stale: bool,
/// Drives [`Inner::go_idle`] once per busy→idle transition. /// Drives [`Inner::go_idle`] once per busy→idle transition.
pub(super) was_busy: bool, pub(super) was_busy: bool,
pub(super) write_conn: Option<Connection>, pub(super) write_conn: Option<Connection>,
@ -207,13 +221,14 @@ impl Inner {
CoordCmd::ConfigChanged(new) => { CoordCmd::ConfigChanged(new) => {
let want_auto = new.indexing.auto_index; let want_auto = new.indexing.auto_index;
let actions = diff_actions(&self.config, &new); let actions = diff_actions(&self.config, &new);
let rewatch = self.watcher.is_none() || watch_inputs_differ(&self.config, &new);
self.config = new; self.config = new;
if let Err(e) = self.reload_filters() { if let Err(e) = self.reload_filters() {
crate::log_warn!("coordinator: {}", e); crate::log_warn!("coordinator: {}", e);
} }
// The write connection may point at an old database_path. // The write connection may point at an old database_path.
self.write_conn = None; self.write_conn = None;
self.files_at = None; self.files_stale = true;
if !actions.requires_rebuild && !actions.work.is_empty() { if !actions.requires_rebuild && !actions.work.is_empty() {
self.start_work(actions.work); self.start_work(actions.work);
} }
@ -221,7 +236,8 @@ impl Inner {
(true, false) => self.enter_auto(), (true, false) => self.enter_auto(),
(false, true) => self.enter_manual_stopped(), (false, true) => self.enter_manual_stopped(),
// Still automatic: the watcher follows the new config. // Still automatic: the watcher follows the new config.
(true, true) => self.start_watcher(), (true, true) if rewatch => self.start_watcher(),
(true, true) => {}
(false, false) => {} (false, false) => {}
} }
self.refresh_last_full_index(); self.refresh_last_full_index();
@ -229,7 +245,7 @@ impl Inner {
CoordCmd::RebuildIndex => { CoordCmd::RebuildIndex => {
let db = self.db_path(); let db = self.db_path();
self.write_conn = None; self.write_conn = None;
self.files_at = None; self.files_stale = true;
self.pending_work = None; self.pending_work = None;
self.reconcile_done = None; self.reconcile_done = None;
if let Err(e) = self.indexing.delete_index_for_rebuild(&db) { if let Err(e) = self.indexing.delete_index_for_rebuild(&db) {
@ -260,7 +276,7 @@ impl Inner {
shared.files = Some(0); shared.files = Some(0);
shared.root_counts = Arc::new(Vec::new()); shared.root_counts = Arc::new(Vec::new());
drop(shared); drop(shared);
self.files_at = None; self.files_stale = true;
} }
CoordCmd::UpdatePaths(paths) => { CoordCmd::UpdatePaths(paths) => {
// Only paths under an indexed root: nothing downstream checks, // Only paths under an indexed root: nothing downstream checks,
@ -275,11 +291,8 @@ impl Inner {
let spelled = path.to_string_lossy(); let spelled = path.to_string_lossy();
prefixes.iter().any(|lo| spelled.starts_with(lo.as_str())) prefixes.iter().any(|lo| spelled.starts_with(lo.as_str()))
}; };
paths self.targeted
.into_iter() .push(paths.into_iter().filter(under_root).filter_map(verb_for));
.filter(under_root)
.filter_map(verb_for)
.for_each(|event| enqueue(&mut self.targeted, event));
self.was_busy = true; self.was_busy = true;
} }
CoordCmd::Shutdown => unreachable!("handled in run()"), CoordCmd::Shutdown => unreachable!("handled in run()"),
@ -313,7 +326,7 @@ impl Inner {
self.defer_runs(); self.defer_runs();
} }
self.refresh_last_full_index(); self.refresh_last_full_index();
self.files_at = None; self.files_stale = true;
if self.mode == IndexMode::ManualRunning { if self.mode == IndexMode::ManualRunning {
self.mode = IndexMode::ManualStopped; self.mode = IndexMode::ManualStopped;
} }
@ -321,8 +334,10 @@ impl Inner {
self.refresh_file_count(); self.refresh_file_count();
// Ahead of every gate: rows a user is looking at right now. // Ahead of every gate: rows a user is looking at right now. Settled
if !self.targeted.is_empty() { // like the watcher's queue, or a growing file on screen, re-sent on
// every live refresh, is re-read, hashed and re-indexed every tick.
if !self.targeted.events.is_empty() && self.targeted.settled(&self.watcher_config) {
self.apply_targeted(); self.apply_targeted();
} }
@ -369,6 +384,8 @@ impl Inner {
return; return;
} }
self.was_busy = false; self.was_busy = false;
// Whatever made it busy may have added or removed rows.
self.files_stale = true;
self.write_conn = None; self.write_conn = None;
crate::platform::release_free_heap(); crate::platform::release_free_heap();
} }
@ -486,7 +503,7 @@ impl Inner {
self.pending.clear(); self.pending.clear();
// Resume points for `targeted` survive: that queue outlives this call. // Resume points for `targeted` survive: that queue outlives this call.
self.resume_from self.resume_from
.retain(|p, _| self.targeted.contains_key(p)); .retain(|p, _| self.targeted.events.contains_key(p));
} }
/// Apply as much of the queue as fits in [`APPLY_BUDGET`], removals first: /// Apply as much of the queue as fits in [`APPLY_BUDGET`], removals first:
@ -510,49 +527,64 @@ impl Inner {
self.pending.after_partial_apply(); self.pending.after_partial_apply();
} }
/// Drain one queue's removals, then its upserts, within [`APPLY_BUDGET`]. /// Drain one queue's removals, then its upserts, within [`APPLY_BUDGET`],
/// `targeted` picks the queue and the failure policy: pending failures /// in one transaction. `targeted` picks the queue and the failure policy:
/// escalate to [`Inner::needs_full_run`], targeted ones only log. /// pending failures escalate to [`Inner::needs_full_run`], targeted ones
/// only log. Measured against a transaction per file (`benches/index.rs`,
/// `directory_event`, one event over 500 small text files): 250 →
/// 88 ms with every file changed, 1.55 → 1.09 ms with none.
fn apply_queue(&mut self, conn: &mut Connection, targeted: bool) { fn apply_queue(&mut self, conn: &mut Connection, targeted: bool) {
let deadline = Instant::now() + APPLY_BUDGET; let deadline = Instant::now() + APPLY_BUDGET;
let chunk = self.config.processing.batch_size.max(1); let chunk = self.config.processing.batch_size.max(1);
let tx = match conn.transaction() {
Ok(tx) => tx,
Err(e) => {
self.needs_full_run |= apply_failed(targeted, "begin", e);
return;
}
};
let queue = if targeted { let queue = if targeted {
&mut self.targeted &mut self.targeted.events
} else { } else {
&mut self.pending.events &mut self.pending.events
}; };
let removals: Vec<PathBuf> = queue // Moved out, not cloned: what the budget leaves goes back below, so a
.iter() // storm's queue is not copied whole on every slice, and the map it
.filter(|(_, ev)| is_removal(ev)) // leaves behind holds only what is left. Measured (200k targeted
.map(|(p, _)| p.clone()) // removals): drained in 10.0 → 9.1 s, peak RSS 85 → 81 MB.
.collect(); let (mut removals, mut upserts) = (Vec::new(), Vec::new());
for (path, ev) in std::mem::take(queue) {
if is_removal(&ev) {
removals.push(path);
} else {
upserts.push((path, ev));
}
}
let mut removed = 0;
for batch in removals.chunks(chunk) { for batch in removals.chunks(chunk) {
if let Err(e) = crate::incremental::remove_paths(conn, batch, chunk) { if let Err(e) = crate::incremental::remove_paths(&tx, batch) {
// The batch leaves the queue either way: retrying a failed // The batch leaves the queue either way: retrying a failed
// write every tick is worse, and the full run recovers the rows. // write every tick is worse, and the full run recovers the rows.
self.needs_full_run |= apply_failed(targeted, "remove", e); self.needs_full_run |= apply_failed(targeted, "remove", e);
} }
for path in batch { removed += batch.len();
queue.remove(path);
}
if Instant::now() >= deadline { if Instant::now() >= deadline {
break; break;
} }
} }
queue.extend(
removals
.drain(removed..)
.map(|path| (path.clone(), FsEvent::Remove(path))),
);
let mut upserts = upserts.into_iter();
if Instant::now() < deadline { if Instant::now() < deadline {
let upserts: Vec<PathBuf> = queue for (path, ev) in upserts.by_ref() {
.iter()
.filter(|(_, ev)| !is_removal(ev))
.map(|(p, _)| p.clone())
.collect();
for path in upserts {
let Some(ev) = queue.remove(&path) else {
continue;
};
match apply_fs_event( match apply_fs_event(
conn, &tx,
&ev, &ev,
&self.config, &self.config,
&self.ignore, &self.ignore,
@ -582,6 +614,10 @@ impl Inner {
} }
} }
} }
queue.extend(upserts);
if let Err(e) = tx.commit() {
self.needs_full_run |= apply_failed(targeted, "commit", e);
}
} }
/// Apply the by-name queue: the paths a frontend is displaying. Never /// Apply the by-name queue: the paths a frontend is displaying. Never
@ -603,6 +639,7 @@ impl Inner {
}; };
self.apply_queue(&mut conn, true); self.apply_queue(&mut conn, true);
self.write_conn = Some(conn); self.write_conn = Some(conn);
self.targeted.after_partial_apply();
} }
fn ensure_write_conn(&mut self) -> Result<Connection, String> { fn ensure_write_conn(&mut self) -> Result<Connection, String> {
@ -684,8 +721,9 @@ impl Inner {
// target (`aliased_paths`), and those rows would leak until a rebuild. // target (`aliased_paths`), and those rows would leak until a rebuild.
self.pending.keep_removals(); self.pending.keep_removals();
// A stale resume point would `skip` entries of an unrelated walk. // A stale resume point would `skip` entries of an unrelated walk.
self.resume_from self.resume_from.retain(|p, _| {
.retain(|p, _| self.pending.events.contains_key(p) || self.targeted.contains_key(p)); self.pending.events.contains_key(p) || self.targeted.events.contains_key(p)
});
if let Err(e) = self if let Err(e) = self
.indexing .indexing
.start_indexing(roots, self.db_path(), self.config.clone()) .start_indexing(roots, self.db_path(), self.config.clone())
@ -864,8 +902,16 @@ impl Inner {
self.set_watcher_status(WatcherStatus::Off); self.set_watcher_status(WatcherStatus::Off);
} }
/// Publish `status`, waking the frontend when it changed: registration
/// can finish minutes after startup with no input to repaint on, and a
/// frontend polling for it would repaint the whole time.
fn set_watcher_status(&self, status: WatcherStatus) { fn set_watcher_status(&self, status: WatcherStatus) {
crate::lock_ok(&self.shared).watcher = status; let mut shared = crate::lock_ok(&self.shared);
if shared.watcher != status {
shared.watcher = status;
drop(shared);
(self.notify)();
}
} }
fn db_path(&self) -> String { fn db_path(&self) -> String {
@ -883,20 +929,18 @@ impl Inner {
Ok(()) Ok(())
} }
/// Re-read the published row count, at most every [`FILE_COUNT_INTERVAL`]. /// Re-read the published row count once something may have changed it.
/// Called only from the idle half of [`Inner::tick`], so it cannot run /// Called only from the idle half of [`Inner::tick`], so it cannot run
/// while a full run holds the database; `COUNT(*)` is still a key scan, /// while a full run holds the database; `COUNT(*)` is still a key scan,
/// hence the interval and the interrupt guard. /// hence the interrupt guard. It used to run every 30 s whatever the
/// mode, opening a connection each time to recount an unchanged index.
fn refresh_file_count(&mut self) { fn refresh_file_count(&mut self) {
if self if !self.files_stale {
.files_at
.is_some_and(|at| at.elapsed() < FILE_COUNT_INTERVAL)
{
return; return;
} }
// Stamped before the read: a failing count must back off like a // Cleared before the read: a failing count must not retry every
// successful one, or a missing index means an open attempt every tick. // tick, or a missing index means an open attempt every second.
self.files_at = Some(Instant::now()); self.files_stale = false;
let Ok(conn) = db::open_existing(&self.db_path(), false) else { let Ok(conn) = db::open_existing(&self.db_path(), false) else {
return; return;
@ -962,7 +1006,7 @@ impl Inner {
let busy = reconcile.is_some() || self.indexing.is_active(); let busy = reconcile.is_some() || self.indexing.is_active();
let mut shared = crate::lock_ok(&self.shared); let mut shared = crate::lock_ok(&self.shared);
shared.mode = self.mode; shared.mode = self.mode;
shared.queued_events = self.pending.events.len() + self.targeted.len(); shared.queued_events = self.pending.events.len() + self.targeted.events.len();
shared.reconcile = reconcile; shared.reconcile = reconcile;
drop(shared); drop(shared);

97
crates/quicksearch-core/src/coordinator/tests.rs
View file

@ -350,12 +350,10 @@ fn stopping_cancels_a_queued_walk_but_not_a_queued_prune() {
coord.shutdown(); coord.shutdown();
} }
/// Short debounce windows so trailing-edge events flush within test /// Short debounce windows so trailing-edge events apply within test
/// timeouts (production defaults are 30 s / 2 s). /// timeouts (production defaults are 2 s / 30 s).
fn fast_watcher() -> WatcherConfig { fn fast_watcher() -> WatcherConfig {
WatcherConfig { WatcherConfig {
throttle_window: Duration::from_millis(300),
tick_interval: Duration::from_millis(100),
pending_settle: Duration::from_millis(200), pending_settle: Duration::from_millis(200),
pending_max_defer: Duration::from_secs(3), pending_max_defer: Duration::from_secs(3),
..WatcherConfig::default() ..WatcherConfig::default()
@ -473,6 +471,38 @@ fn update_paths_indexes_one_file_with_indexing_stopped() {
coord.shutdown(); coord.shutdown();
} }
/// A path re-sent on every live refresh, as a growing file on screen is,
/// waits for the queue to settle instead of being re-indexed every tick.
#[test]
fn update_paths_waits_for_the_queue_to_settle() {
let f = Fixture::new(false);
std::fs::write(f.dir.join("seed.txt"), "initial content").unwrap();
f.seed_index();
let settle = WatcherConfig {
pending_settle: Duration::from_secs(1),
pending_max_defer: Duration::from_secs(20),
..WatcherConfig::default()
};
let coord = start_coord_watching(f.config.clone(), settle);
let added = f.dir.join("growing.log");
std::fs::write(&added, "first line").unwrap();
let spam_until = std::time::Instant::now() + Duration::from_secs(3);
while std::time::Instant::now() < spam_until {
coord.update_paths(vec![added.clone()]);
std::thread::sleep(Duration::from_millis(100));
}
assert_eq!(
f.stored_mtime(&added),
None,
"applied while the path was still being re-sent"
);
wait_for("the settled insert", Duration::from_secs(20), || {
f.stored_mtime(&added).is_some()
});
coord.shutdown();
}
#[test] #[test]
fn update_paths_leaves_a_row_that_already_agrees_alone() { fn update_paths_leaves_a_row_that_already_agrees_alone() {
let f = Fixture::new(false); let f = Fixture::new(false);
@ -582,6 +612,38 @@ fn auto_mode_runs_initial_index_and_applies_watcher_events() {
coord.shutdown(); coord.shutdown();
} }
/// The published count follows the coordinator's own writes, and an idle
/// coordinator does not go back to recount: a row slipped in behind its back
/// stays unreported until something it does changes the index.
#[test]
fn the_file_count_is_refreshed_by_writes_not_by_time() {
let f = Fixture::new(true);
std::fs::write(f.dir.join("seed.txt"), "initial content").unwrap();
let coord = start_coord_watching(f.config.clone(), fast_watcher());
wait_for("initial auto index", Duration::from_secs(20), || {
coord.state().files == Some(1)
});
std::fs::write(f.dir.join("later.txt"), "arrived later").unwrap();
wait_for(
"the count to follow an incremental add",
Duration::from_secs(20),
|| coord.state().files == Some(2),
);
let conn = db::open_existing(&f.db.to_string_lossy(), true).unwrap();
conn.execute(
"INSERT INTO files (name, parent, size, mtime, type, content_state)
VALUES ('ghost.txt', '/nowhere/', 0, 0, 0, 3)",
[],
)
.unwrap();
drop(conn);
std::thread::sleep(Duration::from_secs(3));
assert_eq!(coord.state().files, Some(2), "an idle tick recounted");
coord.shutdown();
}
#[test] #[test]
fn auto_mode_reports_an_active_watcher() { fn auto_mode_reports_an_active_watcher() {
let f = Fixture::new(true); let f = Fixture::new(true);
@ -882,6 +944,33 @@ fn an_unreadable_path_is_not_treated_as_deleted() {
std::fs::remove_file(&present).ok(); std::fs::remove_file(&present).ok();
} }
/// Only what the watcher registers with restarts it; a theme or search
/// setting must leave a large tree's watches alone.
#[test]
fn only_watch_inputs_restart_the_watcher() {
let old = Config::default();
let differs = |edit: fn(&mut Config)| {
let mut new = old.clone();
edit(&mut new);
inner::watch_inputs_differ(&old, &new)
};
assert!(!differs(|_| {}));
assert!(!differs(|c| c.search.debounce_ms += 100));
assert!(!differs(|c| c.processing.batch_size += 1));
assert!(!differs(|c| c.indexing.reindex_interval_minutes += 1));
assert!(differs(|c| c
.paths
.indexing_paths
.push("/elsewhere".into())));
assert!(differs(
|c| c.indexing.include_hidden = !c.indexing.include_hidden
));
assert!(differs(
|c| c.indexing.follow_symlinks = !c.indexing.follow_symlinks
));
assert!(differs(|c| c.indexing.ignore_patterns.push("*.tmp".into())));
}
#[test] #[test]
fn enqueue_last_wins_and_rename_splits() { fn enqueue_last_wins_and_rename_splits() {
let mut pending = HashMap::new(); let mut pending = HashMap::new();

55
crates/quicksearch-core/src/db/repo.rs
View file

@ -275,13 +275,11 @@ impl DocDecoder {
const ZSTD_LEVEL: i32 = 3; const ZSTD_LEVEL: i32 = 3;
/// Reusable compression context for the `documents_text` sidecar; one per /// Reusable compression context for the `documents_text` sidecar; one per
/// batch. Measured (`benches/index.rs`, group `zstd_encode`). /// indexing run. Measured (`benches/index.rs`, group `zstd_encode`).
pub struct DocEncoder { pub struct DocEncoder {
ctx: zstd::bulk::Compressor<'static>, ctx: zstd::bulk::Compressor<'static>,
/// One row's compressed output, reused. `zstd`'s `WriteBuf` for `Vec` /// The last row's compressed output, reused, so the writer allocates
/// writes from offset 0 and sets the length, so a body cannot be /// nothing per document once it has grown.
/// compressed straight into a shared arena; it lands here and is copied
/// across, with no allocation once both buffers have grown.
row: Vec<u8>, row: Vec<u8>,
} }
@ -300,14 +298,12 @@ impl DocEncoder {
.map_err(|e| format!("zstd encode: {}", e)) .map_err(|e| format!("zstd encode: {}", e))
} }
/// Append the compressed form of `text` to `arena`, returning where it /// The compressed form of `text`, valid until the next call; `None` for an
/// landed, so a whole batch's bodies share one allocation instead of one /// empty body, which gets no `documents_text` row.
/// `Vec` per document. pub fn body(&mut self, text: &str) -> Result<Option<&[u8]>, String> {
pub fn encode_into( if text.is_empty() {
&mut self, return Ok(None);
text: &str, }
arena: &mut Vec<u8>,
) -> Result<std::ops::Range<usize>, String> {
self.row.clear(); self.row.clear();
// `compress_to_buffer` writes into the buffer's capacity and fails // `compress_to_buffer` writes into the buffer's capacity and fails
// rather than growing it, so the room has to be there first. // rather than growing it, so the room has to be there first.
@ -315,9 +311,7 @@ impl DocEncoder {
self.ctx self.ctx
.compress_to_buffer(text.as_bytes(), &mut self.row) .compress_to_buffer(text.as_bytes(), &mut self.row)
.map_err(|e| format!("zstd encode: {}", e))?; .map_err(|e| format!("zstd encode: {}", e))?;
let start = arena.len(); Ok(Some(&self.row))
arena.extend_from_slice(&self.row);
Ok(start..arena.len())
} }
} }
@ -739,10 +733,16 @@ pub fn rows_in_range_page(
/// Reset every failed file to pending and drop the failure records, so the /// Reset every failed file to pending and drop the failure records, so the
/// coming run re-attempts them. Returns how many rows were reset. /// coming run re-attempts them. Returns how many rows were reset.
///
/// Found through `failed_files`, which holds exactly the failed rows (see
/// [`STATE_FAILED`]): no index covers `content_state = 2`, so asking `files`
/// scanned the table every run. Measured at 373k rows, 287 failed: 16 →
/// 0.3 ms with the table cached.
pub fn retry_failed_files(tx: &Transaction<'_>) -> Result<usize, String> { pub fn retry_failed_files(tx: &Transaction<'_>) -> Result<usize, String> {
let reset = exec( let reset = exec(
tx, tx,
"UPDATE files SET content_state = ?1 WHERE content_state = ?2", "UPDATE files SET content_state = ?1 \
WHERE id IN (SELECT file_id FROM failed_files) AND content_state = ?2",
params![STATE_PENDING, STATE_FAILED], params![STATE_PENDING, STATE_FAILED],
|| "reset failed files to pending".to_string(), || "reset failed files to pending".to_string(),
)?; )?;
@ -895,9 +895,22 @@ pub fn analyzed_file_count(conn: &Connection) -> Option<i64> {
rows.max() rows.max()
} }
/// Checkpoint, VACUUM, `PRAGMA optimize`, checkpoint; the trailing checkpoint /// Create `idx_files_hash` if it is missing: after a fresh index's first run,
/// empties what VACUUM's copy-back and `optimize` wrote to the log. Returns /// in one sorted build. It is not in [`crate::db::schema::SCHEMA_CURRENT`]:
/// whether it vacuumed. /// a SHA-256 key lands on a random leaf, so kept up through a bulk load, every
/// walk commit rewrote hundreds of the index's pages. Measured (`memprobe
/// cold`): bytes written 4.77 → 2.87 GB over `/usr` (373k files), 938 → 855
/// MiB over the cargo registry (41k), CPU within noise, and about 10 MiB more
/// peak for the sort. Only [`crate::search::find_duplicate_groups`] reads it,
/// and until the build it sorts the table instead.
pub fn build_hash_index(conn: &Connection) -> Result<(), String> {
conn.execute_batch("CREATE INDEX IF NOT EXISTS idx_files_hash ON files(hash);")
.map_err(|e| format!("build hash index: {}", e))
}
/// Checkpoint, [`build_hash_index`], VACUUM, `PRAGMA optimize`, checkpoint;
/// the trailing checkpoint empties what VACUUM's copy-back and `optimize`
/// wrote to the log. Returns whether it vacuumed.
/// ///
/// Run on a connection from [`crate::db::open::open_maintenance`], never the /// Run on a connection from [`crate::db::open::open_maintenance`], never the
/// indexer's. VACUUM's temporary database goes in the directory of `db_path`, /// indexer's. VACUUM's temporary database goes in the directory of `db_path`,
@ -922,6 +935,8 @@ pub fn maintain(conn: &Connection, db_path: &str) -> Result<bool, String> {
#[cfg(feature = "probe")] #[cfg(feature = "probe")]
probe.step("maintain checkpoint"); probe.step("maintain checkpoint");
build_hash_index(conn)?;
let page_count = pragma_number(conn, "page_count")?; let page_count = pragma_number(conn, "page_count")?;
let freelist = pragma_number(conn, "freelist_count")?; let freelist = pragma_number(conn, "freelist_count")?;

53
crates/quicksearch-core/src/db/repo_tests.rs
View file

@ -224,29 +224,18 @@ fn a_compressed_body_carries_its_uncompressed_length() {
} }
} }
/// Several rows into one arena, as the writer does. `zstd`'s `WriteBuf` for /// Several rows through one encoder, as the writer does: each body must come
/// `Vec` writes from offset zero and sets the length, so compressing straight /// back byte-identical from the slice handed out for it, whatever the size
/// into a shared arena would overwrite the previous row. Each body must come /// of the rows before it, and an empty body gets none.
/// back byte-identical out of its own range.
#[test] #[test]
fn an_arena_keeps_every_row_it_is_given() { fn one_encoder_serves_every_row_it_is_given() {
let mut enc = DocEncoder::new().unwrap(); let mut enc = DocEncoder::new().unwrap();
let bodies = [ let long = "a considerably longer second document ".repeat(512);
"the first document", for text in ["the first document", "", long.as_str(), "third"] {
"", let Some(blob) = enc.body(text).unwrap() else {
"a considerably longer second document ".repeat(512).as_str(), assert!(text.is_empty(), "only an empty body has no blob");
"third", continue;
] };
.map(str::to_string);
let mut arena = Vec::new();
let mut ranges = Vec::new();
for text in &bodies {
ranges.push(enc.encode_into(text, &mut arena).unwrap());
}
for (text, at) in bodies.iter().zip(&ranges) {
let blob = &arena[at.clone()];
assert_eq!( assert_eq!(
raw_text_len(blob), raw_text_len(blob),
Some(text.len() as u64), Some(text.len() as u64),
@ -254,28 +243,10 @@ fn an_arena_keeps_every_row_it_is_given() {
); );
assert_eq!( assert_eq!(
DocDecoder::new().unwrap().decode(blob), DocDecoder::new().unwrap().decode(blob),
Some(text.as_str()), Some(text),
"a row did not survive sharing the arena" "a row did not survive the reused buffer"
); );
} }
// The ranges tile the arena in order: a gap or an overlap means one row
// landed on another.
let mut next = 0;
for at in &ranges {
assert_eq!(at.start, next, "rows must be contiguous");
next = at.end;
}
assert_eq!(next, arena.len(), "the arena holds exactly the four bodies");
// Reuse: a second batch must not read the first one's bytes.
arena.clear();
let at = enc.encode_into("a fresh batch", &mut arena).unwrap();
assert_eq!(at.start, 0);
assert_eq!(
DocDecoder::new().unwrap().decode(&arena[at]),
Some("a fresh batch")
);
} }
#[test] #[test]

4
crates/quicksearch-core/src/db/schema.rs
View file

@ -177,6 +177,9 @@ pub const PRAGMAS_WALK_READER: &str = "
/// filename query already does: `searchtime` at 300k rows, 25 ms with or /// filename query already does: `searchtime` at 300k rows, 25 ms with or
/// without a filter, where the indexes had made a one-day filter 0.06 ms. /// without a filter, where the indexes had made a one-day filter 0.06 ms.
/// ///
/// `idx_files_hash` is left out on purpose and built after the first run; see
/// [`crate::db::repo::build_hash_index`].
///
/// `documents_text` holds a row only when the extractor produced text (an /// `documents_text` holds a row only when the extractor produced text (an
/// audio file with empty tags is done but has none), so `files.content_state` /// audio file with empty tags is done but has none), so `files.content_state`
/// is the authority on "content indexed". The uncompressed length is not /// is the authority on "content indexed". The uncompressed length is not
@ -203,7 +206,6 @@ CREATE TABLE files (
); );
CREATE UNIQUE INDEX idx_files_parent ON files(parent, name); CREATE UNIQUE INDEX idx_files_parent ON files(parent, name);
CREATE INDEX idx_files_hash ON files(hash);
CREATE INDEX idx_files_content_pending ON files(id) WHERE content_state = 0; CREATE INDEX idx_files_content_pending ON files(id) WHERE content_state = 0;
CREATE TABLE failed_files ( CREATE TABLE failed_files (

18
crates/quicksearch-core/src/extract/audio.rs
View file

@ -4,6 +4,7 @@
use std::path::Path; use std::path::Path;
use lofty::{ use lofty::{
config::ParseOptions,
file::TaggedFileExt, file::TaggedFileExt,
probe::Probe, probe::Probe,
tag::{Accessor, ItemKey}, tag::{Accessor, ItemKey},
@ -24,8 +25,14 @@ impl Extractor for AudioExtractor {
out: &mut String, out: &mut String,
_scratch: &mut Scratch, _scratch: &mut Scratch,
) -> Result<(), ExtractError> { ) -> Result<(), ExtractError> {
// Tags only: the audio properties (duration, bitrate) cost a seek to
// the end and, in an MP3 without a Xing header, a frame-sync search
// back from it, and fail a file whose audio is cut short. Measured
// (`extractprobe`, 18 generated 3-minute MP3s and 40 Ogg sounds):
// `lseek` + `read` calls 2,567 → 790, cached throughput 1.4–2.3x.
let tagged = Probe::open(path) let tagged = Probe::open(path)
.map_err(|e| format!("lofty probe {}: {}", path.display(), e))? .map_err(|e| format!("lofty probe {}: {}", path.display(), e))?
.options(ParseOptions::new().read_properties(false))
.read() .read()
.map_err(|e| format!("lofty read {}: {}", path.display(), e))?; .map_err(|e| format!("lofty read {}: {}", path.display(), e))?;
@ -125,6 +132,17 @@ mod tests {
assert_eq!(out, "Blue Monday New Order Power Corruption Synthpop"); assert_eq!(out, "Blue Monday New Order Power Corruption Synthpop");
} }
/// A download cut short keeps its tag: the audio after it is never read.
#[test]
fn a_truncated_stream_still_yields_its_tags() {
let path = write_mp3("audio-truncated", &[("TIT2", "Half Written")]);
let bytes = std::fs::read(&path).unwrap();
// The tag, the first frame's header and a few bytes of its body.
let tag_end = bytes.len() - 4 * 417;
std::fs::write(&path, &bytes[..tag_end + 10]).unwrap();
assert_eq!(extract(&path).expect("extract"), "Half Written");
}
#[test] #[test]
fn an_untagged_file_yields_empty_text() { fn an_untagged_file_yields_empty_text() {
let path = write_mp3("audio-untagged", &[]); let path = write_mp3("audio-untagged", &[]);

28
crates/quicksearch-core/src/extract/mod.rs
View file

@ -72,12 +72,10 @@ impl Limits {
pub struct Scratch { pub struct Scratch {
/// The head bytes a walk worker hashes and sniffs each file from. /// The head bytes a walk worker hashes and sniffs each file from.
head: Vec<u8>, head: Vec<u8>,
/// One container member or one whole file, for a parser that will not /// One whole file, for a parser that will not take a reader. Grows to
/// take a reader. Grows to the largest member the worker has met and /// the largest file the worker has met and stays there.
/// stays there, bounded by [`Limits::inflate`].
bytes: Vec<u8>, bytes: Vec<u8>,
/// quick-xml's per-event buffer. Separate from `bytes` because a /// quick-xml's per-event buffer.
/// container walk holds both at once.
events: Vec<u8>, events: Vec<u8>,
/// XLSX's shared-string table. Cleared between files; the entries keep /// XLSX's shared-string table. Cleared between files; the entries keep
/// their capacity, which is most of what a table costs. /// their capacity, which is most of what a table costs.
@ -110,24 +108,22 @@ impl Scratch {
&self.head &self.head
} }
/// The raw-bytes buffer: one container member, one OLE stream, or one /// The raw-bytes buffer: one whole file for a format whose parser will
/// whole file for a format whose parser will not take a reader. The /// not take a reader. The caller clears it before filling.
/// caller clears it before filling.
pub(crate) fn bytes_mut(&mut self) -> &mut Vec<u8> { pub(crate) fn bytes_mut(&mut self) -> &mut Vec<u8> {
&mut self.bytes &mut self.bytes
} }
/// The member and event buffers together, which a container walk holds /// quick-xml's event buffer, for a container's XML members.
/// at once; two `&mut self` accessors could not lend both. pub(crate) fn events(&mut self) -> &mut Vec<u8> {
pub(crate) fn container_bufs(&mut self) -> (&mut Vec<u8>, &mut Vec<u8>) { &mut self.events
(&mut self.bytes, &mut self.events)
} }
/// The same pair plus the shared-string table, cleared, which XLSX needs /// The event buffer plus the shared-string table, cleared, which XLSX
/// all at once. /// needs at once; two `&mut self` accessors could not lend both.
pub(crate) fn xlsx_bufs(&mut self) -> (&mut Vec<u8>, &mut Vec<u8>, &mut Vec<String>) { pub(crate) fn xlsx_bufs(&mut self) -> (&mut Vec<u8>, &mut Vec<String>) {
self.strings.clear(); self.strings.clear();
(&mut self.bytes, &mut self.events, &mut self.strings) (&mut self.events, &mut self.strings)
} }
} }

125
crates/quicksearch-core/src/extract/office.rs
View file

@ -218,59 +218,53 @@ fn open_container(path: &Path) -> Result<Archive, Box<dyn Error>> {
Ok(ZipArchive::new(BufReader::new(File::open(path)?))?) Ok(ZipArchive::new(BufReader::new(File::open(path)?))?)
} }
/// Inflate one whole member into `buf`, the worker's buffer, reused across /// One member, inflated as the parser reads it, bounded by `limit`
/// members and files so members cost no allocation after the first document. /// ([`super::Limits::inflate`]): inflation stops where parsing does, at the
/// Bounded by `limit` ([`super::Limits::inflate`]); the buffer never shrinks, /// text limit, and nothing holds a whole member. Measured against inflating
/// so one hostile document leaves the worker holding up to `limit` for the /// each member into a per-worker buffer that kept its largest size
/// rest of the pass. /// (`extractprobe`, 1.25 MB `document.xml` members): peak live per worker
fn member_bytes<R: Read + Seek>( /// 2.3 → 0.4 MiB, VmHWM 20.5 → 14.4 MiB, throughput unchanged.
archive: &mut ZipArchive<R>, fn member<'a, R: Read + Seek>(
archive: &'a mut ZipArchive<R>,
name: &str, name: &str,
limit: usize, limit: usize,
buf: &mut Vec<u8>, ) -> Result<impl BufRead + 'a, Box<dyn Error>> {
) -> Result<(), Box<dyn Error>> { Ok(BufReader::new(archive.by_name(name)?.take(limit as u64)))
buf.clear();
archive
.by_name(name)?
.take(limit as u64)
.read_to_end(buf)?;
Ok(())
} }
/// Names of the `.xml` members under `prefix`, in archive order, not /// Names of the `.xml` members under `prefix`, in number order: shorter
/// `file_names()`, which iterates a hash map and would shuffle the slides of /// first, so `slide2.xml` comes before `slide10.xml`. From the central
/// every presentation. /// directory the archive already holds; naming members through `by_index`
fn xml_members_under<R: Read + Seek>( /// seeks to and reads each one's local header, about 3 `lseek` and 2 reads a
archive: &mut ZipArchive<R>, /// member, twice for a deck. Measured (`extractprobe`, 20 copies of a real
prefix: &str, /// 86-member deck plus 4 other documents): `lseek` + `read` calls 22,289 →
) -> Result<Vec<String>, Box<dyn Error>> { /// 4,444, throughput 1,800 → 2,330 files/s.
let mut names = Vec::new(); fn xml_members_under<R: Read + Seek>(archive: &ZipArchive<R>, prefix: &str) -> Vec<String> {
for i in 0..archive.len() { let mut names: Vec<String> = archive
let name = archive.by_index(i)?.name().to_string(); .file_names()
if name.starts_with(prefix) && name.ends_with(".xml") { .filter(|name| name.starts_with(prefix) && name.ends_with(".xml"))
names.push(name); .map(str::to_string)
} .collect();
} names.sort_by(|a, b| a.len().cmp(&b.len()).then_with(|| a.cmp(b)));
Ok(names) names
} }
/// A format whose whole text lives in one member under one spec. /// A format whose whole text lives in one member under one spec.
fn single_member( fn single_member(
path: &Path, path: &Path,
member: &str, name: &str,
spec: &TextSpec, spec: &TextSpec,
out: &mut String, out: &mut String,
scratch: &mut Scratch, scratch: &mut Scratch,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
let limits = scratch.limits(); let limits = scratch.limits();
let mut archive = open_container(path)?; let mut archive = open_container(path)?;
let (bytes, events) = scratch.container_bufs(); let xml = member(&mut archive, name, limits.inflate)?;
member_bytes(&mut archive, member, limits.inflate, bytes)?; collect_xml_text(xml, spec, out, limits.text, scratch.events())
collect_xml_text(bytes.as_slice(), spec, out, limits.text, events)
} }
/// Concatenate what `collect` gets out of each `.xml` member under `prefix`, /// Concatenate what `collect` gets out of each `.xml` member under `prefix`,
/// in archive order, stopping once the text reaches [`super::Limits::text`], /// in number order, stopping once the text reaches [`super::Limits::text`],
/// the point past which the caller would discard it anyway. /// the point past which the caller would discard it anyway.
fn collect_members( fn collect_members(
archive: &mut Archive, archive: &mut Archive,
@ -279,7 +273,7 @@ fn collect_members(
limit: usize, limit: usize,
mut collect: impl FnMut(&mut Archive, &str, &mut String) -> Result<(), Box<dyn Error>>, mut collect: impl FnMut(&mut Archive, &str, &mut String) -> Result<(), Box<dyn Error>>,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
for name in xml_members_under(archive, prefix)? { for name in xml_members_under(archive, prefix) {
if out.len() >= limit { if out.len() >= limit {
break; break;
} }
@ -300,15 +294,15 @@ fn collect_members(
fn extract_pptx(path: &Path, out: &mut String, scratch: &mut Scratch) -> Result<(), Box<dyn Error>> { fn extract_pptx(path: &Path, out: &mut String, scratch: &mut Scratch) -> Result<(), Box<dyn Error>> {
let limits = scratch.limits(); let limits = scratch.limits();
let mut archive = open_container(path)?; let mut archive = open_container(path)?;
let (bytes, events) = scratch.container_bufs(); let events = scratch.events();
collect_members( collect_members(
&mut archive, &mut archive,
"ppt/slides/slide", "ppt/slides/slide",
out, out,
limits.text, limits.text,
|archive, name, out| { |archive, name, out| {
member_bytes(archive, name, limits.inflate, bytes)?; let xml = member(archive, name, limits.inflate)?;
collect_xml_text(bytes.as_slice(), &PPTX, out, limits.text, events)?; collect_xml_text(xml, &PPTX, out, limits.text, events)?;
out.push_str("\n--- New Slide ---\n"); out.push_str("\n--- New Slide ---\n");
Ok(()) Ok(())
}, },
@ -321,9 +315,9 @@ fn extract_pptx(path: &Path, out: &mut String, scratch: &mut Scratch) -> Result<
out, out,
limits.text, limits.text,
|archive, name, out| { |archive, name, out| {
member_bytes(archive, name, limits.inflate, bytes)?; let xml = member(archive, name, limits.inflate)?;
out.push_str("\n--- Speaker Notes ---\n"); out.push_str("\n--- Speaker Notes ---\n");
collect_xml_text(bytes.as_slice(), &PPTX, out, limits.text, events) collect_xml_text(xml, &PPTX, out, limits.text, events)
}, },
) )
} }
@ -338,14 +332,13 @@ fn extract_pptx(path: &Path, out: &mut String, scratch: &mut Scratch) -> Result<
fn shared_strings<R: Read + Seek>( fn shared_strings<R: Read + Seek>(
archive: &mut ZipArchive<R>, archive: &mut ZipArchive<R>,
limit: usize, limit: usize,
bytes: &mut Vec<u8>,
events: &mut Vec<u8>, events: &mut Vec<u8>,
strings: &mut Vec<String>, strings: &mut Vec<String>,
) { ) {
if member_bytes(archive, "xl/sharedStrings.xml", limit, bytes).is_err() { let Ok(xml) = member(archive, "xl/sharedStrings.xml", limit) else {
return; return;
} };
let mut reader = Reader::from_reader(bytes.as_slice()); let mut reader = Reader::from_reader(xml);
// No `trim_text`; see `collect_xml_text`. // No `trim_text`; see `collect_xml_text`.
events.clear(); events.clear();
let mut in_text = false; let mut in_text = false;
@ -481,18 +474,18 @@ fn collect_sheet<R: BufRead>(
fn extract_xlsx(path: &Path, out: &mut String, scratch: &mut Scratch) -> Result<(), Box<dyn Error>> { fn extract_xlsx(path: &Path, out: &mut String, scratch: &mut Scratch) -> Result<(), Box<dyn Error>> {
let limits = scratch.limits(); let limits = scratch.limits();
let mut archive = open_container(path)?; let mut archive = open_container(path)?;
// All three at once: the sheets are read while the table is live, and // Both at once: the sheets are read while the table is live, and
// separate `&mut scratch` borrows cannot overlap. // separate `&mut scratch` borrows cannot overlap.
let (bytes, events, strings) = scratch.xlsx_bufs(); let (events, strings) = scratch.xlsx_bufs();
shared_strings(&mut archive, limits.inflate, bytes, events, strings); shared_strings(&mut archive, limits.inflate, events, strings);
collect_members( collect_members(
&mut archive, &mut archive,
"xl/worksheets/sheet", "xl/worksheets/sheet",
out, out,
limits.text, limits.text,
|archive, name, out| { |archive, name, out| {
member_bytes(archive, name, limits.inflate, bytes)?; let xml = member(archive, name, limits.inflate)?;
collect_sheet(bytes.as_slice(), strings, out, limits.text, events) collect_sheet(xml, strings, out, limits.text, events)
}, },
) )
} }
@ -796,7 +789,7 @@ mod tests {
</sheetData></worksheet>"; </sheetData></worksheet>";
#[test] #[test]
fn pptx_marks_each_slide_and_keeps_archive_order() { fn pptx_marks_each_slide() {
let p = container( let p = container(
"pptx", "pptx",
"pptx", "pptx",
@ -811,6 +804,36 @@ mod tests {
); );
} }
/// Slides come out in number order whatever order the archive stores
/// them in, and `slide10` is not read between `slide1` and `slide2`.
#[test]
fn pptx_slides_come_out_in_number_order() {
let slide = |text: &str| {
format!(
"<p:sld><p:cSld><p:spTree><p:sp><p:txBody>\
<a:p><a:r><a:t>{text}</a:t></a:r></a:p>\
</p:txBody></p:sp></p:spTree></p:cSld></p:sld>"
)
};
let (s10, s2, s1) = (slide("ten"), slide("two"), slide("one"));
let p = container(
"pptx-order",
"pptx",
&[
("ppt/slides/slide10.xml", s10.as_str()),
("ppt/slides/slide2.xml", s2.as_str()),
("ppt/slides/slide1.xml", s1.as_str()),
],
);
let text = extract_document_text(&p, "pptx").unwrap();
let slides: Vec<&str> = text
.split("\n--- New Slide ---\n")
.map(str::trim)
.filter(|s| !s.is_empty())
.collect();
assert_eq!(slides, ["one", "two", "ten"]);
}
/// Speaker notes live in parts of their own that the slide text never /// Speaker notes live in parts of their own that the slide text never
/// mentions. /// mentions.
#[test] #[test]

5
crates/quicksearch-core/src/extract/plaintext.rs
View file

@ -56,6 +56,11 @@ pub struct PlaintextExtractor;
/// Read `size` bytes from `f`, never more than `cap`. A short read is not an /// Read `size` bytes from `f`, never more than `cap`. A short read is not an
/// error: a file that shrank keeps its prefix. /// error: a file that shrank keeps its prefix.
///
/// Zero-filled, and still the faster shape: `take(size).read_to_end` into a
/// reserved `Vec` skips the fill but starts at 8 KiB reads and doubles, so
/// over `/usr/include` it made 18,431 `read` calls against 15,419 here and
/// ran 8% slower (`extractprobe`). Measured and rejected.
fn read_sized(f: &mut File, size: usize, cap: usize, path: &Path) -> Result<Vec<u8>, ExtractError> { fn read_sized(f: &mut File, size: usize, cap: usize, path: &Path) -> Result<Vec<u8>, ExtractError> {
let size = size.min(cap); let size = size.min(cap);
let mut buf = vec![0u8; size]; let mut buf = vec![0u8; size];

252
crates/quicksearch-core/src/file_handling/batch.rs
View file

@ -10,112 +10,53 @@ use super::*;
use crate::config::Config; use crate::config::Config;
use crate::db::repo; use crate::db::repo;
/// One writer's compressed sidecars: a batch's blobs end to end in `arena`, /// Write one batch of prepared records in one transaction through
/// with `slots[i]` saying where row `i`'s is, that it has none, or that its /// `write_row`, storing any inline text on the row, which both row writers
/// compression failed (kept per row rather than failing the batch). /// leave pending. `enc` is the run's one encoder (see [`store_extracted`]).
/// ///
/// One per indexing run, reused by every batch and turn the writer takes: the /// Writing an inline row as done outright would save the claim's `UPDATE`,
/// encoder because a zstd context per chunk is wasted CPU (`benches/index.rs`, /// but it would be a second way out of pending beside
/// `zstd_encode`), the arena so the writer allocates nothing per document. /// [`repo::complete_pending`], for 0.5–1.1 µs a row: about 0.1% of a cold
/// Measured against one per writer turn (`indexprobe cold`, 10k docs of /// run's CPU, below `memprobe`'s noise. Measured and rejected.
/// 9–20 KiB): VmHWM 60–65 → 44–50 MiB, CPU ~2% lower.
pub struct Bodies {
enc: repo::DocEncoder,
arena: Vec<u8>,
slots: Vec<Result<Option<std::ops::Range<usize>>, String>>,
}
impl Bodies {
pub fn new() -> Result<Bodies, String> {
Ok(Bodies {
enc: repo::DocEncoder::new()?,
arena: Vec::new(),
slots: Vec::new(),
})
}
/// Compress one chunk's bodies before the caller takes the connection, so
/// the lock covers only the SQL.
fn fill<'a>(&mut self, texts: impl Iterator<Item = Option<&'a str>>) {
self.arena.clear();
self.slots.clear();
for text in texts {
let slot = match text {
Some(t) if !t.is_empty() => self.enc.encode_into(t, &mut self.arena).map(Some),
_ => Ok(None),
};
self.slots.push(slot);
}
}
/// Row `i`'s blob, or why there is none.
fn get(&self, i: usize) -> Result<Option<&[u8]>, &str> {
match &self.slots[i] {
Ok(Some(at)) => Ok(Some(&self.arena[at.clone()])),
Ok(None) => Ok(None),
Err(e) => Err(e),
}
}
}
/// The skeleton behind [`process_batch_updates`] and
/// [`process_batch_inserts`]: compress each chunk's bodies outside the lock,
/// write its rows in one transaction through `write_row`, and store any
/// inline text on the row, which both row writers leave pending.
fn write_prepared_records( fn write_prepared_records(
conn_mutex: &Arc<Mutex<Connection>>, conn_mutex: &Arc<Mutex<Connection>>,
records: &[OwnedNewFile], records: &[OwnedNewFile],
stop_flag: &Arc<AtomicBool>, stop_flag: &Arc<AtomicBool>,
config: &Config, enc: &mut repo::DocEncoder,
bodies: &mut Bodies,
write_row: impl Fn(&rusqlite::Transaction<'_>, &OwnedNewFile) -> Result<Option<i64>, String>, write_row: impl Fn(&rusqlite::Transaction<'_>, &OwnedNewFile) -> Result<Option<i64>, String>,
) -> Result<(), String> { ) -> Result<(), String> {
// `.max(1)`: `chunks(0)` panics on the indexing thread, so a hand-edited let conn = crate::lock_ok(conn_mutex);
// `batch_size = 0` would wedge indexing while the UI reads "Running". let tx = conn
let chunk_size = config.processing.batch_size.max(1); .unchecked_transaction()
for batch in records.chunks(chunk_size) { .map_err(|e| format!("Failed to begin transaction: {}", e))?;
for rec in records {
if stop_flag.load(Ordering::Relaxed) { if stop_flag.load(Ordering::Relaxed) {
return Ok(()); return Ok(());
} }
let id = write_row(&tx, rec)?;
// Outside the lock; see `Bodies::fill`. if let (Some(id), Some(text)) = (id, rec.inline_text.as_deref()) {
bodies.fill(batch.iter().map(|r| r.inline_text.as_deref())); match enc.body(text) {
let conn = crate::lock_ok(conn_mutex); Ok(zstd) => {
let tx = conn repo::store_pending_text(&tx, id, text, zstd)?;
.unchecked_transaction()
.map_err(|e| format!("Failed to begin transaction: {}", e))?;
for (i, rec) in batch.iter().enumerate() {
if stop_flag.load(Ordering::Relaxed) {
return Ok(());
}
let id = write_row(&tx, rec)?;
if let (Some(id), Some(text)) = (id, rec.inline_text.as_deref()) {
match bodies.get(i) {
Ok(zstd) => {
repo::store_pending_text(&tx, id, text, zstd)?;
}
Err(e) => crate::log_warn!("compress text for {}: {}", rec.path(), e),
} }
Err(e) => crate::log_warn!("compress text for {}: {}", rec.path(), e),
} }
} }
tx.commit()
.map_err(|e| format!("Failed to commit transaction: {}", e))?;
} }
Ok(()) tx.commit()
.map_err(|e| format!("Failed to commit transaction: {}", e))
} }
/// Write already-prepared records for files whose content changed. No /// Write one batch of already-prepared records for files whose content
/// filesystem I/O; records carrying inline text are stored complete here, /// changed. No filesystem I/O; records carrying inline text are stored
/// the rest stay pending. /// complete here, the rest stay pending.
pub fn process_batch_updates( pub fn process_batch_updates(
conn_mutex: &Arc<Mutex<Connection>>, conn_mutex: &Arc<Mutex<Connection>>,
files_to_update: &[OwnedNewFile], files_to_update: &[OwnedNewFile],
stop_flag: &Arc<AtomicBool>, stop_flag: &Arc<AtomicBool>,
config: &Config, enc: &mut repo::DocEncoder,
bodies: &mut Bodies,
) -> Result<(), String> { ) -> Result<(), String> {
if files_to_update.is_empty() { if files_to_update.is_empty() {
return Ok(()); return Ok(());
@ -124,45 +65,37 @@ pub fn process_batch_updates(
// The row leaves this closure pending whenever it carries inline text: // The row leaves this closure pending whenever it carries inline text:
// `update_file_basic` cleared a DONE row's content and reset its state // `update_file_basic` cleared a DONE row's content and reset its state
// in this same transaction, and the insert fallback created it pending. // in this same transaction, and the insert fallback created it pending.
write_prepared_records( write_prepared_records(conn_mutex, files_to_update, stop_flag, enc, |tx, rec| {
conn_mutex, let updated = repo::update_file_basic(tx, &rec.as_new_file()).map_err(|e| {
files_to_update, format!(
stop_flag, "Failed to update file record + clear stale content for {}: {}",
config, rec.path(),
bodies, e
|tx, rec| { )
let updated = repo::update_file_basic(tx, &rec.as_new_file()).map_err(|e| { })?;
format!(
"Failed to update file record + clear stale content for {}: {}",
rec.path(),
e
)
})?;
// No row matched: dropping the update would leave the mtime stale // No row matched: dropping the update would leave the mtime stale
// and the file re-hashed on every run forever. // and the file re-hashed on every run forever.
match updated { match updated {
None => { None => {
crate::log_warn!( crate::log_warn!(
"no indexed row matched {} during update; inserting instead", "no indexed row matched {} during update; inserting instead",
rec.path() rec.path()
); );
repo::insert_file(tx, &rec.as_new_file()) repo::insert_file(tx, &rec.as_new_file())
.map_err(|e| format!("Failed to insert file record: {}", e)) .map_err(|e| format!("Failed to insert file record: {}", e))
}
some => Ok(some),
} }
}, some => Ok(some),
) }
})
} }
/// Write already-prepared records for newly discovered files. /// Write one batch of already-prepared records for newly discovered files.
pub fn process_batch_inserts( pub fn process_batch_inserts(
conn_mutex: &Arc<Mutex<Connection>>, conn_mutex: &Arc<Mutex<Connection>>,
files_to_insert: &[OwnedNewFile], files_to_insert: &[OwnedNewFile],
stop_flag: &Arc<AtomicBool>, stop_flag: &Arc<AtomicBool>,
config: &Config, enc: &mut repo::DocEncoder,
bodies: &mut Bodies,
) -> Result<(), String> { ) -> Result<(), String> {
if files_to_insert.is_empty() { if files_to_insert.is_empty() {
return Ok(()); return Ok(());
@ -170,17 +103,10 @@ pub fn process_batch_inserts(
// `insert_file` returned `Some` only by creating the row, pending when it // `insert_file` returned `Some` only by creating the row, pending when it
// needs content, so it cannot carry content from anywhere. // needs content, so it cannot carry content from anywhere.
write_prepared_records( write_prepared_records(conn_mutex, files_to_insert, stop_flag, enc, |tx, rec| {
conn_mutex, repo::insert_file(tx, &rec.as_new_file())
files_to_insert, .map_err(|e| format!("Failed to insert file record: {}", e))
stop_flag, })
config,
bodies,
|tx, rec| {
repo::insert_file(tx, &rec.as_new_file())
.map_err(|e| format!("Failed to insert file record: {}", e))
},
)
} }
/// Delete the rows a completed run found no file behind; returns how many /// Delete the rows a completed run found no file behind; returns how many
@ -200,6 +126,10 @@ pub fn process_batch_inserts(
/// Extraction may still be running (its writes serialize on this connection /// Extraction may still be running (its writes serialize on this connection
/// mutex), so between chunks a doomed row can flip PENDING→DONE; resolved any /// mutex), so between chunks a doomed row can flip PENDING→DONE; resolved any
/// earlier, its posting would outlive the row. /// earlier, its posting would outlive the row.
///
/// Not worth folding the lookup into the delete (`DELETE … RETURNING` per
/// path): the lookups are 35–52 ms of 1.4–1.5 s over the 40k paths above,
/// where the commit, FTS5 writing its tombstones, is 1.0–1.2 s.
pub fn cleanup_stale_index_entries( pub fn cleanup_stale_index_entries(
conn_mutex: &Arc<Mutex<Connection>>, conn_mutex: &Arc<Mutex<Connection>>,
stale_paths: &[String], stale_paths: &[String],
@ -357,11 +287,6 @@ pub fn count_extract_scope(
}) })
} }
/// Rows per compression chunk and per transaction inside [`store_extracted`]:
/// half of what a writer turn may hand in (`pipeline::READY_TOPUP` is 64), so
/// a full turn commits twice and each hold of the connection stays short.
const STORE_CHUNK: usize = 32;
/// What one [`store_extracted`] call did with the rows it was handed. /// What one [`store_extracted`] call did with the rows it was handed.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct Stored { pub struct Stored {
@ -373,57 +298,56 @@ pub struct Stored {
} }
/// Write already-extracted rows, all that runs with the connection held, /// Write already-extracted rows, all that runs with the connection held,
/// until `deadline`. FTS5 tokenization happens here: the writer thread's /// until `deadline`, in one transaction. FTS5 tokenization happens here: the
/// dominant cost. A turn overruns the deadline by at most one /// writer thread's dominant cost. A turn overruns the deadline by at most one
/// document; at least one row is always consumed unless the run is stopped, /// document; at least one row is always consumed unless the run is stopped,
/// so a caller looping on this cannot spin. A row whose write fails is /// so a caller looping on this cannot spin. A row whose write fails is
/// logged and consumed; its `content_state` stays pending for the next run. /// logged and consumed; its `content_state` stays pending for the next run.
///
/// `enc` is the run's one encoder, compressing each body as its row is
/// written: a zstd context per call is wasted CPU (`benches/index.rs`,
/// `zstd_encode`), and nothing else takes the connection during a run but
/// `stop_indexing`'s checkpoint, so compressing outside the lock bought
/// nothing. Measured against compressing each 32-row chunk into an arena
/// first and committing per chunk (`memprobe cold`): bytes written 855 → 825
/// MiB on the cargo registry (41k files) and 2.88 → 2.81 GB on `/usr` (373k),
/// peak RSS 180–200 → 174–176 MiB, CPU within noise.
pub fn store_extracted( pub fn store_extracted(
conn_mutex: &Arc<Mutex<Connection>>, conn_mutex: &Arc<Mutex<Connection>>,
rows: &[crate::content::ExtractedRow], rows: &[crate::content::ExtractedRow],
stop_flag: &Arc<AtomicBool>, stop_flag: &Arc<AtomicBool>,
deadline: std::time::Instant, deadline: std::time::Instant,
bodies: &mut Bodies, enc: &mut repo::DocEncoder,
) -> Result<Stored, String> { ) -> Result<Stored, String> {
let mut done = Stored::default(); let mut done = Stored::default();
for chunk in rows.chunks(STORE_CHUNK) { if stop_flag.load(Ordering::Relaxed) {
if stop_flag.load(Ordering::Relaxed) { return Ok(done);
break; }
} let conn = crate::lock_ok(conn_mutex);
// Outside the lock; see `Bodies::fill`. let tx = conn
bodies.fill( .unchecked_transaction()
chunk .map_err(|e| format!("Failed to begin transaction: {}", e))?;
.iter() for row in rows {
.map(|r| crate::file_handling::outcome_body(&r.outcome)), // Counted before anything can skip it: a failed row still leaves.
); done.consumed += 1;
let conn = crate::lock_ok(conn_mutex); let body = crate::file_handling::outcome_body(&row.outcome).map(|text| enc.body(text));
let tx = conn match body.transpose() {
.unchecked_transaction() Err(e) => crate::log_warn!("compress text for {}: {}", row.path(), e),
.map_err(|e| format!("Failed to begin transaction: {}", e))?; Ok(zstd) => {
let mut cut = false; match repo::complete_pending(&tx, row.file_id, &row.outcome, zstd.flatten()) {
for (i, row) in chunk.iter().enumerate() {
// Counted before anything can skip it: a failed row still leaves.
done.consumed += 1;
match bodies.get(i) {
Err(e) => crate::log_warn!("compress text for {}: {}", row.path(), e),
Ok(zstd) => match repo::complete_pending(&tx, row.file_id, &row.outcome, zstd) {
Ok(true) => done.written += 1, Ok(true) => done.written += 1,
// No longer pending: stored or deleted since it was paged. // No longer pending: stored or deleted since it was paged.
Ok(false) => {} Ok(false) => {}
Err(e) => crate::log_warn!("content indexing for {}: {}", row.path(), e), Err(e) => crate::log_warn!("content indexing for {}: {}", row.path(), e),
}, }
}
if stop_flag.load(Ordering::Relaxed) || std::time::Instant::now() >= deadline {
cut = true;
break;
} }
} }
tx.commit() if stop_flag.load(Ordering::Relaxed) || std::time::Instant::now() >= deadline {
.map_err(|e| format!("Failed to commit transaction: {}", e))?;
if cut {
break; break;
} }
} }
tx.commit()
.map_err(|e| format!("Failed to commit transaction: {}", e))?;
Ok(done) Ok(done)
} }

3
crates/quicksearch-core/src/file_handling/count_and_extract_tests.rs
View file

@ -235,8 +235,7 @@ fn extract_scope_counts_only_files_an_extractor_claims() {
&conn_mutex, &conn_mutex,
&records, &records,
&stop, &stop,
&config, &mut crate::db::repo::DocEncoder::new().unwrap(),
&mut Bodies::new().unwrap(),
) )
.unwrap(); .unwrap();

4
crates/quicksearch-core/src/file_handling/mod.rs
View file

@ -16,7 +16,7 @@ pub(crate) use batch::extract_scope_prepare;
pub(crate) use batch::max_text_file_size; pub(crate) use batch::max_text_file_size;
pub use batch::{ pub use batch::{
cleanup_stale_index_entries, count_extract_scope, mark_oversize_pending_na, cleanup_stale_index_entries, count_extract_scope, mark_oversize_pending_na,
process_batch_inserts, process_batch_updates, store_extracted, Bodies, ExtractCursor, process_batch_inserts, process_batch_updates, store_extracted, ExtractCursor,
ExtractScope, RootRange, Stored, ExtractScope, RootRange, Stored,
}; };
pub use counting::count_tree_entries_fast; pub use counting::count_tree_entries_fast;
@ -29,7 +29,7 @@ pub(crate) use paths::{
}; };
pub(crate) use records::mtime_secs; pub(crate) use records::mtime_secs;
pub use records::{ pub use records::{
classify_by_mtime, classify_for_indexing, content_extractable, decide_content, classify_by_mtime, content_extractable, decide_content,
extract_and_store, fts_begin_bulk_write, fts_begin_tombstone_burst, fts_end_tombstone_burst, extract_and_store, fts_begin_bulk_write, fts_begin_tombstone_burst, fts_end_tombstone_burst,
fts_finalize_after_text_indexing, get_file_hash, hash_failure_counts, outcome_body, fts_finalize_after_text_indexing, get_file_hash, hash_failure_counts, outcome_body,
prepare_file_record, reset_run_warnings, ContentOutcome, DirRows, FileIndexAction, prepare_file_record, reset_run_warnings, ContentOutcome, DirRows, FileIndexAction,

12
crates/quicksearch-core/src/file_handling/records.rs
View file

@ -26,14 +26,10 @@ pub enum FileIndexAction {
Insert, Insert,
} }
/// Decide what Phase 1 should do with a file. A file whose parent is not this /// Decide what Phase 1 should do with a file from its stored mtime, `None`
/// directory (a resolved symlink target) must not be classified here: it /// when it has no row. For a resolved symlink target that is the row under
/// would read as `Insert`, and `INSERT OR IGNORE` would silently not update /// the target's own path: its link's directory rows would read it as
/// it. Use [`classify_by_mtime`] for those. /// `Insert`, and `INSERT OR IGNORE` would silently not update it.
pub fn classify_for_indexing(name: &str, mtime: u64, rows: &DirRows) -> FileIndexAction {
classify_by_mtime(rows.get(name).copied(), mtime)
}
pub fn classify_by_mtime(stored: Option<u64>, mtime: u64) -> FileIndexAction { pub fn classify_by_mtime(stored: Option<u64>, mtime: u64) -> FileIndexAction {
match stored { match stored {
Some(known) if known == mtime => FileIndexAction::Skip, Some(known) if known == mtime => FileIndexAction::Skip,

27
crates/quicksearch-core/src/file_handling/tests.rs
View file

@ -170,31 +170,8 @@ fn filtered_walk_hidden_root_still_walked() {
} }
#[test] #[test]
fn classify_uses_mtime_against_the_existing_index() { fn classify_uses_mtime_against_the_stored_row() {
let mut rows = DirRows::new(); // No row: new. Same mtime: nothing to do. Changed: re-read.
rows.insert("known.txt".to_string(), 100);
assert_eq!(
classify_for_indexing("new.txt", 100, &rows),
FileIndexAction::Insert,
"a name absent from the directory's rows is new"
);
assert_eq!(
classify_for_indexing("known.txt", 100, &rows),
FileIndexAction::Skip,
"same mtime means nothing to do"
);
assert_eq!(
classify_for_indexing("known.txt", 101, &rows),
FileIndexAction::Update,
"a changed mtime means re-read"
);
}
#[test]
fn classify_by_mtime_matches_the_name_keyed_path() {
// Resolved symlink targets take this route: their row is found by
// exact path.
assert_eq!(classify_by_mtime(None, 100), FileIndexAction::Insert); assert_eq!(classify_by_mtime(None, 100), FileIndexAction::Insert);
assert_eq!(classify_by_mtime(Some(100), 100), FileIndexAction::Skip); assert_eq!(classify_by_mtime(Some(100), 100), FileIndexAction::Skip);
assert_eq!(classify_by_mtime(Some(99), 100), FileIndexAction::Update); assert_eq!(classify_by_mtime(Some(99), 100), FileIndexAction::Update);

178
crates/quicksearch-core/src/incremental.rs
View file

@ -6,7 +6,7 @@ use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Instant; use std::time::Instant;
use rusqlite::Connection; use rusqlite::Transaction;
use crate::config::{Config, IgnoreSet}; use crate::config::{Config, IgnoreSet};
use crate::db::repo; use crate::db::repo;
@ -40,19 +40,19 @@ impl Budget<'_> {
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Applied { pub enum Applied {
Done, Done,
/// Budget spent. What was written is committed; the caller should re-queue /// Budget spent. What was written stays written; the caller should
/// the same event with `resume_from` set to `done`. /// re-queue the same event with `resume_from` set to `done`.
Unfinished { Unfinished {
done: usize, done: usize,
}, },
} }
/// Apply one filesystem event to the index. Missing files are treated as /// Apply one filesystem event to the index, inside the caller's transaction,
/// no-ops (the pending Remove event resolves them); unchanged mtimes /// which may hold many events. Missing files are treated as no-ops (the
/// short-circuit without touching the DB. See [`Budget`] for the one event /// pending Remove event resolves them); unchanged mtimes short-circuit without
/// that is not small. /// touching the DB. See [`Budget`] for the one event that is not small.
pub fn apply_fs_event( pub fn apply_fs_event(
conn: &mut Connection, tx: &Transaction<'_>,
event: &FsEvent, event: &FsEvent,
config: &Config, config: &Config,
ignore: &IgnoreSet, ignore: &IgnoreSet,
@ -61,18 +61,18 @@ pub fn apply_fs_event(
) -> Result<Applied, String> { ) -> Result<Applied, String> {
match event { match event {
FsEvent::Create(p) | FsEvent::Modify(p) => { FsEvent::Create(p) | FsEvent::Modify(p) => {
upsert_path(conn, p, config, ignore, registry, budget) upsert_path(tx, p, config, ignore, registry, budget)
} }
FsEvent::Remove(p) => remove_path(conn, p).map(|()| Applied::Done), FsEvent::Remove(p) => remove_paths(tx, std::slice::from_ref(p)).map(|()| Applied::Done),
FsEvent::Rename { from, to } => { FsEvent::Rename { from, to } => {
remove_path(conn, from)?; remove_paths(tx, std::slice::from_ref(from))?;
upsert_path(conn, to, config, ignore, registry, budget) upsert_path(tx, to, config, ignore, registry, budget)
} }
} }
} }
fn upsert_path( fn upsert_path(
conn: &mut Connection, tx: &Transaction<'_>,
path: &Path, path: &Path,
config: &Config, config: &Config,
ignore: &IgnoreSet, ignore: &IgnoreSet,
@ -127,17 +127,37 @@ fn upsert_path(
if budget.spent() { if budget.spent() {
return Ok(Applied::Unfinished { done }); return Ok(Applied::Unfinished { done });
} }
upsert_file(conn, entry.path(), config, registry)?; upsert_file(tx, entry.path(), config, registry)?;
done += 1; done += 1;
} }
Ok(Applied::Done) Ok(Applied::Done)
} else { } else {
upsert_file(conn, path, config, registry).map(|()| Applied::Done) upsert_file(tx, path, config, registry).map(|()| Applied::Done)
}
}
/// Run `write` all or nothing inside the caller's transaction: a file whose
/// writes fail part-way must not keep a row that claims text its posting
/// never got.
fn atomically(
tx: &Transaction<'_>,
write: impl FnOnce() -> Result<(), String>,
) -> Result<(), String> {
tx.execute_batch("SAVEPOINT one_path")
.map_err(|e| format!("begin incremental savepoint: {}", e))?;
match write() {
Ok(()) => tx
.execute_batch("RELEASE one_path")
.map_err(|e| format!("release incremental savepoint: {}", e)),
Err(e) => {
let _ = tx.execute_batch("ROLLBACK TO one_path; RELEASE one_path");
Err(e)
}
} }
} }
fn upsert_file( fn upsert_file(
conn: &mut Connection, tx: &Transaction<'_>,
path: &Path, path: &Path,
config: &Config, config: &Config,
registry: &Registry, registry: &Registry,
@ -153,16 +173,12 @@ fn upsert_file(
return Ok(()); return Ok(());
}; };
let tx = conn
.transaction()
.map_err(|e| format!("begin incremental tx: {}", e))?;
// Settled before the file is opened, as the full walk settles it: an // Settled before the file is opened, as the full walk settles it: an
// unchanged file (a re-sent event, a `chmod`, the live view re-checking a // unchanged file (a re-sent event, a `chmod`, the live view re-checking a
// row it shows) costs one lookup, never a read and a hash. Measured // row it shows) costs one lookup, never a read and a hash. Measured
// (`benches/index.rs`, `unchanged_event`, 500 small text files): 3.3 ms // (`benches/index.rs`, `directory_event::unchanged`, 500 small text
// hashing and sniffing each against 1.4 ms. // files): 3.3 ms hashing and sniffing each against 1.4 ms.
let stored = repo::mtime_for_path(&tx, &db_path)?; let stored = repo::mtime_for_path(tx, &db_path)?;
if stored.is_some() && stored == mtime_secs(&meta) { if stored.is_some() && stored == mtime_secs(&meta) {
return Ok(()); return Ok(());
} }
@ -171,67 +187,54 @@ fn upsert_file(
let Some(rec) = prepare_file_record(&db_path, &meta, config, registry, &mut scratch) else { let Some(rec) = prepare_file_record(&db_path, &meta, config, registry, &mut scratch) else {
return Ok(()); return Ok(());
}; };
let written = match stored { atomically(tx, || {
Some(_) => repo::update_file_basic(&tx, &rec.as_new_file())?, let written = match stored {
None => repo::insert_file(&tx, &rec.as_new_file())?, Some(_) => repo::update_file_basic(tx, &rec.as_new_file())?,
}; None => repo::insert_file(tx, &rec.as_new_file())?,
// `None`: lost a race with another writer; the row that won is current };
// enough. // `None`: lost a race with another writer; the row that won is
let Some(file_id) = written else { // current enough.
return Ok(()); let Some(file_id) = written else {
}; return Ok(());
};
// A row that needs no content is already NA: both writers above set its // A row that needs no content is already NA: both writers above set
// state from `rec`. That gate is also the only size gate on this path: // its state from `rec`. That gate is also the only size gate on this
// `decide_content` has none, and falling through would read a // path: `decide_content` has none, and falling through would read a
// multi-gigabyte `.txt` whole into memory. // multi-gigabyte `.txt` whole into memory.
if rec.needs_content { if rec.needs_content {
match rec.inline_text.as_deref() { match rec.inline_text.as_deref() {
// `prepare_file_record` already read the whole file. // `prepare_file_record` already read the whole file.
Some(text) => { Some(text) => {
let zstd = repo::encode_one(text)?; let zstd = repo::encode_one(text)?;
repo::store_pending_text(&tx, file_id, text, zstd.as_deref())?; repo::store_pending_text(tx, file_id, text, zstd.as_deref())?;
}
None => extract_and_store(tx, file_id, rec.path(), rec.mime, registry, config)?,
} }
None => extract_and_store(&tx, file_id, rec.path(), rec.mime, registry, config)?,
} }
} Ok(())
})
tx.commit()
.map_err(|e| format!("commit incremental tx: {}", e))
} }
fn remove_path(conn: &mut Connection, path: &Path) -> Result<(), String> { /// Delete `paths` and everything indexed beneath them, inside the caller's
remove_paths(conn, std::slice::from_ref(&path.to_path_buf()), 1) /// transaction. Pays off only when the caller has reduced the set to its
} /// *roots*: then `rm -rf dir/` is a fixed handful of range-driven statements
/// ([`repo::delete_subtree`]) rather than five per file.
/// Delete `paths` and everything indexed beneath them, in transactions of at pub fn remove_paths(tx: &Transaction<'_>, paths: &[std::path::PathBuf]) -> Result<(), String> {
/// most `chunk` paths. Pays off only when the caller has reduced the set to for path in paths {
/// its *roots*: then `rm -rf dir/` is a fixed handful of range-driven // A path the index cannot spell was never indexed, so there is
/// statements ([`repo::delete_subtree`]) rather than five per file. // nothing to delete; its lossy spelling names another file (see
pub fn remove_paths( // `path_to_db_string`), whose row and subtree this would delete.
conn: &mut Connection, if path.to_str().is_none() {
paths: &[std::path::PathBuf], continue;
chunk: usize,
) -> Result<(), String> {
for batch in paths.chunks(chunk.max(1)) {
let tx = conn
.transaction()
.map_err(|e| format!("begin incremental tx: {}", e))?;
for path in batch {
// A path the index cannot spell was never indexed, so there is
// nothing to delete; its lossy spelling names another file (see
// `path_to_db_string`), whose row and subtree this would delete.
if path.to_str().is_none() {
continue;
}
// The stored key is canonicalized, but the file is already gone,
// so `canonicalize` cannot run; `db_key_for_missing_path` approximates it.
let path_str = db_key_for_missing_path(path);
repo::delete_file_by_path(&tx, &path_str)?;
repo::delete_subtree(&tx, &RootRange::for_root(&path_str))?;
} }
tx.commit() // The stored key is canonicalized, but the file is already gone, so
.map_err(|e| format!("commit incremental tx: {}", e))?; // `canonicalize` cannot run; `db_key_for_missing_path` approximates it.
let path_str = db_key_for_missing_path(path);
atomically(tx, || {
repo::delete_file_by_path(tx, &path_str)?;
repo::delete_subtree(tx, &RootRange::for_root(&path_str)).map(|_| ())
})?;
} }
Ok(()) Ok(())
} }
@ -240,7 +243,7 @@ pub fn remove_paths(
mod tests { mod tests {
use super::*; use super::*;
use crate::db::open_or_recreate; use crate::db::open_or_recreate;
use rusqlite::OptionalExtension; use rusqlite::{Connection, OptionalExtension};
use std::time::Duration; use std::time::Duration;
struct Fixture { struct Fixture {
@ -281,8 +284,9 @@ mod tests {
} }
fn apply_resuming(&mut self, event: &FsEvent, budget: Duration, from: usize) -> Applied { fn apply_resuming(&mut self, event: &FsEvent, budget: Duration, from: usize) -> Applied {
apply_fs_event( let tx = self.conn.transaction().unwrap();
&mut self.conn, let applied = apply_fs_event(
&tx,
event, event,
&self.config, &self.config,
&self.ignore, &self.ignore,
@ -293,7 +297,9 @@ mod tests {
resume_from: from, resume_from: from,
}, },
) )
.unwrap() .unwrap();
tx.commit().unwrap();
applied
} }
fn write(&self, name: &str, content: &str) -> std::path::PathBuf { fn write(&self, name: &str, content: &str) -> std::path::PathBuf {
@ -429,7 +435,9 @@ mod tests {
]; ];
// Collapsed to its root, as the coordinator's `collapse_pending_removals` does. // Collapsed to its root, as the coordinator's `collapse_pending_removals` does.
let roots = vec![reported[0].clone()]; let roots = vec![reported[0].clone()];
remove_paths(&mut f.conn, &roots, 200).unwrap(); let tx = f.conn.transaction().unwrap();
remove_paths(&tx, &roots).unwrap();
tx.commit().unwrap();
assert_eq!(f.counts(), (1, 1, 1), "only tree2 survives"); assert_eq!(f.counts(), (1, 1, 1), "only tree2 survives");
let survivor = f.canonical(&f.dir.join("tree2").join("keep.txt")); let survivor = f.canonical(&f.dir.join("tree2").join("keep.txt"));
assert!(f.row(&survivor).is_some()); assert!(f.row(&survivor).is_some());
@ -690,8 +698,9 @@ mod tests {
} }
std::fs::write(bad.join("inside.txt"), "body").unwrap(); std::fs::write(bad.join("inside.txt"), "body").unwrap();
let tx = f.conn.transaction().unwrap();
let outcome = apply_fs_event( let outcome = apply_fs_event(
&mut f.conn, &tx,
&FsEvent::Create(bad), &FsEvent::Create(bad),
&f.config, &f.config,
&f.ignore, &f.ignore,
@ -707,6 +716,7 @@ mod tests {
Ok(Applied::Done), Ok(Applied::Done),
"an unindexable subtree is not a failure" "an unindexable subtree is not a failure"
); );
tx.commit().unwrap();
assert_eq!(f.counts().0, 0, "and nothing under it is indexed"); assert_eq!(f.counts().0, 0, "and nothing under it is indexed");
} }

23
crates/quicksearch-core/src/indexing/pipeline.rs
View file

@ -16,8 +16,8 @@ use crate::extract::Registry;
use crate::file_handling::{ use crate::file_handling::{
cleanup_stale_index_entries, count_tree_entries_fast, fts_begin_bulk_write, cleanup_stale_index_entries, count_tree_entries_fast, fts_begin_bulk_write,
fts_finalize_after_text_indexing, mark_oversize_pending_na, normalize_root_string, fts_finalize_after_text_indexing, mark_oversize_pending_na, normalize_root_string,
process_batch_inserts, process_batch_updates, store_extracted, Bodies, ExtractCursor, process_batch_inserts, process_batch_updates, store_extracted, ExtractCursor, ExtractScope,
ExtractScope, FileIndexAction, OwnedNewFile, RootRange, FileIndexAction, OwnedNewFile, RootRange,
}; };
use crate::walk::{thread_count_for, walk_indexable_files, ParallelWalk, TryNext, WalkEvent}; use crate::walk::{thread_count_for, walk_indexable_files, ParallelWalk, TryNext, WalkEvent};
@ -153,8 +153,7 @@ type WriteBatch = fn(
&Arc<Mutex<Connection>>, &Arc<Mutex<Connection>>,
&[OwnedNewFile], &[OwnedNewFile],
&Arc<AtomicBool>, &Arc<AtomicBool>,
&Config, &mut crate::db::repo::DocEncoder,
&mut Bodies,
) -> Result<(), String>; ) -> Result<(), String>;
/// Write `rows` through `write`, then clear them; on an error they stay. /// Write `rows` through `write`, then clear them; on an error they stay.
@ -163,13 +162,7 @@ fn flush(
rows: &mut Vec<OwnedNewFile>, rows: &mut Vec<OwnedNewFile>,
write: WriteBatch, write: WriteBatch,
) -> Result<(), String> { ) -> Result<(), String> {
write( write(&cx.conn_mutex, rows, cx.stop_flag, &mut cx.encoder)?;
&cx.conn_mutex,
rows,
cx.stop_flag,
cx.config,
&mut cx.bodies,
)?;
rows.clear(); rows.clear();
Ok(()) Ok(())
} }
@ -525,7 +518,7 @@ impl RootPipeline {
ready, ready,
cx.stop_flag, cx.stop_flag,
deadline, deadline,
&mut cx.bodies, &mut cx.encoder,
)?; )?;
if stored.consumed > 0 { if stored.consumed > 0 {
// The last row *written*, not the last fetched. The whole path, // The last row *written*, not the last fetched. The whole path,
@ -574,8 +567,8 @@ pub(super) struct RunCx<'a> {
/// Writer time one root's turn may take before the round moves on /// Writer time one root's turn may take before the round moves on
/// ([`crate::config::ProcessingConfig::writer_turn_slice_ms`]). /// ([`crate::config::ProcessingConfig::writer_turn_slice_ms`]).
pub(super) slice: Duration, pub(super) slice: Duration,
/// The run's one set of compression buffers, shared by every write. /// The run's one compression context, shared by every write.
pub(super) bodies: Bodies, pub(super) encoder: crate::db::repo::DocEncoder,
/// Rows with no file behind them, per-directory plus the vanished sweep. /// Rows with no file behind them, per-directory plus the vanished sweep.
pub(super) stale_candidates: Vec<String>, pub(super) stale_candidates: Vec<String>,
/// Paths reached via symlink; their rows may live outside every root. /// Paths reached via symlink; their rows may live outside every root.
@ -600,7 +593,7 @@ impl<'a> RunCx<'a> {
registry: Arc::new(Registry::default_set()), registry: Arc::new(Registry::default_set()),
batch: config.processing.batch_size.max(1), batch: config.processing.batch_size.max(1),
slice: Duration::from_millis(config.processing.writer_turn_slice_ms), slice: Duration::from_millis(config.processing.writer_turn_slice_ms),
bodies: Bodies::new()?, encoder: crate::db::repo::DocEncoder::new()?,
stale_candidates: Vec::new(), stale_candidates: Vec::new(),
aliased_paths: HashSet::new(), aliased_paths: HashSet::new(),
stale_cleanup_ok: true, stale_cleanup_ok: true,

8
crates/quicksearch-core/src/live.rs
View file

@ -474,12 +474,10 @@ impl Loop {
return WindowUpdate::Unchanged; return WindowUpdate::Unchanged;
}; };
let cut = match tier { let cut = match tier {
// Unlike in the passes, the body may have stopped matching here.
ContentTier::Exact => { ContentTier::Exact => {
// Unlike in the passes, the body may have stopped matching crate::search::cascade::body_match(&query.pattern, text, &mut String::new())
// here; an unmarked window says so. .map(|m| m.snippet)
let folded = text.to_ascii_lowercase();
crate::search::cascade::text_snippet(&query.pattern, text, &folded)
.filter(|snip| !snip.ranges.is_empty())
} }
ContentTier::Fuzzy => match &self.fuzzy { ContentTier::Fuzzy => match &self.fuzzy {
Some(bitap) => crate::search::cascade::fuzzy_snippet(bitap, text).map(|(_, s)| s), Some(bitap) => crate::search::cascade::fuzzy_snippet(bitap, text).map(|(_, s)| s),

74
crates/quicksearch-core/src/query/pattern.rs
View file

@ -12,7 +12,6 @@ use regex::{Regex, RegexBuilder};
use super::translator::TranslateError; use super::translator::TranslateError;
use crate::search::prefilter::Required; use crate::search::prefilter::Required;
use crate::snippet;
/// Keeps a hostile pattern (`a{1000000}{1000}`) from ballooning the program. /// Keeps a hostile pattern (`a{1000000}{1000}`) from ballooning the program.
const REGEX_SIZE_LIMIT: usize = 4 << 20; const REGEX_SIZE_LIMIT: usize = 4 << 20;
@ -79,6 +78,21 @@ impl WildcardPattern {
&self.anchored_cs &self.anchored_cs
} }
} }
/// Non-overlapping occurrences in `text`, stopping at 1000 where the
/// cascade's `count_frac` saturates, and the first of them, in one sweep.
pub fn count_and_first(
&self,
text: &str,
case_insensitive: bool,
) -> (usize, Option<Range<usize>>) {
let mut found = self
.search(case_insensitive)
.find_iter(text)
.take(COUNT_CAP);
let first = found.next().map(|m| m.range());
(first.as_ref().map_or(0, |_| 1 + found.count()), first)
}
} }
impl TermPattern { impl TermPattern {
@ -246,38 +260,6 @@ impl TermPattern {
TermPattern::Wildcard(w) => w.search(case_insensitive).find(text).map(|m| m.range()), TermPattern::Wildcard(w) => w.search(case_insensitive).find(text).map(|m| m.range()),
} }
} }
/// [`TermPattern::find_first`], case-insensitive, on a pre-folded haystack.
pub fn find_first_folded(&self, folded: &str) -> Option<Range<usize>> {
match self {
TermPattern::Empty => None,
TermPattern::Literal(l) => {
let pos = folded.find(&l.folded)?;
Some(pos..pos + l.text.len())
}
TermPattern::Wildcard(w) => w.search_ci.find(folded).map(|m| m.range()),
}
}
/// Case-insensitive [`TermPattern::count`] against an already-folded haystack.
pub fn count_folded(&self, folded: &str) -> usize {
match self {
TermPattern::Empty => 0,
TermPattern::Literal(l) => snippet::count_occurrences(folded, &l.folded),
TermPattern::Wildcard(w) => w.search_ci.find_iter(folded).take(COUNT_CAP).count(),
}
}
/// Case-sensitive, non-overlapping occurrence count; the case-insensitive
/// one is [`TermPattern::count_folded`]. Wildcards stop at 1000, where the
/// cascade's `count_frac` saturates anyway.
pub fn count(&self, text: &str) -> usize {
match self {
TermPattern::Empty => 0,
TermPattern::Literal(l) => snippet::count_occurrences(text, &l.text),
TermPattern::Wildcard(w) => w.search_cs.find_iter(text).take(COUNT_CAP).count(),
}
}
} }
/// A compiled `regex:` query. Case-insensitive by default (override with /// A compiled `regex:` query. Case-insensitive by default (override with
@ -369,9 +351,13 @@ impl RegexQuery {
self.re.find(text).map(|m| m.range()) self.re.find(text).map(|m| m.range())
} }
/// Non-overlapping occurrence count, capped at 1000. /// Non-overlapping occurrences, capped at 1000, and the first of them,
pub fn count(&self, text: &str) -> usize { /// in one sweep. Against a count and then a second search for the first
self.re.find_iter(text).take(COUNT_CAP).count() /// (`searchtime`, cargo registry): within noise to 3% faster.
pub fn count_and_first(&self, text: &str) -> (usize, Option<Range<usize>>) {
let mut found = self.re.find_iter(text).take(COUNT_CAP);
let first = found.next().map(|m| m.range());
(first.as_ref().map_or(0, |_| 1 + found.count()), first)
} }
} }
@ -425,7 +411,6 @@ mod tests {
assert!(p.is_empty(), "{:?}", parts); assert!(p.is_empty(), "{:?}", parts);
assert!(!p.whole_match("anything", true)); assert!(!p.whole_match("anything", true));
assert!(p.find_first("anything", true).is_none()); assert!(p.find_first("anything", true).is_none());
assert_eq!(p.count("anything"), 0);
} }
} }
@ -512,10 +497,14 @@ mod tests {
#[test] #[test]
fn count_is_nonoverlapping_and_capped() { fn count_is_nonoverlapping_and_capped() {
let p = TermPattern::build(&[part("a*b", true)]).unwrap(); let TermPattern::Wildcard(w) = TermPattern::build(&[part("a*b", true)]).unwrap() else {
assert_eq!(p.count("ab ab ab"), 3); panic!("a star makes a wildcard");
};
assert_eq!(w.count_and_first("xx ab ab ab", false), (3, Some(3..5)));
let many = "ab ".repeat(2000); let many = "ab ".repeat(2000);
assert_eq!(p.count(&many), 1000); assert_eq!(w.count_and_first(&many, false).0, 1000);
assert_eq!(w.count_and_first("AB aB", false), (0, None), "exact case");
assert_eq!(w.count_and_first("AB aB", true), (2, Some(0..2)));
} }
#[test] #[test]
@ -527,8 +516,6 @@ mod tests {
assert_eq!(p.find_first("my Report.pdf", false), Some(3..9)); assert_eq!(p.find_first("my Report.pdf", false), Some(3..9));
assert_eq!(p.find_first("my report.pdf", true), Some(3..9)); assert_eq!(p.find_first("my report.pdf", true), Some(3..9));
assert_eq!(p.find_first("my report.pdf", false), None); assert_eq!(p.find_first("my report.pdf", false), None);
assert_eq!(p.count("report Report"), 1);
assert_eq!(p.count_folded("report report"), 2);
} }
#[test] #[test]
@ -585,7 +572,8 @@ mod tests {
let r = RegexQuery::new("b[aeiou]d").unwrap(); let r = RegexQuery::new("b[aeiou]d").unwrap();
let hay = "bad bed bodkin"; let hay = "bad bed bodkin";
assert_eq!(r.find_first(hay), Some(0..3)); assert_eq!(r.find_first(hay), Some(0..3));
assert_eq!(r.count(hay), 3); assert_eq!(r.count_and_first(hay), (3, Some(0..3)));
assert_eq!(r.count_and_first("nothing here"), (0, None));
} }
#[test] #[test]

106
crates/quicksearch-core/src/search/cascade.rs
View file

@ -48,39 +48,72 @@ const CANCEL_CHECK_ROWS: usize = 256;
/// Snippet window budget: the GUI trims to column width; mouseover shows the rest. /// Snippet window budget: the GUI trims to column width; mouseover shows the rest.
const SNIPPET_WINDOW_BYTES: usize = 600; const SNIPPET_WINDOW_BYTES: usize = 600;
const SNIPPET_OPTS: snippet::Options = snippet::Options { /// How one document body matched the term: stage 5 when it occurs exactly
approx_bytes: SNIPPET_WINDOW_BYTES, /// as typed, 6 when only in another case; `count` is what the stage ranks
}; /// on, `snippet` the window around the first occurrence.
#[derive(Debug, Clone)]
/// The Content Match snippet for one document body, cut exactly as the pub struct BodyMatch {
/// full-text passes cut it; shared with [`crate::live`] so a re-cut matches. pub stage: u8,
/// `folded` must be `text` ASCII-lowercased; the fold preserves byte length, pub count: usize,
/// so offsets found in it slice `text`. pub snippet: snippet::Snippet,
pub fn text_snippet(
pattern: &crate::query::pattern::TermPattern,
text: &str,
folded: &str,
) -> Option<snippet::Snippet> {
text_snippet_counted(pattern, text, folded)
.map(|(snip, _)| snip)
.or_else(|| {
pattern.find_first_folded(folded).map(|r| {
// A greedy pattern can match megabytes; clamp before the window.
let r = clamp_match_range(text, r, SNIPPET_WINDOW_BYTES);
snippet::window_around(text, (r.start, r.end), &SNIPPET_OPTS)
})
})
} }
/// [`text_snippet`] for a literal pattern, plus the case-insensitive /// The term's match in one document body, verified and cut exactly as the
/// occurrence count found on the way. `None` when the pattern is not literal. /// full-text pass does; `None` when it does not occur. Shared with
pub fn text_snippet_counted( /// [`crate::live`], so a re-cut matches. `lower` is reused scratch for the
/// folded body a literal is searched in.
///
/// One sweep per body where there were three: a literal counts both cases in
/// its one folded scan, and a wildcard is never folded, its exact-case regex
/// giving the count and the first match together. Measured against folding
/// every body, then a folded scan, a case-sensitive recount and, for a
/// wildcard, a third scan for the window (`searchtime`, cargo registry, 41k
/// files): `Hash*Map` 103.5–104.2 → 97.4–98.0 ms, `Serial*ize` 56–59 → 53.5;
/// literals within ±2% (`serialize` 38.6–39.2 → 38.7–39.2).
pub fn body_match(
pattern: &crate::query::pattern::TermPattern, pattern: &crate::query::pattern::TermPattern,
text: &str, text: &str,
folded: &str, lower: &mut String,
) -> Option<(snippet::Snippet, usize)> { ) -> Option<BodyMatch> {
let term = pattern.literal_folded()?; use crate::query::pattern::TermPattern;
Some(snippet::extract_folded(text, folded, term, &SNIPPET_OPTS)) match pattern {
TermPattern::Empty => None,
TermPattern::Literal(_) => {
let (typed, term) = (pattern.literal()?, pattern.literal_folded()?);
fold_into(lower, text);
let found = snippet::extract_folded(text, lower, term, typed, SNIPPET_WINDOW_BYTES)?;
let (stage, count) = match found.exact {
0 => (6, found.count),
exact => (5, exact),
};
Some(BodyMatch {
stage,
count,
snippet: found.snippet,
})
}
TermPattern::Wildcard(w) => {
let (stage, (count, first)) = match w.count_and_first(text, false) {
(0, _) => (6, w.count_and_first(text, true)),
exact => (5, exact),
};
// A greedy pattern can match megabytes; clamp before the window.
let r = clamp_match_range(text, first?, SNIPPET_WINDOW_BYTES);
Some(BodyMatch {
stage,
count,
snippet: snippet::window_around(text, (r.start, r.end), SNIPPET_WINDOW_BYTES),
})
}
}
}
/// Fold `text` into `dst`, reusing its allocation. The ASCII fold is
/// byte-length preserving, so folded offsets are valid in the original.
fn fold_into(dst: &mut String, text: &str) {
dst.clear();
dst.push_str(text);
dst.make_ascii_lowercase();
} }
/// The fuzzy full-text match in one document body: occurrence count within /// The fuzzy full-text match in one document body: occurrence count within
@ -92,7 +125,7 @@ pub fn fuzzy_snippet(
text: &str, text: &str,
) -> Option<(usize, snippet::Snippet)> { ) -> Option<(usize, snippet::Snippet)> {
let (count, first) = bitap.count_and_first(text.as_bytes()); let (count, first) = bitap.count_and_first(text.as_bytes());
first.map(|range| (count, snippet::window_around(text, range, &SNIPPET_OPTS))) first.map(|range| (count, snippet::window_around(text, range, SNIPPET_WINDOW_BYTES)))
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
@ -465,13 +498,20 @@ impl<'a> Cx<'a> {
self.flush_pass(deferred.hits, deferred.overflowed); self.flush_pass(deferred.hits, deferred.overflowed);
} }
/// Sort + cut back to display-limit room once past the cap; true if anything dropped. /// Cut a deferred buffer back to display-limit room once past the cap;
/// true if anything dropped. Partitioned, not sorted: [`Cx::flush_pass`]
/// sorts what is left. Measured against a full sort (`searchtime`, cargo
/// registry, terms every path holds): `1949cf` 25–30 → 21–22 ms, `src`
/// 40–42 → 37–39, `crates` 89–99 → 86–87.
fn enforce_cap(&self, buf: &mut Vec<SearchHit>) -> bool { fn enforce_cap(&self, buf: &mut Vec<SearchHit>) -> bool {
if buf.len() <= self.buffer_cap() { if buf.len() <= self.buffer_cap() {
return false; return false;
} }
buf.sort_by(rank_order); let keep = self.remaining();
buf.truncate(self.remaining()); if keep > 0 {
buf.select_nth_unstable_by(keep - 1, rank_order);
}
buf.truncate(keep);
true true
} }
} }

70
crates/quicksearch-core/src/search/cascade/passes.rs
View file

@ -84,14 +84,6 @@ fn fts_match_sql(filter_sql: &str) -> String {
) )
} }
/// Fold `text` into `dst`, reusing its allocation. The ASCII fold is
/// byte-length preserving, so folded offsets are valid in the original.
fn fold_into(dst: &mut String, text: &str) {
dst.clear();
dst.push_str(text);
dst.make_ascii_lowercase();
}
/// The segment pass A's path-tier prefilter runs on: the longest one /// The segment pass A's path-tier prefilter runs on: the longest one
/// containing no path separator, or `None` when every segment has one. Any /// containing no path separator, or `None` when every segment has one. Any
/// single segment is a superset test, and a separator-free one cannot span /// single segment is a superset test, and a separator-free one cannot span
@ -157,7 +149,6 @@ impl<'a> Cx<'a> {
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
let mut buf: Vec<SearchHit> = Vec::new(); let mut buf: Vec<SearchHit> = Vec::new();
let mut overflowed = false;
let mut deferred = Deferred::default(); let mut deferred = Deferred::default();
let mut scanned = 0usize; let mut scanned = 0usize;
let mut clock = FlushClock::new(); let mut clock = FlushClock::new();
@ -189,8 +180,9 @@ impl<'a> Cx<'a> {
match classify(self, row, file_id, &path, &path[name_at..])? { match classify(self, row, file_id, &path, &path[name_at..])? {
RowHit::Skip => {} RowHit::Skip => {}
RowHit::Emit(hit) => { RowHit::Emit(hit) => {
// No cap here: `flush_if_due` hands the buffer over at
// every `batch` rows, so only deferred hits accumulate.
buf.push(hit); buf.push(hit);
overflowed |= self.enforce_cap(&mut buf);
self.flush_if_due(&mut buf, &mut clock); self.flush_if_due(&mut buf, &mut clock);
// Stop once the display limit is full. Recorded, not // Stop once the display limit is full. Recorded, not
// turned into `limited` here: `flush_pass` owns that call. // turned into `limited` here: `flush_pass` owns that call.
@ -213,7 +205,7 @@ impl<'a> Cx<'a> {
Some(slot) => *self.deferred_mut(slot) = deferred, Some(slot) => *self.deferred_mut(slot) = deferred,
None => debug_assert!(deferred.hits.is_empty(), "deferred hits with no slot"), None => debug_assert!(deferred.hits.is_empty(), "deferred hits with no slot"),
} }
self.flush_pass(buf, overflowed); self.flush_pass(buf, false);
// After the flush, so it cannot be undone by one that truncated nothing; // After the flush, so it cannot be undone by one that truncated nothing;
// without this a cut set can report itself complete. // without this a cut set can report itself complete.
self.limited |= cut_short; self.limited |= cut_short;
@ -329,41 +321,24 @@ impl<'a> Cx<'a> {
let Some(text) = stored_text(row, &mut doc)? else { let Some(text) = stored_text(row, &mut doc)? else {
return Ok(RowHit::Skip); return Ok(RowHit::Skip);
}; };
// Fold once: count, first match and snippet all need it, even on let Some(m) = super::body_match(pattern, text, &mut lower) else {
// rows about to be dropped. return Ok(RowHit::Skip);
fold_into(&mut lower, text);
let (count, stage, snip) = match super::text_snippet_counted(pattern, text, &lower) {
Some((snip, count_ci)) => {
if count_ci == 0 {
return Ok(RowHit::Skip);
}
let count_cs = pattern.count(text);
if count_cs > 0 {
(count_cs, 5, Some(snip))
} else {
(count_ci, 6, Some(snip))
}
}
None => {
let count_cs = pattern.count(text);
let (count, stage) = if count_cs > 0 {
(count_cs, 5)
} else {
let count_ci = pattern.count_folded(&lower);
if count_ci == 0 {
return Ok(RowHit::Skip);
}
(count_ci, 6)
};
(count, stage, super::text_snippet(pattern, text, &lower))
}
}; };
let rank = stage as f64 + count_frac(count); let rank = m.stage as f64 + count_frac(m.count);
if !cx.regex_accepts(file_id, path, Some(text))? { if !cx.regex_accepts(file_id, path, Some(text))? {
return Ok(RowHit::Skip); return Ok(RowHit::Skip);
} }
row_hit(row, file_id, path, name, rank, stage, snip, false) row_hit(
row,
file_id,
path,
name,
rank,
m.stage,
Some(m.snippet),
false,
)
}) })
} }
@ -529,15 +504,12 @@ impl<'a> Cx<'a> {
let Some(text) = stored_text(row, &mut doc)? else { let Some(text) = stored_text(row, &mut doc)? else {
return Ok(RowHit::Skip); return Ok(RowHit::Skip);
}; };
let count = re.count(text); let (count, Some(first)) = re.count_and_first(text) else {
if count == 0 {
return Ok(RowHit::Skip); return Ok(RowHit::Skip);
} };
// A greedy user regex can match megabytes; clamp before the window. // A greedy user regex can match megabytes; clamp before the window.
let snip = re.find_first(text).map(|r| { let r = clamp_match_range(text, first, SNIPPET_WINDOW_BYTES);
let r = clamp_match_range(text, r, SNIPPET_WINDOW_BYTES); let snip = snippet::window_around(text, (r.start, r.end), SNIPPET_WINDOW_BYTES);
snippet::window_around(text, (r.start, r.end), &SNIPPET_OPTS)
});
row_hit( row_hit(
row, row,
file_id, file_id,
@ -545,7 +517,7 @@ impl<'a> Cx<'a> {
name, name,
6.0 + count_frac(count), 6.0 + count_frac(count),
6, 6,
snip, Some(snip),
false, false,
) )
}) })

28
crates/quicksearch-core/src/search/duplicates.rs
View file

@ -15,6 +15,10 @@
//! database in RAM (the gap widens on cold or encrypted storage). An innocent //! database in RAM (the gap widens on cold or encrypted storage). An innocent
//! edit can lose this, so a test reads the query plan back and fails on //! edit can lose this, so a test reads the query plan back and fails on
//! anything but a covering scan. //! anything but a covering scan.
//!
//! The index exists only once a run has finished (see
//! [`crate::db::repo::build_hash_index`]); before that the same query sorts
//! the table, slower but with the same answer.
use std::cmp::Ordering; use std::cmp::Ordering;
use std::collections::BinaryHeap; use std::collections::BinaryHeap;
@ -239,6 +243,8 @@ mod tests {
.expect("unique path"); .expect("unique path");
} }
tx.commit().unwrap(); tx.commit().unwrap();
// As a finished run's maintenance leaves it.
crate::db::repo::build_hash_index(&conn).unwrap();
(scratch, p) (scratch, p)
} }
@ -282,6 +288,28 @@ mod tests {
assert_eq!(groups[1].members[0].1, "a1.txt", "members path-ordered"); assert_eq!(groups[1].members[0].1, "a1.txt", "members path-ordered");
} }
/// A fresh index has no hash index until its first run's maintenance
/// builds one; the listing must not depend on it, only get faster.
#[test]
fn groups_are_the_same_before_the_hash_index_is_built() {
let (_dir, p) = seed_db();
let indexed = find_duplicate_groups(&p, 10).unwrap();
let conn = db::open_existing(&p, true).unwrap();
conn.execute_batch("DROP INDEX idx_files_hash;").unwrap();
assert_eq!(find_duplicate_groups(&p, 10).unwrap(), indexed);
crate::db::repo::maintain(&conn, &p).unwrap();
let rebuilt: i64 = conn
.query_row(
"SELECT COUNT(*) FROM sqlite_master WHERE name = 'idx_files_hash'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(rebuilt, 1, "maintenance builds the missing index");
assert_eq!(find_duplicate_groups(&p, 10).unwrap(), indexed);
}
#[test] #[test]
fn limit_keeps_the_largest_groups() { fn limit_keeps_the_largest_groups() {
let (_dir, p) = seed_db(); let (_dir, p) = seed_db();

161
crates/quicksearch-core/src/search/fuzzy.rs
View file

@ -6,12 +6,18 @@
//! byte are set), so neither side is folded. Measured: folding the haystack //! byte are set), so neither side is folded. Measured: folding the haystack
//! cost a copy per scanned field and per stored document. //! cost a copy per scanned field and per stored document.
use std::ops::ControlFlow;
use crate::query::split::CascadeQuery; use crate::query::split::CascadeQuery;
/// Registers: one per error count `0..=k`. [`edit_budget`]'s ladder against a /// Registers: one per error count `0..=k`. [`edit_budget`]'s ladder against a
/// 64-byte pattern caps `k` at 21; [`Bitap::new`] rejects more. /// 64-byte pattern caps `k` at 21; [`Bitap::new`] rejects more.
const MAX_REGISTERS: usize = 22; const MAX_REGISTERS: usize = 22;
/// [`Bitap::step`]'s register count, taken from `k` at run time rather than
/// fixed by [`Bitap::each_match`]'s dispatch.
const RUNTIME_K: usize = usize::MAX;
pub struct Bitap { pub struct Bitap {
/// `masks[c]` has bit `i` set iff `pattern[i] == c` ignoring ASCII case. /// `masks[c]` has bit `i` set iff `pattern[i] == c` ignoring ASCII case.
masks: [u64; 256], masks: [u64; 256],
@ -19,6 +25,8 @@ pub struct Bitap {
rev_masks: [u64; 256], rev_masks: [u64; 256],
len: usize, len: usize,
k: usize, k: usize,
/// The bit that says the whole pattern has matched: `1 << (len - 1)`.
done: u64,
} }
impl Bitap { impl Bitap {
@ -44,6 +52,7 @@ impl Bitap {
rev_masks, rev_masks,
len: pattern.len(), len: pattern.len(),
k, k,
done: 1 << (pattern.len() - 1),
}) })
} }
@ -56,31 +65,75 @@ impl Bitap {
} }
} }
/// Advance all registers by one haystack byte; returns the smallest error /// Advance all registers by one haystack byte; `true` when the full
/// count d at which the full pattern just matched. `masks` selects the /// pattern just matched within `k` errors. `masks` selects the scan
/// scan direction: [`Bitap::masks`] forwards, [`Bitap::rev_masks`] back. /// direction: [`Bitap::masks`] forwards, [`Bitap::rev_masks`] back.
#[inline] ///
fn step(&self, masks: &[u64; 256], r: &mut [u64], byte: u8) -> Option<usize> { /// Registers only grow with the error count (`r[d] ⊇ r[d-1]`, from
/// `reset` on by induction), so `r[k]` alone answers "matched", and
/// [`Bitap::errors`] finds how well only when it did. Measured against
/// testing every register every byte (`searchtime`, cargo registry, 41k
/// files): `~recieve` 361 → 310 ms, `~serialze` 453 → 397, `~HashMpa`
/// 569 → 517.
#[inline(always)]
fn step<const K: usize>(
&self,
masks: &[u64; 256],
r: &mut [u64; MAX_REGISTERS],
byte: u8,
) -> bool {
let k = if K == RUNTIME_K { self.k } else { K };
let mask = masks[byte as usize]; let mask = masks[byte as usize];
let done = 1u64 << (self.len - 1);
let mut hit = None;
let mut prev_old = r[0]; // R_old[d-1] for the d-th iteration let mut prev_old = r[0]; // R_old[d-1] for the d-th iteration
r[0] = ((r[0] << 1) | 1) & mask; r[0] = ((r[0] << 1) | 1) & mask;
if r[0] & done != 0 { for d in 1..=k {
hit = Some(0);
}
for d in 1..=self.k {
let old = r[d]; let old = r[d];
r[d] = (((old << 1) | 1) & mask) // extend a ≤d-error state r[d] = (((old << 1) | 1) & mask) // extend a ≤d-error state
| prev_old // insertion in text | prev_old // insertion in text
| (prev_old << 1) // substitution | (prev_old << 1) // substitution
| ((r[d - 1] << 1) | 1); // deletion (pattern byte skipped) | ((r[d - 1] << 1) | 1); // deletion (pattern byte skipped)
prev_old = old; prev_old = old;
if hit.is_none() && r[d] & done != 0 { }
hit = Some(d); r[k] & self.done != 0
}
/// Call `hit` with the end of every match a forward scan finds and the
/// registers there, which it may reset; stop when it breaks. The budgets
/// default searches use get a copy with the register count fixed, so the
/// loop unrolls. Measured against the runtime count, on top of the note
/// at [`Bitap::step`]: `~recieve` 307–322 → 195–199 ms, `~serialze`
/// 391–398 → 239–240, `~HashMpa` 497–505 → 302–304.
fn each_match(
&self,
hay: &[u8],
hit: impl FnMut(usize, &mut [u64; MAX_REGISTERS]) -> ControlFlow<()>,
) {
match self.k {
1 => self.scan::<1>(hay, hit),
2 => self.scan::<2>(hay, hit),
_ => self.scan::<RUNTIME_K>(hay, hit),
}
}
fn scan<const K: usize>(
&self,
hay: &[u8],
mut hit: impl FnMut(usize, &mut [u64; MAX_REGISTERS]) -> ControlFlow<()>,
) {
let mut r = [0u64; MAX_REGISTERS];
self.reset(&mut r);
for (i, &b) in hay.iter().enumerate() {
if self.step::<K>(&self.masks, &mut r, b) && hit(i + 1, &mut r).is_break() {
return;
} }
} }
hit }
/// The fewest errors the match [`Bitap::step`] just reported has.
fn errors(&self, r: &[u64; MAX_REGISTERS]) -> usize {
(0..=self.k)
.find(|&d| r[d] & self.done != 0)
.unwrap_or(self.k)
} }
/// Where the match that ended at `end` with `errors` edits began: the same /// Where the match that ended at `end` with `errors` edits began: the same
@ -95,10 +148,8 @@ impl Bitap {
let mut r = [0u64; MAX_REGISTERS]; let mut r = [0u64; MAX_REGISTERS];
self.reset(&mut r); self.reset(&mut r);
for (back, &b) in hay[floor..end].iter().rev().enumerate() { for (back, &b) in hay[floor..end].iter().rev().enumerate() {
if self self.step::<RUNTIME_K>(&self.rev_masks, &mut r, b);
.step(&self.rev_masks, &mut r, b) if r[errors] & self.done != 0 {
.is_some_and(|d| d <= errors)
{
return end - (back + 1); return end - (back + 1);
} }
} }
@ -121,7 +172,8 @@ impl Bitap {
let mut best = (errors, end); let mut best = (errors, end);
let mut probe = *r; let mut probe = *r;
for (ahead, &b) in hay[end..].iter().take(errors).enumerate() { for (ahead, &b) in hay[end..].iter().take(errors).enumerate() {
if let Some(d) = self.step(&self.masks, &mut probe, b) { if self.step::<RUNTIME_K>(&self.masks, &mut probe, b) {
let d = self.errors(&probe);
if d < best.0 { if d < best.0 {
best = (d, end + ahead + 1); best = (d, end + ahead + 1);
} }
@ -133,21 +185,18 @@ impl Bitap {
/// The smallest edit distance (≤ k) at which the pattern occurs in `hay`, /// The smallest edit distance (≤ k) at which the pattern occurs in `hay`,
/// and that occurrence's byte range: the span a frontend marks. /// and that occurrence's byte range: the span a frontend marks.
pub fn best_distance_and_first(&self, hay: &[u8]) -> Option<(usize, (usize, usize))> { pub fn best_distance_and_first(&self, hay: &[u8]) -> Option<(usize, (usize, usize))> {
let mut r = [0u64; MAX_REGISTERS];
self.reset(&mut r);
let mut best: Option<(usize, usize)> = None; let mut best: Option<(usize, usize)> = None;
for (i, &b) in hay.iter().enumerate() { self.each_match(hay, |end, r| {
if let Some(d) = self.step(&self.masks, &mut r, b) { let d = self.errors(r);
let end = i + 1; if best.is_none_or(|(cur, _)| d < cur) {
if d == 0 { best = Some((d, end));
best = Some((0, end));
break;
}
if best.is_none_or(|(cur, _)| d < cur) {
best = Some((d, end));
}
} }
} if d == 0 {
ControlFlow::Break(())
} else {
ControlFlow::Continue(())
}
});
best.map(|(d, end)| (d, (self.match_start(hay, end, d), end))) best.map(|(d, end)| (d, (self.match_start(hay, end, d), end)))
} }
@ -155,20 +204,17 @@ impl Bitap {
/// one's byte range in `hay`. After each hit the automaton resets, so /// one's byte range in `hay`. After each hit the automaton resets, so
/// overlapping suffix matches don't inflate counts. /// overlapping suffix matches don't inflate counts.
pub fn count_and_first(&self, hay: &[u8]) -> (usize, Option<(usize, usize)>) { pub fn count_and_first(&self, hay: &[u8]) -> (usize, Option<(usize, usize)>) {
let mut r = [0u64; MAX_REGISTERS];
self.reset(&mut r);
let mut count = 0usize; let mut count = 0usize;
let mut first: Option<(usize, usize)> = None; let mut first: Option<(usize, usize)> = None;
for (i, &b) in hay.iter().enumerate() { self.each_match(hay, |end, r| {
if let Some(d) = self.step(&self.masks, &mut r, b) { count += 1;
count += 1; if first.is_none() {
if first.is_none() { let (errors, end) = self.refine_end(hay, r, end, self.errors(r));
let (errors, end) = self.refine_end(hay, &r, i + 1, d); first = Some((self.match_start(hay, end, errors), end));
first = Some((self.match_start(hay, end, errors), end));
}
self.reset(&mut r);
} }
} self.reset(r);
ControlFlow::Continue(())
});
(count, first) (count, first)
} }
} }
@ -517,6 +563,35 @@ mod tests {
} }
} }
/// The budgets with a fixed register count must report exactly the
/// matches, and the errors at each, that the runtime-count loop does.
#[test]
fn fixed_register_counts_agree_with_the_runtime_loop() {
fn matches(b: &Bitap, hay: &[u8], fixed: bool) -> Vec<(usize, usize)> {
let mut out = Vec::new();
let hit = |end, r: &mut [u64; MAX_REGISTERS]| {
out.push((end, b.errors(r)));
b.reset(r);
ControlFlow::Continue(())
};
if fixed {
b.each_match(hay, hit);
} else {
b.scan::<RUNTIME_K>(hay, hit);
}
out
}
let hay = b"the quick brown fox jumps over the lazy dog; The Quikc Browm Fx";
for pattern in ["quick", "brown", "fox", "lazy dog", "jumps", "QUICK"] {
for k in 1..=2 {
let b = Bitap::new(pattern.as_bytes(), k).unwrap();
let fixed = matches(&b, hay, true);
assert!(!fixed.is_empty(), "{pattern} occurs at k={k}");
assert_eq!(fixed, matches(&b, hay, false), "{pattern} k={k}");
}
}
}
#[test] #[test]
fn matches_brute_force_oracle() { fn matches_brute_force_oracle() {
let mut seed: u64 = 0x2545F4914F6CDD1D; let mut seed: u64 = 0x2545F4914F6CDD1D;

7
crates/quicksearch-core/src/search/mod.rs
View file

@ -100,9 +100,6 @@ impl SearchHit {
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub enum SearchUpdate { pub enum SearchUpdate {
Started {
generation: u64,
},
Hits { Hits {
generation: u64, generation: u64,
hits: Vec<SearchHit>, hits: Vec<SearchHit>,
@ -121,8 +118,7 @@ pub enum SearchUpdate {
impl SearchUpdate { impl SearchUpdate {
pub fn generation(&self) -> u64 { pub fn generation(&self) -> u64 {
match self { match self {
SearchUpdate::Started { generation } SearchUpdate::Hits { generation, .. }
| SearchUpdate::Hits { generation, .. }
| SearchUpdate::Completed { generation, .. } | SearchUpdate::Completed { generation, .. }
| SearchUpdate::Error { generation, .. } => *generation, | SearchUpdate::Error { generation, .. } => *generation,
} }
@ -409,7 +405,6 @@ impl Worker {
fn handle(&mut self, req: SearchRequest) { fn handle(&mut self, req: SearchRequest) {
let generation = req.generation; let generation = req.generation;
self.send(SearchUpdate::Started { generation });
match self.search(&req) { match self.search(&req) {
Ok(Some(Outcome { total, limited })) => self.send(SearchUpdate::Completed { Ok(Some(Outcome { total, limited })) => self.send(SearchUpdate::Completed {
generation, generation,

276
crates/quicksearch-core/src/snippet.rs
View file

@ -3,19 +3,6 @@
//! markup. Matching folds ASCII case only, as full-text verification does: //! markup. Matching folds ASCII case only, as full-text verification does:
//! `cafe` neither finds nor marks `café`. //! `cafe` neither finds nor marks `café`.
#[derive(Debug, Clone)]
pub struct Options {
/// Approximate byte budget for the returned window: a soft target, since
/// matches expand the window so a hit is never cut off.
pub approx_bytes: usize,
}
impl Default for Options {
fn default() -> Self {
Options { approx_bytes: 200 }
}
}
/// A context window from a document plus the match positions inside it. /// A context window from a document plus the match positions inside it.
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub struct Snippet { pub struct Snippet {
@ -40,35 +27,46 @@ impl Snippet {
} }
} }
/// Extract against a haystack the caller has already ASCII-folded. `folded` /// What [`extract_folded`] found in one body.
/// must be `text.to_ascii_lowercase()`: the fold preserves byte length, so #[derive(Debug, Clone)]
/// offsets found in it slice the original. pub struct Found {
pub snippet: Snippet,
/// Occurrences in any ASCII case, counted before touching ones were
/// coalesced: what the full-text pass ranks on.
pub count: usize,
/// How many of those are spelled exactly as typed.
pub exact: usize,
}
/// Find `term` in a haystack the caller has already ASCII-folded, and cut the
/// window around its first occurrence. `folded` must be
/// `text.to_ascii_lowercase()` and `term` must be `typed` ASCII-lowercased:
/// the fold preserves byte length, so an offset found in `folded` slices
/// `text`, where the occurrence is compared with `typed`. `None` when the
/// term does not occur; a blank term never does, but a padded one is searched
/// as given, never trimmed.
/// ///
/// Returns the window and how many occurrences of `term` it found before /// `exact` comes from the case-insensitive occurrences rather than a second,
/// touching ranges were coalesced, the count the full-text pass ranks on. A /// case-sensitive search: the same count for any term that cannot overlap
/// blank term matches nothing, but a padded term is searched as given, never /// itself, and at most one short per overlap for one that can (`aA` in `aaA`).
/// trimmed. pub fn extract_folded(
pub fn extract_folded(text: &str, folded: &str, term: &str, opts: &Options) -> (Snippet, usize) { text: &str,
folded: &str,
term: &str,
typed: &str,
budget: usize,
) -> Option<Found> {
debug_assert_eq!(folded.len(), text.len(), "ASCII folding preserves length"); debug_assert_eq!(folded.len(), text.len(), "ASCII folding preserves length");
if text.is_empty() { debug_assert_eq!(term, typed.to_ascii_lowercase(), "`term` is `typed` folded");
return (Snippet::empty(), 0);
}
if term.trim().is_empty() { if term.trim().is_empty() {
return (head_window(text, opts.approx_bytes), 0); return None;
} }
// Borrow when the term is already folded, as every cascade call's is,
// to avoid an allocation per candidate row.
let pattern: std::borrow::Cow<'_, str> = if term.bytes().any(|b| b.is_ascii_uppercase()) {
std::borrow::Cow::Owned(term.to_ascii_lowercase())
} else {
std::borrow::Cow::Borrowed(term)
};
// `memmem`, not `str::match_indices`: measured faster on whole-body // `memmem`, not `str::match_indices`: measured faster on whole-body
// scans. The walk runs to the end (the count must see every occurrence), // scans. The walk runs to the end (the counts must see every occurrence),
// but matches past the window's right edge are not stored. // but matches past the window's right edge are not stored.
let mut matches: Vec<(usize, usize)> = Vec::new(); let mut matches: Vec<(usize, usize)> = Vec::new();
let mut found = 0usize; let (mut count, mut exact) = (0usize, 0usize);
// The window's right edge once the first match fixes it, from the same // The window's right edge once the first match fixes it, from the same
// [`window_bounds`] the window itself uses. // [`window_bounds`] the window itself uses.
let mut keep_below: Option<usize> = None; let mut keep_below: Option<usize> = None;
@ -76,29 +74,32 @@ pub fn extract_folded(text: &str, folded: &str, term: &str, opts: &Options) -> (
// (`abab…` for `ab`) that starts inside the window and continues past // (`abab…` for `ab`) that starts inside the window and continues past
// it stays intact for `coalesce_overlapping` and the expansion step. // it stays intact for `coalesce_overlapping` and the expansion step.
let mut chain_end = 0usize; let mut chain_end = 0usize;
for at in memchr::memmem::find_iter(folded.as_bytes(), pattern.as_bytes()) { for at in memchr::memmem::find_iter(folded.as_bytes(), term.as_bytes()) {
// Before any dropping: two abutting occurrences are two hits for // Before any dropping: two abutting occurrences are two hits for
// ranking even though they paint as one highlight. // ranking even though they paint as one highlight.
found += 1; count += 1;
let bound = *keep_below.get_or_insert_with(|| window_bounds(text, at, opts.approx_bytes).1); if text.as_bytes()[at..at + term.len()] == *typed.as_bytes() {
exact += 1;
}
let bound = *keep_below.get_or_insert_with(|| window_bounds(text, at, budget).1);
if at >= bound && at > chain_end { if at >= bound && at > chain_end {
// Past the edge and not chained to anything kept: keep counting, // Past the edge and not chained to anything kept: keep counting,
// stop storing. // stop storing.
continue; continue;
} }
chain_end = at + pattern.len(); chain_end = at + term.len();
matches.push((at, chain_end)); matches.push((at, chain_end));
} }
if matches.is_empty() { if matches.is_empty() {
return (head_window(text, opts.approx_bytes), 0); return None;
} }
// One term's occurrences arrive in order and never overlap; only // One term's occurrences arrive in order and never overlap; only
// abutting ones merge. // abutting ones merge.
let matches = coalesce_overlapping(matches); let matches = coalesce_overlapping(matches);
let (win_start, mut win_end) = window_bounds(text, matches[0].0, opts.approx_bytes); let (win_start, mut win_end) = window_bounds(text, matches[0].0, budget);
// A match straddling the right edge is fully included, not cut mid-hit. // A match straddling the right edge is fully included, not cut mid-hit.
if let Some((_, end)) = matches.iter().rfind(|(s, _)| *s < win_end) { if let Some((_, end)) = matches.iter().rfind(|(s, _)| *s < win_end) {
if *end > win_end { if *end > win_end {
@ -117,15 +118,16 @@ pub fn extract_folded(text: &str, folded: &str, term: &str, opts: &Options) -> (
}) })
.collect(); .collect();
( Some(Found {
Snippet { snippet: Snippet {
window: text[win_start..win_end].to_string(), window: text[win_start..win_end].to_string(),
ranges, ranges,
truncated_start: win_start > 0, truncated_start: win_start > 0,
truncated_end: win_end < text.len(), truncated_end: win_end < text.len(),
}, },
found, count,
) exact,
})
} }
/// The window cut around a match that starts at `first`: from a third of /// The window cut around a match that starts at `first`: from a third of
@ -174,12 +176,12 @@ pub fn whole_field(text: &str, range: (usize, usize)) -> Snippet {
/// Build a snippet window around one known match range in `text`. The range /// Build a snippet window around one known match range in `text`. The range
/// is clamped into `text` and widened to char boundaries. /// is clamped into `text` and widened to char boundaries.
pub fn window_around(text: &str, range: (usize, usize), opts: &Options) -> Snippet { pub fn window_around(text: &str, range: (usize, usize), budget: usize) -> Snippet {
if text.is_empty() { if text.is_empty() {
return Snippet::empty(); return Snippet::empty();
} }
let (ms, me) = aligned_range(text, range); let (ms, me) = aligned_range(text, range);
let (win_start, win_end) = window_bounds(text, ms, opts.approx_bytes); let (win_start, win_end) = window_bounds(text, ms, budget);
// The whole match stays inside the window. // The whole match stays inside the window.
let win_end = win_end.max(me); let win_end = win_end.max(me);
@ -196,26 +198,6 @@ pub fn window_around(text: &str, range: (usize, usize), opts: &Options) -> Snipp
} }
} }
/// Count non-overlapping occurrences of `term` in `text`, byte for byte: a
/// case-insensitive count passes both already folded.
pub fn count_occurrences(text: &str, term: &str) -> usize {
if term.is_empty() || text.len() < term.len() {
return 0;
}
memchr::memmem::find_iter(text.as_bytes(), term.as_bytes()).count()
}
/// The first `n` bytes of `text` (char-aligned) as a match-less window.
fn head_window(text: &str, n: usize) -> Snippet {
let cut = text.floor_char_boundary(n);
Snippet {
window: text[..cut].to_string(),
ranges: Vec::new(),
truncated_start: false,
truncated_end: cut < text.len(),
}
}
/// Merge adjacent/overlapping ranges. Input must be sorted by start. /// Merge adjacent/overlapping ranges. Input must be sorted by start.
fn coalesce_overlapping(v: Vec<(usize, usize)>) -> Vec<(usize, usize)> { fn coalesce_overlapping(v: Vec<(usize, usize)>) -> Vec<(usize, usize)> {
let mut out = Vec::with_capacity(v.len()); let mut out = Vec::with_capacity(v.len());
@ -239,80 +221,87 @@ fn coalesce_overlapping(v: Vec<(usize, usize)>) -> Vec<(usize, usize)> {
mod tests { mod tests {
use super::*; use super::*;
fn extract(text: &str, term: &str, opts: &Options) -> Snippet { /// [`extract_folded`] with the folding done here; `typed` as given.
extract_folded(text, &text.to_ascii_lowercase(), term, opts).0 fn find(text: &str, typed: &str, budget: usize) -> Option<Found> {
let term = typed.to_ascii_lowercase();
extract_folded(text, &text.to_ascii_lowercase(), &term, typed, budget)
} }
fn extract_counted(text: &str, term: &str, opts: &Options) -> (Snippet, usize) { fn extract(text: &str, term: &str, budget: usize) -> Snippet {
extract_folded(text, &text.to_ascii_lowercase(), term, opts) find(text, term, budget).expect("the term occurs").snippet
} }
fn opts_small() -> Options { fn extract_counted(text: &str, term: &str, budget: usize) -> (Snippet, usize) {
Options { approx_bytes: 40 } let found = find(text, term, budget).expect("the term occurs");
(found.snippet, found.count)
} }
/// Non-overlapping occurrences, byte for byte.
fn memmem_count(text: &str, term: &str) -> usize {
memchr::memmem::find_iter(text.as_bytes(), term.as_bytes()).count()
}
/// A window budget small enough that tests see truncation.
const SMALL: usize = 40;
/// The count is occurrences, not the highlights they coalesce into. /// The count is occurrences, not the highlights they coalesce into.
#[test] #[test]
fn extract_folded_counts_occurrences_not_ranges() { fn extract_folded_counts_occurrences_not_ranges() {
let text = "abab and ab"; let text = "abab and ab";
let (s, n) = extract_counted(text, "ab", &opts_small()); let (s, n) = extract_counted(text, "ab", SMALL);
assert_eq!(n, 3, "three occurrences"); assert_eq!(n, 3, "three occurrences");
assert_eq!( assert_eq!(n, memmem_count(text, "ab"));
n,
count_occurrences(text, "ab"),
"must agree with count_occurrences"
);
assert_eq!(s.ranges.len(), 2, "the abutting pair paints as one range"); assert_eq!(s.ranges.len(), 2, "the abutting pair paints as one range");
} }
/// A disagreement here would change which rows survive the pass. /// A disagreement here would change which rows survive the pass, or
/// rank them: both counts must match separate searches, one over the
/// folded body and one over the body as stored.
#[test] #[test]
fn extract_folded_count_agrees_with_count_occurrences() { fn extract_folded_counts_agree_with_separate_searches() {
let cases: &[(&str, &str)] = &[ let cases: &[(&str, &str)] = &[
("no hits at all", "zzz"),
("one hit here", "hit"), ("one hit here", "hit"),
("Hit hit HIT", "hit"), ("Hit hit HIT", "hit"),
("Hit hit HIT", "HIT"),
("Hit hit HIT", "hIT"),
("aaaa", "aa"), ("aaaa", "aa"),
("", "x"), ("ünïcode ÜNÏCODE ünïcode", "ünïcode"),
("short", "much longer than the haystack"), ("Needle and needle and NEEDLE", "Needle"),
("ünïcode ünïcode", "ünïcode"),
]; ];
for (text, term) in cases { for (text, typed) in cases {
let found = find(text, typed, SMALL).expect("the term occurs");
let folded = text.to_ascii_lowercase(); let folded = text.to_ascii_lowercase();
let (_, n) = extract_folded(text, &folded, term, &opts_small()); let term = typed.to_ascii_lowercase();
assert_eq!( assert_eq!(
n, found.count,
count_occurrences(&folded, &term.to_ascii_lowercase()), memmem_count(&folded, &term),
"count mismatch for {:?} in {:?}", "{typed:?} in {text:?}"
term, );
text assert_eq!(
found.exact,
memmem_count(text, typed),
"{typed:?} in {text:?}"
); );
} }
} }
/// The owning and borrowing branches of the fold must find the same
/// matches.
#[test] #[test]
fn extract_folded_needle_case_does_not_change_matches() { fn a_term_that_does_not_occur_finds_nothing() {
let text = "The Needle and the needle"; for (text, term) in [
let folded = text.to_ascii_lowercase(); ("no hits at all", "zzz"),
let (upper, n_upper) = extract_folded(text, &folded, "Needle", &opts_small()); ("", "x"),
let (lower, n_lower) = extract_folded(text, &folded, "needle", &opts_small()); ("short", "much longer than the haystack"),
assert_eq!(n_upper, 2); ("alpha beta gamma", "nomatch"),
assert_eq!(n_upper, n_lower); ] {
assert_eq!(upper.ranges, lower.ranges); assert!(find(text, term, SMALL).is_none(), "{term:?} in {text:?}");
assert_eq!(upper.window, lower.window); }
} }
/// An empty needle would match at every byte offset. /// An empty needle would match at every byte offset.
#[test] #[test]
fn extract_folded_ignores_blank_terms() { fn extract_folded_ignores_blank_terms() {
let text = "some text";
let folded = text.to_ascii_lowercase();
for term in ["", " ", "\t"] { for term in ["", " ", "\t"] {
let (s, n) = extract_folded(text, &folded, term, &opts_small()); assert!(find("some text", term, SMALL).is_none(), "{term:?}");
assert_eq!(n, 0, "blank term {:?} counted", term);
assert!(s.ranges.is_empty(), "blank term {:?} highlighted", term);
} }
} }
@ -320,10 +309,8 @@ mod tests {
#[test] #[test]
fn extract_folded_does_not_trim_a_padded_term() { fn extract_folded_does_not_trim_a_padded_term() {
let text = "needle needlework"; let text = "needle needlework";
let folded = text.to_ascii_lowercase(); let (_, n) = extract_counted(text, "needle ", SMALL);
let (_, n) = extract_folded(text, &folded, "needle ", &opts_small());
assert_eq!(n, 1, "only the occurrence followed by a space"); assert_eq!(n, 1, "only the occurrence followed by a space");
assert_eq!(n, count_occurrences(&folded, "needle "));
} }
/// The `Snippet::ranges` contract egui's LayoutJob sections rely on. /// The `Snippet::ranges` contract egui's LayoutJob sections rely on.
@ -349,7 +336,7 @@ mod tests {
fn occurrences_past_the_window_are_counted_but_not_kept() { fn occurrences_past_the_window_are_counted_but_not_kept() {
let unit = "needle filler filler "; let unit = "needle filler filler ";
let text = unit.repeat(400); let text = unit.repeat(400);
let (snip, found) = extract_counted(&text, "needle", &opts_small()); let (snip, found) = extract_counted(&text, "needle", SMALL);
assert_eq!(found, 400, "every occurrence must still be counted"); assert_eq!(found, 400, "every occurrence must still be counted");
assert_ranges_valid(&snip); assert_ranges_valid(&snip);
@ -378,7 +365,7 @@ mod tests {
#[test] #[test]
fn a_coalescing_chain_is_not_cut_at_the_window_edge() { fn a_coalescing_chain_is_not_cut_at_the_window_edge() {
let text = "ab".repeat(400); let text = "ab".repeat(400);
let (snip, found) = extract_counted(&text, "ab", &opts_small()); let (snip, found) = extract_counted(&text, "ab", SMALL);
assert_eq!(found, 400, "non-overlapping occurrences, all counted"); assert_eq!(found, 400, "non-overlapping occurrences, all counted");
assert_ranges_valid(&snip); assert_ranges_valid(&snip);
@ -403,46 +390,23 @@ mod tests {
); );
} }
#[test]
fn empty_text_returns_empty() {
let s = extract("", "foo", &Options::default());
assert_eq!(s, Snippet::empty());
}
#[test]
fn a_blank_term_returns_head_marked_truncated() {
let long = "abcdefghijklmnop".repeat(10);
let s = extract(&long, "", &opts_small());
assert!(s.truncated_end);
assert!(!s.truncated_start);
assert!(s.ranges.is_empty());
assert!(s.window.len() <= 40);
}
#[test]
fn a_blank_term_untruncated() {
let s = extract("short text", "", &opts_small());
assert_eq!(s.window, "short text");
assert!(!s.truncated_end && !s.truncated_start);
}
#[test] #[test]
fn simple_match_range() { fn simple_match_range() {
let s = extract("the quick brown fox", "quick", &opts_small()); let s = extract("the quick brown fox", "quick", SMALL);
assert_eq!(marked(&s), vec!["quick"]); assert_eq!(marked(&s), vec!["quick"]);
assert_ranges_valid(&s); assert_ranges_valid(&s);
} }
#[test] #[test]
fn case_insensitive_match_reports_original_case() { fn case_insensitive_match_reports_original_case() {
let s = extract("The QUICK brown fox", "quick", &opts_small()); let s = extract("The QUICK brown fox", "quick", SMALL);
assert_eq!(marked(&s), vec!["QUICK"]); assert_eq!(marked(&s), vec!["QUICK"]);
} }
#[test] #[test]
fn window_truncation_flags_on_both_sides() { fn window_truncation_flags_on_both_sides() {
let text = "prefix ".repeat(20) + "MATCH in middle " + &"suffix ".repeat(20); let text = "prefix ".repeat(20) + "MATCH in middle " + &"suffix ".repeat(20);
let s = extract(&text, "MATCH", &opts_small()); let s = extract(&text, "MATCH", SMALL);
assert!(s.truncated_start); assert!(s.truncated_start);
assert!(s.truncated_end); assert!(s.truncated_end);
assert_eq!(marked(&s), vec!["MATCH"]); assert_eq!(marked(&s), vec!["MATCH"]);
@ -450,27 +414,15 @@ mod tests {
#[test] #[test]
fn match_at_start_not_truncated_left() { fn match_at_start_not_truncated_left() {
let s = extract( let s = extract("MATCH right at the start of this paragraph", "match", SMALL);
"MATCH right at the start of this paragraph",
"match",
&opts_small(),
);
assert!(!s.truncated_start); assert!(!s.truncated_start);
assert_eq!(s.ranges[0].0, 0); assert_eq!(s.ranges[0].0, 0);
} }
#[test]
fn no_match_returns_head_without_ranges() {
let text = "alpha beta gamma delta epsilon zeta eta theta iota kappa";
let s = extract(text, "nomatch", &opts_small());
assert!(s.ranges.is_empty());
assert!(s.window.starts_with("alpha"));
}
#[test] #[test]
fn utf8_boundaries_hold_with_multibyte_text() { fn utf8_boundaries_hold_with_multibyte_text() {
let text = "café café café café café café café café café café"; let text = "café café café café café café café café café café";
let s = extract(text, "café", &opts_small()); let s = extract(text, "café", SMALL);
assert!(!s.ranges.is_empty()); assert!(!s.ranges.is_empty());
assert_ranges_valid(&s); assert_ranges_valid(&s);
for m in marked(&s) { for m in marked(&s) {
@ -482,7 +434,7 @@ mod tests {
fn match_near_right_edge_is_fully_included() { fn match_near_right_edge_is_fully_included() {
let prefix = "x".repeat(30); let prefix = "x".repeat(30);
let text = format!("{}{}", prefix, "LONGMATCHTERMTEXT"); let text = format!("{}{}", prefix, "LONGMATCHTERMTEXT");
let s = extract(&text, "LONGMATCHTERMTEXT", &opts_small()); let s = extract(&text, "LONGMATCHTERMTEXT", SMALL);
assert_eq!(marked(&s), vec!["LONGMATCHTERMTEXT"]); assert_eq!(marked(&s), vec!["LONGMATCHTERMTEXT"]);
} }
@ -490,7 +442,7 @@ mod tests {
fn window_around_basic() { fn window_around_basic() {
let text = "prefix ".repeat(20) + "NEEDLE" + &" suffix".repeat(20); let text = "prefix ".repeat(20) + "NEEDLE" + &" suffix".repeat(20);
let at = text.find("NEEDLE").unwrap(); let at = text.find("NEEDLE").unwrap();
let s = window_around(&text, (at, at + 6), &opts_small()); let s = window_around(&text, (at, at + 6), SMALL);
assert_eq!(marked(&s), vec!["NEEDLE"]); assert_eq!(marked(&s), vec!["NEEDLE"]);
assert!(s.truncated_start && s.truncated_end); assert!(s.truncated_start && s.truncated_end);
assert_ranges_valid(&s); assert_ranges_valid(&s);
@ -498,28 +450,18 @@ mod tests {
#[test] #[test]
fn window_around_clamps_out_of_bounds() { fn window_around_clamps_out_of_bounds() {
let s = window_around("tiny", (2, 999), &opts_small()); let s = window_around("tiny", (2, 999), SMALL);
assert_eq!(s.window, "tiny"); assert_eq!(s.window, "tiny");
assert_eq!(s.ranges, vec![(2, 4)]); assert_eq!(s.ranges, vec![(2, 4)]);
// Fully out-of-range → no ranges, but never a panic. // Fully out-of-range → no ranges, but never a panic.
let s = window_around("tiny", (999, 1000), &opts_small()); let s = window_around("tiny", (999, 1000), SMALL);
assert!(s.ranges.is_empty()); assert!(s.ranges.is_empty());
} }
#[test] #[test]
fn window_around_aligns_multibyte_boundaries() { fn window_around_aligns_multibyte_boundaries() {
let text = "ééééééééé needle ééééééééé"; let text = "ééééééééé needle ééééééééé";
let s = window_around(text, (1, 3), &opts_small()); let s = window_around(text, (1, 3), SMALL);
assert_ranges_valid(&s); assert_ranges_valid(&s);
} }
#[test]
fn count_occurrences_cases() {
assert_eq!(count_occurrences("aaaa", "aaa"), 1, "non-overlapping");
assert_eq!(count_occurrences("abcABC", "abc"), 1, "byte for byte");
assert_eq!(count_occurrences("", "x"), 0);
assert_eq!(count_occurrences("xyz", ""), 0);
assert_eq!(count_occurrences("no hits here", "zzz"), 0);
assert_eq!(count_occurrences("ab ab ab", "ab"), 3);
}
} }

2
crates/quicksearch-core/src/testutil.rs
View file

@ -452,6 +452,8 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) {
} }
} }
tx.commit().unwrap(); tx.commit().unwrap();
// As a finished run's maintenance leaves it.
crate::db::repo::build_hash_index(&conn).unwrap();
conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);").ok(); conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);").ok();
} }

160
crates/quicksearch-core/src/walk.rs
View file

@ -16,8 +16,8 @@ use std::thread::JoinHandle;
use crate::config::{Config, IgnoreSet}; use crate::config::{Config, IgnoreSet};
use crate::extract::Registry; use crate::extract::Registry;
use crate::file_handling::{ use crate::file_handling::{
classify_by_mtime, classify_for_indexing, dir_to_db_parent, path_to_db_string, classify_by_mtime, dir_to_db_parent, path_to_db_string, prepare_file_record, DirRows,
prepare_file_record, DirRows, FileIndexAction, OwnedNewFile, UnreadableDirs, FileIndexAction, OwnedNewFile, UnreadableDirs,
}; };
mod pool; mod pool;
@ -30,6 +30,10 @@ use pool::{Found, PrefetchWork, Queue, Shared};
/// Files one worker takes for itself before handing the rest to the pool. /// Files one worker takes for itself before handing the rest to the pool.
const FILES_PER_JOB: usize = 128; const FILES_PER_JOB: usize = 128;
/// Events a root's walk may run ahead of the writer. Each can carry up to
/// `hash_length` of inline text, but the run's peak is set during extraction:
/// measured at 512 (`memprobe cold`, `/usr`, 373k files), peak RSS 343–355
/// against 293–309 MiB here and the walk no faster. Measured and rejected.
const CHANNEL_CAP: usize = 4096; const CHANNEL_CAP: usize = 4096;
const LOCAL_THREADS: usize = 4; const LOCAL_THREADS: usize = 4;
@ -77,11 +81,14 @@ pub enum WalkEvent {
struct PendingFile { struct PendingFile {
path: PathBuf, path: PathBuf,
cached: Option<crate::platform::CachedMetadata>, cached: Option<crate::platform::CachedMetadata>,
/// The stored mtime, taken out of the directory's rows as it was listed;
/// `None` when the file has no row yet.
stored: Option<u64>,
} }
enum Job { enum Job {
Dir(PathBuf, Arc<DirRows>), Dir(PathBuf, DirRows),
Files(Vec<PendingFile>, Arc<DirRows>), Files(Vec<PendingFile>),
/// A resolved symlink target, with the stored mtime for its own path. /// A resolved symlink target, with the stored mtime for its own path.
Alias(PathBuf, Option<u64>), Alias(PathBuf, Option<u64>),
} }
@ -146,9 +153,12 @@ pub fn reset_run_warnings() {
/// for the pool, the remaining files come back for this worker, `stale` gets /// for the pool, the remaining files come back for this worker, `stale` gets
/// the paths whose row has no file behind it. A directory that cannot be read /// the paths whose row has no file behind it. A directory that cannot be read
/// returns before reconciling: it must not read as an empty one. /// returns before reconciling: it must not read as an empty one.
///
/// Each listed file takes its row out of `rows`, carrying the stored mtime
/// with it, so what is left afterwards is exactly the stale set.
fn read_directory( fn read_directory(
dir: &Path, dir: &Path,
rows: &Arc<DirRows>, mut rows: DirRows,
ctx: &Ctx, ctx: &Ctx,
found: &mut Vec<Found>, found: &mut Vec<Found>,
stale: &mut Vec<String>, stale: &mut Vec<String>,
@ -169,17 +179,11 @@ fn read_directory(
} }
}; };
// Every `continue` in the loop must be a genuine "not indexable", or the // Every `continue` in the loop must be a genuine "not indexable", or the
// stale diff below deletes live rows. Only names that have a row matter to // stale diff below deletes live rows: only a name taken out of `rows`
// that diff, so `present` borrows them from `rows`: no allocation per // keeps its row. Measured against marking names present in a set borrowed
// entry, and none at all for a directory with no rows yet. Measured // from `rows` and looking each one up again in `prepare` (`indexprobe
// (`indexprobe cold`, 100k files): 8.2 → 7.2 allocations per file, // warm`, `/usr`, 373k files): 9.3–9.8 → 8.8–9.3 allocations per file,
// churn 42 → 35 MiB; time did not move, being `stat`-bound. // churn 246–257 → 231–240 MiB; time within noise, being `stat`-bound.
let mut present: HashSet<&str> = HashSet::new();
let mut mark_present = |name: &str| {
if let Some((known, _)) = rows.get_key_value(name) {
present.insert(known.as_str());
}
};
let mut unreadable_entry = false; let mut unreadable_entry = false;
let mut files = Vec::new(); let mut files = Vec::new();
@ -198,17 +202,22 @@ fn read_directory(
} }
}; };
let name = entry.file_name(); // The name borrowed from the path, which the entry allocates either
// way: `file_name()` would allocate a second copy. With the path's
// buffer becoming its `String` in `prepare`, measured (`indexprobe
// warm`, `/usr`, 373k files): 9.2–9.6 → 6.6–6.9 allocations per
// file, churn 240–247 → 203–210 MiB, user CPU 0.92–1.01 → 0.83–0.92 s.
let path = entry.path();
// The walk's one screen for names the index cannot spell (invalid // The walk's one screen for names the index cannot spell (invalid
// UTF-8 on Unix, unpaired UTF-16 surrogates on Windows; see // UTF-8 on Unix, unpaired UTF-16 surrogates on Windows; see
// `path_to_db_string`), skipped by choice. Leaving the entry out of // `path_to_db_string`), skipped by choice. Its row cannot exist: no
// `present` is safe: no stored row can carry such a name. // stored row can carry such a name.
let Some(name) = name.to_str() else { let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
if UNREPRESENTABLE_WARNINGS.allow() { if UNREPRESENTABLE_WARNINGS.allow() {
crate::log_warn!( crate::log_warn!(
"Skipping {:?} (name is not valid UTF-8, so it cannot be stored, hashed \ "Skipping {:?} (name is not valid UTF-8, so it cannot be stored, hashed \
or text-indexed)", or text-indexed)",
entry.path() path
); );
} }
continue; continue;
@ -233,7 +242,7 @@ fn read_directory(
crate::log_info!( crate::log_info!(
"skipping {}: hidden attribute set (enable \"include hidden \ "skipping {}: hidden attribute set (enable \"include hidden \
files\" to index it)", files\" to index it)",
entry.path().display() path.display()
); );
} }
continue; continue;
@ -242,13 +251,12 @@ fn read_directory(
} }
// Ignored, or the index's own database and sidecars (see // Ignored, or the index's own database and sidecars (see
// `file_handling::index_file_set`). `continue`, not `skipped`: the // `file_handling::index_file_set`). `continue`, not `skipped`: the
// name stays out of `present`, so old rows for the index fall to the // row stays in `rows`, so old rows for the index fall to the stale
// stale sweep; `skipped` would keep them. // sweep; `skipped` would keep them.
if ctx.ignore.matches_component(name) { if ctx.ignore.matches_component(name) {
ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed); ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed);
continue; continue;
} }
let path = entry.path();
if ctx.ignore.matches_path_pattern(&path) || ctx.index_files.contains(&path) { if ctx.ignore.matches_path_pattern(&path) || ctx.index_files.contains(&path) {
ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed); ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed);
continue; continue;
@ -256,13 +264,13 @@ fn read_directory(
// `file_type` is the cached `d_type` from the directory read. // `file_type` is the cached `d_type` from the directory read.
match entry.file_type() { match entry.file_type() {
// Not marked present: a name that was a file last run and is a // Row left in `rows`: a name that was a file last run and is a
// directory now should lose its row. // directory now should lose its row.
Ok(ft) if ft.is_dir() => found.push(Found::Dir(path)), Ok(ft) if ft.is_dir() => found.push(Found::Dir(path)),
// Directory and file targets gate together, or the walkers // Directory and file targets gate together, or the walkers
// disagree: `filtered_walk` follows neither kind, so a file target // disagree: `filtered_walk` follows neither kind, so a file target
// followed only here would never update between runs. Never // followed only here would never update between runs. Row left
// marked present: the target's row belongs to its own directory. // in `rows`: the target's row belongs to its own directory.
Ok(ft) if ft.is_symlink() => { Ok(ft) if ft.is_symlink() => {
if ctx.follow_symlinks { if ctx.follow_symlinks {
if let Some(target) = resolve_link(&path, ctx) { if let Some(target) = resolve_link(&path, ctx) {
@ -271,29 +279,31 @@ fn read_directory(
} }
} }
Ok(_) => { Ok(_) => {
mark_present(name); let stored = rows.remove(name);
// `None` on Unix and on any reparse point. // `None` on Unix and on any reparse point.
let cached = crate::platform::entry_cached_metadata(|| entry.metadata().ok()); let cached = crate::platform::entry_cached_metadata(|| entry.metadata().ok());
files.push(PendingFile { path, cached }); files.push(PendingFile {
path,
cached,
stored,
});
}
// Type unknown: taken out so an existing row survives.
Err(_) => {
rows.remove(name);
} }
// Type unknown: mark it present so an existing row survives.
Err(_) => mark_present(name),
} }
} }
if !unreadable_entry { if !unreadable_entry {
stale.extend( stale.extend(rows.into_keys().map(|name| path_to_db_string(&dir.join(name))));
rows.keys()
.filter(|name| !present.contains(name.as_str()))
.map(|name| path_to_db_string(&dir.join(name))),
);
} }
// Spread a wide directory across the pool, keeping the tail for ourselves // Spread a wide directory across the pool, keeping the tail for ourselves
// so the entries the read just warmed are handled now. // so the entries the read just warmed are handled now.
while files.len() > FILES_PER_JOB { while files.len() > FILES_PER_JOB {
let chunk = files.split_off(files.len() - FILES_PER_JOB); let chunk = files.split_off(files.len() - FILES_PER_JOB);
found.push(Found::Files(chunk, rows.clone())); found.push(Found::Files(chunk));
} }
files files
} }
@ -333,38 +343,35 @@ fn resolve_link(path: &Path, ctx: &Ctx) -> Option<Found> {
} }
} }
/// How a file's stored mtime is to be found.
enum Known<'a> {
InDir(&'a DirRows),
/// Resolved by exact path: a symlink target's row lives under a
/// different parent.
Exact(Option<u64>),
}
/// At most one `stat`, then classify; only files that will be written get /// At most one `stat`, then classify; only files that will be written get
/// opened, and small text files are finished outright. "At most": on Windows /// opened, and small text files are finished outright. "At most": on Windows
/// [`PendingFile::cached`] may already hold the answer. /// [`PendingFile::cached`] may already hold the answer.
/// ///
/// `scratch` is this worker's, carrying the head buffer every hash and MIME /// `aliased`: reached by resolving a symlink, so its row lives under a
/// sniff reads into. /// different parent than the link's. `scratch` is this worker's, carrying the
/// head buffer every hash and MIME sniff reads into.
fn prepare( fn prepare(
file: PendingFile, file: PendingFile,
known: Known<'_>, aliased: bool,
ctx: &Ctx, ctx: &Ctx,
scratch: &mut crate::extract::Scratch, scratch: &mut crate::extract::Scratch,
) -> WalkedFile { ) -> WalkedFile {
let PendingFile { path, cached } = file; let PendingFile {
// Every route here has already screened the path for UTF-8; see path,
// `path_to_db_string`. cached,
debug_assert!( stored,
path.to_str().is_some(), } = file;
"an unrepresentable path reached prepare(): {:?}", // Every route here has already screened the path for UTF-8, and every
path // walk path is a normalized root or link target with names joined on, so
); // it already is its stored spelling: the buffer becomes the `String`
let db_path = path_to_db_string(&path); // without a copy (see `path_to_db_string`).
let aliased = matches!(known, Known::Exact(_)); let db_path = path.into_os_string().into_string().unwrap_or_else(|path| {
debug_assert!(false, "an unrepresentable path reached prepare(): {:?}", path);
path_to_db_string(Path::new(&path))
});
debug_assert_eq!(db_path, path_to_db_string(Path::new(&db_path)));
let Ok(meta) = crate::platform::metadata_or_stat(&path, cached) else { let Ok(meta) = crate::platform::metadata_or_stat(Path::new(&db_path), cached) else {
// Seen but unreadable: a transient stat failure must not read as // Seen but unreadable: a transient stat failure must not read as
// "deleted". // "deleted".
return WalkedFile::skipped(db_path, aliased); return WalkedFile::skipped(db_path, aliased);
@ -373,17 +380,7 @@ fn prepare(
return WalkedFile::skipped(db_path, aliased); return WalkedFile::skipped(db_path, aliased);
}; };
let action = match known { let action = classify_by_mtime(stored, mtime);
Known::InDir(rows) => {
// `to_str`, never lossy; the screen makes it always `Some`.
let name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or_default();
classify_for_indexing(name, mtime, rows)
}
Known::Exact(stored) => classify_by_mtime(stored, mtime),
};
let record = match action { let record = match action {
// Unchanged: never opened or hashed, so it stays at one syscall. // Unchanged: never opened or hashed, so it stays at one syscall.
FileIndexAction::Skip => None, FileIndexAction::Skip => None,
@ -414,20 +411,24 @@ fn worker(shared: &Shared, ctx: &Ctx, tx: &mpsc::SyncSender<WalkEvent>) {
let delivered = match job { let delivered = match job {
Job::Dir(dir, rows) => { Job::Dir(dir, rows) => {
let (mut found, mut stale) = (Vec::new(), Vec::new()); let (mut found, mut stale) = (Vec::new(), Vec::new());
let files = read_directory(&dir, &rows, ctx, &mut found, &mut stale); let files = read_directory(&dir, rows, ctx, &mut found, &mut stale);
// Hand the subdirectories over before doing our own per-file // Hand the subdirectories over before doing our own per-file
// work, so the rest of the pool never idles behind one worker. // work, so the rest of the pool never idles behind one worker.
slot.finish(found); slot.finish(found);
deliver(files, &rows, stale, ctx, tx, &mut scratch) deliver(files, stale, ctx, tx, &mut scratch)
} }
Job::Files(files, rows) => { Job::Files(files) => {
slot.finish(Vec::new()); slot.finish(Vec::new());
deliver(files, &rows, Vec::new(), ctx, tx, &mut scratch) deliver(files, Vec::new(), ctx, tx, &mut scratch)
} }
Job::Alias(path, stored) => { Job::Alias(path, stored) => {
slot.finish(Vec::new()); slot.finish(Vec::new());
let file = PendingFile { path, cached: None }; let file = PendingFile {
let walked = prepare(file, Known::Exact(stored), ctx, &mut scratch); path,
cached: None,
stored,
};
let walked = prepare(file, true, ctx, &mut scratch);
tx.send(WalkEvent::File(walked)).is_ok() tx.send(WalkEvent::File(walked)).is_ok()
} }
}; };
@ -443,7 +444,6 @@ fn worker(shared: &Shared, ctx: &Ctx, tx: &mpsc::SyncSender<WalkEvent>) {
/// failed: not an error. /// failed: not an error.
fn deliver( fn deliver(
files: Vec<PendingFile>, files: Vec<PendingFile>,
rows: &DirRows,
stale: Vec<String>, stale: Vec<String>,
ctx: &Ctx, ctx: &Ctx,
tx: &mpsc::SyncSender<WalkEvent>, tx: &mpsc::SyncSender<WalkEvent>,
@ -456,7 +456,7 @@ fn deliver(
if ctx.stop_flag.load(Ordering::Relaxed) { if ctx.stop_flag.load(Ordering::Relaxed) {
return false; return false;
} }
let walked = prepare(file, Known::InDir(rows), ctx, scratch); let walked = prepare(file, false, ctx, scratch);
tx.send(WalkEvent::File(walked)).is_ok() tx.send(WalkEvent::File(walked)).is_ok()
}) })
} }
@ -478,7 +478,7 @@ fn prefetcher(shared: &Shared, db_path: &str) {
while let Some(work) = shared.take_prefetch() { while let Some(work) = shared.take_prefetch() {
let job = match work { let job = match work {
PrefetchWork::Dir(dir) => crate::db::repo::dir_rows(&conn, &dir_to_db_parent(&dir)) PrefetchWork::Dir(dir) => crate::db::repo::dir_rows(&conn, &dir_to_db_parent(&dir))
.map(|rows| Job::Dir(dir, Arc::new(rows))), .map(|rows| Job::Dir(dir, rows)),
PrefetchWork::Alias(path) => { PrefetchWork::Alias(path) => {
crate::db::repo::mtime_for_path(&conn, &path_to_db_string(&path)) crate::db::repo::mtime_for_path(&conn, &path_to_db_string(&path))
.map(|stored| Job::Alias(path, stored)) .map(|stored| Job::Alias(path, stored))

9
crates/quicksearch-core/src/walk/pool.rs
View file

@ -17,7 +17,8 @@ pub(super) struct Queue {
/// end-of-walk proof in [`Shared::take`]. /// end-of-walk proof in [`Shared::take`].
pub(super) prefetching: bool, pub(super) prefetching: bool,
/// What [`PREFETCH_AHEAD`] bounds. Not `jobs.len()`: that also counts /// What [`PREFETCH_AHEAD`] bounds. Not `jobs.len()`: that also counts
/// `Job::Files` chunks, which carry no rows and would starve prefetching. /// `Job::Files` chunks, which hold no directory's rows and would starve
/// prefetching.
pub(super) dirs_ready: usize, pub(super) dirs_ready: usize,
pub(super) active: usize, pub(super) active: usize,
/// Canonical directories already queued: collapses overlapping roots, /// Canonical directories already queued: collapses overlapping roots,
@ -173,8 +174,8 @@ pub(super) enum PrefetchWork {
pub(super) enum Found { pub(super) enum Found {
Dir(PathBuf), Dir(PathBuf),
Alias(PathBuf), Alias(PathBuf),
/// Overflow files from the directory just read, which already has rows. /// Overflow files from the directory just read, each carrying its row.
Files(Vec<PendingFile>, Arc<DirRows>), Files(Vec<PendingFile>),
} }
impl Shared { impl Shared {
@ -192,7 +193,7 @@ impl Shared {
} }
} }
Found::Alias(path) => q.needs_alias.push(path), Found::Alias(path) => q.needs_alias.push(path),
Found::Files(files, rows) => q.jobs.push(Job::Files(files, rows)), Found::Files(files) => q.jobs.push(Job::Files(files)),
} }
} }
q.active -= 1; q.active -= 1;

52
crates/quicksearch-core/src/walk/tests.rs
View file

@ -177,9 +177,9 @@ fn a_non_utf8_name_is_dropped_at_the_entry() {
); );
} }
/// A `continue` in `read_directory` leaves the name out of `present`, and /// A `continue` in `read_directory` leaves the name's row in `rows`, and
/// everything left out of `present` is deleted; the row spelled with the lossy /// every row left there is deleted; the row spelled with the lossy name
/// name belongs to a different file, which must mark itself present. /// belongs to a different file, which must take its own row out.
#[test] #[test]
fn dropping_a_bad_entry_deletes_nothing() { fn dropping_a_bad_entry_deletes_nothing() {
let root = tmp_tree("nonutf8-stale"); let root = tmp_tree("nonutf8-stale");
@ -659,6 +659,50 @@ fn a_directory_reports_rows_with_no_file_behind_them() {
); );
} }
/// A directory wider than one job is spread across the pool, and each file
/// takes its stored mtime with it: every chunk must classify against the
/// file's own row, and only the row with no file is stale.
#[test]
fn every_file_of_a_wide_directory_is_classified_against_its_own_row() {
let root = tmp_tree("wide");
let total = FILES_PER_JOB * 2 + 7;
let path = |i: usize| root.join(format!("f{:04}.txt", i));
for i in 0..total {
touch(&path(i), b"x");
}
let mtime =
|i: usize| crate::file_handling::mtime_secs(&fs::metadata(path(i)).unwrap()).unwrap();
// Thirds: unchanged, changed, never indexed.
let mut rows: Vec<(String, u64)> = (0..total)
.filter(|i| i % 3 != 2)
.map(|i| (path_to_db_string(&path(i)), mtime(i) + (i % 3) as u64))
.collect();
let gone = path_to_db_string(&root.join("gone.txt"));
rows.push((gone.clone(), 1));
let db = db_with("wide", &rows);
let (mut files, mut stale) = (Vec::new(), Vec::new());
for event in start(&root, &db) {
match event {
WalkEvent::File(f) => files.push(f),
WalkEvent::Stale(paths) => stale.extend(paths),
}
}
assert_eq!(files.len(), total);
for f in &files {
let name = Path::new(&f.path).file_name().unwrap().to_str().unwrap();
let i: usize = name[1..5].parse().unwrap();
let want = [
FileIndexAction::Skip,
FileIndexAction::Update,
FileIndexAction::Insert,
][i % 3];
assert_eq!(f.action, want, "{}", name);
}
assert_eq!(stale, vec![gone], "only the row with no file behind it");
}
#[test] #[test]
#[cfg(unix)] #[cfg(unix)]
fn an_unreadable_directory_reports_nothing_stale() { fn an_unreadable_directory_reports_nothing_stale() {
@ -819,7 +863,7 @@ fn the_index_and_its_sidecars_are_never_walked() {
/// A stored row for the index itself has to go, or the live watcher and /// A stored row for the index itself has to go, or the live watcher and
/// duplicate verification keep opening the file; a skip would keep the row, /// duplicate verification keep opening the file; a skip would keep the row,
/// so the walk leaves the name out of the directory's `present` set instead. /// so the walk leaves the name's row in the directory's `rows` instead.
#[test] #[test]
fn a_stored_row_for_the_index_falls_to_the_stale_sweep() { fn a_stored_row_for_the_index_falls_to_the_stale_sweep() {
let root = crate::testutil::scratch_dir_canonical("walk-self-index-stale"); let root = crate::testutil::scratch_dir_canonical("walk-self-index-stale");

201
crates/quicksearch-core/src/watcher.rs
View file

@ -6,12 +6,12 @@
//! the whole watcher fails: a half-watched root looks live while going //! the whole watcher fails: a half-watched root looks live while going
//! silently stale. //! silently stale.
use std::collections::{BTreeSet, HashMap}; use std::collections::BTreeSet;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{mpsc, Arc, Mutex}; use std::sync::{mpsc, Arc, Mutex};
use std::thread::{self, JoinHandle}; use std::thread::{self, JoinHandle};
use std::time::{Duration, Instant}; use std::time::Duration;
use notify::event::{ModifyKind, RenameMode}; use notify::event::{ModifyKind, RenameMode};
use notify::{ use notify::{
@ -116,20 +116,15 @@ fn fmt_cap(cap: usize) -> String {
} }
} }
/// Tuning for the whole filesystem-event pipeline. /// Tuning for the whole filesystem-event pipeline. The watcher forwards
/// events as they arrive; the coordinator's queue coalesces them by path and
/// decides when to apply them.
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct WatcherConfig { pub struct WatcherConfig {
/// How long the coordinator's queue must go quiet before it is applied. /// How long the coordinator's queue must go quiet before it is applied.
pub pending_settle: Duration, pub pending_settle: Duration,
/// Ceiling on how long `pending_settle` may hold the queue back. /// Ceiling on how long `pending_settle` may hold the queue back.
pub pending_max_defer: Duration, pub pending_max_defer: Duration,
/// Per-directory debounce window.
pub throttle_window: Duration,
pub tick_interval: Duration,
/// Directories *selected* per tick; [`FLUSH_BUDGET`] is the real limit.
pub max_dirs_per_tick: usize,
/// When to garbage-collect stale throttle entries (idle > window * N).
pub prune_max_age_multiplier: u32,
/// Directory budget; see [`DEFAULT_MAX_WATCHED_DIRS`]. /// Directory budget; see [`DEFAULT_MAX_WATCHED_DIRS`].
pub max_watched_dirs: usize, pub max_watched_dirs: usize,
} }
@ -139,10 +134,6 @@ impl Default for WatcherConfig {
Self { Self {
pending_settle: Duration::from_secs(2), pending_settle: Duration::from_secs(2),
pending_max_defer: Duration::from_secs(30), pending_max_defer: Duration::from_secs(30),
throttle_window: Duration::from_secs(30),
tick_interval: Duration::from_millis(500),
max_dirs_per_tick: 512,
prune_max_age_multiplier: 10,
max_watched_dirs: DEFAULT_MAX_WATCHED_DIRS, max_watched_dirs: DEFAULT_MAX_WATCHED_DIRS,
} }
} }
@ -156,8 +147,6 @@ struct WatchRegistry {
cap: usize, cap: usize,
} }
/// How long one flush pass may spend handing events to the sink.
const FLUSH_BUDGET: Duration = Duration::from_millis(50);
const WATCH_MODE: RecursiveMode = if crate::platform::WATCH_ROOTS_RECURSIVELY { const WATCH_MODE: RecursiveMode = if crate::platform::WATCH_ROOTS_RECURSIVELY {
RecursiveMode::Recursive RecursiveMode::Recursive
@ -240,7 +229,9 @@ fn add_watch(reg: &mut WatchRegistry, dir: &Path) -> Result<(), WatchError> {
/// Handle to a running watcher. Dropping calls [`Self::stop`] implicitly. /// Handle to a running watcher. Dropping calls [`Self::stop`] implicitly.
pub struct Watcher { pub struct Watcher {
stop_flag: Arc<AtomicBool>, /// The loop's own channel: `None` ends it. The notify handle holds the
/// other sender, so the channel never closes while the loop runs.
stop_tx: mpsc::Sender<Option<NotifyEvent>>,
handle: Option<JoinHandle<()>>, handle: Option<JoinHandle<()>>,
dir_count: Arc<AtomicUsize>, dir_count: Arc<AtomicUsize>,
degraded: Degraded, degraded: Degraded,
@ -261,7 +252,8 @@ impl Watcher {
config: WatcherConfig, config: WatcherConfig,
sink: EventSink, sink: EventSink,
) -> Result<Self, WatchError> { ) -> Result<Self, WatchError> {
let (tx, rx) = mpsc::channel::<NotifyEvent>(); let (tx, rx) = mpsc::channel::<Option<NotifyEvent>>();
let stop_tx = tx.clone();
let degraded: Degraded = Arc::new(Mutex::new(None)); let degraded: Degraded = Arc::new(Mutex::new(None));
let degraded_cb = degraded.clone(); let degraded_cb = degraded.clone();
let raw = RecommendedWatcher::new( let raw = RecommendedWatcher::new(
@ -274,7 +266,7 @@ impl Watcher {
crate::lock_ok(&degraded_cb).get_or_insert(WatchError::Overflowed); crate::lock_ok(&degraded_cb).get_or_insert(WatchError::Overflowed);
return; return;
} }
let _ = tx.send(ev); let _ = tx.send(Some(ev));
} }
Err(e) => { Err(e) => {
// The limit can also be hit asynchronously, when notify // The limit can also be hit asynchronously, when notify
@ -330,11 +322,8 @@ impl Watcher {
} }
let dir_count = Arc::new(AtomicUsize::new(crate::lock_ok(&registry).dirs.len())); let dir_count = Arc::new(AtomicUsize::new(crate::lock_ok(&registry).dirs.len()));
let stop_flag = Arc::new(AtomicBool::new(false));
let ctx = LoopCtx { let ctx = LoopCtx {
sink, sink,
config,
stop: stop_flag.clone(),
registry: registry.clone(), registry: registry.clone(),
filters, filters,
roots, roots,
@ -344,7 +333,7 @@ impl Watcher {
let handle = thread::spawn(move || run_loop(rx, ctx)); let handle = thread::spawn(move || run_loop(rx, ctx));
Ok(Self { Ok(Self {
stop_flag, stop_tx,
handle: Some(handle), handle: Some(handle),
dir_count, dir_count,
degraded, degraded,
@ -376,7 +365,7 @@ impl Watcher {
/// Stop the background thread and join it; safe to call multiple times. /// Stop the background thread and join it; safe to call multiple times.
pub fn stop(&mut self) { pub fn stop(&mut self) {
self.stop_flag.store(true, Ordering::Relaxed); let _ = self.stop_tx.send(None);
if let Some(h) = self.handle.take() { if let Some(h) = self.handle.take() {
let _ = h.join(); let _ = h.join();
} }
@ -400,8 +389,6 @@ impl std::fmt::Debug for Watcher {
struct LoopCtx { struct LoopCtx {
sink: EventSink, sink: EventSink,
config: WatcherConfig,
stop: Arc<AtomicBool>,
registry: Arc<Mutex<WatchRegistry>>, registry: Arc<Mutex<WatchRegistry>>,
filters: WatchFilters, filters: WatchFilters,
/// Configured roots; "hidden" is judged relative to them, since a root /// Configured roots; "hidden" is judged relative to them, since a root
@ -412,56 +399,21 @@ struct LoopCtx {
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum QueuedOp { enum Op {
Create, Create,
Modify, Modify,
Remove, Remove,
} }
#[derive(Debug)] /// Forward each interesting event to the sink as it arrives, blocking in
struct DirThrottleEntry { /// between; `None` is [`Watcher::stop`]. Nothing is held back here: the
record_time: Instant, /// coordinator's queue coalesces by path and waits for `pending_settle`
queue: HashMap<PathBuf, QueuedOp>, /// (at most `pending_max_defer`). A per-directory throttle here used to hold
/// Flush on the next tick regardless of window age; set for the first /// a directory's second event for 30 s and woke this thread twice a second
/// event in a previously-idle directory so it reacts fast. /// while idle.
immediate: bool, fn run_loop(rx: mpsc::Receiver<Option<NotifyEvent>>, ctx: LoopCtx) {
} while let Ok(Some(ev)) = rx.recv() {
handle_notify_event(&ev, &ctx);
/// Ticks between sweeps of idle throttle entries.
const PRUNE_EVERY_TICKS: u32 = 20;
fn run_loop(rx: mpsc::Receiver<NotifyEvent>, ctx: LoopCtx) {
let mut throttle: HashMap<PathBuf, DirThrottleEntry> = HashMap::new();
let mut tick_counter: u32 = 0;
while !ctx.stop.load(Ordering::Relaxed) {
let deadline = Instant::now() + ctx.config.tick_interval;
while !ctx.stop.load(Ordering::Relaxed) {
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
break;
}
match rx.recv_timeout(remaining) {
Ok(ev) => handle_notify_event(&ev, &mut throttle, &ctx),
Err(mpsc::RecvTimeoutError::Timeout) => break,
Err(mpsc::RecvTimeoutError::Disconnected) => return,
}
}
if ctx.stop.load(Ordering::Relaxed) {
break;
}
flush_ready(&mut throttle, &ctx.sink, &ctx.config);
tick_counter = tick_counter.wrapping_add(1);
if tick_counter.is_multiple_of(PRUNE_EVERY_TICKS) {
let max_age = ctx
.config
.throttle_window
.saturating_mul(ctx.config.prune_max_age_multiplier);
prune_stale(&mut throttle, max_age);
}
} }
} }
@ -525,7 +477,7 @@ fn unwatch_tree(ctx: &LoopCtx, path: &Path) {
/// Whether an event for `path` is worth queueing at all: the same predicate /// Whether an event for `path` is worth queueing at all: the same predicate
/// the walk applies. Under a recursive root watch it alone keeps /// the walk applies. Under a recursive root watch it alone keeps
/// `node_modules` churn out of the throttle map. A path the index cannot /// `node_modules` churn out of the coordinator's queue. A path the index cannot
/// spell is dropped (see `path_to_db_string`). /// spell is dropped (see `path_to_db_string`).
fn is_event_interesting(ctx: &LoopCtx, path: &Path) -> bool { fn is_event_interesting(ctx: &LoopCtx, path: &Path) -> bool {
path.to_str().is_some() path.to_str().is_some()
@ -533,11 +485,7 @@ fn is_event_interesting(ctx: &LoopCtx, path: &Path) -> bool {
&& (ctx.filters.include_hidden || !path_has_hidden_component_under(path, &ctx.roots)) && (ctx.filters.include_hidden || !path_has_hidden_component_under(path, &ctx.roots))
} }
fn handle_notify_event( fn handle_notify_event(ev: &NotifyEvent, ctx: &LoopCtx) {
ev: &NotifyEvent,
throttle: &mut HashMap<PathBuf, DirThrottleEntry>,
ctx: &LoopCtx,
) {
let op = match ev.kind { let op = match ev.kind {
EventKind::Modify(ModifyKind::Name(RenameMode::Both)) if ev.paths.len() == 2 => { EventKind::Modify(ModifyKind::Name(RenameMode::Both)) if ev.paths.len() == 2 => {
// A rename is only uninteresting when *both* ends are: out of an // A rename is only uninteresting when *both* ends are: out of an
@ -556,13 +504,9 @@ fn handle_notify_event(
} }
// Split renames degrade to Create/Remove; notify 6.x does not bundle // Split renames degrade to Create/Remove; notify 6.x does not bundle
// halves by cookie uniformly across backends. // halves by cookie uniformly across backends.
EventKind::Create(_) | EventKind::Modify(ModifyKind::Name(RenameMode::To)) => { EventKind::Create(_) | EventKind::Modify(ModifyKind::Name(RenameMode::To)) => Op::Create,
QueuedOp::Create EventKind::Remove(_) | EventKind::Modify(ModifyKind::Name(RenameMode::From)) => Op::Remove,
} EventKind::Modify(_) => Op::Modify,
EventKind::Remove(_) | EventKind::Modify(ModifyKind::Name(RenameMode::From)) => {
QueuedOp::Remove
}
EventKind::Modify(_) => QueuedOp::Modify,
_ => return, _ => return,
}; };
@ -570,83 +514,20 @@ fn handle_notify_event(
if !is_event_interesting(ctx, p) { if !is_event_interesting(ctx, p) {
continue; continue;
} }
// Keep the watch set in step before debouncing: a directory created // Keep the watch set in step before the event is applied: a
// now may be populated before its window expires and needs its watch. // directory created now may be populated at once and needs its watch.
match op { let event = match op {
QueuedOp::Create => watch_if_new_dir(ctx, p), Op::Create => {
QueuedOp::Remove => unwatch_tree(ctx, p), watch_if_new_dir(ctx, p);
QueuedOp::Modify => {} FsEvent::Create(p.clone())
}
enqueue(throttle, p.clone(), op);
}
}
fn enqueue(throttle: &mut HashMap<PathBuf, DirThrottleEntry>, path: PathBuf, op: QueuedOp) {
let dir = path
.parent()
.map(|p| p.to_path_buf())
.unwrap_or_else(|| path.clone());
let entry = throttle.entry(dir).or_insert_with(|| DirThrottleEntry {
record_time: Instant::now(),
queue: HashMap::new(),
immediate: true,
});
// Created and deleted within one window: the file never surfaces.
if op == QueuedOp::Remove && entry.queue.get(&path) == Some(&QueuedOp::Create) {
entry.queue.remove(&path);
} else {
entry.queue.insert(path, op);
}
}
fn flush_ready(
throttle: &mut HashMap<PathBuf, DirThrottleEntry>,
sink: &EventSink,
config: &WatcherConfig,
) {
let now = Instant::now();
let mut ready: Vec<PathBuf> = Vec::new();
for (dir, entry) in throttle.iter() {
let age = now.saturating_duration_since(entry.record_time);
if entry.immediate || (!entry.queue.is_empty() && age >= config.throttle_window) {
ready.push(dir.clone());
if ready.len() >= config.max_dirs_per_tick {
break;
} }
} Op::Remove => {
} unwatch_tree(ctx, p);
let deadline = now + FLUSH_BUDGET; FsEvent::Remove(p.clone())
for dir in ready {
if Instant::now() >= deadline {
break;
}
if let Some(entry) = throttle.get_mut(&dir) {
let drained: Vec<(PathBuf, QueuedOp)> = entry.queue.drain().collect();
entry.immediate = false;
entry.record_time = now;
for (path, op) in drained {
let ev = match op {
QueuedOp::Create => FsEvent::Create(path),
QueuedOp::Modify => FsEvent::Modify(path),
QueuedOp::Remove => FsEvent::Remove(path),
};
sink(ev);
} }
} Op::Modify => FsEvent::Modify(p.clone()),
} };
} (ctx.sink)(event);
fn prune_stale(throttle: &mut HashMap<PathBuf, DirThrottleEntry>, max_age: Duration) {
let now = Instant::now();
throttle.retain(|_, entry| {
!entry.queue.is_empty()
|| entry.immediate
|| now.saturating_duration_since(entry.record_time) < max_age
});
// `retain` keeps the table sized for the busiest moment this map has ever
// seen; a single unpack burst would otherwise hold that memory for days.
if throttle.is_empty() {
throttle.shrink_to_fit();
} }
} }

140
crates/quicksearch-core/src/watcher_tests.rs
View file

@ -18,19 +18,11 @@ fn default_filters() -> WatchFilters {
} }
} }
fn fast_config() -> WatcherConfig { /// The default config with a directory budget of `n`.
WatcherConfig {
throttle_window: Duration::from_millis(50),
tick_interval: Duration::from_millis(20),
..WatcherConfig::default()
}
}
/// [`fast_config`] with a directory budget of `n`.
fn capped(n: usize) -> WatcherConfig { fn capped(n: usize) -> WatcherConfig {
WatcherConfig { WatcherConfig {
max_watched_dirs: n, max_watched_dirs: n,
..fast_config() ..WatcherConfig::default()
} }
} }
@ -39,9 +31,9 @@ fn watch_with(dir: &Path, config: WatcherConfig, sink: EventSink) -> Result<Watc
Watcher::start(vec![dir.to_path_buf()], default_filters(), config, sink) Watcher::start(vec![dir.to_path_buf()], default_filters(), config, sink)
} }
/// [`watch_with`] at the fast defaults, into a sink nobody reads. /// [`watch_with`] at the defaults, into a sink nobody reads.
fn watch(dir: &Path) -> Watcher { fn watch(dir: &Path) -> Watcher {
watch_with(dir, fast_config(), sink_to_vec().0).unwrap() watch_with(dir, WatcherConfig::default(), sink_to_vec().0).unwrap()
} }
/// Seeds `dirs` directly so set logic is tested without real kernel watches. /// Seeds `dirs` directly so set logic is tested without real kernel watches.
@ -72,88 +64,44 @@ fn remove_tree_skips_the_scan_for_a_path_that_is_not_watched() {
); );
} }
/// Nothing holds an event back: a directory's second change reaches the
/// sink as promptly as its first. A per-directory throttle once held it for
/// 30 s; the coordinator's queue does the debouncing now.
#[test] #[test]
fn enqueue_create_then_remove_cancels() { fn every_event_reaches_the_sink_as_it_happens() {
let mut map: HashMap<PathBuf, DirThrottleEntry> = HashMap::new(); let dir = crate::testutil::scratch_dir("prompt");
let p = PathBuf::from("/tmp/a.txt");
enqueue(&mut map, p.clone(), QueuedOp::Create);
enqueue(&mut map, p.clone(), QueuedOp::Remove);
let entry = map.get(p.parent().unwrap()).unwrap();
assert!(entry.queue.is_empty(), "Create then Remove should cancel");
}
#[test]
fn modify_after_modify_is_one() {
let mut map: HashMap<PathBuf, DirThrottleEntry> = HashMap::new();
let p = PathBuf::from("/tmp/a.txt");
enqueue(&mut map, p.clone(), QueuedOp::Modify);
enqueue(&mut map, p.clone(), QueuedOp::Modify);
let entry = map.get(p.parent().unwrap()).unwrap();
assert_eq!(entry.queue.len(), 1);
}
#[test]
fn flush_ready_leading_edge_fires_immediately() {
let mut map: HashMap<PathBuf, DirThrottleEntry> = HashMap::new();
enqueue(&mut map, PathBuf::from("/tmp/a.txt"), QueuedOp::Create);
let (sink, got) = sink_to_vec(); let (sink, got) = sink_to_vec();
let config = WatcherConfig::default(); let mut w = watch_with(&dir, WatcherConfig::default(), sink).unwrap();
flush_ready(&mut map, &sink, &config);
let got = got.lock().unwrap();
assert_eq!(got.len(), 1);
assert!(matches!(got[0], FsEvent::Create(_)));
}
#[test] let seen = |name: &str| {
fn flush_ready_respects_max_dirs_per_tick() { let path = dir.join(name);
let mut map: HashMap<PathBuf, DirThrottleEntry> = HashMap::new(); let deadline = std::time::Instant::now() + Duration::from_secs(5);
for i in 0..10 { while std::time::Instant::now() < deadline {
enqueue( let found = got
&mut map, .lock()
PathBuf::from(format!("/dir{}/a", i)), .unwrap()
QueuedOp::Create, .iter()
); .any(|e| matches!(e, FsEvent::Create(p) | FsEvent::Modify(p) if p == &path));
} if found {
let (sink, got) = sink_to_vec(); return true;
let config = WatcherConfig { }
max_dirs_per_tick: 3, std::thread::sleep(Duration::from_millis(10));
..WatcherConfig::default() }
false
}; };
flush_ready(&mut map, &sink, &config); std::fs::write(dir.join("first.txt"), "a").unwrap();
// Each dir contributes one event because each entry has one path. assert!(seen("first.txt"));
assert_eq!(got.lock().unwrap().len(), 3); let second = std::time::Instant::now();
} std::fs::write(dir.join("second.txt"), "b").unwrap();
assert!(seen("second.txt"));
#[test] assert!(
fn prune_stale_drops_empty_old_entries() { second.elapsed() < Duration::from_secs(2),
let mut map: HashMap<PathBuf, DirThrottleEntry> = HashMap::new(); "the second event in a directory took {:?}",
map.insert( second.elapsed()
PathBuf::from("/tmp"),
DirThrottleEntry {
record_time: Instant::now() - Duration::from_secs(3600),
queue: HashMap::new(),
immediate: false,
},
); );
prune_stale(&mut map, Duration::from_secs(1));
assert!(map.is_empty());
}
#[test] w.stop();
fn prune_stale_keeps_active_entries() { std::fs::remove_dir_all(&dir).ok();
let mut map: HashMap<PathBuf, DirThrottleEntry> = HashMap::new();
let mut queue = HashMap::new();
queue.insert(PathBuf::from("/tmp/a"), QueuedOp::Modify);
map.insert(
PathBuf::from("/tmp"),
DirThrottleEntry {
record_time: Instant::now() - Duration::from_secs(3600),
queue,
immediate: false,
},
);
prune_stale(&mut map, Duration::from_secs(1));
assert_eq!(map.len(), 1);
} }
#[test] #[test]
@ -161,7 +109,7 @@ fn e2e_create_modify_remove_surfaces() {
let dir = crate::testutil::scratch_dir("e2e"); let dir = crate::testutil::scratch_dir("e2e");
let (sink, got) = sink_to_vec(); let (sink, got) = sink_to_vec();
let mut w = watch_with(&dir, fast_config(), sink).unwrap(); let mut w = watch_with(&dir, WatcherConfig::default(), sink).unwrap();
let f = dir.join("hello.txt"); let f = dir.join("hello.txt");
std::fs::write(&f, "hi").unwrap(); std::fs::write(&f, "hi").unwrap();
@ -200,7 +148,7 @@ fn events_for_an_unrepresentable_name_never_surface() {
std::fs::remove_file(&bad).unwrap(); std::fs::remove_file(&bad).unwrap();
let (sink, got) = sink_to_vec(); let (sink, got) = sink_to_vec();
let mut w = watch_with(&dir, fast_config(), sink).unwrap(); let mut w = watch_with(&dir, WatcherConfig::default(), sink).unwrap();
std::fs::write(&bad, "hi").unwrap(); std::fs::write(&bad, "hi").unwrap();
std::thread::sleep(Duration::from_millis(150)); std::thread::sleep(Duration::from_millis(150));
@ -270,7 +218,13 @@ fn include_hidden_registers_dotted_dirs() {
include_hidden: true, include_hidden: true,
..default_filters() ..default_filters()
}; };
let w = Watcher::start(vec![dir.clone()], filters, fast_config(), sink_to_vec().0).unwrap(); let w = Watcher::start(
vec![dir.clone()],
filters,
WatcherConfig::default(),
sink_to_vec().0,
)
.unwrap();
assert_eq!(w.watched_dirs(), 2, "root + .hidden"); assert_eq!(w.watched_dirs(), 2, "root + .hidden");
drop(w); drop(w);
@ -303,7 +257,7 @@ fn an_unreadable_directory_is_skipped_not_fatal() {
std::fs::create_dir_all(&locked).unwrap(); std::fs::create_dir_all(&locked).unwrap();
crate::platform::deny_read(&locked).unwrap(); crate::platform::deny_read(&locked).unwrap();
let started = watch_with(&dir, fast_config(), sink_to_vec().0); let started = watch_with(&dir, WatcherConfig::default(), sink_to_vec().0);
crate::platform::restore_read(&locked).ok(); crate::platform::restore_read(&locked).ok();
let w = started.expect("an unreadable directory must not fail the watcher"); let w = started.expect("an unreadable directory must not fail the watcher");
@ -351,7 +305,7 @@ fn a_tree_inside_the_cap_registers() {
fn a_directory_created_after_start_is_watched() { fn a_directory_created_after_start_is_watched() {
let dir = crate::testutil::scratch_dir("newdir"); let dir = crate::testutil::scratch_dir("newdir");
let (sink, got) = sink_to_vec(); let (sink, got) = sink_to_vec();
let mut w = watch_with(&dir, fast_config(), sink).unwrap(); let mut w = watch_with(&dir, WatcherConfig::default(), sink).unwrap();
assert_eq!(w.watched_dirs(), 1, "only the root to begin with"); assert_eq!(w.watched_dirs(), 1, "only the root to begin with");
let sub = dir.join("later"); let sub = dir.join("later");

44
crates/quicksearch-core/tests/common/mod.rs
View file

@ -14,10 +14,13 @@ pub use quicksearch_core::testutil::{touch, Lcg, BODY_TERM};
const INDEX_TIMEOUT: Duration = Duration::from_secs(120); const INDEX_TIMEOUT: Duration = Duration::from_secs(120);
/// One full indexing run, awaited via the `last_full_index` marker, which is /// One full indexing run, awaited until `Idle`, then checked for the
/// written only on a successful finish. Polling for `Idle` would race: a small /// `last_full_index` marker, which is written only on a successful finish.
/// tree finishes between two polls, leaving `Idle` ambiguous between "not ///
/// started yet" and "already done". /// `Idle` and not the marker ends the wait: the marker lands before the
/// run's tail and the post-run VACUUM, which would otherwise still be
/// writing when the caller's next statement runs. `Idle` is unambiguous
/// because `start_indexing` claims the status before it returns.
pub struct IndexOnce<'a> { pub struct IndexOnce<'a> {
pub db: &'a Path, pub db: &'a Path,
pub roots: Vec<String>, pub roots: Vec<String>,
@ -25,8 +28,8 @@ pub struct IndexOnce<'a> {
/// Delete any existing completion marker first. Off for suites indexing a /// Delete any existing completion marker first. Off for suites indexing a
/// database whose lifecycle they are themselves testing. /// database whose lifecycle they are themselves testing.
pub fresh_marker: bool, pub fresh_marker: bool,
/// Poll the marker through the keyed open: a plain open cannot read an /// Read the marker through the keyed open: a plain open cannot read an
/// encrypted index and would never observe its own completion. /// encrypted index.
pub encrypted: bool, pub encrypted: bool,
} }
@ -48,28 +51,29 @@ impl IndexOnce<'_> {
.unwrap(); .unwrap();
let deadline = Instant::now() + INDEX_TIMEOUT; let deadline = Instant::now() + INDEX_TIMEOUT;
let mut done = false; loop {
while Instant::now() < deadline { match service.get_status() {
if let IndexingStatus::Error(e) = service.get_status() { IndexingStatus::Idle => break,
panic!("indexing failed: {}", e); IndexingStatus::Error(e) => panic!("indexing failed: {}", e),
} _ => {}
if self.db.exists() && self.completed() {
done = true;
break;
} }
assert!(
Instant::now() < deadline,
"indexing did not finish within {:?}",
INDEX_TIMEOUT
);
std::thread::sleep(Duration::from_millis(10)); std::thread::sleep(Duration::from_millis(10));
} }
assert!(done, "indexing did not finish within {:?}", INDEX_TIMEOUT); assert!(self.completed(), "the run ended without finishing");
service.stop_indexing().unwrap();
} }
/// Whether the marker is present; mid-creation is simply "not yet". /// Whether the marker is present.
fn completed(&self) -> bool { fn completed(&self) -> bool {
let conn = if self.encrypted { let conn = if self.encrypted {
db::open_existing(&self.db.to_string_lossy(), false).ok() db::open_existing(&self.db.to_string_lossy(), false).unwrap()
} else { } else {
rusqlite::Connection::open(self.db).ok() rusqlite::Connection::open(self.db).unwrap()
}; };
conn.is_some_and(|c| db::repo::get_last_full_index(&c).is_some()) db::repo::get_last_full_index(&conn).is_some()
} }
} }

5
crates/quicksearch-gui/src/app/status_bar.rs
View file

@ -146,11 +146,6 @@ impl QuickSearchApp {
{ {
ctx.request_repaint_after(Duration::from_millis(250)); ctx.request_repaint_after(Duration::from_millis(250));
} }
// Watcher registration walks every root, so its verdict can land
// minutes after startup, with no input to repaint on.
if matches!(state.watcher, WatcherStatus::Starting) {
ctx.request_repaint_after(Duration::from_millis(500));
}
} }
} }

136
crates/quicksearch-gui/src/hotkey/mod.rs
View file

@ -52,13 +52,16 @@ struct Registry {
} }
enum Backend { enum Backend {
/// Nothing registered: no shortcut set, a Wayland session (see the /// A Wayland session (see the module docs): nothing is ever registered.
/// module docs), or the backend never started.
Idle, Idle,
/// Windows and X11. /// Windows and X11, holding no key, and so no manager either: on X11 one
/// polls the display twenty times a second for as long as it exists,
/// whether or not it holds anything.
Ready,
/// Windows and X11, holding `registered`.
Grab { Grab {
manager: GlobalHotKeyManager, manager: GlobalHotKeyManager,
registered: Option<HotKey>, registered: HotKey,
}, },
} }
@ -75,34 +78,27 @@ pub fn init(ctx: &egui::Context, setting: &str) {
} }
})); }));
let (registry, started) = match choose_backend() { // No X11 grab as a fallback under Wayland: it succeeds inside a Wayland
Ok(backend) => ( // session and then only fires while an XWayland window has focus, which
Registry { // looks broken.
backend, let backend = if crate::activate::raise::is_wayland() {
status: Status::Disabled, Backend::Idle
}, } else {
true, Backend::Ready
),
Err(message) => {
quicksearch_core::log_warn!("global shortcut: {}", message);
(
Registry {
backend: Backend::Idle,
status: Status::Error(message),
},
false,
)
}
}; };
REGISTRY.with_borrow_mut(|slot| *slot = Some(registry)); REGISTRY.with_borrow_mut(|slot| {
if started { *slot = Some(Registry {
apply(setting); backend,
} status: Status::Disabled,
})
});
apply(setting);
} }
/// Registers `setting`, releasing whatever was registered before. An /// Registers `setting`, releasing whatever was registered before. An
/// unparseable or refused shortcut is reported through [`status`], never by /// unparseable or refused shortcut is reported through [`status`], never by
/// failing: not worth blocking a config the user has already applied. /// failing: not worth blocking a config the user has already applied. On the
/// event-loop thread, like [`init`]: registering creates the manager.
pub fn apply(setting: &str) { pub fn apply(setting: &str) {
REGISTRY.with_borrow_mut(|slot| { REGISTRY.with_borrow_mut(|slot| {
let Some(registry) = slot.as_mut() else { let Some(registry) = slot.as_mut() else {
@ -156,53 +152,43 @@ pub fn status() -> Status {
} }
impl Backend { impl Backend {
/// Hold `wanted` and nothing else. `None` releases without registering. /// Hold `wanted` and nothing else. `None` releases without registering,
/// and drops the manager with it.
fn register(&mut self, wanted: Option<Binding>) -> Result<(), String> { fn register(&mut self, wanted: Option<Binding>) -> Result<(), String> {
match self { if matches!(self, Backend::Idle) {
Backend::Idle => Ok(()), return Ok(());
Backend::Grab { }
manager, if let Backend::Grab {
registered, manager,
} => { registered,
if let Some(old) = registered.take() { } = std::mem::replace(self, Backend::Ready)
// Worth reporting, not worth refusing the new binding over. {
if let Err(e) = manager.unregister(old) { // Worth reporting, not worth refusing the new binding over.
quicksearch_core::log_warn!("releasing the old global shortcut: {}", e); if let Err(e) = manager.unregister(registered) {
} quicksearch_core::log_warn!("releasing the old global shortcut: {}", e);
}
let Some(binding) = wanted else {
return Ok(());
};
let hotkey: HotKey = binding
.to_string()
.parse()
.map_err(|e| format!("{} is not a usable shortcut: {}", binding, e))?;
manager.register(hotkey).map_err(|e| match e {
global_hotkey::Error::AlreadyRegistered(_) => {
format!("another application is already using {}", binding)
}
other => format!("{} could not be registered: {}", binding, other),
})?;
*registered = Some(hotkey);
Ok(())
} }
} }
} let Some(binding) = wanted else {
} return Ok(());
};
/// A Wayland session gets no backend (see the module docs). No X11 grab as a let hotkey: HotKey = binding
/// fallback either: it succeeds inside a Wayland session and then only fires .to_string()
/// while an XWayland window has focus, which looks broken. .parse()
fn choose_backend() -> Result<Backend, String> { .map_err(|e| format!("{} is not a usable shortcut: {}", binding, e))?;
if crate::activate::raise::is_wayland() { let manager = GlobalHotKeyManager::new()
return Ok(Backend::Idle); .map_err(|e| format!("global shortcuts are unavailable: {}", e))?;
} manager.register(hotkey).map_err(|e| match e {
GlobalHotKeyManager::new() global_hotkey::Error::AlreadyRegistered(_) => {
.map(|manager| Backend::Grab { format!("another application is already using {}", binding)
}
other => format!("{} could not be registered: {}", binding, other),
})?;
*self = Backend::Grab {
manager, manager,
registered: None, registered: hotkey,
}) };
.map_err(|e| format!("global shortcuts are unavailable: {}", e)) Ok(())
}
} }
#[cfg(test)] #[cfg(test)]
@ -226,5 +212,15 @@ mod tests {
backend.register(Some("Ctrl+Shift+F".parse().unwrap())), backend.register(Some("Ctrl+Shift+F".parse().unwrap())),
Ok(()) Ok(())
); );
assert!(matches!(backend, Backend::Idle));
}
/// Holding no key must not start a manager: on X11 its thread polls the
/// display until it is dropped.
#[test]
fn releasing_holds_no_manager() {
let mut backend = Backend::Ready;
assert_eq!(backend.register(None), Ok(()));
assert!(matches!(backend, Backend::Ready));
} }
} }

1
crates/quicksearch-gui/src/search_tab.rs
View file

@ -568,7 +568,6 @@ impl SearchTab {
return; return;
} }
match update { match update {
SearchUpdate::Started { .. } => {}
SearchUpdate::Hits { hits, .. } => { SearchUpdate::Hits { hits, .. } => {
// An empty batch is not a result; the cascade never sends one, // An empty batch is not a result; the cascade never sends one,
// but the event is public and this is what it would mean. // but the event is public and this is what it would mean.

6
crates/quicksearch-gui/src/search_tab/tests.rs
View file

@ -53,7 +53,11 @@ fn timed_frame(ctx: &egui::Context, tab: &mut SearchTab) {
/// Per-visible-row frame cost, printed rather than asserted. Measured: 1000 /// Per-visible-row frame cost, printed rather than asserted. Measured: 1000
/// rows with content snippets cost 615 µs/frame (<4% of a 60 fps budget), so /// rows with content snippets cost 615 µs/frame (<4% of a 60 fps budget), so
/// the Content Match cut is not memoized; re-measure here before optimizing /// the Content Match cut is not memoized; re-measure here before optimizing
/// the row renderer. /// the row renderer. Re-measured for formatting each hit's size, date, rank
/// and flattened window once on arrival instead of every frame: rows cost
/// 231 µs/frame (names only) and 434 µs (content snippets), against about
/// 20 ms for a whole frame under software GL (llvmpipe), so caching could
/// save at most ~2% of a fade or cursor-blink frame. Not adopted.
/// ///
/// ```text /// ```text
/// QSB_RENDER_PERF=1 cargo test --release -p quicksearch-gui -- render_perf --nocapture /// QSB_RENDER_PERF=1 cargo test --release -p quicksearch-gui -- render_perf --nocapture