diff --git a/Cargo.lock b/Cargo.lock index 44907f9..f9e00ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3179,7 +3179,7 @@ dependencies = [ [[package]] name = "quicksearch-core" -version = "0.8.5" +version = "0.8.8" dependencies = [ "argon2", "ctrlc", @@ -3187,6 +3187,7 @@ dependencies = [ "globset", "infer", "kamadak-exif", + "libc", "lofty", "lopdf 0.32.0", "mime_guess", @@ -3207,7 +3208,7 @@ dependencies = [ [[package]] name = "quicksearch-gui" -version = "0.8.5" +version = "0.8.8" dependencies = [ "chrono", "eframe", diff --git a/Cargo.toml b/Cargo.toml index 0411d92..b239e05 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ ] [workspace.package] -version = "0.8.6" +version = "0.8.8" edition = "2021" license = "GPL-3.0-or-later" authors = ["Jeremy "] diff --git a/README.md b/README.md index 26604c3..a514bf6 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,10 @@ the package is installed. indexed folder list, full-text extension filters, ignore patterns, and the indexing options. Stopping switches to manual mode and saves that (`indexing.auto_index`), so a stopped index stays stopped across - restarts until you return to automatic. + restarts until you return to automatic. The index size beside the + status heading totals the database and its `-wal`/`-shm` sidecars, + refreshed every ten seconds; hovering it lists the ways to make it + smaller. - **Duplicates**: files sharing a content hash, grouped. - **Logs**: the lines the app would have printed to a terminal — warnings from indexing, folder watching and opening files, newest last, with a @@ -264,7 +267,11 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. - **Storage** (`db/`): SQLite via rusqlite (bundled SQLCipher build — identical to stock SQLite until a key is applied), WAL mode so the - single writer never blocks streaming read-only searches. `files` holds + single writer never blocks streaming read-only searches. A run forces a + `wal_checkpoint(TRUNCATE)` every `processing.maximum_wal_size` bytes of + log, because SQLite's own autocheckpoint can only reset the log at an + instant no reader holds it — and a run keeps a reader per root querying + throughout, so left alone the log grows for the whole run. `files` holds metadata (name, path, size, mtime, hash, MIME/type bitmask, per-row index state); `searchabletext` is a *contentless* FTS5 table (postings only, configurable tokenizer, trigram by default); canonical extracted @@ -284,8 +291,11 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. tags, EXIF; see `extract/`) for FTS. Files no larger than `processing.hash_length` skip that second pass entirely: the head the walk reads to hash them is already their whole content, so a plaintext body is - extracted in the same `read` and stored complete. Progress streams through - a polled `IndexingStatus`. + extracted in the same `read` and stored complete. Every run ends — whether + it completed or was stopped — with an optimize pass on its own connection: + checkpoint, VACUUM if the file has at least 10% slack to reclaim, `PRAGMA + optimize`, checkpoint again. Progress streams through a polled + `IndexingStatus`, which reads `Optimizing` for the duration of that pass. - **Coordinator** (`coordinator.rs`): the object binaries construct. Owns the `IndexingService`, the debouncing filesystem watcher (`watcher.rs`), and the mode state machine (Auto / Manual, persisted as diff --git a/config_example.toml b/config_example.toml index 1c4487a..34f469b 100644 --- a/config_example.toml +++ b/config_example.toml @@ -21,16 +21,23 @@ indexing_paths = ["~"] database_path = "~/.local/share/quicksearch/index.sqlite" [indexing] -# The indexing mode. true = automatic: filesystem watchers apply changes as -# they happen and a full reindex runs every reindex_interval_minutes (the -# watcher catches changes as they happen, so that interval only needs to be +# The indexing mode. true = automatic: filesystem watchers apply changes a +# couple of seconds after they happen — the queue waits for a burst to settle +# so that deleting a folder costs one operation instead of one per file — and +# a full reindex runs every reindex_interval_minutes (the watcher catches +# changes as they happen, so that interval only needs to be # often enough to cover whatever the watcher missed). false = manual: # nothing is indexed until you ask for it. The Stop and Return to Automatic # buttons on the Manage Index tab write this value, so the mode you left the # app in is the mode it starts in. auto_index = true reindex_interval_minutes = 1440 -# Follow symbolic links during directory walks. +# Follow symbolic links during directory walks. Applies to links pointing at +# files as well as at directories: with this off a symlink is not resolved at +# all, so its target is never indexed — which matters because a target can +# live outside every folder listed above. A resolved target is stored under +# its own real path, not the link's. Changing this changes what is in the +# index, so it prompts for a rebuild. follow_symlinks = false # Index hidden files and directories. That means dot-files everywhere, and # additionally anything carrying the Hidden or System attribute on Windows @@ -80,9 +87,17 @@ maximum_text_size = 262144 # Files larger than this skip text extraction entirely (bytes). maximum_text_file_size = 2097152 # Files per batch during walks / inserts / extraction. -batch_size = 200 +batch_size = 500 # Files per transaction for incremental FTS updates. fts_update_batch_size = 1000 +# How large the write-ahead log (index.sqlite-wal) may grow during an +# indexing run before the indexer forces a checkpoint (bytes). SQLite copies +# the log into the index on its own but can only reset it when no reader is +# mid-query, and a run keeps one reader per root busy throughout — so +# unattended the log grows for the whole run and can end up larger than the +# index. Set to 0 to disable forced checkpoints; any other value below +# 16777216 is raised to it. +maximum_wal_size = 536870912 # FTS5 tokenizer: 'trigram' (substring matching, the default; gets # remove_diacritics 1 appended), 'unicode61', 'porter', or a full FTS5 # option string. See https://www.sqlite.org/fts5.html#tokenizers diff --git a/crates/quicksearch-core/Cargo.toml b/crates/quicksearch-core/Cargo.toml index cb052ee..af1c7aa 100644 --- a/crates/quicksearch-core/Cargo.toml +++ b/crates/quicksearch-core/Cargo.toml @@ -43,6 +43,13 @@ zstd = "0.13" globset = "0.4" regex = "1" +# `nice()`, for dropping indexing threads to background scheduling priority. +# Linux schedules per task, so it affects only the calling thread. Already in +# the lockfile transitively (rusqlite, getrandom), so naming it directly +# compiles nothing new. +[target.'cfg(unix)'.dependencies] +libc = "0.2" + # `GetDriveTypeW` (a mapped drive letter is the only way to spot an SMB share # that isn't written as UNC) plus the FILE_ATTRIBUTE_* constants for hidden # detection. Pinned to 0.52 deliberately: walkdir → winapi-util already @@ -50,8 +57,12 @@ regex = "1" [target.'cfg(windows)'.dependencies] # GetDriveTypeW and the FILE_ATTRIBUTE_* constants live in # Win32_Storage_FileSystem; DRIVE_REMOTE, oddly, is filed under -# Win32_System_WindowsProgramming. +# Win32_System_WindowsProgramming. Win32_System_Threading carries +# SetThreadPriority and THREAD_MODE_BACKGROUND_BEGIN, and both it and +# GetCurrentThread need Win32_Foundation for HANDLE/BOOL. windows-sys = { version = "0.52", features = [ + "Win32_Foundation", "Win32_Storage_FileSystem", + "Win32_System_Threading", "Win32_System_WindowsProgramming", ] } diff --git a/crates/quicksearch-core/examples/walkprobe.rs b/crates/quicksearch-core/examples/walkprobe.rs index 2335e14..434594b 100644 --- a/crates/quicksearch-core/examples/walkprobe.rs +++ b/crates/quicksearch-core/examples/walkprobe.rs @@ -31,7 +31,7 @@ //! cache (or, on a share, the client's attribute cache), so the second is the //! one to compare. use std::sync::atomic::AtomicBool; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use std::time::{Instant, UNIX_EPOCH}; use quicksearch_core::config::{Config, IgnoreSet}; @@ -118,7 +118,7 @@ fn parallel(root: &str, config: &Config, db_path: &str) -> (usize, usize) { db_path, config.clone(), Arc::new(Registry::default_set()), - Arc::new(Mutex::new(false)), + Arc::new(AtomicBool::new(false)), Arc::new(AtomicBool::new(false)), 4, ) { diff --git a/crates/quicksearch-core/src/cli.rs b/crates/quicksearch-core/src/cli.rs index 191787f..c1f2221 100644 --- a/crates/quicksearch-core/src/cli.rs +++ b/crates/quicksearch-core/src/cli.rs @@ -173,10 +173,11 @@ pub fn index_size_breakdown(db_path: &str) -> Result { }) } -/// Count files with `content_state = 0` (pending). Distinct from -/// `files_row_count − searchabletext_row_count`, which over-counts because -/// files whose content doesn't apply (binary formats, too-large files) sit -/// with `content_state = 3` (NA) forever and never become FTS rows. +/// Count files with `content_state = 0` (pending) — files an extractor claims +/// whose text has not been read yet. Files nothing extracts (binary formats, +/// too-large files) are written `content_state = 3` (NA) when the walk records +/// them and are never counted here, so this is outstanding work rather than +/// `files_row_count − searchabletext_row_count`, which counts those forever. /// /// Used by the Baloo compat daemon to report the "Files waiting for content /// indexing" figure both to balooctl and to the LMDB mirror. @@ -263,6 +264,7 @@ mod tests { mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, + needs_content: true, }, ) .unwrap() @@ -281,6 +283,7 @@ mod tests { mime: None, ftype: FileType::EMPTY, hash: None, + needs_content: false, }, ) .unwrap() @@ -366,6 +369,7 @@ mod tests { mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, + needs_content: true, }, ) .unwrap() @@ -424,6 +428,7 @@ mod tests { mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, + needs_content: true, }, ) .unwrap() diff --git a/crates/quicksearch-core/src/config.rs b/crates/quicksearch-core/src/config.rs index 8f1e8e1..dfddbe7 100644 --- a/crates/quicksearch-core/src/config.rs +++ b/crates/quicksearch-core/src/config.rs @@ -61,7 +61,8 @@ pub struct IndexingConfig { /// supports. Non-empty = only files with these extensions get content /// extraction/FTS; everything else is still listed for filename search /// (`content_state = NA`). Entries are case-insensitive, with or - /// without a leading dot. + /// without a leading dot. Applied at walk time, when the row is written — + /// which is why changing this forces a rebuild (see [`diff_actions`]). pub content_extensions: Vec, /// Excluded from the index entirely — never even listed. A pattern /// without `/` matches any single path component (so `.git` prunes @@ -69,9 +70,11 @@ pub struct IndexingConfig { /// matched against the full path. Glob syntax (`*`, `?`, `[..]`). pub ignore_patterns: Vec, /// Per-root walker thread override, keyed by the root string exactly - /// as it appears in `indexing_paths`. Absent or 0 = auto-detect - /// (4 for local storage, 16 for network mounts). Read at run start; - /// a change applies to the next run. + /// as it appears in `indexing_paths` — the indexer resolves both sides + /// to the same canonical path before matching, so a key that spells its + /// root as `~/docs`, `/docs/` or a symlink still applies. Absent or + /// 0 = auto-detect (4 for local storage, 16 for network mounts). Read + /// at run start; a change applies to the next run. pub root_workers: std::collections::HashMap, } @@ -95,6 +98,18 @@ pub struct ProcessingConfig { pub maximum_text_file_size: u64, pub batch_size: usize, pub fts_update_batch_size: usize, + /// How large the write-ahead log may grow during a run before the indexer + /// forces a checkpoint, in bytes. `0` disables forced checkpoints; + /// anything else is raised to [`MINIMUM_WAL_SIZE`] at the use site. + /// + /// SQLite's own autocheckpoint copies the log into the index continuously + /// but can only *reset* it when no reader is mid-query, and a run keeps a + /// reader per root busy from start to finish — so left alone the log grows + /// for the whole run and can end up larger than the index itself. This is + /// a stall-frequency knob, not a throughput one: by the time a forced + /// checkpoint fires there is almost nothing left to copy, so it costs lock + /// acquisition and little else. + pub maximum_wal_size: u64, pub tokenize: String, /// When `true` (default), extracted text is stored zstd-compressed in /// `documents_text` so search results can render snippet/highlight @@ -107,6 +122,12 @@ pub struct ProcessingConfig { pub store_text_for_snippets: bool, } +/// Floor on a non-zero [`ProcessingConfig::maximum_wal_size`]. A checkpoint +/// costs a lock acquisition that can wait on a running search, so a cap set +/// low enough to fire every round would trade a large log for a stalled +/// writer. Zero still means "never force one". +pub const MINIMUM_WAL_SIZE: u64 = 1024 * 1024 * 16; + /// Fuzzy edit distances above this are allowed but warned about: matches /// become dominated by coincidence and every fuzzy pass slows down. pub const FUZZY_EDITS_WARN_ABOVE: usize = 3; @@ -177,8 +198,9 @@ impl Default for ProcessingConfig { hash_length: 1024 * 8, maximum_text_size: 1024 * 256, maximum_text_file_size: 1024 * 1024 * 2, - batch_size: 200, + batch_size: 500, fts_update_batch_size: 1000, + maximum_wal_size: 1024 * 1024 * 512, tokenize: "trigram".to_string(), store_text_for_snippets: true, } @@ -650,6 +672,11 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions { let requires_rebuild = old.processing.hash_length != new.processing.hash_length || old.processing.tokenize != new.processing.tokenize || old.indexing.include_hidden != new.indexing.include_hidden + // Not merely a walk-behaviour knob: with links off, a symlink's target + // is not indexed at all, so turning it off leaves rows for files that + // are no longer in scope — including ones whose real parent lies + // outside every root, which no sweep will ever reach. + || old.indexing.follow_symlinks != new.indexing.follow_symlinks || old.indexing.ignore_patterns != new.indexing.ignore_patterns || old.indexing.content_extensions != new.indexing.content_extensions || old.security.password_protected != new.security.password_protected @@ -657,9 +684,7 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions { || roots_changed; ConfigActions { requires_rebuild, - restart_watcher: requires_rebuild - || roots_changed - || old.indexing.follow_symlinks != new.indexing.follow_symlinks, + restart_watcher: requires_rebuild || roots_changed, search_db_changed: old.paths.database_path != new.paths.database_path, } } @@ -727,7 +752,7 @@ mod tests { fs::write(&path, "[paths]\nindexing_paths=[\"/x\"]\ndatabase_path=\"db.sqlite\"\n").unwrap(); let cfg = Config::load_from(&path).unwrap(); assert_eq!(cfg.paths.indexing_paths, vec!["/x".to_string()]); - assert_eq!(cfg.processing.batch_size, 200, "missing sections default"); + assert_eq!(cfg.processing.batch_size, 500, "missing sections default"); assert_eq!(cfg.search.debounce_ms, 150); assert!((cfg.ui.scale - 1.1).abs() < f32::EPSILON); fs::remove_dir_all(&dir).ok(); @@ -975,10 +1000,12 @@ mod tests { let a = diff_actions(&base, &c); assert!(a.requires_rebuild && a.restart_watcher); + // `follow_symlinks` decides what is in the index, not just how the walk + // moves, so it rebuilds as well as restarting the watcher. let mut c = base.clone(); c.indexing.follow_symlinks = true; let a = diff_actions(&base, &c); - assert!(!a.requires_rebuild && a.restart_watcher); + assert!(a.requires_rebuild && a.restart_watcher); let mut c = base.clone(); c.paths.database_path = "/elsewhere.sqlite".into(); @@ -988,6 +1015,7 @@ mod tests { let mut c = base.clone(); c.search.display_limit = 5000; c.processing.batch_size = 999; + c.processing.maximum_wal_size = 0; c.indexing.auto_index = false; c.indexing.reindex_interval_minutes = 5; let a = diff_actions(&base, &c); diff --git a/crates/quicksearch-core/src/content.rs b/crates/quicksearch-core/src/content.rs new file mode 100644 index 0000000..1c5ed74 --- /dev/null +++ b/crates/quicksearch-core/src/content.rs @@ -0,0 +1,634 @@ +//! Parallel content extraction for one indexing root. +//! +//! The second half of a root's pipeline, and the sibling of [`crate::walk`]: +//! a pool of worker threads produces finished work over a bounded channel, and +//! the single writer drains it round-robin against every other root. +//! +//! That shape is the whole point. Extraction used to run *on* the writer +//! thread, a batch of files at a time, with the database connection held for +//! the duration — so one root reading a network share or a stack of large PDFs +//! stopped every other root dead for as long as its batch took, and blocked +//! the watcher's incremental writes with it. Roots are meant to be +//! independent; walking already was, and this makes the rest of the pipeline +//! match. +//! +//! The split within the pass mirrors the walk's too: **one feeder thread owns +//! the only database connection**, paging through the root's pending rows, +//! while N workers do nothing but filesystem work. A connection per worker +//! would multiply SQLite's page cache by the pool size, which is exactly what +//! [`crate::db::schema::PRAGMAS_WALK_READER`] exists to avoid. +//! +//! Deliberately std-only (`std::thread` + `std::sync::mpsc`), matching the +//! house style of [`crate::walk`] and [`crate::watcher`]. + +use std::sync::atomic::AtomicBool; +use std::sync::{mpsc, Arc, Condvar, Mutex}; +use std::thread::{self, JoinHandle}; + +use crate::config::Config; +use crate::extract::Registry; +use crate::file_handling::{decide_content, ContentOutcome, ExtractCursor}; +use crate::indexing::should_abort; +use crate::walk::{try_recv_next, TryNext, WorkerStats}; + +/// Finished rows waiting for the writer. +/// +/// Far shallower than the walk's 4096. An [`ExtractedRow`] carries up to +/// `maximum_text_size` of extracted text — 256 KiB by default, some 32× a +/// `WalkedFile` — so the walk's depth would put gigabytes in flight. At 32 the +/// ceiling is ~8 MiB per root, plus whatever the workers hold mid-document. +const READY_CAP: usize = 32; + +/// Rows fetched but not yet claimed by a worker. +/// +/// Small for the same reason [`crate::walk`] caps its prefetch: the feeder +/// must not run arbitrarily far ahead of a pool that is bound by file I/O. +/// These are only ids and paths, so the cost is the strings, not the text. +const QUEUE_AHEAD: usize = 256; + +/// How many rows the feeder fetches per query. Large enough that a slow root +/// is not paying a round trip per file, small enough to stay inside +/// [`QUEUE_AHEAD`]. +const FEED_PAGE: usize = 128; + +/// One file's extracted content, ready to be written. +#[derive(Debug)] +pub struct ExtractedRow { + pub file_id: i64, + /// The `files.name` the FTS row is indexed under. + pub name: String, + pub outcome: ContentOutcome, +} + +/// A row the feeder handed to the pool: everything a worker needs, and nothing +/// that would make it touch the database. +#[derive(Debug)] +struct Pending { + file_id: i64, + name: String, + path: String, + mime: Option, +} + +#[derive(Default)] +struct Queue { + rows: Vec, + /// Set while the feeder is mid-query, holding rows that are in neither the + /// queue nor a worker. Without it a worker could see an empty queue + /// between two pages and declare the pass finished early. + feeding: bool, + /// True once the feeder has read the last page. + drained: bool, + done: bool, +} + +struct Shared { + queue: Mutex, + idle: Condvar, +} + +impl Shared { + /// Claim a row, blocking while the feeder might still produce more. + /// + /// `None` only when the queue is empty *and* the feeder is finished — at + /// that instant nobody is left who could add another row. + fn take(&self) -> Option { + let mut q = self.queue.lock().unwrap(); + loop { + if q.done { + return None; + } + if let Some(row) = q.rows.pop() { + // The feeder may be parked behind QUEUE_AHEAD. + self.idle.notify_all(); + return Some(row); + } + if q.drained && !q.feeding { + q.done = true; + self.idle.notify_all(); + return None; + } + q = self.idle.wait(q).unwrap(); + } + } + + /// Claim the right to fetch one page, or `None` once the pass is over. + /// Parks while the queue is already [`QUEUE_AHEAD`] deep. + fn take_feed_slot(&self) -> Option<()> { + let mut q = self.queue.lock().unwrap(); + loop { + if q.done || q.drained { + return None; + } + if q.rows.len() < QUEUE_AHEAD { + q.feeding = true; + return Some(()); + } + q = self.idle.wait(q).unwrap(); + } + } + + /// Publish a page and clear the in-flight flag together, under one lock — + /// the indivisibility [`Shared::take`]'s end-of-pass test relies on. + fn finish_feed(&self, rows: Vec, last_page: bool) { + let mut q = self.queue.lock().unwrap(); + // Reversed: `take` pops from the back, and rows should reach workers + // in id order so a partial run leaves a contiguous prefix done. + q.rows.extend(rows.into_iter().rev()); + q.feeding = false; + if last_page { + q.drained = true; + } + self.idle.notify_all(); + } + + fn shutdown(&self) { + let mut q = self.queue.lock().unwrap(); + q.done = true; + q.drained = true; + self.idle.notify_all(); + } +} + +/// A running content pass. Draining it yields finished rows; dropping it stops +/// the workers and joins them. +pub struct ContentPass { + rx: Option>, + handles: Vec>, + feeder: Option>, + shared: Arc, + stats: WorkerStats, +} + +impl ContentPass { + /// Non-blocking pull, for the writer loop multiplexing several roots. + pub fn try_next(&mut self) -> TryNext { + try_recv_next(self.rx.as_ref()) + } + + /// A cheap, cloneable handle for reading pool activity while the pass is + /// mutably borrowed by the writer loop. The sibling of + /// [`crate::walk::ParallelWalk::worker_stats`], and for the same reason: + /// once a root reaches extraction its walk pool is gone, so the progress + /// display has to read this one instead. + pub fn worker_stats(&self) -> WorkerStats { + self.stats.clone() + } + + /// Join the workers and report whether every one finished cleanly. + /// + /// The caller needs this for the same reason the walk does: a dead worker + /// and a finished worker both close the channel, so from the receiving end + /// they are indistinguishable. + pub fn finish(&mut self) -> bool { + // Dropping the receiver first releases any worker parked in `send`. + self.rx = None; + self.shared.shutdown(); + let mut clean = true; + for handle in self.handles.drain(..) { + if handle.join().is_err() { + clean = false; + } + } + if let Some(handle) = self.feeder.take() { + if handle.join().is_err() { + clean = false; + } + } + clean + } +} + +impl Drop for ContentPass { + fn drop(&mut self) { + self.shared.shutdown(); + // No-op if the caller already called `finish`. + self.finish(); + } +} + +/// Page the root's pending rows into the queue from one read-only connection. +/// +/// A failed query ends the pass rather than retrying: the rows stay +/// `content_state = 0` and the next run picks them up, which is the same +/// outcome as being interrupted. +fn feeder(shared: &Shared, db_path: &str, mut cursor: ExtractCursor, max_size: i64) { + let conn = match crate::db::open::open_walk_reader(db_path) { + Ok(conn) => conn, + Err(e) => { + crate::log_warn!("content reader: {}", e); + shared.shutdown(); + return; + } + }; + + while shared.take_feed_slot().is_some() { + let page = match crate::db::repo::pending_content_page( + &conn, + &cursor, + max_size, + FEED_PAGE as i64, + ) { + Ok(page) => page, + Err(e) => { + crate::log_warn!("{}", e); + shared.shutdown(); + return; + } + }; + let last_page = page.len() < FEED_PAGE; + if let Some((id, _, _, _)) = page.last() { + cursor.last_id = *id; + } + let rows = page + .into_iter() + .map(|(file_id, name, path, mime)| Pending { + file_id, + name, + path, + mime, + }) + .collect(); + shared.finish_feed(rows, last_page); + if last_page { + return; + } + } +} + +fn worker( + shared: &Shared, + tx: &mpsc::SyncSender, + registry: &Registry, + config: &Config, + stop_flag: &Arc, + suspend_flag: &Arc, + stats: &WorkerStats, +) { + while let Some(row) = shared.take() { + // Held for the whole of `decide_content` — reading and parsing a + // document is exactly the work the progress line is reporting. + let _busy = stats.enter(); + if should_abort(stop_flag, suspend_flag) { + shared.shutdown(); + return; + } + let outcome = decide_content(&row.path, row.mime.as_deref(), registry, config); + let sent = tx.send(ExtractedRow { + file_id: row.file_id, + name: row.name, + outcome, + }); + if sent.is_err() { + // Receiver gone: the run was stopped or failed. Not an error. + shared.shutdown(); + return; + } + } +} + +/// Extract every pending row under `cursor`'s range, in parallel. +/// +/// `workers` is the root's own count — the same value its walk uses, because +/// extraction over a share is round-trip bound for the same reason walking is. +/// Clamped to 1..=64. +#[allow(clippy::too_many_arguments)] +pub fn extract_content( + db_path: &str, + cursor: &ExtractCursor, + registry: Arc, + config: Config, + stop_flag: Arc, + suspend_flag: Arc, + workers: usize, +) -> ContentPass { + let shared = Arc::new(Shared { + queue: Mutex::new(Queue::default()), + idle: Condvar::new(), + }); + let max_size = i64::try_from(config.processing.maximum_text_file_size).unwrap_or(i64::MAX); + + let (tx, rx) = mpsc::sync_channel(READY_CAP); + let stats = WorkerStats::new(workers.clamp(1, 64)); + let handles = (0..stats.total()) + .map(|_| { + let (shared, tx) = (shared.clone(), tx.clone()); + let (registry, config) = (registry.clone(), config.clone()); + let (stop_flag, suspend_flag) = (stop_flag.clone(), suspend_flag.clone()); + let stats = stats.clone(); + thread::spawn(move || { + crate::platform::set_background_priority(); + worker(&shared, &tx, ®istry, &config, &stop_flag, &suspend_flag, &stats) + }) + }) + .collect(); + // The workers must hold the only senders, or `try_recv` never reports the + // end of the pass. The feeder deliberately holds none: it produces rows to + // extract, not extracted rows. + drop(tx); + + let feeder_handle = { + let (shared, db_path, cursor) = (shared.clone(), db_path.to_string(), cursor.clone()); + thread::spawn(move || { + crate::platform::set_background_priority(); + feeder(&shared, &db_path, cursor, max_size) + }) + }; + + ContentPass { + rx: Some(rx), + handles, + feeder: Some(feeder_handle), + shared, + stats, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::db::open_or_recreate; + use crate::db::repo::{self, insert_file, NewFile}; + use crate::file_handling::{extract_scope_prepare, store_extracted}; + use crate::mime::FileType; + use std::path::{Path, PathBuf}; + use std::time::UNIX_EPOCH; + + fn tmp(tag: &str) -> PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!( + "qs-content-{}-{}-{}", + tag, + std::process::id(), + std::time::SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + p + } + + /// A tree of `n` text files under `root/sub`, plus an index holding a + /// pending row for each. + fn seed(tag: &str, dirs: &[(&str, usize)]) -> (PathBuf, PathBuf) { + let tree = tmp(&format!("{}-tree", tag)); + let db = tmp(&format!("{}-db", tag)); + let mut conn = open_or_recreate(db.to_str().unwrap(), "trigram").unwrap(); + let tx = conn.transaction().unwrap(); + for (dir, n) in dirs { + let d = tree.join(dir); + std::fs::create_dir_all(&d).unwrap(); + for i in 0..*n { + let f = d.join(format!("f{:04}.txt", i)); + std::fs::write(&f, format!("sphinx of black quartz {} {}", dir, i)).unwrap(); + insert_file( + &tx, + &NewFile { + name: f.file_name().unwrap().to_str().unwrap(), + path: f.to_str().unwrap(), + parent: d.to_str().unwrap(), + size: std::fs::metadata(&f).unwrap().len(), + mtime: 1, + inode: None, + device_id: None, + mime: Some("text/plain"), + ftype: FileType::TEXT, + hash: None, + needs_content: true, + }, + ) + .unwrap() + .expect("unique path"); + } + } + tx.commit().unwrap(); + drop(conn); + (tree, db) + } + + fn pass_for(tree: &Path, db: &Path, sub: &str, workers: usize) -> ContentPass { + extract_content( + db.to_str().unwrap(), + &ExtractCursor::for_root(tree.join(sub).to_str().unwrap()), + Arc::new(Registry::default_set()), + Config::default(), + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicBool::new(false)), + workers, + ) + } + + /// Drain a pass to exhaustion, blocking between polls the way the writer + /// loop's outer sleep does. + fn drain(pass: &mut ContentPass) -> Vec { + let mut out = Vec::new(); + loop { + match pass.try_next() { + TryNext::Item(row) => out.push(row), + TryNext::Empty => thread::sleep(std::time::Duration::from_millis(1)), + TryNext::Finished => return out, + } + } + } + + #[test] + fn every_pending_row_is_yielded_exactly_once() { + let (tree, db) = seed("once", &[("r1", 250)]); + let mut pass = pass_for(&tree, &db, "r1", 4); + let rows = drain(&mut pass); + assert!(pass.finish(), "no worker panicked"); + + assert_eq!(rows.len(), 250); + let ids: std::collections::HashSet = rows.iter().map(|r| r.file_id).collect(); + assert_eq!(ids.len(), 250, "no id may be yielded twice"); + assert!( + rows.iter() + .all(|r| matches!(r.outcome, ContentOutcome::Done { .. })), + "every plaintext file extracts" + ); + + std::fs::remove_dir_all(&tree).ok(); + std::fs::remove_file(&db).ok(); + } + + /// The pass is scoped by the cursor's path range, so a sibling root's rows + /// are never touched. (Moved here with the extraction itself: the scoping + /// is now the feeder's query, not the writer's.) + #[test] + fn the_pass_is_scoped_to_its_root_range() { + let (tree, db) = seed("scope", &[("r1", 3), ("r2", 3)]); + let conn_mutex = Arc::new(Mutex::new( + open_or_recreate(db.to_str().unwrap(), "trigram").unwrap(), + )); + let config = Config::default(); + let cursor = ExtractCursor::for_root(tree.join("r1").to_str().unwrap()); + + let scope = extract_scope_prepare(&conn_mutex, &cursor, &config).unwrap(); + assert_eq!(scope.pending, 3, "only r1's files are in range"); + assert_eq!(scope.already_done, 0, "nothing extracted yet"); + + let mut pass = pass_for(&tree, &db, "r1", 2); + let rows = drain(&mut pass); + assert!(pass.finish()); + assert_eq!(rows.len(), 3); + + let stop = Arc::new(AtomicBool::new(false)); + assert_eq!( + store_extracted(&conn_mutex, &rows, &stop, &config).unwrap(), + 3 + ); + + let state = |p: &Path| -> i64 { + conn_mutex + .lock() + .unwrap() + .query_row( + "SELECT content_state FROM files WHERE path = ?1", + rusqlite::params![p.to_str().unwrap()], + |r| r.get(0), + ) + .unwrap() + }; + assert_eq!(state(&tree.join("r1/f0000.txt")), repo::STATE_DONE); + assert_eq!( + state(&tree.join("r2/f0000.txt")), + repo::STATE_PENDING, + "out-of-range row untouched" + ); + + // A second run over the unchanged root reports it already extracted, + // so progress reads "3 of 3" rather than "0 of 0". + let scope2 = extract_scope_prepare(&conn_mutex, &cursor, &config).unwrap(); + assert_eq!((scope2.pending, scope2.already_done), (0, 3)); + + let hits: i64 = conn_mutex + .lock() + .unwrap() + .query_row( + "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH '\"sphinx\"'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(hits, 3); + + std::fs::remove_dir_all(&tree).ok(); + std::fs::remove_file(&db).ok(); + } + + #[test] + fn an_empty_range_terminates_immediately() { + // The "nothing to do at t=0" corner: every worker must observe the pass + // as finished rather than waiting for rows that will never arrive. + let (tree, db) = seed("empty", &[("r1", 2)]); + let mut pass = pass_for(&tree, &db, "nonexistent", 4); + assert!(drain(&mut pass).is_empty()); + assert!(pass.finish()); + std::fs::remove_dir_all(&tree).ok(); + std::fs::remove_file(&db).ok(); + } + + #[test] + fn an_already_stopped_pass_does_not_run_to_completion() { + let (tree, db) = seed("stop", &[("r1", 400)]); + let mut pass = extract_content( + db.to_str().unwrap(), + &ExtractCursor::for_root(tree.join("r1").to_str().unwrap()), + Arc::new(Registry::default_set()), + Config::default(), + Arc::new(AtomicBool::new(true)), + Arc::new(AtomicBool::new(false)), + 4, + ); + assert!(drain(&mut pass).len() < 400); + assert!(pass.finish()); + std::fs::remove_dir_all(&tree).ok(); + std::fs::remove_file(&db).ok(); + } + + #[test] + fn dropping_the_pass_early_does_not_hang() { + // Workers blocked in `send` must be released by the receiver going + // away, or `Drop` would join threads that never wake. + let (tree, db) = seed("early-drop", &[("r1", 500)]); + let mut pass = pass_for(&tree, &db, "r1", 4); + // Pull one, leave the rest queued and the channel full. + loop { + match pass.try_next() { + TryNext::Item(_) => break, + TryNext::Empty => thread::sleep(std::time::Duration::from_millis(1)), + TryNext::Finished => break, + } + } + drop(pass); // must return, not deadlock + std::fs::remove_dir_all(&tree).ok(); + std::fs::remove_file(&db).ok(); + } + + #[test] + fn repeated_passes_agree_on_the_result_set() { + // The termination protocol is racy by nature; run it enough times under + // real contention that a premature exit would show up. + let (tree, db) = seed("repeat", &[("r1", 120)]); + for run in 0..20 { + let mut pass = pass_for(&tree, &db, "r1", 4); + let rows = drain(&mut pass); + assert!(pass.finish(), "run {}", run); + assert_eq!(rows.len(), 120, "run {}", run); + } + std::fs::remove_dir_all(&tree).ok(); + std::fs::remove_file(&db).ok(); + } + + /// The pass counts its own busy threads, which is what the progress line + /// shows once a root leaves the walk behind. + #[test] + fn the_pool_reports_its_own_activity() { + let (tree, db) = seed("stats", &[("r1", 300)]); + let mut pass = pass_for(&tree, &db, "r1", 4); + let stats = pass.worker_stats(); + assert_eq!(stats.total(), 4); + + // Nothing is drained here, so the ready channel fills and every worker + // ends up parked mid-row inside `send` — busy by the definition the + // display uses, and there are more rows than the channel holds, so all + // four get there. + let mut peak = 0; + for _ in 0..500 { + peak = peak.max(stats.active()); + if peak == 4 { + break; + } + thread::sleep(std::time::Duration::from_millis(2)); + } + assert_eq!(peak, 4, "every worker busy while the channel is full"); + + assert_eq!(drain(&mut pass).len(), 300); + assert!(pass.finish()); + assert_eq!(stats.active(), 0, "a finished pool is idle"); + + std::fs::remove_dir_all(&tree).ok(); + std::fs::remove_file(&db).ok(); + } + + /// A file that vanished between the walk and extraction is a failure with + /// a reason, not a silent skip: the row records why so it is not retried + /// forever. + #[test] + fn a_missing_file_is_reported_as_failed() { + let (tree, db) = seed("missing", &[("r1", 2)]); + std::fs::remove_file(tree.join("r1/f0000.txt")).unwrap(); + let mut pass = pass_for(&tree, &db, "r1", 2); + let rows = drain(&mut pass); + assert!(pass.finish()); + assert_eq!(rows.len(), 2, "both rows are still reported"); + assert_eq!( + rows.iter() + .filter(|r| matches!(r.outcome, ContentOutcome::Failed(_))) + .count(), + 1 + ); + std::fs::remove_dir_all(&tree).ok(); + std::fs::remove_file(&db).ok(); + } +} diff --git a/crates/quicksearch-core/src/coordinator.rs b/crates/quicksearch-core/src/coordinator.rs index c6c7027..cb8b4f9 100644 --- a/crates/quicksearch-core/src/coordinator.rs +++ b/crates/quicksearch-core/src/coordinator.rs @@ -27,7 +27,7 @@ use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{mpsc, Arc, Mutex}; use std::thread::JoinHandle; -use std::time::Duration; +use std::time::{Duration, Instant}; use rusqlite::Connection; @@ -142,6 +142,8 @@ impl IndexCoordinator { watcher_rx: None, watcher_gen: 0, pending: HashMap::new(), + last_event_at: None, + pending_since: None, needs_full_run: false, saw_running: false, write_conn: None, @@ -259,6 +261,50 @@ fn enqueue(pending: &mut HashMap, event: FsEvent) { } } +/// Whether a queued event is a removal. The queue only ever holds +/// Create/Modify/Remove — [`enqueue`] splits renames into their halves. +fn is_removal(event: &FsEvent) -> bool { + matches!(event, FsEvent::Remove(_)) +} + +/// Drop queued removals that a queued removal of one of their ancestors +/// already covers, in place. +/// +/// `rm -rf dir/` reports `dir` and every path beneath it; applying `dir` sweeps +/// the whole range, so the descendants are duplicate work. Collapsing here — +/// rather than at application time — is what keeps a mass deletion from +/// tripping [`PENDING_OVERFLOW`] and forcing a redundant full run. +/// +/// Only removals collapse against removals. A `Create` under a removed +/// directory is a re-creation and must survive: removals are applied first, so +/// it lands afterwards and the row is correct either way. +fn collapse_pending_removals(pending: &mut HashMap) { + if pending.values().filter(|e| is_removal(e)).take(2).count() < 2 { + return; + } + let removed: std::collections::HashSet = pending + .iter() + .filter(|(_, ev)| is_removal(ev)) + .map(|(p, _)| p.clone()) + .collect(); + // `Path::ancestors` walks whole components, so `/a/bc` is never treated as + // living under `/a/b` — the rule `remove_tree` and `UnreadableDirs::covers` + // also use. + pending.retain(|path, ev| { + !is_removal(ev) || !path.ancestors().skip(1).any(|a| removed.contains(a)) + }); +} + +/// How long [`Inner::apply_pending`] may hold the command loop before handing +/// the rest of the queue to the next tick. +/// +/// A long run over a busy tree can accumulate tens of thousands of events, and +/// draining them all inline would block Stop and Apply behind the backlog. A +/// deadline bounds that latency without throttling throughput the way a fixed +/// event budget would; `queued_events` already reports the remainder, so a +/// multi-tick drain is visible rather than silent. +const APPLY_BUDGET: Duration = Duration::from_millis(250); + struct Inner { config: Config, indexing: Arc, @@ -271,6 +317,12 @@ struct Inner { watcher_rx: Option)>>, watcher_gen: u64, pending: HashMap, + /// When the most recent event arrived; the burst is over once this is + /// `pending_settle` old. + last_event_at: Option, + /// When the oldest un-applied event arrived, so a steady trickle cannot + /// defer application past `pending_max_defer`. + pending_since: Option, needs_full_run: bool, /// A start was requested; set false once the service reports running, /// so idle-after-running transitions are detectable. @@ -363,9 +415,13 @@ impl Inner { let status = self.indexing.get_status(); match status { - IndexingStatus::Running { .. } | IndexingStatus::Stopping => { + IndexingStatus::Running { .. } + | IndexingStatus::Stopping + | IndexingStatus::Optimizing => { // Single-writer rule: never touch the DB while a full run - // is active; the queue drains on a later tick. + // is active; the queue drains on a later tick. Optimizing + // counts — it holds a write transaction over the whole file + // for as long as the rewrite takes. self.saw_running = true; return; } @@ -384,12 +440,12 @@ impl Inner { if self.mode != IndexMode::Auto { if self.mode == IndexMode::ManualStopped { - self.pending.clear(); + self.clear_pending(); } return; } - if !self.pending.is_empty() && !self.needs_full_run { + if !self.pending.is_empty() && !self.needs_full_run && self.pending_settled() { self.apply_pending(); } @@ -399,19 +455,58 @@ impl Inner { } fn drain_events(&mut self) { + let mut received = false; while let Ok(ev) = self.event_rx.try_recv() { enqueue(&mut self.pending, ev); + received = true; + } + if received { + let now = Instant::now(); + self.last_event_at = Some(now); + self.pending_since.get_or_insert(now); + // Before the overflow test, not after: an `rm -rf` of half a + // million files collapses to a handful of directory roots, and + // measuring the queue by its raw event count would throw all of + // them away and schedule a full run instead. + collapse_pending_removals(&mut self.pending); } if self.pending.len() > PENDING_OVERFLOW { // Replaying a storm one file at a time is slower than one // incremental full run (unchanged files skip on mtime). - self.pending.clear(); + self.clear_pending(); self.needs_full_run = true; } } + /// Drop the queue and the timers that describe it, so a stale + /// `pending_since` cannot force an immediate apply of the next event. + fn clear_pending(&mut self) { + self.pending.clear(); + self.last_event_at = None; + self.pending_since = None; + } + + /// Whether the queue has gone quiet long enough to be worth applying, or + /// has waited long enough that it must be applied regardless. + fn pending_settled(&self) -> bool { + let quiet = self + .last_event_at + .is_none_or(|t| t.elapsed() >= self.watcher_config.pending_settle); + let overdue = self + .pending_since + .is_some_and(|t| t.elapsed() >= self.watcher_config.pending_max_defer); + quiet || overdue + } + + /// Apply as much of the queue as fits in [`APPLY_BUDGET`], removals first. + /// + /// Removals lead deliberately. The queue is an unordered map, so the old + /// arbitrary application order could delete a row a `Create` in the same + /// batch had just written. With removals first both interleavings converge + /// on the truth: a delete-then-recreate ends with the file present, and a + /// create-then-delete ends with it absent, because the upsert half consults + /// the filesystem and finds nothing there. fn apply_pending(&mut self) { - let events: Vec = self.pending.drain().map(|(_, ev)| ev).collect(); let conn = match self.ensure_write_conn() { Ok(conn) => conn, Err(e) => { @@ -423,13 +518,57 @@ impl Inner { }; // Borrow dance: pull the connection out while applying. let mut conn = conn; - for ev in &events { - if let Err(e) = apply_fs_event(&mut conn, ev, &self.config, &self.ignore, &self.registry) - { - crate::log_warn!("coordinator: apply {:?}: {}", ev, e); + let deadline = Instant::now() + APPLY_BUDGET; + let chunk = self.config.processing.batch_size.max(1); + + let removals: Vec = self + .pending + .iter() + .filter(|(_, ev)| is_removal(ev)) + .map(|(p, _)| p.clone()) + .collect(); + for batch in removals.chunks(chunk) { + if let Err(e) = crate::incremental::remove_paths(&mut conn, batch, chunk) { + crate::log_warn!("coordinator: remove: {}", e); + } + for path in batch { + self.pending.remove(path); + } + if Instant::now() >= deadline { + break; } } + + if Instant::now() < deadline { + let upserts: Vec = self + .pending + .iter() + .filter(|(_, ev)| !is_removal(ev)) + .map(|(p, _)| p.clone()) + .collect(); + for path in upserts { + let Some(ev) = self.pending.remove(&path) else { + continue; + }; + if let Err(e) = + apply_fs_event(&mut conn, &ev, &self.config, &self.ignore, &self.registry) + { + crate::log_warn!("coordinator: apply {:?}: {}", ev, e); + } + if Instant::now() >= deadline { + break; + } + } + } + self.write_conn = Some(conn); + // Whatever is left goes to the next tick, and goes immediately: the + // pause was ours, not the filesystem's, so it must not re-arm the + // settle window. + self.last_event_at = None; + if self.pending.is_empty() { + self.pending_since = None; + } } fn ensure_write_conn(&mut self) -> Result { @@ -478,7 +617,18 @@ impl Inner { // The full run owns the DB (and may wipe/rebuild the file). self.write_conn = None; self.needs_full_run = false; - self.pending.clear(); + // Queued creates and modifies are dropped — the walk about to start + // rediscovers every one of them. Removals are kept, because it cannot: + // a deletion under a directory the walk fails to read is deliberately + // left alone by `unreadable.covers`, and one of a symlink target whose + // real parent lies outside every root is exempted by `aliased_paths`. + // Those rows would otherwise leak until a rebuild. Re-applying a + // removal the walk did happen to catch is a harmless no-op. + self.pending.retain(|_, ev| is_removal(ev)); + if self.pending.is_empty() { + self.last_event_at = None; + self.pending_since = None; + } if let Err(e) = self .indexing .start_indexing(roots, self.db_path(), self.config.clone()) @@ -486,18 +636,10 @@ impl Inner { crate::log_warn!("coordinator: start indexing: {}", e); return; } - // Give the service's command thread a moment to flip the status; - // small trees can finish between two coordinator ticks, and the - // finished-run bookkeeping keys off `saw_running`. - for _ in 0..200 { - if !matches!( - self.indexing.get_status(), - IndexingStatus::Idle | IndexingStatus::Error(_) - ) { - break; - } - std::thread::sleep(Duration::from_millis(10)); - } + // `start_indexing` claims the Running status before it returns, so + // `get_status()` is already authoritative — no poll, and no window in + // which this thread could believe the service idle and start writing + // to a database the run is about to reopen. self.saw_running = true; } @@ -516,7 +658,7 @@ impl Inner { self.mode = IndexMode::ManualStopped; self.config.indexing.auto_index = false; self.stop_watcher(); - self.pending.clear(); + self.clear_pending(); let status = self.indexing.get_status(); if !matches!(status, IndexingStatus::Idle | IndexingStatus::Error(_)) { // Signal only — waiting up to 5 s here would stall every @@ -775,11 +917,13 @@ mod tests { } /// Short debounce windows so trailing-edge events flush within test - /// timeouts (production default is a 30 s window). + /// timeouts (production defaults are 30 s / 2 s). fn fast_watcher() -> WatcherConfig { WatcherConfig { throttle_window: Duration::from_millis(300), tick_interval: Duration::from_millis(100), + pending_settle: Duration::from_millis(200), + pending_max_defer: Duration::from_secs(3), ..WatcherConfig::default() } } @@ -983,6 +1127,114 @@ mod tests { std::fs::remove_dir_all(&extra_root).ok(); } + /// A `rm -rf` reports the directory *and* everything under it. Only the + /// directory needs applying — its range sweep covers the rest — and + /// collapsing before the overflow test is what stops a large deletion from + /// discarding the queue and forcing a full run. + #[test] + fn collapsing_reduces_a_tree_deletion_to_its_root() { + let mut pending = HashMap::new(); + let dir = PathBuf::from("/x/tree"); + enqueue(&mut pending, FsEvent::Remove(dir.clone())); + for i in 0..500 { + enqueue( + &mut pending, + FsEvent::Remove(dir.join(format!("sub{}/f{}.txt", i % 5, i))), + ); + enqueue(&mut pending, FsEvent::Remove(dir.join(format!("sub{}", i % 5)))); + } + // Not under the removed tree, and not a removal: both must survive. + enqueue(&mut pending, FsEvent::Remove(PathBuf::from("/x/treehouse"))); + enqueue(&mut pending, FsEvent::Create(dir.join("reborn.txt"))); + + collapse_pending_removals(&mut pending); + + let mut left: Vec = pending + .keys() + .map(|p| p.to_string_lossy().into_owned()) + .collect(); + left.sort(); + assert_eq!( + left, + vec![ + "/x/tree".to_string(), + "/x/tree/reborn.txt".to_string(), + "/x/treehouse".to_string(), + ], + "only the removal root, the re-creation, and the prefix sibling remain" + ); + } + + #[test] + fn collapsing_a_queue_without_removals_is_a_no_op() { + let mut pending = HashMap::new(); + enqueue(&mut pending, FsEvent::Create(PathBuf::from("/x/a"))); + enqueue(&mut pending, FsEvent::Modify(PathBuf::from("/x/a/b"))); + collapse_pending_removals(&mut pending); + assert_eq!(pending.len(), 2); + } + + /// Deleting a populated directory must land as one queued removal, not one + /// per file, and the rows must actually go. + #[test] + fn a_deleted_directory_is_applied_as_a_single_collapsed_removal() { + let f = Fixture::new(true); + let tree = f.dir.join("tree"); + for i in 0..40 { + let p = tree.join(format!("sub{}/f{:03}.txt", i % 4, i)); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(&p, format!("body {}", i)).unwrap(); + } + std::fs::write(f.dir.join("keep.txt"), "survivor").unwrap(); + + let coord = + IndexCoordinator::start_with_watcher_config(f.config.clone(), fast_watcher()).unwrap(); + wait_for("initial index", Duration::from_secs(30), || { + f.file_count() == 41 + }); + + std::fs::remove_dir_all(&tree).unwrap(); + wait_for("subtree removed from the index", Duration::from_secs(30), || { + f.file_count() == 1 + }); + + coord.shutdown(); + } + + /// The queue must not be applied while a full run owns the database, and + /// the events must survive to be applied once it finishes. + #[test] + fn a_deletion_during_a_full_run_is_queued_then_applied() { + let f = Fixture::new(false); // manual: runs happen only when asked + for i in 0..300 { + let p = f.dir.join(format!("d{}/f{:03}.txt", i % 6, i)); + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(&p, format!("body {}", i)).unwrap(); + } + + let coord = + IndexCoordinator::start_with_watcher_config(f.config.clone(), fast_watcher()).unwrap(); + coord.reindex_now(); + wait_for("first run", Duration::from_secs(30), || { + coord.state().last_full_index.is_some() && f.file_count() == 300 + }); + + // Auto mode so the watcher is live, then delete while a run is going. + coord.set_mode(IndexMode::Auto); + wait_for("watcher active", Duration::from_secs(30), || { + matches!(coord.state().watcher, WatcherStatus::Active { .. }) + }); + coord.reindex_now(); + std::fs::remove_dir_all(f.dir.join("d0")).unwrap(); + + // 300 files, 50 of them under d0. + wait_for("deletion applied after the run", Duration::from_secs(60), || { + f.file_count() == 250 + }); + + coord.shutdown(); + } + #[test] fn enqueue_last_wins_and_rename_splits() { let mut pending = HashMap::new(); diff --git a/crates/quicksearch-core/src/db/mod.rs b/crates/quicksearch-core/src/db/mod.rs index a3fd5af..d87d855 100644 --- a/crates/quicksearch-core/src/db/mod.rs +++ b/crates/quicksearch-core/src/db/mod.rs @@ -15,6 +15,6 @@ pub mod schema; pub use key::{process_key_hex, set_process_key}; pub use open::{ - open_existing, open_or_recreate, verify_process_key, CURRENT_SCHEMA_VERSION, - KEY_MISMATCH_PREFIX, + index_needs_rebuild, open_existing, open_or_recreate, verify_process_key, + CURRENT_SCHEMA_VERSION, KEY_MISMATCH_PREFIX, }; diff --git a/crates/quicksearch-core/src/db/open.rs b/crates/quicksearch-core/src/db/open.rs index b248811..69a309c 100644 --- a/crates/quicksearch-core/src/db/open.rs +++ b/crates/quicksearch-core/src/db/open.rs @@ -18,8 +18,8 @@ use std::path::Path; use rusqlite::{params, Connection, OpenFlags, OptionalExtension}; use super::schema::{ - effective_tokenizer, fts_create_sql, PRAGMAS_FAST, PRAGMAS_READONLY, PRAGMAS_WALK_READER, - SCHEMA_CURRENT, + effective_tokenizer, fts_create_sql, PRAGMAS_FAST, PRAGMAS_MAINTENANCE, PRAGMAS_READONLY, + PRAGMAS_WALK_READER, SCHEMA_CURRENT, }; use crate::security::IndexKey; @@ -33,7 +33,7 @@ pub const KEY_MISMATCH_PREFIX: &str = "KEY_MISMATCH: "; /// a way that makes an old DB unreadable by new code. Any such bump /// causes existing indexes to be wiped on next open — there's no /// migration path by design. -pub const CURRENT_SCHEMA_VERSION: u32 = 3; +pub const CURRENT_SCHEMA_VERSION: u32 = 4; /// Open `db_path`, applying fast-path pragmas, and ensure the on-disk /// schema matches what this build expects. If it doesn't, delete the @@ -114,6 +114,17 @@ pub fn open_walk_reader(db_path: &str) -> Result { ) } +/// A writable connection for post-run compaction, and the only one that may +/// VACUUM. See [`PRAGMAS_MAINTENANCE`] for why it cannot be the indexer's. +pub fn open_maintenance(db_path: &str) -> Result { + open_keyed_with_pragmas( + db_path, + true, + super::key::process_key().as_ref(), + PRAGMAS_MAINTENANCE, + ) +} + pub(crate) fn open_existing_keyed( db_path: &str, write: bool, @@ -155,8 +166,63 @@ fn open_keyed_with_pragmas( /// index. Used by the GUI unlock screen and the CLI prompt loop before any /// service starts; the error carries [`KEY_MISMATCH_PREFIX`] on a wrong /// password. +/// +/// Answers **only** the key question. It deliberately does not go through +/// [`open_existing`], which additionally demands a current schema — a +/// different question, with a different owner. Whether the stored schema is +/// current is the *indexer's* business, and its answer is to wipe and rebuild +/// ([`open_or_recreate`]); an unlock screen has nothing useful to do with it. +/// +/// Conflating the two made every schema bump present itself to anyone using +/// password protection as an unlock failure, with no way past the gate even +/// with the correct password: +/// +/// ```text +/// index at …/index.sqlite is not a compatible QuickSearch index +/// (schema v4 expected); refusing to modify it. Re-index to rebuild. +/// ``` +/// +/// An unprotected install in the same state starts fine and rebuilds on its +/// first run; this keeps the protected one behaving the same way. pub fn verify_process_key(db_path: &str) -> Result<(), String> { - open_existing(db_path, false).map(|_| ()) + verify_key(db_path, super::key::process_key().as_ref()) +} + +/// Whether an existing index will be discarded and rebuilt by the next +/// indexing run because it was written under a different schema version. +/// +/// The GUI asks this at startup so it can *say so*. The wipe is otherwise +/// silent: `open_or_recreate` replaces the file, every search fails in the +/// meantime, and the only visible explanation is a schema-version error +/// string — which reads like data loss rather than a version upgrade. +/// +/// `false` for anything this cannot positively establish: no file yet, a file +/// the process key does not open, or a database that cannot be queried at all. +/// Announcing a reset that is not happening would be worse than saying nothing. +pub fn index_needs_rebuild(db_path: &str) -> bool { + let Ok(conn) = Connection::open_with_flags( + db_path, + OpenFlags::SQLITE_OPEN_NO_MUTEX | OpenFlags::SQLITE_OPEN_READ_ONLY, + ) else { + return false; + }; + if key_and_probe(&conn, db_path, super::key::process_key().as_ref()).is_err() { + return false; + } + // Only `Ok(false)` — a version we read and it differed, or a layout with no + // `schema_info` at all. An `Err` means we could not tell. + matches!(schema_version_current(&conn), Ok(false)) +} + +pub(crate) fn verify_key(db_path: &str, key: Option<&IndexKey>) -> Result<(), String> { + // Read-only and no CREATE: verifying a key must never bring a database + // into existence, and must never modify one. + let conn = Connection::open_with_flags( + db_path, + OpenFlags::SQLITE_OPEN_NO_MUTEX | OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|e| format!("Failed to open database at {}: {}", db_path, e))?; + key_and_probe(&conn, db_path, key) } /// Apply the SQLCipher key (if any) and force the first page off disk. @@ -627,6 +693,84 @@ mod tests { std::fs::remove_file(&p).ok(); } + /// The one difference that makes the maintenance profile exist. + /// + /// SQLCipher is compiled `-DSQLITE_TEMP_STORE=2`, under which SQLite puts + /// temporary databases in memory for any `temp_store` but an explicit + /// `FILE` (1). VACUUM builds the whole replacement index in that temporary + /// database, so on the indexer's connection — which sets `MEMORY` — it + /// would try to hold a rebuilt multi-gigabyte index in RAM. + #[test] + fn maintenance_opens_keep_temporaries_on_disk() { + let p = tmp_db_path(); + { + let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + let indexer: i64 = conn.query_row("PRAGMA temp_store", [], |r| r.get(0)).unwrap(); + assert_eq!(indexer, 2, "the indexer's own profile is MEMORY"); + } + + let conn = open_maintenance(p.to_str().unwrap()).unwrap(); + let store: i64 = conn.query_row("PRAGMA temp_store", [], |r| r.get(0)).unwrap(); + assert_eq!(store, 1, "maintenance must build its temporaries on disk"); + + // And the directory those temporaries land in is steerable, which is + // what keeps them off a RAM-backed /tmp. Deprecated but present. + let dir = p.parent().unwrap().to_string_lossy().into_owned(); + conn.execute_batch(&format!("PRAGMA temp_store_directory = '{}';", dir)) + .unwrap(); + let set: String = conn + .query_row("PRAGMA temp_store_directory", [], |r| r.get(0)) + .unwrap(); + assert_eq!(set, dir); + conn.execute_batch("PRAGMA temp_store_directory = '';").unwrap(); + + drop(conn); + std::fs::remove_file(&p).ok(); + } + + /// Drives the GUI's "your index is being reset" modal, so a false positive + /// announces a wipe that is not happening and a false negative lets one + /// happen in silence. + #[test] + fn index_needs_rebuild_only_when_the_schema_really_differs() { + let p = tmp_db_path(); + assert!( + !index_needs_rebuild(p.to_str().unwrap()), + "no file yet is a fresh install, not a reset" + ); + + { + let _ = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + } + assert!( + !index_needs_rebuild(p.to_str().unwrap()), + "a current index is not going to be rebuilt" + ); + + // Age it, exactly as a version bump does. + { + let conn = open_existing(p.to_str().unwrap(), true).unwrap(); + conn.execute("UPDATE schema_info SET value = '1' WHERE key = 'version'", []) + .unwrap(); + } + assert!(index_needs_rebuild(p.to_str().unwrap())); + + // A pre-`schema_info` layout counts too. + std::fs::remove_file(&p).ok(); + { + let conn = Connection::open(&p).unwrap(); + conn.execute("CREATE TABLE files (id INTEGER PRIMARY KEY, name TEXT)", []) + .unwrap(); + } + assert!(index_needs_rebuild(p.to_str().unwrap())); + + // Not a database at all: we cannot tell, so we say nothing. + std::fs::write(&p, [0x5a; 4096]).unwrap(); + assert!(!index_needs_rebuild(p.to_str().unwrap())); + + std::fs::remove_file(&p).ok(); + } + #[test] fn open_existing_errors_on_missing_file() { let p = tmp_db_path(); diff --git a/crates/quicksearch-core/src/db/repo.rs b/crates/quicksearch-core/src/db/repo.rs index ae8654e..f4bcf98 100644 --- a/crates/quicksearch-core/src/db/repo.rs +++ b/crates/quicksearch-core/src/db/repo.rs @@ -36,12 +36,20 @@ pub struct NewFile<'a> { pub mime: Option<&'a str>, pub ftype: FileType, pub hash: Option<&'a [u8]>, + /// Whether the content pass has work to do for this file. `false` means + /// the row is born `STATE_NA` — nothing claims its MIME, the + /// `content_extensions` filter excludes it, or it is over + /// `maximum_text_file_size`. Decided at walk time by + /// [`crate::file_handling::content_extractable`]. + pub needs_content: bool, } /// Insert a new file row, returning its id. `basic_state` is set to DONE -/// (row existing *is* the basic-index state); `content_state` is PENDING -/// unless the MIME maps to a type we won't extract text from, in which case -/// the caller can later set it to NA. +/// (row existing *is* the basic-index state); `content_state` comes from +/// `needs_content` — PENDING for a file an extractor will claim, NA for one it +/// won't. Deciding here rather than on a content worker is what keeps PENDING +/// meaning "real work outstanding", which the extraction progress denominator +/// and [`pending_content_page`] both depend on. /// /// Uses `INSERT OR IGNORE` so a UNIQUE(path) collision (which indicates the /// caller fed the same path twice in one run) becomes a silent no-op @@ -49,12 +57,14 @@ pub struct NewFile<'a> { /// expected to dedupe visits upstream; this is a defense-in-depth backstop. pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result, String> { let rows = tx - .execute( + .prepare_cached( "INSERT OR IGNORE INTO files ( name, path, parent, size, mtime, inode, device_id, mime, type, basic_state, content_state, hash ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", - params![ + ) + .and_then(|mut stmt| { + stmt.execute(params![ f.name, f.path, f.parent, @@ -65,10 +75,10 @@ pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result, f.mime, f.ftype.bits() as i64, STATE_DONE, - STATE_PENDING, + initial_content_state(f), f.hash, - ], - ) + ]) + }) .map_err(|e| format!("insert file {}: {}", f.path, e))?; if rows == 0 { // Existing row with the same path (e.g. duplicate visit within the run). @@ -77,44 +87,58 @@ pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result, Ok(Some(tx.last_insert_rowid())) } -/// Update a file's metadata in place (same path, changed size/mtime/hash). -/// Clears any extracted content so the text-indexing pass re-processes it. +/// The `content_state` a freshly written row starts in. Shared by +/// [`insert_file`] and [`update_file_basic`] so a file that reappears as an +/// update lands in the same state it would have as an insert. +fn initial_content_state(f: &NewFile<'_>) -> i64 { + if f.needs_content { + STATE_PENDING + } else { + STATE_NA + } +} + +/// Update a file's metadata in place (same path, changed size/mtime/hash) and +/// reset its content state from `f.needs_content`, clearing any extracted +/// content so the text-indexing pass re-processes it. +/// +/// Writes `size`, `mtime`, `hash`, `mime`, `type`, `content_state` and +/// `failure_msg` — and only those. `name`, `parent`, `inode` and `device_id` +/// are not refreshed here despite being present on the `NewFile`: the row is +/// found by path, and the first two are functions of it. pub fn update_file_basic( tx: &Transaction<'_>, - path: &str, - size: u64, - mtime: u64, - hash: Option<&[u8]>, - mime: Option<&str>, - ftype: FileType, + f: &NewFile<'_>, ) -> Result, String> { + // One statement, not a lookup then an update: `RETURNING` hands back the + // id of the row it just wrote, and a miss is simply no row returned. let id: Option = tx - .query_row( - "SELECT id FROM files WHERE path = ?1", - params![path], - |r| r.get(0), + .prepare_cached( + "UPDATE files + SET size = ?1, mtime = ?2, hash = ?3, mime = ?4, type = ?5, + content_state = ?6, failure_msg = NULL + WHERE path = ?7 + RETURNING id", ) - .optional() - .map_err(|e| format!("lookup file id {}: {}", path, e))?; + .and_then(|mut stmt| { + stmt.query_row( + params![ + f.size as i64, + f.mtime as i64, + f.hash, + f.mime, + f.ftype.bits() as i64, + initial_content_state(f), + f.path, + ], + |r| r.get(0), + ) + .optional() + }) + .map_err(|e| format!("update file {}: {}", f.path, e))?; let Some(id) = id else { return Ok(None); }; - tx.execute( - "UPDATE files - SET size = ?1, mtime = ?2, hash = ?3, mime = ?4, type = ?5, - content_state = ?6, failure_msg = NULL - WHERE id = ?7", - params![ - size as i64, - mtime as i64, - hash, - mime, - ftype.bits() as i64, - STATE_PENDING, - id, - ], - ) - .map_err(|e| format!("update file {}: {}", path, e))?; // Any prior extracted content is stale — remove it along with its FTS row. remove_content_for_id(tx, id)?; Ok(Some(id)) @@ -141,19 +165,17 @@ pub fn set_content_done( remove_content_for_id(tx, file_id)?; for (k, v) in properties { - tx.execute( - "INSERT INTO properties(file_id, key, value) VALUES (?1, ?2, ?3)", - params![file_id, k, v], - ) - .map_err(|e| format!("insert property {}={}: {}", k, v, e))?; + tx.prepare_cached("INSERT INTO properties(file_id, key, value) VALUES (?1, ?2, ?3)") + .and_then(|mut stmt| stmt.execute(params![file_id, k, v])) + .map_err(|e| format!("insert property {}={}: {}", k, v, e))?; } let props_blob = encode_properties_for_fts(properties); // Contentless FTS5 still accepts values on INSERT — the tokenizer needs // them — it simply doesn't persist the raw column values. - tx.execute( + tx.prepare_cached( "INSERT INTO searchabletext(rowid, name, text, properties) VALUES (?1, ?2, ?3, ?4)", - params![file_id, name, text, props_blob], ) + .and_then(|mut stmt| stmt.execute(params![file_id, name, text, props_blob])) .map_err(|e| format!("insert FTS row {}: {}", file_id, e))?; // Skip the compressed sidecar when: the config disables snippet storage @@ -163,20 +185,19 @@ pub fn set_content_done( if store_text && !text.is_empty() { let compressed = zstd::encode_all(text.as_bytes(), ZSTD_LEVEL) .map_err(|e| format!("zstd encode for file {}: {}", file_id, e))?; - tx.execute( + tx.prepare_cached( "INSERT INTO documents_text(file_id, text_zstd, text_len) VALUES (?1, ?2, ?3)", - params![file_id, compressed, text.len() as i64], ) + .and_then(|mut stmt| stmt.execute(params![file_id, compressed, text.len() as i64])) .map_err(|e| format!("insert documents_text {}: {}", file_id, e))?; } - tx.execute( - "UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2", - params![STATE_DONE, file_id], - ) - .map_err(|e| format!("update content_state DONE {}: {}", file_id, e))?; + tx.prepare_cached("UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2") + .and_then(|mut stmt| stmt.execute(params![STATE_DONE, file_id])) + .map_err(|e| format!("update content_state DONE {}: {}", file_id, e))?; // Clear any prior failed-file record. - tx.execute("DELETE FROM failed_files WHERE file_id = ?1", params![file_id]) + tx.prepare_cached("DELETE FROM failed_files WHERE file_id = ?1") + .and_then(|mut stmt| stmt.execute(params![file_id])) .map_err(|e| format!("clear failed_files {}: {}", file_id, e))?; Ok(()) } @@ -198,15 +219,13 @@ pub fn set_content_failed( .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0) as i64; - tx.execute( - "UPDATE files SET content_state = ?1, failure_msg = ?2 WHERE id = ?3", - params![STATE_FAILED, reason, file_id], - ) - .map_err(|e| format!("update content_state FAILED {}: {}", file_id, e))?; - tx.execute( + tx.prepare_cached("UPDATE files SET content_state = ?1, failure_msg = ?2 WHERE id = ?3") + .and_then(|mut stmt| stmt.execute(params![STATE_FAILED, reason, file_id])) + .map_err(|e| format!("update content_state FAILED {}: {}", file_id, e))?; + tx.prepare_cached( "INSERT OR REPLACE INTO failed_files(file_id, reason, ts) VALUES (?1, ?2, ?3)", - params![file_id, reason, now], ) + .and_then(|mut stmt| stmt.execute(params![file_id, reason, now])) .map_err(|e| format!("insert failed_files {}: {}", file_id, e))?; Ok(()) } @@ -214,12 +233,11 @@ pub fn set_content_failed( /// Mark content extraction as not applicable (e.g. binary format we don't /// support). The file row still contributes to filename search. pub fn set_content_na(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> { - tx.execute( - "UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2", - params![STATE_NA, file_id], - ) - .map_err(|e| format!("update content_state NA {}: {}", file_id, e))?; - tx.execute("DELETE FROM failed_files WHERE file_id = ?1", params![file_id]) + tx.prepare_cached("UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2") + .and_then(|mut stmt| stmt.execute(params![STATE_NA, file_id])) + .map_err(|e| format!("update content_state NA {}: {}", file_id, e))?; + tx.prepare_cached("DELETE FROM failed_files WHERE file_id = ?1") + .and_then(|mut stmt| stmt.execute(params![file_id])) .map_err(|e| format!("clear failed_files {}: {}", file_id, e))?; Ok(()) } @@ -227,21 +245,60 @@ pub fn set_content_na(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> /// Delete a file row by path, keeping FTS in sync. Returns whether a row was /// removed. pub fn delete_file_by_path(tx: &Transaction<'_>, path: &str) -> Result { + // `RETURNING` gives the id of the row that went, so the dependent tables + // can be cleared without a separate lookup first. let id: Option = tx - .query_row( - "SELECT id FROM files WHERE path = ?1", - params![path], - |r| r.get(0), - ) - .optional() - .map_err(|e| format!("lookup {} for delete: {}", path, e))?; + .prepare_cached("DELETE FROM files WHERE path = ?1 RETURNING id") + .and_then(|mut stmt| stmt.query_row(params![path], |r| r.get(0)).optional()) + .map_err(|e| format!("delete file {}: {}", path, e))?; let Some(id) = id else { return Ok(false) }; remove_content_for_id(tx, id)?; - tx.execute("DELETE FROM files WHERE id = ?1", params![id]) - .map_err(|e| format!("delete file {}: {}", path, e))?; Ok(true) } +/// Delete every row whose path falls in the half-open range `[lo, hi)`, +/// keeping FTS, `documents_text`, `properties` and `failed_files` in step. +/// Returns how many `files` rows went. +/// +/// The bulk counterpart to [`delete_file_by_path`], for removing a whole +/// directory at once. Five statements regardless of how many files the range +/// holds — where deleting them one at a time costs ~5 *per file* — and the +/// range is a plain index seek on `UNIQUE(files.path)`, so this is +/// `SEARCH … (path>? AND path, lo: &str, hi: &str) -> Result { + // Every dependent table is keyed by the file id, so they share one + // sub-select; `files` itself goes last, once nothing references it. + for (table, key) in [ + ("searchabletext", "rowid"), + ("documents_text", "file_id"), + ("properties", "file_id"), + ("failed_files", "file_id"), + ] { + let sql = format!( + "DELETE FROM {} WHERE {} IN \ + (SELECT id FROM files WHERE path >= ?1 AND path < ?2)", + table, key + ); + tx.prepare_cached(&sql) + .and_then(|mut stmt| stmt.execute(params![lo, hi])) + .map_err(|e| format!("delete {} under {}: {}", table, lo, e))?; + } + let removed = tx + .prepare_cached("DELETE FROM files WHERE path >= ?1 AND path < ?2") + .and_then(|mut stmt| stmt.execute(params![lo, hi])) + .map_err(|e| format!("delete files under {}: {}", lo, e))?; + Ok(removed) +} + /// Every indexed file directly inside `parent`, as `name -> mtime`. /// /// The walk's unit of classification. Keyed by name rather than full path @@ -268,6 +325,47 @@ pub fn dir_rows( Ok(out) } +/// One page of rows still awaiting content extraction under `cursor`'s range, +/// as `(id, name, path, mime)` ordered by id. +/// +/// Keyset, not `OFFSET`: `id > cursor.last_id` means each page is an index +/// seek rather than a re-scan of everything already handed out, and — because +/// the cursor only moves forward — a row is served exactly once even though +/// the writer is concurrently flipping `content_state` behind the reader. The +/// `content_state = 0` predicate is the belt to that braces, not the +/// mechanism. +pub fn pending_content_page( + conn: &Connection, + cursor: &crate::file_handling::ExtractCursor, + max_size: i64, + limit: i64, +) -> Result)>, String> { + let mut stmt = conn + .prepare_cached( + "SELECT id, name, path, mime FROM files + WHERE content_state = 0 AND size <= ?1 AND id > ?2 + AND path >= ?3 AND path < ?4 + ORDER BY id + LIMIT ?5", + ) + .map_err(|e| format!("prepare pending content query: {}", e))?; + let rows = stmt + .query_map( + params![max_size, cursor.last_id, cursor.lo, cursor.hi, limit], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + )) + }, + ) + .map_err(|e| format!("query pending content: {}", e))?; + rows.collect::, _>>() + .map_err(|e| format!("read pending content row: {}", e)) +} + /// The stored mtime for one exact path, or `None` if it isn't indexed. /// /// For files the walk reaches by a spelling whose parent isn't the directory @@ -329,18 +427,16 @@ pub fn paths_in_dir(conn: &Connection, parent: &str) -> Result, Stri pub fn remove_content_for_id(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> { // `contentless_delete=1` on the FTS5 table makes this work without // re-supplying the old column values (it tombstones the rowid). - tx.execute( - "DELETE FROM searchabletext WHERE rowid = ?1", - params![file_id], - ) - .map_err(|e| format!("FTS delete row {}: {}", file_id, e))?; - tx.execute( - "DELETE FROM documents_text WHERE file_id = ?1", - params![file_id], - ) - .map_err(|e| format!("delete documents_text {}: {}", file_id, e))?; - tx.execute("DELETE FROM properties WHERE file_id = ?1", params![file_id]) - .map_err(|e| format!("delete properties {}: {}", file_id, e))?; + for (table, key) in [ + ("searchabletext", "rowid"), + ("documents_text", "file_id"), + ("properties", "file_id"), + ] { + let sql = format!("DELETE FROM {} WHERE {} = ?1", table, key); + tx.prepare_cached(&sql) + .and_then(|mut stmt| stmt.execute(params![file_id])) + .map_err(|e| format!("delete {} for {}: {}", table, file_id, e))?; + } Ok(()) } @@ -359,14 +455,115 @@ fn encode_properties_for_fts(props: &[(String, String)]) -> String { buf } +/// Free pages, as a percentage of the file, that make a [`maintain`] VACUUM +/// worth its cost. Below this the file is rewritten for nothing: a run that +/// changed little leaves little slack, and rewriting a multi-gigabyte index +/// to reclaim a few megabytes is minutes of I/O for no gain. +const VACUUM_MIN_SLACK_PERCENT: i64 = 10; + +/// Flush the whole WAL into the main database and truncate the log to zero +/// bytes. `Err` means the log was *not* emptied. +/// +/// `execute`/`execute_batch` discard the pragma's result row, and that row is +/// the only place SQLite reports that a checkpoint gave up — an incomplete +/// checkpoint is not an error, it is a number in a row nobody read. That is +/// how a log can grow past the index it journals without a word in the logs. +/// +/// The signal is the *log* column (frames left in the WAL), not `busy`: a +/// TRUNCATE that cannot take the writer lock silently downgrades itself to +/// PASSIVE and still reports `busy = 0` with the log untouched. A real restart +/// sets `mxFrame` to 0. A database not in WAL mode reports -1, hence `<= 0`. +pub fn checkpoint_truncate(conn: &Connection) -> Result<(), String> { + let (busy, log): (i64, i64) = conn + .query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| { + Ok((r.get(0)?, r.get(1)?)) + }) + .map_err(|e| format!("wal checkpoint: {}", e))?; + if log <= 0 { + Ok(()) + } else { + Err(format!( + "wal checkpoint incomplete: busy={}, {} frames left in the log", + busy, log + )) + } +} + /// Flush the WAL into the main DB file and close. Call on clean shutdown so /// the next open starts with an empty log. WAL mode itself is persistent in /// the file — deliberately left on. pub fn checkpoint_and_close(conn: Connection) { - let _ = conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);"); + if let Err(e) = checkpoint_truncate(&conn) { + crate::log_warn!("{}", e); + } drop(conn); } +/// Land the log, reclaim the file's slack, and refresh the query planner's +/// statistics. Returns whether it vacuumed. +/// +/// The sequence is checkpoint → VACUUM → `PRAGMA optimize` → checkpoint. The +/// trailing checkpoint is not a repeat of the first: VACUUM's copy-back pushes +/// every page of the rebuilt file through the log, and `optimize` writes +/// `sqlite_stat1`, so leaving without one would trade the slack just reclaimed +/// for a log the size of the index. +/// +/// Run on a connection from [`crate::db::open::open_maintenance`], never on +/// the indexer's: VACUUM builds the replacement database in the temp store, +/// and under the indexer's `temp_store = MEMORY` that means building the whole +/// index in RAM. +/// +/// `db_dir` is where the temporary database goes, and it must be the index's +/// own directory. `temp_store = FILE` alone resolves through `SQLITE_TMPDIR`, +/// `TMPDIR`, `/var/tmp`, then `/tmp` — and `/tmp` is a RAM-backed tmpfs on many +/// Linux systems, which would put us straight back where we started. Keeping +/// the temporary database beside the index also puts it on a volume already +/// known to hold one. +/// +/// Peak transient space on that volume is roughly three times the index: the +/// original, the replacement being built beside it, and the log that VACUUM's +/// copy-back runs through. Running out is a failed VACUUM, not a damaged +/// index — the transaction rolls back. +pub fn maintain(conn: &Connection, db_dir: &str) -> Result { + // Best-effort: a reader that briefly pins the log is no reason to skip + // the compaction below, which does not need the log empty to start. + if let Err(e) = checkpoint_truncate(conn) { + crate::log_warn!("{}", e); + } + + let page_count: i64 = conn + .query_row("PRAGMA page_count", [], |r| r.get(0)) + .map_err(|e| format!("read page_count: {}", e))?; + let freelist: i64 = conn + .query_row("PRAGMA freelist_count", [], |r| r.get(0)) + .map_err(|e| format!("read freelist_count: {}", e))?; + + let vacuumed = freelist * 100 >= page_count * VACUUM_MIN_SLACK_PERCENT; + if vacuumed { + // `temp_store_directory` is a deprecated pragma that writes a global, + // so it is set for the VACUUM and cleared straight after rather than + // left standing for every other connection in the process. + let escaped = db_dir.replace('\'', "''"); + conn.execute_batch(&format!("PRAGMA temp_store_directory = '{}';", escaped)) + .map_err(|e| format!("set temp dir for vacuum: {}", e))?; + let outcome = conn + .execute_batch("VACUUM;") + .map_err(|e| format!("vacuum: {}", e)); + let _ = conn.execute_batch("PRAGMA temp_store_directory = '';"); + outcome?; + } + + // Re-analyses only the tables whose shape has drifted far enough to matter, + // so it is close to free on a run that changed little. A run that added + // millions of rows is exactly when the planner's old statistics start + // choosing the wrong index for a search. + conn.execute_batch("PRAGMA optimize;") + .map_err(|e| format!("optimize: {}", e))?; + + checkpoint_truncate(conn)?; + Ok(vacuumed) +} + /// Read the `last_full_index` marker (unix seconds of the last *successful* /// full indexing run) from `schema_info`. Absent key — fresh DB, or a DB /// from before this marker existed — means "never". @@ -396,6 +593,9 @@ pub fn set_last_full_index(conn: &Connection, ts: u64) -> Result<(), String> { #[cfg(test)] mod tests { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + use super::*; use crate::db::open_or_recreate; @@ -431,6 +631,7 @@ mod tests { mime: Some("text/plain"), ftype: FileType::TEXT, hash: Some(&[1, 2, 3]), + needs_content: true, }, ) .unwrap() @@ -477,60 +678,106 @@ mod tests { } #[test] - fn update_resets_content_state() { + fn insert_writes_content_state_from_needs_content() { + // The whole point of the field: a row nothing will extract is born NA, + // so "pending" downstream — the extraction denominator, the feeder's + // page query, the Baloo pending count — means real outstanding work. let p = tmp_path(); let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); - let id = { - let tx = conn.transaction().unwrap(); - let id = insert_file( - &tx, - &NewFile { - name: "a.txt", - path: "/tmp/a.txt", - parent: "/tmp", - size: 10, - mtime: 1, - inode: None, - device_id: None, - mime: None, - ftype: FileType::EMPTY, - hash: None, - }, + let tx = conn.transaction().unwrap(); + let mut row = NewFile { + name: "claimed.txt", + path: "/tmp/claimed.txt", + parent: "/tmp", + size: 1, + mtime: 1, + inode: None, + device_id: None, + mime: Some("text/plain"), + ftype: FileType::TEXT, + hash: None, + needs_content: true, + }; + let claimed = insert_file(&tx, &row).unwrap().expect("unique path"); + row.name = "unclaimed.mp4"; + row.path = "/tmp/unclaimed.mp4"; + row.mime = Some("video/mp4"); + row.needs_content = false; + let unclaimed = insert_file(&tx, &row).unwrap().expect("unique path"); + + let state = |id: i64| -> i64 { + tx.query_row( + "SELECT content_state FROM files WHERE id = ?1", + params![id], + |r| r.get(0), ) .unwrap() - .expect("unique path"); + }; + assert_eq!(state(claimed), STATE_PENDING); + assert_eq!(state(unclaimed), STATE_NA); + + drop(tx); + drop(conn); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn update_writes_content_state_from_needs_content() { + let p = tmp_path(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + let mut row = NewFile { + name: "a.txt", + path: "/tmp/a.txt", + parent: "/tmp", + size: 10, + mtime: 1, + inode: None, + device_id: None, + mime: None, + ftype: FileType::EMPTY, + hash: None, + needs_content: false, + }; + let id = { + let tx = conn.transaction().unwrap(); + let id = insert_file(&tx, &row).unwrap().expect("unique path"); set_content_done(&tx, id, "a.txt", "old text", &[], true).unwrap(); tx.commit().unwrap(); id }; + let content_state = |conn: &Connection| -> i64 { + conn.query_row( + "SELECT content_state FROM files WHERE id = ?1", + params![id], + |r| r.get(0), + ) + .unwrap() + }; + + // Rewritten as something an extractor claims: back to pending, and the + // stale FTS row goes with it. { let tx = conn.transaction().unwrap(); - let got = update_file_basic( - &tx, - "/tmp/a.txt", - 20, - 2, - None, - Some("text/plain"), - FileType::TEXT, - ) - .unwrap(); + row.size = 20; + row.mtime = 2; + row.mime = Some("text/plain"); + row.ftype = FileType::TEXT; + row.needs_content = true; + let got = update_file_basic(&tx, &row).unwrap(); assert_eq!(got, Some(id)); tx.commit().unwrap(); } - - let (state, content): (i64, i64) = conn + let basic: i64 = conn .query_row( - "SELECT basic_state, content_state FROM files WHERE id = ?1", + "SELECT basic_state FROM files WHERE id = ?1", params![id], - |r| Ok((r.get(0)?, r.get(1)?)), + |r| r.get(0), ) .unwrap(); - assert_eq!(state, STATE_DONE); - assert_eq!(content, STATE_PENDING); + assert_eq!(basic, STATE_DONE); + assert_eq!(content_state(&conn), STATE_PENDING); - // FTS row for the stale content should be gone. let fts_hits: i64 = conn .query_row( "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'old'", @@ -540,6 +787,18 @@ mod tests { .unwrap(); assert_eq!(fts_hits, 0); + // And rewritten as something nothing claims: NA, not pending. Without + // this the row would re-enter the content pass on every run. + { + let tx = conn.transaction().unwrap(); + row.mtime = 3; + row.mime = Some("video/mp4"); + row.needs_content = false; + update_file_basic(&tx, &row).unwrap().expect("row still there"); + tx.commit().unwrap(); + } + assert_eq!(content_state(&conn), STATE_NA); + drop(conn); std::fs::remove_file(&p).ok(); } @@ -563,6 +822,7 @@ mod tests { mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, + needs_content: true, }; let id1 = insert_file(&tx, &row).unwrap().expect("first insert"); let id2 = insert_file(&tx, &row).unwrap(); @@ -585,6 +845,134 @@ mod tests { std::fs::remove_file(&p).ok(); } + #[test] + fn delete_subtree_clears_every_dependent_table() { + let p = tmp_path(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + let add = |tx: &Transaction<'_>, path: &str| -> i64 { + let name = path.rsplit('/').next().unwrap(); + let parent = &path[..path.rfind('/').unwrap()]; + let id = insert_file( + tx, + &NewFile { + name, + path, + parent, + size: 1, + mtime: 1, + inode: None, + device_id: None, + mime: Some("text/plain"), + ftype: FileType::TEXT, + hash: None, + needs_content: true, + }, + ) + .unwrap() + .expect("unique path"); + set_content_done(tx, id, name, "body text", &[("k".into(), "v".into())], true).unwrap(); + id + }; + + { + let tx = conn.transaction().unwrap(); + add(&tx, "/tree/a.txt"); + add(&tx, "/tree/deep/b.txt"); + let failed = add(&tx, "/tree/deep/c.txt"); + set_content_failed(&tx, failed, "bad parse").unwrap(); + // Outside the range: a prefix sibling, and a LIKE-metacharacter + // neighbour that a `LIKE 'tree_%'` sweep would have swallowed. + add(&tx, "/tree2/keep.txt"); + add(&tx, "/treeX/keep.txt"); + tx.commit().unwrap(); + } + + let range = crate::file_handling::ExtractCursor::for_root("/tree"); + let removed = { + let tx = conn.transaction().unwrap(); + // The directory row itself does not exist (only files are indexed), + // so everything here comes from the range. + let n = delete_subtree(&tx, &range.lo, &range.hi).unwrap(); + tx.commit().unwrap(); + n + }; + assert_eq!(removed, 3, "three files under /tree"); + + let count = |sql: &str| -> i64 { conn.query_row(sql, [], |r| r.get(0)).unwrap() }; + assert_eq!(count("SELECT COUNT(*) FROM files"), 2, "siblings survive"); + assert_eq!(count("SELECT COUNT(*) FROM searchabletext"), 2); + assert_eq!(count("SELECT COUNT(*) FROM documents_text"), 2); + assert_eq!(count("SELECT COUNT(*) FROM properties"), 2); + assert_eq!( + count("SELECT COUNT(*) FROM failed_files"), + 0, + "the failed row went with its file" + ); + + let survivors: Vec = { + let mut stmt = conn.prepare("SELECT path FROM files ORDER BY path").unwrap(); + let v = stmt + .query_map([], |r| r.get::<_, String>(0)) + .unwrap() + .map(|r| r.unwrap()) + .collect(); + v + }; + assert_eq!(survivors, vec!["/tree2/keep.txt", "/treeX/keep.txt"]); + + drop(conn); + std::fs::remove_file(&p).ok(); + } + + /// The walk's row prefetcher runs `dir_rows` once per directory, against a + /// deliberately tiny page cache, so it must not have to touch the table + /// heap at all. `idx_files_parent` carries `name` and `mtime` for exactly + /// this; trimming it back to `(parent)` would silently reintroduce a row + /// fetch per entry. + #[test] + fn dir_rows_is_served_entirely_from_the_index() { + let p = tmp_path(); + let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + let plan: String = conn + .query_row( + "EXPLAIN QUERY PLAN SELECT name, mtime FROM files WHERE parent = ?1", + params!["/some/dir"], + |r| r.get(3), + ) + .unwrap(); + assert!( + plan.contains("COVERING INDEX idx_files_parent"), + "dir_rows must be index-only, got: {}", + plan + ); + drop(conn); + std::fs::remove_file(&p).ok(); + } + + /// The whole point of the range form: it is an index seek. `LIKE … ESCAPE` + /// cannot use the index (SQLite disables the LIKE optimisation whenever an + /// ESCAPE clause is present), so the old sweep read every path in the table + /// on every deletion event. + #[test] + fn the_subtree_range_is_an_index_seek_not_a_scan() { + let p = tmp_path(); + let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + let plan: String = conn + .query_row( + "EXPLAIN QUERY PLAN DELETE FROM files WHERE path >= ?1 AND path < ?2", + params!["/tree/", "/tree0"], + |r| r.get(3), + ) + .unwrap(); + assert!( + plan.contains("SEARCH") && !plan.contains("SCAN"), + "range delete must seek, got: {}", + plan + ); + drop(conn); + std::fs::remove_file(&p).ok(); + } + #[test] fn last_full_index_round_trip() { let p = tmp_path(); @@ -618,6 +1006,7 @@ mod tests { mime: None, ftype: FileType::EMPTY, hash: None, + needs_content: false, }, ) .unwrap(); @@ -638,6 +1027,249 @@ mod tests { std::fs::remove_file(&p).ok(); } + /// Bulk rows, cheap to write, enough of them to make the file grow. + fn seed_rows(conn: &mut Connection, range: std::ops::Range) { + let tx = conn.transaction().unwrap(); + for i in range { + let path = format!("/tmp/bulk/{}.txt", i); + let name = format!("{}.txt", i); + let id = insert_file( + &tx, + &NewFile { + name: &name, + path: &path, + parent: "/tmp/bulk", + size: 1, + mtime: 1, + inode: None, + device_id: None, + mime: Some("text/plain"), + ftype: FileType::TEXT, + hash: None, + needs_content: true, + }, + ) + .unwrap() + .unwrap(); + set_content_done(&tx, id, &name, &"lorem ipsum dolor sit amet ".repeat(64), &[], true) + .unwrap(); + } + tx.commit().unwrap(); + } + + fn wal_bytes(p: &std::path::Path) -> u64 { + std::fs::metadata(format!("{}-wal", p.display())) + .map(|m| m.len()) + .unwrap_or(0) + } + + #[test] + fn checkpoint_truncate_empties_the_log() { + let p = tmp_path(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + seed_rows(&mut conn, 0..200); + assert!(wal_bytes(&p) > 0, "the writes should be sitting in the log"); + + checkpoint_truncate(&conn).expect("nothing is holding the log back"); + assert_eq!(wal_bytes(&p), 0, "a completed TRUNCATE leaves no log"); + + drop(conn); + std::fs::remove_file(&p).ok(); + } + + /// The case the old `execute_batch("PRAGMA wal_checkpoint(TRUNCATE)")` + /// reported as success: a reader pins the log, SQLite declines to reset + /// it, and the only word of it is in the result row. + #[test] + fn checkpoint_truncate_reports_an_incomplete_checkpoint() { + let p = tmp_path(); + let mut writer = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + seed_rows(&mut writer, 0..200); + // Long enough to prove the point, short enough not to stall the suite. + writer + .busy_timeout(std::time::Duration::from_millis(100)) + .unwrap(); + + let reader = crate::db::open_existing(p.to_str().unwrap(), false).unwrap(); + let mut stmt = reader.prepare("SELECT id FROM files").unwrap(); + let mut rows = stmt.query([]).unwrap(); + rows.next().unwrap().expect("a row to hold the snapshot on"); + + let err = checkpoint_truncate(&writer).expect_err("a reader holds the log open"); + assert!(err.contains("incomplete"), "unexpected message: {}", err); + assert!(wal_bytes(&p) > 0, "and the log is still there"); + + drop(rows); + drop(stmt); + drop(reader); + drop(writer); + std::fs::remove_file(&p).ok(); + } + + /// The mechanism the whole in-run checkpoint exists for. + /// + /// SQLite's autocheckpoint copies committed frames into the database + /// continuously, but the log is only *reset* when the writer opens a + /// transaction at an instant no reader holds a read mark — and it tries + /// that lock exactly once, with no retry. A reader querying back to back, + /// which is what an indexing run's per-root prefetcher is, keeps that + /// instant from arriving, so the log appends for as long as the run lasts. + /// An explicit checkpoint retries the same lock under `busy_timeout` and + /// gets it. + #[test] + fn a_busy_reader_defeats_the_autocheckpoint_but_not_a_forced_one() { + // Bare `files` rows: no text, so no zstd and no tokenising. The log + // grows from committed frames, and frames are what starves the reset — + // paying for extraction here would only make the test slow. + fn seed_bare(conn: &mut Connection, range: std::ops::Range) { + let tx = conn.transaction().unwrap(); + for i in range { + let path = format!("/tmp/bare/{}.txt", i); + let name = format!("{}.txt", i); + insert_file( + &tx, + &NewFile { + name: &name, + path: &path, + parent: "/tmp/bare", + size: i as u64, + mtime: 1, + inode: None, + device_id: None, + mime: None, + ftype: FileType::TEXT, + hash: Some(&[0u8; 32]), + needs_content: false, + }, + ) + .unwrap(); + } + tx.commit().unwrap(); + } + + fn run(p: &std::path::Path, force_every: usize) -> u64 { + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + seed_bare(&mut conn, 0..500); + + // Stands in for a walk prefetcher: short reads, no gaps. + let stop: Arc = Arc::new(AtomicBool::new(false)); + let reader = { + let (path, stop) = (p.to_path_buf(), stop.clone()); + std::thread::spawn(move || { + let conn = crate::db::open_existing(path.to_str().unwrap(), false).unwrap(); + while !stop.load(Ordering::Relaxed) { + let _: i64 = conn + .query_row( + "SELECT COUNT(*) FROM files WHERE parent = '/tmp/bare'", + [], + |r| r.get(0), + ) + .unwrap(); + } + }) + }; + + let mut peak = 0u64; + for batch in 0..60 { + let lo = 1000 + batch * 300; + seed_bare(&mut conn, lo..lo + 300); + if force_every > 0 && batch % force_every == force_every - 1 { + let _ = checkpoint_truncate(&conn); + } + peak = peak.max(wal_bytes(p)); + } + + stop.store(true, Ordering::Relaxed); + reader.join().unwrap(); + drop(conn); + peak + } + + let unbounded = tmp_path(); + let left_alone = run(&unbounded, 0); + std::fs::remove_file(&unbounded).ok(); + + let bounded = tmp_path(); + let forced = run(&bounded, 4); + std::fs::remove_file(&bounded).ok(); + + eprintln!("peak WAL: autocheckpoint only {}, forced {}", left_alone, forced); + assert!( + forced * 2 < left_alone, + "forcing checkpoints did not bound the log: {} vs {}", + forced, + left_alone + ); + } + + #[test] + fn maintain_vacuums_when_slack_is_significant() { + let p = tmp_path(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + seed_rows(&mut conn, 0..2000); + checkpoint_truncate(&conn).unwrap(); + let before = std::fs::metadata(&p).unwrap().len(); + + { + let tx = conn.transaction().unwrap(); + for i in 0..1900 { + delete_file_by_path(&tx, &format!("/tmp/bulk/{}.txt", i)).unwrap(); + } + tx.commit().unwrap(); + } + drop(conn); + + let conn = crate::db::open::open_maintenance(p.to_str().unwrap()).unwrap(); + let freelist: i64 = conn + .query_row("PRAGMA freelist_count", [], |r| r.get(0)) + .unwrap(); + assert!(freelist > 0, "the deletions should have freed pages"); + + let dir = p.parent().unwrap().to_string_lossy().into_owned(); + assert!(maintain(&conn, &dir).unwrap(), "that much slack is worth a vacuum"); + assert_eq!(wal_bytes(&p), 0, "the vacuum's own writes are checkpointed too"); + assert!( + std::fs::metadata(&p).unwrap().len() < before, + "the file should have shrunk" + ); + // The surviving rows are still there and still searchable. + let n: i64 = conn + .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) + .unwrap(); + assert_eq!(n, 100); + let hits: i64 = conn + .query_row( + "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'lorem'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(hits, 100, "the FTS index survived the rewrite"); + + drop(conn); + std::fs::remove_file(&p).ok(); + } + + #[test] + fn maintain_skips_vacuum_on_a_tight_file() { + let p = tmp_path(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + seed_rows(&mut conn, 0..200); + drop(conn); + + let conn = crate::db::open::open_maintenance(p.to_str().unwrap()).unwrap(); + let dir = p.parent().unwrap().to_string_lossy().into_owned(); + assert!( + !maintain(&conn, &dir).unwrap(), + "a file with no slack is not worth rewriting" + ); + // The checkpoint is not conditional on the vacuum, though. + assert_eq!(wal_bytes(&p), 0); + + drop(conn); + std::fs::remove_file(&p).ok(); + } + #[test] fn set_content_failed_writes_failed_table() { let p = tmp_path(); @@ -657,6 +1289,7 @@ mod tests { mime: None, ftype: FileType::EMPTY, hash: None, + needs_content: false, }, ) .unwrap() diff --git a/crates/quicksearch-core/src/db/schema.rs b/crates/quicksearch-core/src/db/schema.rs index dd66e7a..58628ad 100644 --- a/crates/quicksearch-core/src/db/schema.rs +++ b/crates/quicksearch-core/src/db/schema.rs @@ -11,7 +11,9 @@ /// writers exist (full index runs and the coordinator's incremental /// updates) and they're serialized by design; `busy_timeout` is a backstop, /// not a coordination mechanism. A clean shutdown truncates the log via -/// [`super::repo::checkpoint_and_close`]. +/// [`super::repo::checkpoint_and_close`], and a long run truncates it +/// periodically as it goes (see `maximum_wal_size`) — SQLite's own +/// autocheckpoint backfills but cannot reset a log that readers are touching. pub const PRAGMAS_FAST: &str = " PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL; @@ -21,6 +23,27 @@ pub const PRAGMAS_FAST: &str = " PRAGMA foreign_keys = ON; "; +/// Pragmas for the connection that compacts the index after a run. +/// +/// [`PRAGMAS_FAST`] but for `temp_store`, and that one difference is the whole +/// reason this profile exists. VACUUM builds the replacement database in the +/// temp store; SQLCipher is compiled `-DSQLITE_TEMP_STORE=2`, under which +/// anything but an explicit `FILE` puts that database in memory. Vacuuming a +/// multi-gigabyte index on the indexer's connection would try to hold the +/// entire rebuilt index in RAM. See [`super::repo::maintain`], which also +/// points the temp directory at the index's own volume. +/// +/// The smaller page cache is because this connection does one bulk copy and +/// then closes; the 40 MiB the indexer keeps hot buys it nothing. +pub const PRAGMAS_MAINTENANCE: &str = " + PRAGMA journal_mode = WAL; + PRAGMA synchronous = NORMAL; + PRAGMA busy_timeout = 5000; + PRAGMA cache_size = 2000; + PRAGMA temp_store = FILE; + PRAGMA foreign_keys = ON; +"; + /// Pragmas safe to apply on a read-only connection, where `journal_mode` /// and `synchronous` can't be changed on the file. Used by /// [`super::open::open_existing`] for read-only opens; write paths get the @@ -84,7 +107,15 @@ CREATE TABLE files ( hash BLOB ); -CREATE INDEX idx_files_parent ON files(parent); +-- Covering, not just `(parent)`. The walk's row prefetcher issues +-- `SELECT name, mtime FROM files WHERE parent = ?` once per directory — the +-- hottest read in a full run — and with the bare index that is an index probe +-- plus a table-row fetch per entry. Those fetches are cold by design: the walk +-- reader deliberately runs on a 1 MiB page cache (see `PRAGMAS_WALK_READER`). +-- Carrying `name` and `mtime` in the index makes it an index-only scan. +-- `parent` stays leading, so `SELECT DISTINCT parent` range scans and +-- `paths_in_dir` are unaffected. +CREATE INDEX idx_files_parent ON files(parent, name, mtime); CREATE INDEX idx_files_mtime ON files(mtime); CREATE INDEX idx_files_type ON files(type); CREATE INDEX idx_files_mime ON files(mime); diff --git a/crates/quicksearch-core/src/extract/mod.rs b/crates/quicksearch-core/src/extract/mod.rs index d4f14f3..07ee7ca 100644 --- a/crates/quicksearch-core/src/extract/mod.rs +++ b/crates/quicksearch-core/src/extract/mod.rs @@ -121,6 +121,27 @@ impl Registry { self } + /// The extractor that claims `mime`, if any. The one place dispatch + /// happens, so every question about a MIME — "who handles it", "does + /// anyone handle it", "handle it from these bytes" — gets the same answer. + fn find(&self, mime: &str) -> Option<&dyn Extractor> { + let lower = mime.to_ascii_lowercase(); + self.extractors + .iter() + .find(|e| e.supports(&lower)) + .map(|e| &**e) + } + + /// Whether any extractor claims `mime` — the same question + /// [`Registry::extract`] answers by returning `Ok(None)`, asked without + /// touching the file. This is what lets the walk decide a row's + /// `content_state` up front instead of queueing it for a worker that will + /// only mark it not-applicable (see + /// [`crate::file_handling::content_extractable`]). + pub fn supports(&self, mime: &str) -> bool { + self.find(mime).is_some() + } + /// Look up a handler for `mime` and run it against `path`. Returns /// `Ok(None)` if no extractor claims the MIME — the caller should then /// decide whether the file is "not applicable" (text state NA). @@ -129,13 +150,7 @@ impl Registry { path: &Path, mime: &str, ) -> Result, ExtractError> { - let lower = mime.to_ascii_lowercase(); - for e in &self.extractors { - if e.supports(&lower) { - return e.extract(path).map(Some); - } - } - Ok(None) + self.find(mime).map(|e| e.extract(path)).transpose() } /// [`Registry::extract`] for a file whose complete contents the caller @@ -151,13 +166,7 @@ impl Registry { mime: &str, head: &[u8], ) -> Option> { - let lower = mime.to_ascii_lowercase(); - for e in &self.extractors { - if e.supports(&lower) { - return e.extract_from_head(path, head); - } - } - None + self.find(mime).and_then(|e| e.extract_from_head(path, head)) } /// The default set wired up for Set A: plaintext, office docs, PDF, @@ -223,6 +232,43 @@ mod tests { } } + #[test] + fn supports_agrees_with_extract_dispatch() { + // `supports` is the cheap form of the question `extract` answers with + // `Ok(None)`. They must agree for every MIME, or the walk would write + // a content state the content pass then contradicts. The path does not + // exist, so a claimed MIME surfaces as `Err`, not `Ok(None)` — which is + // exactly the distinction under test. + let r = Registry::default_set(); + let missing = Path::new("/nonexistent/quicksearch-supports-probe"); + for mime in [ + "text/plain", + "TEXT/PLAIN", + "text/x-rust", + "application/json", + "APPLICATION/PDF", + "application/pdf", + "audio/mpeg", + "Image/JPEG", + "application/msword", + "application/vnd.oasis.opendocument.text", + // Real MIMEs with no extractor: the population the fix is about. + "video/mp4", + "application/zip", + "application/x-executable", + "application/octet-stream", + "", + ] { + let claimed = !matches!(r.extract(missing, mime), Ok(None)); + assert_eq!( + r.supports(mime), + claimed, + "supports and extract disagree about {:?}", + mime + ); + } + } + #[test] fn properties_sorted_is_deterministic() { let c = ExtractedContent::with_text("hi") diff --git a/crates/quicksearch-core/src/file_handling.rs b/crates/quicksearch-core/src/file_handling.rs index f6d1291..8ed0a2e 100644 --- a/crates/quicksearch-core/src/file_handling.rs +++ b/crates/quicksearch-core/src/file_handling.rs @@ -1,4 +1,4 @@ -use std::sync::atomic::AtomicBool; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Mutex, Arc}; use std::fs::File; use std::io::Read; @@ -532,16 +532,16 @@ fn get_file_hash( Ok((hasher.finalize().to_vec(), head)) } -/// Nudge FTS5 to merge its index segments. Best-effort optimization; any -/// error is logged but not fatal. -pub fn fts_finalize_after_text_indexing(conn: &Connection) -> Result<(), String> { +/// Nudge FTS5 to merge its index segments. Best-effort: the only failure +/// possible is logged and swallowed, so there is nothing for a caller to +/// handle — hence no `Result`. +pub fn fts_finalize_after_text_indexing(conn: &Connection) { if let Err(e) = conn.execute( "INSERT INTO searchabletext(searchabletext, rank) VALUES('automerge', 8)", [], ) { crate::log_warn!("FTS automerge failed (non-fatal): {}", e); } - Ok(()) } /// An owned, fully-derived file record: everything needed to insert or @@ -566,6 +566,11 @@ pub struct OwnedNewFile { /// one, and the walk's channel is bounded at `CHANNEL_CAP`, so this adds /// at most `CHANNEL_CAP * hash_length` of in-flight memory. pub inline_text: Option, + /// Whether the content pass has anything to do for this file — see + /// [`content_extractable`], plus the `maximum_text_file_size` gate. `false` + /// means the row is born `STATE_NA`, so "pending" counts only real work and + /// the pass never queues a file just to write it off. + pub needs_content: bool, } impl OwnedNewFile { @@ -581,6 +586,7 @@ impl OwnedNewFile { mime: self.mime.as_deref(), ftype: self.ftype, hash: Some(&self.hash), + needs_content: self.needs_content, } } } @@ -636,27 +642,27 @@ pub fn prepare_file_record( let mime = guess_mime_from_head(Path::new(path), &head); let ftype = mime.as_deref().map(mime_to_type).unwrap_or(FileType::EMPTY); + // Decided once, here, where the sniffed MIME, the size, the config and the + // registry are all in hand — and stored as the row's `content_state`, so + // "pending" downstream means a file that really needs reading rather than + // one the content pass would only mark not-applicable. + let needs_content = size <= config.processing.maximum_text_file_size + && content_extractable(Path::new(path), mime.as_deref(), config, registry); + // When the head is the whole file, an extractor that works from bytes can // finish the job now and spare the content pass an open/read/close. Any // condition that does not hold simply leaves this `None`, and the file // stays pending exactly as before — including invalid UTF-8, which the // content pass records as a failure with a reason. - let inline_text = mime.as_deref().and_then(|m| { - // The head is the whole file only up to `hash_length`. The - // `maximum_text_file_size` gate is the content pass's own (see - // `extract_scope_prepare`), repeated so both paths agree even when a - // config sets it below `hash_length`. + let inline_text = mime.as_deref().filter(|_| needs_content).and_then(|m| { + // The head is the whole file only up to `hash_length`. // // Size 0 is excluded rather than treated as "trivially complete": // procfs, sysfs and some FUSE mounts report it for files that do have // content, and inlining would store empty text for them. The content // pass reads those correctly, and an actually-empty file costs the // same there as it ever did. - if size == 0 - || size > config.processing.hash_length as u64 - || size > config.processing.maximum_text_file_size - || !crate::config::content_allowed(Path::new(path), config) - { + if size == 0 || size > config.processing.hash_length as u64 { return None; } match registry.extract_complete_head(Path::new(path), m, &head) { @@ -686,6 +692,7 @@ pub fn prepare_file_record( ftype, hash, inline_text, + needs_content, }) } @@ -729,10 +736,71 @@ pub fn extract_and_store( registry: &Registry, config: &Config, ) -> Result<(), String> { + let outcome = decide_content(path, mime, registry, config); + store_content_outcome(tx, file_id, name, &outcome, config) +} + +/// What should be written for one file's content, decided without touching +/// the database. +/// +/// Split from the write so the deciding — which opens the file, reads it and +/// runs an extractor over it — can happen on a worker thread while the single +/// writer holds nothing. See [`crate::content`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ContentOutcome { + /// Text (already truncated to `maximum_text_size`) and sorted properties. + Done { + text: String, + properties: Vec<(String, String)>, + }, + /// No extractor claims the MIME, or `content_extensions` excludes it. + NotApplicable, + /// The extractor ran and failed; the reason goes on the row. + Failed(String), +} + +/// Whether extraction will produce anything for this file: the +/// `content_extensions` filter allows it, and some extractor claims its MIME. +/// +/// The single predicate behind both the `content_state` a row is born with +/// (see [`prepare_file_record`]) and [`decide_content`]'s not-applicable +/// early-out, so the two can never drift. That matters more than it looks: if +/// they disagreed, a file the walk wrote off as NA would silently never be +/// full-text indexed. +/// +/// Deliberately size-free. The content pass's workers never see a size — the +/// `maximum_text_file_size` gate lives in the feeder's query +/// ([`crate::db::repo::pending_content_page`]) and in [`extract_scope_prepare`] +/// — so folding it in here would be a gate only one of the two callers could +/// honour. +pub fn content_extractable( + path: &Path, + mime: Option<&str>, + config: &Config, + registry: &Registry, +) -> bool { + crate::config::content_allowed(path, config) + && mime.is_some_and(|m| registry.supports(m)) +} + +/// Read `path` and decide what its content row should say. No database access, +/// no locks held — this is the expensive half. +/// +/// `mime` is authoritative, including when it is `None`: see +/// [`extract_and_store`]. +pub fn decide_content( + path: &str, + mime: Option<&str>, + registry: &Registry, + config: &Config, +) -> ContentOutcome { let p = Path::new(path); - if !crate::config::content_allowed(p, config) { - return repo::set_content_na(tx, file_id); + if !content_extractable(p, mime, config, registry) { + return ContentOutcome::NotApplicable; } + // `content_extractable` just established that some extractor claims this + // MIME, so the `Ok(None)` arm below is unreachable; it stays as the honest + // answer should that ever stop holding. let result = match mime { Some(m) => registry.extract(p, m), None => Ok(None), @@ -740,20 +808,39 @@ pub fn extract_and_store( match result { Ok(Some(mut content)) => { if content.text.len() > config.processing.maximum_text_size { - content.text = safe_truncate_string(&content.text, config.processing.maximum_text_size); + content.text = + safe_truncate_string(&content.text, config.processing.maximum_text_size); + } + ContentOutcome::Done { + properties: content.properties_sorted(), + text: content.text, } - let props = content.properties_sorted(); - repo::set_content_done( - tx, - file_id, - name, - &content.text, - &props, - config.processing.store_text_for_snippets, - ) } - Ok(None) => repo::set_content_na(tx, file_id), - Err(reason) => repo::set_content_failed(tx, file_id, &reason), + Ok(None) => ContentOutcome::NotApplicable, + Err(reason) => ContentOutcome::Failed(reason), + } +} + +/// Apply a decision from [`decide_content`]. The cheap half: pure database +/// writes, so this is all that runs with the connection held. +pub fn store_content_outcome( + tx: &rusqlite::Transaction<'_>, + file_id: i64, + name: &str, + outcome: &ContentOutcome, + config: &Config, +) -> Result<(), String> { + match outcome { + ContentOutcome::Done { text, properties } => repo::set_content_done( + tx, + file_id, + name, + text, + properties, + config.processing.store_text_for_snippets, + ), + ContentOutcome::NotApplicable => repo::set_content_na(tx, file_id), + ContentOutcome::Failed(reason) => repo::set_content_failed(tx, file_id, reason), } } @@ -769,7 +856,7 @@ pub fn extract_and_store( pub fn process_batch_updates( conn_mutex: &Arc>, files_to_update: &[OwnedNewFile], - stop_flag: &Arc>, + stop_flag: &Arc, config: &Config, ) -> Result<(), String> { if files_to_update.is_empty() { @@ -779,7 +866,7 @@ pub fn process_batch_updates( let fts_batch = config.processing.fts_update_batch_size.max(1); for batch in files_to_update.chunks(fts_batch) { - if *stop_flag.lock().unwrap() { + if stop_flag.load(Ordering::Relaxed) { return Ok(()); } @@ -789,22 +876,13 @@ pub fn process_batch_updates( .map_err(|e| format!("Failed to begin transaction: {}", e))?; for rec in batch.iter() { - if *stop_flag.lock().unwrap() { + if stop_flag.load(Ordering::Relaxed) { drop(tx); drop(conn); return Ok(()); } - let updated = repo::update_file_basic( - &tx, - &rec.path, - rec.size, - rec.mtime, - Some(rec.hash.as_slice()), - rec.mime.as_deref(), - rec.ftype, - ) - .map_err(|e| { + let updated = repo::update_file_basic(&tx, &rec.as_new_file()).map_err(|e| { format!( "Failed to update file record + clear stale content for {}: {}", rec.path, e @@ -867,7 +945,7 @@ pub(crate) fn store_inline_text( pub fn process_batch_inserts( conn_mutex: &Arc>, files_to_insert: &[OwnedNewFile], - stop_flag: &Arc>, + stop_flag: &Arc, config: &Config, ) -> Result<(), String> { if files_to_insert.is_empty() { @@ -875,7 +953,7 @@ pub fn process_batch_inserts( } for batch in files_to_insert.chunks(config.processing.batch_size) { - if *stop_flag.lock().unwrap() { + if stop_flag.load(Ordering::Relaxed) { return Ok(()); } @@ -885,7 +963,7 @@ pub fn process_batch_inserts( .map_err(|e| format!("Failed to begin transaction: {}", e))?; for rec in batch.iter() { - if *stop_flag.lock().unwrap() { + if stop_flag.load(Ordering::Relaxed) { drop(tx); drop(conn); return Ok(()); @@ -904,52 +982,76 @@ pub fn process_batch_inserts( Ok(()) } +/// Delete the rows a completed run found no file behind, in chunked +/// transactions. Returns how many went. +/// +/// The chunking is not just about transaction size: the connection is shared +/// with every search and status query, and `should_abort` *blocks* while the +/// indexer is suspended. Checking it inside the transaction — as this used to — +/// pinned the connection for the whole suspension and froze the GUI behind it. +/// So the stop flag, which never blocks, is what guards the inner loop, and +/// suspension is only ever observed between chunks with nothing held. pub fn cleanup_stale_index_entries( conn_mutex: &Arc>, stale_paths: &[String], - stop_flag: &Arc>, + stop_flag: &Arc, suspend_flag: &Arc, + config: &Config, ) -> Result { if stale_paths.is_empty() { return Ok(0); } - - let conn = conn_mutex.lock().unwrap(); - let tx = conn - .unchecked_transaction() - .map_err(|e| format!("Failed to begin stale cleanup transaction: {}", e))?; - + let chunk = config.processing.batch_size.max(1); let mut deleted_count = 0usize; - for path in stale_paths { + + for batch in stale_paths.chunks(chunk) { + // Outside the lock, so a suspend parks here rather than mid-transaction. if should_abort(stop_flag, suspend_flag) { - let _ = tx.commit(); - drop(conn); return Ok(deleted_count); } - - if repo::delete_file_by_path(&tx, path).map_err(|e| { - format!( - "Failed to remove stale index entry for {}: {}", - path, e - ) - })? { - deleted_count += 1; + let conn = conn_mutex.lock().unwrap(); + let tx = conn + .unchecked_transaction() + .map_err(|e| format!("Failed to begin stale cleanup transaction: {}", e))?; + for path in batch { + if stop_flag.load(Ordering::Relaxed) { + break; + } + if repo::delete_file_by_path(&tx, path) + .map_err(|e| format!("Failed to remove stale index entry for {}: {}", path, e))? + { + deleted_count += 1; + } + } + tx.commit() + .map_err(|e| format!("Failed to commit stale cleanup transaction: {}", e))?; + if stop_flag.load(Ordering::Relaxed) { + return Ok(deleted_count); } } - tx.commit() - .map_err(|e| format!("Failed to commit stale cleanup transaction: {}", e))?; - if deleted_count > 0 && !should_abort(stop_flag, suspend_flag) { - fts_finalize_after_text_indexing(&conn)?; + let conn = conn_mutex.lock().unwrap(); + fts_finalize_after_text_indexing(&conn); } Ok(deleted_count) } -/// Keyset cursor for per-root content extraction. `lo`/`hi` bound the -/// root's path range: `[root + "/", root + "0")` — `'0'` is `'/' + 1`, so -/// the pair is a pure index range on `UNIQUE(files.path)`. +/// Keyset cursor bounding everything stored beneath one directory. +/// +/// `lo`/`hi` are the half-open path range `[dir + SEP, dir + (SEP + 1))`, so +/// the pair is a pure index range on `UNIQUE(files.path)` — `SEARCH … (path>? +/// AND path ExtractCursor { - let base = root.trim_end_matches('/'); + const SEP: char = std::path::MAIN_SEPARATOR; + // Both separators are trimmed, not just the platform's: a config or a + // watcher event may spell a directory either way, and a trailing one + // would otherwise be doubled into the bounds. + let base = root.trim_end_matches(['/', '\\']); + let next = char::from_u32(SEP as u32 + 1).expect("separator successor is a valid char"); ExtractCursor { last_id: 0, - lo: format!("{}/", base), - hi: format!("{}0", base), + lo: format!("{}{}", base, SEP), + hi: format!("{}{}", base, next), } } } @@ -973,6 +1080,11 @@ impl ExtractCursor { /// and rows whose text is already searchable from earlier runs. Progress /// displays show their sum so an unchanged root reads as fully extracted /// rather than "extracted 0". +/// +/// Both halves count only files an extractor claims: rows nothing will extract +/// are written `NA` at walk time (see [`content_extractable`]) and so appear in +/// neither. That is what makes the sum a denominator for the *work*, rather +/// than for every file under the root. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ExtractScope { pub pending: usize, @@ -980,8 +1092,15 @@ pub struct ExtractScope { } /// Prepare a root's extraction scope: flip oversize pending rows to NA -/// (idempotent; also handles a `maximum_text_file_size` lowered between -/// runs) and count what is pending vs. already extracted in the range. +/// (idempotent) and count what is pending vs. already extracted in the range. +/// +/// [`prepare_file_record`] already applies `maximum_text_file_size` when it +/// writes a row, so the sweep finds nothing on a database this build filled. +/// It stays for the two cases that decision cannot cover: a +/// `maximum_text_file_size` *lowered* between runs (which, unlike +/// `content_extensions`, does not force a rebuild — see +/// [`crate::config::diff_actions`]), and rows left pending by an older build +/// that marked every file pending regardless. pub fn extract_scope_prepare( conn_mutex: &Arc>, cursor: &ExtractCursor, @@ -1017,87 +1136,47 @@ pub fn extract_scope_prepare( }) } -/// Run ONE bounded batch of content extraction within the cursor's range. -/// Returns rows processed; 0 means the range is drained (or the run is -/// stopping). `on_file` receives each file's name for progress display. -/// Designed to be pumped by the per-root writer loop, so one root's -/// extraction interleaves with other roots' walks and extractions. -pub fn extract_one_batch( +/// Write a batch of already-extracted rows. +/// +/// The cheap half of the content pass: [`crate::content`] does the reading and +/// the extracting on its own worker threads, and this is all that runs with the +/// connection held. Chunked so each transaction — and so each hold of the +/// connection lock — stays short, exactly as [`process_batch_inserts`] does. +/// +/// Returns how many rows were written. A row whose write fails is logged and +/// skipped rather than failing the run: its `content_state` stays pending, so +/// the next run retries it. +pub fn store_extracted( conn_mutex: &Arc>, - cursor: &mut ExtractCursor, - registry: &Registry, + rows: &[crate::content::ExtractedRow], + stop_flag: &Arc, config: &Config, - stop_flag: &Arc>, - suspend_flag: &Arc, - on_file: &mut dyn FnMut(&str), ) -> Result { - if should_abort(stop_flag, suspend_flag) { + if rows.is_empty() { return Ok(0); } - let max_size = i64::try_from(config.processing.maximum_text_file_size).unwrap_or(i64::MAX); - let batch_limit = config.processing.batch_size.max(1) as i64; - - let batch: Vec<(i64, String, String, Option)> = { + let mut written = 0usize; + for batch in rows.chunks(config.processing.batch_size.max(1)) { + if stop_flag.load(Ordering::Relaxed) { + return Ok(written); + } let conn = conn_mutex.lock().unwrap(); - let mut stmt = conn - .prepare( - "SELECT id, name, path, mime FROM files - WHERE content_state = 0 AND size <= ?1 AND id > ?2 - AND path >= ?3 AND path < ?4 - ORDER BY id - LIMIT ?5", - ) - .map_err(|e| format!("Failed to prepare text indexing query: {}", e))?; - let rows = stmt - .query_map( - rusqlite::params![max_size, cursor.last_id, cursor.lo, cursor.hi, batch_limit], - |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - row.get::<_, Option>(3)?, - )) - }, - ) - .map_err(|e| format!("Failed to query files for text indexing: {}", e))?; - rows.collect::, _>>() - .map_err(|e| format!("Failed to read file row: {}", e))? - }; - - if batch.is_empty() { - return Ok(0); - } - - let mut processed = 0usize; - let conn = conn_mutex.lock().unwrap(); - let tx = conn - .unchecked_transaction() - .map_err(|e| format!("Failed to begin transaction: {}", e))?; - for (file_id, fname, fpath, fmime) in batch.iter() { - if *stop_flag.lock().unwrap() { - break; + let tx = conn + .unchecked_transaction() + .map_err(|e| format!("Failed to begin transaction: {}", e))?; + for row in batch { + if let Err(e) = + store_content_outcome(&tx, row.file_id, &row.name, &row.outcome, config) + { + crate::log_warn!("content indexing for {}: {}", row.name, e); + continue; + } + written += 1; } - on_file(fname); - if let Err(e) = extract_and_store( - &tx, - *file_id, - fname, - fpath, - fmime.as_deref(), - registry, - config, - ) { - crate::log_warn!("content indexing for {}: {}", fpath, e); - } - // Advance only past what was actually processed, so a stop - // mid-batch never skips rows (they stay pending for the next run). - cursor.last_id = *file_id; - processed += 1; + tx.commit() + .map_err(|e| format!("Failed to commit transaction: {}", e))?; } - tx.commit() - .map_err(|e| format!("Failed to commit transaction: {}", e))?; - Ok(processed) + Ok(written) } #[cfg(test)] @@ -1310,6 +1389,54 @@ mod tests { ); } + /// The range must bracket paths spelled with the *platform's* separator. + /// + /// The bug this pins: `hi` was hard-coded as `'/' + 1` = `'0'`, so on + /// Windows every stored `C:\Users\me\…` path sorted above it and the range + /// was empty — content extraction and the vanished-directory sweep both + /// silently did nothing. + #[test] + fn extract_cursor_brackets_paths_under_its_root() { + use std::path::MAIN_SEPARATOR as SEP; + + let root = format!("{}Users{}me", root_prefix(), SEP); + let c = ExtractCursor::for_root(&root); + let inside = format!("{}{}docs{}a.txt", root, SEP, SEP); + assert!( + inside.as_str() >= c.lo.as_str() && inside.as_str() < c.hi.as_str(), + "{:?} must fall inside [{:?}, {:?})", + inside, + c.lo, + c.hi + ); + + // The directory itself is *not* in the range (the range is what lives + // beneath it), and a prefix sibling is outside it. + assert!(root.as_str() < c.lo.as_str()); + let sibling = format!("{}Users{}mexico{}a.txt", root_prefix(), SEP, SEP); + assert!( + !(sibling.as_str() >= c.lo.as_str() && sibling.as_str() < c.hi.as_str()), + "{:?} is a sibling of {:?}, not a child", + sibling, + root + ); + + // A trailing separator of either flavour must not double up. + for spelled in [format!("{}/", root), format!("{}\\", root)] { + let c2 = ExtractCursor::for_root(&spelled); + assert_eq!((&c2.lo, &c2.hi), (&c.lo, &c.hi), "spelled {:?}", spelled); + } + } + + /// An absolute-path prefix for the running platform. + fn root_prefix() -> String { + if cfg!(windows) { + r"C:\".to_string() + } else { + "/".to_string() + } + } + /// A Remove event names a path that is already gone, so the key for it has /// to be built from the deepest ancestor that still resolves. #[test] @@ -1437,9 +1564,6 @@ mod tests { #[cfg(test)] mod count_and_extract_tests { use super::*; - use crate::db::open_or_recreate; - use crate::db::repo::{insert_file, NewFile}; - use crate::mime::FileType; use std::sync::atomic::{AtomicBool, Ordering}; fn tmp(tag: &str) -> std::path::PathBuf { @@ -1492,102 +1616,161 @@ mod count_and_extract_tests { assert!(cancel.load(Ordering::Relaxed)); } + /// The invariant the whole "decide at walk time" design rests on. If these + /// two ever disagree, a file the walk wrote off as NA would silently never + /// be full-text indexed — a far worse bug than the inflated denominator + /// this replaced. Fails the moment someone edits one predicate and not the + /// other. #[test] - fn extract_one_batch_is_scoped_to_its_root_range() { - let tree = tmp("extract-tree"); - std::fs::create_dir_all(tree.join("r1")).unwrap(); - std::fs::create_dir_all(tree.join("r2")).unwrap(); - let f1 = tree.join("r1/inside.txt"); - let f2 = tree.join("r2/outside.txt"); - std::fs::write(&f1, "sphinx of black quartz").unwrap(); - std::fs::write(&f2, "judge my vow").unwrap(); - - let db = tmp("extract-db"); - let mut conn = open_or_recreate(db.to_str().unwrap(), "trigram").unwrap(); - { - let tx = conn.transaction().unwrap(); - for f in [&f1, &f2] { - insert_file( - &tx, - &NewFile { - name: f.file_name().unwrap().to_str().unwrap(), - path: f.to_str().unwrap(), - parent: f.parent().unwrap().to_str().unwrap(), - size: std::fs::metadata(f).unwrap().len(), - mtime: 1, - inode: None, - device_id: None, - mime: Some("text/plain"), - ftype: FileType::TEXT, - hash: None, - }, - ) - .unwrap() - .expect("unique"); - } - tx.commit().unwrap(); - } - let conn_mutex = Arc::new(Mutex::new(conn)); - + fn content_extractable_is_decide_contents_not_applicable() { + let root = tmp("extractable"); + std::fs::create_dir_all(&root).unwrap(); + let mut cfg = Config::default(); let registry = Registry::default_set(); - let config = Config::default(); - let stop = Arc::new(Mutex::new(false)); - let suspend = Arc::new(AtomicBool::new(false)); - let mut cursor = ExtractCursor::for_root(tree.join("r1").to_str().unwrap()); - let scope = extract_scope_prepare(&conn_mutex, &cursor, &config).unwrap(); - assert_eq!(scope.pending, 1, "only r1's file is in range"); - assert_eq!(scope.already_done, 0, "nothing extracted yet"); + // Real files, because `decide_content` runs the extractor for anything + // it claims and its answer must be a genuine one. + let cases = [ + "notes.txt", + "data.json", + "schema.sql", + "song.mp3", + "photo.jpg", + "movie.mp4", + "archive.zip", + "blob.bin", + "noextension", + ]; + for name in cases { + let p = root.join(name); + std::fs::write(&p, b"plain bytes with no magic").unwrap(); + } - let mut seen_names = Vec::new(); - loop { - let n = extract_one_batch( - &conn_mutex, - &mut cursor, - ®istry, - &config, - &stop, - &suspend, - &mut |name| seen_names.push(name.to_string()), - ) - .unwrap(); - if n == 0 { - break; + // Once with the filter off (the default: everything the registry + // claims), once with it narrowed to `.txt`. + for filter in [Vec::new(), vec!["txt".to_string()]] { + cfg.indexing.content_extensions = filter.clone(); + for name in cases { + let p = root.join(name); + let path = p.to_str().unwrap(); + let mime = guess_mime_from_head(&p, b"plain bytes with no magic"); + let claimed = content_extractable(&p, mime.as_deref(), &cfg, ®istry); + let outcome = decide_content(path, mime.as_deref(), ®istry, &cfg); + assert_eq!( + claimed, + outcome != ContentOutcome::NotApplicable, + "{} (mime {:?}, filter {:?}): predicate says {}, decide_content says {:?}", + name, + mime, + filter, + claimed, + outcome + ); } } - assert_eq!(seen_names, vec!["inside.txt".to_string()]); - let conn = conn_mutex.lock().unwrap(); - let state = |path: &std::path::Path| -> i64 { - conn.query_row( - "SELECT content_state FROM files WHERE path = ?1", - rusqlite::params![path.to_str().unwrap()], - |r| r.get(0), - ) - .unwrap() + std::fs::remove_dir_all(&root).ok(); + } + + #[test] + fn prepare_file_record_marks_only_claimable_files() { + let root = tmp("claimable"); + std::fs::create_dir_all(&root).unwrap(); + let registry = Registry::default_set(); + let needs = |cfg: &Config, name: &str| -> bool { + let p = root.join(name); + let meta = std::fs::metadata(&p).unwrap(); + prepare_file_record(p.to_str().unwrap(), &meta, cfg, ®istry) + .expect("regular file") + .needs_content }; - assert_eq!(state(&f1), repo::STATE_DONE, "in-range row extracted"); - assert_eq!(state(&f2), repo::STATE_PENDING, "out-of-range row untouched"); - drop(conn); - // A second run over the unchanged root must report the file as - // already extracted, so progress reads "1 of 1", never "0 of 0". - let cursor2 = ExtractCursor::for_root(tree.join("r1").to_str().unwrap()); - let scope2 = extract_scope_prepare(&conn_mutex, &cursor2, &config).unwrap(); - assert_eq!(scope2.pending, 0); - assert_eq!(scope2.already_done, 1); - let conn = conn_mutex.lock().unwrap(); - let hits: i64 = conn - .query_row( - "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH '\"sphinx\"'", - [], - |r| r.get(0), - ) + for name in ["notes.txt", "song.mp3", "movie.mp4", "blob.bin"] { + std::fs::write(root.join(name), b"body").unwrap(); + } + let big = root.join("huge.txt"); + std::fs::write(&big, vec![b'x'; 4096]).unwrap(); + + let cfg = Config::default(); + assert!(needs(&cfg, "notes.txt"), "plaintext is claimed"); + assert!(needs(&cfg, "song.mp3"), "audio tags are content too"); + assert!(!needs(&cfg, "movie.mp4"), "no extractor claims video"); + assert!(!needs(&cfg, "blob.bin"), "unsniffable: no MIME, no extractor"); + + // Over `maximum_text_file_size`, so the content pass would never read + // it even though plaintext claims the MIME. + let mut small_cap = Config::default(); + small_cap.processing.maximum_text_file_size = 1024; + assert!(!needs(&small_cap, "huge.txt")); + assert!(needs(&cfg, "huge.txt"), "and claimed under the default cap"); + + // The `content_extensions` allowlist excludes it. + let mut only_md = Config::default(); + only_md.indexing.content_extensions = vec!["md".to_string()]; + assert!(!needs(&only_md, "notes.txt")); + + std::fs::remove_dir_all(&root).ok(); + } + + /// The headline regression test: the number the manage-index tab shows as + /// the extraction denominator, measured where `indexing.rs` measures it — + /// after the walk's inserts, before any content pass runs. + #[test] + fn extract_scope_counts_only_files_an_extractor_claims() { + let root = tmp("denominator"); + std::fs::create_dir_all(&root).unwrap(); + let mut db = root.clone(); + db.set_extension("sqlite"); + + // 3 files an extractor claims, 5 it never will. Big enough that the + // old behaviour (every row pending) can't coincide with the new one. + let claimed = ["a.txt", "b.json", "c.mp3"]; + let unclaimed = ["d.mp4", "e.zip", "f.bin", "g.exe", "h"]; + for name in claimed.iter().chain(unclaimed.iter()) { + std::fs::write(root.join(name), b"body bytes, no magic").unwrap(); + } + + let config = Config::default(); + let registry = Registry::default_set(); + let records: Vec = claimed + .iter() + .chain(unclaimed.iter()) + .map(|name| { + let p = root.join(name); + let meta = std::fs::metadata(&p).unwrap(); + prepare_file_record(p.to_str().unwrap(), &meta, &config, ®istry) + .expect("regular file") + }) + .collect(); + + let conn_mutex = Arc::new(Mutex::new( + crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap(), + )); + let stop = Arc::new(AtomicBool::new(false)); + process_batch_inserts(&conn_mutex, &records, &stop, &config).unwrap(); + + let cursor = ExtractCursor::for_root(root.to_str().unwrap()); + let scope = extract_scope_prepare(&conn_mutex, &cursor, &config).unwrap(); + + // `extract_total` in the GUI. The two small text-ish files were + // finished inline by the walk so they land in `already_done`; the mp3 + // needs the disk pass. Either way the denominator is the claimed set — + // before this was decided at walk time it read 8, every indexed file. + assert_eq!( + (scope.pending, scope.already_done), + (1, 2), + "denominator must be the files needing text, not every indexed file" + ); + assert_eq!(scope.pending + scope.already_done, claimed.len()); + + let total: i64 = conn_mutex + .lock() + .unwrap() + .query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) .unwrap(); - assert_eq!(hits, 1); + assert_eq!(total as usize, claimed.len() + unclaimed.len()); - drop(conn); - std::fs::remove_dir_all(&tree).ok(); + std::fs::remove_dir_all(&root).ok(); std::fs::remove_file(&db).ok(); } } diff --git a/crates/quicksearch-core/src/incremental.rs b/crates/quicksearch-core/src/incremental.rs index 0d4f618..a7e84ca 100644 --- a/crates/quicksearch-core/src/incremental.rs +++ b/crates/quicksearch-core/src/incremental.rs @@ -17,15 +17,14 @@ use std::path::Path; use rusqlite::{Connection, OptionalExtension}; -use crate::config::{content_allowed, Config, IgnoreSet}; +use crate::config::{Config, IgnoreSet}; use crate::db::repo; use crate::extract::Registry; use crate::file_handling::{ - db_key_for_missing_path, extract_and_store, filtered_walk, UnreadableDirs, + db_key_for_missing_path, extract_and_store, filtered_walk, ExtractCursor, UnreadableDirs, prepare_file_record_from_path, store_inline_text, }; use crate::platform::path_has_hidden_component_under; -use crate::query::translator::like_subtree_pattern; use crate::watcher::FsEvent; /// Apply one filesystem event to the index. Missing files are treated as @@ -120,15 +119,7 @@ fn upsert_file( let file_id = match existing { Some((_, mtime)) if mtime.max(0) as u64 == rec.mtime => return Ok(()), Some((id, _)) => { - repo::update_file_basic( - &tx, - &rec.path, - rec.size, - rec.mtime, - Some(&rec.hash), - rec.mime.as_deref(), - rec.ftype, - )?; + repo::update_file_basic(&tx, &rec.as_new_file())?; id } None => match repo::insert_file(&tx, &rec.as_new_file())? { @@ -139,9 +130,13 @@ fn upsert_file( }, }; - if rec.size > config.processing.maximum_text_file_size - || !content_allowed(Path::new(&rec.path), config) - { + // The insert/update above already wrote NA; re-asserting it costs one + // indexed UPDATE on a path that handles one file per event, and keeps this + // correct on its own terms rather than on `insert_file`'s. It is also the + // only size gate on this path — `decide_content` has none, so falling + // through would hand a multi-gigabyte `.txt` to the plaintext extractor, + // which reads the whole file into memory. + if !rec.needs_content { repo::set_content_na(&tx, file_id)?; } else if let Some(text) = rec.inline_text.as_deref() { // Small enough that `prepare_file_record_from_path` already read the @@ -163,34 +158,68 @@ fn upsert_file( } fn remove_path(conn: &mut Connection, path: &Path) -> Result<(), String> { - // The insert side stores a canonicalized path, so the raw event spelling - // is not a usable key — but the file is already gone, so `canonicalize` - // cannot be called on it directly either. - let path_str = db_key_for_missing_path(path); - let tx = conn - .transaction() - .map_err(|e| format!("begin incremental tx: {}", e))?; + remove_paths(conn, std::slice::from_ref(&path.to_path_buf()), 1) +} - repo::delete_file_by_path(&tx, &path_str)?; - - // Directory removals surface as one event for the directory itself — - // sweep everything indexed beneath it. - let subtree: Vec = { - let mut stmt = tx - .prepare("SELECT path FROM files WHERE path LIKE ?1 ESCAPE '\\'") - .map_err(|e| e.to_string())?; - let rows = stmt - .query_map(rusqlite::params![like_subtree_pattern(&path_str)], |r| { - r.get::<_, String>(0) - }) - .map_err(|e| e.to_string())?; - rows.collect::, _>>().map_err(|e| e.to_string())? - }; - for p in &subtree { - repo::delete_file_by_path(&tx, p)?; +/// Drop removals that a removal of one of their ancestors already covers. +/// +/// `rm -rf dir/` reports `dir` *and* every file beneath it. Since removing +/// `dir` sweeps its whole path range, each descendant event is duplicate work — +/// 10 000 of them for a 10 000-file tree. +/// +/// Ancestor membership is tested against a set rather than by sorting, which +/// keeps this obviously correct: no ordering argument to get wrong, and +/// `Path::ancestors` walks whole components, so `/a/bc` is never treated as +/// living under `/a/b` — the same rule `remove_tree` and +/// `UnreadableDirs::covers` use. Paths are shallow, so the cost is linear. +pub fn collapse_removal_roots(paths: Vec) -> Vec { + if paths.len() < 2 { + return paths; } + let all: std::collections::HashSet<&Path> = paths.iter().map(|p| p.as_path()).collect(); + paths + .iter() + .filter(|p| !p.ancestors().skip(1).any(|a| all.contains(a))) + .cloned() + .collect() +} - tx.commit().map_err(|e| format!("commit incremental tx: {}", e)) +/// Delete `paths` and everything indexed beneath them, in transactions of at +/// most `chunk` paths. +/// +/// A mass deletion is the case this exists for. Callers hand over the *roots* +/// of the removal set (see [`collapse_removal_roots`]), so `rm -rf dir/` is one +/// path here rather than one per file, and each one costs a fixed handful of +/// range-driven statements ([`repo::delete_subtree`]) instead of a full table +/// scan plus five statements per file. +/// +/// Chunking bounds how long any single transaction holds the connection, the +/// same way `process_batch_inserts` bounds the indexer's writes. +pub fn remove_paths( + conn: &mut Connection, + paths: &[std::path::PathBuf], + chunk: usize, +) -> Result<(), String> { + for batch in paths.chunks(chunk.max(1)) { + let tx = conn + .transaction() + .map_err(|e| format!("begin incremental tx: {}", e))?; + for path in batch { + // The insert side stores a canonicalized path, so the raw event + // spelling is not a usable key — but the file is already gone, so + // `canonicalize` cannot be called on it directly either. + let path_str = db_key_for_missing_path(path); + // The path itself, whether it was a file or a directory... + repo::delete_file_by_path(&tx, &path_str)?; + // ...then everything beneath it, for a directory removal, which + // surfaces as a single event for the directory. + let range = ExtractCursor::for_root(&path_str); + repo::delete_subtree(&tx, &range.lo, &range.hi)?; + } + tx.commit() + .map_err(|e| format!("commit incremental tx: {}", e))?; + } + Ok(()) } #[cfg(test)] @@ -297,6 +326,81 @@ mod tests { } } + fn collapse(paths: &[&str]) -> Vec { + let mut out: Vec = + collapse_removal_roots(paths.iter().map(std::path::PathBuf::from).collect()) + .iter() + .map(|p| p.to_string_lossy().into_owned()) + .collect(); + out.sort(); + out + } + + #[test] + fn removal_roots_collapse_to_the_shallowest_ancestor() { + // `rm -rf dir/` reports the directory and every file beneath it; + // removing the directory already sweeps its whole range. + assert_eq!( + collapse(&["/dir", "/dir/a.txt", "/dir/b/c.txt", "/dir/b"]), + vec!["/dir"] + ); + + // Component-wise, so a name-prefix sibling is not swallowed. + assert_eq!( + collapse(&["/a/b", "/a/bc"]), + vec!["/a/b", "/a/bc"], + "/a/bc does not live under /a/b" + ); + assert_eq!( + collapse(&["/a/b", "/a/b.txt"]), + vec!["/a/b", "/a/b.txt"], + "a sibling file sorting between a dir and its children survives" + ); + + // Unrelated removals all survive; order of input does not matter. + assert_eq!( + collapse(&["/x/deep/f", "/y", "/x"]), + vec!["/x", "/y"], + "/x/deep/f is covered by /x, /y is independent" + ); + + // Degenerate inputs. + assert!(collapse(&[]).is_empty()); + assert_eq!(collapse(&["/only"]), vec!["/only"]); + } + + /// The collapse must not change what ends up deleted — only how much work + /// it takes to get there. + #[test] + fn collapsed_removal_deletes_the_same_rows_as_the_full_set() { + let mut f = Fixture::new(); + f.write("tree/a.txt", "alpha"); + f.write("tree/deep/b.txt", "beta"); + f.write("tree2/keep.txt", "survivor"); + let tree = f.dir.join("tree"); + f.apply(&FsEvent::Create(tree.clone())); + f.apply(&FsEvent::Create(f.dir.join("tree2"))); + assert_eq!(f.counts().0, 3); + + let canonical_tree = f.canonical(&tree); + std::fs::remove_dir_all(&tree).unwrap(); + + // What a real `rm -rf` produces: the directory plus every path under it. + let reported: Vec = vec![ + canonical_tree.clone().into(), + format!("{}/deep", canonical_tree).into(), + format!("{}/a.txt", canonical_tree).into(), + format!("{}/deep/b.txt", canonical_tree).into(), + ]; + let roots = collapse_removal_roots(reported); + assert_eq!(roots.len(), 1, "one range covers the whole tree"); + + remove_paths(&mut f.conn, &roots, 200).unwrap(); + assert_eq!(f.counts(), (1, 1, 1), "only tree2 survives"); + let survivor = f.canonical(&f.dir.join("tree2").join("keep.txt")); + assert!(f.row(&survivor).is_some()); + } + #[test] fn create_indexes_file_and_content() { let mut f = Fixture::new(); @@ -414,6 +518,27 @@ mod tests { assert_eq!(f.counts(), (1, 0, 0)); } + /// The third way a file can end up NA — no extractor claims its MIME — + /// which the filter and oversize tests either side of this one don't + /// cover. A row left pending here would be re-fed to the content pass on + /// every subsequent run and counted in the extraction denominator forever. + #[test] + fn an_unclaimed_mime_is_na_in_the_watcher_path() { + let mut f = Fixture::new(); + // `.mp4` sniffs as video/mp4 by extension; nothing extracts video. + let vid = f.write("clip.mp4", "not really an mp4, and it needn't be"); + f.apply(&FsEvent::Create(vid.clone())); + + let canonical = f.canonical(&vid); + let (_, _, content_state) = f.row(&canonical).expect("row listed"); + assert_eq!( + content_state, + repo::STATE_NA, + "filename indexed, nothing to extract" + ); + assert_eq!(f.counts(), (1, 0, 0)); + } + /// A Remove event whose path is spelled differently from the stored key /// must still delete the row. `dir/./f.txt` and `dir/f.txt` are the same /// file; only the canonicalized spelling is in the index. diff --git a/crates/quicksearch-core/src/indexing.rs b/crates/quicksearch-core/src/indexing.rs index ae81e66..3683f69 100644 --- a/crates/quicksearch-core/src/indexing.rs +++ b/crates/quicksearch-core/src/indexing.rs @@ -2,15 +2,15 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, mpsc}; use std::thread; use std::time::{Duration, Instant}; -use std::collections::HashSet; -use rusqlite::{params, Connection, OptionalExtension}; +use std::collections::{HashMap, HashSet}; +use rusqlite::{params, Connection, InterruptHandle, OptionalExtension}; use crate::extract::Registry; use crate::file_handling::{ cleanup_stale_index_entries, count_tree_entries_fast, - extract_one_batch, extract_scope_prepare, + store_extracted, fts_finalize_after_text_indexing, process_batch_inserts, process_batch_updates, @@ -20,9 +20,7 @@ use crate::file_handling::{ OwnedNewFile, }; use crate::config::Config; -use crate::walk::{ - thread_count_for, walk_indexable_files, ParallelWalk, TryNext, WalkEvent, WorkerStats, -}; +use crate::walk::{thread_count_for, walk_indexable_files, ParallelWalk, TryNext, WalkEvent}; use crate::db; use crate::db::repo; @@ -43,21 +41,94 @@ pub enum RootPhase { pub struct RootProgress { pub root: String, pub phase: RootPhase, - /// Files the walk has seen so far. + /// Files the walk has seen so far. Final and exact once the root leaves + /// [`RootPhase::Walking`] — see [`RootProgress::walk_denominator`]. pub walked: usize, - /// Concurrent `find`-based denominator; `None` until the count lands. + /// Concurrent `find`-based *estimate*; `None` until the count lands. + /// Counts tree entries, not walkable files, so it reads high — read it + /// through [`RootProgress::walk_denominator`] rather than directly. pub walk_total: Option, /// Rows with searchable text: extracted in earlier runs plus this one. pub extracted: usize, - /// The root's whole searchable set: pending + already-extracted rows - /// at the moment the walk finished. + /// The root's whole searchable set: pending + already-extracted rows at + /// the moment the walk finished. Files no extractor claims are already + /// `NA` by then, so this is the count of files that have or will have + /// text — not the count of files under the root. pub extract_total: usize, pub current_file: Option, - /// Walker threads busy right now / pool size. + /// Threads busy right now / pool size, for whichever pool this root's + /// current phase is running — the walk's while walking, the content + /// pass's while extracting. Both zero once the root is done: its threads + /// are gone, and reporting a dead pool's size is how the status line came + /// to read "0/44 workers". pub active_workers: usize, pub total_workers: usize, } +impl RootProgress { + /// This root's walk-phase contribution to a progress denominator. + /// + /// While the walk runs, the only figure available is the concurrent + /// `find` count, which counts tree *entries* — directories, hidden + /// entries and ignore-pruned subtrees included — where `walked` counts + /// only the files the walk emits. On a home directory that estimate runs + /// over 1.6x high. Once the walk ends `walked` is final and exact, so the + /// estimate is dropped: keeping it is what stopped the bar reaching 100%. + pub fn walk_denominator(&self) -> Option { + match self.phase { + // Never below what has already been walked: an estimate the walk + // has overtaken is provably wrong, and a bar pinned at 100% + // mid-walk reads as a hang. + RootPhase::Walking => self.walk_total.map(|t| t.max(self.walked)), + RootPhase::Extracting | RootPhase::Done => Some(self.walked), + } + } +} + +/// Files processed and the run's total, across every root. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OverallProgress { + /// Both halves of the work: every file the walks have seen, plus every + /// row with searchable text. A file is counted once for each, so this is + /// a work-units figure rather than a file count. + pub processed: usize, + /// `None` while a still-walking root has no count yet; no root past its + /// walk can withhold one, so a run always gains a total in the end. + pub total: Option, +} + +impl OverallProgress { + /// Completed share, clamped to 1. `None` when there is nothing to + /// divide by — an unknown total, or a run with no work in it at all. + pub fn fraction(&self) -> Option { + match self.total { + Some(total) if total > 0 => Some((self.processed as f64 / total as f64).min(1.0)), + _ => None, + } + } +} + +/// Aggregate every root's progress into the one pair the status bar shows. +/// +/// The extraction half needs no estimate: `extract_total` is queried exactly, +/// from the rows themselves, the moment a root's walk ends. Before that it is +/// zero and the root contributes only its walk — which is also the only part +/// of it the GUI reports. +pub fn overall_progress(roots: &[RootProgress]) -> OverallProgress { + let processed = roots.iter().map(|r| r.walked + r.extracted).sum(); + let mut total = Some(0usize); + for r in roots { + match (total, r.walk_denominator()) { + (Some(acc), Some(walk)) => total = Some(acc + walk + r.extract_total), + _ => { + total = None; + break; + } + } + } + OverallProgress { processed, total } +} + #[derive(Debug, Clone)] pub enum IndexingStatus { Idle, @@ -66,6 +137,12 @@ pub enum IndexingStatus { roots: Vec, }, Stopping, + /// Compacting and re-analysing the index after a run — see + /// [`IndexingService::run_maintenance`]. Distinct from `Running` because + /// it holds the database with no per-file progress to show, and distinct + /// from `Idle` because the single-writer rule still applies: the + /// coordinator must stay off the file until this clears. + Optimizing, Error(String), } @@ -141,28 +218,85 @@ fn sweep_unvisited_parents( Ok(()) } -#[derive(Debug)] +/// One placeholder progress row per root, so the GUI has structure to draw +/// between the click and the writer loop's first real numbers. +fn starting_roots(paths: &[String]) -> Vec { + paths + .iter() + .map(|p| RootProgress { + root: p.clone(), + phase: RootPhase::Walking, + walked: 0, + walk_total: None, + extracted: 0, + extract_total: 0, + current_file: Some("Starting…".to_string()), + active_workers: 0, + total_workers: 0, + }) + .collect() +} + +// No `Debug`: rusqlite's `InterruptHandle` has none, and nothing formats the +// service anyway. pub struct IndexingService { status: Arc>, command_tx: mpsc::Sender, db_connection: Arc>>>>, suspend_flag: Arc, + /// Set while [`IndexingStatus::Optimizing`] holds, so the one caller that + /// cannot wait out a VACUUM can cut it short. See + /// [`IndexingService::cancel_optimizing`]. + maintenance: Arc>>, _handle: thread::JoinHandle<()>, } /// Polling interval for `should_abort` while suspended. const SUSPEND_POLL_MS: u64 = 100; +/// `indexing.root_workers` rekeyed from the spellings the user typed to the +/// canonical roots the indexer walks, so an override survives a `~`, a +/// trailing slash, a relative path or a symlinked root. Entries naming a +/// folder that is no longer indexed are dropped. +fn resolved_root_workers(config: &Config) -> HashMap { + config + .paths + .indexing_paths + .iter() + .zip(config.resolved_indexing_paths()) + .filter_map(|(raw, resolved)| { + let workers = config.indexing.root_workers.get(raw).copied()?; + Some((normalize_root_string(&resolved.to_string_lossy()), workers)) + }) + .collect() +} + +/// Size of the write-ahead log on disk, or 0 if it is absent. +/// +/// A run watches this because SQLite will not bound it. The autocheckpoint +/// copies committed frames into the index continuously, but the log only +/// *shrinks* when the writer opens a transaction at an instant no reader holds +/// a read mark — a lock SQLite tries once, without retrying. A run keeps a +/// reader per root querying from start to finish (one read per directory while +/// walking, one per page of rows while extracting), so that instant does not +/// come, and the log appends for the length of the run: on a large tree it +/// ends up larger than the index it journals. An explicit checkpoint retries +/// the same lock under `busy_timeout` and wins, which is why `run_indexing` +/// forces one every `maximum_wal_size` bytes. +fn wal_len(path: &str) -> u64 { + std::fs::metadata(path).map(|m| m.len()).unwrap_or(0) +} + /// Combined stop/suspend check used by worker loops. Returns `true` iff the /// caller should abort the operation. While the suspend flag is set and stop /// is not, this parks the thread by sleeping in short increments so a later /// `resume()` unblocks it. Cheap to call in tight loops. pub(crate) fn should_abort( - stop: &Arc>, + stop: &Arc, suspend: &Arc, ) -> bool { loop { - if *stop.lock().unwrap() { + if stop.load(Ordering::Relaxed) { return true; } if !suspend.load(Ordering::Relaxed) { @@ -172,34 +306,6 @@ pub(crate) fn should_abort( } } -/// Set process priority for background operation -// fn set_background_priority() { -// #[cfg(windows)] -// { -// use std::os::windows::raw::HANDLE; - -// // Windows implementation -// extern "system" { -// fn GetCurrentProcess() -> HANDLE; -// fn SetPriorityClass(hprocess: HANDLE, dwpriorityclass: u32) -> i32; -// } - -// const BELOW_NORMAL_PRIORITY_CLASS: u32 = 0x00004000; -// unsafe { -// SetPriorityClass(GetCurrentProcess(), BELOW_NORMAL_PRIORITY_CLASS); -// } -// } - -// #[cfg(unix)] -// { -// // Unix implementation -// use std::os::unix::process::CommandExt; -// unsafe { -// libc::nice(10); // Lower priority -// } -// } -// } - /// Flips an [`AtomicBool`] when dropped. Held by `run_indexing` so the /// per-root count subprocesses die on every exit path of a run. struct CancelOnDrop(Arc); @@ -214,32 +320,60 @@ impl Drop for CancelOnDrop { struct RootPipeline { root: String, walk: ParallelWalk, - stats: WorkerStats, /// Concurrent `find` count; 0 = not yet known. count_total: Arc, + /// Threads this root gets, for the walk and then for extraction: both are + /// round-trip bound on a share for the same reason, so both use the value + /// `thread_count_for` / `root_workers` produced. + workers: usize, pending_updates: Vec, pending_inserts: Vec, walked: usize, walk_clean: bool, phase: RootPhase, - extract: Option, + /// The running content pass, once this root's walk has finished. + content: Option, extract_total: usize, extracted: usize, current_file: Option, } +impl RootPipeline { + /// Busy threads / pool size for the pool this root is currently running. + /// + /// A root outlives its walk — the `ParallelWalk` stays in the struct after + /// its workers exit — so the pool has to be chosen by phase, not read from + /// whichever handle happens to be at hand. + fn worker_counts(&self) -> (usize, usize) { + let stats = match self.phase { + RootPhase::Walking => Some(self.walk.worker_stats()), + RootPhase::Extracting => self.content.as_ref().map(|p| p.worker_stats()), + RootPhase::Done => None, + }; + stats.map_or((0, 0), |s| (s.active(), s.total())) + } +} + impl IndexingService { pub fn new() -> Self { let status = Arc::new(Mutex::new(IndexingStatus::Idle)); let (command_tx, command_rx) = mpsc::channel(); let db_connection = Arc::new(Mutex::new(None)); let suspend_flag = Arc::new(AtomicBool::new(false)); + let maintenance = Arc::new(Mutex::new(None)); let status_clone = status.clone(); let db_connection_clone = db_connection.clone(); let suspend_clone = suspend_flag.clone(); + let maintenance_clone = maintenance.clone(); let handle = thread::spawn(move || { - Self::indexing_thread(status_clone, command_rx, db_connection_clone, suspend_clone); + Self::indexing_thread( + status_clone, + command_rx, + db_connection_clone, + suspend_clone, + maintenance_clone, + ); }); IndexingService { @@ -247,10 +381,25 @@ impl IndexingService { command_tx, db_connection, suspend_flag, + maintenance, _handle: handle, } } + /// Cut short an [`IndexingStatus::Optimizing`] pass. No-op otherwise. + /// + /// Stop deliberately does *not* do this — optimizing is what runs after a + /// run stops. This exists for the one caller that cannot wait: deleting + /// the index for a rebuild, where a VACUUM still holding the file would + /// fail the delete outright on Windows. An interrupted VACUUM rolls back. + pub fn cancel_optimizing(&self) { + if let Ok(slot) = self.maintenance.lock() { + if let Some(handle) = slot.as_ref() { + handle.interrupt(); + } + } + } + /// Pause the indexer. All worker loops that call [`should_abort`] will /// block until [`resume`](Self::resume) is called. No-op if already /// suspended. Does not stop the worker — stop_indexing is still the way @@ -271,7 +420,17 @@ impl IndexingService { /// Start indexing one or more roots. Paths are walked in order; duplicate /// or nested roots are de-duplicated by the indexer. At least one path is - /// required. + /// required. Returns `Err` if a run is already in flight. + /// + /// The `Idle → Running` transition happens **here**, synchronously, not on + /// the service's command thread. Callers use [`get_status`](Self::get_status) + /// to enforce the single-writer rule, and the command thread cannot flip the + /// status until it has finished joining the *previous* run's thread — which + /// can take arbitrarily long. A caller that polled for the flip would give + /// up and start writing to a database this run is about to reopen (and + /// possibly wipe). Claiming the status under one lock before sending closes + /// that window entirely, and makes "already running" a reportable error + /// rather than a silently dropped command. pub fn start_indexing( &self, paths: Vec, @@ -281,9 +440,27 @@ impl IndexingService { if paths.is_empty() { return Err("start_indexing requires at least one path".into()); } + { + let mut status = self.status.lock().unwrap(); + if matches!( + *status, + IndexingStatus::Running { .. } | IndexingStatus::Stopping + ) { + return Err("indexing is already running".into()); + } + *status = IndexingStatus::Running { + start_time: Instant::now(), + roots: starting_roots(&paths), + }; + } self.command_tx .send(IndexingCommand::Start { paths, db_path, config }) - .map_err(|e| format!("Failed to send start command: {}", e)) + .map_err(|e| { + // The service is gone; leave the status honest rather than + // stuck on a run that will never happen. + *self.status.lock().unwrap() = IndexingStatus::Idle; + format!("Failed to send start command: {}", e) + }) } /// Signal a running index pass to stop without waiting for it. Used @@ -318,7 +495,9 @@ impl IndexingService { if let Ok(mut db_opt) = self.db_connection.lock() { if let Some(db_conn_arc) = db_opt.take() { if let Ok(conn) = db_conn_arc.lock() { - let _ = conn.execute("PRAGMA wal_checkpoint(TRUNCATE);", ()); + if let Err(e) = crate::db::repo::checkpoint_truncate(&conn) { + crate::log_warn!("{}", e); + } } } } @@ -353,13 +532,22 @@ impl IndexingService { // Stop any running indexing first self.stop_indexing() .map_err(|e| format!("Failed to stop indexing: {}", e))?; + // And cut short the optimize pass that follows it — this is the one + // caller that cannot wait it out, since the file it is about to delete + // is the file that pass has open. + self.cancel_optimizing(); // Wait for indexing to actually stop let mut attempts = 0; while attempts < 50 { // Wait up to 5 seconds match self.get_status() { IndexingStatus::Idle => break, - IndexingStatus::Stopping | IndexingStatus::Running { .. } => { + // Optimizing holds the file about to be deleted, so it is + // waited on like a run; `cancel_optimizing` above is what + // keeps that wait short. + IndexingStatus::Stopping + | IndexingStatus::Running { .. } + | IndexingStatus::Optimizing => { std::thread::sleep(std::time::Duration::from_millis(100)); attempts += 1; } @@ -384,42 +572,25 @@ impl IndexingService { command_rx: mpsc::Receiver, db_connection: Arc>>>>, suspend_flag: Arc, + maintenance: Arc>>, ) { - let stop_flag = Arc::new(Mutex::new(false)); + let stop_flag = Arc::new(AtomicBool::new(false)); let mut indexing_handle: Option> = None; while let Ok(command) = command_rx.recv() { match command { IndexingCommand::Start { paths, db_path, config } => { - if matches!(*status.lock().unwrap(), IndexingStatus::Running { .. }) { - continue; // Already running - } + // `start_indexing` already claimed the status and rejected + // a concurrent start, so there is nothing to re-check here. - // Join any previous indexing thread + // Join any previous indexing thread. This can block for as + // long as that run takes to wind down, which is exactly why + // the status flip does not live here. if let Some(handle) = indexing_handle.take() { let _ = handle.join(); } - *stop_flag.lock().unwrap() = false; - // One placeholder row per root so the GUI has structure - // to draw before the writer loop publishes real numbers. - *status.lock().unwrap() = IndexingStatus::Running { - start_time: Instant::now(), - roots: paths - .iter() - .map(|p| RootProgress { - root: p.clone(), - phase: RootPhase::Walking, - walked: 0, - walk_total: None, - extracted: 0, - extract_total: 0, - current_file: Some("Starting…".to_string()), - active_workers: 0, - total_workers: 0, - }) - .collect(), - }; + stop_flag.store(false, Ordering::Relaxed); // Run indexing in a separate thread let status_clone = status.clone(); @@ -430,26 +601,42 @@ impl IndexingService { let db_connection_clone = db_connection.clone(); let suspend_clone = suspend_flag.clone(); + let maintenance_clone = maintenance.clone(); indexing_handle = Some(thread::spawn(move || { - if let Err(e) = Self::run_indexing(&status_clone, &paths_owned, &db_path_owned, &stop_flag_clone, &suspend_clone, &config_owned, &db_connection_clone) { - *status_clone.lock().unwrap() = IndexingStatus::Error(e); - } else { - // Only set to Idle if we weren't stopped - if !*stop_flag_clone.lock().unwrap() { - *status_clone.lock().unwrap() = IndexingStatus::Idle; - } - } + // The writer thread: every DB write and every text + // extraction a run performs happens here. + crate::platform::set_background_priority(); + let result = Self::run_indexing(&status_clone, &paths_owned, &db_path_owned, &stop_flag_clone, &suspend_clone, &config_owned, &db_connection_clone); - // Clear the database connection when indexing completes + // Released before maintenance, not after: VACUUM needs + // its own connection (see `db::open::open_maintenance`) + // and two writable connections on one file would only + // contend. if let Ok(mut db_opt) = db_connection_clone.lock() { *db_opt = None; } + + match result { + Err(e) => *status_clone.lock().unwrap() = IndexingStatus::Error(e), + // Stopped runs included: a run cut short still + // leaves a log to land and, if it got as far as + // deleting rows, slack to reclaim. + Ok(()) => { + *status_clone.lock().unwrap() = IndexingStatus::Optimizing; + Self::run_maintenance(&db_path_owned, &maintenance_clone); + *status_clone.lock().unwrap() = IndexingStatus::Idle; + } + } })); } IndexingCommand::Stop => { - if matches!(*status.lock().unwrap(), IndexingStatus::Running { .. }) { - *status.lock().unwrap() = IndexingStatus::Stopping; - *stop_flag.lock().unwrap() = true; + // Only a run is stoppable. Optimizing is what happens + // *after* a run stops, so a Stop landing during it has + // nothing left to ask for. + let mut guard = status.lock().unwrap(); + if matches!(*guard, IndexingStatus::Running { .. }) { + *guard = IndexingStatus::Stopping; + stop_flag.store(true, Ordering::Relaxed); } } } @@ -461,11 +648,51 @@ impl IndexingService { } } + /// Optimize the index once a run ends, completed or stopped: land the log, + /// reclaim the file's slack, refresh the planner's statistics. + /// Best-effort — nothing here can fail a run that is already over, so every + /// outcome is a log line. + /// + /// Runs on its own connection, after the run's writer is closed. VACUUM on + /// the indexer's connection would build the replacement index in RAM; see + /// [`crate::db::schema::PRAGMAS_MAINTENANCE`]. + /// + /// Deliberately not cancelled by the stop flag: Stop ends *indexing*, and + /// this is what runs afterwards. `interrupt` is the one way out, for the + /// caller that cannot wait — see [`Self::cancel_optimizing`]. + fn run_maintenance(db_path: &str, interrupt: &Arc>>) { + let conn = match crate::db::open::open_maintenance(db_path) { + Ok(conn) => conn, + Err(e) => { + crate::log_warn!("optimize: {}", e); + return; + } + }; + if let Ok(mut slot) = interrupt.lock() { + *slot = Some(conn.get_interrupt_handle()); + } + + let dir = std::path::Path::new(db_path) + .parent() + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default(); + let outcome = crate::db::repo::maintain(&conn, &dir); + + if let Ok(mut slot) = interrupt.lock() { + *slot = None; + } + match outcome { + Ok(true) => crate::log_info!("optimized the index and reclaimed unused space"), + Ok(false) => {} + Err(e) => crate::log_warn!("optimize failed (non-fatal): {}", e), + } + } + fn run_indexing( status: &Arc>, paths: &[String], db_path: &str, - stop_flag: &Arc>, + stop_flag: &Arc, suspend_flag: &Arc, config: &Config, db_connection: &Arc>>>>, @@ -482,21 +709,17 @@ impl IndexingService { let mut seen_roots = HashSet::new(); let roots: Vec = paths .iter() - .map(|p| { - std::path::Path::new(p) - .canonicalize() - .ok() - // Same spelling rules as `files.path`: a hand-rolled - // four-character strip turns `\\?\UNC\server\share` into - // `UNC\server\share`, which is not a path — and no longer - // looks like a share, so the root would silently walk with - // the local thread count instead of the network one. - .map(|c| path_to_db_string(&c)) - .unwrap_or_else(|| p.clone()) - }) + .map(|p| normalize_root_string(p)) .filter(|p| seen_roots.insert(p.clone())) .collect(); + // Per-root worker overrides, rekeyed to match `roots`. The config + // stores them under the root exactly as the user typed it, which is + // not what a canonicalized root looks like once a `~`, a trailing + // slash or a symlink is involved — and a lookup that misses is + // invisible, it just quietly walks with the auto-detected count. + let worker_overrides = resolved_root_workers(config); + // Open and migrate the database to the current schema version. let conn = db::open_or_recreate(db_path, &config.processing.tokenize)?; @@ -538,9 +761,7 @@ impl IndexingService { for root in &roots { let ignore = crate::config::IgnoreSet::compile(&config.indexing.ignore_patterns) .map_err(|e| format!("ignore patterns: {}", e))?; - let workers = config - .indexing - .root_workers + let workers = worker_overrides .get(root) .copied() .filter(|w| *w > 0) @@ -558,7 +779,6 @@ impl IndexingService { suspend_flag.clone(), workers, ); - let stats = walk.worker_stats(); let count_total = Arc::new(AtomicUsize::new(0)); { @@ -566,6 +786,7 @@ impl IndexingService { let cancel = count_cancel.clone(); let total = count_total.clone(); let _ = thread::Builder::new().name("qs-count".into()).spawn(move || { + crate::platform::set_background_priority(); match count_tree_entries_fast(&root, &cancel) { // A genuinely empty root stores 1 so "known" stays // distinguishable from the 0 = unknown sentinel; an @@ -583,14 +804,14 @@ impl IndexingService { pipelines.push(RootPipeline { root: root.clone(), walk, - stats, count_total, pending_updates: Vec::new(), pending_inserts: Vec::new(), walked: 0, walk_clean: true, phase: RootPhase::Walking, - extract: None, + workers, + content: None, extract_total: 0, extracted: 0, current_file: None, @@ -602,19 +823,22 @@ impl IndexingService { let publish = |pipelines: &[RootPipeline]| { let roots: Vec = pipelines .iter() - .map(|p| RootProgress { - root: p.root.clone(), - phase: p.phase, - walked: p.walked, - walk_total: match p.count_total.load(Ordering::Relaxed) { - 0 => None, - n => Some(n), - }, - extracted: p.extracted, - extract_total: p.extract_total, - current_file: p.current_file.clone(), - active_workers: p.stats.active(), - total_workers: p.stats.total(), + .map(|p| { + let (active_workers, total_workers) = p.worker_counts(); + RootProgress { + root: p.root.clone(), + phase: p.phase, + walked: p.walked, + walk_total: match p.count_total.load(Ordering::Relaxed) { + 0 => None, + n => Some(n), + }, + extracted: p.extracted, + extract_total: p.extract_total, + current_file: p.current_file.clone(), + active_workers, + total_workers, + } }) .collect(); if let Ok(mut g) = status.lock() { @@ -641,11 +865,22 @@ impl IndexingService { // Paths reached by resolving a symlink, whose row lives under a // parent that may be outside every root. let mut aliased_paths: HashSet = HashSet::new(); - let mut aborted = false; + // Set by whichever of the two `break`s the loop leaves through, so a + // run that was stopped is never mistaken for a completed one. + let aborted; let mut stale_cleanup_ok = true; let mut cleanup_done = false; let mut stale_deleted = 0usize; let mut rr = 0usize; + // Log size at which to force a checkpoint, re-armed after every + // attempt. See [`wal_len`] for why the run has to do this itself + // rather than leave it to SQLite's autocheckpoint. + let wal_path = format!("{}-wal", db_path); + let wal_cap = match config.processing.maximum_wal_size { + 0 => 0, + n => n.max(crate::config::MINIMUM_WAL_SIZE), + }; + let mut checkpoint_at = wal_cap; loop { if should_abort(stop_flag, suspend_flag) { @@ -745,7 +980,7 @@ impl IndexingService { ); stale_cleanup_ok = false; p.phase = RootPhase::Done; - } else if *stop_flag.lock().unwrap() { + } else if stop_flag.load(Ordering::Relaxed) { p.phase = RootPhase::Done; } else { let cursor = ExtractCursor::for_root(&p.root); @@ -761,7 +996,18 @@ impl IndexingService { if scope.pending == 0 { p.phase = RootPhase::Done; } else { - p.extract = Some(cursor); + // Starts only now: the rows have to + // exist before the feeder can page + // over them. + p.content = Some(crate::content::extract_content( + db_path, + &cursor, + registry.clone(), + config.clone(), + stop_flag.clone(), + suspend_flag.clone(), + p.workers, + )); p.phase = RootPhase::Extracting; } } @@ -773,27 +1019,43 @@ impl IndexingService { progressed |= took > 0; } RootPhase::Extracting => { - let cursor = p.extract.as_mut().expect("extracting root has a cursor"); - let mut last_file: Option = None; - let processed = extract_one_batch( - &conn_mutex, - cursor, - ®istry, - config, - stop_flag, - suspend_flag, - &mut |name| last_file = Some(name.to_string()), - )?; - if last_file.is_some() { - p.current_file = last_file; + // The same shape as the walking arm: drain up to a + // quantum of finished work, then write it. Reading and + // extracting happen on this root's own pool, so a slow + // root occupies the writer only for as long as its + // commits take. + let pass = p.content.as_mut().expect("extracting root has a pass"); + let mut batch: Vec = Vec::new(); + let mut finished = false; + while batch.len() < quantum { + match pass.try_next() { + TryNext::Item(row) => batch.push(row), + TryNext::Empty => break, + TryNext::Finished => { + finished = true; + break; + } + } } - if processed == 0 { - p.extract = None; + if let Some(row) = batch.last() { + p.current_file = Some(row.name.clone()); + } + let took = batch.len(); + p.extracted += store_extracted(&conn_mutex, &batch, stop_flag, config)?; + if finished { + // Join before deciding: workers close the channel + // when they stop for any reason, so a panic and a + // finished pass look identical from here. + if !pass.finish() { + crate::log_warn!( + "a content worker for {} terminated abnormally", + p.root + ); + } + p.content = None; p.phase = RootPhase::Done; - } else { - p.extracted += processed; } - progressed = true; + progressed |= finished || took > 0; } RootPhase::Done => {} } @@ -805,7 +1067,7 @@ impl IndexingService { // symlinks. Runs at most once per run, on this writer thread. if !cleanup_done && pipelines.iter().all(|p| p.phase != RootPhase::Walking) { cleanup_done = true; - let stopped = *stop_flag.lock().unwrap(); + let stopped = stop_flag.load(Ordering::Relaxed); if stale_cleanup_ok && !stopped { // Directories that vanished entirely are never read, so // per-directory reconciliation never sees them; only a @@ -827,7 +1089,20 @@ impl IndexingService { // the sweep skips parents beneath one. Re-checking here // as well would put the same rule in two places, free to // drift, and the tests could not tell which one held. - let stale_paths: Vec = stale_candidates.drain(..).collect(); + // + // The aliased filter *is* applied to both, though. The + // sweep does its own, but per-directory reconciliation can + // produce an aliased path too: a symlink target that lives + // in a walked directory but is itself hidden or + // ignore-matched is skipped before `present.insert`, so it + // reads as stale — while the alias route inserted it moments + // earlier. Without this the row is written and deleted on + // every single run. One filter, in the one place the + // deletions are assembled. + let stale_paths: Vec = stale_candidates + .drain(..) + .filter(|p| !aliased_paths.contains(p)) + .collect(); let unreadable_count: usize = pipelines .iter() .map(|p| p.walk.unreadable().paths().len()) @@ -850,6 +1125,7 @@ impl IndexingService { stale_paths.as_slice(), stop_flag, suspend_flag, + config, )?; } } @@ -858,7 +1134,42 @@ impl IndexingService { publish(&pipelines); + // After `publish`, so the GUI's last snapshot is fresh going into + // a checkpoint that may block for `busy_timeout`. + // + // `progressed` is the cheap half of the test: this thread is the + // only writer during a run, so a round that wrote nothing cannot + // have grown the log, and without the gate this would stat the + // file at 500 Hz through the idle backoff below. The stop flag is + // the other half — `stop_indexing` wants this same connection, and + // a checkpoint starting as the user hits Stop would sit in front + // of it for five seconds. + if wal_cap > 0 + && progressed + && !stop_flag.load(Ordering::Relaxed) + && wal_len(&wal_path) >= checkpoint_at + { + { + let conn = conn_mutex.lock().unwrap(); + if let Err(e) = crate::db::repo::checkpoint_truncate(&conn) { + crate::log_warn!("{}", e); + } + } + // Re-armed from what is actually on disk, not from zero: a + // checkpoint that lost the race to a running search then costs + // one attempt per further `wal_cap` of growth instead of + // retrying into every round for the rest of the run. + checkpoint_at = wal_len(&wal_path) + wal_cap; + } + if pipelines.iter().all(|p| p.phase == RootPhase::Done) { + // A stop can land *inside* the pass above — `TryNext::Finished` + // and a drained content pass both mark a root Done — so + // every root can reach Done without the top-of-loop check ever + // seeing the flag. Re-read it here or a cut-short run would be + // stamped as a completed full index and suppress the next + // periodic reindex for the whole interval. + aborted = stop_flag.load(Ordering::Relaxed); break; } if !progressed { @@ -874,20 +1185,27 @@ impl IndexingService { process_batch_inserts(&conn_mutex, &p.pending_inserts, stop_flag, config)?; p.pending_inserts.clear(); } - if let Ok(mut status_guard) = status.lock() { - *status_guard = IndexingStatus::Idle; - } // Deliberately no stale cleanup: a partial walk has a partial // seen set, and deleting everything it did not reach would // empty most of the index. + // + // The final status is the caller's to publish: a stopped run is + // still followed by an optimize pass, so this is not yet Idle. return Ok(()); } + if stale_deleted > 0 { + crate::log_info!( + "removed {} stale index entr{}", + stale_deleted, + if stale_deleted == 1 { "y" } else { "ies" } + ); + } + // FTS housekeeping once per completed run (cheap if nothing changed). - let _ = stale_deleted; { let conn = conn_mutex.lock().unwrap(); - fts_finalize_after_text_indexing(&conn)?; + fts_finalize_after_text_indexing(&conn); } // Stamp the successful run so the coordinator can schedule the next @@ -922,6 +1240,13 @@ impl IndexingService { ("indexing_path", normalize_root_string(indexing_path)), ("tokenize", config.processing.tokenize.clone()), ("include_hidden", config.indexing.include_hidden.to_string()), + // Decides whether symlink targets are in the index at all, so a + // change leaves rows that no longer belong — the rebuild prompt has + // to be able to name it. + ( + "follow_symlinks", + config.indexing.follow_symlinks.to_string(), + ), ( "ignore_patterns", sorted_joined(&config.indexing.ignore_patterns), @@ -982,6 +1307,12 @@ impl IndexingService { /// Canonicalize a root string for storage/comparison, stripping the Windows /// UNC prefix. Multi-root strings (newline-joined) fail canonicalize and /// pass through verbatim, which still compares consistently. +/// +/// The UNC strip is [`path_to_db_string`]'s, not a hand-rolled one: chopping +/// four characters would turn `\\?\UNC\server\share` into +/// `UNC\server\share`, which is not a path — and no longer looks like a +/// share, so the root would walk with the local thread count instead of the +/// network one. fn normalize_root_string(indexing_path: &str) -> String { let path = std::path::Path::new(indexing_path) .canonicalize() @@ -1002,3 +1333,296 @@ impl Default for IndexingService { } } + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp_dir(tag: &str) -> std::path::PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!( + "quicksearch-idx-{}-{}-{}", + tag, + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&p).unwrap(); + // The temp dir itself may sit behind a symlink (/tmp -> /private/tmp). + p.canonicalize().unwrap() + } + + fn config_with(roots: Vec, overrides: &[(&str, usize)]) -> Config { + let mut cfg = Config::default(); + cfg.paths.indexing_paths = roots; + for (root, workers) in overrides { + cfg.indexing + .root_workers + .insert((*root).to_string(), *workers); + } + cfg + } + + #[test] + fn an_override_survives_a_trailing_slash() { + let dir = tmp_dir("slash"); + let spelled = format!("{}/", dir.display()); + let cfg = config_with(vec![spelled.clone()], &[(&spelled, 24)]); + assert_eq!( + resolved_root_workers(&cfg).get(&normalize_root_string(&dir.to_string_lossy())), + Some(&24), + "the walk canonicalizes the root; the override must follow" + ); + std::fs::remove_dir_all(&dir).ok(); + } + + #[cfg(unix)] + #[test] + fn an_override_survives_a_symlinked_root() { + let dir = tmp_dir("symlink"); + let target = dir.join("real"); + let link = dir.join("link"); + std::fs::create_dir_all(&target).unwrap(); + std::os::unix::fs::symlink(&target, &link).unwrap(); + + let spelled = link.to_string_lossy().into_owned(); + let cfg = config_with(vec![spelled.clone()], &[(&spelled, 12)]); + let resolved = resolved_root_workers(&cfg); + assert_eq!( + resolved.get(&normalize_root_string(&target.to_string_lossy())), + Some(&12) + ); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn overrides_for_folders_that_are_no_longer_indexed_are_dropped() { + let dir = tmp_dir("stale"); + let kept = dir.to_string_lossy().into_owned(); + let cfg = config_with(vec![kept.clone()], &[(&kept, 8), ("/gone", 32)]); + let resolved = resolved_root_workers(&cfg); + assert_eq!(resolved.len(), 1, "{:?}", resolved); + assert_eq!(resolved.values().next(), Some(&8)); + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn a_root_without_an_override_gets_no_entry() { + let dir = tmp_dir("auto"); + let root = dir.to_string_lossy().into_owned(); + let cfg = config_with(vec![root], &[]); + assert!(resolved_root_workers(&cfg).is_empty(), "absent = auto"); + std::fs::remove_dir_all(&dir).ok(); + } + + /// The progress line reports whichever pool the root is currently + /// running. Reading the walk's stats in every phase made an extracting + /// root report "0 busy" out of a walk pool that had already exited, and + /// let finished roots keep inflating the total. + #[test] + fn worker_counts_follow_the_phase() { + let dir = tmp_dir("phase-workers"); + std::fs::write(dir.join("a.txt"), "hello").unwrap(); + let db_path = dir.join("index.db").to_string_lossy().into_owned(); + drop(db::open_or_recreate(&db_path, "trigram").unwrap()); + + let root = dir.to_string_lossy().into_owned(); + let stop = Arc::new(AtomicBool::new(false)); + let suspend = Arc::new(AtomicBool::new(false)); + let walk = walk_indexable_files( + std::slice::from_ref(&root), + false, + false, + crate::config::IgnoreSet::compile(&[]).unwrap(), + &db_path, + Config::default(), + Arc::new(Registry::default_set()), + stop.clone(), + suspend.clone(), + 3, + ); + // An empty range, so the pass ends immediately — but its pool size is + // fixed when it is built, which is what the display reports. + let content = crate::content::extract_content( + &db_path, + &ExtractCursor::for_root(&dir.join("nothing").to_string_lossy()), + Arc::new(Registry::default_set()), + Config::default(), + stop, + suspend, + 2, + ); + + let mut p = RootPipeline { + root, + walk, + count_total: Arc::new(AtomicUsize::new(0)), + workers: 3, + pending_updates: Vec::new(), + pending_inserts: Vec::new(), + walked: 0, + walk_clean: true, + phase: RootPhase::Walking, + content: Some(content), + extract_total: 0, + extracted: 0, + current_file: None, + }; + + assert_eq!(p.worker_counts().1, 3, "walking: the walk's own pool"); + p.phase = RootPhase::Extracting; + assert_eq!(p.worker_counts().1, 2, "extracting: the content pool"); + p.phase = RootPhase::Done; + assert_eq!(p.worker_counts(), (0, 0), "a finished root runs nothing"); + + drop(p); + std::fs::remove_dir_all(&dir).ok(); + } + + /// A root's progress with only the fields the denominator rules read. + fn progress(phase: RootPhase, walked: usize, walk_total: Option) -> RootProgress { + RootProgress { + root: "/r".to_string(), + phase, + walked, + walk_total, + extracted: 0, + extract_total: 0, + current_file: None, + active_workers: 0, + total_workers: 0, + } + } + + #[test] + fn a_walking_root_falls_back_to_the_find_estimate() { + let p = progress(RootPhase::Walking, 100, Some(1000)); + assert_eq!(p.walk_denominator(), Some(1000)); + } + + #[test] + fn a_walking_root_without_a_count_yet_has_no_denominator() { + let p = progress(RootPhase::Walking, 100, None); + assert_eq!(p.walk_denominator(), None); + } + + /// An estimate the walk has already overtaken is provably wrong, and a bar + /// pinned at 100% while the walk is still running reads as a hang. + #[test] + fn an_overtaken_estimate_is_raised_to_the_walked_count() { + let p = progress(RootPhase::Walking, 1500, Some(1000)); + assert_eq!(p.walk_denominator(), Some(1500)); + } + + /// The bug this whole rule exists for: `find` counts tree entries — + /// directories, hidden files, ignore-pruned subtrees — where `walked` + /// counts only walkable files, so the estimate reads far high. Once the + /// walk ends the exact number is in hand and the estimate must go. + #[test] + fn a_root_past_its_walk_uses_the_exact_count() { + for phase in [RootPhase::Extracting, RootPhase::Done] { + assert_eq!( + progress(phase, 261_088, Some(6_677_062)).walk_denominator(), + Some(261_088), + "{:?} must not keep the estimate", + phase + ); + assert_eq!( + progress(phase, 261_088, None).walk_denominator(), + Some(261_088), + "{:?} needs no estimate to have landed", + phase + ); + } + } + + #[test] + fn overall_progress_sums_both_halves_of_every_root() { + let mut walking = progress(RootPhase::Walking, 100, Some(1000)); + let mut extracting = progress(RootPhase::Extracting, 500, Some(9999)); + extracting.extracted = 200; + extracting.extract_total = 400; + walking.extracted = 0; + + let o = overall_progress(&[walking, extracting]); + assert_eq!(o.processed, 100 + 500 + 200); + // 1000 (estimate) + 500 (exact) + 400 (extraction scope). + assert_eq!(o.total, Some(1900)); + } + + #[test] + fn one_uncounted_walking_root_leaves_the_whole_total_unknown() { + let known = progress(RootPhase::Done, 10, Some(10)); + let unknown = progress(RootPhase::Walking, 5, None); + let o = overall_progress(&[known, unknown]); + assert_eq!(o.processed, 15); + assert_eq!(o.total, None); + assert_eq!(o.fraction(), None); + } + + /// Roots past their walk carry their own totals, so a run whose counts + /// never landed still gains a percentage once the walks end. + #[test] + fn a_run_past_its_walks_needs_no_estimate_at_all() { + let roots = [ + progress(RootPhase::Done, 10, None), + progress(RootPhase::Extracting, 5, None), + ]; + assert_eq!(overall_progress(&roots).total, Some(15)); + } + + /// The regression: with the `find` estimate held past the walk, the run + /// below finished at 7,999,707 / 10,562,418 = 76% and the bar never + /// filled. These are the real figures from that run. + #[test] + fn a_finished_run_reaches_exactly_one_hundred_percent() { + let roots: Vec = [ + (261_088usize, 238_929usize), + (45_202, 10_339), + (2_000_000, 2_574_506), + (300_000, 221_641), + (1_508_061, 839_941), + ] + .iter() + .map(|&(walked, extracted)| { + let mut p = progress(RootPhase::Done, walked, Some(walked * 2)); + p.extracted = extracted; + p.extract_total = extracted; + p + }) + .collect(); + + let o = overall_progress(&roots); + assert_eq!(o.processed, 4_114_351 + 3_885_356); + assert_eq!(o.total, Some(o.processed), "the estimate must be gone"); + assert_eq!(o.fraction(), Some(1.0)); + } + + #[test] + fn a_run_with_nothing_to_do_has_no_fraction_to_show() { + let o = overall_progress(&[progress(RootPhase::Done, 0, None)]); + assert_eq!(o.total, Some(0)); + assert_eq!(o.fraction(), None, "no division by zero"); + } + + /// `walked` can outrun a denominator that was exact when taken — a root + /// re-walked through symlink aliases, say. The bar must stop at full. + #[test] + fn the_fraction_never_exceeds_one() { + let mut p = progress(RootPhase::Done, 10, None); + p.extracted = 100; + let o = overall_progress(&[p]); + assert_eq!(o.processed, 110); + assert_eq!(o.total, Some(10)); + assert_eq!(o.fraction(), Some(1.0)); + } + + #[test] + fn a_run_with_no_roots_is_complete_rather_than_unknown() { + let o = overall_progress(&[]); + assert_eq!(o.processed, 0); + assert_eq!(o.total, Some(0)); + } +} diff --git a/crates/quicksearch-core/src/lib.rs b/crates/quicksearch-core/src/lib.rs index bfc28cc..c6b6032 100644 --- a/crates/quicksearch-core/src/lib.rs +++ b/crates/quicksearch-core/src/lib.rs @@ -1,5 +1,6 @@ pub mod cli; pub mod config; +pub mod content; pub mod coordinator; pub mod db; pub mod document_extraction; diff --git a/crates/quicksearch-core/src/platform.rs b/crates/quicksearch-core/src/platform.rs index 74b231a..d354752 100644 --- a/crates/quicksearch-core/src/platform.rs +++ b/crates/quicksearch-core/src/platform.rs @@ -227,6 +227,41 @@ pub const WATCH_ROOTS_RECURSIVELY: bool = cfg!(windows); /// on both sides, consistently. pub const PATH_COLLATION: &str = if cfg!(windows) { "NOCASE" } else { "BINARY" }; +/// Drop the **calling thread** to background scheduling priority. +/// +/// Per-thread, not per-process. The GUI shares this process, so lowering the +/// process would slow the very window the user is watching progress in — the +/// point is to yield to the foreground, not to throttle ourselves. Called by +/// the threads that do indexing work and by nobody else; the search worker, +/// the coordinator and the watcher exist to answer promptly and keep normal +/// priority. +/// +/// Best-effort and idempotent: a refusal is not worth reporting, since the +/// only consequence is that indexing competes on equal terms. +pub fn set_background_priority() { + #[cfg(target_os = "linux")] + { + // Linux schedules per task, so `nice` moves this thread alone. + // Deliberately not `setpriority(PRIO_PROCESS, 0, …)`, which is + // process-wide on the BSDs and would take the GUI with it. + unsafe { libc::nice(10) }; + } + #[cfg(windows)] + { + use windows_sys::Win32::System::Threading::{ + GetCurrentThread, SetThreadPriority, THREAD_MODE_BACKGROUND_BEGIN, + }; + // Background *mode*, not merely a lower priority number: it drops I/O + // priority as well, which is what actually keeps a walk from starving + // the foreground on a spinning disk. + unsafe { SetThreadPriority(GetCurrentThread(), THREAD_MODE_BACKGROUND_BEGIN) }; + } + // Elsewhere (macOS, BSD): deliberately nothing. `nice` there applies to the + // whole process, so it would hit the GUI. The right call is + // `pthread_set_qos_class_self_np(QOS_CLASS_UTILITY, 0)`, which is worth + // adding on its own terms rather than approximating here. +} + /// How long to keep retrying a delete that fails because something else holds /// the file open. #[cfg(windows)] @@ -351,6 +386,16 @@ mod tests { assert!(!is_unc_string("")); } + /// Not observable cross-platform beyond "it did not blow up", which is + /// still worth pinning: the call is `unsafe` on both real targets, and it + /// runs at the top of every walker thread, so it must also be safe to + /// repeat. + #[test] + fn background_priority_is_best_effort_and_repeatable() { + set_background_priority(); + set_background_priority(); + } + #[test] fn collation_matches_like_case_folding() { // LIKE folds ASCII case on every platform; the `=` half of a path diff --git a/crates/quicksearch-core/src/query/ast.rs b/crates/quicksearch-core/src/query/ast.rs index b3fe14d..83da903 100644 --- a/crates/quicksearch-core/src/query/ast.rs +++ b/crates/quicksearch-core/src/query/ast.rs @@ -1,5 +1,6 @@ -//! Query AST shared between parser and translator. +//! Shared query vocabulary. +/// The comparison in a `key op value` filter, as the lexer emits it. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Op { /// `key:value` — substring / FTS MATCH semantics. @@ -15,44 +16,3 @@ pub enum Op { /// `key>=value` Ge, } - -#[derive(Debug, Clone, PartialEq)] -pub enum Term { - /// An unquoted word or quoted phrase. Feeds the FTS MATCH expression. - Literal(String), - /// A structured filter such as `type:Audio` or `modified:>=2024-01-01`. - Property { key: String, op: Op, value: String }, - And(Vec), - Or(Vec), -} - -impl Term { - /// Combine two terms with AND, flattening to avoid a deeply nested tree. - pub fn and(a: Term, b: Term) -> Term { - match (a, b) { - (Term::And(mut xs), Term::And(ys)) => { - xs.extend(ys); - Term::And(xs) - } - (Term::And(mut xs), other) | (other, Term::And(mut xs)) => { - xs.push(other); - Term::And(xs) - } - (a, b) => Term::And(vec![a, b]), - } - } - - pub fn or(a: Term, b: Term) -> Term { - match (a, b) { - (Term::Or(mut xs), Term::Or(ys)) => { - xs.extend(ys); - Term::Or(xs) - } - (Term::Or(mut xs), other) | (other, Term::Or(mut xs)) => { - xs.push(other); - Term::Or(xs) - } - (a, b) => Term::Or(vec![a, b]), - } - } -} diff --git a/crates/quicksearch-core/src/query/mod.rs b/crates/quicksearch-core/src/query/mod.rs index 66596dc..0de87ee 100644 --- a/crates/quicksearch-core/src/query/mod.rs +++ b/crates/quicksearch-core/src/query/mod.rs @@ -1,33 +1,33 @@ -//! Structured query parser and SQL translator. +//! Search-box input → the ranked cascade's term plus its structured filters. //! -//! Input syntax (a deliberate subset of KDE Baloo's query language — just -//! enough to be useful standalone; full Baloo grammar lives in the Set B -//! compat layer): +//! The entry point is [`split_for_cascade`], which splits raw input into a +//! single search phrase and zero or more `key:value` filters. There is no +//! boolean grammar by design: the cascade ranks one phrase, so `AND`, `OR` and +//! parentheses are ordinary words rather than operators (see [`split`]). //! -//! - plain words: `foo bar` (implicit AND) -//! - quoted phrases: `"hello world"` -//! - boolean operators: `AND`, `OR` (case-sensitive) -//! - grouping: `(a OR b)` -//! - structured filters: -//! - `type:Audio` / `type:Image` / `type:Document` / `type:Text` / `type:Video` -//! / `type:Archive` / `type:Spreadsheet` / `type:Presentation` / `type:Folder` -//! - `modified:>=2024-01-01`, `modified:<2023-12-01`, `modified:=2024-05-20` -//! (also accepts `modified>=2024-01-01` without the colon) -//! - `path:/some/dir` — matches files with that directory as their parent -//! or any ancestor. +//! Recognized filters: //! -//! The entry point is [`parse_and_build`], which accepts a query string and -//! returns a ready-to-execute [`SqlQuery`]. +//! - `type:Audio` / `Image` / `Video` / `Document` / `Text` / `Archive` / +//! `Spreadsheet` / `Presentation` / `Folder` +//! - `modified:>=2024-01-01`, `modified:<2023-12-01`, `modified:=2024-05-20` +//! (`mtime:` is a synonym; `=` matches the whole day) +//! - `path:/some/dir` — that directory or any beneath it (`folder:`, +//! `includefolder:` are synonyms) +//! - `name:report` / `filename:report` — filename substring, `*` globs +//! - `mime:application/pdf` — exact MIME type +//! - `regex:…` — compiled in [`pattern`] and matched in Rust, never in SQL +//! +//! Each recognized filter becomes a [`translator::FilterFragment`] that the +//! cascade ANDs onto every stage; everything else joins the term. pub mod ast; pub mod lexer; -pub mod parser; pub mod pattern; pub mod split; pub mod translator; -pub use ast::{Op, Term}; +pub use ast::Op; pub use lexer::tokenize_spanned; pub use pattern::{RegexQuery, TermPattern}; pub use split::{split_for_cascade, CascadeQuery}; -pub use translator::{parse_and_build, SqlQuery, TranslateError}; +pub use translator::TranslateError; diff --git a/crates/quicksearch-core/src/query/parser.rs b/crates/quicksearch-core/src/query/parser.rs deleted file mode 100644 index 77bbadb..0000000 --- a/crates/quicksearch-core/src/query/parser.rs +++ /dev/null @@ -1,284 +0,0 @@ -//! Recursive-descent parser: token stream → [`Term`] tree. -//! -//! Grammar (implicit AND between adjacent terms): -//! ```text -//! expr := or_expr -//! or_expr := and_expr ("OR" and_expr)* -//! and_expr := atom ("AND"? atom)* -//! atom := "(" expr ")" | property | literal -//! property := ident ":"|"="|"<"|"<="|">"|">=" value -//! literal := WORD | QUOTED -//! ``` - -use super::ast::{Op, Term}; -use super::lexer::{tokenize, LexError, Token}; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ParseError { - Lex(LexError), - Unexpected { at: usize, reason: String }, - Empty, -} - -impl std::fmt::Display for ParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ParseError::Lex(e) => write!(f, "{}", e), - ParseError::Unexpected { at, reason } => { - write!(f, "parse error at token {}: {}", at, reason) - } - ParseError::Empty => write!(f, "empty query"), - } - } -} - -impl std::error::Error for ParseError {} - -pub fn parse(input: &str) -> Result { - let tokens = tokenize(input).map_err(ParseError::Lex)?; - let trimmed: Vec<_> = tokens.into_iter().collect(); - if trimmed.is_empty() { - return Err(ParseError::Empty); - } - let mut p = Parser { tokens: trimmed, pos: 0 }; - let term = p.parse_or()?; - if p.pos < p.tokens.len() { - return Err(ParseError::Unexpected { - at: p.pos, - reason: format!("trailing token {:?}", p.tokens[p.pos]), - }); - } - Ok(term) -} - -struct Parser { - tokens: Vec, - pos: usize, -} - -impl Parser { - fn peek(&self) -> Option<&Token> { - self.tokens.get(self.pos) - } - - fn bump(&mut self) -> Option { - let t = self.tokens.get(self.pos).cloned(); - if t.is_some() { - self.pos += 1; - } - t - } - - fn parse_or(&mut self) -> Result { - let mut left = self.parse_and()?; - while matches!(self.peek(), Some(Token::Or)) { - self.bump(); - let right = self.parse_and()?; - left = Term::or(left, right); - } - Ok(left) - } - - fn parse_and(&mut self) -> Result { - let mut left = self.parse_atom()?; - loop { - match self.peek() { - Some(Token::And) => { - self.bump(); - let right = self.parse_atom()?; - left = Term::and(left, right); - } - // Implicit AND: adjacent atoms without an operator. - Some(Token::Word(_)) - | Some(Token::Quoted(_)) - | Some(Token::LParen) => { - let right = self.parse_atom()?; - left = Term::and(left, right); - } - _ => break, - } - } - Ok(left) - } - - fn parse_atom(&mut self) -> Result { - match self.bump() { - Some(Token::LParen) => { - let inner = self.parse_or()?; - match self.bump() { - Some(Token::RParen) => Ok(inner), - other => Err(ParseError::Unexpected { - at: self.pos, - reason: format!("expected ')' got {:?}", other), - }), - } - } - Some(Token::Quoted(s)) => Ok(Term::Literal(s)), - Some(Token::Word(w)) => { - // If followed by an operator, this word is a property key. - if let Some(Token::Op(op)) = self.peek().cloned() { - self.bump(); - // `key:>=value` — an Op followed by another Op becomes the - // effective comparator; the original colon is "separator". - let effective_op = if op == Op::Contains { - if let Some(Token::Op(inner_op)) = self.peek().cloned() { - self.bump(); - inner_op - } else { - Op::Contains - } - } else { - op - }; - let value = match self.bump() { - Some(Token::Word(v)) => v, - Some(Token::Quoted(v)) => v, - other => { - return Err(ParseError::Unexpected { - at: self.pos, - reason: format!("expected property value got {:?}", other), - }) - } - }; - Ok(Term::Property { - key: w, - op: effective_op, - value, - }) - } else { - Ok(Term::Literal(w)) - } - } - other => Err(ParseError::Unexpected { - at: self.pos, - reason: format!("expected term, got {:?}", other), - }), - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn lit(s: &str) -> Term { - Term::Literal(s.into()) - } - - fn prop(k: &str, op: Op, v: &str) -> Term { - Term::Property { - key: k.into(), - op, - value: v.into(), - } - } - - #[test] - fn single_word() { - assert_eq!(parse("foo").unwrap(), lit("foo")); - } - - #[test] - fn quoted_phrase() { - assert_eq!(parse(r#""hello world""#).unwrap(), lit("hello world")); - } - - #[test] - fn implicit_and() { - assert_eq!( - parse("foo bar").unwrap(), - Term::And(vec![lit("foo"), lit("bar")]) - ); - } - - #[test] - fn explicit_and() { - assert_eq!( - parse("foo AND bar").unwrap(), - Term::And(vec![lit("foo"), lit("bar")]) - ); - } - - #[test] - fn or_has_lower_precedence_than_and() { - assert_eq!( - parse("a b OR c").unwrap(), - Term::Or(vec![ - Term::And(vec![lit("a"), lit("b")]), - lit("c") - ]) - ); - } - - #[test] - fn parens_override_precedence() { - assert_eq!( - parse("a (b OR c)").unwrap(), - Term::And(vec![lit("a"), Term::Or(vec![lit("b"), lit("c")])]) - ); - } - - #[test] - fn property_contains() { - assert_eq!( - parse("type:Audio").unwrap(), - prop("type", Op::Contains, "Audio") - ); - } - - #[test] - fn property_comparator_via_colon() { - assert_eq!( - parse("modified:>=2024-01-01").unwrap(), - prop("modified", Op::Ge, "2024-01-01") - ); - } - - #[test] - fn property_comparator_bare() { - assert_eq!( - parse("modified>=2024-01-01").unwrap(), - prop("modified", Op::Ge, "2024-01-01") - ); - } - - #[test] - fn mixed_filter_and_literal() { - assert_eq!( - parse("type:Audio beatles").unwrap(), - Term::And(vec![ - prop("type", Op::Contains, "Audio"), - lit("beatles") - ]) - ); - } - - #[test] - fn nested_or() { - assert_eq!( - parse("(a OR b) AND (c OR d)").unwrap(), - Term::And(vec![ - Term::Or(vec![lit("a"), lit("b")]), - Term::Or(vec![lit("c"), lit("d")]) - ]) - ); - } - - #[test] - fn empty_query_is_error() { - assert!(matches!(parse(" "), Err(ParseError::Empty))); - } - - #[test] - fn trailing_token_is_error() { - assert!(parse("a )").is_err()); - } - - #[test] - fn property_value_may_be_quoted() { - assert_eq!( - parse(r#"path:"/tmp with space""#).unwrap(), - prop("path", Op::Contains, "/tmp with space") - ); - } -} diff --git a/crates/quicksearch-core/src/query/pattern.rs b/crates/quicksearch-core/src/query/pattern.rs index 0adeb64..d79cc29 100644 --- a/crates/quicksearch-core/src/query/pattern.rs +++ b/crates/quicksearch-core/src/query/pattern.rs @@ -236,6 +236,38 @@ impl TermPattern { } } + /// Case-insensitive [`find_first`] against an already-folded haystack. + /// + /// The literal path would otherwise fold the haystack itself, and the + /// cascade's full-text passes need the same fold for counting, searching + /// and snippet extraction — three copies of a document that can run to + /// `maximum_text_size`. Folding is byte-length preserving, so the returned + /// range is valid in the unfolded original too. + pub fn find_first_folded(&self, folded: &str) -> Option> { + match self { + TermPattern::Empty => None, + TermPattern::Literal(l) => { + let pos = folded.find(&l.folded)?; + Some(pos..pos + l.text.len()) + } + // The regex engine folds as it matches, so it needs no help and + // allocates nothing either way. + TermPattern::Wildcard(w) => w.search_ci.find(folded).map(|m| m.range()), + } + } + + /// Case-insensitive [`count`] against an already-folded haystack. See + /// [`TermPattern::find_first_folded`]. + pub fn count_folded(&self, folded: &str) -> usize { + match self { + TermPattern::Empty => 0, + // Both sides are already folded, so an exact scan *is* the + // case-insensitive one — and it allocates nothing. + TermPattern::Literal(l) => snippet::count_occurrences(folded, &l.folded, true), + TermPattern::Wildcard(w) => w.search_ci.find_iter(folded).take(COUNT_CAP).count(), + } + } + /// Non-overlapping occurrence count, capped at 1000 (the cascade's /// `count_frac` saturates there anyway). pub fn count(&self, text: &str, case_insensitive: bool) -> usize { diff --git a/crates/quicksearch-core/src/query/translator.rs b/crates/quicksearch-core/src/query/translator.rs index 118b89c..a17a02a 100644 --- a/crates/quicksearch-core/src/query/translator.rs +++ b/crates/quicksearch-core/src/query/translator.rs @@ -1,38 +1,16 @@ -//! AST → executable SQL. +//! `key op value` filters → composable SQL fragments. +//! +//! One filter becomes one [`FilterFragment`]: a WHERE fragment over table alias +//! `f` with anonymous `?` placeholders, plus the values to bind. The search +//! cascade ANDs those fragments onto every stage's query, so they have to +//! compose by plain appending — which anonymous placeholders do and numbered +//! ones would not. -use super::ast::{Op, Term}; -use super::parser::{parse, ParseError}; +use super::ast::Op; use crate::mime::FileType; -/// A prepared SQL statement plus its positional parameters. Parameters are -/// rusqlite `Value` for convenience at the call site. Every `?N` in `sql` -/// corresponds to `params[N-1]`. -#[derive(Debug, Clone)] -pub struct SqlQuery { - pub sql: String, - pub params: Vec, -} - -/// Sort strategy for the result set. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Sort { - /// Newest first by file modification time. - ByMtimeDesc, - /// FTS5 relevance rank. Only sensible when the query has FTS terms. - ByRank, - /// No ORDER BY clause. - None, -} - -impl Default for Sort { - fn default() -> Self { - Sort::ByMtimeDesc - } -} - #[derive(Debug, Clone)] pub enum TranslateError { - Parse(ParseError), UnknownProperty(String), BadDate(String), BadRegex(String), @@ -45,7 +23,6 @@ pub enum TranslateError { impl std::fmt::Display for TranslateError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - TranslateError::Parse(e) => write!(f, "{}", e), TranslateError::UnknownProperty(k) => write!(f, "unknown property '{}'", k), TranslateError::BadDate(s) => write!(f, "bad date '{}'", s), TranslateError::BadRegex(s) => write!(f, "regex error: {}", s), @@ -58,121 +35,6 @@ impl std::fmt::Display for TranslateError { impl std::error::Error for TranslateError {} -/// One-shot: parse the input string and build the SQL. Limit/offset are -/// applied at the SQL level; `None` means "no limit". -pub fn parse_and_build( - input: &str, - limit: Option, - offset: u32, - sort: Sort, -) -> Result { - let ast = parse(input).map_err(TranslateError::Parse)?; - build(&ast, limit, offset, sort) -} - -/// Translate an already-parsed AST into SQL. -pub fn build( - ast: &Term, - limit: Option, - offset: u32, - sort: Sort, -) -> Result { - let mut b = Builder::default(); - let where_sql = b.translate(ast)?; - - // The FTS branch joins a CTE of matching rowids; the structured branch - // stands alone. A query may have both — in that case we intersect at the - // file.id level. - let needs_fts_join = !b.fts_parts.is_empty(); - - // Structured placeholders were numbered assuming no FTS param. When an - // FTS MATCH is prepended as ?1, bump every `?N` in the generated WHERE - // clause by 1 so the positional bindings line up. - let where_sql = if needs_fts_join { - shift_placeholders(&where_sql) - } else { - where_sql - }; - - let mut sql = String::new(); - if needs_fts_join { - sql.push_str( - "WITH fts_hits AS (SELECT rowid FROM searchabletext WHERE searchabletext MATCH ?1) ", - ); - b.all_params - .insert(0, rusqlite::types::Value::Text(fts_expr(&b.fts_parts))); - } - sql.push_str("SELECT f.id, f.name, f.path FROM files f "); - if needs_fts_join { - sql.push_str("JOIN fts_hits ON fts_hits.rowid = f.id "); - } - if !where_sql.is_empty() { - sql.push_str("WHERE "); - sql.push_str(&where_sql); - sql.push(' '); - } - match sort { - Sort::ByMtimeDesc => sql.push_str("ORDER BY f.mtime DESC "), - Sort::ByRank if needs_fts_join => sql.push_str("ORDER BY rank "), - Sort::ByRank | Sort::None => {} - } - if let Some(n) = limit { - sql.push_str(&format!("LIMIT {} ", n)); - } - if offset > 0 { - sql.push_str(&format!("OFFSET {}", offset)); - } - - Ok(SqlQuery { - sql: sql.trim_end().to_string(), - params: b.all_params, - }) -} - -#[derive(Default)] -struct Builder { - /// Tokens that will be joined into the single FTS MATCH expression. - fts_parts: Vec, - /// Everything else (structured filters) as a WHERE clause. - all_params: Vec, -} - -#[derive(Debug, Clone)] -enum FtsFragment { - /// A plain word or phrase to be included as an AND'd token. - And(String), - /// Grouped alternation, already rendered with internal ORs. - OrGroup(Vec), -} - -fn fts_expr(parts: &[FtsFragment]) -> String { - // Render `(a AND b AND (c OR d))` where a/b/c/d are quoted phrases. - let mut out = String::new(); - let mut first = true; - for p in parts { - if !first { - out.push_str(" AND "); - } - first = false; - match p { - FtsFragment::And(s) => out.push_str("e_phrase(s)), - FtsFragment::OrGroup(items) => { - out.push('('); - let mut first_item = true; - for item in items { - if !first_item { - out.push_str(" OR "); - } - first_item = false; - out.push_str("e_phrase(item)); - } - out.push(')'); - } - } - } - out -} - /// Escape a phrase for FTS5 MATCH. FTS5 itself uses doubled quotes for /// literal quotes inside a quoted phrase; wrapping in quotes renders all /// other MATCH metacharacters (`( ) * :` etc.) inert. Injection-safe by @@ -190,92 +52,6 @@ pub fn quote_phrase(s: &str) -> String { buf } -impl Builder { - /// Translate a sub-term, returning its contribution to the SQL WHERE - /// clause. FTS parts are accumulated in `self.fts_parts` (not returned - /// here) since they collapse into a single MATCH expression at the top. - fn translate(&mut self, t: &Term) -> Result { - match t { - Term::Literal(s) => { - self.fts_parts.push(FtsFragment::And(s.clone())); - Ok(String::new()) - } - Term::Property { key, op, value } => self.translate_property(key, *op, value), - Term::And(children) => { - let mut pieces = Vec::new(); - for c in children { - let p = self.translate(c)?; - if !p.is_empty() { - pieces.push(p); - } - } - Ok(join_with("AND", &pieces)) - } - Term::Or(children) => { - // An OR of pure-literal children collapses into one FTS OR-group. - if children.iter().all(|c| matches!(c, Term::Literal(_))) { - let group: Vec = children - .iter() - .map(|c| match c { - Term::Literal(s) => s.clone(), - _ => unreachable!(), - }) - .collect(); - self.fts_parts.push(FtsFragment::OrGroup(group)); - return Ok(String::new()); - } - // Mixed OR — each branch becomes a separate sub-Builder whose - // WHERE fragments we OR together. FTS branches cannot mix - // with structured branches cleanly at the SQL level here; - // keep it simple by requiring that mixed-OR branches produce - // structured-only WHERE fragments. - let mut pieces = Vec::new(); - for c in children { - let before = self.fts_parts.len(); - let p = self.translate(c)?; - if self.fts_parts.len() > before { - return Err(TranslateError::UnknownProperty( - "OR mixing FTS and structured terms is not supported in Set A".into(), - )); - } - if !p.is_empty() { - pieces.push(p); - } - } - Ok(format!("({})", join_with("OR", &pieces))) - } - } - } - - /// Delegate to the shared [`build_filter`] fragment builder, then - /// convert its anonymous `?` placeholders to this builder's numbered - /// scheme (params[0] is reserved for the FTS MATCH when one exists; - /// `build` shifts numbers afterwards). - fn translate_property( - &mut self, - key: &str, - op: Op, - value: &str, - ) -> Result { - let frag = build_filter(key, op, value, false)?; - let mut params = frag.params.into_iter(); - let mut out = String::with_capacity(frag.sql.len() + 8); - for c in frag.sql.chars() { - if c == '?' { - let v = params - .next() - .expect("FilterFragment placeholder/param counts match"); - self.all_params.push(v); - out.push('?'); - out.push_str(&self.all_params.len().to_string()); - } else { - out.push(c); - } - } - Ok(out) - } -} - /// A structured-filter fragment over table alias `f`: SQL with anonymous /// `?` placeholders plus the values they bind. Anonymous placeholders /// compose by simple appending — the search cascade tacks fragments onto @@ -295,9 +71,8 @@ pub fn is_filter_key(key: &str) -> bool { ) } -/// Translate one `key op value` filter into a [`FilterFragment`]. The -/// single source of filter semantics, shared by the legacy numbered -/// [`build`] path and the cascade's [`super::split`]. +/// Translate one `key op value` filter into a [`FilterFragment`]. The single +/// source of filter semantics, reached through [`super::split`]. /// /// `glob` marks a value whose unquoted `*` should act as a wildcard — only /// `name:`/`filename:` honor it; every other key treats the star literally. @@ -457,14 +232,6 @@ pub fn like_subtree_pattern(dir: &str) -> String { ) } -fn join_with(sep: &str, pieces: &[String]) -> String { - pieces - .iter() - .map(|p| p.clone()) - .collect::>() - .join(&format!(" {} ", sep)) -} - /// Parse a date string. Accepts `YYYY-MM-DD`. Returns unix seconds at 00:00 UTC. fn parse_date_to_unix(s: &str) -> Option { // Minimal parser: split on '-' into y/m/d integers. @@ -475,130 +242,144 @@ fn parse_date_to_unix(s: &str) -> Option { let y: i64 = parts[0].parse().ok()?; let m: u32 = parts[1].parse().ok()?; let d: u32 = parts[2].parse().ok()?; - if !(1..=12).contains(&m) || !(1..=31).contains(&d) || !(1970..=9999).contains(&y) { + if !(1..=12).contains(&m) || !(1970..=9999).contains(&y) { + return None; + } + // Against the real month length, not a flat 1..=31: `civil_to_unix` happily + // rolls 2024-02-31 over into March, so a typo would silently filter on a + // date the user never typed rather than being reported. + if d < 1 || d > days_in_month(y, m) { return None; } // Compute unix seconds using the days-since-epoch formula. Some(civil_to_unix(y, m as i64, d as i64)) } +/// Days in a Gregorian month, February by the leap rule. +fn days_in_month(year: i64, month: u32) -> u32 { + match month { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + 2 if year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) => 29, + 2 => 28, + _ => 0, + } +} + /// Howard Hinnant's civil-from-days algorithm, converting (year, month, day) /// in the Gregorian calendar to days since 1970-01-01. fn civil_to_unix(y: i64, m: i64, d: i64) -> i64 { let y = if m <= 2 { y - 1 } else { y }; let era = if y >= 0 { y } else { y - 399 } / 400; - let yoe = (y - era * 400) as i64; + let yoe = y - era * 400; let doy = (153 * (if m > 2 { m - 3 } else { m + 9 }) + 2) / 5 + d - 1; let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; let days = era * 146_097 + doe - 719_468; days * 86_400 } -/// Rewrite placeholder indices after an FTS MATCH parameter was inserted at -/// position 0. Called automatically in [`build`] when needed — but since we -/// append placeholders numerically during translation and prepend the FTS -/// param afterwards, every explicit `?N` in the generated SQL is off by one -/// when FTS is present. -fn shift_placeholders(sql: &str) -> String { - // Find every `?N` and bump N by 1. Only ASCII digits after `?`; skip - // anonymous `?` (which rusqlite won't mix with numbered anyway). - let bytes = sql.as_bytes(); - let mut out = String::with_capacity(sql.len()); - let mut i = 0; - while i < bytes.len() { - if bytes[i] == b'?' && i + 1 < bytes.len() && bytes[i + 1].is_ascii_digit() { - out.push('?'); - let mut j = i + 1; - while j < bytes.len() && bytes[j].is_ascii_digit() { - j += 1; - } - let n: u64 = std::str::from_utf8(&bytes[i + 1..j]) - .unwrap() - .parse() - .unwrap(); - out.push_str(&(n + 1).to_string()); - i = j; - } else { - out.push(bytes[i] as char); - i += 1; - } - } - out -} - - #[cfg(test)] mod tests { use super::*; - fn build_q(input: &str) -> SqlQuery { - parse_and_build(input, None, 0, Sort::None).expect("build") + /// The filter builder is reached through `split_for_cascade`, which is + /// where these values come from in production; calling it directly here + /// keeps the tests about filter semantics rather than about tokenizing. + fn frag(key: &str, op: Op, value: &str) -> FilterFragment { + build_filter(key, op, value, false).expect("builds") } #[test] - fn single_literal() { - let q = build_q("foo"); - assert!(q.sql.contains("searchabletext MATCH ?")); + fn type_filter_masks_the_bits() { + let f = frag("type", Op::Contains, "Audio"); + assert_eq!(f.sql, "(f.type & ?) != 0"); assert_eq!( - q.params, - vec![rusqlite::types::Value::Text("\"foo\"".into())] - ); - } - - #[test] - fn implicit_and_joins_as_fts_and() { - let q = build_q("foo bar"); - assert!(q.sql.contains("searchabletext MATCH ?")); - let m: &rusqlite::types::Value = &q.params[0]; - if let rusqlite::types::Value::Text(s) = m { - assert_eq!(s, "\"foo\" AND \"bar\""); - } else { - panic!(); - } - } - - #[test] - fn or_of_literals_becomes_fts_group() { - let q = build_q("a OR b"); - if let rusqlite::types::Value::Text(s) = &q.params[0] { - assert_eq!(s, "(\"a\" OR \"b\")"); - } else { - panic!(); - } - } - - #[test] - fn type_filter_only() { - let q = build_q("type:Audio"); - assert!(q.sql.contains("(f.type & ?")); - assert_eq!( - q.params, + f.params, vec![rusqlite::types::Value::Integer(FileType::AUDIO.bits() as i64)] ); } #[test] - fn modified_ge() { - let q = build_q("modified:>=2024-01-01"); - assert!(q.sql.contains("f.mtime >= ?")); - assert_eq!(q.params.len(), 1); + fn modified_comparisons_and_day_ranges() { + let f = frag("modified", Op::Ge, "2024-01-01"); + assert!(f.sql.contains("f.mtime >= ?")); + assert_eq!(f.params.len(), 1); + + // `=` means the whole day, not the second. + let f = frag("modified", Op::Eq, "2024-05-20"); + assert!(f.sql.contains("f.mtime >= ?") && f.sql.contains("f.mtime < ?")); + assert_eq!(f.params.len(), 2); } #[test] - fn modified_eq_is_day_range() { - let q = build_q("modified:=2024-05-20"); - assert!(q.sql.contains("f.mtime >= ?") && q.sql.contains("f.mtime < ?")); - assert_eq!(q.params.len(), 2); + fn path_filter_covers_the_folder_and_its_subtree() { + let f = frag("path", Op::Contains, "/home/me/docs"); + assert!(f.sql.contains("f.parent = ?")); + assert!(f.sql.contains("f.parent LIKE ?")); + // The LIKE half must declare its escape character; without the clause + // a Windows separator would be eaten as an escape. + assert!(f.sql.contains("ESCAPE '\\'"), "{}", f.sql); } #[test] - fn path_filter() { - let q = build_q("path:/home/me/docs"); - assert!(q.sql.contains("f.parent = ?")); - assert!(q.sql.contains("f.parent LIKE ?")); - // The LIKE half must be escaped and declare its escape character; - // without the clause a Windows separator would be eaten as an escape. - assert!(q.sql.contains("ESCAPE '\\'"), "{}", q.sql); + fn unsupported_shapes_error() { + assert!(matches!( + build_filter("artist", Op::Contains, "beatles", false), + Err(TranslateError::UnknownProperty(_)) + )); + assert!(matches!( + build_filter("modified", Op::Ge, "not-a-date", false), + Err(TranslateError::BadDate(_)) + )); + assert!(matches!( + build_filter("type", Op::Contains, "NotAThing", false), + Err(TranslateError::UnknownProperty(_)) + )); + // Only `name:` accepts a bare `contains`; an ordering operator on it + // is meaningless. + assert!(matches!( + build_filter("name", Op::Ge, "report", false), + Err(TranslateError::UnsupportedOp { .. }) + )); + } + + /// `civil_to_unix` rolls an impossible day over into the next month, so a + /// typo would otherwise filter on a date the user never typed — silently, + /// since the query still runs. + #[test] + fn impossible_dates_are_rejected_rather_than_rolled_over() { + for bad in [ + "2024-02-30", + "2023-02-29", // not a leap year + "2024-04-31", + "2024-01-00", + "2024-13-01", + "1969-12-31", // before the epoch + ] { + assert!( + parse_date_to_unix(bad).is_none(), + "{} should be rejected", + bad + ); + } + for good in ["2024-02-29", "2000-02-29", "2024-01-31", "2024-04-30"] { + assert!(parse_date_to_unix(good).is_some(), "{} should parse", good); + } + // Centurial leap rule: 1900 is not a leap year, 2000 is. + assert!(parse_date_to_unix("1900-02-29").is_none()); + } + + #[test] + fn is_filter_key_covers_exactly_the_supported_keys() { + for key in [ + "type", "modified", "mtime", "path", "folder", "includefolder", "name", "filename", + "mime", "TYPE", "Path", + ] { + assert!(is_filter_key(key), "{} should be a filter key", key); + } + for key in ["artist", "regex", "size", ""] { + assert!(!is_filter_key(key), "{} should not be a filter key", key); + } } /// The subtree pattern is the one place the separator and the LIKE escape @@ -693,31 +474,6 @@ mod tests { } } - #[test] - fn combined_fts_and_type() { - let q = build_q("type:Audio beatles"); - assert!(q.sql.contains("searchabletext MATCH ?")); - assert!(q.sql.contains("(f.type & ?")); - // FTS param first, then type bits. - assert_eq!(q.params.len(), 2); - assert_eq!( - q.params[0], - rusqlite::types::Value::Text("\"beatles\"".into()) - ); - } - - #[test] - fn unknown_property_errors() { - let err = parse_and_build("artist:beatles", None, 0, Sort::None).unwrap_err(); - assert!(matches!(err, TranslateError::UnknownProperty(_))); - } - - #[test] - fn bad_date_errors() { - let err = parse_and_build("modified:>=not-a-date", None, 0, Sort::None).unwrap_err(); - assert!(matches!(err, TranslateError::BadDate(_))); - } - #[test] fn civil_conversion() { // 2024-01-01 → 1704067200 unix @@ -728,122 +484,4 @@ mod tests { assert_eq!(civil_to_unix(2000, 2, 29), 951_782_400); } - #[test] - fn limit_offset_sort_render() { - let q = parse_and_build("foo", Some(10), 20, Sort::ByMtimeDesc).unwrap(); - assert!(q.sql.contains("ORDER BY f.mtime DESC")); - assert!(q.sql.contains("LIMIT 10")); - assert!(q.sql.contains("OFFSET 20")); - } - - // End-to-end: run a generated query against a real SQLite DB to verify - // placeholder shifting after FTS prepending is correct. - #[test] - fn end_to_end_combined_filter_executes() { - use crate::db::{ - open_or_recreate, - repo::{insert_file, set_content_done, NewFile}, - }; - use crate::mime::FileType; - use rusqlite::types::ToSql; - - let mut p = std::env::temp_dir(); - p.push(format!( - "qs-query-e2e-{}-{}.sqlite", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_nanos() - )); - let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); - - // Two audio files, one document. Only the "audio with beatles" file - // should be returned for `type:Audio beatles`. - let ids: Vec = { - let tx = conn.transaction().unwrap(); - let a = insert_file( - &tx, - &NewFile { - name: "beatles-track.mp3", - path: "/music/beatles-track.mp3", - parent: "/music", - size: 100, - mtime: 1_700_000_000, - inode: None, - device_id: None, - mime: Some("audio/mpeg"), - ftype: FileType::AUDIO, - hash: None, - }, - ) - .unwrap() - .expect("unique path"); - set_content_done( - &tx, - a, - "beatles-track.mp3", - "beatles hey jude", - &[("artist".into(), "The Beatles".into())], - true, - ) - .unwrap(); - let b = insert_file( - &tx, - &NewFile { - name: "bach.flac", - path: "/music/bach.flac", - parent: "/music", - size: 100, - mtime: 1_700_000_000, - inode: None, - device_id: None, - mime: Some("audio/flac"), - ftype: FileType::AUDIO, - hash: None, - }, - ) - .unwrap() - .expect("unique path"); - set_content_done(&tx, b, "bach.flac", "bach prelude", &[], true).unwrap(); - let c = insert_file( - &tx, - &NewFile { - name: "notes.txt", - path: "/docs/notes.txt", - parent: "/docs", - size: 50, - mtime: 1_700_000_000, - inode: None, - device_id: None, - mime: Some("text/plain"), - ftype: FileType::TEXT, - hash: None, - }, - ) - .unwrap() - .expect("unique path"); - set_content_done(&tx, c, "notes.txt", "beatles biography", &[], true).unwrap(); - tx.commit().unwrap(); - vec![a, b, c] - }; - - let q = parse_and_build("type:Audio beatles", None, 0, Sort::ByMtimeDesc).unwrap(); - let rows: Vec<(i64, String, String)> = { - let mut stmt = conn.prepare(&q.sql).expect("prepare"); - let bind: Vec<&dyn ToSql> = q.params.iter().map(|v| v as &dyn ToSql).collect(); - stmt.query_map(bind.as_slice(), |r| { - Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?)) - }) - .unwrap() - .map(|r| r.unwrap()) - .collect() - }; - - assert_eq!(rows.len(), 1, "expected only beatles-track.mp3; got {:?}", rows); - assert_eq!(rows[0].0, ids[0]); - - drop(conn); - std::fs::remove_file(&p).ok(); - } } diff --git a/crates/quicksearch-core/src/search/cascade.rs b/crates/quicksearch-core/src/search/cascade.rs index 0d020ee..80f8bb2 100644 --- a/crates/quicksearch-core/src/search/cascade.rs +++ b/crates/quicksearch-core/src/search/cascade.rs @@ -1,7 +1,6 @@ //! The ranked search cascade. //! -//! One term, four table scans, eleven ranks. Rank base = stage number, so -//! later stages only ever append to a rank-sorted result list: +//! One term, four table scans, eleven ranks. Rank base = stage number: //! //! | rank | meaning | scan | //! |-----:|----------------------------------|------| @@ -45,9 +44,23 @@ //! Every scan appends the caller's structured-filter SQL (anonymous //! placeholders over alias `f`) and checks the generation counter as it //! streams; a bumped generation aborts mid-statement. +//! +//! # Batches are ordered within themselves, not against each other +//! +//! A pass hands hits over *while it scans* (see [`FLUSH_INTERVAL`]) rather than +//! at the end. On a large index a single pass runs for seconds, and holding its +//! results back means the UI shows nothing for that whole time even though a +//! rank-1 filename match may have turned up in the first few milliseconds. +//! +//! The cost is that a scan finds hits in table order, not rank order: batch two +//! can contain something better than anything in batch one. Each batch is +//! sorted before it goes, but the consumer owns the ordering *across* batches — +//! the GUI keeps its table sorted by whichever column is keyed and places each +//! arrival where that sort requires. Do not assume arrival order is rank order. use std::collections::HashSet; use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; use rusqlite::Connection; @@ -206,6 +219,19 @@ struct Deferred { overflowed: bool, } +/// Longest a pass may sit on hits before handing them over. +/// +/// A pass is a whole-table scan, and on a large index that runs for seconds. +/// Waiting for it to finish means the GUI shows nothing for that whole time, +/// even when a rank-1 filename match turned up in the first few milliseconds. +/// Draining on a clock rather than only on a full buffer is what makes a +/// *sparse* query responsive too — six matches out of seven million still +/// paint as the scan reaches them. +/// +/// Short enough to land two or three batches inside the GUI's 250 ms result +/// fade, so the fade reveals a filling list rather than an empty one. +const FLUSH_INTERVAL: Duration = Duration::from_millis(80); + struct Cx<'a> { conn: &'a Connection, query: &'a CascadeQuery, @@ -223,6 +249,39 @@ struct Cx<'a> { sink: &'a mut dyn FnMut(Vec), } +/// Drives [`Cx::flush_if_due`]: when this pass last handed hits over, and +/// whether it has handed over anything at all. +struct FlushClock { + last: Instant, + sent_anything: bool, +} + +impl FlushClock { + fn new() -> FlushClock { + FlushClock { + last: Instant::now(), + sent_anything: false, + } + } + + /// Whether a buffer of `len` hits should go now. + /// + /// The first batch of a pass goes the moment there is anything to send, so + /// a single early hit paints immediately instead of waiting out an + /// interval it has no reason to. + fn due(&self, len: usize, batch: usize) -> bool { + if len == 0 { + return false; + } + !self.sent_anything || len >= batch || self.last.elapsed() >= FLUSH_INTERVAL + } + + fn mark_sent(&mut self) { + self.last = Instant::now(); + self.sent_anything = true; + } +} + impl<'a> Cx<'a> { fn cancelled(&self) -> bool { self.generation != self.latest_gen.load(Ordering::Relaxed) @@ -288,6 +347,24 @@ impl<'a> Cx<'a> { Ok(re.is_match(&String::from_utf8_lossy(&raw))) } + /// Hand `buf` over mid-scan if it is due, leaving it empty when it goes. + /// + /// Partial and final flushes are the same operation — [`flush_pass`] sorts + /// what it is given and emits it — so a pass streams simply by calling this + /// each time round its row loop. Ordering *between* batches is the caller's + /// problem: the GUI keeps its table sorted by whichever column is keyed, so + /// a better-ranked hit found later lands above the ones already shown. + fn flush_if_due(&mut self, buf: &mut Vec, clock: &mut FlushClock) { + if !clock.due(buf.len(), self.options.batch.max(1)) { + return; + } + let batch = std::mem::take(buf); + // `overflowed` belongs to the pass as a whole, not to one batch; the + // final flush reports it. + self.flush_pass(batch, false); + clock.mark_sent(); + } + /// Sort a finished pass buffer, truncate to what's left of the display /// limit, and stream it out in `options.batch`-sized events. fn flush_pass(&mut self, mut buf: Vec, overflowed: bool) { @@ -387,6 +464,7 @@ impl<'a> Cx<'a> { let mut overflowed = false; let mut path_overflowed = false; let mut scanned = 0usize; + let mut clock = FlushClock::new(); while let Some(row) = rows.next().map_err(|e| e.to_string())? { scanned += 1; if scanned % CANCEL_CHECK_ROWS == 0 && self.cancelled() { @@ -451,6 +529,7 @@ impl<'a> Cx<'a> { } else { buf.push(hit); overflowed |= self.enforce_cap(&mut buf); + self.flush_if_due(&mut buf, &mut clock); } } drop(rows); @@ -529,6 +608,7 @@ impl<'a> Cx<'a> { let snippet_opts = snippet::Options { approx_chars: SNIPPET_WINDOW_CHARS }; let mut buf: Vec = Vec::new(); let mut overflowed = false; + let mut clock = FlushClock::new(); while let Some(row) = rows.next().map_err(|e| e.to_string())? { // Decompression dominates: check every row. if self.cancelled() { @@ -546,14 +626,22 @@ impl<'a> Cx<'a> { let (rank, stage, snip) = match &text { Some(text) => { - let (count, stage, ci) = { + // Fold once. The case-insensitive count, the first-match + // search and the snippet extraction all need it, and each + // used to make its own copy of a document that can run to + // `maximum_text_size`. The trigram tokenizer is + // case-insensitive, so nearly every candidate takes this + // path — it is a per-row cost, not a per-hit one. + let mut folded: Option = None; + let (count, stage) = { let count_cs = pattern.count(text, false); if count_cs > 0 { - (count_cs, 5, false) + (count_cs, 5) } else { - let count_ci = pattern.count(text, true); + let lower = folded.insert(text.to_ascii_lowercase()); + let count_ci = pattern.count_folded(lower); if count_ci > 0 { - (count_ci, 6, true) + (count_ci, 6) } else { // Folded/unordered FTS candidate: the // pattern never occurs — drop it. @@ -563,9 +651,15 @@ impl<'a> Cx<'a> { }; // Literal terms keep the richer multi-occurrence // extract; a wildcard match marks its own first range. + let lower = folded.unwrap_or_else(|| text.to_ascii_lowercase()); let snip = match pattern.literal() { - Some(term) => Some(snippet::extract(text, &[term], &snippet_opts)), - None => pattern.find_first(text, ci).map(|r| { + Some(term) => Some(snippet::extract_folded( + text, + &lower, + &[term], + &snippet_opts, + )), + None => pattern.find_first_folded(&lower).map(|r| { let r = clamp_match_range(text, r, SNIPPET_WINDOW_CHARS); snippet::window_around(text, (r.start, r.end), &snippet_opts) }), @@ -601,6 +695,7 @@ impl<'a> Cx<'a> { snippet: snip, }); overflowed |= self.enforce_cap(&mut buf); + self.flush_if_due(&mut buf, &mut clock); } drop(rows); if self.cancelled() { @@ -644,6 +739,7 @@ impl<'a> Cx<'a> { let mut overflowed = false; let mut path_overflowed = false; let mut scanned = 0usize; + let mut clock = FlushClock::new(); while let Some(row) = rows.next().map_err(|e| e.to_string())? { scanned += 1; if scanned % 1024 == 0 && self.cancelled() { @@ -656,14 +752,18 @@ impl<'a> Cx<'a> { continue; } // The name is the better match when both fire, so it wins and - // only a name miss falls through to the path tier. + // only a name miss falls through to the path tier. Distance and + // match range come from one sweep — taking them separately meant a + // second full scan of the same buffer for every hit. let folded_name = name.to_ascii_lowercase(); - let (rank, field, folded_field) = match bitap.best_distance(folded_name.as_bytes()) { - Some(distance) => (7.0 + 0.1 * distance as f64, &name, folded_name), + let (rank, field, range) = match bitap.best_distance_and_first(folded_name.as_bytes()) { + Some((distance, range)) => (7.0 + 0.1 * distance as f64, &name, range), None if with_paths => { let folded_path = path.to_ascii_lowercase(); - match bitap.best_distance(folded_path.as_bytes()) { - Some(distance) => (11.0 + 0.1 * distance as f64, &path, folded_path), + match bitap.best_distance_and_first(folded_path.as_bytes()) { + Some((distance, range)) => { + (11.0 + 0.1 * distance as f64, &path, range) + } None => continue, } } @@ -675,15 +775,13 @@ impl<'a> Cx<'a> { // Mark the approximate matched span in the matched field for // the GUI's [matched field] rendering. window_around clamps // and aligns. - let snip = bitap.count_and_first(folded_field.as_bytes()).1.map(|range| { - snippet::window_around( - field, - range, - &snippet::Options { - approx_chars: field.len().saturating_mul(2).max(8), - }, - ) - }); + let snip = Some(snippet::window_around( + field, + range, + &snippet::Options { + approx_chars: field.len().saturating_mul(2).max(8), + }, + )); let is_path_tier = rank >= 11.0; let hit = SearchHit { file_id, @@ -701,6 +799,7 @@ impl<'a> Cx<'a> { } else { buf.push(hit); overflowed |= self.enforce_cap(&mut buf); + self.flush_if_due(&mut buf, &mut clock); } } drop(rows); @@ -746,6 +845,7 @@ impl<'a> Cx<'a> { let snippet_opts = snippet::Options { approx_chars: SNIPPET_WINDOW_CHARS }; let mut buf: Vec = Vec::new(); let mut overflowed = false; + let mut clock = FlushClock::new(); while let Some(row) = rows.next().map_err(|e| e.to_string())? { if self.cancelled() { return Ok(false); @@ -785,6 +885,7 @@ impl<'a> Cx<'a> { snippet: snip, }); overflowed |= self.enforce_cap(&mut buf); + self.flush_if_due(&mut buf, &mut clock); } drop(rows); if self.cancelled() { @@ -816,6 +917,7 @@ impl<'a> Cx<'a> { let mut overflowed = false; let mut path_overflowed = false; let mut scanned = 0usize; + let mut clock = FlushClock::new(); while let Some(row) = rows.next().map_err(|e| e.to_string())? { scanned += 1; if scanned % 1024 == 0 && self.cancelled() { @@ -858,6 +960,7 @@ impl<'a> Cx<'a> { } else { buf.push(hit); overflowed |= self.enforce_cap(&mut buf); + self.flush_if_due(&mut buf, &mut clock); } } drop(rows); @@ -890,6 +993,7 @@ impl<'a> Cx<'a> { let snippet_opts = snippet::Options { approx_chars: SNIPPET_WINDOW_CHARS }; let mut buf: Vec = Vec::new(); let mut overflowed = false; + let mut clock = FlushClock::new(); while let Some(row) = rows.next().map_err(|e| e.to_string())? { // Decompression dominates: check every row. if self.cancelled() { @@ -926,6 +1030,7 @@ impl<'a> Cx<'a> { snippet: snip, }); overflowed |= self.enforce_cap(&mut buf); + self.flush_if_due(&mut buf, &mut clock); } drop(rows); if self.cancelled() { diff --git a/crates/quicksearch-core/src/search/duplicates.rs b/crates/quicksearch-core/src/search/duplicates.rs index e47fe29..65791f7 100644 --- a/crates/quicksearch-core/src/search/duplicates.rs +++ b/crates/quicksearch-core/src/search/duplicates.rs @@ -115,6 +115,9 @@ mod tests { mime: None, ftype: FileType::TEXT, hash, + // No MIME, so nothing claims it — and duplicate detection + // never looks at content anyway. + needs_content: false, }, ) .unwrap() diff --git a/crates/quicksearch-core/src/search/fuzzy.rs b/crates/quicksearch-core/src/search/fuzzy.rs index 13e88c9..f44b4cf 100644 --- a/crates/quicksearch-core/src/search/fuzzy.rs +++ b/crates/quicksearch-core/src/search/fuzzy.rs @@ -11,6 +11,15 @@ //! convention). Patterns are limited to 64 bytes by the machine word; the //! cascade skips fuzzy stages for longer terms. +/// Registers the automaton needs: one per error count `0..=k`. +/// +/// `k` is bounded by [`edit_budget`]'s one-edit-per-three-characters ladder +/// against a pattern the machine word caps at 64 bytes, so it never exceeds 21 +/// however hostile `fuzzy_max_edits` is (pinned by +/// `edit_budget_stays_within_the_bitap_word_size`). [`Bitap::new`] rejects +/// anything larger, so the array is always big enough. +const MAX_REGISTERS: usize = 22; + pub struct Bitap { /// `masks[c]` has bit `i` set iff `pattern[i] == c`. masks: [u64; 256], @@ -22,9 +31,10 @@ pub struct Bitap { impl Bitap { /// `None` when the pattern is empty, longer than 64 bytes, or the edit - /// budget reaches the word size (`initial_registers` shifts by `k`). + /// budget does not fit the registers (`reset` shifts by `k`, and the + /// register array holds [`MAX_REGISTERS`]). pub fn new(pattern: &[u8], k: usize) -> Option { - if pattern.is_empty() || pattern.len() > 64 || k >= 64 { + if pattern.is_empty() || pattern.len() > 64 || k >= MAX_REGISTERS { return None; } let mut masks = [0u64; 256]; @@ -38,14 +48,19 @@ impl Bitap { }) } - /// Fresh per-distance state registers. Bit `i` of `r[d]` set means "a + /// Reset the per-distance state registers. Bit `i` of `r[d]` set means "a /// match of pattern[..=i] with ≤ d errors ends at the current text /// position". With d errors the first d pattern bytes can be deleted /// before any text is read, hence the pre-set low bits. - fn initial_registers(&self) -> Vec { - (0..=self.k) - .map(|d| if d == 0 { 0 } else { (1u64 << d) - 1 }) - .collect() + /// + /// Writes into a caller-owned array rather than returning a `Vec`: the + /// fuzzy passes call this once per scanned row — millions of them on a + /// whole-table sweep — and a heap allocation per row is pure overhead for + /// at most [`MAX_REGISTERS`] words. + fn reset(&self, r: &mut [u64; MAX_REGISTERS]) { + for (d, reg) in r.iter_mut().enumerate().take(self.k + 1) { + *reg = if d == 0 { 0 } else { (1u64 << d) - 1 }; + } } /// Advance all registers by one haystack byte. Returns the smallest @@ -78,15 +93,30 @@ impl Bitap { /// Minimum edit distance (≤ k) of any occurrence of the pattern in /// `hay`, or `None` if nothing matches within k edits. pub fn best_distance(&self, hay: &[u8]) -> Option { - let mut r = self.initial_registers(); - let mut best: Option = None; - for &b in hay { + self.best_distance_and_first(hay).map(|(d, _)| d) + } + + /// [`best_distance`](Self::best_distance) plus the first match's + /// approximate byte range, from one sweep. + /// + /// The filename pass needs both — the distance to rank by, the range to + /// mark in the snippet — and taking them separately meant a second full + /// scan of the same buffer for every hit. The range carries the same + /// caveat as [`count_and_first`](Self::count_and_first): it assumes a + /// pattern-length match, so edits can shift the true start by up to `k`. + pub fn best_distance_and_first(&self, hay: &[u8]) -> Option<(usize, (usize, usize))> { + let mut r = [0u64; MAX_REGISTERS]; + self.reset(&mut r); + let mut best: Option<(usize, (usize, usize))> = None; + for (i, &b) in hay.iter().enumerate() { if let Some(d) = self.step(&mut r, b) { + let end = i + 1; + let range = (end.saturating_sub(self.len), end); if d == 0 { - return Some(0); + return Some((0, range)); } - if best.map_or(true, |cur| d < cur) { - best = Some(d); + if best.is_none_or(|(cur, _)| d < cur) { + best = Some((d, range)); } } } @@ -100,7 +130,8 @@ impl Bitap { /// The reported range assumes pattern-length matches — edits can shift /// the true start by up to k bytes, which is fine for snippet windows. pub fn count_and_first(&self, hay: &[u8]) -> (usize, Option<(usize, usize)>) { - let mut r = self.initial_registers(); + let mut r = [0u64; MAX_REGISTERS]; + self.reset(&mut r); let mut count = 0usize; let mut first: Option<(usize, usize)> = None; for (i, &b) in hay.iter().enumerate() { @@ -110,9 +141,7 @@ impl Bitap { let end = i + 1; first = Some((end.saturating_sub(self.len), end)); } - for (d, reg) in r.iter_mut().enumerate() { - *reg = if d == 0 { 0 } else { (1u64 << d) - 1 }; - } + self.reset(&mut r); } } (count, first) @@ -183,10 +212,12 @@ mod tests { #[test] fn oversized_k_is_rejected_not_shifted() { - // `initial_registers` shifts by k; k >= 64 would overflow u64. + // `reset` shifts by k and writes k+1 registers, so both the u64 shift + // and the fixed array have to be respected. + assert!(Bitap::new(b"abc", MAX_REGISTERS).is_none()); assert!(Bitap::new(b"abc", 64).is_none()); assert!(Bitap::new(b"abc", usize::MAX).is_none()); - assert!(Bitap::new(b"abc", 63).is_some()); + assert!(Bitap::new(b"abc", MAX_REGISTERS - 1).is_some()); } #[test] diff --git a/crates/quicksearch-core/src/snippet.rs b/crates/quicksearch-core/src/snippet.rs index b67f1dc..b820d32 100644 --- a/crates/quicksearch-core/src/snippet.rs +++ b/crates/quicksearch-core/src/snippet.rs @@ -63,6 +63,18 @@ impl Snippet { /// `terms` (ASCII-case-insensitive). With no terms or no matches, returns /// the head of the text as the window with no ranges. pub fn extract(text: &str, terms: &[&str], opts: &Options) -> Snippet { + extract_folded(text, &text.to_ascii_lowercase(), terms, opts) +} + +/// [`extract`] against a haystack the caller has already ASCII-folded. +/// +/// The search cascade folds each document once per row and then counts, +/// searches and extracts from that one buffer; folding again here would copy +/// up to `maximum_text_size` a second time for every hit. `folded` must be +/// `text.to_ascii_lowercase()` — the fold is byte-length preserving, which is +/// what lets offsets found in it slice the original. +pub fn extract_folded(text: &str, folded: &str, terms: &[&str], opts: &Options) -> Snippet { + debug_assert_eq!(folded.len(), text.len(), "ASCII folding preserves length"); if text.is_empty() { return Snippet::empty(); } @@ -75,10 +87,9 @@ pub fn extract(text: &str, terms: &[&str], opts: &Options) -> Snippet { return head_window(text, opts.approx_chars); } - // Case-fold once; we do all positioning on the folded buffer and slice - // from the original. Both buffers have identical byte layout because - // `to_ascii_lowercase` only touches ASCII letters. - let folded = text.to_ascii_lowercase(); + // All positioning happens on the folded buffer; slices come from the + // original. Both have identical byte layout because `to_ascii_lowercase` + // only touches ASCII letters. let folded_bytes = folded.as_bytes(); let mut matches: Vec<(usize, usize)> = Vec::new(); diff --git a/crates/quicksearch-core/src/walk.rs b/crates/quicksearch-core/src/walk.rs index ea624a2..603781e 100644 --- a/crates/quicksearch-core/src/walk.rs +++ b/crates/quicksearch-core/src/walk.rs @@ -178,12 +178,12 @@ struct Shared { /// observational, for progress display: two relaxed atomic ops per /// *job* (a directory read plus up to [`FILES_PER_JOB`] files), so it /// costs nothing the queue mutex didn't already. - busy: AtomicUsize, + stats: WorkerStats, } /// Decrements the busy count however the worker leaves its job — including /// early returns on stop and panics. -struct BusyGuard<'a>(&'a AtomicUsize); +pub(crate) struct BusyGuard<'a>(&'a AtomicUsize); impl Drop for BusyGuard<'_> { fn drop(&mut self) { @@ -191,17 +191,32 @@ impl Drop for BusyGuard<'_> { } } -/// Lock-free view of walker activity for progress displays. +/// Lock-free view of a worker pool's activity for progress displays. +/// +/// Shared by the walk and by [`crate::content`]: a root runs one pool and then +/// the other, and the progress line has to report whichever is live — a pool +/// whose threads have exited reads as busy 0, which is only the truth while +/// that pool is the one running. #[derive(Clone)] pub struct WorkerStats { - shared: Arc, + busy: Arc, total: usize, } impl WorkerStats { + pub(crate) fn new(total: usize) -> Self { + WorkerStats { busy: Arc::new(AtomicUsize::new(0)), total } + } + + /// Count the calling thread as busy until the returned guard drops. + pub(crate) fn enter(&self) -> BusyGuard<'_> { + self.busy.fetch_add(1, Ordering::Relaxed); + BusyGuard(&self.busy) + } + /// Workers doing work right now (the rest are parked). pub fn active(&self) -> usize { - self.shared.busy.load(Ordering::Relaxed).min(self.total) + self.busy.load(Ordering::Relaxed).min(self.total) } pub fn total(&self) -> usize { @@ -378,7 +393,7 @@ struct Ctx { /// from bytes saves the content pass an open/read/close per file. registry: Arc, unreadable: UnreadableDirs, - stop_flag: Arc>, + stop_flag: Arc, suspend_flag: Arc, } @@ -458,16 +473,28 @@ fn read_directory( // directory now *should* lose its row. Ok(ft) if ft.is_dir() => found.push(Found::Dir(path)), Ok(ft) if ft.is_symlink() => { + // With links off there is nothing here to index — targets + // included. Returning before `canonicalize` also drops a + // readlink chain and a stat per symlink, so honouring the + // setting costs fewer syscalls than ignoring it, not more. + // + // Both kinds have to be gated together, or the two walkers + // disagree: `filtered_walk` (which the watcher and the + // incremental path use) passes `follow_links` straight to + // walkdir and follows neither. A file target followed here but + // not there is indexed by every full run and never updated + // between them — and, if it resolves outside every configured + // root, indexed despite living somewhere the user never asked + // us to look. + if !ctx.follow_symlinks { + continue; + } // Resolve aliases where they are found. The target's canonical // path is what the index stores, and pushing only canonical // directories is what keeps `seen_dirs` able to break cycles. if let Ok(target) = path.canonicalize() { match fs::metadata(&target) { - Ok(m) if m.is_dir() => { - if ctx.follow_symlinks { - found.push(Found::Dir(target)); - } - } + Ok(m) if m.is_dir() => found.push(Found::Dir(target)), // The row for a resolved target belongs to the // target's own directory, not this one, so it is not // marked present here and cannot be classified @@ -589,8 +616,7 @@ fn prepare(path: PathBuf, known: Known<'_>, ctx: &Ctx) -> WalkedFile { fn worker(shared: &Shared, ctx: &Ctx, tx: &mpsc::SyncSender) { while let Some((job, slot)) = shared.take() { - shared.busy.fetch_add(1, Ordering::Relaxed); - let _busy = BusyGuard(&shared.busy); + let _busy = shared.stats.enter(); if should_abort(&ctx.stop_flag, &ctx.suspend_flag) { shared.shutdown(); return; @@ -728,11 +754,11 @@ impl ParallelWalk { /// A cheap, cloneable handle for reading worker activity while the /// walk's iterator is mutably borrowed by a `for` loop. + /// + /// Meaningful only while the walk is running: once the workers exit, the + /// pool size stays but the busy count is permanently zero. pub fn worker_stats(&self) -> WorkerStats { - WorkerStats { - shared: self.shared.clone(), - total: self.handles.len(), - } + self.shared.stats.clone() } /// Join the workers and report whether every one of them finished @@ -766,27 +792,37 @@ impl ParallelWalk { } } -/// Result of a non-blocking pull from a walk. -pub enum TryNext { - Item(WalkEvent), - /// Nothing ready right now; the walk is still running. +/// Result of a non-blocking pull from a producer pool. +/// +/// Shared by every pass the writer loop multiplexes — the walk here and the +/// content pass in [`crate::content`] — so that draining one root's work reads +/// identically whichever pass it came from. +pub enum TryNext { + Item(T), + /// Nothing ready right now; the pass is still running. Empty, - /// The walk has ended (all workers exited, for any reason). + /// The pass has ended (all workers exited, for any reason). Finished, } +/// Translate a non-blocking channel pull into [`TryNext`]. `None` is a +/// receiver the owner already dropped, which reads as finished. +pub(crate) fn try_recv_next(rx: Option<&mpsc::Receiver>) -> TryNext { + match rx { + None => TryNext::Finished, + Some(rx) => match rx.try_recv() { + Ok(item) => TryNext::Item(item), + Err(mpsc::TryRecvError::Empty) => TryNext::Empty, + Err(mpsc::TryRecvError::Disconnected) => TryNext::Finished, + }, + } +} + impl ParallelWalk { /// Non-blocking variant of `next`, for callers multiplexing several /// walks (the per-root writer loop). - pub fn try_next(&mut self) -> TryNext { - match &self.rx { - None => TryNext::Finished, - Some(rx) => match rx.try_recv() { - Ok(file) => TryNext::Item(file), - Err(mpsc::TryRecvError::Empty) => TryNext::Empty, - Err(mpsc::TryRecvError::Disconnected) => TryNext::Finished, - }, - } + pub fn try_next(&mut self) -> TryNext { + try_recv_next(self.rx.as_ref()) } } @@ -823,7 +859,7 @@ pub fn walk_indexable_files( db_path: &str, config: Config, registry: Arc, - stop_flag: Arc>, + stop_flag: Arc, suspend_flag: Arc, workers: usize, ) -> ParallelWalk { @@ -862,7 +898,7 @@ pub fn walk_indexable_files( let shared = Arc::new(Shared { queue: Mutex::new(queue), idle: Condvar::new(), - busy: AtomicUsize::new(0), + stats: WorkerStats::new(threads), }); let ctx = Arc::new(Ctx { follow_symlinks, @@ -883,7 +919,12 @@ pub fn walk_indexable_files( let handles = (0..threads) .map(|_| { let (shared, ctx, tx) = (shared.clone(), ctx.clone(), tx.clone()); - thread::spawn(move || worker(&shared, &ctx, &tx)) + thread::spawn(move || { + // Walker threads are the bulk of a run's CPU and I/O; the + // foreground must stay ahead of them. + crate::platform::set_background_priority(); + worker(&shared, &ctx, &tx) + }) }) .collect(); // The workers must hold the only senders, or `recv` never reports the end @@ -893,7 +934,10 @@ pub fn walk_indexable_files( let prefetch = { let (shared, db_path) = (shared.clone(), db_path.to_string()); - thread::spawn(move || prefetcher(&shared, &db_path)) + thread::spawn(move || { + crate::platform::set_background_priority(); + prefetcher(&shared, &db_path) + }) }; ParallelWalk { rx: Some(rx), handles, prefetch: Some(prefetch), shared, ctx } @@ -999,7 +1043,7 @@ mod tests { db.to_str().unwrap(), Config::default(), Arc::new(Registry::default_set()), - Arc::new(Mutex::new(false)), + Arc::new(AtomicBool::new(false)), Arc::new(AtomicBool::new(false)), 4, )) @@ -1181,7 +1225,7 @@ mod tests { empty_db("unreadable-dir").to_str().unwrap(), Config::default(), Arc::new(Registry::default_set()), - Arc::new(Mutex::new(false)), + Arc::new(AtomicBool::new(false)), Arc::new(AtomicBool::new(false)), 4, ); @@ -1240,15 +1284,52 @@ mod tests { ) .unwrap(); - let files = walk(&root, &empty_db("symlink-file")); + let files = walk_with(&root, &empty_db("symlink-file"), true, false); let paths: HashSet<&String> = files.iter().map(|f| &f.path).collect(); assert_eq!(paths.len(), 1, "both routes report one canonical path"); let canonical = path_to_db_string(&root.join("real/target.txt").canonicalize().unwrap()); assert_eq!(*paths.into_iter().next().unwrap(), canonical, "the target, not the alias"); + + // The alias itself is still reported, so its row is never mistaken for + // deleted — it is reported under the *target's* path. + assert_eq!(files.len(), 2, "seen twice, spelled once"); + assert!(files.iter().any(|f| f.aliased), "the link route is marked"); fs::remove_dir_all(&root).ok(); } + /// The counterpart, and the reason both symlink kinds are gated together: + /// `filtered_walk` — which the watcher and the incremental path use — + /// passes `follow_links` to walkdir and follows neither kind. A file link + /// followed only here would be re-indexed by every full run and never + /// updated between them, and its target may sit outside every root. + #[test] + #[cfg(unix)] + fn a_file_symlink_is_not_followed_when_links_are_off() { + let root = tmp_tree("symlink-off"); + touch(&root.join("real/target.txt")); + fs::create_dir_all(root.join("links")).unwrap(); + std::os::unix::fs::symlink(root.join("real/target.txt"), root.join("links/alias.txt")) + .unwrap(); + // A target outside the walked tree: with links off it must not be + // reachable at all. + let outside = tmp_tree("symlink-off-outside"); + touch(&outside.join("elsewhere.txt")); + std::os::unix::fs::symlink(outside.join("elsewhere.txt"), root.join("links/out.txt")) + .unwrap(); + + let files = walk_with(&root, &empty_db("symlink-off"), false, false); + assert_eq!( + names(&files), + vec!["target.txt"], + "only the real file, reached directly" + ); + assert!(!files.iter().any(|f| f.aliased), "nothing was resolved"); + + fs::remove_dir_all(&root).ok(); + fs::remove_dir_all(&outside).ok(); + } + #[test] fn hidden_and_ignored_entries_are_pruned() { let root = tmp_tree("prune"); @@ -1272,7 +1353,7 @@ mod tests { empty_db("prune").to_str().unwrap(), Config::default(), Arc::new(Registry::default_set()), - Arc::new(Mutex::new(false)), + Arc::new(AtomicBool::new(false)), Arc::new(AtomicBool::new(false)), 4, )); @@ -1311,7 +1392,7 @@ mod tests { db.to_str().unwrap(), Config::default(), Arc::new(Registry::default_set()), - Arc::new(Mutex::new(false)), + Arc::new(AtomicBool::new(false)), Arc::new(AtomicBool::new(false)), 4, )); @@ -1348,7 +1429,7 @@ mod tests { db.to_str().unwrap(), Config::default(), Arc::new(Registry::default_set()), - Arc::new(Mutex::new(false)), + Arc::new(AtomicBool::new(false)), Arc::new(AtomicBool::new(false)), 4, )); @@ -1377,7 +1458,7 @@ mod tests { touch(&root.join(format!("f{:04}.txt", i))); } - let stop = Arc::new(Mutex::new(true)); + let stop = Arc::new(AtomicBool::new(true)); let files: Vec = files_only(walk_indexable_files( &[root.to_string_lossy().into_owned()], false, @@ -1412,7 +1493,7 @@ mod tests { empty_db("early-drop").to_str().unwrap(), Config::default(), Arc::new(Registry::default_set()), - Arc::new(Mutex::new(false)), + Arc::new(AtomicBool::new(false)), Arc::new(AtomicBool::new(false)), 4, ); @@ -1438,7 +1519,7 @@ mod tests { empty_db("overlap").to_str().unwrap(), Config::default(), Arc::new(Registry::default_set()), - Arc::new(Mutex::new(false)), + Arc::new(AtomicBool::new(false)), Arc::new(AtomicBool::new(false)), 4, )); @@ -1486,7 +1567,7 @@ mod tests { empty_db("finish").to_str().unwrap(), Config::default(), Arc::new(Registry::default_set()), - Arc::new(Mutex::new(false)), + Arc::new(AtomicBool::new(false)), Arc::new(AtomicBool::new(false)), 4, ); diff --git a/crates/quicksearch-core/src/watcher.rs b/crates/quicksearch-core/src/watcher.rs index a0300ef..6d7a02e 100644 --- a/crates/quicksearch-core/src/watcher.rs +++ b/crates/quicksearch-core/src/watcher.rs @@ -157,8 +157,25 @@ fn fmt_cap(cap: usize) -> String { } } +/// Tuning for the whole filesystem-event pipeline: the watcher's own +/// per-directory debounce, and the coordinator's queue on the far side of it. +/// Both halves are debounce knobs on one path, so they live together and +/// tests that want a fast pipeline shorten them in one place. #[derive(Debug, Clone)] pub struct WatcherConfig { + /// How long the coordinator's queue must go quiet before it is applied. + /// + /// An `rm -rf` arrives as a burst; applying it in waves as it lands means + /// no wave sees the whole set to collapse against, and each pays its own + /// costs. Waiting for quiet trades a little latency for one pass over the + /// complete picture. Creates and modifies benefit too — the queue is + /// last-event-wins per path, so a build that rewrites the same files + /// repeatedly collapses for free. + pub pending_settle: Duration, + /// Ceiling on how long [`WatcherConfig::pending_settle`] may hold the + /// queue back. A steady trickle of changes never goes quiet, and must not + /// starve. + pub pending_max_defer: Duration, /// Per-directory debounce window. Bursts of events in the same directory /// collapse to one flush after this interval of quiet. pub throttle_window: Duration, @@ -177,6 +194,8 @@ pub struct WatcherConfig { impl Default for WatcherConfig { fn default() -> Self { Self { + pending_settle: Duration::from_secs(2), + pending_max_defer: Duration::from_secs(30), throttle_window: Duration::from_secs(30), tick_interval: Duration::from_millis(500), max_dirs_per_tick: 64, @@ -244,6 +263,17 @@ impl WatchRegistry { /// directories on its own; unwatching anyway keeps notify's internal /// descriptor map from growing across a long session of directory churn. fn remove_tree(&mut self, dir: &Path) -> usize { + // The scan below is O(watched dirs), and the event loop calls this for + // every Remove — files included. Deleting a directory of 10k files + // would otherwise cost 10k passes over a set capped at 128k entries. + // + // Exact, not a heuristic: registration always walks top-down (see + // `register_tree` and startup), so a watched directory beneath `dir` + // implies `dir` itself is watched. An unwatched `dir` therefore has no + // watched descendants and there is nothing to find. + if !self.dirs.contains(dir) { + return 0; + } let doomed: Vec = self .dirs .iter() @@ -464,13 +494,8 @@ struct LoopCtx { degraded: Degraded, } -/// A queued event, deduplicated per path within a window. -#[derive(Debug, Clone)] -struct QueuedEvent { - op: QueuedOp, -} - -#[derive(Debug, Clone, Copy)] +/// A queued operation, deduplicated per path within a window. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] enum QueuedOp { Create, Modify, @@ -484,7 +509,7 @@ struct DirThrottleEntry { record_time: Instant, /// Per-path pending op. Same path seen twice in a window keeps only the /// latest op — coalescing a rename-as-create+modify spam into one event. - queue: HashMap, + queue: HashMap, /// If true, the next tick flushes regardless of window age. Set for the /// first event in a previously-idle directory so it reacts fast. immediate: bool, @@ -694,14 +719,12 @@ fn enqueue( immediate: true, }); // Coalesce: Remove after Create → drop both. Modify after Modify → one Modify. - match (op, entry.queue.get(&path).map(|q| q.op)) { + match (op, entry.queue.get(&path).copied()) { (QueuedOp::Remove, Some(QueuedOp::Create)) => { entry.queue.remove(&path); } _ => { - entry - .queue - .insert(path, QueuedEvent { op }); + entry.queue.insert(path, op); } } } @@ -726,11 +749,7 @@ fn flush_ready( } for dir in ready { if let Some(entry) = throttle.get_mut(&dir) { - let drained: Vec<(PathBuf, QueuedOp)> = entry - .queue - .drain() - .map(|(p, q)| (p, q.op)) - .collect(); + let drained: Vec<(PathBuf, QueuedOp)> = entry.queue.drain().collect(); entry.immediate = false; entry.record_time = now; for (path, op) in drained { @@ -801,6 +820,42 @@ mod tests { dir } + /// A registry with `dirs` seeded directly, so the pure set logic can be + /// tested without registering real kernel watches. + fn registry_with(dirs: &[&str]) -> WatchRegistry { + let raw = RecommendedWatcher::new(|_res| {}, NotifyConfig::default()).unwrap(); + WatchRegistry { + raw, + dirs: dirs.iter().map(PathBuf::from).collect(), + cap: 64, + } + } + + /// Most `Remove` events name *files*, which are never watched. Returning + /// before the O(watched) scan is what keeps deleting a 10k-file directory + /// off 10k passes over a set capped at 128k entries. + #[test] + fn remove_tree_skips_the_scan_for_a_path_that_is_not_watched() { + let mut reg = registry_with(&["/a/b", "/a/b/c", "/a/bc"]); + + // A file inside a watched directory: not watched itself, and nothing + // can live beneath it. + assert_eq!(reg.remove_tree(Path::new("/a/b/file.txt")), 0); + assert_eq!(reg.dirs.len(), 3, "the watch set is untouched"); + + // An entirely unrelated path is likewise a no-op. + assert_eq!(reg.remove_tree(Path::new("/elsewhere")), 0); + assert_eq!(reg.dirs.len(), 3); + + // The watched directory itself still takes its descendants with it — + // and only its descendants: /a/bc is a sibling, not a child. + assert_eq!(reg.remove_tree(Path::new("/a/b")), 2); + assert_eq!( + reg.dirs.iter().collect::>(), + vec![&PathBuf::from("/a/bc")] + ); + } + #[test] fn enqueue_create_then_remove_cancels() { let mut map: HashMap = HashMap::new(); @@ -866,10 +921,7 @@ mod tests { fn prune_stale_keeps_active_entries() { let mut map: HashMap = HashMap::new(); let mut queue = HashMap::new(); - queue.insert( - PathBuf::from("/tmp/a"), - QueuedEvent { op: QueuedOp::Modify }, - ); + queue.insert(PathBuf::from("/tmp/a"), QueuedOp::Modify); map.insert( PathBuf::from("/tmp"), DirThrottleEntry { diff --git a/crates/quicksearch-core/tests/cascade.rs b/crates/quicksearch-core/tests/cascade.rs index 33a78a5..4d1a383 100644 --- a/crates/quicksearch-core/tests/cascade.rs +++ b/crates/quicksearch-core/tests/cascade.rs @@ -55,6 +55,7 @@ impl Seeder { mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, + needs_content: true, }, ) .unwrap() @@ -73,20 +74,47 @@ impl Seeder { /// Run the cascade synchronously, collecting every batch. Returns /// (flattened hits in emission order, outcome). +/// Run a search and return its hits in rank order. +/// +/// The cascade streams batches *while* each pass scans, so arrival order is +/// table order, not rank order — batch two can hold something better than +/// anything in batch one. Ordering across batches belongs to the consumer, and +/// this mirrors what the GUI does with the default sort key, so the ranking +/// assertions below stay about ranking rather than about scan order. +/// +/// Use [`run_collect_batches`] to assert on the stream itself. fn run_collect( conn: &rusqlite::Connection, input: &str, options: &SearchOptions, ) -> (Vec, cascade::Outcome) { + let (batches, outcome) = run_collect_batches(conn, input, options); + let mut hits: Vec = batches.into_iter().flatten().collect(); + hits.sort_by(|a, b| { + a.rank + .partial_cmp(&b.rank) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.name.cmp(&b.name)) + .then_with(|| a.path.cmp(&b.path)) + }); + (hits, outcome) +} + +/// The raw batch stream, one `Vec` per sink call. +fn run_collect_batches( + conn: &rusqlite::Connection, + input: &str, + options: &SearchOptions, +) -> (Vec>, cascade::Outcome) { let split = split_for_cascade(input).expect("split"); let latest = AtomicU64::new(7); - let mut hits = Vec::new(); + let mut batches = Vec::new(); let outcome = cascade::run(conn, &split, options, 7, &latest, &mut |batch| { - hits.extend(batch) + batches.push(batch) }) .expect("cascade run") .expect("not cancelled"); - (hits, outcome) + (batches, outcome) } fn fuzzy_options() -> SearchOptions { @@ -1013,3 +1041,115 @@ fn service_reports_missing_db_as_error() { assert!(got_error, "missing index must surface as a search error"); service.shutdown(); } + +/// The point of the whole change: a pass hands hits over *while* it scans, so +/// the UI has something to show long before the scan ends. +/// +/// Proven by ordering rather than by batch count — `flush_pass` has always +/// chunked its output, so counting sink calls proves nothing. Pass A scans in +/// `files.path` order, so seeding a *worse* match at an early path and a +/// *better* one at a late path separates the two designs: emitting at the end +/// sorts them and leads with rank 1, while streaming hands over the rank-3 hit +/// before the scan has even reached the rank-1 one. +/// +/// That is the trade being made deliberately: arrival order is scan order, and +/// ordering across batches belongs to the consumer. +#[test] +fn a_pass_hands_hits_over_before_the_scan_reaches_the_end() { + let p = tmp_db("stream"); + let mut s = Seeder::new(&p, true); + let early_worse = s.add("my_zebra_file.txt", "/aaa", 1, None); // rank 3 + for i in 0..400 { + s.add(&format!("filler{:04}.txt", i), "/mmm", i as u64 + 2, None); + } + let late_better = s.add("zebra", "/zzz", 999, None); // rank 1 + let conn = s.done(); + + let options = SearchOptions { + batch: 100, + ..SearchOptions::default() + }; + let (batches, outcome) = run_collect_batches(&conn, "zebra", &options); + + assert_eq!(outcome.total, 2, "both matches still reach the sink"); + assert_eq!( + batches.first().map(|b| b.as_slice()).and_then(|b| b.first()).map(|h| h.file_id), + Some(early_worse), + "the early hit should have gone out before the scan found the better one; \ + batch sizes: {:?}", + batches.iter().map(|b| b.len()).collect::>() + ); + + // ...and sorting the stream the way the GUI does still puts rank 1 on top. + let (hits, _) = run_collect(&conn, "zebra", &options); + assert_eq!( + hits.iter().map(|h| h.file_id).collect::>(), + vec![late_better, early_worse], + "the consumer's sort restores rank order" + ); + + drop(conn); + std::fs::remove_file(&p).ok(); +} + +/// The time bound, which a size-only rule would miss: a query matching a +/// handful of rows out of many still paints them as the scan reaches them +/// rather than at the end. +#[test] +fn a_sparse_match_still_streams_before_the_scan_ends() { + let p = tmp_db("sparse"); + let mut s = Seeder::new(&p, true); + // Three needles, spread through a haystack far larger than one batch. + for i in 0..3000 { + let name = if i % 1000 == 500 { + format!("needle{:04}.txt", i) + } else { + format!("hay{:04}.txt", i) + }; + s.add(&name, "/d", i as u64 + 1, Some("body")); + } + let conn = s.done(); + + let options = SearchOptions { + batch: 100, + ..SearchOptions::default() + }; + let (batches, outcome) = run_collect_batches(&conn, "needle", &options); + + assert_eq!(outcome.total, 3, "all three needles found"); + assert_eq!(batches.iter().map(|b| b.len()).sum::(), 3); + assert!( + !batches.is_empty() && batches[0].len() < 3, + "the first needle should not have waited for the other two; batches: {:?}", + batches.iter().map(|b| b.len()).collect::>() + ); +} + +/// Streaming must not change *what* a search returns, only when. The rank +/// ordering assertions elsewhere in this file are the detailed version; this +/// pins the set and the count against a mixed-tier query. +#[test] +fn streaming_does_not_change_the_result_set() { + let p = tmp_db("stream-set"); + let mut s = Seeder::new(&p, true); + let exact = s.add("zebra", "/d", 1, Some("nothing here")); + let sub = s.add("my_zebra_file.txt", "/d", 2, Some("nothing here")); + let content = s.add("unrelated.txt", "/d", 3, Some("a zebra in the text")); + let conn = s.done(); + + for batch in [1usize, 2, 100] { + let options = SearchOptions { + batch, + ..SearchOptions::default() + }; + let (hits, outcome) = run_collect(&conn, "zebra", &options); + let ids: Vec = hits.iter().map(|h| h.file_id).collect(); + assert_eq!( + ids, + vec![exact, sub, content], + "batch size {} must not change ranking", + batch + ); + assert_eq!(outcome.total, 3, "batch size {}", batch); + } +} diff --git a/crates/quicksearch-core/tests/encrypted.rs b/crates/quicksearch-core/tests/encrypted.rs index 610ff09..11a8893 100644 --- a/crates/quicksearch-core/tests/encrypted.rs +++ b/crates/quicksearch-core/tests/encrypted.rs @@ -107,9 +107,50 @@ fn encrypted_index_lifecycle() { "plaintext content leaked into the encrypted file" ); + // --- Optimizing a keyed index: VACUUM keeps it encrypted. --- + // + // VACUUM rewrites the whole file through a temporary database that + // SQLCipher has to key from the main one. If it did not, the rewrite would + // hand back a plaintext index — silently, and only for protected users. + // + // The slack is manufactured: this tree is two files, and `maintain` only + // rewrites a file with something to reclaim. + { + let conn = db::open_existing(&db_path.to_string_lossy(), true).unwrap(); + conn.execute_batch( + "INSERT INTO files (name, path, parent, size, mtime, type, basic_state, content_state) + WITH RECURSIVE n(i) AS ( + SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 20000 + ) + SELECT 'p' || i, '/pad/' || i, '/pad', 0, 0, 0, 1, 3 FROM n; + DELETE FROM files WHERE parent = '/pad';", + ) + .unwrap(); + drop(conn); + + let conn = db::open::open_maintenance(&db_path.to_string_lossy()).unwrap(); + let dir = data.to_string_lossy().into_owned(); + assert!( + quicksearch_core::db::repo::maintain(&conn, &dir).unwrap(), + "that much slack should have been reclaimed" + ); + drop(conn); + + assert_ne!( + &header(&db_path), + b"SQLite format 3\0", + "the vacuum's replacement file must still be encrypted" + ); + assert_eq!( + match_count(&db_path, "zebrapayload"), + 1, + "and still searchable under the same key" + ); + } + // --- Wrong password / no password: tagged error, file intact. --- let before = std::fs::read(&db_path).unwrap(); - db::set_process_key(Some(wrong_key)); + db::set_process_key(Some(wrong_key.clone())); let err = db::verify_process_key(&db_path.to_string_lossy()).unwrap_err(); assert!(err.starts_with(db::KEY_MISMATCH_PREFIX), "got: {err}"); db::set_process_key(None); @@ -121,6 +162,56 @@ fn encrypted_index_lifecycle() { "failed unlocks must never modify the index" ); + // --- A stale schema must not read as a locked index. --- + // + // The reported failure: after a schema bump, a password-protected install + // could not start at all — the correct password was rejected with + // "not a compatible QuickSearch index (schema v4 expected)", because the + // unlock gate verified the key by opening the index the way a *consumer* + // does, which also insists the schema be current. An unprotected install + // in the same state starts and rebuilds on its first run; the protected + // one had no way past the gate. + // + // Whether the schema is current belongs to the indexer, which answers it + // by wiping and rebuilding. Unlocking only has to answer "does this key + // open the file?". + { + db::set_process_key(Some(key.clone())); + // Age the stored schema, exactly as a version bump would. + let conn = db::open_existing(&db_path.to_string_lossy(), true).unwrap(); + conn.execute( + "UPDATE schema_info SET value = '1' WHERE key = 'version'", + [], + ) + .unwrap(); + drop(conn); + + // The right password still unlocks... + db::verify_process_key(&db_path.to_string_lossy()).expect( + "a stale schema must not make the correct password look wrong", + ); + // ...and the wrong one is still refused, with the same tagged error — + // the relaxation must not have turned the check into a rubber stamp. + db::set_process_key(Some(wrong_key.clone())); + let err = db::verify_process_key(&db_path.to_string_lossy()).unwrap_err(); + assert!(err.starts_with(db::KEY_MISMATCH_PREFIX), "got: {err}"); + + // Consumers still refuse a stale index, which is what sends the + // indexer down its rebuild path. + db::set_process_key(Some(key.clone())); + let err = db::open_existing(&db_path.to_string_lossy(), false).unwrap_err(); + assert!(err.contains("not a compatible QuickSearch index"), "got: {err}"); + + // And the rebuild comes back encrypted and searchable under the same + // key, so the whole path a real user walks is covered. + index_once(&root, &db_path, &config); + assert_ne!(&header(&db_path), b"SQLite format 3\0"); + assert_eq!(match_count(&db_path, "zebrapayload"), 1); + + // Hand the next section the unkeyed state it expects. + db::set_process_key(None); + } + // --- Disable: delete + rebuild produces a plaintext index. --- let service = IndexingService::new(); service diff --git a/crates/quicksearch-core/tests/full_index.rs b/crates/quicksearch-core/tests/full_index.rs index c19ff64..2ddf486 100644 --- a/crates/quicksearch-core/tests/full_index.rs +++ b/crates/quicksearch-core/tests/full_index.rs @@ -7,10 +7,12 @@ //! stale — and only appears on the second run. use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use quicksearch_core::config::Config; -use quicksearch_core::indexing::{IndexingService, IndexingStatus}; +use quicksearch_core::file_handling::{extract_scope_prepare, ExtractCursor}; +use quicksearch_core::indexing::{IndexingService, IndexingStatus, RootPhase}; fn tmp_dir(tag: &str) -> PathBuf { let mut p = std::env::temp_dir(); @@ -270,6 +272,137 @@ fn stopping_mid_run_deletes_nothing() { std::fs::remove_dir_all(&db_dir).ok(); } +#[test] +fn a_stamped_run_has_finished_its_stale_cleanup() { + // `last_full_index` is what the coordinator schedules the next periodic + // reindex from. Stamping it for a run that was cut short suppresses + // reindexing for the whole interval (24 h by default) — and the damage is + // concrete: stale cleanup is skipped when the run is stopped, so rows for + // files that no longer exist stay in the index and keep turning up in + // search results until something else forces a rebuild. + // + // The hole this guards: the writer loop set `aborted` only at the *top* of + // an iteration, while the "every root is Done" exit sits at the bottom and + // breaks directly. A stop landing inside the pass — or inside stale cleanup + // itself, which returns early and leaves rows behind — reached that bottom + // break with `aborted` still false and stamped the run as complete. + // + // The assertion is one-sided on purpose, so timing can never make it fail + // spuriously: a stamp *always* has to mean cleanup finished, whether the + // stop landed inside the window or never landed at all. + let root = tmp_dir("stop-stamp"); + let db_dir = tmp_dir("stop-stamp-db"); + let db = db_dir.join("index.sqlite"); + let mut config = test_config(); + // Nothing to extract, so a root goes Walking → Done in one pass and the + // run's whole tail is the stale cleanup this test wants to interrupt. + config.processing.maximum_text_file_size = 0; + + const FILES: usize = 8000; + for i in 0..FILES { + touch(&root.join(format!("d{}/f{:05}.txt", i % 25, i)), b"body"); + } + index_once(&root, &db, &config); + assert_eq!(rows(&db).len(), FILES); + + // Every file vanishes, so the next run has FILES stale rows to delete — + // a tail long enough for a stop to land inside it. + for i in 0..FILES { + std::fs::remove_file(root.join(format!("d{}/f{:05}.txt", i % 25, i))).unwrap(); + } + + let marker = |db: &Path| -> Option { + let conn = rusqlite::Connection::open(db).ok()?; + quicksearch_core::db::repo::get_last_full_index(&conn) + }; + + for delay_ms in [2u64, 5, 10, 20, 35, 60, 100, 200] { + { + let conn = rusqlite::Connection::open(&db).unwrap(); + conn.execute("DELETE FROM schema_info WHERE key = 'last_full_index'", []) + .unwrap(); + } + assert_eq!(marker(&db), None, "stamp cleared before the run"); + + let service = IndexingService::new(); + service + .start_indexing( + vec![root.to_string_lossy().into_owned()], + db.to_string_lossy().into_owned(), + config.clone(), + ) + .unwrap(); + std::thread::sleep(Duration::from_millis(delay_ms)); + service.stop_indexing().unwrap(); + drop(service); + std::thread::sleep(Duration::from_millis(300)); + + if marker(&db).is_some() { + assert_eq!( + rows(&db).len(), + 0, + "delay {}ms: the run stamped itself complete but left stale rows behind", + delay_ms + ); + // Cleanup finished, so there is nothing left for later delays to + // interrupt; the rest of the sweep would be vacuous. + break; + } + } + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +#[test] +fn starting_a_run_claims_the_status_before_it_returns() { + // The coordinator enforces the single-writer rule by polling + // `get_status()`. That is only sound if the Running transition has already + // happened when `start_indexing` returns — it used to be performed by the + // service's command thread, *after* it joined the previous run's handle, + // so a caller could see Idle and start writing to the database this run is + // about to reopen (and possibly wipe). + let root = tmp_dir("start-claims"); + let db_dir = tmp_dir("start-claims-db"); + let db = db_dir.join("index.sqlite"); + let config = test_config(); + touch(&root.join("a.txt"), b"body"); + + let service = IndexingService::new(); + service + .start_indexing( + vec![root.to_string_lossy().into_owned()], + db.to_string_lossy().into_owned(), + config.clone(), + ) + .unwrap(); + + // No sleep, no poll: the very next observation must already be Running. + assert!( + matches!(service.get_status(), IndexingStatus::Running { .. }), + "status must be claimed synchronously, got {:?}", + service.get_status() + ); + + // And a second start is a reportable error rather than a silently + // dropped command. + let err = service + .start_indexing( + vec![root.to_string_lossy().into_owned()], + db.to_string_lossy().into_owned(), + config.clone(), + ) + .unwrap_err(); + assert!(err.contains("already running"), "got: {}", err); + + service.stop_indexing().unwrap(); + drop(service); + std::thread::sleep(Duration::from_millis(250)); + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + #[test] fn a_wide_tree_indexes_every_file_exactly_once() { // Exercises the parallel walk's chunking and termination against a real @@ -444,7 +577,10 @@ fn a_symlink_target_in_an_unwalked_directory_survives_reindexing() { let outside = tmp_dir("alias-outside"); let db_dir = tmp_dir("alias-db"); let db = db_dir.join("index.sqlite"); - let config = test_config(); + // Aliases only exist when links are followed; with the default (off) a + // symlink is not resolved at all, which the tail of this test checks. + let mut config = test_config(); + config.indexing.follow_symlinks = true; touch(&root.join("normal.txt"), b"inside the root"); @@ -471,6 +607,16 @@ fn a_symlink_target_in_an_unwalked_directory_survives_reindexing() { index_once(&root, &db, &config); assert_eq!(rows(&db), first, "an aliased row must survive a re-index"); + // And the other half of the setting: with links off, neither target is + // indexed — including the one outside the root, which the user never asked + // us to look at. This is also what keeps the full run in agreement with + // `filtered_walk`, which the watcher uses and which follows neither kind. + let db2 = db_dir.join("links-off.sqlite"); + index_once(&root, &db2, &test_config()); + let off: Vec = rows(&db2).into_iter().map(|(p, _, _)| p).collect(); + assert_eq!(off.len(), 1, "only the ordinary file: {:?}", off); + assert!(off[0].ends_with("normal.txt")); + std::fs::remove_dir_all(&root).ok(); std::fs::remove_dir_all(&outside).ok(); std::fs::remove_dir_all(&db_dir).ok(); @@ -486,7 +632,8 @@ fn a_modified_symlink_target_is_updated_not_silently_ignored() { let outside = tmp_dir("alias-mod-outside"); let db_dir = tmp_dir("alias-mod-db"); let db = db_dir.join("index.sqlite"); - let config = test_config(); + let mut config = test_config(); + config.indexing.follow_symlinks = true; let target = outside.join("target.txt"); touch(&target, b"first body"); @@ -735,6 +882,69 @@ fn undecodable_small_files_are_reported_as_failures_not_silently_skipped() { std::fs::remove_dir_all(&db_dir).ok(); } +/// End-to-end version of the fix: the extraction denominator the manage-index +/// tab renders is `extract_total`, and it must count files that need text — +/// not every indexed file. Asserted through a real `IndexingService` run so it +/// covers the walk, the batch writers and `extract_scope_prepare` together. +#[test] +fn the_extraction_denominator_counts_only_files_that_need_text() { + let root = tmp_dir("denominator"); + let db_dir = tmp_dir("denominator-db"); + let db = db_dir.join("index.sqlite"); + + // Three files an extractor claims, seven it never will. `big.txt` is the + // interesting one: larger than `hash_length`, so the walk cannot finish it + // inline and it is the only row the content pass actually opens. + for name in ["a.txt", "b.json"] { + touch(&root.join(name), b"body bytes with no magic"); + } + touch(&root.join("big.txt"), &vec![b'z'; 32 * 1024]); + for name in ["d.mp4", "e.zip", "f.bin", "g.exe", "h.iso", "i.so", "j"] { + touch(&root.join(name), b"body bytes with no magic"); + } + + let config = Config::default(); + index_once(&root, &db, &config); + + let conn = rusqlite::Connection::open(&db).unwrap(); + let count = |state: i64| -> i64 { + conn.query_row( + "SELECT COUNT(*) FROM files WHERE content_state = ?1", + [state], + |r| r.get(0), + ) + .unwrap() + }; + assert_eq!(count(0), 0, "a finished run leaves nothing pending"); + assert_eq!(count(1), 3, "the claimed files have text"); + assert_eq!(count(3), 7, "the rest are NA, and were NA from the walk on"); + drop(conn); + + // The exact call `indexing.rs` makes to fill `RootProgress::extract_total`, + // run against the index the full pass just produced. Asserted here rather + // than by sampling the live status, which cannot be observed reliably: a + // ten-file tree finishes between two polls. + let conn = Arc::new(Mutex::new( + // Writable: the scope call's first act is the idempotent oversize sweep. + quicksearch_core::db::open_existing(db.to_str().unwrap(), true).unwrap(), + )); + let cursor = ExtractCursor::for_root(root.to_str().unwrap()); + let scope = extract_scope_prepare(&conn, &cursor, &config).unwrap(); + assert_eq!( + (scope.pending, scope.already_done), + (0, 3), + "extract_total is the searchable set, not the file count" + ); + // Which is what the row renders: "3 / 3" on an unchanged re-run. Before + // this was decided at walk time it read "10 / 10", seven of them files + // with nothing to extract. + assert_eq!(scope.pending + scope.already_done, 3); + drop(conn); + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + #[test] fn an_empty_file_is_done_with_no_snippet_sidecar() { let root = tmp_dir("inline-empty"); @@ -822,3 +1032,258 @@ fn contentless_mode_still_indexes_inlined_files_without_storing_bodies() { std::fs::remove_dir_all(&root).ok(); std::fs::remove_dir_all(&db_dir).ok(); } + +/// A slow root must not stall the others. +/// +/// This is the complaint stated directly: one root doing heavy extraction used +/// to occupy the single writer thread — and the database connection — for a +/// whole batch of files at a time, during which no other root's walk was +/// drained at all. Their walker threads filled their channels and blocked. +/// +/// So the assertion is about *stalls*, not throughput. Throughput would be the +/// wrong measure: writing is serial by construction (one SQLite connection), +/// so on a local disk the writer, not extraction, is the bottleneck and a +/// wall-clock comparison would mostly measure the machine. +#[test] +fn a_heavy_root_does_not_stall_a_light_one() { + // HEAVY: few files, each big enough that reading it is real work, with a + // small `maximum_text_size` so the cost lands in extraction rather than in + // the writer's tokenising. + let heavy = tmp_dir("stall-heavy"); + let body: Vec = "sphinx of black quartz judge my vow ".repeat(40_000).into_bytes(); + for i in 0..200 { + touch(&heavy.join(format!("d{}/big{:04}.txt", i % 8, i)), &body); + } + // LIGHT: a wide tree of tiny files, so its walk runs long enough to sample + // and its progress counter moves finely. + let light = tmp_dir("stall-light"); + for i in 0..6000 { + touch(&light.join(format!("d{}/f{:05}.txt", i % 60, i)), b"x"); + } + + let db_dir = tmp_dir("stall-db"); + let db = db_dir.join("index.sqlite"); + let mut config = test_config(); + config.processing.maximum_text_size = 1024; + config.processing.maximum_text_file_size = 8 * 1024 * 1024; + + let service = IndexingService::new(); + service + .start_indexing( + vec![ + heavy.to_string_lossy().into_owned(), + light.to_string_lossy().into_owned(), + ], + db.to_string_lossy().into_owned(), + config.clone(), + ) + .unwrap(); + + // Sample the light root's progress while the heavy one is extracting, and + // keep the longest interval over which it did not move. + let mut worst = Duration::ZERO; + let mut last_change = Instant::now(); + let mut last_seen = 0usize; + let mut sampled_together = false; + let deadline = Instant::now() + Duration::from_secs(120); + while Instant::now() < deadline { + match service.get_status() { + IndexingStatus::Running { roots, .. } => { + let heavy_p = roots.iter().find(|r| r.root.contains("stall-heavy")); + let light_p = roots.iter().find(|r| r.root.contains("stall-light")); + if let (Some(h), Some(l)) = (heavy_p, light_p) { + let light_busy = l.phase != RootPhase::Done; + if h.phase == RootPhase::Extracting && light_busy { + sampled_together = true; + let now = l.walked + l.extracted; + if now != last_seen { + last_seen = now; + last_change = Instant::now(); + } else { + worst = worst.max(last_change.elapsed()); + } + } + } + } + IndexingStatus::Error(e) => panic!("indexing failed: {}", e), + _ => break, + } + std::thread::sleep(Duration::from_millis(2)); + } + service.stop_indexing().unwrap(); + drop(service); + + assert!( + sampled_together, + "never observed the two roots overlapping; the fixture is not exercising the case" + ); + // Measured on this fixture: ~20 ms with the extraction pools, ~130 ms when + // the file reading is forced back onto the writer thread (and unbounded in + // the real failure, where the heavy root is on a network share). The bound + // sits between, with several times the observed headroom. + // + // The fixed design's stall does not grow with the heavy root's cost — it is + // one round-robin pass plus one commit — so making that root heavier only + // widens the margin. + eprintln!("longest light-root stall while heavy extracted: {:?}", worst); + assert!( + worst < Duration::from_millis(100), + "the light root stalled for {:?} while the heavy root extracted", + worst + ); + + std::fs::remove_dir_all(&heavy).ok(); + std::fs::remove_dir_all(&light).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +/// The write-ahead log must not grow for the length of a run. +/// +/// SQLite's autocheckpoint copies committed frames into the index but can only +/// *reset* the log at an instant no reader holds a read mark — a lock it tries +/// once, without retrying. A run keeps a reader per root querying continuously, +/// so that instant does not come and the log appends until the run ends: the +/// case that prompted this was a 12.5 GiB index carrying a 21.6 GiB log. +/// +/// So the assertion is about the *peak while running*. It has to be sampled +/// in flight — `stop_indexing` and the post-run maintenance both truncate the +/// log on the way out, so a reading taken afterwards proves nothing about what +/// happened during. +#[test] +fn the_wal_stays_bounded_during_a_run() { + let root = tmp_dir("wal-bound"); + // Wide and text-heavy: every file lands in the FTS index, which is what + // actually fills the log. + let body: Vec = "sphinx of black quartz judge my vow ".repeat(200).into_bytes(); + for i in 0..4000 { + touch(&root.join(format!("d{}/f{:05}.txt", i % 40, i)), &body); + } + + let db_dir = tmp_dir("wal-bound-db"); + let db = db_dir.join("index.sqlite"); + let wal = db_dir.join("index.sqlite-wal"); + let mut config = test_config(); + // The floor `MINIMUM_WAL_SIZE` clamps to, so the cap is exercised many + // times over a fixture this size rather than once at the very end. + config.processing.maximum_wal_size = 16 * 1024 * 1024; + + let service = IndexingService::new(); + service + .start_indexing( + vec![root.to_string_lossy().into_owned()], + db.to_string_lossy().into_owned(), + config.clone(), + ) + .unwrap(); + + let mut peak = 0u64; + let mut checkpointed = false; + let mut last = 0u64; + let deadline = Instant::now() + Duration::from_secs(120); + while Instant::now() < deadline { + let len = std::fs::metadata(&wal).map(|m| m.len()).unwrap_or(0); + peak = peak.max(len); + // A drop in length is a checkpoint that ran mid-run; without one the + // bound below could be met simply by the fixture being too small. + if len + 1024 * 1024 < last { + checkpointed = true; + } + last = len; + match service.get_status() { + IndexingStatus::Running { .. } => {} + IndexingStatus::Error(e) => panic!("indexing failed: {}", e), + _ => break, + } + std::thread::sleep(Duration::from_millis(2)); + } + // Let the maintenance pass finish before tearing the service down. + let idle_by = Instant::now() + Duration::from_secs(120); + while Instant::now() < idle_by && !matches!(service.get_status(), IndexingStatus::Idle) { + std::thread::sleep(Duration::from_millis(10)); + } + let after = std::fs::metadata(&wal).map(|m| m.len()).unwrap_or(0); + drop(service); + + eprintln!("peak WAL during the run: {} bytes", peak); + assert!( + checkpointed, + "the log never shrank mid-run; the fixture is not exercising the cap" + ); + // Generously above the 16 MiB cap: the check runs between round-robin + // rounds, so a round's worth of commits can land on top of it, and a + // checkpoint that loses a lock race defers to the next cap of growth. + assert!( + peak < 96 * 1024 * 1024, + "the log peaked at {} bytes against a 16 MiB cap", + peak + ); + assert_eq!(after, 0, "the optimize pass leaves an empty log behind"); + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +/// Stopping a run does not skip the optimize pass. +/// +/// A run cut short is exactly when the log is at its largest and nothing else +/// will come along to land it: the writer connection closes, and the next run +/// may be hours away. So Stop ends the *indexing*, and the pass that follows +/// runs either way — visible as `Optimizing` until it is done. +#[test] +fn a_stopped_run_is_still_optimized() { + let root = tmp_dir("stop-optimize"); + let body: Vec = "sphinx of black quartz judge my vow ".repeat(200).into_bytes(); + for i in 0..4000 { + touch(&root.join(format!("d{}/f{:05}.txt", i % 40, i)), &body); + } + + let db_dir = tmp_dir("stop-optimize-db"); + let db = db_dir.join("index.sqlite"); + let wal = db_dir.join("index.sqlite-wal"); + + let service = IndexingService::new(); + service + .start_indexing( + vec![root.to_string_lossy().into_owned()], + db.to_string_lossy().into_owned(), + test_config(), + ) + .unwrap(); + + // Let it get far enough in to have written something worth landing. + let deadline = Instant::now() + Duration::from_secs(120); + while Instant::now() < deadline { + if std::fs::metadata(&wal).map(|m| m.len()).unwrap_or(0) > 512 * 1024 { + break; + } + if let IndexingStatus::Error(e) = service.get_status() { + panic!("indexing failed: {}", e); + } + std::thread::sleep(Duration::from_millis(2)); + } + service.request_stop(); + + let mut saw_optimizing = false; + let idle_by = Instant::now() + Duration::from_secs(120); + loop { + match service.get_status() { + IndexingStatus::Optimizing => saw_optimizing = true, + IndexingStatus::Idle => break, + IndexingStatus::Error(e) => panic!("indexing failed: {}", e), + _ => {} + } + assert!(Instant::now() < idle_by, "the stopped run never reached Idle"); + std::thread::sleep(Duration::from_millis(1)); + } + + assert!(saw_optimizing, "a stopped run must still publish Optimizing"); + assert_eq!( + std::fs::metadata(&wal).map(|m| m.len()).unwrap_or(0), + 0, + "the optimize pass must land the stopped run's log" + ); + drop(service); + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} diff --git a/crates/quicksearch-gui/src/app.rs b/crates/quicksearch-gui/src/app.rs index 6491bb5..5e097ae 100644 --- a/crates/quicksearch-gui/src/app.rs +++ b/crates/quicksearch-gui/src/app.rs @@ -4,11 +4,11 @@ use std::sync::mpsc; use std::time::{Duration, Instant}; -use quicksearch_core::cli::{index_counts, IndexCounts}; +use quicksearch_core::cli::IndexCounts; use quicksearch_core::config::{diff_actions, nested_roots, Config, SecurityConfig}; use quicksearch_core::coordinator::{IndexMode, IndexerState, WatcherStatus}; use quicksearch_core::db; -use quicksearch_core::indexing::{ConfigChange, IndexingStatus, RootPhase}; +use quicksearch_core::indexing::{overall_progress, ConfigChange, IndexingStatus, RootPhase}; use quicksearch_core::search::SearchOptions; use quicksearch_core::security::{derive_key, generate_salt, salt_to_hex, IndexKey}; use quicksearch_core::watcher::WatchError; @@ -22,6 +22,7 @@ use crate::logs_tab::LogsTab; use crate::manage_tab::ManageTab; use crate::options::{OptionsWindow, SecurityAction}; use crate::search_tab::SearchTab; +use crate::unlock::KeySource; #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Tab { @@ -50,6 +51,12 @@ pub struct QuickSearchApp { /// Nested roots found in the loaded config (startup validation); shown /// as a modal over the Manage tab until dismissed. nested_prompt: Option>, + /// How this session's key was obtained, for wording that refers to it. + key_source: KeySource, + /// Set when the index on disk was written by a different schema version + /// and the next run will replace it; see + /// [`QuickSearchApp::stale_index_prompt_ui`]. + stale_index_prompt: bool, /// Set when the watcher gave up on the directory budget and live /// updates are off; see [`QuickSearchApp::check_watch_cap_warning`]. watch_cap_prompt: Option, @@ -100,6 +107,7 @@ impl QuickSearchApp { cfg: Config, config_error: Option, initial_query: Option, + key_source: KeySource, ) -> Result { // Compact styling: results density is the whole point. ctx.style_mut(|style| { @@ -108,6 +116,13 @@ impl QuickSearchApp { }); ctx.set_zoom_factor(clamp_scale(cfg.ui.scale)); + // Probed *before* the backend exists. The coordinator can begin a full + // run — and with it the wipe — the moment it starts, and afterwards + // there is nothing left on disk to tell an upgrade apart from a fresh + // install. + let stale_index_prompt = + db::index_needs_rebuild(&cfg.resolved_database_path().to_string_lossy()); + let backend = Backend::start(&cfg, ctx.clone())?; let fuzzy = cfg.search.fuzzy_default; // Startup validation: a hand-edited config can nest roots, which @@ -137,6 +152,8 @@ impl QuickSearchApp { rebuild_prompt: None, clear_prompt: false, nested_prompt, + key_source, + stale_index_prompt, watch_cap_prompt: None, security_prompt: None, config_error, @@ -259,6 +276,28 @@ impl QuickSearchApp { Err(_) => break, } } + // Status-bar counts worker. + if let Some(rx) = &self.backend.counts_job { + match rx.try_recv() { + Ok(counts) => { + self.counts = Some((Instant::now(), counts)); + self.backend.counts_job = None; + } + Err(mpsc::TryRecvError::Empty) => {} + // The worker gave up (missing or unreadable index). Keep the + // last known figures but restamp them, so the next attempt + // waits its turn instead of respawning a thread every frame. + Err(mpsc::TryRecvError::Disconnected) => { + let last = self.counts.map(|(_, c)| c).unwrap_or(IndexCounts { + files: 0, + content_done: 0, + content_pending: 0, + }); + self.counts = Some((Instant::now(), last)); + self.backend.counts_job = None; + } + } + } // Duplicates worker. if let Some(rx) = &self.backend.dup_job { match rx.try_recv() { @@ -349,14 +388,10 @@ impl QuickSearchApp { .map(|(at, _)| at.elapsed() > Duration::from_secs(5)) .unwrap_or(true); if stale { - let db = self.cfg.resolved_database_path(); - let counts = - index_counts(&db.to_string_lossy()).unwrap_or(IndexCounts { - files: 0, - content_done: 0, - content_pending: 0, - }); - self.counts = Some((Instant::now(), counts)); + // Kicked off, not awaited: the result lands through + // `drain_events` on a later frame. + let cfg = self.cfg.clone(); + self.backend.start_index_counts(&cfg, ctx.clone()); } let files = self.counts.map(|(_, c)| c.files).unwrap_or(0); ui.label( @@ -377,25 +412,25 @@ impl QuickSearchApp { IndexingStatus::Stopping => { ui.label(egui::RichText::new("Stopping indexing…").small()); } + IndexingStatus::Optimizing => { + ui.label(egui::RichText::new("Optimizing index…").small()); + } IndexingStatus::Running { roots, .. } => { let done = roots.iter().filter(|r| r.phase == RootPhase::Done).count(); - let processed: usize = roots.iter().map(|r| r.walked + r.extracted).sum(); - let totals_known = roots.iter().all(|r| r.walk_total.is_some()); - let denominator: usize = roots - .iter() - .map(|r| r.walk_total.unwrap_or(0) + r.extract_total) - .sum(); + let progress = overall_progress(roots); + let frac = progress.fraction(); - let mut text = if totals_known && denominator > 0 { - let frac = (processed as f64 / denominator as f64).min(1.0); - format!( + let mut text = match (progress.total, frac) { + (Some(total), Some(frac)) => format!( "Indexing {} / {} ({:.0}%)", - group_thousands(processed as u64), - group_thousands(denominator as u64), + group_thousands(progress.processed as u64), + group_thousands(total as u64), frac * 100.0 - ) - } else { - format!("Indexing · {} files", group_thousands(processed as u64)) + ), + _ => format!( + "Indexing · {} files", + group_thousands(progress.processed as u64) + ), }; if roots.len() > 1 { text.push_str(&format!(" · {}/{} roots done", done, roots.len())); @@ -409,11 +444,13 @@ impl QuickSearchApp { text.push_str(&format!(" · {}/{} workers", active, total_workers)); } ui.label(egui::RichText::new(text).small()); - if totals_known && denominator > 0 { - let frac = (processed as f32 / denominator as f32).clamp(0.0, 1.0); - ui.add(egui::ProgressBar::new(frac).desired_width(120.0)); - } else { - ui.add(egui::Spinner::new().size(12.0)); + match frac { + Some(frac) => { + ui.add(egui::ProgressBar::new(frac as f32).desired_width(120.0)); + } + None => { + ui.add(egui::Spinner::new().size(12.0)); + } } } } @@ -781,6 +818,26 @@ impl QuickSearchApp { } } + /// Tell the user their index is being replaced because it belongs to an + /// older version — rather than letting them discover it as a run of failed + /// searches. + /// + /// This is the one modal that is not a question. The old index genuinely + /// cannot be read by this build, so there is no "keep it" branch to offer; + /// the button starts the rebuild rather than merely dismissing, which is + /// what makes the promise true in manual mode as well as automatic. + fn stale_index_prompt_ui(&mut self, ctx: &egui::Context) { + if !self.stale_index_prompt { + return; + } + if stale_index_window(ctx, self.key_source) { + self.stale_index_prompt = false; + self.backend.coordinator.rebuild_index(); + self.counts = None; + self.dups.state = DupState::NotLoaded; + } + } + fn watch_cap_prompt_ui(&mut self, ctx: &egui::Context) { let Some(reason) = &self.watch_cap_prompt else { return; @@ -904,6 +961,60 @@ fn pin_live_fields(new: &mut Config, live: &Config) { new.indexing.auto_index = live.indexing.auto_index; } +/// The stale-index window's body. Returns whether the user asked for the +/// rebuild. +/// +/// Split out from the method so it can be rendered, and its button clicked, in +/// a headless egui context — building a whole [`QuickSearchApp`] would start a +/// coordinator and a watcher. +fn stale_index_window(ctx: &egui::Context, key_source: KeySource) -> bool { + let mut rebuild = false; + egui::Window::new("Index reset for this version") + .collapsible(false) + .resizable(false) + .anchor(egui::Align2::CENTER_CENTER, [0.0, 0.0]) + .show(ctx, |ui| { + ui.set_max_width(440.0); + ui.label( + "Your search index was created by an older version of QuickSearch, \ + which this version cannot read. It is being reset and rebuilt from \ + scratch. Don't worry, this is very Quick!", + ); + // Only says "you just entered" when that actually happened: with + // the key remembered on this device there was no prompt to type + // into, and naming one the user never saw would just confuse. + let reassurance = match key_source { + KeySource::Unprotected => None, + KeySource::Prompt => Some( + "The rebuilt index is encrypted with the password you just \ + entered, so it stays password protected.", + ), + KeySource::Keychain => Some( + "The rebuilt index is encrypted with the key remembered on \ + this device, so it stays password protected.", + ), + }; + if let Some(text) = reassurance { + ui.add_space(4.0); + ui.label(text); + } + ui.add_space(4.0); + ui.label( + egui::RichText::new( + "Your files are not touched. Searches return incomplete results \ + until the rebuild finishes; progress is on the Manage Index tab.", + ) + .small() + .weak(), + ); + ui.add_space(4.0); + if ui.button("Rebuild now").clicked() { + rebuild = true; + } + }); + rebuild +} + /// A stored/current config value for the rebuild prompt; list values are /// already newline-joined and render as-is, empty means unset. fn display_value(value: &str) -> String { @@ -1034,6 +1145,10 @@ impl eframe::App for QuickSearchApp { self.security_prompt_ui(ctx); self.clear_prompt_ui(ctx); self.nested_prompt_ui(ctx); + // Ahead of the watch-cap warning: on a fresh upgrade both can be true, + // and "your index is being rebuilt" is the one that explains what the + // user is actually looking at. + self.stale_index_prompt_ui(ctx); self.watch_cap_prompt_ui(ctx); } @@ -1046,6 +1161,68 @@ impl eframe::App for QuickSearchApp { mod tests { use super::*; + fn frame(ctx: &egui::Context, source: KeySource, events: Vec) -> bool { + let input = egui::RawInput { + screen_rect: Some(egui::Rect::from_min_size( + egui::Pos2::ZERO, + egui::vec2(1000.0, 700.0), + )), + events, + ..Default::default() + }; + let mut clicked = false; + let _ = ctx.run(input, |ctx| clicked = stale_index_window(ctx, source)); + clicked + } + + fn click_at(pos: egui::Pos2) -> Vec { + [true, false] + .into_iter() + .map(|pressed| egui::Event::PointerButton { + pos, + button: egui::PointerButton::Primary, + pressed, + modifiers: egui::Modifiers::default(), + }) + .collect() + } + + /// The modal the user sees after unlocking onto an index from an older + /// version. It must render under every key source — each produces a + /// different height — and its one button must actually report the click, + /// since that click is what starts the rebuild. A dead button would leave + /// the promise ("it is being rebuilt") unkept in manual mode. + #[test] + fn the_stale_index_modal_renders_and_its_button_fires() { + for source in [ + KeySource::Unprotected, + KeySource::Prompt, + KeySource::Keychain, + ] { + let ctx = egui::Context::default(); + assert!( + !frame(&ctx, source, Vec::new()), + "an untouched frame must not request a rebuild" + ); + + // Sweep the window for the button rather than hard-coding its + // position: it is left-aligned at the bottom of a centre-anchored + // window whose height depends on which sentence is shown, and + // pinning coordinates would make this a layout test. + let mut fired = None; + 'sweep: for y in (230..480).step_by(3) { + for x in (250..760).step_by(6) { + let pos = egui::pos2(x as f32, y as f32); + if frame(&ctx, source, click_at(pos)) { + fired = Some(pos); + break 'sweep; + } + } + } + assert!(fired.is_some(), "no clickable Rebuild button for {source:?}"); + } + } + #[test] fn a_stale_draft_cannot_revert_the_indexing_mode_or_security() { // The draft as it was when the editor last synced: automatic diff --git a/crates/quicksearch-gui/src/backend.rs b/crates/quicksearch-gui/src/backend.rs index cd83996..c45901a 100644 --- a/crates/quicksearch-gui/src/backend.rs +++ b/crates/quicksearch-gui/src/backend.rs @@ -7,6 +7,7 @@ use std::sync::{mpsc, Arc}; +use quicksearch_core::cli::IndexCounts; use quicksearch_core::config::Config; use quicksearch_core::coordinator::IndexCoordinator; use quicksearch_core::search::{DuplicateGroup, SearchService, SearchUpdate}; @@ -17,6 +18,8 @@ pub struct Backend { pub search: Option, pub search_rx: mpsc::Receiver, pub dup_job: Option, String>>>, + /// In-flight status-bar count; see [`Backend::start_index_counts`]. + pub counts_job: Option>, } impl Backend { @@ -37,9 +40,35 @@ impl Backend { search: Some(search), search_rx, dup_job: None, + counts_job: None, }) } + /// Refresh the status bar's "N files indexed" on a worker thread. + /// + /// Three `COUNT(*)` scans, and on a multi-million-row index the unfiltered + /// one alone reads the whole primary key. Running it inline in `update()` + /// froze a frame every refresh for a number that is purely decorative — so + /// it goes the same way the duplicates scan does. No-op while one is + /// already in flight. + pub fn start_index_counts(&mut self, config: &Config, ctx: egui::Context) { + if self.counts_job.is_some() { + return; + } + let (tx, rx) = mpsc::channel(); + let db = config.resolved_database_path(); + std::thread::spawn(move || { + // A missing or unreadable index is not worth reporting here: the + // status bar has nothing useful to say about it that the indexing + // state does not already say. + if let Ok(counts) = quicksearch_core::cli::index_counts(&db.to_string_lossy()) { + let _ = tx.send(counts); + } + ctx.request_repaint(); + }); + self.counts_job = Some(rx); + } + pub fn search(&self) -> &SearchService { self.search.as_ref().expect("search service alive") } diff --git a/crates/quicksearch-gui/src/main.rs b/crates/quicksearch-gui/src/main.rs index cd49408..317993f 100644 --- a/crates/quicksearch-gui/src/main.rs +++ b/crates/quicksearch-gui/src/main.rs @@ -77,8 +77,17 @@ fn main() { // verified key means no prompt at all. Anything else starts locked — // the unlock screen owns password entry, bad-salt reporting, and the // forgot-password escape hatch. No index is touched until unlocked. - let start_unlocked = - !config.security.password_protected || unlock::try_keychain_unlock(&config); + // + // `None` means "start locked". The two unlocked cases stay distinct + // rather than collapsing to a bool, because anything the app later says + // *about* the key has to know whether the user typed one. + let key_source = if !config.security.password_protected { + Some(unlock::KeySource::Unprotected) + } else if unlock::try_keychain_unlock(&config) { + Some(unlock::KeySource::Keychain) + } else { + None + }; let native_options = eframe::NativeOptions { viewport: egui::ViewportBuilder::default() @@ -94,11 +103,12 @@ fn main() { "QuickSearch", native_options, Box::new(move |cc| { - let gate = if start_unlocked { - unlock::Gate::running(&cc.egui_ctx, config, config_error, initial_query) - .map_err(Box::::from)? - } else { - unlock::Gate::locked(config, config_error, initial_query) + let gate = match key_source { + Some(source) => { + unlock::Gate::running(&cc.egui_ctx, config, config_error, initial_query, source) + .map_err(Box::::from)? + } + None => unlock::Gate::locked(config, config_error, initial_query), }; Ok(Box::new(gate) as Box) }), diff --git a/crates/quicksearch-gui/src/manage_tab.rs b/crates/quicksearch-gui/src/manage_tab.rs index 833ff8c..802ac97 100644 --- a/crates/quicksearch-gui/src/manage_tab.rs +++ b/crates/quicksearch-gui/src/manage_tab.rs @@ -1,11 +1,14 @@ //! The Manage Index tab: detailed status, mode controls, indexed roots, //! and the content/ignore filter editors. +use std::path::Path; +use std::time::{Duration, Instant}; + use quicksearch_core::config::Config; use quicksearch_core::coordinator::{IndexMode, IndexerState, WatcherStatus}; use quicksearch_core::indexing::{IndexingStatus, RootPhase, RootProgress}; -use crate::format::{fmt_interval, fmt_rate, group_thousands, middle_truncate}; +use crate::format::{fmt_interval, fmt_rate, group_thousands, human_size, middle_truncate}; use crate::tracker::SpeedTracker; /// What the tab asks the app to do after this frame. @@ -35,6 +38,8 @@ pub struct ManageTab { baseline: Option, /// Draft of the roots/filters edited in-place. draft: Option, + /// Cached on-disk footprint of the index, restatted on a timer. + db_size: DbSizeProbe, } impl ManageTab { @@ -47,6 +52,7 @@ impl ManageTab { root_error: None, baseline: None, draft: None, + db_size: DbSizeProbe::default(), } } @@ -59,7 +65,9 @@ impl ManageTab { let total: usize = roots.iter().map(|r| r.walked + r.extracted).sum(); self.speed.record(total); } - IndexingStatus::Idle | IndexingStatus::Error(_) => self.speed.reset(), + IndexingStatus::Idle | IndexingStatus::Error(_) | IndexingStatus::Optimizing => { + self.speed.reset() + } _ => {} } } @@ -121,7 +129,14 @@ impl ManageTab { .auto_shrink([false; 2]) .show(ui, |ui| { // --- Status --------------------------------------------------- - ui.heading(egui::RichText::new("Status").strong()); + ui.horizontal(|ui| { + ui.heading(egui::RichText::new("Status").strong()); + // Right-aligned, clear of the progress text below it — + // that text changes width every frame during a run. + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + db_size_label(ui, self.db_size.size(config, Instant::now())); + }); + }); status_panel(ui, state, &self.speed); watch_panel(ui, state, config); ui.add_space(8.0); @@ -232,6 +247,8 @@ impl ManageTab { storage, 16 on network mounts. Takes effect on \ the next indexing run.", ); + #[cfg(test)] + tests::record_widget("workers", &response); if response.changed() { if workers == 0 { draft.indexing.root_workers.remove(root); @@ -280,9 +297,11 @@ impl ManageTab { } } }); - if let Some(err) = &self.root_error { - ui.colored_label(ui.visuals().error_fg_color, err); - } + crate::ui_util::stable_section(ui, |ui| { + if let Some(err) = &self.root_error { + ui.colored_label(ui.visuals().error_fg_color, err); + } + }); ui.separator(); // --- Filters --------------------------------------------------- @@ -297,27 +316,32 @@ impl ManageTab { ); cols[1].label("Ignore patterns (excluded entirely):"); let mut remove_pat: Option = None; - for (i, pat) in draft.indexing.ignore_patterns.iter().enumerate() { - cols[1].horizontal(|ui| { - ui.with_layout( - egui::Layout::right_to_left(egui::Align::Center), - |ui| { - if ui.small_button("Remove").clicked() { - remove_pat = Some(i); - } - ui.with_layout( - egui::Layout::left_to_right(egui::Align::Center), - |ui| { - ui.monospace(pat); - }, - ); - }, - ); - }); - } - if draft.indexing.ignore_patterns.is_empty() { - cols[1].label(egui::RichText::new("No ignore patterns.").small().weak()); - } + // The list grows and shrinks — including from outside + // this tab — so it is kept off the id of the editor + // below it (see `ui_util::stable_section`). + crate::ui_util::stable_section(&mut cols[1], |ui| { + for (i, pat) in draft.indexing.ignore_patterns.iter().enumerate() { + ui.horizontal(|ui| { + ui.with_layout( + egui::Layout::right_to_left(egui::Align::Center), + |ui| { + if ui.small_button("Remove").clicked() { + remove_pat = Some(i); + } + ui.with_layout( + egui::Layout::left_to_right(egui::Align::Center), + |ui| { + ui.monospace(pat); + }, + ); + }, + ); + }); + } + if draft.indexing.ignore_patterns.is_empty() { + ui.label(egui::RichText::new("No ignore patterns.").small().weak()); + } + }); if let Some(i) = remove_pat { draft.indexing.ignore_patterns.remove(i); } @@ -363,13 +387,13 @@ impl ManageTab { ); ui.add_space(8.0); - if ui - .add(crate::ui_util::bordered_button( - "Apply & Save", - crate::ui_util::BLUE, - )) - .clicked() - { + let apply = ui.add(crate::ui_util::bordered_button( + "Apply & Save", + crate::ui_util::BLUE, + )); + #[cfg(test)] + tests::record_widget("apply", &apply); + if apply.clicked() { let mut new_config = draft.clone(); new_config.indexing.content_extensions = parse_lines(&self.ext_filter_text); let roots = new_config.paths.indexing_paths.clone(); @@ -383,10 +407,111 @@ impl ManageTab { }); crate::ui_util::more_below_hint(ui, &scroll); + // Nothing else asks for repaints while the app sits idle, so without + // this the size would freeze at whatever it read when the pointer + // last moved. One frame per interval, and only while this tab is the + // one on screen. + ui.ctx().request_repaint_after(DB_SIZE_REFRESH); + actions } } +/// How often the index files are re-statted. The number moves slowly even +/// during a run, so this is deliberately lazy: it costs three stats and one +/// repaint, and asking any more often would buy nothing. +const DB_SIZE_REFRESH: Duration = Duration::from_secs(10); + +/// Total on-disk footprint of the index: the database plus its `-wal` and +/// `-shm` sidecars. The `-wal` file is why the database alone will not do — +/// mid-run it can hold hundreds of megabytes the database does not show yet. +/// +/// A file that is not there counts as zero: there is no database at all +/// before the first run, and the sidecars exist only while a connection is +/// open. +fn measure_db_size(db: &Path) -> u64 { + // Only regular files: a misconfigured path pointing at a directory + // would otherwise report that directory's own inode size as an index. + let len = |path: &Path| { + std::fs::metadata(path) + .map(|m| if m.is_file() { m.len() } else { 0 }) + .unwrap_or(0) + }; + let name = db.file_name().and_then(|s| s.to_str()).unwrap_or(""); + // No `-journal`: the index runs in WAL mode, so a rollback journal is + // not part of a live database. + len(db) + + ["-wal", "-shm"] + .iter() + .map(|suffix| len(&db.with_file_name(format!("{}{}", name, suffix)))) + .sum::() +} + +/// Caches the last measurement so the tab can ask for it every frame. +#[derive(Default)] +struct DbSizeProbe { + /// Configured (unresolved) path the cached size belongs to. + path: String, + bytes: u64, + measured_at: Option, +} + +impl DbSizeProbe { + /// The cached size, restatted when it has gone stale or the configured + /// database path changed under it. `now` is a parameter so the refresh + /// cadence can be tested without sleeping. + fn size(&mut self, config: &Config, now: Instant) -> u64 { + let expired = self + .measured_at + .is_none_or(|at| now.duration_since(at) >= DB_SIZE_REFRESH); + if expired || self.path != config.paths.database_path { + // Resolving the path is only worth doing on a real refresh. + self.bytes = measure_db_size(&config.resolved_database_path()); + self.path = config.paths.database_path.clone(); + self.measured_at = Some(now); + } + self.bytes + } +} + +/// The index's footprint, with the levers for shrinking it on hover: a user +/// who finds the number too large looks here first, and every lever named +/// is either on this tab or in Options. +fn db_size_label(ui: &mut egui::Ui, bytes: u64) { + let response = ui.label(format!("Index size: {}", human_size(bytes))); + #[cfg(test)] + tests::record_widget("db-size", &response); + response.on_hover_ui(db_size_tooltip); +} + +fn db_size_tooltip(ui: &mut egui::Ui) { + ui.set_max_width(440.0); + ui.strong("To reduce the index size"); + for lever in [ + "Add ignore filters for files and folders you never search — the ignore \ + pattern list further down this tab.", + "Remove indexed folders you do not need, in Indexed folders above.", + "Narrow the full-text extension whitelist, so text is only extracted \ + from the file types you actually search.", + "Turn off \"Store text for snippets\" in Options: full-text search keeps \ + working, but without previews, occurrence ranking or fuzzy matching \ + inside file contents.", + "Lower \"Max text file size\" and \"Max stored text\" in Options.", + ] { + ui.label(format!("• {}", lever)); + } + ui.add_space(6.0); + ui.label( + egui::RichText::new( + "The file does not shrink on its own: freed space is reused by the \ + index rather than returned to the disk. To hand it back after \ + narrowing the filters, use Clear index… and reindex.", + ) + .small() + .weak(), + ); +} + /// Append a root to the draft unless it would duplicate or nest with an /// existing one; the rejection reason lands in `error`. fn try_add_root(draft: &mut Config, candidate: String, error: &mut Option) -> bool { @@ -419,7 +544,14 @@ fn parse_lines(text: &str) -> Vec { /// Live-update health. Permanent counterpart to the one-time modal: the /// modal is dismissed and remembered per root, but "live updates are off" /// stays true and must remain discoverable. +/// +/// Wrapped for the same reason as [`status_panel`]: this panel is empty in +/// manual mode and a line long otherwise. fn watch_panel(ui: &mut egui::Ui, state: &IndexerState, config: &Config) { + crate::ui_util::stable_section(ui, |ui| watch_contents(ui, state, config)); +} + +fn watch_contents(ui: &mut egui::Ui, state: &IndexerState, config: &Config) { match &state.watcher { // Manual mode already says "stopped" in the controls row; repeating // it here would be noise. @@ -454,7 +586,15 @@ fn watch_panel(ui: &mut egui::Ui, state: &IndexerState, config: &Config) { } } +/// Live progress. Its widget count tracks the run (roots come and go, the +/// rate line and the current file appear and vanish), so it renders inside +/// a [`crate::ui_util::stable_section`]: without one, every id below it — +/// including the per-root worker fields — would move mid-run. fn status_panel(ui: &mut egui::Ui, state: &IndexerState, speed: &SpeedTracker) { + crate::ui_util::stable_section(ui, |ui| status_contents(ui, state, speed)); +} + +fn status_contents(ui: &mut egui::Ui, state: &IndexerState, speed: &SpeedTracker) { match &state.activity { IndexingStatus::Idle => { // Relative wording makes even a milliseconds-fast run visibly @@ -471,6 +611,11 @@ fn status_panel(ui: &mut egui::Ui, state: &IndexerState, speed: &SpeedTracker) { IndexingStatus::Stopping => { ui.label("Stopping…"); } + IndexingStatus::Optimizing => { + // No per-file progress exists to show: this is one bulk rewrite + // of the whole file, and on a large index it runs for minutes. + ui.label("Optimizing index; reclaiming unused space…"); + } IndexingStatus::Running { roots, .. } => { for root in roots { root_row(ui, root); @@ -500,7 +645,7 @@ fn root_row(ui: &mut egui::Ui, r: &RootProgress) { ui.label("indexing"); divider(ui); let workers = format!("{}/{} workers", r.active_workers, r.total_workers); - match r.walk_total { + match r.walk_denominator() { Some(total) if total > 0 => { let frac = (r.walked as f32 / total as f32).clamp(0.0, 1.0); ui.label(format!( @@ -535,10 +680,12 @@ fn root_row(ui: &mut egui::Ui, r: &RootProgress) { 1.0 }; ui.label(format!( - "{} / {} ({:.0}%)", + "{} / {} ({:.0}%) · {}/{} workers", group_thousands(r.extracted as u64), group_thousands(r.extract_total as u64), - frac * 100.0 + frac * 100.0, + r.active_workers, + r.total_workers )); ui.add(egui::ProgressBar::new(frac).desired_width(160.0)); } @@ -567,6 +714,542 @@ fn root_row(ui: &mut egui::Ui, r: &RootProgress) { mod tests { use super::*; + use std::cell::RefCell; + + // Driving the real tab through a headless egui context is the only way + // to test widget identity, and identity is exactly what these tests are + // about — so the widgets under test report themselves here. + thread_local! { + static WIDGETS: RefCell> = + const { RefCell::new(Vec::new()) }; + } + + pub(super) fn record_widget(tag: &'static str, response: &egui::Response) { + WIDGETS.with(|w| w.borrow_mut().push((tag, response.id, response.rect))); + } + + fn widget(tag: &str) -> (egui::Id, egui::Rect) { + WIDGETS.with(|w| { + w.borrow() + .iter() + .find(|(t, _, _)| *t == tag) + .map(|(_, id, rect)| (*id, *rect)) + .unwrap_or_else(|| panic!("{} widget was not drawn", tag)) + }) + } + + fn idle_state() -> IndexerState { + IndexerState { + mode: IndexMode::Auto, + activity: IndexingStatus::Idle, + last_full_index: Some(0), + queued_events: 0, + watcher: WatcherStatus::Active { dirs: 10 }, + } + } + + /// A run in progress. `current_file` and the number of roots are the + /// parts that come and go from frame to frame in a real run. + fn running_state(roots: &[&str], current_file: Option<&str>) -> IndexerState { + state_with( + roots + .iter() + .map(|root| RootProgress { + root: (*root).to_string(), + phase: RootPhase::Walking, + walked: 100, + walk_total: Some(1000), + extracted: 0, + extract_total: 0, + current_file: current_file.map(str::to_string), + active_workers: 4, + total_workers: 4, + }) + .collect(), + ) + } + + /// A run whose roots are described one by one, for the rows whose + /// contents — not just their widget ids — are under test. + fn state_with(roots: Vec) -> IndexerState { + IndexerState { + mode: IndexMode::Auto, + activity: IndexingStatus::Running { + start_time: std::time::Instant::now(), + roots, + }, + last_full_index: Some(0), + queued_events: 0, + watcher: WatcherStatus::Active { dirs: 10 }, + } + } + + fn raw_input(events: Vec) -> egui::RawInput { + egui::RawInput { + screen_rect: Some(egui::Rect::from_min_size( + egui::Pos2::ZERO, + egui::vec2(1000.0, 900.0), + )), + events, + ..Default::default() + } + } + + /// One frame of the real tab, with `events` delivered to it. + fn frame( + ctx: &egui::Context, + tab: &mut ManageTab, + cfg: &Config, + state: &IndexerState, + events: Vec, + ) -> ManageActions { + WIDGETS.with(|w| w.borrow_mut().clear()); + let mut actions = ManageActions::default(); + let _ = ctx.run(raw_input(events), |ctx| { + egui::CentralPanel::default().show(ctx, |ui| { + actions = tab.ui(ui, state, cfg); + }); + }); + actions + } + + /// Every string the tab actually painted this frame. Labels carry no + /// widget id worth recording, so the rendered text is read back off the + /// shapes — the only place a number the user sees can be checked. + fn frame_text(ctx: &egui::Context, tab: &mut ManageTab, state: &IndexerState) -> Vec { + frame_text_with(ctx, tab, &cfg_with_root(), state) + } + + fn frame_text_with( + ctx: &egui::Context, + tab: &mut ManageTab, + cfg: &Config, + state: &IndexerState, + ) -> Vec { + WIDGETS.with(|w| w.borrow_mut().clear()); + let out = ctx.run(raw_input(vec![]), |ctx| { + egui::CentralPanel::default().show(ctx, |ui| { + tab.ui(ui, state, cfg); + }); + }); + let mut text = Vec::new(); + for clipped in &out.shapes { + collect_text(&clipped.shape, &mut text); + } + text + } + + fn collect_text(shape: &egui::epaint::Shape, out: &mut Vec) { + match shape { + egui::epaint::Shape::Text(t) => out.push(t.galley.text().to_string()), + egui::epaint::Shape::Vec(shapes) => { + for s in shapes { + collect_text(s, out); + } + } + _ => {} + } + } + + fn pointer(pos: egui::Pos2, pressed: bool) -> egui::Event { + egui::Event::PointerButton { + pos, + button: egui::PointerButton::Primary, + pressed, + modifiers: egui::Modifiers::NONE, + } + } + + fn click_at(pos: egui::Pos2) -> Vec { + vec![ + egui::Event::PointerMoved(pos), + pointer(pos, true), + pointer(pos, false), + ] + } + + fn cfg_with_root() -> Config { + let mut cfg = Config::default(); + cfg.paths.indexing_paths = vec!["/data".into()]; + cfg + } + + fn staged_workers(tab: &ManageTab) -> Option { + tab.draft + .as_ref() + .unwrap() + .indexing + .root_workers + .get("/data") + .copied() + } + + /// The per-root worker field must keep the same widget id however the + /// status above it changes: egui hangs focus and in-progress text off + /// that id, so a field that is renamed mid-run silently drops the edit. + #[test] + fn the_worker_field_keeps_its_identity_as_the_status_changes() { + let ctx = egui::Context::default(); + let mut tab = ManageTab::new(); + let cfg = cfg_with_root(); + + frame(&ctx, &mut tab, &cfg, &running_state(&["/data"], None), vec![]); + let (baseline, _) = widget("workers"); + + for state in [ + running_state(&["/data"], Some("/data/file")), + running_state(&["/data", "/other"], None), + idle_state(), + IndexerState { + watcher: WatcherStatus::Off, + ..idle_state() + }, + IndexerState { + activity: IndexingStatus::Error("boom".into()), + ..idle_state() + }, + ] { + frame(&ctx, &mut tab, &cfg, &state, vec![]); + assert_eq!( + widget("workers").0, + baseline, + "status change moved the worker field" + ); + } + } + + /// Click the field, type a count, click Apply & Save — while a run is + /// reporting progress the whole time. + #[test] + fn a_typed_worker_count_reaches_the_applied_config() { + let ctx = egui::Context::default(); + let mut tab = ManageTab::new(); + let cfg = cfg_with_root(); + + frame(&ctx, &mut tab, &cfg, &running_state(&["/data"], None), vec![]); + let field = widget("workers").1.center(); + frame( + &ctx, + &mut tab, + &cfg, + &running_state(&["/data"], None), + click_at(field), + ); + // The run starts reporting a file: one more label above the field. + let busy = running_state(&["/data"], Some("/data/file")); + frame(&ctx, &mut tab, &cfg, &busy, vec![]); + frame( + &ctx, + &mut tab, + &cfg, + &busy, + vec![egui::Event::Text("8".into())], + ); + assert_eq!(staged_workers(&tab), Some(8), "typed count was not staged"); + + let apply = widget("apply").1.center(); + let mut actions = frame(&ctx, &mut tab, &cfg, &busy, click_at(apply)); + if actions.apply_config.is_none() { + // egui fires a click on release; give it the follow-up frame. + actions = frame(&ctx, &mut tab, &cfg, &busy, vec![]); + } + let applied = actions.apply_config.expect("Apply & Save produced a config"); + assert_eq!(applied.indexing.root_workers.get("/data"), Some(&8)); + } + + /// The other way to set the field: drag it. + #[test] + fn a_dragged_worker_count_is_staged() { + let ctx = egui::Context::default(); + let mut tab = ManageTab::new(); + let cfg = cfg_with_root(); + let busy = running_state(&["/data"], None); + + frame(&ctx, &mut tab, &cfg, &busy, vec![]); + let field = widget("workers").1.center(); + frame( + &ctx, + &mut tab, + &cfg, + &busy, + vec![egui::Event::PointerMoved(field), pointer(field, true)], + ); + // Drag right across frames, with the status changing underneath. + let mut pos = field; + for state in [ + running_state(&["/data"], Some("/data/a")), + running_state(&["/data"], None), + running_state(&["/data"], Some("/data/b")), + ] { + pos.x += 4.0; + frame( + &ctx, + &mut tab, + &cfg, + &state, + vec![egui::Event::PointerMoved(pos)], + ); + } + frame(&ctx, &mut tab, &cfg, &busy, vec![pointer(pos, false)]); + assert!( + staged_workers(&tab).is_some_and(|w| w > 0), + "dragging staged nothing: {:?}", + staged_workers(&tab) + ); + } + + fn root_progress(phase: RootPhase, walked: usize, walk_total: Option) -> RootProgress { + RootProgress { + root: "/data".to_string(), + phase, + walked, + walk_total, + extracted: 0, + extract_total: 0, + current_file: None, + active_workers: 4, + total_workers: 4, + } + } + + /// The `find` count is an estimate of *tree entries*, so it runs far + /// ahead of the files a walk actually emits. Once the walk ends the exact + /// number is in hand, and the row must show that instead — the estimate + /// is what kept the bar short of full for the rest of the run. + #[test] + fn a_finished_root_reports_its_exact_count_not_the_estimate() { + let ctx = egui::Context::default(); + let mut tab = ManageTab::new(); + let mut done = root_progress(RootPhase::Done, 261_088, Some(6_677_062)); + done.extracted = 238_929; + done.active_workers = 0; + done.total_workers = 0; + + let text = frame_text(&ctx, &mut tab, &state_with(vec![done])).join(" | "); + assert!( + text.contains("indexed 261,088, extracted 238,929"), + "finished row: {}", + text + ); + assert!( + !text.contains("6,677,062"), + "the stale estimate is still on screen: {}", + text + ); + } + + /// While the walk runs the estimate is all there is, so it is shown — + /// but never below what has already been walked, or the row would sit at + /// 100% with the walk still going and read as a hang. + #[test] + fn a_walking_root_shows_the_estimate_raised_to_what_it_has_walked() { + let ctx = egui::Context::default(); + let mut tab = ManageTab::new(); + + let honest = frame_text( + &ctx, + &mut tab, + &state_with(vec![root_progress(RootPhase::Walking, 100, Some(1000))]), + ) + .join(" | "); + assert!(honest.contains("100 / 1,000 (10%)"), "{}", honest); + + let overtaken = frame_text( + &ctx, + &mut tab, + &state_with(vec![root_progress(RootPhase::Walking, 1500, Some(1000))]), + ) + .join(" | "); + assert!( + overtaken.contains("1,500 / 1,500 (100%)"), + "an overtaken estimate must be raised, not shown: {}", + overtaken + ); + } + + /// No count has landed yet: an indeterminate row, not a fabricated one. + #[test] + fn a_walking_root_without_a_count_shows_no_denominator() { + let ctx = egui::Context::default(); + let mut tab = ManageTab::new(); + let text = frame_text( + &ctx, + &mut tab, + &state_with(vec![root_progress(RootPhase::Walking, 100, None)]), + ) + .join(" | "); + assert!(text.contains("100 files"), "{}", text); + assert!(!text.contains(" / "), "invented a denominator: {}", text); + } + + /// A scratch directory of its own for each size test, so two of them + /// running in parallel cannot see each other's files. + fn scratch_dir(tag: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("qs-dbsize-{}-{}", std::process::id(), tag)); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).expect("scratch dir"); + dir + } + + fn write_bytes(path: &Path, len: usize) { + std::fs::write(path, vec![b'x'; len]).expect("write"); + } + + /// A config whose database is `path`, for the probe and the rendered row. + fn cfg_with_db(path: &Path) -> Config { + let mut cfg = cfg_with_root(); + cfg.paths.database_path = path.to_string_lossy().into_owned(); + cfg + } + + /// Each of the three files SQLite keeps for one database is added, and + /// nothing else that happens to sit beside them is. + #[test] + fn db_size_counts_the_database_and_both_sidecars() { + let dir = scratch_dir("parts"); + let db = dir.join("index.sqlite"); + write_bytes(&db, 4096); + assert_eq!(measure_db_size(&db), 4096, "the database itself"); + write_bytes(&dir.join("index.sqlite-wal"), 1024); + assert_eq!(measure_db_size(&db), 5120, "-wal was not added"); + write_bytes(&dir.join("index.sqlite-shm"), 32); + assert_eq!(measure_db_size(&db), 5152, "-shm was not added"); + + // Decoys: a rollback journal (never present in WAL mode) and an + // unrelated neighbour. + write_bytes(&dir.join("index.sqlite-journal"), 999); + write_bytes(&dir.join("index.sqlite.bak"), 777); + assert_eq!(measure_db_size(&db), 5152, "a decoy was counted"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Before the first indexing run there is no database at all, and a + /// closed database has no sidecars — neither is an error. + #[test] + fn a_missing_database_measures_zero() { + let dir = scratch_dir("missing"); + let db = dir.join("index.sqlite"); + assert_eq!(measure_db_size(&db), 0); + assert_eq!(measure_db_size(&dir), 0, "a directory"); + + write_bytes(&db, 100); + assert_eq!(measure_db_size(&db), 100, "sidecars are optional"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// The probe answers from cache until the interval is up: `ui()` asks it + /// every frame, and three stats per frame is churn nobody needs. + #[test] + fn the_probe_caches_until_the_refresh_interval_is_up() { + let dir = scratch_dir("cache"); + let db = dir.join("index.sqlite"); + write_bytes(&db, 100); + let cfg = cfg_with_db(&db); + let mut probe = DbSizeProbe::default(); + let t0 = Instant::now(); + + assert_eq!(probe.size(&cfg, t0), 100); + write_bytes(&db, 5000); + assert_eq!( + probe.size(&cfg, t0 + DB_SIZE_REFRESH / 2), + 100, + "restatted before the interval was up" + ); + assert_eq!( + probe.size(&cfg, t0 + DB_SIZE_REFRESH), + 5000, + "the refresh never happened" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A database path edited in Options must not keep showing the old + /// database's size for the rest of the interval. + #[test] + fn the_probe_follows_a_changed_database_path() { + let dir = scratch_dir("moved"); + let first = dir.join("index.sqlite"); + let second = dir.join("other.sqlite"); + write_bytes(&first, 100); + write_bytes(&second, 7000); + let mut probe = DbSizeProbe::default(); + let t0 = Instant::now(); + + assert_eq!(probe.size(&cfg_with_db(&first), t0), 100); + assert_eq!( + probe.size(&cfg_with_db(&second), t0), + 7000, + "a new path must restat at once, not wait out the interval" + ); + let _ = std::fs::remove_dir_all(&dir); + } + + /// The number reaches the screen, and it is the total of all three files + /// rather than the database on its own. + #[test] + fn the_status_row_shows_the_total_index_size() { + let dir = scratch_dir("render"); + let db = dir.join("index.sqlite"); + write_bytes(&db, 3_000_000); + write_bytes(&dir.join("index.sqlite-wal"), 200_000); + write_bytes(&dir.join("index.sqlite-shm"), 32_768); + + let ctx = egui::Context::default(); + let mut tab = ManageTab::new(); + let text = frame_text_with(&ctx, &mut tab, &cfg_with_db(&db), &idle_state()).join(" | "); + assert!( + text.contains("Index size: 3.2 MB"), + "the size is not on screen: {}", + text + ); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Hovering is the whole point of the readout: the number tells the user + /// the index is big, the tooltip tells them what to do about it. Each + /// lever named here has to keep matching a control that exists. + #[test] + fn hovering_the_size_explains_how_to_shrink_the_index() { + let dir = scratch_dir("hover"); + let db = dir.join("index.sqlite"); + write_bytes(&db, 2048); + + let ctx = egui::Context::default(); + // egui holds tooltips back for a third of a second, and frames here + // are only 1/60 s of simulated time apart. + ctx.style_mut(|s| s.interaction.tooltip_delay = 0.0); + let mut tab = ManageTab::new(); + let cfg = cfg_with_db(&db); + + frame_text_with(&ctx, &mut tab, &cfg, &idle_state()); + let at = widget("db-size").1.center(); + frame( + &ctx, + &mut tab, + &cfg, + &idle_state(), + vec![egui::Event::PointerMoved(at)], + ); + // The tooltip is painted in the frame after the pointer lands. + let text = frame_text_with(&ctx, &mut tab, &cfg, &idle_state()).join(" | "); + + assert!( + text.contains("To reduce the index size"), + "no tooltip: {}", + text + ); + for lever in [ + "ignore filters", + "Indexed folders", + "whitelist", + "Store text for snippets", + "Options", + ] { + assert!(text.contains(lever), "tooltip never mentions {}", lever); + } + let _ = std::fs::remove_dir_all(&dir); + } + fn synced_tab(config: &Config) -> ManageTab { let mut tab = ManageTab::new(); tab.sync_editors(config); diff --git a/crates/quicksearch-gui/src/options.rs b/crates/quicksearch-gui/src/options.rs index ceb5929..8d3c804 100644 --- a/crates/quicksearch-gui/src/options.rs +++ b/crates/quicksearch-gui/src/options.rs @@ -308,6 +308,18 @@ pub fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section ui.add(egui::DragValue::new(&mut config.processing.batch_size).range(10..=100_000)); ui.end_row(); + ui.label("Max WAL size (bytes)"); + ui.add( + egui::DragValue::new(&mut config.processing.maximum_wal_size) + .range(0u64..=8_589_934_592u64), + ) + .on_hover_text( + "How large index.sqlite-wal may grow during a run before the \ + indexer forces a checkpoint. 0 disables forced checkpoints; \ + anything below 16 MiB is raised to it.", + ); + ui.end_row(); + ui.label("Store text for snippets"); ui.checkbox(&mut config.processing.store_text_for_snippets, "") .on_hover_text( @@ -365,9 +377,14 @@ pub fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section ); ui.end_row(); }); - if let Some(warning) = config.search.fuzzy_edits_warning() { - ui.colored_label(ui.visuals().warn_fg_color, warning); - } + // A warning that comes and goes as the value is edited would + // otherwise move every widget below it in the window; see + // `ui_util::stable_section`. + crate::ui_util::stable_section(ui, |ui| { + if let Some(warning) = config.search.fuzzy_edits_warning() { + ui.colored_label(ui.visuals().warn_fg_color, warning); + } + }); } } } diff --git a/crates/quicksearch-gui/src/search_tab.rs b/crates/quicksearch-gui/src/search_tab.rs index 4268b38..ccf8420 100644 --- a/crates/quicksearch-gui/src/search_tab.rs +++ b/crates/quicksearch-gui/src/search_tab.rs @@ -41,6 +41,35 @@ pub struct SearchActions { pub save_fuzzy_default: Option, } +/// Add `incoming` to `set`, keeping at most `limit` of them — the best by +/// **rank**, whatever column the table is currently sorted by. +/// +/// Retention and display are separate questions. What to keep is about +/// relevance; what order to show it in is the user's choice. That distinction +/// only started to matter once the cascade began streaming: hits now arrive in +/// table order, so a cap that simply stopped accepting at `limit` would fill +/// the table with whatever the scan happened to reach first and never show the +/// good ones. Dropping the worst-ranked instead means a rank-1 hit found late +/// in a scan still displaces a rank-10 one found early. +fn admit( + set: &mut Vec, + incoming: Vec, + limit: usize, + limited: &mut bool, +) { + set.extend(incoming); + if set.len() > limit { + set.sort_by(|a, b| { + a.rank + .partial_cmp(&b.rank) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.path.cmp(&b.path)) + }); + set.truncate(limit); + *limited = true; + } +} + pub struct SearchTab { pub query: String, pub fuzzy: bool, @@ -142,31 +171,31 @@ impl SearchTab { SearchUpdate::Hits { hits, .. } => { if self.swap_pending { // Old results are still fading out; hold the new ones. - for hit in hits { - if self.staging.len() >= display_limit { - self.limited = true; - break; - } - self.staging_has_snippets |= hit.snippet.is_some(); - self.staging.push(hit); - } + // Nothing of this generation is displayed yet, so there is + // no selection to keep. + self.staging_has_snippets |= hits.iter().any(|h| h.snippet.is_some()); + admit(&mut self.staging, hits, display_limit, &mut self.limited); } else { - // Post-swap stream: later cascade passes append live. - for hit in hits { - if self.results.len() >= display_limit { - self.limited = true; - break; - } - self.has_snippets |= hit.snippet.is_some(); - self.results.push(hit); - } - // Arrival order *is* rank order, so the default sort - // needs no work; anything else re-sorts on the set. - if self.sort != (SortKey::Rank, true) { - self.sort_dirty = true; - } else { - self.order = (0..self.results.len() as u32).collect(); - } + self.has_snippets |= hits.iter().any(|h| h.snippet.is_some()); + // A row can be selected while batches are still arriving, + // and admitting them may reorder or drop rows out from + // under its index, so carry the selection by file id. + let selected_id = self + .selected + .and_then(|i| self.results.get(i as usize)) + .map(|h| h.file_id); + admit(&mut self.results, hits, display_limit, &mut self.limited); + self.selected = selected_id.and_then(|id| { + self.results + .iter() + .position(|h| h.file_id == id) + .map(|i| i as u32) + }); + // Arrival order is *scan* order — the cascade streams each + // pass as it runs, so a better-ranked hit can turn up after + // a worse one. Re-establish the table's own order on every + // batch, under whichever column is keyed. + self.sort_dirty = true; } } SearchUpdate::Completed { limited, .. } => { @@ -213,11 +242,14 @@ impl SearchTab { SortKey::Size => a.size.cmp(&b.size), SortKey::Modified => a.mtime.cmp(&b.mtime), }; - if ascending { - ord - } else { - ord.reverse() - } + let ord = if ascending { ord } else { ord.reverse() }; + // Break ties on the path, which is unique, so the order is total. + // Without it a stable sort falls back to insertion order — and + // that is now the order batches happened to stream in, so equal + // keys would shuffle under the pointer on every arrival. The + // tiebreak stays ascending regardless of the key's direction; it + // is there for stability, not as a second sort the user asked for. + ord.then_with(|| a.path.cmp(&b.path)) }); // Selection follows the file, not the visual slot. self.selected = selected_id.and_then(|id| { @@ -395,11 +427,9 @@ impl SearchTab { self.has_snippets = self.staging_has_snippets; self.selected = None; self.swap_pending = false; - if self.sort == (SortKey::Rank, true) { - self.order = (0..self.results.len() as u32).collect(); - } else { - self.sort_dirty = true; - } + // Staged hits arrived in scan order too, so the table has to be + // ordered here as well — including under the default key. + self.sort_dirty = true; } if self.sort_dirty { @@ -425,6 +455,10 @@ impl SearchTab { let text_height = egui::TextStyle::Body.resolve(ui.style()).size + 4.0; let mut open_ignore_dialog: Option = None; let mut hovered_now: Option = None; + // Moved out of `self` rather than cloned. The row closure needs `&mut + // self` for selection and hover, so a field read would conflict — but + // a plain local does not, and this runs every frame. + let order = std::mem::take(&mut self.order); let table_scroll = ui .push_id("results", |ui| { @@ -465,7 +499,6 @@ impl SearchTab { header.col(|ui| self.sort_header(ui, SortKey::Rank, "Rank")); }) .body(|body| { - let order = self.order.clone(); body.rows(text_height, order.len(), |mut row| { let display_ix = row.index(); let result_ix = order[display_ix] as usize; @@ -487,13 +520,17 @@ impl SearchTab { .push(ui.label(egui::RichText::new(&hit.path).weak())); }); if self.has_snippets { - let snippet = hit.snippet.clone(); + // Borrowed, not cloned: a snippet window runs + // to 600 characters and this is per visible row + // per frame. The hover job is built inside the + // closure, so an un-hovered row builds nothing. + let snippet = hit.snippet.as_ref(); // Name and path matches show a whole field, so // they render bracketed: [matched field]. let whole_field = hit.stage <= 4 || hit.stage == 7 || hit.stage >= 9; row.col(|ui| { - if let Some(snip) = &snippet { + if let Some(snip) = snippet { let width = ui.available_width(); let job = centered_match_job(ui, snip, width, whole_field); let mut response = ui @@ -505,10 +542,9 @@ impl SearchTab { ) .inner; if !snip.ranges.is_empty() { - let hover = snip.clone(); response = response.on_hover_ui(|ui| { ui.set_max_width(520.0); - let job = snippet_job(ui, &hover, 10); + let job = snippet_job(ui, snip, 10); ui.label(job); }); } @@ -592,6 +628,8 @@ impl SearchTab { }) }) .inner; + // Put the permutation back for the next frame. + self.order = order; crate::ui_util::more_below_hint(ui, &table_scroll); self.hovered_row = hovered_now; @@ -900,8 +938,14 @@ fn centered_match_job( // centered single-line cell. Flatten them to spaces — a byte-for-byte // ASCII replacement, so the match ranges stay valid. The mouseover // renders the original window untouched. - let flattened = snip.window.replace(['\n', '\r', '\t'], " "); - let window = flattened.as_str(); + // + // Only copied when there is something to flatten: name and path snippets + // never contain these, and this runs per visible row per frame. + let flattened: Option = snip + .window + .contains(['\n', '\r', '\t']) + .then(|| snip.window.replace(['\n', '\r', '\t'], " ")); + let window = flattened.as_deref().unwrap_or(&snip.window); let (start, end) = match snip.ranges.first().copied() { Some((a, b)) => { let match_chars = window[a..b].chars().count(); @@ -1086,6 +1130,208 @@ mod tests { .collect() } + fn hit(id: i64, name: &str, rank: f64, size: u64) -> SearchHit { + SearchHit { + file_id: id, + name: name.to_string(), + path: format!("/d/{name}"), + size, + mtime: 1_700_000_000, + rank, + stage: rank as u8, + snippet: None, + } + } + + fn batch(tab: &mut SearchTab, hits: Vec) { + tab.apply_update( + SearchUpdate::Hits { + generation: tab.generation, + hits, + }, + 1000, + ); + if tab.sort_dirty { + tab.resort(); + } + } + + /// A tab already past the fade, so batches land straight in `results`. + fn streaming_tab() -> SearchTab { + let mut tab = SearchTab::new(false); + tab.focus_query = false; + tab.query = "zebra".into(); + tab.swap_pending = false; + tab + } + + fn displayed(tab: &SearchTab) -> Vec<&str> { + tab.order + .iter() + .map(|&i| tab.results[i as usize].name.as_str()) + .collect() + } + + /// The cascade streams each pass as it scans, so a better-ranked hit can + /// arrive after a worse one. The table has to stay ordered by its keyed + /// column as batches land, not merely append them. + #[test] + fn later_batches_land_in_the_tables_sort_order() { + let mut tab = streaming_tab(); + batch(&mut tab, vec![hit(1, "middling.txt", 4.0, 50)]); + assert_eq!(displayed(&tab), vec!["middling.txt"]); + + // A rank-1 hit found later in the scan belongs on top. + batch(&mut tab, vec![hit(2, "best.txt", 1.0, 10)]); + assert_eq!(displayed(&tab), vec!["best.txt", "middling.txt"]); + + // And a rank-10 one belongs at the bottom, not wherever it arrived. + batch(&mut tab, vec![hit(3, "worst.txt", 10.0, 90)]); + assert_eq!( + displayed(&tab), + vec!["best.txt", "middling.txt", "worst.txt"] + ); + } + + /// Rank is only the default. Under any other key, arrivals slot into that + /// key's order. + #[test] + fn batches_respect_a_non_rank_sort_key() { + let mut tab = streaming_tab(); + tab.sort = (SortKey::Name, true); + batch(&mut tab, vec![hit(1, "mango.txt", 1.0, 50)]); + batch(&mut tab, vec![hit(2, "apple.txt", 9.0, 10)]); + batch(&mut tab, vec![hit(3, "zucchini.txt", 2.0, 90)]); + assert_eq!( + displayed(&tab), + vec!["apple.txt", "mango.txt", "zucchini.txt"], + "name order, not arrival or rank order" + ); + + tab.sort = (SortKey::Size, false); + tab.sort_dirty = true; + tab.resort(); + assert_eq!(displayed(&tab), vec!["zucchini.txt", "mango.txt", "apple.txt"]); + } + + /// The user may re-key the sort at any time, including while results are + /// still arriving: rows already shown must re-order, and later batches + /// must land under the new key. + #[test] + fn re_keying_the_sort_mid_stream_reorders_everything() { + let mut tab = streaming_tab(); + batch(&mut tab, vec![hit(1, "delta.txt", 1.0, 30)]); + batch(&mut tab, vec![hit(2, "alpha.txt", 5.0, 10)]); + assert_eq!(displayed(&tab), vec!["delta.txt", "alpha.txt"], "rank order"); + + // Header click, mid-search. + tab.sort = (SortKey::Name, true); + tab.sort_dirty = true; + tab.resort(); + assert_eq!( + displayed(&tab), + vec!["alpha.txt", "delta.txt"], + "rows that already arrived re-order under the new key" + ); + + batch(&mut tab, vec![hit(3, "bravo.txt", 2.0, 20)]); + assert_eq!( + displayed(&tab), + vec!["alpha.txt", "bravo.txt", "delta.txt"], + "and the next batch lands under it too" + ); + } + + /// At the display cap, retention stays keyed on rank even when the table + /// is shown in another order — otherwise a broad query fills up with + /// whatever the scan reached first and never shows the good hits. + #[test] + fn a_late_better_hit_displaces_the_worst_at_the_cap() { + let mut tab = streaming_tab(); + tab.sort = (SortKey::Name, true); + let limit = 3; + + let send = |tab: &mut SearchTab, hits: Vec| { + tab.apply_update( + SearchUpdate::Hits { + generation: tab.generation, + hits, + }, + limit, + ); + if tab.sort_dirty { + tab.resort(); + } + }; + + send( + &mut tab, + vec![ + hit(1, "aaa.txt", 9.0, 10), + hit(2, "bbb.txt", 8.0, 20), + hit(3, "ccc.txt", 7.0, 30), + ], + ); + assert_eq!(displayed(&tab), vec!["aaa.txt", "bbb.txt", "ccc.txt"]); + assert!(!tab.limited); + + // Full. A rank-1 arrival must still get in, evicting rank 9. + send(&mut tab, vec![hit(4, "zzz.txt", 1.0, 40)]); + assert!(tab.limited, "the cap was hit"); + assert_eq!(tab.results.len(), limit); + assert_eq!( + displayed(&tab), + vec!["bbb.txt", "ccc.txt", "zzz.txt"], + "worst rank dropped, display still in name order" + ); + } + + /// Batches arriving during the fade get the same treatment; the ordering + /// problem must not simply move inside the 250 ms window. + #[test] + fn staged_batches_are_ordered_once_the_fade_swaps() { + let mut tab = SearchTab::new(false); + tab.focus_query = false; + tab.query = "zebra".into(); + tab.on_search_started(1); + assert!(tab.swap_pending); + + for h in [hit(1, "worst.txt", 9.0, 10), hit(2, "best.txt", 1.0, 20)] { + tab.apply_update( + SearchUpdate::Hits { + generation: 1, + hits: vec![h], + }, + 1000, + ); + } + assert!(tab.results.is_empty(), "still staged behind the fade"); + + // What the fade does when it reaches zero. + tab.results = std::mem::take(&mut tab.staging); + tab.swap_pending = false; + tab.sort_dirty = true; + tab.resort(); + assert_eq!(displayed(&tab), vec!["best.txt", "worst.txt"]); + } + + /// A selected row is identified by file id, so it survives both the + /// re-ordering and the eviction that a new batch can cause. + #[test] + fn the_selection_follows_its_file_across_batches() { + let mut tab = streaming_tab(); + batch(&mut tab, vec![hit(1, "chosen.txt", 5.0, 10)]); + tab.selected = Some(0); + + batch(&mut tab, vec![hit(2, "better.txt", 1.0, 20)]); + let sel = tab.selected.expect("still selected"); + assert_eq!( + tab.results[sel as usize].file_id, + 1, + "selection follows the file, not the slot" + ); + } + #[test] fn rows_respond_over_selectable_label_text() { let ctx = egui::Context::default(); diff --git a/crates/quicksearch-gui/src/tracker.rs b/crates/quicksearch-gui/src/tracker.rs index 09cf8ca..25b2c33 100644 --- a/crates/quicksearch-gui/src/tracker.rs +++ b/crates/quicksearch-gui/src/tracker.rs @@ -8,18 +8,20 @@ //! computable), and measures against `now` so the estimate decays during //! stalls instead of freezing at the last burst. +use std::collections::VecDeque; use std::time::{Duration, Instant}; const HISTORY: Duration = Duration::from_secs(60); pub struct SpeedTracker { - /// (when, counter value) — appended only on counter change. - points: Vec<(Instant, usize)>, + /// (when, counter value) — appended only on counter change. A deque + /// because pruning drops from the front, which is O(n) on a `Vec`. + points: VecDeque<(Instant, usize)>, } impl SpeedTracker { pub fn new() -> SpeedTracker { - SpeedTracker { points: Vec::new() } + SpeedTracker { points: VecDeque::new() } } /// Reset between phases (each phase restarts its counter). @@ -32,18 +34,18 @@ impl SpeedTracker { } fn record_at(&mut self, now: Instant, files_processed: usize) { - match self.points.last() { + match self.points.back() { Some(&(_, last)) if last == files_processed => return, // Counter went backwards — a new phase started without an // explicit reset. Some(&(_, last)) if files_processed < last => self.points.clear(), _ => {} } - self.points.push((now, files_processed)); + self.points.push_back((now, files_processed)); // Prune old points, but always keep at least two so a slow but // steady rate never becomes unmeasurable. while self.points.len() > 2 && now.duration_since(self.points[0].0) > HISTORY { - self.points.remove(0); + self.points.pop_front(); } } @@ -54,8 +56,8 @@ impl SpeedTracker { } fn files_per_sec_at(&self, now: Instant) -> Option { - let (t0, c0) = *self.points.first()?; - let (_, c1) = *self.points.last()?; + let (t0, c0) = *self.points.front()?; + let (_, c1) = *self.points.back()?; if self.points.len() < 2 { return None; } diff --git a/crates/quicksearch-gui/src/ui_util.rs b/crates/quicksearch-gui/src/ui_util.rs index 00006a4..c19f120 100644 --- a/crates/quicksearch-gui/src/ui_util.rs +++ b/crates/quicksearch-gui/src/ui_util.rs @@ -20,6 +20,25 @@ pub fn bordered_button( egui::Button::new(text).stroke(egui::Stroke::new(1.5, color)) } +/// Render a section whose widget count changes from frame to frame inside +/// its own child `Ui`. +/// +/// egui derives a widget's id from how many widgets precede it in the same +/// `Ui`. A section that emits, say, one label while idle and four rows plus +/// a progress line while working therefore *renames* every widget below it +/// the moment its content changes — and a `DragValue` or `TextEdit` whose +/// id changes loses keyboard focus and whatever the user was typing. A +/// child `Ui` costs the parent exactly one id no matter what goes inside +/// it, so everything below keeps its identity. +/// +/// Wrapping does not help the section's *own* widgets: a child `Ui` mixes +/// the parent's counter into its id, so an unstable section cannot be made +/// stable from the inside. Live text and progress belong in a wrapped +/// section; editable fields belong outside one. +pub fn stable_section(ui: &mut egui::Ui, contents: impl FnOnce(&mut egui::Ui) -> R) -> R { + ui.vertical(contents).inner +} + /// Whether `pattern` is usable as an ignore pattern. `IgnoreSet::compile` /// silently *skips* patterns that trim to nothing, so emptiness is checked /// here with the same trimming rules compile applies. diff --git a/crates/quicksearch-gui/src/unlock.rs b/crates/quicksearch-gui/src/unlock.rs index 2236e0b..0001426 100644 --- a/crates/quicksearch-gui/src/unlock.rs +++ b/crates/quicksearch-gui/src/unlock.rs @@ -21,6 +21,21 @@ use zeroize::{Zeroize, Zeroizing}; use crate::app::QuickSearchApp; use crate::keychain; +/// Where this session's index key came from. +/// +/// The app needs it for anything that *refers* to the key: telling someone +/// "the password you just entered" is wrong when they never typed one, because +/// the keychain answered before the window opened. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeySource { + /// The index is not password-protected. + Unprotected, + /// Typed at the unlock prompt during this session. + Prompt, + /// Supplied by the OS keychain, with no prompt shown. + Keychain, +} + /// The application shell handed to eframe: locked (unlock screen) or /// running (the real app). pub enum Gate { @@ -36,8 +51,9 @@ impl Gate { cfg: Config, config_error: Option, initial_query: Option, + key_source: KeySource, ) -> Result { - QuickSearchApp::new(ctx, cfg, config_error, initial_query) + QuickSearchApp::new(ctx, cfg, config_error, initial_query, key_source) .map(|app| Gate::Running(Box::new(app))) } @@ -99,8 +115,13 @@ pub fn try_keychain_unlock(cfg: &Config) -> bool { enum Mode { /// An index exists: the password must open it. Unlock, - /// Protection is on but no index file exists yet — the typed password - /// (with confirmation) becomes the one the new index is built under. + /// Protection is on but no index file exists yet, so the typed password + /// becomes the one the new index is built under. + /// + /// Still not a *new* password: this mode is only reachable with a salt + /// already in the config, so one was chosen previously and this is the + /// user re-entering it. Choosing a genuinely new password happens in the + /// Options window, which does its own confirmation. Create, /// `password_protected = true` but the salt is missing or corrupt; no /// password can help. Only the reset escape hatch applies. @@ -113,9 +134,14 @@ pub struct UnlockScreen { initial_query: Option, mode: Mode, password: String, - confirm: String, remember: bool, error: Option, + /// Put the caret in the password field on the next frame. Set once at + /// startup and again after a failed attempt, so the user can retype + /// straight away — but *not* every frame: re-focusing unconditionally + /// traps the caret, and nothing else on the screen can be tabbed to or + /// clicked into. + focus_password: bool, /// In-flight Argon2 derivation (+ verification) on a worker thread. job: Option>>, forgot_confirm: bool, @@ -140,9 +166,9 @@ impl UnlockScreen { initial_query, mode, password: String::new(), - confirm: String::new(), remember, error: None, + focus_password: true, job: None, forgot_confirm: false, } @@ -159,6 +185,9 @@ impl UnlockScreen { } else { e }); + // The field was cleared on submit, so put the caret back + // in it rather than making the user click before retrying. + self.focus_password = true; } } } @@ -206,15 +235,6 @@ impl UnlockScreen { .hint_text("Password") .desired_width(240.0), ); - if matches!(self.mode, Mode::Create) { - ui.add( - egui::TextEdit::singleline(&mut self.confirm) - .id(confirm_field_id()) - .password(true) - .hint_text("Confirm password") - .desired_width(240.0), - ); - } ui.add_space(4.0); ui.checkbox(&mut self.remember, "Remember on this device") .on_hover_text( @@ -231,8 +251,9 @@ impl UnlockScreen { let entered = field.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); submitted = clicked || entered; - if !busy && !field.has_focus() && !submitted { + if self.focus_password && !busy { field.request_focus(); + self.focus_password = false; } }); if busy { @@ -292,19 +313,12 @@ impl UnlockScreen { self.error = Some("The password may not be empty.".to_string()); return; } - if self.password != self.confirm { - self.error = Some("Passwords do not match.".to_string()); - return; - } } let Ok(salt) = self.cfg.security.salt_bytes() else { return; // BrokenSalt mode never reaches submit }; let password = Zeroizing::new(std::mem::take(&mut self.password)); - self.confirm.zeroize(); - self.confirm.clear(); purge_text_state(ctx, pw_field_id()); - purge_text_state(ctx, confirm_field_id()); let verify_against = match self.mode { Mode::Unlock => Some(self.cfg.resolved_database_path()), @@ -352,11 +366,20 @@ impl UnlockScreen { /// Construct the real app; on failure stay locked and show why. fn launch(&mut self, ctx: &egui::Context) -> Option { + // Reaching here means either the password was just typed, or the + // "forgot password" path disabled protection on the way. The config + // says which, and no caller has to remember to pass it. + let key_source = if self.cfg.security.password_protected { + KeySource::Prompt + } else { + KeySource::Unprotected + }; match QuickSearchApp::new( ctx, self.cfg.clone(), self.config_error.take(), self.initial_query.take(), + key_source, ) { Ok(app) => Some(app), Err(e) => { @@ -420,7 +443,6 @@ impl UnlockScreen { impl Drop for UnlockScreen { fn drop(&mut self) { self.password.zeroize(); - self.confirm.zeroize(); } } @@ -428,10 +450,6 @@ fn pw_field_id() -> egui::Id { egui::Id::new("unlock-password") } -fn confirm_field_id() -> egui::Id { - egui::Id::new("unlock-confirm") -} - /// Drop egui's retained state for a password field — its text buffer and /// undo history — so the plaintext doesn't outlive the submit. fn purge_text_state(ctx: &egui::Context, id: egui::Id) { @@ -456,3 +474,83 @@ fn delete_index_files(db_path: &std::path::Path) -> Result<(), String> { } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + /// A protected config whose salt parses, so the screen lands in a real + /// password mode rather than `BrokenSalt`. + fn locked_config() -> Config { + let mut cfg = Config::default(); + cfg.security.password_protected = true; + cfg.security.salt = Some("0f1e2d3c4b5a69788796a5b4c3d2e1f0".to_string()); + cfg.paths.database_path = std::env::temp_dir() + .join(format!("qs-unlock-test-{}.sqlite", std::process::id())) + .to_string_lossy() + .into_owned(); + cfg + } + + fn frame(ctx: &egui::Context, screen: &mut UnlockScreen) { + let input = egui::RawInput { + screen_rect: Some(egui::Rect::from_min_size( + egui::Pos2::ZERO, + egui::vec2(900.0, 600.0), + )), + ..Default::default() + }; + let _ = ctx.run(input, |ctx| { + // `update` only builds the app on a successful unlock, which needs + // a derived key — so with nothing submitted this stays on screen. + assert!(screen.update(ctx).is_none()); + }); + } + + /// The caret starts in the password field, and — the regression — can then + /// leave it. + /// + /// The screen used to call `request_focus()` on every frame the field did + /// not have focus, which yanked the caret back the instant anything else + /// took it. Nothing else on the screen could be tabbed to or clicked into. + #[test] + fn the_password_field_takes_focus_once_and_then_releases_it() { + let ctx = egui::Context::default(); + let mut screen = UnlockScreen::new(locked_config(), None, None); + + frame(&ctx, &mut screen); + assert!( + ctx.memory(|m| m.has_focus(pw_field_id())), + "the caret should start in the password field" + ); + + // Whatever the user clicks or tabs to next takes focus away. + ctx.memory_mut(|m| m.surrender_focus(pw_field_id())); + frame(&ctx, &mut screen); + assert!( + !ctx.memory(|m| m.has_focus(pw_field_id())), + "focus was stolen back; the caret is trapped in the password field" + ); + + // And it stays released across further frames. + frame(&ctx, &mut screen); + assert!(!ctx.memory(|m| m.has_focus(pw_field_id()))); + } + + /// A failed attempt is the one case that *should* re-focus: the field was + /// cleared on submit, so the user would otherwise have to click before + /// retyping. + #[test] + fn a_failed_attempt_puts_the_caret_back() { + let ctx = egui::Context::default(); + let mut screen = UnlockScreen::new(locked_config(), None, None); + frame(&ctx, &mut screen); + ctx.memory_mut(|m| m.surrender_focus(pw_field_id())); + frame(&ctx, &mut screen); + assert!(!ctx.memory(|m| m.has_focus(pw_field_id()))); + + screen.focus_password = true; // what the error path sets + frame(&ctx, &mut screen); + assert!(ctx.memory(|m| m.has_focus(pw_field_id()))); + } +}