diff --git a/DESIGN.md b/DESIGN.md index 479c8c3..bbfe26d 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -21,12 +21,16 @@ memory, memory churn, and indexing and search speed. The index is a single SQLite database: file metadata, an FTS5 full-text index over document contents, and the extracted text itself, stored zstd-compressed -for snippets and ranking. The database runs in WAL mode with exactly one +for snippets, ranking and verification. The database runs in WAL mode with exactly one writer at a time, so read-only searches are never blocked by indexing. Schema changes wipe and rebuild the index rather than migrating: the index is a cache of the filesystem, so rebuilding is always safe, and only the indexer may do it. FTS5 runs in trigram mode for full-text search and also speeds up regex -and fuzzy searches. Tuning this database is central to QuickSearch's +and fuzzy searches. By default it records no word positions: every candidate +is checked against the stored text anyway, so positions only narrow the +candidates, and dropping them makes the index several times smaller and +faster to build, for content searches up to two or three times slower. A +setting trades the other way. Tuning this database is central to QuickSearch's performance. ## The search path never stalls diff --git a/config_example.toml b/config_example.toml index 1d311e0..e04c453 100644 --- a/config_example.toml +++ b/config_example.toml @@ -94,17 +94,20 @@ writer_turn_slice_ms = 100 # searches that run alongside indexing and lengthens crash recovery. The # default favours faster runs. maximum_wal_size = 2147483648 -# FTS5 tokenizer: 'trigram' (substring matching, the default; gets -# remove_diacritics 1 appended), 'unicode61', 'porter', or a full FTS5 option -# string. See https://www.sqlite.org/fts5.html#tokenizers. Changing it -# rebuilds the index. +# FTS5 tokenizer: 'trigram' (substring matching, the default), 'unicode61', +# 'porter', or a full FTS5 option string. See +# https://www.sqlite.org/fts5.html#tokenizers. Matching is exact apart from +# ASCII case: "cafe" does not find "café". Changing it rebuilds the index. tokenize = "trigram" -# Store extracted text (zstd-compressed) alongside the FTS index. Off, the -# index shrinks considerably, but search loses snippet previews, occurrence -# ranking, case verification and fuzzy full-text. Turning it off discards -# the stored text at once; turning it on re-extracts, because the text of -# files already indexed was never kept. -store_text_for_snippets = true +# Build the full-text index without word positions. Several times smaller +# and faster to build (1.1 GB of source code: a 219 MiB index instead of +# 1.3 GB, built a third faster), and searches return exactly the same files, +# because every candidate is checked against the stored text. The cost is +# more candidates to check: searching contents for a word can take up to +# twice as long, for a phrase of common words up to three times. Only the +# trigram tokenizer can use it; other tokenizers always record positions. +# Changing it rebuilds the index. +compact_text_index = true [security] # Encrypt the index with a password (SQLCipher). The password is asked for at diff --git a/crates/quicksearch-core/benches/index.rs b/crates/quicksearch-core/benches/index.rs index b095e14..3ab714a 100644 --- a/crates/quicksearch-core/benches/index.rs +++ b/crates/quicksearch-core/benches/index.rs @@ -84,7 +84,7 @@ mod text_pipeline { let path = Path::new("/tmp/bench/notes.txt"); bencher.bench(|| { let looks = textenc::looks_like_text(divan::black_box(head)); - let text = textenc::decode_text(head.to_vec(), path).unwrap(); + let text = textenc::decode_text(head.to_vec(), path, false).unwrap(); (looks, text.len()) }); } @@ -101,7 +101,7 @@ mod text_pipeline { fn decode_only(bencher: Bencher) { let head = corpus::text_head(); let path = Path::new("/tmp/bench/notes.txt"); - bencher.bench(|| textenc::decode_text(divan::black_box(head).to_vec(), path)); + bencher.bench(|| textenc::decode_text(divan::black_box(head).to_vec(), path, false)); } } @@ -167,7 +167,8 @@ mod update_narrowing { fn seeded_na(tag: &str) -> (Scratch, Connection) { let (dir, p) = Scratch::db(tag); let mut conn = - quicksearch_core::db::open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + quicksearch_core::db::open_or_recreate(p.to_str().unwrap(), &Default::default()) + .unwrap(); let tx = conn.transaction().unwrap(); let names: Vec = (0..ROWS).map(|i| format!("f{:04}.txt", i)).collect(); for name in &names { @@ -233,3 +234,54 @@ mod update_narrowing { }); } } + +/// A watcher event for a directory whose files are all indexed and unchanged: +/// a re-sent `Create`, a `chmod -R`, the live view re-checking what it shows. +/// Nothing is written, so the cost is whatever it reads to learn that; the +/// full walk's answer is one `stat` per file. +mod unchanged_event { + use super::*; + use quicksearch_core::config::{Config, IgnoreSet}; + use quicksearch_core::extract::Registry; + use quicksearch_core::incremental::{apply_fs_event, Budget}; + use quicksearch_core::testutil::Scratch; + use quicksearch_core::watcher::FsEvent; + use std::sync::atomic::AtomicBool; + use std::time::{Duration, Instant}; + + const FILES: usize = 500; + + #[divan::bench] + fn directory_create(bencher: Bencher) { + let (dir, db) = Scratch::db("bench-unchanged-event"); + let tree = db.with_file_name("tree"); + std::fs::create_dir_all(&tree).unwrap(); + for i in 0..FILES { + std::fs::write( + tree.join(format!("f{:04}.txt", i)), + corpus::text(1 << 10, 4), + ) + .unwrap(); + } + let mut conn = + quicksearch_core::db::open_or_recreate(db.to_str().unwrap(), &Default::default()) + .unwrap(); + let config = Config::default(); + let ignore = IgnoreSet::compile(&config.indexing.ignore_patterns).unwrap(); + let registry = Registry::default_set(); + let event = FsEvent::Create(tree.clone()); + let cancel = AtomicBool::new(false); + let mut apply = move || { + let budget = Budget { + deadline: Instant::now() + Duration::from_secs(3600), + cancel: &cancel, + resume_from: 0, + }; + apply_fs_event(&mut conn, &event, &config, &ignore, ®istry, &budget).unwrap() + }; + // Indexed once; every timed run finds it unchanged. + apply(); + bencher.bench_local(apply); + drop(dir); + } +} diff --git a/crates/quicksearch-core/benches/index_alloc.rs b/crates/quicksearch-core/benches/index_alloc.rs index 3a390bf..9764835 100644 --- a/crates/quicksearch-core/benches/index_alloc.rs +++ b/crates/quicksearch-core/benches/index_alloc.rs @@ -183,7 +183,7 @@ fn compression_traffic_per_chunk(config: &Config) { let per_row = measure(|| { (0..CHUNK) - .map(|_| repo::encode_one(&doc, true).expect("encode")) + .map(|_| repo::encode_one(&doc).expect("encode")) .collect::>() }); // What `Bodies::fill` does: one encoder and one arena for the chunk. diff --git a/crates/quicksearch-core/benches/search.rs b/crates/quicksearch-core/benches/search.rs index eaeb839..40d41bf 100644 --- a/crates/quicksearch-core/benches/search.rs +++ b/crates/quicksearch-core/benches/search.rs @@ -177,10 +177,10 @@ mod substring { let (s, b) = snippet::extract_folded( divan::black_box(text), divan::black_box(folded), - &[corpus::NEEDLE], + corpus::NEEDLE, &opts, ); - let a = pattern.count(text, false); + let a = pattern.count(text); (a, b, s.ranges.len()) }); } @@ -194,9 +194,9 @@ mod substring { let folded = corpus::text_folded(size, 4); let opts = snippet::Options { approx_bytes: 600 }; bencher.bench(|| { - let a = pattern.count(divan::black_box(text), false); + let a = pattern.count(divan::black_box(text)); let b = pattern.count_folded(divan::black_box(folded)); - let (s, _) = snippet::extract_folded(text, folded, &[corpus::NEEDLE], &opts); + let (s, _) = snippet::extract_folded(text, folded, corpus::NEEDLE, &opts); (a, b, s.ranges.len()) }); } @@ -216,7 +216,7 @@ mod snippet_extract { snippet::extract_folded( divan::black_box(text), divan::black_box(folded), - &[corpus::NEEDLE], + corpus::NEEDLE, &opts, ) }); diff --git a/crates/quicksearch-core/examples/contentprobe.rs b/crates/quicksearch-core/examples/contentprobe.rs index 58b7d65..c28f160 100644 --- a/crates/quicksearch-core/examples/contentprobe.rs +++ b/crates/quicksearch-core/examples/contentprobe.rs @@ -397,9 +397,7 @@ fn run_stage_with( continue; } - // `scope::apply_page`'s decision. `restore_text` is false because this - // probe only narrows the filter, so the second arm reduces to "wants - // content but has none". + // `scope::apply_page`'s decision. let at = Instant::now(); pending.clear(); for row in &rows { @@ -463,9 +461,8 @@ fn write_page(tx: &Connection, stage: Stage, pending: &Pending, t: &mut Timing) (&pending.all, &pending.all) }; - // The per-row clears are `repo::remove_content_for_id` followed by - // `set_state_clearing_failure`'s `failed_files` delete, one bound - // statement at a time. They prepare through the connection's cache, as + // The per-row clears are `repo::remove_content_for_id` followed by a + // `failed_files` delete, one bound statement at a time. They prepare through the connection's cache, as // `repo::exec` does; preparing afresh per row would price a mistake the // product does not make. if stage.clears() { diff --git a/crates/quicksearch-core/examples/pruneprobe.rs b/crates/quicksearch-core/examples/pruneprobe.rs index f35e6e7..62c8a14 100644 --- a/crates/quicksearch-core/examples/pruneprobe.rs +++ b/crates/quicksearch-core/examples/pruneprobe.rs @@ -674,10 +674,10 @@ fn main() { // The page cache. `PRAGMAS_INCREMENTAL` gives the reconcile a flat // 4 MiB for a pass across the whole index. A delete also maintains - // `idx_files_mtime`, `idx_files_mime` and `idx_files_hash`, three - // b-trees in orders uncorrelated with `(parent, name)`, so every - // doomed row scatters ~3 page touches that a 512-page cache cannot - // hold. Read the `misses` column. Both commit policies are swept, + // `idx_files_hash`, a b-tree in an order uncorrelated with + // `(parent, name)`, so every doomed row scatters a page touch that a + // 512-page cache cannot hold (three, before `idx_files_mtime` and + // `idx_files_mime` were dropped). Read the `misses` column. Both commit policies are swept, // because a cache that holds the working set may make commit cadence // stop mattering. println!("\n page cache, at +fts(all) [{} = shipped]:", SHIPPED_MIB); diff --git a/crates/quicksearch-core/examples/searchtime.rs b/crates/quicksearch-core/examples/searchtime.rs index 1b85dc6..78989c7 100644 --- a/crates/quicksearch-core/examples/searchtime.rs +++ b/crates/quicksearch-core/examples/searchtime.rs @@ -5,8 +5,12 @@ //! ```text //! cargo build -p quicksearch-core --example searchtime --release //! ./target/release/examples/searchtime /path/to/index.db +//! ./target/release/examples/searchtime /path/to/index.db serialize "fn main" ~recieve //! ``` //! +//! Queries after the path replace the built-in set, for a corpus whose +//! vocabulary the defaults do not hit; a leading `~` runs one fuzzy. +//! //! Queries run against one held connection, as the worker holds one; each //! is timed best-of-N so a scheduling hiccup is not the headline. @@ -21,9 +25,14 @@ use quicksearch_core::testutil; const RUNS: u32 = 5; /// The content queries go through however many segments the write side -/// left behind; the filename query is the control and must not move. +/// left behind; the filename query is the control and must not move. The +/// filter rows price `modified:` and `mime:` against that control: neither +/// column has an index, so they must cost no more than the scan it already is. const QUERIES: &[(&str, bool, &str)] = &[ ("filename (control)", false, "doc42"), + ("filename + modified", false, "doc42 modified:>=2025-06-01"), + ("filename + one day", false, "doc42 modified:2025-06-01"), + ("filename + mime", false, "doc42 mime:application/pdf"), ("content, common", false, "mountain"), ("content, rare", false, "quartzite"), ("content, two words", false, "ocean forest"), @@ -55,8 +64,24 @@ fn main() { ); println!("{:<22} {:>10} {:>8}", "query", "best", "hits"); + let custom: Vec<(String, bool, String)> = std::env::args() + .skip(2) + .map(|q| match q.strip_prefix('~') { + Some(term) => (q.clone(), true, term.to_string()), + None => (q.clone(), false, q), + }) + .collect(); + let queries: Vec<(&str, bool, &str)> = if custom.is_empty() { + QUERIES.to_vec() + } else { + custom + .iter() + .map(|(label, fuzzy, query)| (label.as_str(), *fuzzy, query.as_str())) + .collect() + }; + let mut total = Duration::ZERO; - for (label, fuzzy, query) in QUERIES { + for &(label, fuzzy, query) in &queries { // The display limit makes this unfair: `scan_pass` stops when the // limit fills, and candidates stream in insertion order, so which // documents drew low ids changes bytes read 13x. A `QSB_LIMIT` past @@ -66,7 +91,7 @@ fn main() { .and_then(|v| v.parse().ok()) .unwrap_or(1000); let options = SearchOptions { - fuzzy: *fuzzy, + fuzzy, limit, ..SearchOptions::default() }; diff --git a/crates/quicksearch-core/examples/walkprobe.rs b/crates/quicksearch-core/examples/walkprobe.rs index f6113f5..212a031 100644 --- a/crates/quicksearch-core/examples/walkprobe.rs +++ b/crates/quicksearch-core/examples/walkprobe.rs @@ -40,7 +40,7 @@ fn main() { std::process::id() )); let _ = std::fs::remove_file(&db); - quicksearch_core::db::open_or_recreate(db.to_str().unwrap(), &config.processing.tokenize) + quicksearch_core::db::open_or_recreate(db.to_str().unwrap(), &config.processing) .expect("scratch index"); let existing = DirRows::new(); diff --git a/crates/quicksearch-core/src/cli.rs b/crates/quicksearch-core/src/cli.rs index a7eead9..16353e8 100644 --- a/crates/quicksearch-core/src/cli.rs +++ b/crates/quicksearch-core/src/cli.rs @@ -197,7 +197,8 @@ pub fn clear_path(db_path: &str, path: &str) -> Result { mod tests { use super::*; use crate::db::open_or_recreate; - use crate::db::repo::{insert_file, set_content_done, set_content_failed, NewFile}; + use crate::db::repo::{complete_pending, insert_file, store_pending_text, NewFile}; + use crate::file_handling::ContentOutcome; use crate::mime::FileType; use crate::testutil::{zstd_of, Scratch}; @@ -226,13 +227,15 @@ mod tests { } fn seed_fixture(db_path: &str) -> (i64, i64) { - let mut conn = open_or_recreate(db_path, "trigram").unwrap(); + let mut conn = open_or_recreate(db_path, &Default::default()).unwrap(); let (a, b) = { let tx = conn.transaction().unwrap(); let a = row(&tx, "a.txt", true); - set_content_done(&tx, a, "hello", zstd_of("hello").as_deref()).unwrap(); - let b = row(&tx, "b.bin", false); - set_content_failed(&tx, b, "bad extract").unwrap(); + store_pending_text(&tx, a, "hello", zstd_of("hello").as_deref()).unwrap(); + // Pending, not NA: only an extraction that ran can fail. + let b = row(&tx, "b.bin", true); + let failed = ContentOutcome::Failed("bad extract".into()); + assert!(complete_pending(&tx, b, &failed, None).unwrap()); tx.commit().unwrap(); (a, b) }; @@ -279,7 +282,7 @@ mod tests { let _ = seed_fixture(dbp); assert_eq!(pending_content_count(dbp).unwrap(), 0); - let mut conn = open_or_recreate(dbp, "trigram").unwrap(); + let mut conn = open_or_recreate(dbp, &Default::default()).unwrap(); { let tx = conn.transaction().unwrap(); // Claimed by an extractor, text not read yet: the backlog. @@ -321,12 +324,12 @@ mod tests { fn documents_text_ratio_reports_savings_on_compressible_prose() { // Guards against silently swapping compression for a pass-through. let (_dir, p) = Scratch::db("cli"); - let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + let mut conn = open_or_recreate(p.to_str().unwrap(), &Default::default()).unwrap(); { let tx = conn.transaction().unwrap(); let id = row(&tx, "big.txt", true); let prose = "the quick brown fox jumps over the lazy dog. ".repeat(500); - set_content_done(&tx, id, &prose, zstd_of(&prose).as_deref()).unwrap(); + store_pending_text(&tx, id, &prose, zstd_of(&prose).as_deref()).unwrap(); tx.commit().unwrap(); } drop(conn); @@ -359,7 +362,11 @@ mod tests { let (_dir, p) = Scratch::db("cli"); let dbp = p.to_str().unwrap(); { - let mut conn = open_or_recreate(dbp, "unicode61").unwrap(); + let unicode61 = crate::config::ProcessingConfig { + tokenize: "unicode61".into(), + ..Default::default() + }; + let mut conn = open_or_recreate(dbp, &unicode61).unwrap(); let tx = conn.transaction().unwrap(); for name in ["a.txt", "b.txt"] { row(&tx, name, true); diff --git a/crates/quicksearch-core/src/config/diff.rs b/crates/quicksearch-core/src/config/diff.rs index b7eda8c..fc2d080 100644 --- a/crates/quicksearch-core/src/config/diff.rs +++ b/crates/quicksearch-core/src/config/diff.rs @@ -30,11 +30,6 @@ pub struct IndexWork { /// back to pending; newly excluded ones lose text and FTS but keep their /// row. pub reconcile_content: bool, - /// `store_text_for_snippets` turned on; rows extracted under the old - /// setting kept no text, so they must run again. - pub restore_text: bool, - /// `store_text_for_snippets` turned off. - pub drop_text: bool, /// Newly in-scope files exist only on disk, so only a walk can find them. pub reindex: bool, } @@ -56,8 +51,6 @@ impl IndexWork { self.drop_aliases |= other.drop_aliases; self.prune_scope |= other.prune_scope; self.reconcile_content |= other.reconcile_content; - self.restore_text |= other.restore_text; - self.drop_text |= other.drop_text; self.reindex |= other.reindex; } @@ -67,14 +60,12 @@ impl IndexWork { || self.drop_aliases || self.prune_scope || self.reconcile_content - || self.restore_text - || self.drop_text } /// Whether applying this scans rows under each surviving root, instead /// of only deleting whole ranges. pub fn scans_rows(&self) -> bool { - self.prune_scope || self.reconcile_content || self.restore_text || self.drop_text + self.prune_scope || self.reconcile_content } /// The plan in one line for the log, naming what changed. @@ -88,8 +79,6 @@ impl IndexWork { self.reconcile_content, "changed which files have their text read", ), - (self.restore_text, "snippet text turned on"), - (self.drop_text, "snippet text turned off"), ] .into_iter() .filter_map(|(changed, part)| changed.then_some(part)) @@ -174,12 +163,13 @@ fn ignore_set(c: &Config) -> BTreeSet<&str> { pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions { let search_db_changed = old.paths.database_path != new.paths.database_path; - // The only three: a different key makes the file unreadable, the - // tokenizer is baked into the FTS table definition, and `hash_length` - // decides what bytes a stored hash covers. (`use_keychain` changes only - // where the key is remembered.) + // The only four: a different key makes the file unreadable, the + // tokenizer and the position setting are baked into the FTS table + // definition, and `hash_length` decides what bytes a stored hash covers. + // (`use_keychain` changes only where the key is remembered.) let requires_rebuild = old.processing.hash_length != new.processing.hash_length || old.processing.tokenize != new.processing.tokenize + || old.processing.fts_positions() != new.processing.fts_positions() || old.security.password_protected != new.security.password_protected || old.security.salt != new.security.salt; if requires_rebuild { @@ -200,9 +190,6 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions { let new_content = content_filter_set(&new_idx.content_extensions); let old_cap = old.processing.maximum_text_file_size; let new_cap = new.processing.maximum_text_file_size; - let old_store = old.processing.store_text_for_snippets; - let new_store = new.processing.store_text_for_snippets; - let restore_text = !old_store && new_store; let work = IndexWork { drop_roots: old_roots.difference(&new_roots).cloned().collect(), @@ -215,8 +202,6 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions { drop_aliases: old_idx.follow_symlinks && !new_idx.follow_symlinks, // See the field: the size cap decides what the extension filter does. reconcile_content: old_content != new_content || old_cap != new_cap, - restore_text, - drop_text: old_store && !new_store, // Widening only adds files, and a file absent from the index is not // findable from it: only a walk produces those rows. Re-extraction // needs a run too, since the content pass runs as part of one; a @@ -226,8 +211,7 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions { || (!old_idx.include_hidden && new_idx.include_hidden) || (!old_idx.follow_symlinks && new_idx.follow_symlinks) || filter_widened(&old_content, &new_content) - || new_cap > old_cap - || restore_text, + || new_cap > old_cap, }; ConfigActions { diff --git a/crates/quicksearch-core/src/config/mod.rs b/crates/quicksearch-core/src/config/mod.rs index 552736a..e407bd0 100644 --- a/crates/quicksearch-core/src/config/mod.rs +++ b/crates/quicksearch-core/src/config/mod.rs @@ -93,11 +93,30 @@ pub struct ProcessingConfig { /// User-facing guidance is in `config_example.toml`. pub maximum_wal_size: u64, pub tokenize: String, - /// When `true` (default), extracted text is stored zstd-compressed in - /// `documents_text` for snippet previews. When `false` queries return the - /// same hits, but results carry no snippets, can't be case-verified or - /// occurrence-ranked, and fuzzy full-text search is unavailable. - pub store_text_for_snippets: bool, + /// When `true` (default), the full-text index keeps no word positions + /// (FTS5 `detail=none`): a term is looked up as the AND of its trigrams + /// and the full-text pass drops the documents where they are not + /// adjacent, against the stored text. Several times smaller and faster + /// to build, at the cost of checking more candidates per search; see + /// [`crate::db::schema::fts_create_sql`] for the measurements. Only + /// `trigram` can be queried that way; see [`ProcessingConfig::fts_positions`]. + pub compact_text_index: bool, +} + +impl ProcessingConfig { + /// Whether the full-text index is built with word positions. Everything + /// that decides the index's layout asks this, never + /// [`ProcessingConfig::compact_text_index`] directly: a word tokenizer + /// keeps positions whatever that says, because a multi-word phrase has no + /// trigram form to fall back on, and toggling it there changes nothing. + pub fn fts_positions(&self) -> bool { + let trigram = self + .tokenize + .split_whitespace() + .next() + .is_some_and(|name| name.eq_ignore_ascii_case("trigram")); + !(self.compact_text_index && trigram) + } } /// Floor on a non-zero [`ProcessingConfig::maximum_wal_size`]: a checkpoint @@ -231,7 +250,7 @@ impl Default for ProcessingConfig { writer_turn_slice_ms: 100, maximum_wal_size: 1024 * 1024 * 1024 * 2, tokenize: "trigram".to_string(), - store_text_for_snippets: true, + compact_text_index: true, } } } diff --git a/crates/quicksearch-core/src/config/tests.rs b/crates/quicksearch-core/src/config/tests.rs index 96d26ad..3fce3c2 100644 --- a/crates/quicksearch-core/src/config/tests.rs +++ b/crates/quicksearch-core/src/config/tests.rs @@ -671,16 +671,41 @@ fn turning_symlinks_off_drops_aliases_and_on_walks() { ); } -/// Stored text: on re-extracts, off drops the blobs; never a rebuild. +/// The position setting is part of the FTS table definition, so a change +/// rebuilds; but only a change of the layout it produces, and a word +/// tokenizer keeps positions either way. #[test] -fn stored_text_on_re_extracts_and_off_drops_it() { +fn the_compact_index_flag_rebuilds_only_when_the_layout_changes() { let base = wide_base(); - let off = edited(&base, |c| c.processing.store_text_for_snippets = false); - let on = edited(&base, |c| c.processing.store_text_for_snippets = true); - let a = diff_actions(&on, &off); - assert!(a.work.drop_text && !a.work.restore_text && !a.work.reindex); - let a = diff_actions(&off, &on); - assert!(a.work.restore_text && !a.work.drop_text && a.work.reindex); + let full = edited(&base, |c| c.processing.compact_text_index = false); + assert!(diff_actions(&base, &full).requires_rebuild); + assert!(diff_actions(&full, &base).requires_rebuild); + + let words = edited(&base, |c| c.processing.tokenize = "unicode61".into()); + let words_full = edited(&words, |c| c.processing.compact_text_index = false); + assert_eq!( + diff_actions(&words, &words_full), + ConfigActions::default(), + "positions either way: nothing to do" + ); +} + +#[test] +fn only_trigram_can_drop_positions() { + let mut p = ProcessingConfig::default(); + assert!(p.compact_text_index, "compact by default"); + assert!(!p.fts_positions()); + p.tokenize = "trigram case_sensitive 1".into(); + assert!(!p.fts_positions(), "trigram with options is still trigram"); + p.tokenize = " TRIGRAM ".into(); + assert!(!p.fts_positions()); + for word in ["unicode61", "porter", "porter unicode61", "trigrammer"] { + p.tokenize = word.into(); + assert!(p.fts_positions(), "{word} cannot be queried by trigrams"); + } + p.tokenize = "trigram".into(); + p.compact_text_index = false; + assert!(p.fts_positions()); } #[test] @@ -713,7 +738,7 @@ fn merging_two_plans_loses_nothing() { let first = IndexWork { drop_roots: vec!["/gone".into(), "/shared".into()], prune_scope: true, - drop_text: true, + drop_aliases: true, ..IndexWork::default() }; let second = IndexWork { @@ -730,7 +755,7 @@ fn merging_two_plans_loses_nothing() { vec!["/shared", "/also-gone", "/gone"], "every root from both, each once" ); - assert!(merged.prune_scope && merged.drop_text); + assert!(merged.prune_scope && merged.drop_aliases); assert!(merged.reconcile_content && merged.reindex); // Empty merge is a no-op and self-merge is the identity; both make a diff --git a/crates/quicksearch-core/src/content.rs b/crates/quicksearch-core/src/content.rs index 4a88bc1..c2fbc95 100644 --- a/crates/quicksearch-core/src/content.rs +++ b/crates/quicksearch-core/src/content.rs @@ -331,12 +331,18 @@ mod tests { use crate::db::open_or_recreate; use crate::db::repo::{self, insert_file, NewFile}; - use crate::file_handling::{extract_scope_prepare, store_extracted, ExtractScope, Stored}; + use crate::file_handling::{ + extract_scope_prepare, store_extracted, Bodies, ExtractScope, Stored, + }; use crate::mime::FileType; use crate::testutil::Scratch; use std::path::PathBuf; use std::time::{Duration, Instant}; + fn bodies() -> Bodies { + Bodies::new().unwrap() + } + fn tree(s: &Scratch) -> PathBuf { s.join("tree") } @@ -349,7 +355,7 @@ mod tests { /// a pending row for each. The guard owns both. fn seed(tag: &str, dirs: &[(&str, usize)]) -> Scratch { let s = Scratch::dir(tag); - let mut conn = open_or_recreate(db(&s).to_str().unwrap(), "trigram").unwrap(); + let mut conn = open_or_recreate(db(&s).to_str().unwrap(), &Default::default()).unwrap(); let tx = conn.transaction().unwrap(); for (dir, n) in dirs { let d = tree(&s).join(dir); @@ -385,7 +391,7 @@ mod tests { fn writer(s: &Scratch) -> Arc> { Arc::new(Mutex::new( - open_or_recreate(db(s).to_str().unwrap(), "trigram").unwrap(), + open_or_recreate(db(s).to_str().unwrap(), &Default::default()).unwrap(), )) } @@ -422,7 +428,7 @@ mod tests { assert!(pass.finish()); let stop = Arc::new(AtomicBool::new(false)); let far = Instant::now() + Duration::from_secs(60); - store_extracted(conn, &rows, &stop, &Config::default(), far).unwrap() + store_extracted(conn, &rows, &stop, far, &mut bodies()).unwrap() } #[test] @@ -479,15 +485,7 @@ mod tests { "out-of-range row untouched" ); - let hits: i64 = conn_mutex - .lock() - .unwrap() - .query_row( - "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH '\"sphinx\"'", - [], - |r| r.get(0), - ) - .unwrap(); + let hits = crate::testutil::fts_count(&conn_mutex.lock().unwrap(), "sphinx"); assert_eq!(hits, 3); } @@ -545,7 +543,6 @@ mod tests { fn store_extracted_honours_its_deadline_but_always_makes_progress() { let s = seed("deadline", &[("r1", 5)]); let conn_mutex = writer(&s); - let config = Config::default(); let mut pass = pass_for(&s, "r1", 2); let rows = drain(&mut pass); assert!(pass.finish()); @@ -555,7 +552,7 @@ mod tests { // A deadline already gone by: one row, then out. let past = Instant::now() - Duration::from_secs(1); assert_eq!( - store_extracted(&conn_mutex, &rows, &stop, &config, past).unwrap(), + store_extracted(&conn_mutex, &rows, &stop, past, &mut bodies()).unwrap(), Stored { consumed: 1, written: 1 @@ -563,7 +560,7 @@ mod tests { ); let far = Instant::now() + Duration::from_secs(60); assert_eq!( - store_extracted(&conn_mutex, &rows[1..], &stop, &config, far).unwrap(), + store_extracted(&conn_mutex, &rows[1..], &stop, far, &mut bodies()).unwrap(), Stored { consumed: 4, written: 4 @@ -583,7 +580,7 @@ mod tests { // Stopped before it starts: nothing consumed. stop.store(true, std::sync::atomic::Ordering::Relaxed); assert_eq!( - store_extracted(&conn_mutex, &rows, &stop, &config, far).unwrap(), + store_extracted(&conn_mutex, &rows, &stop, far, &mut bodies()).unwrap(), Stored::default() ); } diff --git a/crates/quicksearch-core/src/db/open.rs b/crates/quicksearch-core/src/db/open.rs index f30f4f5..a912e7e 100644 --- a/crates/quicksearch-core/src/db/open.rs +++ b/crates/quicksearch-core/src/db/open.rs @@ -7,10 +7,11 @@ use std::path::Path; use rusqlite::{params, Connection, OpenFlags, OptionalExtension}; use super::schema::{ - effective_tokenizer, fts_create_sql, fts_set_pgsz, pragmas_search, - recommended_search_cache_mib, Profile, PRAGMAS_FAST, PRAGMAS_INCREMENTAL, PRAGMAS_MAINTENANCE, - PRAGMAS_READONLY, PRAGMAS_WALK_READER, SCHEMA_CURRENT, + fts_create_sql, fts_set_pgsz, pragmas_search, recommended_search_cache_mib, Profile, + PRAGMAS_FAST, PRAGMAS_INCREMENTAL, PRAGMAS_MAINTENANCE, PRAGMAS_READONLY, PRAGMAS_WALK_READER, + SCHEMA_CURRENT, }; +use crate::config::ProcessingConfig; use crate::security::IndexKey; /// Prefix tagging every "the key does not fit this file" error, so a wrong @@ -26,18 +27,25 @@ pub const KEY_MISMATCH_PREFIX: &str = "KEY_MISMATCH: "; /// table. Keyed indexes are rebuilt anyway via the profile retry in /// [`open_probed`]; the bump rebuilds unprotected ones too. /// +/// v12 dropped the `mtime` and `mime` indexes (see [`schema::SCHEMA_CURRENT`]). +/// /// [`schema::HMAC_MODE`]: super::schema::HMAC_MODE -pub const CURRENT_SCHEMA_VERSION: u32 = 11; +/// [`schema::SCHEMA_CURRENT`]: super::schema::SCHEMA_CURRENT +pub const CURRENT_SCHEMA_VERSION: u32 = 12; -/// Open `db_path`; on any schema or tokenizer mismatch, delete the file and +/// Open `db_path`; on a schema mismatch, or a full-text layout other than +/// `processing` asks for (tokenizer, positions), delete the file and /// recreate it empty, for the caller to re-index. -pub fn open_or_recreate(db_path: &str, tokenizer: &str) -> Result { - open_or_recreate_keyed(db_path, tokenizer, super::key::process_key().as_ref()) +pub fn open_or_recreate( + db_path: &str, + processing: &ProcessingConfig, +) -> Result { + open_or_recreate_keyed(db_path, processing, super::key::process_key().as_ref()) } pub(crate) fn open_or_recreate_keyed( db_path: &str, - tokenizer: &str, + processing: &ProcessingConfig, key: Option<&IndexKey>, ) -> Result { let path = Path::new(db_path).to_path_buf(); @@ -67,7 +75,7 @@ pub(crate) fn open_or_recreate_keyed( ProfileMatch::Previous(found) => { format!("was built with {} and this build uses {}", found, want) } - ProfileMatch::Current if db_matches_current(&conn, tokenizer)? => return Ok(conn), + ProfileMatch::Current if db_matches_current(&conn, processing)? => return Ok(conn), ProfileMatch::Current => "does not match current schema".to_string(), }; crate::log_warn!( @@ -76,7 +84,7 @@ pub(crate) fn open_or_recreate_keyed( why ); let conn = wipe_and_reopen(conn, &path, key, want)?; - apply_current_schema(&conn, tokenizer, key, want)?; + apply_current_schema(&conn, processing, key, want)?; Ok(conn) } @@ -491,9 +499,9 @@ fn schema_version_current(conn: &Connection) -> Result { Ok(version.as_deref() == Some(&CURRENT_SCHEMA_VERSION.to_string())) } -/// True iff the DB has the current schema version *and* the -/// effective-tokenizer string this caller asked for. -fn db_matches_current(conn: &Connection, tokenizer: &str) -> Result { +/// True iff the DB has the current schema version *and* the full-text layout +/// this caller asked for. +fn db_matches_current(conn: &Connection, processing: &ProcessingConfig) -> Result { if !schema_version_current(conn)? { // Refuse rather than wipe unless the file is recognisably ours: // `database_path` is free text with no confirmation, and a typo @@ -504,8 +512,11 @@ fn db_matches_current(conn: &Connection, tokenizer: &str) -> Result, profile: Profile, ) -> Result<(), String> { conn.execute_batch(SCHEMA_CURRENT) .map_err(|e| format!("Failed to create current schema tables: {}", e))?; - let fts = fts_create_sql(tokenizer); + let positions = processing.fts_positions(); + let fts = fts_create_sql(&processing.tokenize, positions); conn.execute_batch(&fts) .map_err(|e| format!("Failed to create searchabletext: {}", e))?; // Only here: FTS5's leaf size must suit this file's page size and @@ -573,13 +585,16 @@ fn apply_current_schema( .map_err(|e| format!("Failed to set searchabletext pgsz: {}", e))?; let now = crate::log::now_unix(); - let effective = effective_tokenizer(tokenizer); + // `fts_positions` is read back by every search (`repo::fts_positions`), + // which must phrase its queries for the table as it was built. conn.execute( - "INSERT INTO schema_info(key, value) VALUES ('version', ?1), ('created_at', ?2), ('tokenize', ?3)", + "INSERT INTO schema_info(key, value) VALUES ('version', ?1), ('created_at', ?2), \ + ('tokenize', ?3), ('fts_positions', ?4)", params![ CURRENT_SCHEMA_VERSION.to_string(), now.to_string(), - effective + processing.tokenize.trim(), + positions.to_string() ], ) .map_err(|e| format!("Failed to seed schema_info: {}", e))?; diff --git a/crates/quicksearch-core/src/db/open_tests.rs b/crates/quicksearch-core/src/db/open_tests.rs index 49210c9..7747cb2 100644 --- a/crates/quicksearch-core/src/db/open_tests.rs +++ b/crates/quicksearch-core/src/db/open_tests.rs @@ -15,6 +15,24 @@ fn info(conn: &Connection, key: &str) -> String { .unwrap() } +/// Processing settings asking for `tokenize`, the rest default. +fn with_tokenizer(tokenize: &str) -> ProcessingConfig { + ProcessingConfig { + tokenize: tokenize.into(), + ..Default::default() + } +} + +/// The DDL the full-text table was created with. +fn fts_sql(conn: &Connection) -> String { + conn.query_row( + "SELECT sql FROM sqlite_master WHERE name = 'searchabletext'", + [], + |r| r.get(0), + ) + .unwrap() +} + fn file_count(conn: &Connection) -> i64 { conn.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0)) .unwrap() @@ -42,15 +60,15 @@ fn write_v1_index(p: &str, name: &str) { #[test] fn fresh_db_gets_current_version() { let (_dir, p) = db(); - let conn = open_or_recreate(&p, "trigram").unwrap(); + let conn = open_or_recreate(&p, &Default::default()).unwrap(); assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string()); } #[test] fn reopen_is_idempotent() { let (_dir, p) = db(); - drop(open_or_recreate(&p, "trigram").unwrap()); - let conn = open_or_recreate(&p, "trigram").unwrap(); + drop(open_or_recreate(&p, &Default::default()).unwrap()); + let conn = open_or_recreate(&p, &Default::default()).unwrap(); assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string()); } @@ -58,7 +76,7 @@ fn reopen_is_idempotent() { fn older_versioned_db_is_wiped_and_recreated() { let (_dir, p) = db(); write_v1_index(&p, "a.txt"); - let conn = open_or_recreate(&p, "trigram").unwrap(); + let conn = open_or_recreate(&p, &Default::default()).unwrap(); assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string()); assert_eq!(file_count(&conn), 0, "old rows should be wiped"); } @@ -79,7 +97,7 @@ fn legacy_layout_db_is_wiped_and_recreated() { ) .unwrap(); } - let conn = open_or_recreate(&p, "trigram").unwrap(); + let conn = open_or_recreate(&p, &Default::default()).unwrap(); assert_eq!(file_count(&conn), 0); // An unknown column name fails to prepare. conn.prepare("SELECT content_state, type, mime FROM files LIMIT 0") @@ -90,7 +108,7 @@ fn legacy_layout_db_is_wiped_and_recreated() { fn tokenizer_drift_wipes_db() { let (_dir, p) = db(); let first_effective = { - let conn = open_or_recreate(&p, "trigram").unwrap(); + let conn = open_or_recreate(&p, &Default::default()).unwrap(); conn.execute( "INSERT INTO files (name, parent, size, mtime) \ VALUES ('x', '/', 0, 0)", @@ -99,16 +117,77 @@ fn tokenizer_drift_wipes_db() { .unwrap(); info(&conn, "tokenize") }; - let conn = open_or_recreate(&p, "unicode61").unwrap(); + let conn = open_or_recreate(&p, &with_tokenizer("unicode61")).unwrap(); assert_eq!(file_count(&conn), 0, "tokenizer drift should wipe rows"); assert_ne!(first_effective, info(&conn, "tokenize")); } +/// The position setting is part of the table definition, like the +/// tokenizer: a change wipes, and the layout built is the one recorded, +/// which is what every search reads back. +#[test] +fn position_drift_wipes_db_and_the_layout_is_recorded() { + let (_dir, p) = db(); + { + let conn = open_or_recreate(&p, &Default::default()).unwrap(); + assert!(fts_sql(&conn).contains("detail=none"), "compact by default"); + assert_eq!(info(&conn, "fts_positions"), "false"); + assert!(!crate::db::repo::fts_positions(&conn)); + conn.execute( + "INSERT INTO files (name, parent, size, mtime) VALUES ('x', '/', 0, 0)", + [], + ) + .unwrap(); + } + let full = ProcessingConfig { + compact_text_index: false, + ..Default::default() + }; + let conn = open_or_recreate(&p, &full).unwrap(); + assert_eq!(file_count(&conn), 0, "a layout change wipes the rows"); + assert!(!fts_sql(&conn).contains("detail")); + assert_eq!(info(&conn, "fts_positions"), "true"); + assert!(crate::db::repo::fts_positions(&conn)); +} + +/// A word tokenizer cannot be queried by trigrams, so it keeps positions +/// whatever the flag says, and the flag alone then changes nothing. +#[test] +fn a_word_tokenizer_keeps_positions_and_ignores_the_flag() { + let (_dir, p) = db(); + { + let conn = open_or_recreate(&p, &with_tokenizer("unicode61")).unwrap(); + assert!(!fts_sql(&conn).contains("detail")); + assert_eq!(info(&conn, "fts_positions"), "true"); + conn.execute( + "INSERT INTO files (name, parent, size, mtime) VALUES ('x', '/', 0, 0)", + [], + ) + .unwrap(); + } + let flag_off = ProcessingConfig { + compact_text_index: false, + ..with_tokenizer("unicode61") + }; + let conn = open_or_recreate(&p, &flag_off).unwrap(); + assert_eq!(file_count(&conn), 1, "the same layout: nothing to rebuild"); +} + +/// An index from before the layout was recorded kept positions. +#[test] +fn an_unrecorded_layout_reads_as_positions() { + let (_dir, p) = db(); + let conn = open_or_recreate(&p, &Default::default()).unwrap(); + conn.execute("DELETE FROM schema_info WHERE key = 'fts_positions'", []) + .unwrap(); + assert!(crate::db::repo::fts_positions(&conn)); +} + #[test] fn open_existing_reads_nondefault_tokenizer_without_wiping() { let (_dir, p) = db(); { - let conn = open_or_recreate(&p, "unicode61").unwrap(); + let conn = open_or_recreate(&p, &with_tokenizer("unicode61")).unwrap(); conn.execute( "INSERT INTO files (name, parent, size, mtime) \ VALUES ('note', '/', 0, 0)", @@ -129,6 +208,7 @@ fn open_existing_reads_nondefault_tokenizer_without_wiping() { 1, "open_existing must not wipe a non-default-tokenizer DB" ); + // A word tokenizer: `hello` is one token, so a bare MATCH is exact. let hits: i64 = conn .query_row( "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'hello'", @@ -145,7 +225,7 @@ fn open_existing_reads_nondefault_tokenizer_without_wiping() { fn open_or_recreate_creates_missing_parent_dirs() { let dir = Scratch::dir("mkdir"); let db = dir.join("nested/deeper/index.sqlite"); - drop(open_or_recreate(db.to_str().unwrap(), "trigram").unwrap()); + drop(open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap()); assert!(db.exists()); } @@ -156,7 +236,7 @@ fn writable_opens_use_wal_and_it_persists() { conn.query_row("PRAGMA journal_mode", [], |r| r.get(0)) .unwrap() }; - let conn = open_or_recreate(&p, "trigram").unwrap(); + let conn = open_or_recreate(&p, &Default::default()).unwrap(); assert_eq!(mode(&conn).to_lowercase(), "wal"); drop(conn); // WAL is persistent in the file: a later read-only consumer sees it. @@ -173,7 +253,7 @@ fn maintenance_opens_keep_temporaries_on_disk() { conn.query_row("PRAGMA temp_store", [], |r| r.get(0)) .unwrap() }; - let conn = open_or_recreate(&p, "trigram").unwrap(); + let conn = open_or_recreate(&p, &Default::default()).unwrap(); assert_eq!(temp_store(&conn), 2, "the indexer's own profile is MEMORY"); drop(conn); @@ -211,13 +291,13 @@ fn index_needs_rebuild_only_when_the_schema_really_differs() { let cases: [(Fixture, bool, &str); 5] = [ (|_| {}, false, "no file yet is a fresh install, not a reset"), ( - |p| drop(open_or_recreate(p, "trigram").unwrap()), + |p| drop(open_or_recreate(p, &Default::default()).unwrap()), false, "a current index is not going to be rebuilt", ), ( |p| { - drop(open_or_recreate(p, "trigram").unwrap()); + drop(open_or_recreate(p, &Default::default()).unwrap()); open_existing(p, true) .unwrap() .execute( @@ -289,7 +369,7 @@ fn keyed_create_reopen_and_header_is_encrypted() { let (_dir, p) = db(); let key = test_key(0xa1); { - let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap(); + let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&key)).unwrap(); conn.execute( "INSERT INTO files (name, parent, size, mtime) \ VALUES ('secret', '/', 0, 0)", @@ -300,7 +380,7 @@ fn keyed_create_reopen_and_header_is_encrypted() { let head = &file_bytes(&p)[..16]; assert_ne!(head, b"SQLite format 3\0", "file must not be plaintext"); - let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap(); + let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&key)).unwrap(); assert_eq!( file_count(&conn), 1, @@ -315,7 +395,7 @@ fn keyed_create_reopen_and_header_is_encrypted() { fn wrong_key_errors_without_wiping() { let (_dir, p) = db(); { - let conn = open_or_recreate_keyed(&p, "trigram", Some(&test_key(0xa1))).unwrap(); + let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&test_key(0xa1))).unwrap(); conn.execute( "INSERT INTO files (name, parent, size, mtime) \ VALUES ('x', '/', 0, 0)", @@ -329,7 +409,7 @@ fn wrong_key_errors_without_wiping() { assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}"); } // A wrong key is never a "schema mismatch" to answer with a wipe. - let err = open_or_recreate_keyed(&p, "trigram", Some(&test_key(0xb2))).unwrap_err(); + let err = open_or_recreate_keyed(&p, &Default::default(), Some(&test_key(0xb2))).unwrap_err(); assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}"); assert_eq!(before, file_bytes(&p), "file must be byte-identical"); } @@ -337,12 +417,12 @@ fn wrong_key_errors_without_wiping() { #[test] fn missing_key_on_encrypted_db_errors_without_wiping() { let (_dir, p) = db(); - drop(open_or_recreate_keyed(&p, "trigram", Some(&test_key(0xa1))).unwrap()); + drop(open_or_recreate_keyed(&p, &Default::default(), Some(&test_key(0xa1))).unwrap()); let before = file_bytes(&p); let err = open_existing_keyed(&p, false, None).unwrap_err(); assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}"); assert!(err.contains("password-protected"), "got: {err}"); - let err = open_or_recreate_keyed(&p, "trigram", None).unwrap_err(); + let err = open_or_recreate_keyed(&p, &Default::default(), None).unwrap_err(); assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}"); assert_eq!(before, file_bytes(&p)); } @@ -350,13 +430,13 @@ fn missing_key_on_encrypted_db_errors_without_wiping() { #[test] fn key_on_plaintext_db_errors_without_wiping() { let (_dir, p) = db(); - drop(open_or_recreate(&p, "trigram").unwrap()); + drop(open_or_recreate(&p, &Default::default()).unwrap()); let before = file_bytes(&p); let err = open_existing_keyed(&p, false, Some(&test_key(0xa1))).unwrap_err(); assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}"); // The sniffed plaintext header yields the precise diagnosis. assert!(err.contains("not encrypted"), "got: {err}"); - let err = open_or_recreate_keyed(&p, "trigram", Some(&test_key(0xa1))).unwrap_err(); + let err = open_or_recreate_keyed(&p, &Default::default(), Some(&test_key(0xa1))).unwrap_err(); assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}"); assert_eq!(before, file_bytes(&p)); } @@ -377,7 +457,7 @@ fn schema_mismatch_under_key_wipes_and_recreates_encrypted() { key_and_probe(&conn, &p, Some(&key), previous).unwrap(); write_v1_schema_info(&conn); } - let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap(); + let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&key)).unwrap(); assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string()); drop(conn); let head = &file_bytes(&p)[..16]; @@ -401,7 +481,7 @@ fn garbage_file_with_key_reports_mismatch_not_corruption() { fn open_existing_rw_allows_delete() { let (_dir, p) = db(); { - let conn = open_or_recreate(&p, "trigram").unwrap(); + let conn = open_or_recreate(&p, &Default::default()).unwrap(); conn.execute( "INSERT INTO files (name, parent, size, mtime) \ VALUES ('a', '/', 0, 0)", @@ -428,7 +508,7 @@ fn a_fresh_index_and_its_sidecars_are_owner_only() { // test: an existing directory keeps whatever mode its owner chose. let dir = Scratch::dir("open"); let p = dir.join("data").join("index.sqlite"); - let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + let conn = open_or_recreate(p.to_str().unwrap(), &Default::default()).unwrap(); conn.execute( "INSERT INTO files (name, parent, size, mtime) \ VALUES ('a', '/', 0, 0)", @@ -462,7 +542,7 @@ fn maintain_reads_its_pragmas_on_a_keyed_index() { let (_dir, p) = db(); let key = test_key(0xc3); { - let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap(); + let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&key)).unwrap(); conn.execute( "INSERT INTO files (name, parent, size, mtime) VALUES ('x', '/', 0, 0)", [], @@ -484,6 +564,16 @@ fn maintain_reads_its_pragmas_on_a_keyed_index() { ); } +/// A layout with positions: `pgsz` caps a leaf the same in either layout, +/// and position lists fill leaves soonest, so the geometry tests below need +/// the smallest corpus this way. +fn positional() -> ProcessingConfig { + ProcessingConfig { + compact_text_index: false, + ..Default::default() + } +} + /// Write enough text that FTS5 emits several *full* leaves. A corpus of tiny /// documents fits in one part-filled leaf, never reaches the inline-payload /// limit, and would let the tests below pass on a broken geometry. @@ -565,7 +655,7 @@ fn assert_leaves_inline(conn: &Connection, keyed: bool) { fn keyed_fts_leaves_stay_inline() { let (_dir, p) = db(); let key = test_key(0xd4); - let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap(); + let conn = open_or_recreate_keyed(&p, &positional(), Some(&key)).unwrap(); assert_eq!( fts_stored_pgsz(&conn), Some(crate::db::schema::fts_pgsz_for( @@ -638,7 +728,7 @@ fn derived_fts_pgsz_fits_inline_at_every_swept_page_size() { fn a_plain_index_gets_the_derived_pgsz_for_its_page_size() { use crate::db::schema::{fts_pgsz_for, PROFILE}; let (_dir, p) = db(); - let conn = open_or_recreate(&p, "trigram").unwrap(); + let conn = open_or_recreate(&p, &positional()).unwrap(); assert_eq!( fts_stored_pgsz(&conn), Some(fts_pgsz_for(PROFILE, false)), @@ -672,7 +762,7 @@ fn a_foreign_database_is_refused_not_wiped() { ) .unwrap(); } - let err = open_or_recreate(&p, "trigram").unwrap_err(); + let err = open_or_recreate(&p, &Default::default()).unwrap_err(); assert!(err.starts_with(FOREIGN_DB_PREFIX), "got: {err}"); assert!( err.contains("moz_places"), @@ -692,7 +782,7 @@ fn an_empty_database_file_is_still_ours_to_build() { let (_dir, p) = db(); // An empty but real SQLite file, header and all. drop(Connection::open(&p).unwrap()); - let conn = open_or_recreate(&p, "trigram").unwrap(); + let conn = open_or_recreate(&p, &Default::default()).unwrap(); assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string()); } @@ -706,7 +796,7 @@ fn a_legacy_table_marks_a_file_as_ours() { .unwrap() .execute(&format!("CREATE TABLE {} (x INTEGER)", table), []) .unwrap(); - open_or_recreate(&p, "trigram") + open_or_recreate(&p, &Default::default()) .unwrap_or_else(|e| panic!("{table} should read as ours, got: {e}")); } } @@ -723,7 +813,7 @@ fn a_schema_info_without_a_version_row_is_still_ours() { [], ) .unwrap(); - open_or_recreate(&p, "trigram").expect("half-built index is ours"); + open_or_recreate(&p, &Default::default()).expect("half-built index is ours"); } /// A table that merely *shares the name* `schema_info` is not ours. @@ -737,7 +827,7 @@ fn a_foreign_schema_info_is_refused() { conn.execute("INSERT INTO schema_info(revision) VALUES(3)", []) .unwrap(); } - let err = open_or_recreate(&p, "trigram").unwrap_err(); + let err = open_or_recreate(&p, &Default::default()).unwrap_err(); assert!(err.starts_with(FOREIGN_DB_PREFIX), "got: {err}"); let conn = Connection::open(&p).unwrap(); diff --git a/crates/quicksearch-core/src/db/repo.rs b/crates/quicksearch-core/src/db/repo.rs index 15cb642..a1a4cbf 100644 --- a/crates/quicksearch-core/src/db/repo.rs +++ b/crates/quicksearch-core/src/db/repo.rs @@ -3,7 +3,7 @@ use rusqlite::{params, params_from_iter, Connection, OptionalExtension, Transaction}; -use crate::file_handling::{ExtractCursor, RootRange}; +use crate::file_handling::{ContentOutcome, ExtractCursor, RootRange}; use crate::mime::FileType; pub const STATE_PENDING: i64 = 0; @@ -27,30 +27,6 @@ fn exec( .map_err(|e| format!("{}: {}", what(), e)) } -/// Set a file's content state and clear any failure record with it: -/// `list-failed` reads `failed_files` directly, so a stale entry keeps -/// reporting a file broken. [`set_content_failed`] does not use this. -fn set_state_clearing_failure( - tx: &Transaction<'_>, - file_id: i64, - state: i64, - transition: &'static str, -) -> Result<(), String> { - exec( - tx, - "UPDATE files SET content_state = ?1 WHERE id = ?2", - params![state, file_id], - || format!("{} content_state {}", transition, file_id), - )?; - exec( - tx, - "DELETE FROM failed_files WHERE file_id = ?1", - params![file_id], - || format!("clear failed_files {}", file_id), - )?; - Ok(()) -} - #[derive(Debug, Clone)] pub struct NewFile<'a> { pub name: &'a str, @@ -154,9 +130,8 @@ pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result