jeremy/quick_search
jeremy/quick_search
1
0
Fork 0
Code Issues Pull requests Projects Releases 15 Packages Wiki Activity Actions

Removal of positions index from FTS for dramatically reduced index size and improved indexing speed at the cost of slightly slower search of common terms.
Some checks failed
CI / linux (push) Has been cancelled
Details
CI / windows-cross (push) Has been cancelled
Details
CI / release (push) Has been cancelled
Details

Browse source
This commit is contained in:
Jeremy Karst 2026-09-26 16:29:36 -04:00
parent e261e18a35
commit 67dd461587
57 changed files with 1571 additions and 1107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
DESIGN.md
View file

@ -21,12 +21,16 @@ memory, memory churn, and indexing and search speed.
The index is a single SQLite database: file metadata, an FTS5 full-text index
over document contents, and the extracted text itself, stored zstd-compressed
for snippets and ranking. The database runs in WAL mode with exactly one
for snippets, ranking and verification. The database runs in WAL mode with exactly one
writer at a time, so read-only searches are never blocked by indexing. Schema
changes wipe and rebuild the index rather than migrating: the index is a cache
of the filesystem, so rebuilding is always safe, and only the indexer may do
it. FTS5 runs in trigram mode for full-text search and also speeds up regex
and fuzzy searches. Tuning this database is central to QuickSearch's
and fuzzy searches. By default it records no word positions: every candidate
is checked against the stored text anyway, so positions only narrow the
candidates, and dropping them makes the index several times smaller and
faster to build, for content searches up to two or three times slower. A
setting trades the other way. Tuning this database is central to QuickSearch's
performance.
## The search path never stalls

23
config_example.toml
View file

@ -94,17 +94,20 @@ writer_turn_slice_ms = 100
# searches that run alongside indexing and lengthens crash recovery. The
# default favours faster runs.
maximum_wal_size = 2147483648
# FTS5 tokenizer: 'trigram' (substring matching, the default; gets
# remove_diacritics 1 appended), 'unicode61', 'porter', or a full FTS5 option
# string. See https://www.sqlite.org/fts5.html#tokenizers. Changing it
# rebuilds the index.
# FTS5 tokenizer: 'trigram' (substring matching, the default), 'unicode61',
# 'porter', or a full FTS5 option string. See
# https://www.sqlite.org/fts5.html#tokenizers. Matching is exact apart from
# ASCII case: "cafe" does not find "café". Changing it rebuilds the index.
tokenize = "trigram"
# Store extracted text (zstd-compressed) alongside the FTS index. Off, the
# index shrinks considerably, but search loses snippet previews, occurrence
# ranking, case verification and fuzzy full-text. Turning it off discards
# the stored text at once; turning it on re-extracts, because the text of
# files already indexed was never kept.
store_text_for_snippets = true
# Build the full-text index without word positions. Several times smaller
# and faster to build (1.1 GB of source code: a 219 MiB index instead of
# 1.3 GB, built a third faster), and searches return exactly the same files,
# because every candidate is checked against the stored text. The cost is
# more candidates to check: searching contents for a word can take up to
# twice as long, for a phrase of common words up to three times. Only the
# trigram tokenizer can use it; other tokenizers always record positions.
# Changing it rebuilds the index.
compact_text_index = true
[security]
# Encrypt the index with a password (SQLCipher). The password is asked for at

58
crates/quicksearch-core/benches/index.rs
View file

@ -84,7 +84,7 @@ mod text_pipeline {
let path = Path::new("/tmp/bench/notes.txt");
bencher.bench(|| {
let looks = textenc::looks_like_text(divan::black_box(head));
let text = textenc::decode_text(head.to_vec(), path).unwrap();
let text = textenc::decode_text(head.to_vec(), path, false).unwrap();
(looks, text.len())
});
}
@ -101,7 +101,7 @@ mod text_pipeline {
fn decode_only(bencher: Bencher) {
let head = corpus::text_head();
let path = Path::new("/tmp/bench/notes.txt");
bencher.bench(|| textenc::decode_text(divan::black_box(head).to_vec(), path));
bencher.bench(|| textenc::decode_text(divan::black_box(head).to_vec(), path, false));
}
}
@ -167,7 +167,8 @@ mod update_narrowing {
fn seeded_na(tag: &str) -> (Scratch, Connection) {
let (dir, p) = Scratch::db(tag);
let mut conn =
quicksearch_core::db::open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
quicksearch_core::db::open_or_recreate(p.to_str().unwrap(), &Default::default())
.unwrap();
let tx = conn.transaction().unwrap();
let names: Vec<String> = (0..ROWS).map(|i| format!("f{:04}.txt", i)).collect();
for name in &names {
@ -233,3 +234,54 @@ mod update_narrowing {
});
}
}
/// A watcher event for a directory whose files are all indexed and unchanged:
/// a re-sent `Create`, a `chmod -R`, the live view re-checking what it shows.
/// Nothing is written, so the cost is whatever it reads to learn that; the
/// full walk's answer is one `stat` per file.
mod unchanged_event {
use super::*;
use quicksearch_core::config::{Config, IgnoreSet};
use quicksearch_core::extract::Registry;
use quicksearch_core::incremental::{apply_fs_event, Budget};
use quicksearch_core::testutil::Scratch;
use quicksearch_core::watcher::FsEvent;
use std::sync::atomic::AtomicBool;
use std::time::{Duration, Instant};
const FILES: usize = 500;
#[divan::bench]
fn directory_create(bencher: Bencher) {
let (dir, db) = Scratch::db("bench-unchanged-event");
let tree = db.with_file_name("tree");
std::fs::create_dir_all(&tree).unwrap();
for i in 0..FILES {
std::fs::write(
tree.join(format!("f{:04}.txt", i)),
corpus::text(1 << 10, 4),
)
.unwrap();
}
let mut conn =
quicksearch_core::db::open_or_recreate(db.to_str().unwrap(), &Default::default())
.unwrap();
let config = Config::default();
let ignore = IgnoreSet::compile(&config.indexing.ignore_patterns).unwrap();
let registry = Registry::default_set();
let event = FsEvent::Create(tree.clone());
let cancel = AtomicBool::new(false);
let mut apply = move || {
let budget = Budget {
deadline: Instant::now() + Duration::from_secs(3600),
cancel: &cancel,
resume_from: 0,
};
apply_fs_event(&mut conn, &event, &config, &ignore, &registry, &budget).unwrap()
};
// Indexed once; every timed run finds it unchanged.
apply();
bencher.bench_local(apply);
drop(dir);
}
}

2
crates/quicksearch-core/benches/index_alloc.rs
View file

@ -183,7 +183,7 @@ fn compression_traffic_per_chunk(config: &Config) {
let per_row = measure(|| {
(0..CHUNK)
.map(|_| repo::encode_one(&doc, true).expect("encode"))
.map(|_| repo::encode_one(&doc).expect("encode"))
.collect::<Vec<_>>()
});
// What `Bodies::fill` does: one encoder and one arena for the chunk.

10
crates/quicksearch-core/benches/search.rs
View file

@ -177,10 +177,10 @@ mod substring {
let (s, b) = snippet::extract_folded(
divan::black_box(text),
divan::black_box(folded),
&[corpus::NEEDLE],
corpus::NEEDLE,
&opts,
);
let a = pattern.count(text, false);
let a = pattern.count(text);
(a, b, s.ranges.len())
});
}
@ -194,9 +194,9 @@ mod substring {
let folded = corpus::text_folded(size, 4);
let opts = snippet::Options { approx_bytes: 600 };
bencher.bench(|| {
let a = pattern.count(divan::black_box(text), false);
let a = pattern.count(divan::black_box(text));
let b = pattern.count_folded(divan::black_box(folded));
let (s, _) = snippet::extract_folded(text, folded, &[corpus::NEEDLE], &opts);
let (s, _) = snippet::extract_folded(text, folded, corpus::NEEDLE, &opts);
(a, b, s.ranges.len())
});
}
@ -216,7 +216,7 @@ mod snippet_extract {
snippet::extract_folded(
divan::black_box(text),
divan::black_box(folded),
&[corpus::NEEDLE],
corpus::NEEDLE,
&opts,
)
});

9
crates/quicksearch-core/examples/contentprobe.rs
View file

@ -397,9 +397,7 @@ fn run_stage_with(
continue;
}
// `scope::apply_page`'s decision. `restore_text` is false because this
// probe only narrows the filter, so the second arm reduces to "wants
// content but has none".
// `scope::apply_page`'s decision.
let at = Instant::now();
pending.clear();
for row in &rows {
@ -463,9 +461,8 @@ fn write_page(tx: &Connection, stage: Stage, pending: &Pending, t: &mut Timing)
(&pending.all, &pending.all)
};
// The per-row clears are `repo::remove_content_for_id` followed by
// `set_state_clearing_failure`'s `failed_files` delete, one bound
// statement at a time. They prepare through the connection's cache, as
// The per-row clears are `repo::remove_content_for_id` followed by a
// `failed_files` delete, one bound statement at a time. They prepare through the connection's cache, as
// `repo::exec` does; preparing afresh per row would price a mistake the
// product does not make.
if stage.clears() {

8
crates/quicksearch-core/examples/pruneprobe.rs
View file

@ -674,10 +674,10 @@ fn main() {
// The page cache. `PRAGMAS_INCREMENTAL` gives the reconcile a flat
// 4 MiB for a pass across the whole index. A delete also maintains
// `idx_files_mtime`, `idx_files_mime` and `idx_files_hash`, three
// b-trees in orders uncorrelated with `(parent, name)`, so every
// doomed row scatters ~3 page touches that a 512-page cache cannot
// hold. Read the `misses` column. Both commit policies are swept,
// `idx_files_hash`, a b-tree in an order uncorrelated with
// `(parent, name)`, so every doomed row scatters a page touch that a
// 512-page cache cannot hold (three, before `idx_files_mtime` and
// `idx_files_mime` were dropped). Read the `misses` column. Both commit policies are swept,
// because a cache that holds the working set may make commit cadence
// stop mattering.
println!("\n page cache, at +fts(all) [{} = shipped]:", SHIPPED_MIB);

31
crates/quicksearch-core/examples/searchtime.rs
View file

@ -5,8 +5,12 @@
//! ```text
//! cargo build -p quicksearch-core --example searchtime --release
//! ./target/release/examples/searchtime /path/to/index.db
//! ./target/release/examples/searchtime /path/to/index.db serialize "fn main" ~recieve
//! ```
//!
//! Queries after the path replace the built-in set, for a corpus whose
//! vocabulary the defaults do not hit; a leading `~` runs one fuzzy.
//!
//! Queries run against one held connection, as the worker holds one; each
//! is timed best-of-N so a scheduling hiccup is not the headline.
@ -21,9 +25,14 @@ use quicksearch_core::testutil;
const RUNS: u32 = 5;
/// The content queries go through however many segments the write side
/// left behind; the filename query is the control and must not move.
/// left behind; the filename query is the control and must not move. The
/// filter rows price `modified:` and `mime:` against that control: neither
/// column has an index, so they must cost no more than the scan it already is.
const QUERIES: &[(&str, bool, &str)] = &[
("filename (control)", false, "doc42"),
("filename + modified", false, "doc42 modified:>=2025-06-01"),
("filename + one day", false, "doc42 modified:2025-06-01"),
("filename + mime", false, "doc42 mime:application/pdf"),
("content, common", false, "mountain"),
("content, rare", false, "quartzite"),
("content, two words", false, "ocean forest"),
@ -55,8 +64,24 @@ fn main() {
);
println!("{:<22} {:>10} {:>8}", "query", "best", "hits");
let custom: Vec<(String, bool, String)> = std::env::args()
.skip(2)
.map(|q| match q.strip_prefix('~') {
Some(term) => (q.clone(), true, term.to_string()),
None => (q.clone(), false, q),
})
.collect();
let queries: Vec<(&str, bool, &str)> = if custom.is_empty() {
QUERIES.to_vec()
} else {
custom
.iter()
.map(|(label, fuzzy, query)| (label.as_str(), *fuzzy, query.as_str()))
.collect()
};
let mut total = Duration::ZERO;
for (label, fuzzy, query) in QUERIES {
for &(label, fuzzy, query) in &queries {
// The display limit makes this unfair: `scan_pass` stops when the
// limit fills, and candidates stream in insertion order, so which
// documents drew low ids changes bytes read 13x. A `QSB_LIMIT` past
@ -66,7 +91,7 @@ fn main() {
.and_then(|v| v.parse().ok())
.unwrap_or(1000);
let options = SearchOptions {
fuzzy: *fuzzy,
fuzzy,
limit,
..SearchOptions::default()
};

2
crates/quicksearch-core/examples/walkprobe.rs
View file

@ -40,7 +40,7 @@ fn main() {
std::process::id()
));
let _ = std::fs::remove_file(&db);
quicksearch_core::db::open_or_recreate(db.to_str().unwrap(), &config.processing.tokenize)
quicksearch_core::db::open_or_recreate(db.to_str().unwrap(), &config.processing)
.expect("scratch index");
let existing = DirRows::new();

25
crates/quicksearch-core/src/cli.rs
View file

@ -197,7 +197,8 @@ pub fn clear_path(db_path: &str, path: &str) -> Result<bool, String> {
mod tests {
use super::*;
use crate::db::open_or_recreate;
use crate::db::repo::{insert_file, set_content_done, set_content_failed, NewFile};
use crate::db::repo::{complete_pending, insert_file, store_pending_text, NewFile};
use crate::file_handling::ContentOutcome;
use crate::mime::FileType;
use crate::testutil::{zstd_of, Scratch};
@ -226,13 +227,15 @@ mod tests {
}
fn seed_fixture(db_path: &str) -> (i64, i64) {
let mut conn = open_or_recreate(db_path, "trigram").unwrap();
let mut conn = open_or_recreate(db_path, &Default::default()).unwrap();
let (a, b) = {
let tx = conn.transaction().unwrap();
let a = row(&tx, "a.txt", true);
set_content_done(&tx, a, "hello", zstd_of("hello").as_deref()).unwrap();
let b = row(&tx, "b.bin", false);
set_content_failed(&tx, b, "bad extract").unwrap();
store_pending_text(&tx, a, "hello", zstd_of("hello").as_deref()).unwrap();
// Pending, not NA: only an extraction that ran can fail.
let b = row(&tx, "b.bin", true);
let failed = ContentOutcome::Failed("bad extract".into());
assert!(complete_pending(&tx, b, &failed, None).unwrap());
tx.commit().unwrap();
(a, b)
};
@ -279,7 +282,7 @@ mod tests {
let _ = seed_fixture(dbp);
assert_eq!(pending_content_count(dbp).unwrap(), 0);
let mut conn = open_or_recreate(dbp, "trigram").unwrap();
let mut conn = open_or_recreate(dbp, &Default::default()).unwrap();
{
let tx = conn.transaction().unwrap();
// Claimed by an extractor, text not read yet: the backlog.
@ -321,12 +324,12 @@ mod tests {
fn documents_text_ratio_reports_savings_on_compressible_prose() {
// Guards against silently swapping compression for a pass-through.
let (_dir, p) = Scratch::db("cli");
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let mut conn = open_or_recreate(p.to_str().unwrap(), &Default::default()).unwrap();
{
let tx = conn.transaction().unwrap();
let id = row(&tx, "big.txt", true);
let prose = "the quick brown fox jumps over the lazy dog. ".repeat(500);
set_content_done(&tx, id, &prose, zstd_of(&prose).as_deref()).unwrap();
store_pending_text(&tx, id, &prose, zstd_of(&prose).as_deref()).unwrap();
tx.commit().unwrap();
}
drop(conn);
@ -359,7 +362,11 @@ mod tests {
let (_dir, p) = Scratch::db("cli");
let dbp = p.to_str().unwrap();
{
let mut conn = open_or_recreate(dbp, "unicode61").unwrap();
let unicode61 = crate::config::ProcessingConfig {
tokenize: "unicode61".into(),
..Default::default()
};
let mut conn = open_or_recreate(dbp, &unicode61).unwrap();
let tx = conn.transaction().unwrap();
for name in ["a.txt", "b.txt"] {
row(&tx, name, true);

30
crates/quicksearch-core/src/config/diff.rs
View file

@ -30,11 +30,6 @@ pub struct IndexWork {
/// back to pending; newly excluded ones lose text and FTS but keep their
/// row.
pub reconcile_content: bool,
/// `store_text_for_snippets` turned on; rows extracted under the old
/// setting kept no text, so they must run again.
pub restore_text: bool,
/// `store_text_for_snippets` turned off.
pub drop_text: bool,
/// Newly in-scope files exist only on disk, so only a walk can find them.
pub reindex: bool,
}
@ -56,8 +51,6 @@ impl IndexWork {
self.drop_aliases |= other.drop_aliases;
self.prune_scope |= other.prune_scope;
self.reconcile_content |= other.reconcile_content;
self.restore_text |= other.restore_text;
self.drop_text |= other.drop_text;
self.reindex |= other.reindex;
}
@ -67,14 +60,12 @@ impl IndexWork {
|| self.drop_aliases
|| self.prune_scope
|| self.reconcile_content
|| self.restore_text
|| self.drop_text
}
/// Whether applying this scans rows under each surviving root, instead
/// of only deleting whole ranges.
pub fn scans_rows(&self) -> bool {
self.prune_scope || self.reconcile_content || self.restore_text || self.drop_text
self.prune_scope || self.reconcile_content
}
/// The plan in one line for the log, naming what changed.
@ -88,8 +79,6 @@ impl IndexWork {
self.reconcile_content,
"changed which files have their text read",
),
(self.restore_text, "snippet text turned on"),
(self.drop_text, "snippet text turned off"),
]
.into_iter()
.filter_map(|(changed, part)| changed.then_some(part))
@ -174,12 +163,13 @@ fn ignore_set(c: &Config) -> BTreeSet<&str> {
pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions {
let search_db_changed = old.paths.database_path != new.paths.database_path;
// The only three: a different key makes the file unreadable, the
// tokenizer is baked into the FTS table definition, and `hash_length`
// decides what bytes a stored hash covers. (`use_keychain` changes only
// where the key is remembered.)
// The only four: a different key makes the file unreadable, the
// tokenizer and the position setting are baked into the FTS table
// definition, and `hash_length` decides what bytes a stored hash covers.
// (`use_keychain` changes only where the key is remembered.)
let requires_rebuild = old.processing.hash_length != new.processing.hash_length
|| old.processing.tokenize != new.processing.tokenize
|| old.processing.fts_positions() != new.processing.fts_positions()
|| old.security.password_protected != new.security.password_protected
|| old.security.salt != new.security.salt;
if requires_rebuild {
@ -200,9 +190,6 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions {
let new_content = content_filter_set(&new_idx.content_extensions);
let old_cap = old.processing.maximum_text_file_size;
let new_cap = new.processing.maximum_text_file_size;
let old_store = old.processing.store_text_for_snippets;
let new_store = new.processing.store_text_for_snippets;
let restore_text = !old_store && new_store;
let work = IndexWork {
drop_roots: old_roots.difference(&new_roots).cloned().collect(),
@ -215,8 +202,6 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions {
drop_aliases: old_idx.follow_symlinks && !new_idx.follow_symlinks,
// See the field: the size cap decides what the extension filter does.
reconcile_content: old_content != new_content || old_cap != new_cap,
restore_text,
drop_text: old_store && !new_store,
// Widening only adds files, and a file absent from the index is not
// findable from it: only a walk produces those rows. Re-extraction
// needs a run too, since the content pass runs as part of one; a
@ -226,8 +211,7 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions {
|| (!old_idx.include_hidden && new_idx.include_hidden)
|| (!old_idx.follow_symlinks && new_idx.follow_symlinks)
|| filter_widened(&old_content, &new_content)
|| new_cap > old_cap
|| restore_text,
|| new_cap > old_cap,
};
ConfigActions {

31
crates/quicksearch-core/src/config/mod.rs
View file

@ -93,11 +93,30 @@ pub struct ProcessingConfig {
/// User-facing guidance is in `config_example.toml`.
pub maximum_wal_size: u64,
pub tokenize: String,
/// When `true` (default), extracted text is stored zstd-compressed in
/// `documents_text` for snippet previews. When `false` queries return the
/// same hits, but results carry no snippets, can't be case-verified or
/// occurrence-ranked, and fuzzy full-text search is unavailable.
pub store_text_for_snippets: bool,
/// When `true` (default), the full-text index keeps no word positions
/// (FTS5 `detail=none`): a term is looked up as the AND of its trigrams
/// and the full-text pass drops the documents where they are not
/// adjacent, against the stored text. Several times smaller and faster
/// to build, at the cost of checking more candidates per search; see
/// [`crate::db::schema::fts_create_sql`] for the measurements. Only
/// `trigram` can be queried that way; see [`ProcessingConfig::fts_positions`].
pub compact_text_index: bool,
}
impl ProcessingConfig {
/// Whether the full-text index is built with word positions. Everything
/// that decides the index's layout asks this, never
/// [`ProcessingConfig::compact_text_index`] directly: a word tokenizer
/// keeps positions whatever that says, because a multi-word phrase has no
/// trigram form to fall back on, and toggling it there changes nothing.
pub fn fts_positions(&self) -> bool {
let trigram = self
.tokenize
.split_whitespace()
.next()
.is_some_and(|name| name.eq_ignore_ascii_case("trigram"));
!(self.compact_text_index && trigram)
}
}
/// Floor on a non-zero [`ProcessingConfig::maximum_wal_size`]: a checkpoint
@ -231,7 +250,7 @@ impl Default for ProcessingConfig {
writer_turn_slice_ms: 100,
maximum_wal_size: 1024 * 1024 * 1024 * 2,
tokenize: "trigram".to_string(),
store_text_for_snippets: true,
compact_text_index: true,
}
}
}

45
crates/quicksearch-core/src/config/tests.rs
View file

@ -671,16 +671,41 @@ fn turning_symlinks_off_drops_aliases_and_on_walks() {
);
}
/// Stored text: on re-extracts, off drops the blobs; never a rebuild.
/// The position setting is part of the FTS table definition, so a change
/// rebuilds; but only a change of the layout it produces, and a word
/// tokenizer keeps positions either way.
#[test]
fn stored_text_on_re_extracts_and_off_drops_it() {
fn the_compact_index_flag_rebuilds_only_when_the_layout_changes() {
let base = wide_base();
let off = edited(&base, |c| c.processing.store_text_for_snippets = false);
let on = edited(&base, |c| c.processing.store_text_for_snippets = true);
let a = diff_actions(&on, &off);
assert!(a.work.drop_text && !a.work.restore_text && !a.work.reindex);
let a = diff_actions(&off, &on);
assert!(a.work.restore_text && !a.work.drop_text && a.work.reindex);
let full = edited(&base, |c| c.processing.compact_text_index = false);
assert!(diff_actions(&base, &full).requires_rebuild);
assert!(diff_actions(&full, &base).requires_rebuild);
let words = edited(&base, |c| c.processing.tokenize = "unicode61".into());
let words_full = edited(&words, |c| c.processing.compact_text_index = false);
assert_eq!(
diff_actions(&words, &words_full),
ConfigActions::default(),
"positions either way: nothing to do"
);
}
#[test]
fn only_trigram_can_drop_positions() {
let mut p = ProcessingConfig::default();
assert!(p.compact_text_index, "compact by default");
assert!(!p.fts_positions());
p.tokenize = "trigram case_sensitive 1".into();
assert!(!p.fts_positions(), "trigram with options is still trigram");
p.tokenize = " TRIGRAM ".into();
assert!(!p.fts_positions());
for word in ["unicode61", "porter", "porter unicode61", "trigrammer"] {
p.tokenize = word.into();
assert!(p.fts_positions(), "{word} cannot be queried by trigrams");
}
p.tokenize = "trigram".into();
p.compact_text_index = false;
assert!(p.fts_positions());
}
#[test]
@ -713,7 +738,7 @@ fn merging_two_plans_loses_nothing() {
let first = IndexWork {
drop_roots: vec!["/gone".into(), "/shared".into()],
prune_scope: true,
drop_text: true,
drop_aliases: true,
..IndexWork::default()
};
let second = IndexWork {
@ -730,7 +755,7 @@ fn merging_two_plans_loses_nothing() {
vec!["/shared", "/also-gone", "/gone"],
"every root from both, each once"
);
assert!(merged.prune_scope && merged.drop_text);
assert!(merged.prune_scope && merged.drop_aliases);
assert!(merged.reconcile_content && merged.reindex);
// Empty merge is a no-op and self-merge is the identity; both make a

31
crates/quicksearch-core/src/content.rs
View file

@ -331,12 +331,18 @@ mod tests {
use crate::db::open_or_recreate;
use crate::db::repo::{self, insert_file, NewFile};
use crate::file_handling::{extract_scope_prepare, store_extracted, ExtractScope, Stored};
use crate::file_handling::{
extract_scope_prepare, store_extracted, Bodies, ExtractScope, Stored,
};
use crate::mime::FileType;
use crate::testutil::Scratch;
use std::path::PathBuf;
use std::time::{Duration, Instant};
fn bodies() -> Bodies {
Bodies::new().unwrap()
}
fn tree(s: &Scratch) -> PathBuf {
s.join("tree")
}
@ -349,7 +355,7 @@ mod tests {
/// a pending row for each. The guard owns both.
fn seed(tag: &str, dirs: &[(&str, usize)]) -> Scratch {
let s = Scratch::dir(tag);
let mut conn = open_or_recreate(db(&s).to_str().unwrap(), "trigram").unwrap();
let mut conn = open_or_recreate(db(&s).to_str().unwrap(), &Default::default()).unwrap();
let tx = conn.transaction().unwrap();
for (dir, n) in dirs {
let d = tree(&s).join(dir);
@ -385,7 +391,7 @@ mod tests {
fn writer(s: &Scratch) -> Arc<Mutex<rusqlite::Connection>> {
Arc::new(Mutex::new(
open_or_recreate(db(s).to_str().unwrap(), "trigram").unwrap(),
open_or_recreate(db(s).to_str().unwrap(), &Default::default()).unwrap(),
))
}
@ -422,7 +428,7 @@ mod tests {
assert!(pass.finish());
let stop = Arc::new(AtomicBool::new(false));
let far = Instant::now() + Duration::from_secs(60);
store_extracted(conn, &rows, &stop, &Config::default(), far).unwrap()
store_extracted(conn, &rows, &stop, far, &mut bodies()).unwrap()
}
#[test]
@ -479,15 +485,7 @@ mod tests {
"out-of-range row untouched"
);
let hits: i64 = conn_mutex
.lock()
.unwrap()
.query_row(
"SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH '\"sphinx\"'",
[],
|r| r.get(0),
)
.unwrap();
let hits = crate::testutil::fts_count(&conn_mutex.lock().unwrap(), "sphinx");
assert_eq!(hits, 3);
}
@ -545,7 +543,6 @@ mod tests {
fn store_extracted_honours_its_deadline_but_always_makes_progress() {
let s = seed("deadline", &[("r1", 5)]);
let conn_mutex = writer(&s);
let config = Config::default();
let mut pass = pass_for(&s, "r1", 2);
let rows = drain(&mut pass);
assert!(pass.finish());
@ -555,7 +552,7 @@ mod tests {
// A deadline already gone by: one row, then out.
let past = Instant::now() - Duration::from_secs(1);
assert_eq!(
store_extracted(&conn_mutex, &rows, &stop, &config, past).unwrap(),
store_extracted(&conn_mutex, &rows, &stop, past, &mut bodies()).unwrap(),
Stored {
consumed: 1,
written: 1
@ -563,7 +560,7 @@ mod tests {
);
let far = Instant::now() + Duration::from_secs(60);
assert_eq!(
store_extracted(&conn_mutex, &rows[1..], &stop, &config, far).unwrap(),
store_extracted(&conn_mutex, &rows[1..], &stop, far, &mut bodies()).unwrap(),
Stored {
consumed: 4,
written: 4
@ -583,7 +580,7 @@ mod tests {
// Stopped before it starts: nothing consumed.
stop.store(true, std::sync::atomic::Ordering::Relaxed);
assert_eq!(
store_extracted(&conn_mutex, &rows, &stop, &config, far).unwrap(),
store_extracted(&conn_mutex, &rows, &stop, far, &mut bodies()).unwrap(),
Stored::default()
);
}

55
crates/quicksearch-core/src/db/open.rs
View file

@ -7,10 +7,11 @@ use std::path::Path;
use rusqlite::{params, Connection, OpenFlags, OptionalExtension};
use super::schema::{
effective_tokenizer, fts_create_sql, fts_set_pgsz, pragmas_search,
recommended_search_cache_mib, Profile, PRAGMAS_FAST, PRAGMAS_INCREMENTAL, PRAGMAS_MAINTENANCE,
PRAGMAS_READONLY, PRAGMAS_WALK_READER, SCHEMA_CURRENT,
fts_create_sql, fts_set_pgsz, pragmas_search, recommended_search_cache_mib, Profile,
PRAGMAS_FAST, PRAGMAS_INCREMENTAL, PRAGMAS_MAINTENANCE, PRAGMAS_READONLY, PRAGMAS_WALK_READER,
SCHEMA_CURRENT,
};
use crate::config::ProcessingConfig;
use crate::security::IndexKey;
/// Prefix tagging every "the key does not fit this file" error, so a wrong
@ -26,18 +27,25 @@ pub const KEY_MISMATCH_PREFIX: &str = "KEY_MISMATCH: ";
/// table. Keyed indexes are rebuilt anyway via the profile retry in
/// [`open_probed`]; the bump rebuilds unprotected ones too.
///
/// v12 dropped the `mtime` and `mime` indexes (see [`schema::SCHEMA_CURRENT`]).
///
/// [`schema::HMAC_MODE`]: super::schema::HMAC_MODE
pub const CURRENT_SCHEMA_VERSION: u32 = 11;
/// [`schema::SCHEMA_CURRENT`]: super::schema::SCHEMA_CURRENT
pub const CURRENT_SCHEMA_VERSION: u32 = 12;
/// Open `db_path`; on any schema or tokenizer mismatch, delete the file and
/// Open `db_path`; on a schema mismatch, or a full-text layout other than
/// `processing` asks for (tokenizer, positions), delete the file and
/// recreate it empty, for the caller to re-index.
pub fn open_or_recreate(db_path: &str, tokenizer: &str) -> Result<Connection, String> {
open_or_recreate_keyed(db_path, tokenizer, super::key::process_key().as_ref())
pub fn open_or_recreate(
db_path: &str,
processing: &ProcessingConfig,
) -> Result<Connection, String> {
open_or_recreate_keyed(db_path, processing, super::key::process_key().as_ref())
}
pub(crate) fn open_or_recreate_keyed(
db_path: &str,
tokenizer: &str,
processing: &ProcessingConfig,
key: Option<&IndexKey>,
) -> Result<Connection, String> {
let path = Path::new(db_path).to_path_buf();
@ -67,7 +75,7 @@ pub(crate) fn open_or_recreate_keyed(
ProfileMatch::Previous(found) => {
format!("was built with {} and this build uses {}", found, want)
}
ProfileMatch::Current if db_matches_current(&conn, tokenizer)? => return Ok(conn),
ProfileMatch::Current if db_matches_current(&conn, processing)? => return Ok(conn),
ProfileMatch::Current => "does not match current schema".to_string(),
};
crate::log_warn!(
@ -76,7 +84,7 @@ pub(crate) fn open_or_recreate_keyed(
why
);
let conn = wipe_and_reopen(conn, &path, key, want)?;
apply_current_schema(&conn, tokenizer, key, want)?;
apply_current_schema(&conn, processing, key, want)?;
Ok(conn)
}
@ -491,9 +499,9 @@ fn schema_version_current(conn: &Connection) -> Result<bool, String> {
Ok(version.as_deref() == Some(&CURRENT_SCHEMA_VERSION.to_string()))
}
/// True iff the DB has the current schema version *and* the
/// effective-tokenizer string this caller asked for.
fn db_matches_current(conn: &Connection, tokenizer: &str) -> Result<bool, String> {
/// True iff the DB has the current schema version *and* the full-text layout
/// this caller asked for.
fn db_matches_current(conn: &Connection, processing: &ProcessingConfig) -> Result<bool, String> {
if !schema_version_current(conn)? {
// Refuse rather than wipe unless the file is recognisably ours:
// `database_path` is free text with no confirmation, and a typo
@ -504,8 +512,11 @@ fn db_matches_current(conn: &Connection, tokenizer: &str) -> Result<bool, String
}
return Ok(false);
}
let want_tokenize = effective_tokenizer(tokenizer);
Ok(stored_info(conn, "tokenize")?.as_deref() == Some(&*want_tokenize))
Ok(
stored_info(conn, "tokenize")?.as_deref() == Some(processing.tokenize.trim())
&& stored_info(conn, "fts_positions")?.as_deref()
== Some(&*processing.fts_positions().to_string()),
)
}
/// Delete the index at `path` and its SQLite sidecars (`-wal`, `-shm`,
@ -557,13 +568,14 @@ fn wipe_and_reopen(
fn apply_current_schema(
conn: &Connection,
tokenizer: &str,
processing: &ProcessingConfig,
key: Option<&IndexKey>,
profile: Profile,
) -> Result<(), String> {
conn.execute_batch(SCHEMA_CURRENT)
.map_err(|e| format!("Failed to create current schema tables: {}", e))?;
let fts = fts_create_sql(tokenizer);
let positions = processing.fts_positions();
let fts = fts_create_sql(&processing.tokenize, positions);
conn.execute_batch(&fts)
.map_err(|e| format!("Failed to create searchabletext: {}", e))?;
// Only here: FTS5's leaf size must suit this file's page size and
@ -573,13 +585,16 @@ fn apply_current_schema(
.map_err(|e| format!("Failed to set searchabletext pgsz: {}", e))?;
let now = crate::log::now_unix();
let effective = effective_tokenizer(tokenizer);
// `fts_positions` is read back by every search (`repo::fts_positions`),
// which must phrase its queries for the table as it was built.
conn.execute(
"INSERT INTO schema_info(key, value) VALUES ('version', ?1), ('created_at', ?2), ('tokenize', ?3)",
"INSERT INTO schema_info(key, value) VALUES ('version', ?1), ('created_at', ?2), \
('tokenize', ?3), ('fts_positions', ?4)",
params![
CURRENT_SCHEMA_VERSION.to_string(),
now.to_string(),
effective
processing.tokenize.trim(),
positions.to_string()
],
)
.map_err(|e| format!("Failed to seed schema_info: {}", e))?;

154
crates/quicksearch-core/src/db/open_tests.rs
View file

@ -15,6 +15,24 @@ fn info(conn: &Connection, key: &str) -> String {
.unwrap()
}
/// Processing settings asking for `tokenize`, the rest default.
fn with_tokenizer(tokenize: &str) -> ProcessingConfig {
ProcessingConfig {
tokenize: tokenize.into(),
..Default::default()
}
}
/// The DDL the full-text table was created with.
fn fts_sql(conn: &Connection) -> String {
conn.query_row(
"SELECT sql FROM sqlite_master WHERE name = 'searchabletext'",
[],
|r| r.get(0),
)
.unwrap()
}
fn file_count(conn: &Connection) -> i64 {
conn.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
.unwrap()
@ -42,15 +60,15 @@ fn write_v1_index(p: &str, name: &str) {
#[test]
fn fresh_db_gets_current_version() {
let (_dir, p) = db();
let conn = open_or_recreate(&p, "trigram").unwrap();
let conn = open_or_recreate(&p, &Default::default()).unwrap();
assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string());
}
#[test]
fn reopen_is_idempotent() {
let (_dir, p) = db();
drop(open_or_recreate(&p, "trigram").unwrap());
let conn = open_or_recreate(&p, "trigram").unwrap();
drop(open_or_recreate(&p, &Default::default()).unwrap());
let conn = open_or_recreate(&p, &Default::default()).unwrap();
assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string());
}
@ -58,7 +76,7 @@ fn reopen_is_idempotent() {
fn older_versioned_db_is_wiped_and_recreated() {
let (_dir, p) = db();
write_v1_index(&p, "a.txt");
let conn = open_or_recreate(&p, "trigram").unwrap();
let conn = open_or_recreate(&p, &Default::default()).unwrap();
assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string());
assert_eq!(file_count(&conn), 0, "old rows should be wiped");
}
@ -79,7 +97,7 @@ fn legacy_layout_db_is_wiped_and_recreated() {
)
.unwrap();
}
let conn = open_or_recreate(&p, "trigram").unwrap();
let conn = open_or_recreate(&p, &Default::default()).unwrap();
assert_eq!(file_count(&conn), 0);
// An unknown column name fails to prepare.
conn.prepare("SELECT content_state, type, mime FROM files LIMIT 0")
@ -90,7 +108,7 @@ fn legacy_layout_db_is_wiped_and_recreated() {
fn tokenizer_drift_wipes_db() {
let (_dir, p) = db();
let first_effective = {
let conn = open_or_recreate(&p, "trigram").unwrap();
let conn = open_or_recreate(&p, &Default::default()).unwrap();
conn.execute(
"INSERT INTO files (name, parent, size, mtime) \
VALUES ('x', '/', 0, 0)",
@ -99,16 +117,77 @@ fn tokenizer_drift_wipes_db() {
.unwrap();
info(&conn, "tokenize")
};
let conn = open_or_recreate(&p, "unicode61").unwrap();
let conn = open_or_recreate(&p, &with_tokenizer("unicode61")).unwrap();
assert_eq!(file_count(&conn), 0, "tokenizer drift should wipe rows");
assert_ne!(first_effective, info(&conn, "tokenize"));
}
/// The position setting is part of the table definition, like the
/// tokenizer: a change wipes, and the layout built is the one recorded,
/// which is what every search reads back.
#[test]
fn position_drift_wipes_db_and_the_layout_is_recorded() {
let (_dir, p) = db();
{
let conn = open_or_recreate(&p, &Default::default()).unwrap();
assert!(fts_sql(&conn).contains("detail=none"), "compact by default");
assert_eq!(info(&conn, "fts_positions"), "false");
assert!(!crate::db::repo::fts_positions(&conn));
conn.execute(
"INSERT INTO files (name, parent, size, mtime) VALUES ('x', '/', 0, 0)",
[],
)
.unwrap();
}
let full = ProcessingConfig {
compact_text_index: false,
..Default::default()
};
let conn = open_or_recreate(&p, &full).unwrap();
assert_eq!(file_count(&conn), 0, "a layout change wipes the rows");
assert!(!fts_sql(&conn).contains("detail"));
assert_eq!(info(&conn, "fts_positions"), "true");
assert!(crate::db::repo::fts_positions(&conn));
}
/// A word tokenizer cannot be queried by trigrams, so it keeps positions
/// whatever the flag says, and the flag alone then changes nothing.
#[test]
fn a_word_tokenizer_keeps_positions_and_ignores_the_flag() {
let (_dir, p) = db();
{
let conn = open_or_recreate(&p, &with_tokenizer("unicode61")).unwrap();
assert!(!fts_sql(&conn).contains("detail"));
assert_eq!(info(&conn, "fts_positions"), "true");
conn.execute(
"INSERT INTO files (name, parent, size, mtime) VALUES ('x', '/', 0, 0)",
[],
)
.unwrap();
}
let flag_off = ProcessingConfig {
compact_text_index: false,
..with_tokenizer("unicode61")
};
let conn = open_or_recreate(&p, &flag_off).unwrap();
assert_eq!(file_count(&conn), 1, "the same layout: nothing to rebuild");
}
/// An index from before the layout was recorded kept positions.
#[test]
fn an_unrecorded_layout_reads_as_positions() {
let (_dir, p) = db();
let conn = open_or_recreate(&p, &Default::default()).unwrap();
conn.execute("DELETE FROM schema_info WHERE key = 'fts_positions'", [])
.unwrap();
assert!(crate::db::repo::fts_positions(&conn));
}
#[test]
fn open_existing_reads_nondefault_tokenizer_without_wiping() {
let (_dir, p) = db();
{
let conn = open_or_recreate(&p, "unicode61").unwrap();
let conn = open_or_recreate(&p, &with_tokenizer("unicode61")).unwrap();
conn.execute(
"INSERT INTO files (name, parent, size, mtime) \
VALUES ('note', '/', 0, 0)",
@ -129,6 +208,7 @@ fn open_existing_reads_nondefault_tokenizer_without_wiping() {
1,
"open_existing must not wipe a non-default-tokenizer DB"
);
// A word tokenizer: `hello` is one token, so a bare MATCH is exact.
let hits: i64 = conn
.query_row(
"SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'hello'",
@ -145,7 +225,7 @@ fn open_existing_reads_nondefault_tokenizer_without_wiping() {
fn open_or_recreate_creates_missing_parent_dirs() {
let dir = Scratch::dir("mkdir");
let db = dir.join("nested/deeper/index.sqlite");
drop(open_or_recreate(db.to_str().unwrap(), "trigram").unwrap());
drop(open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap());
assert!(db.exists());
}
@ -156,7 +236,7 @@ fn writable_opens_use_wal_and_it_persists() {
conn.query_row("PRAGMA journal_mode", [], |r| r.get(0))
.unwrap()
};
let conn = open_or_recreate(&p, "trigram").unwrap();
let conn = open_or_recreate(&p, &Default::default()).unwrap();
assert_eq!(mode(&conn).to_lowercase(), "wal");
drop(conn);
// WAL is persistent in the file: a later read-only consumer sees it.
@ -173,7 +253,7 @@ fn maintenance_opens_keep_temporaries_on_disk() {
conn.query_row("PRAGMA temp_store", [], |r| r.get(0))
.unwrap()
};
let conn = open_or_recreate(&p, "trigram").unwrap();
let conn = open_or_recreate(&p, &Default::default()).unwrap();
assert_eq!(temp_store(&conn), 2, "the indexer's own profile is MEMORY");
drop(conn);
@ -211,13 +291,13 @@ fn index_needs_rebuild_only_when_the_schema_really_differs() {
let cases: [(Fixture, bool, &str); 5] = [
(|_| {}, false, "no file yet is a fresh install, not a reset"),
(
|p| drop(open_or_recreate(p, "trigram").unwrap()),
|p| drop(open_or_recreate(p, &Default::default()).unwrap()),
false,
"a current index is not going to be rebuilt",
),
(
|p| {
drop(open_or_recreate(p, "trigram").unwrap());
drop(open_or_recreate(p, &Default::default()).unwrap());
open_existing(p, true)
.unwrap()
.execute(
@ -289,7 +369,7 @@ fn keyed_create_reopen_and_header_is_encrypted() {
let (_dir, p) = db();
let key = test_key(0xa1);
{
let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap();
let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&key)).unwrap();
conn.execute(
"INSERT INTO files (name, parent, size, mtime) \
VALUES ('secret', '/', 0, 0)",
@ -300,7 +380,7 @@ fn keyed_create_reopen_and_header_is_encrypted() {
let head = &file_bytes(&p)[..16];
assert_ne!(head, b"SQLite format 3\0", "file must not be plaintext");
let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap();
let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&key)).unwrap();
assert_eq!(
file_count(&conn),
1,
@ -315,7 +395,7 @@ fn keyed_create_reopen_and_header_is_encrypted() {
fn wrong_key_errors_without_wiping() {
let (_dir, p) = db();
{
let conn = open_or_recreate_keyed(&p, "trigram", Some(&test_key(0xa1))).unwrap();
let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&test_key(0xa1))).unwrap();
conn.execute(
"INSERT INTO files (name, parent, size, mtime) \
VALUES ('x', '/', 0, 0)",
@ -329,7 +409,7 @@ fn wrong_key_errors_without_wiping() {
assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}");
}
// A wrong key is never a "schema mismatch" to answer with a wipe.
let err = open_or_recreate_keyed(&p, "trigram", Some(&test_key(0xb2))).unwrap_err();
let err = open_or_recreate_keyed(&p, &Default::default(), Some(&test_key(0xb2))).unwrap_err();
assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}");
assert_eq!(before, file_bytes(&p), "file must be byte-identical");
}
@ -337,12 +417,12 @@ fn wrong_key_errors_without_wiping() {
#[test]
fn missing_key_on_encrypted_db_errors_without_wiping() {
let (_dir, p) = db();
drop(open_or_recreate_keyed(&p, "trigram", Some(&test_key(0xa1))).unwrap());
drop(open_or_recreate_keyed(&p, &Default::default(), Some(&test_key(0xa1))).unwrap());
let before = file_bytes(&p);
let err = open_existing_keyed(&p, false, None).unwrap_err();
assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}");
assert!(err.contains("password-protected"), "got: {err}");
let err = open_or_recreate_keyed(&p, "trigram", None).unwrap_err();
let err = open_or_recreate_keyed(&p, &Default::default(), None).unwrap_err();
assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}");
assert_eq!(before, file_bytes(&p));
}
@ -350,13 +430,13 @@ fn missing_key_on_encrypted_db_errors_without_wiping() {
#[test]
fn key_on_plaintext_db_errors_without_wiping() {
let (_dir, p) = db();
drop(open_or_recreate(&p, "trigram").unwrap());
drop(open_or_recreate(&p, &Default::default()).unwrap());
let before = file_bytes(&p);
let err = open_existing_keyed(&p, false, Some(&test_key(0xa1))).unwrap_err();
assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}");
// The sniffed plaintext header yields the precise diagnosis.
assert!(err.contains("not encrypted"), "got: {err}");
let err = open_or_recreate_keyed(&p, "trigram", Some(&test_key(0xa1))).unwrap_err();
let err = open_or_recreate_keyed(&p, &Default::default(), Some(&test_key(0xa1))).unwrap_err();
assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}");
assert_eq!(before, file_bytes(&p));
}
@ -377,7 +457,7 @@ fn schema_mismatch_under_key_wipes_and_recreates_encrypted() {
key_and_probe(&conn, &p, Some(&key), previous).unwrap();
write_v1_schema_info(&conn);
}
let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap();
let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&key)).unwrap();
assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string());
drop(conn);
let head = &file_bytes(&p)[..16];
@ -401,7 +481,7 @@ fn garbage_file_with_key_reports_mismatch_not_corruption() {
fn open_existing_rw_allows_delete() {
let (_dir, p) = db();
{
let conn = open_or_recreate(&p, "trigram").unwrap();
let conn = open_or_recreate(&p, &Default::default()).unwrap();
conn.execute(
"INSERT INTO files (name, parent, size, mtime) \
VALUES ('a', '/', 0, 0)",
@ -428,7 +508,7 @@ fn a_fresh_index_and_its_sidecars_are_owner_only() {
// test: an existing directory keeps whatever mode its owner chose.
let dir = Scratch::dir("open");
let p = dir.join("data").join("index.sqlite");
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let conn = open_or_recreate(p.to_str().unwrap(), &Default::default()).unwrap();
conn.execute(
"INSERT INTO files (name, parent, size, mtime) \
VALUES ('a', '/', 0, 0)",
@ -462,7 +542,7 @@ fn maintain_reads_its_pragmas_on_a_keyed_index() {
let (_dir, p) = db();
let key = test_key(0xc3);
{
let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap();
let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&key)).unwrap();
conn.execute(
"INSERT INTO files (name, parent, size, mtime) VALUES ('x', '/', 0, 0)",
[],
@ -484,6 +564,16 @@ fn maintain_reads_its_pragmas_on_a_keyed_index() {
);
}
/// A layout with positions: `pgsz` caps a leaf the same in either layout,
/// and position lists fill leaves soonest, so the geometry tests below need
/// the smallest corpus this way.
fn positional() -> ProcessingConfig {
ProcessingConfig {
compact_text_index: false,
..Default::default()
}
}
/// Write enough text that FTS5 emits several *full* leaves. A corpus of tiny
/// documents fits in one part-filled leaf, never reaches the inline-payload
/// limit, and would let the tests below pass on a broken geometry.
@ -565,7 +655,7 @@ fn assert_leaves_inline(conn: &Connection, keyed: bool) {
fn keyed_fts_leaves_stay_inline() {
let (_dir, p) = db();
let key = test_key(0xd4);
let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap();
let conn = open_or_recreate_keyed(&p, &positional(), Some(&key)).unwrap();
assert_eq!(
fts_stored_pgsz(&conn),
Some(crate::db::schema::fts_pgsz_for(
@ -638,7 +728,7 @@ fn derived_fts_pgsz_fits_inline_at_every_swept_page_size() {
fn a_plain_index_gets_the_derived_pgsz_for_its_page_size() {
use crate::db::schema::{fts_pgsz_for, PROFILE};
let (_dir, p) = db();
let conn = open_or_recreate(&p, "trigram").unwrap();
let conn = open_or_recreate(&p, &positional()).unwrap();
assert_eq!(
fts_stored_pgsz(&conn),
Some(fts_pgsz_for(PROFILE, false)),
@ -672,7 +762,7 @@ fn a_foreign_database_is_refused_not_wiped() {
)
.unwrap();
}
let err = open_or_recreate(&p, "trigram").unwrap_err();
let err = open_or_recreate(&p, &Default::default()).unwrap_err();
assert!(err.starts_with(FOREIGN_DB_PREFIX), "got: {err}");
assert!(
err.contains("moz_places"),
@ -692,7 +782,7 @@ fn an_empty_database_file_is_still_ours_to_build() {
let (_dir, p) = db();
// An empty but real SQLite file, header and all.
drop(Connection::open(&p).unwrap());
let conn = open_or_recreate(&p, "trigram").unwrap();
let conn = open_or_recreate(&p, &Default::default()).unwrap();
assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string());
}
@ -706,7 +796,7 @@ fn a_legacy_table_marks_a_file_as_ours() {
.unwrap()
.execute(&format!("CREATE TABLE {} (x INTEGER)", table), [])
.unwrap();
open_or_recreate(&p, "trigram")
open_or_recreate(&p, &Default::default())
.unwrap_or_else(|e| panic!("{table} should read as ours, got: {e}"));
}
}
@ -723,7 +813,7 @@ fn a_schema_info_without_a_version_row_is_still_ours() {
[],
)
.unwrap();
open_or_recreate(&p, "trigram").expect("half-built index is ours");
open_or_recreate(&p, &Default::default()).expect("half-built index is ours");
}
/// A table that merely *shares the name* `schema_info` is not ours.
@ -737,7 +827,7 @@ fn a_foreign_schema_info_is_refused() {
conn.execute("INSERT INTO schema_info(revision) VALUES(3)", [])
.unwrap();
}
let err = open_or_recreate(&p, "trigram").unwrap_err();
let err = open_or_recreate(&p, &Default::default()).unwrap_err();
assert!(err.starts_with(FOREIGN_DB_PREFIX), "got: {err}");
let conn = Connection::open(&p).unwrap();

183
crates/quicksearch-core/src/db/repo.rs
View file

@ -3,7 +3,7 @@
use rusqlite::{params, params_from_iter, Connection, OptionalExtension, Transaction};
use crate::file_handling::{ExtractCursor, RootRange};
use crate::file_handling::{ContentOutcome, ExtractCursor, RootRange};
use crate::mime::FileType;
pub const STATE_PENDING: i64 = 0;
@ -27,30 +27,6 @@ fn exec(
.map_err(|e| format!("{}: {}", what(), e))
}
/// Set a file's content state and clear any failure record with it:
/// `list-failed` reads `failed_files` directly, so a stale entry keeps
/// reporting a file broken. [`set_content_failed`] does not use this.
fn set_state_clearing_failure(
tx: &Transaction<'_>,
file_id: i64,
state: i64,
transition: &'static str,
) -> Result<(), String> {
exec(
tx,
"UPDATE files SET content_state = ?1 WHERE id = ?2",
params![state, file_id],
|| format!("{} content_state {}", transition, file_id),
)?;
exec(
tx,
"DELETE FROM failed_files WHERE file_id = ?1",
params![file_id],
|| format!("clear failed_files {}", file_id),
)?;
Ok(())
}
#[derive(Debug, Clone)]
pub struct NewFile<'a> {
pub name: &'a str,
@ -154,9 +130,8 @@ pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option
}
let new_state = initial_content_state(f);
if old_state != new_state {
// A plain assignment, not `set_state_clearing_failure`: the FAILED
// arm above already took the failure record, and no other state can
// hold one.
// The FAILED arm above already took the failure record, and no
// other state can hold one.
exec(
tx,
"UPDATE files SET content_state = ?1 WHERE id = ?2",
@ -167,29 +142,64 @@ pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option
Ok(Some(id))
}
/// Mark a file's content indexing as complete and write the extracted text
/// atomically. `text_zstd` is the pre-compressed `documents_text` sidecar
/// body, or `None` for no sidecar. Compression is the caller's job: this runs
/// inside the writer's transaction, so compress before taking the lock.
pub fn set_content_done(
tx: &Transaction<'_>,
file_id: i64,
text: &str,
text_zstd: Option<&[u8]>,
) -> Result<(), String> {
remove_content_for_id(tx, file_id)?;
set_content_done_fresh(tx, file_id, text, text_zstd)
/// Move a [`STATE_PENDING`] row to `state`; `false`, writing nothing, when
/// the row is no longer pending (or no longer exists).
fn claim_pending(tx: &Transaction<'_>, file_id: i64, state: i64) -> Result<bool, String> {
let changed = exec(
tx,
"UPDATE files SET content_state = ?1 WHERE id = ?2 AND content_state = ?3",
params![state, file_id, STATE_PENDING],
|| format!("set content_state {} on {}", state, file_id),
)?;
Ok(changed > 0)
}
/// [`set_content_done`] for a row that **provably holds no content yet**,
/// skipping the pre-delete. Use it only where the same transaction has
/// established that; a wrong call leaves a duplicate FTS entry, not an error.
pub fn set_content_done_fresh(
/// Record the outcome of extracting a pending row: the one way content state
/// leaves [`STATE_PENDING`]. `text_zstd` is the pre-compressed
/// `documents_text` body for a `Done` outcome, or `None` for no sidecar;
/// compress before taking the writer's lock. Returns whether the row was
/// still pending, and so whether anything was written.
///
/// Pending is also what makes it cheap: a pending row provably holds no
/// posting, stored body or failure record (see [`STATE_DONE`] and
/// [`STATE_FAILED`]), so nothing is cleared first. A row that stopped being
/// pending while its extraction was in flight (stored by another route, or
/// deleted by stale cleanup) is left alone: writing it anyway would leave a
/// posting with no `files` row behind it, which a reused rowid inherits.
pub fn complete_pending(
tx: &Transaction<'_>,
file_id: i64,
outcome: &ContentOutcome,
text_zstd: Option<&[u8]>,
) -> Result<bool, String> {
match outcome {
ContentOutcome::Done { text } => store_pending_text(tx, file_id, text, text_zstd),
ContentOutcome::NotApplicable => claim_pending(tx, file_id, STATE_NA),
ContentOutcome::Failed(reason) => {
if !claim_pending(tx, file_id, STATE_FAILED)? {
return Ok(false);
}
exec(
tx,
"INSERT INTO failed_files(file_id, reason, ts) VALUES (?1, ?2, ?3)",
params![file_id, reason, crate::log::now_unix() as i64],
|| format!("insert failed_files {}", file_id),
)?;
Ok(true)
}
}
}
/// [`complete_pending`] with extracted text, for callers holding it borrowed.
pub fn store_pending_text(
tx: &Transaction<'_>,
file_id: i64,
text: &str,
text_zstd: Option<&[u8]>,
) -> Result<(), String> {
) -> Result<bool, String> {
if !claim_pending(tx, file_id, STATE_DONE)? {
return Ok(false);
}
// Contentless FTS5 takes the text on INSERT for the tokenizer and does
// not store it.
exec(
@ -198,7 +208,6 @@ pub fn set_content_done_fresh(
params![file_id, text],
|| format!("insert FTS row {}", file_id),
)?;
if let Some(compressed) = text_zstd {
exec(
tx,
@ -207,8 +216,7 @@ pub fn set_content_done_fresh(
|| format!("insert documents_text {}", file_id),
)?;
}
set_state_clearing_failure(tx, file_id, STATE_DONE, "update DONE")
Ok(true)
}
/// Reusable decode buffer and context for the readers of `documents_text`;
@ -313,9 +321,10 @@ impl DocEncoder {
}
}
/// Compress one body, for the writers that handle a single row.
pub fn encode_one(text: &str, store_text: bool) -> Result<Option<Vec<u8>>, String> {
if !store_text || text.is_empty() {
/// Compress one body, for the writers that handle a single row; `None` for
/// an empty one, which gets no `documents_text` row.
pub fn encode_one(text: &str) -> Result<Option<Vec<u8>>, String> {
if text.is_empty() {
return Ok(None);
}
DocEncoder::new()?.encode(text).map(Some)
@ -327,38 +336,6 @@ pub fn raw_text_len(blob: &[u8]) -> Option<u64> {
zstd::zstd_safe::get_frame_content_size(blob).ok().flatten()
}
/// Mark a file's content extraction as failed, keeping the row.
///
/// Clears any posting and stored body first, so text from an earlier version
/// of the file stops matching and [`STATE_DONE`]'s invariant holds. Callers
/// usually reach here on a row that is already clear, so the deletes are
/// normally no-ops.
pub fn set_content_failed(tx: &Transaction<'_>, file_id: i64, reason: &str) -> Result<(), String> {
remove_content_for_id(tx, file_id)?;
let now = crate::log::now_unix() as i64;
exec(
tx,
"UPDATE files SET content_state = ?1 WHERE id = ?2",
params![STATE_FAILED, file_id],
|| format!("update content_state FAILED {}", file_id),
)?;
exec(
tx,
"INSERT OR REPLACE INTO failed_files(file_id, reason, ts) VALUES (?1, ?2, ?3)",
params![file_id, reason, now],
|| format!("insert failed_files {}", file_id),
)?;
Ok(())
}
/// Mark content extraction as not applicable; the row still serves filename
/// search. Clears any posting and stored body first, as
/// [`set_content_failed`] does.
pub fn set_content_na(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> {
remove_content_for_id(tx, file_id)?;
set_state_clearing_failure(tx, file_id, STATE_NA, "update NA")
}
/// Delete a file row by path, keeping FTS in sync; returns whether a row went.
pub fn delete_file_by_path(tx: &Transaction<'_>, path: &str) -> Result<bool, String> {
let Some((parent, name)) = crate::file_handling::split_db_path(path) else {
@ -504,10 +481,9 @@ pub fn delete_ids(
/// Set `content_state` on many rows at once, leaving everything else alone.
///
/// The batch form of [`set_state_clearing_failure`]'s `UPDATE`, without its
/// `failed_files` sweep: a caller that knows the stored states clears the few
/// [`STATE_FAILED`] rows with [`clear_failed_for_ids`]. A caller that does not
/// know them must use the per-row helpers.
/// No `failed_files` sweep: a caller that knows the stored states clears the
/// few [`STATE_FAILED`] rows with [`clear_failed_for_ids`], and their content
/// with [`clear_content_for_ids`].
pub fn set_content_state(tx: &Transaction<'_>, ids: &[i64], state: i64) -> Result<usize, String> {
let mut changed = 0;
for chunk in ids.chunks(DELETE_IDS_CHUNK) {
@ -551,9 +527,8 @@ pub fn clear_content_for_ids(tx: &Transaction<'_>, ids: &[i64]) -> Result<(), St
Ok(())
}
/// Forget the failure records of many rows at once: the batch half of
/// [`set_state_clearing_failure`]. Pass only [`STATE_FAILED`] ids; nothing
/// else holds a record.
/// Forget the failure records of many rows at once. Pass only
/// [`STATE_FAILED`] ids; nothing else holds a record.
pub fn clear_failed_for_ids(tx: &Transaction<'_>, ids: &[i64]) -> Result<(), String> {
exec_chunked(
tx,
@ -586,7 +561,7 @@ pub fn dir_rows(
/// A stored path that remembers where its `name` begins: `parent + name` in
/// one buffer instead of two or three strings. The content pass moves one per
/// file from the feeder to the writer.
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct RowPath {
path: String,
name_at: usize,
@ -612,6 +587,11 @@ impl RowPath {
// `name_at` is the parent's length, so it is a char boundary.
&self.path[self.name_at..]
}
/// The `files.parent` half, trailing separator included.
pub fn parent(&self) -> &str {
&self.path[..self.name_at]
}
}
/// One row the content pass has yet to extract.
@ -757,18 +737,6 @@ pub fn rows_in_range_page(
.map_err(|e| format!("read range page row: {}", e))
}
/// Drop the stored text of the given file ids, leaving their FTS row and
/// `files` row intact: full-text search keeps working, only the snippet
/// source goes away.
pub fn drop_stored_text(tx: &Transaction<'_>, ids: &[i64]) -> Result<usize, String> {
exec_chunked(
tx,
"DELETE FROM documents_text WHERE file_id IN (",
ids,
|n| format!("drop stored text for {} ids", n),
)
}
/// Reset every failed file to pending and drop the failure records, so the
/// coming run re-attempts them. Returns how many rows were reset.
pub fn retry_failed_files(tx: &Transaction<'_>) -> Result<usize, String> {
@ -1092,6 +1060,15 @@ fn set_info(conn: &Connection, key: &str, value: &str) -> rusqlite::Result<usize
)
}
/// Whether the full-text index was built with positions, which decides how
/// a search may phrase its `MATCH` (`search::prefilter::fts_literal`). Read
/// from the index rather than the config, which may already ask for the
/// other layout before the rebuild that brings it. An index that never
/// recorded it predates the compact layout, so kept positions.
pub fn fts_positions(conn: &Connection) -> bool {
get_info(conn, "fts_positions").is_none_or(|v| v != "false")
}
/// Unix seconds of the last *successful* full indexing run; `None` means
/// "never".
pub fn get_last_full_index(conn: &Connection) -> Option<u64> {

202
crates/quicksearch-core/src/db/repo_tests.rs
View file

@ -9,7 +9,7 @@ use crate::testutil::{zstd_of, Scratch};
fn fresh() -> (Scratch, String, Connection) {
let (dir, path) = Scratch::db("repo");
let path = path.to_str().unwrap().to_string();
let conn = open_or_recreate(&path, "trigram").unwrap();
let conn = open_or_recreate(&path, &Default::default()).unwrap();
(dir, path, conn)
}
@ -76,14 +76,9 @@ fn fts_rows(conn: &Connection) -> i64 {
count(conn, "SELECT COUNT(*) FROM searchabletext")
}
/// Rows whose body matches the FTS query `term`.
/// Rows whose body the FTS index matches for `term`.
fn matches(conn: &Connection, term: &str) -> i64 {
conn.query_row(
"SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH ?1",
[term],
|r| r.get(0),
)
.unwrap()
crate::testutil::fts_count(conn, term)
}
fn state(conn: &Connection, id: i64) -> i64 {
@ -93,6 +88,22 @@ fn state(conn: &Connection, id: i64) -> i64 {
.unwrap()
}
/// Store `text` on a pending row, which must still be pending.
fn done(tx: &Transaction<'_>, id: i64, text: &str) {
assert!(store_pending_text(tx, id, text, zstd_of(text).as_deref()).unwrap());
}
/// Fail a row as a later run would: a changed file clears it back to pending
/// (the state `update_file_basic` leaves), then its extraction fails.
fn fail(tx: &Transaction<'_>, id: i64, reason: &str) {
if state(tx, id) == STATE_DONE {
clear_content_for_ids(tx, &[id]).unwrap();
}
set_content_state(tx, &[id], STATE_PENDING).unwrap();
let failed = ContentOutcome::Failed(reason.to_string());
assert!(complete_pending(tx, id, &failed, None).unwrap());
}
/// Every stored path, in `(parent, name)` order.
fn survivors(conn: &Connection) -> Vec<String> {
let mut stmt = conn
@ -114,18 +125,18 @@ fn seeded(conn: &mut Connection, paths: &[&str]) -> std::collections::HashMap<St
.iter()
.map(|path| {
let id = insert_at(tx, path, true);
set_content_done(tx, id, "body text", zstd_of("body text").as_deref()).unwrap();
done(tx, id, "body text");
(path.to_string(), id)
})
.collect()
})
}
/// `set_content_done_fresh` skips the pre-delete, so its safety is the
/// caller's claim that the row is clean. A duplicate `searchabletext` row
/// shows up as a file appearing twice in results, never as an error.
/// [`complete_pending`] clears nothing first, so its safety is that only a
/// pending row is written. A duplicate `searchabletext` row shows up as a file
/// appearing twice in results, never as an error.
#[test]
fn the_fresh_content_write_leaves_exactly_one_fts_row() {
fn a_pending_write_leaves_exactly_one_fts_row() {
let row = |mtime| NewFile {
size: 10,
mtime,
@ -135,13 +146,12 @@ fn the_fresh_content_write_leaves_exactly_one_fts_row() {
let id = with_tx(&mut conn, |tx| {
let id = insert_file(tx, &row(1)).unwrap().unwrap();
set_content_done_fresh(tx, id, "sphinx quartz", zstd_of("sphinx quartz").as_deref())
.unwrap();
done(tx, id, "sphinx quartz");
// The update path: the sequence `process_batch_updates` performs.
let same = update_file_basic(tx, &row(2)).unwrap().unwrap();
assert_eq!(same, id, "the same row");
set_content_done_fresh(tx, id, "sphinx onyx", zstd_of("sphinx onyx").as_deref()).unwrap();
done(tx, id, "sphinx onyx");
id
});
@ -150,16 +160,43 @@ fn the_fresh_content_write_leaves_exactly_one_fts_row() {
assert_eq!(matches(&conn, "onyx"), 1, "the new body is searchable");
assert_eq!(matches(&conn, "quartz"), 0, "the old body is gone");
// The idempotent entry point, which the content pass and the watcher use,
// still repairs a row that really does hold content.
with_tx(&mut conn, |tx| {
set_content_done(tx, id, "sphinx jasper", zstd_of("sphinx jasper").as_deref()).unwrap()
// A second write to a row that is no longer pending (the content pass
// holding a row the walk's inline text already stored) writes nothing.
let stored = with_tx(&mut conn, |tx| {
store_pending_text(tx, id, "sphinx jasper", zstd_of("sphinx jasper").as_deref()).unwrap()
});
assert!(!stored);
assert_eq!(fts_rows(&conn), 1);
assert_eq!(count(&conn, "SELECT COUNT(*) FROM documents_text"), 1);
assert_eq!(matches(&conn, "jasper"), 1);
assert_eq!(matches(&conn, "onyx"), 0);
assert_eq!(matches(&conn, "jasper"), 0);
assert_eq!(matches(&conn, "onyx"), 1);
}
/// Stale cleanup can delete a pending row while its extraction is in flight.
/// Writing the outcome anyway would leave a posting with no `files` row,
/// matching searches forever and inherited by whichever file reuses the id.
#[test]
fn an_outcome_for_a_row_deleted_in_flight_writes_nothing() {
let (_dir, _path, mut conn) = fresh();
let (text_id, failed_id) = with_tx(&mut conn, |tx| {
let a = insert_at(tx, "/gone/a.txt", true);
let b = insert_at(tx, "/gone/b.txt", true);
delete_ids(tx, &[a, b], &[]).unwrap();
(a, b)
});
with_tx(&mut conn, |tx| {
let text = ContentOutcome::Done {
text: "sphinx quartz".into(),
};
assert!(
!complete_pending(tx, text_id, &text, zstd_of("sphinx quartz").as_deref()).unwrap()
);
let failed = ContentOutcome::Failed("boom".into());
assert!(!complete_pending(tx, failed_id, &failed, None).unwrap());
});
assert_eq!(fts_rows(&conn), 0, "no orphan posting");
assert_eq!(count(&conn, "SELECT COUNT(*) FROM documents_text"), 0);
assert_eq!(count(&conn, "SELECT COUNT(*) FROM failed_files"), 0);
}
/// The size report reads each body's length out of its zstd frame header,
@ -252,17 +289,10 @@ fn insert_update_delete_round_trip() {
..text_row("/tmp/", "a.txt")
};
let id = insert_file(tx, &row).unwrap().expect("unique path");
set_content_done(tx, id, "hello world", zstd_of("hello world").as_deref()).unwrap();
done(tx, id, "hello world");
});
let hit: i64 = conn
.query_row(
"SELECT rowid FROM searchabletext WHERE searchabletext MATCH 'hello'",
[],
|r| r.get(0),
)
.unwrap();
assert!(hit > 0);
assert_eq!(matches(&conn, "hello"), 1);
with_tx(&mut conn, |tx| {
assert!(delete_file_by_path(tx, "/tmp/a.txt").unwrap())
@ -294,11 +324,11 @@ fn update_writes_content_state_from_needs_content() {
let (_dir, _path, mut conn) = fresh();
let row = NewFile {
size: 10,
..na_row("/tmp/", "a.txt")
..text_row("/tmp/", "a.txt")
};
let id = with_tx(&mut conn, |tx| {
let id = insert_file(tx, &row).unwrap().expect("unique path");
set_content_done(tx, id, "old text", zstd_of("old text").as_deref()).unwrap();
done(tx, id, "old text");
id
});
@ -365,7 +395,7 @@ fn delete_subtree_clears_every_dependent_table() {
],
);
with_tx(&mut conn, |tx| {
set_content_failed(tx, ids["/tree/deep/c.txt"], "bad parse").unwrap()
fail(tx, ids["/tree/deep/c.txt"], "bad parse")
});
let range = RootRange::for_root("/tree");
@ -441,9 +471,7 @@ fn delete_ids_clears_every_dependent_table() {
&mut conn,
&["/t/a.txt", "/t/b.log", "/t/deep/c.log", "/t/keep.txt"],
);
with_tx(&mut conn, |tx| {
set_content_failed(tx, ids["/t/deep/c.log"], "bad parse").unwrap()
});
with_tx(&mut conn, |tx| fail(tx, ids["/t/deep/c.log"], "bad parse"));
let doomed = vec![ids["/t/b.log"], ids["/t/deep/c.log"]];
// `c.log` is FAILED and so holds no posting: the distinction the second
@ -499,7 +527,7 @@ fn updating_clears_content_exactly_for_the_states_that_hold_it() {
let ids = seeded(&mut conn, &["/t/done.txt", "/t/failed.txt", "/t/keep.txt"]);
let (na_id, pending_id) = with_tx(&mut conn, |tx| {
// Clears failed.txt's posting too; the narrowing rests on that.
set_content_failed(tx, ids["/t/failed.txt"], "bad parse").unwrap();
fail(tx, ids["/t/failed.txt"], "bad parse");
let na = insert_file(tx, &na_row("/t/", "na.bin"))
.unwrap()
.expect("unique path");
@ -589,25 +617,6 @@ fn id_and_state_for_path_reads_the_stored_state() {
);
}
/// Dropping stored text must cost the file its snippets and nothing else.
#[test]
fn drop_stored_text_keeps_the_file_searchable() {
let (_dir, _path, mut conn) = fresh();
let ids = seeded(&mut conn, &["/t/a.txt", "/t/b.txt"]);
with_tx(&mut conn, |tx| {
drop_stored_text(tx, &[ids["/t/a.txt"]]).unwrap()
});
assert_eq!(count(&conn, "SELECT COUNT(*) FROM documents_text"), 1);
assert_eq!(count(&conn, "SELECT COUNT(*) FROM files"), 2);
assert_eq!(fts_rows(&conn), 2);
assert_eq!(
matches(&conn, "body"),
2,
"both files still match on content"
);
}
/// The reconciliation scan must serve every row in the range exactly once,
/// in order, and stop at the range bound rather than at a name prefix.
#[test]
@ -742,7 +751,7 @@ fn seed_rows(conn: &mut Connection, range: std::ops::Range<usize>) {
let id = insert_file(tx, &text_row("/tmp/bulk/", &name))
.unwrap()
.unwrap();
set_content_done(tx, id, &body, zstd_of(&body).as_deref()).unwrap();
done(tx, id, &body);
}
});
}
@ -788,7 +797,15 @@ fn checkpoint_truncate_reports_an_incomplete_checkpoint() {
/// [`crate::config::MINIMUM_WAL_SIZE`]; this shows the limit works.
#[test]
fn journal_size_limit_gives_the_space_back_after_a_blocked_truncate() {
let (_dir, p, mut writer) = fresh();
// With positions: the test needs more than 16 MiB of log, and position
// lists write it from the fewest rows.
let (_dir, p) = Scratch::db("repo");
let p = p.to_str().unwrap().to_string();
let positional = crate::config::ProcessingConfig {
compact_text_index: false,
..Default::default()
};
let mut writer = open_or_recreate(&p, &positional).unwrap();
seed_rows(&mut writer, 0..10);
writer
.busy_timeout(std::time::Duration::from_millis(100))
@ -864,7 +881,7 @@ fn a_busy_reader_defeats_the_autocheckpoint_but_not_a_forced_one() {
}
fn run(p: &str, force_every: usize) -> u64 {
let mut conn = open_or_recreate(p, "trigram").unwrap();
let mut conn = open_or_recreate(p, &Default::default()).unwrap();
seed_bare(&mut conn, 0..500);
// Stands in for a walk prefetcher: short reads, no gaps.
@ -1006,15 +1023,11 @@ fn maintain_skips_vacuum_on_a_tight_file() {
}
#[test]
fn set_content_failed_writes_failed_table() {
fn a_failed_extraction_records_its_reason() {
let (_dir, _path, mut conn) = fresh();
let id = with_tx(&mut conn, |tx| {
let row = NewFile {
size: 0,
..na_row("/tmp/", "oops.bin")
};
let id = insert_file(tx, &row).unwrap().expect("unique path");
set_content_failed(tx, id, "bad parse").unwrap();
let id = insert_at(tx, "/tmp/oops.txt", true);
fail(tx, id, "bad parse");
id
});
@ -1029,11 +1042,8 @@ fn set_content_failed_writes_failed_table() {
assert_eq!(state(&conn, id), STATE_FAILED);
}
/// A way out of `STATE_DONE`, applied to the row with the given id.
type Transition = fn(&Transaction<'_>, i64);
/// Every transition out of [`STATE_DONE`] takes the posting and the stored
/// body with it, so its invariant holds however a row got where it is.
/// Every way a changed file leaves [`STATE_DONE`] takes the posting and the
/// stored body with it, so its invariant holds however a row got where it is.
///
/// `count_root_counts_the_fts_rows_it_says_it_does` checks rows that were
/// never DONE, which passes even for a transition that leaves a stale posting
@ -1044,31 +1054,22 @@ type Transition = fn(&Transaction<'_>, i64);
fn leaving_done_always_takes_the_posting_with_it() {
let (_dir, _path, mut conn) = fresh();
// Each row starts DONE, with a posting and a stored body, and then takes
// one of the three ways out.
let leave: [(&str, Transition); 3] = [
("/t/failed.txt", |tx, id| {
set_content_failed(tx, id, "bad parse").unwrap()
}),
("/t/na.txt", |tx, id| set_content_na(tx, id).unwrap()),
("/t/rewritten.txt", |tx, _id| {
// The shape a changed file takes: metadata updated in place.
// Each row starts DONE, with a posting and a stored body, and is then
// rewritten in place: still text (back to pending), or no longer
// anything an extractor claims (NA).
for (name, needs_content) in [("rewritten.txt", true), ("now-binary.txt", false)] {
with_tx(&mut conn, |tx| {
let id = insert_at(tx, &format!("/t/{}", name), true);
done(tx, id, "body text");
let row = NewFile {
size: 99,
mtime: 99,
..text_row("/t/", "rewritten.txt")
needs_content,
..text_row("/t/", name)
};
update_file_basic(tx, &row)
.unwrap()
.expect("the row is there");
}),
];
for (path, transition) in leave {
with_tx(&mut conn, |tx| {
let id = insert_at(tx, path, true);
set_content_done(tx, id, "body text", zstd_of("body text").as_deref()).unwrap();
transition(tx, id);
});
}
@ -1094,12 +1095,12 @@ fn count_root_counts_the_fts_rows_it_says_it_does() {
// Two searchable, and one of each way a row can fail to be.
for path in ["/tree/a.txt", "/tree/b.txt"] {
let id = insert_at(tx, path, true);
set_content_done(tx, id, "body text", zstd_of("body text").as_deref()).unwrap();
done(tx, id, "body text");
}
let failed = insert_at(tx, "/tree/c.bin", true);
set_content_failed(tx, failed, "bad parse").unwrap();
fail(tx, failed, "bad parse");
let na = insert_at(tx, "/tree/d.iso", true);
set_content_na(tx, na).unwrap();
assert!(complete_pending(tx, na, &ContentOutcome::NotApplicable, None).unwrap());
insert_at(tx, "/tree/e.txt", true); // still pending
});
@ -1116,7 +1117,7 @@ fn count_root_counts_the_fts_rows_it_says_it_does() {
// root's figure.
with_tx(&mut conn, |tx| {
let id = insert_at(tx, "/elsewhere/f.txt", true);
set_content_done(tx, id, "body text", zstd_of("body text").as_deref()).unwrap();
done(tx, id, "body text");
});
assert_eq!(fts_rows(&conn), 3);
assert_eq!(
@ -1226,11 +1227,10 @@ fn deleting_a_file_row_cascades_the_fk_tables() {
let (_dir, _path, mut conn) = fresh();
seeded(&mut conn, &["/casc/a.txt", "/casc/b.txt"]);
// The failure goes on a row that never extracted, the only way a run
// reaches `set_content_failed`, so the FTS figures below stay about the
// cascade.
// reaches a failure, so the FTS figures below stay about the cascade.
with_tx(&mut conn, |tx| {
let never = insert_at(tx, "/casc/c.bin", true);
set_content_failed(tx, never, "boom").unwrap();
fail(tx, never, "boom");
});
assert_eq!(count(&conn, "SELECT COUNT(*) FROM documents_text"), 2);
assert_eq!(count(&conn, "SELECT COUNT(*) FROM failed_files"), 1);
@ -1260,7 +1260,7 @@ fn retry_failed_files_resets_state_and_clears_records() {
let (_dir, _path, mut conn) = fresh();
let ids = seeded(&mut conn, &["/retry/bad.txt", "/retry/good.txt"]);
with_tx(&mut conn, |tx| {
set_content_failed(tx, ids["/retry/bad.txt"], "parser choked").unwrap()
fail(tx, ids["/retry/bad.txt"], "parser choked")
});
assert_eq!(with_tx(&mut conn, |tx| retry_failed_files(tx).unwrap()), 1);
@ -1281,8 +1281,8 @@ fn a_changed_file_clears_its_failure_record() {
let (_dir, _path, mut conn) = fresh();
let ids = seeded(&mut conn, &["/chg/a.txt", "/chg/b.txt"]);
with_tx(&mut conn, |tx| {
set_content_failed(tx, ids["/chg/a.txt"], "boom").unwrap();
set_content_failed(tx, ids["/chg/b.txt"], "boom").unwrap();
fail(tx, ids["/chg/a.txt"], "boom");
fail(tx, ids["/chg/b.txt"], "boom");
});
let changed = |name: &'static str, needs_content: bool| NewFile {

79
crates/quicksearch-core/src/db/schema.rs
View file

@ -170,6 +170,13 @@ pub const PRAGMAS_WALK_READER: &str = "
/// Measured: an index cost ~10% on inserts and 805 pages at 300k rows, pages
/// that compete with the ones search wants resident.
///
/// Nor do `mtime` and `mime`, which only the `modified:` and `mime:` filters
/// could seek, while every insert, update and delete paid for both. Measured
/// (`indexprobe cold`, 100k tiny files): index 23.0 → 19.5 MiB, wall time
/// 0.47 → 0.41 s. The filters now cost what the `files` scan behind every
/// filename query already does: `searchtime` at 300k rows, 25 ms with or
/// without a filter, where the indexes had made a one-day filter 0.06 ms.
///
/// `documents_text` holds a row only when the extractor produced text (an
/// audio file with empty tags is done but has none), so `files.content_state`
/// is the authority on "content indexed". The uncompressed length is not
@ -196,8 +203,6 @@ CREATE TABLE files (
);
CREATE UNIQUE INDEX idx_files_parent ON files(parent, name);
CREATE INDEX idx_files_mtime ON files(mtime);
CREATE INDEX idx_files_mime ON files(mime);
CREATE INDEX idx_files_hash ON files(hash);
CREATE INDEX idx_files_content_pending ON files(id) WHERE content_state = 0;
@ -481,31 +486,44 @@ pub const FTS5_DEFAULT_PGSZ: i64 = 4050;
///
/// One column: only document bodies are ever MATCHed, and a `name` column
/// would cost (len − 2) postings per file.
pub fn fts_create_sql(tokenizer: &str) -> String {
let effective = effective_tokenizer(tokenizer);
///
/// `positions` is [`crate::config::ProcessingConfig::fts_positions`]. Without
/// them (`detail=none`, the default through `compact_text_index`) the table
/// answers only single-token phrases, so the search side asks for a term as
/// the AND of its trigrams (`search::prefilter::fts_literal`) and the
/// full-text pass drops the documents where they are not adjacent, which it
/// can because text is always stored. Measured on 1.1 GB of crate sources
/// (`indexprobe cold`, then `searchtime`), identical results either way:
///
/// | | positions | none |
/// |---|---|---|
/// | index | 1291 MiB | **219 MiB** |
/// | build | 37.4 s | **24.6 s** |
/// | written through the WAL | 4.1 GB | **0.7 GB** |
/// | candidates per true hit | 1 | 1.0–2x a word, 8–13x a phrase |
/// | warm search time | 1 | 0.9–1.8x a word, 2.8x a phrase |
///
/// The slowest was a phrase of common words (`impl Iterator for`, 24 → 67
/// ms), whose trigrams occur in many documents that lack the phrase.
///
/// The configured tokenizer is applied as written. Plain `trigram` once got
/// `remove_diacritics 1` appended, but full-text verification folds ASCII
/// only (`tests/cascade.rs`, `diacritic_fts_candidates_are_dropped`), so the
/// extra candidates it matched were always dropped again. Dropping it measured
/// neutral on indexing time (`indexprobe cold`, 5k accented documents).
pub fn fts_create_sql(tokenizer: &str, positions: bool) -> String {
format!(
"CREATE VIRTUAL TABLE searchabletext USING fts5(\
text, \
tokenize='{}', \
content='', \
contentless_delete=1\
contentless_delete=1{}\
);",
effective.replace('\'', "''")
tokenizer.trim().replace('\'', "''"),
if positions { "" } else { ", detail=none" }
)
}
/// Map a user-facing tokenizer name to the FTS5 option string applied: plain
/// `trigram` gets `remove_diacritics 1` (so `cafe` matches `café`); any
/// explicit option string is used verbatim.
pub fn effective_tokenizer(tokenizer: &str) -> String {
let trimmed = tokenizer.trim();
if trimmed.eq_ignore_ascii_case("trigram") {
"trigram remove_diacritics 1".to_string()
} else {
trimmed.to_string()
}
}
#[cfg(test)]
mod tests {
use super::*;
@ -630,24 +648,19 @@ mod tests {
}
#[test]
fn plain_trigram_gets_accent_stripping() {
assert_eq!(
effective_tokenizer("trigram"),
"trigram remove_diacritics 1"
);
assert_eq!(
effective_tokenizer(" trigram "),
"trigram remove_diacritics 1"
fn the_tokenizer_is_applied_as_written() {
assert!(fts_create_sql(" trigram ", true).contains("tokenize='trigram',"));
assert!(fts_create_sql("unicode61", true).contains("tokenize='unicode61',"));
assert!(
fts_create_sql("trigram remove_diacritics 1", true)
.contains("tokenize='trigram remove_diacritics 1',"),
"an explicit option string still passes through"
);
}
#[test]
fn explicit_tokenizers_pass_through() {
assert_eq!(
effective_tokenizer("trigram remove_diacritics 0"),
"trigram remove_diacritics 0"
);
assert_eq!(effective_tokenizer("porter"), "porter");
assert_eq!(effective_tokenizer("unicode61"), "unicode61");
fn positions_decide_the_detail_option() {
assert!(!fts_create_sql("trigram", true).contains("detail"));
assert!(fts_create_sql("trigram", false).contains(", detail=none)"));
}
}

122
crates/quicksearch-core/src/extract/pdf.rs
View file

@ -51,17 +51,19 @@ impl Extractor for PdfExtractor {
/// has been measured holding tens of megabytes. Input and output are
/// bounded by the limits, but the middle is `pdf_extract`'s with no
/// scratch to reuse, so peak for PDFs alone is `workers × amplification`,
/// one pool per root.
/// one pool per root. Pages stop being rendered once the text reaches
/// `limits().text`.
fn extract(
&self,
path: &Path,
out: &mut String,
_scratch: &mut Scratch,
scratch: &mut Scratch,
) -> Result<(), ExtractError> {
let limit = scratch.limits().text;
// Loading is inside the guard too: a panic outside it takes the thread.
install_quiet_panic_hook();
SUPPRESS_PANIC_PRINT.with(|flag| flag.set(true));
let result = std::panic::catch_unwind(|| extract_one_pass(path));
let result = std::panic::catch_unwind(|| extract_one_pass(path, limit));
SUPPRESS_PANIC_PRINT.with(|flag| flag.set(false));
let text = result
.map_err(|panic| format!("pdf_extract panicked: {}", panic_message(&*panic)))??;
@ -70,8 +72,36 @@ impl Extractor for PdfExtractor {
}
}
/// `pdf_extract::extract_text`'s body (load, decrypt, `output_doc`) spelled out.
fn extract_one_pass(path: &Path) -> Result<String, ExtractError> {
/// The text sink, full at `limit` bytes. Once the text reaches what the
/// caller keeps, the next write fails, which ends `output_doc` rather than
/// rendering pages whose text would only be truncated away. Measured
/// (`extractprobe`, text-only PDFs of 684 and ~340 pages at the default
/// 256 KiB limit): 1.7 s → 0.8 s of CPU.
struct Capped<'a> {
text: &'a mut String,
limit: usize,
}
impl std::fmt::Write for Capped<'_> {
fn write_str(&mut self, s: &str) -> std::fmt::Result {
if self.text.len() >= self.limit {
return Err(std::fmt::Error);
}
self.text.push_str(s);
Ok(())
}
}
impl pdf_extract::ConvertToFmt for Capped<'_> {
type Writer = Self;
fn convert(self) -> Self {
self
}
}
/// `pdf_extract::extract_text`'s body (load, decrypt, `output_doc`) spelled
/// out, stopping once `limit` bytes of text are in.
fn extract_one_pass(path: &Path, limit: usize) -> Result<String, ExtractError> {
let mut doc = Document::load(path).map_err(|e| format!("pdf_extract: {}", e))?;
// Empty-password decrypt only; nothing on this path can prompt for a real one.
if doc.is_encrypted() {
@ -79,11 +109,19 @@ fn extract_one_pass(path: &Path) -> Result<String, ExtractError> {
}
let mut text = String::new();
{
let mut sink = PlainTextOutput::new(&mut text);
pdf_extract::output_doc(&doc, &mut sink).map_err(|e| format!("pdf_extract: {}", e))?;
let rendered = {
let mut sink = PlainTextOutput::new(Capped {
text: &mut text,
limit,
});
pdf_extract::output_doc(&doc, &mut sink)
};
match rendered {
Ok(()) => Ok(text),
// The sink filled: the stop is ours, not a failure.
Err(_) if text.len() >= limit => Ok(text),
Err(e) => Err(format!("pdf_extract: {}", e)),
}
Ok(text)
}
#[cfg(test)]
@ -195,6 +233,72 @@ mod tests {
assert!(out.contains("Body Only"));
}
/// Rendering stops once the text limit is reached: a later page's text
/// is never produced, not produced and cut.
#[test]
fn extraction_stops_at_the_text_limit() {
let mut doc = Document::with_version("1.5");
let font = doc.add_object(dictionary! {
"Type" => "Font",
"Subtype" => "Type1",
"BaseFont" => "Helvetica",
});
let resources = doc.add_object(dictionary! {
"Font" => dictionary! { "F1" => font },
});
let pages_id = doc.new_object_id();
let kids: Vec<Object> = ["alphapage", "bravopage", "charliepage", "deltapage"]
.iter()
.map(|word| {
let content = format!("BT /F1 24 Tf 72 720 Td ({}) Tj ET", word);
let contents = doc.add_object(Stream::new(dictionary! {}, content.into_bytes()));
doc.add_object(dictionary! {
"Type" => "Page",
"Parent" => pages_id,
"Contents" => contents,
"MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
})
.into()
})
.collect();
doc.objects.insert(
pages_id,
Object::Dictionary(dictionary! {
"Type" => "Pages",
"Count" => kids.len() as i64,
"Kids" => kids,
"Resources" => resources,
}),
);
let catalog = doc.add_object(dictionary! {
"Type" => "Catalog",
"Pages" => pages_id,
});
doc.trailer.set("Root", catalog);
let path = crate::testutil::scratch_dir("pdf-limit").join("pages.pdf");
doc.save(&path).expect("write fixture pdf");
let whole = extract(&path).expect("extract");
assert!(
whole.contains("deltapage"),
"every page renders: {:?}",
whole
);
let mut config = crate::config::Config::default();
config.processing.maximum_text_size = 12;
let mut out = String::new();
PdfExtractor
.extract(&path, &mut out, &mut Scratch::new(&config))
.expect("a stop at the limit is not a failure");
assert!(out.contains("alphapage"), "got {:?}", out);
assert!(
!out.contains("deltapage"),
"rendered past the limit: {:?}",
out
);
}
#[test]
fn malformed_pdf_fails_without_panicking_the_process() {
let path = crate::testutil::scratch_dir("pdf-malformed").join("broken.pdf");

61
crates/quicksearch-core/src/extract/plaintext.rs
View file

@ -38,11 +38,17 @@ pub(crate) const EXTRA_TEXT_MIMES: &[&str] = &[
"message/rfc822",
];
/// Decode `bytes` into `out`. Valid UTF-8 is moved, not copied, which is why
/// plaintext reads into a fresh buffer rather than the worker's scratch:
/// reusing one would trade an allocation for a full-length `memcpy` per file.
fn decode_into(bytes: Vec<u8>, path: &Path, out: &mut String) -> Result<(), ExtractError> {
super::append(out, crate::textenc::decode_text(bytes, path)?);
/// Decode `bytes`, a prefix of the file when `truncated`, into `out`. Valid
/// UTF-8 is moved, not copied, which is why plaintext reads into a fresh
/// buffer rather than the worker's scratch: reusing one would trade an
/// allocation for a full-length `memcpy` per file.
fn decode_into(
bytes: Vec<u8>,
path: &Path,
truncated: bool,
out: &mut String,
) -> Result<(), ExtractError> {
super::append(out, crate::textenc::decode_text(bytes, path, truncated)?);
Ok(())
}
@ -79,9 +85,12 @@ impl Extractor for PlaintextExtractor {
out: &mut String,
scratch: &mut Scratch,
) -> Result<(), ExtractError> {
// The pass never offers a file above `limits().read`, so this cap only
// bites for a file that grew since the walk or reports a false size.
let cap = scratch.limits().read;
// Only the prefix that can survive truncation to `limits().text` is
// read: UTF-16 is the widest input per byte of UTF-8 output (2:1), so
// no file contributes more than twice the limit. Reading a 40 MiB log
// whole to keep 256 KiB of it was the content pass's peak memory.
let limits = scratch.limits();
let cap = limits.read.min(limits.text.saturating_mul(2));
let mut f =
File::open(path).map_err(|e| format!("plaintext read {}: {}", path.display(), e))?;
let size = f
@ -96,10 +105,12 @@ impl Extractor for PlaintextExtractor {
f.take(cap as u64)
.read_to_end(&mut buf)
.map_err(|e| format!("plaintext read {}: {}", path.display(), e))?;
return decode_into(buf, path, out);
let truncated = buf.len() == cap;
return decode_into(buf, path, truncated, out);
}
decode_into(read_sized(&mut f, size, cap, path)?, path, out)
let buf = read_sized(&mut f, size, cap, path)?;
decode_into(buf, path, size > cap, out)
}
/// The head belongs to the walk worker and is reused for the next file,
@ -301,6 +312,36 @@ mod tests {
std::fs::remove_file(&p).ok();
}
/// Twice the text limit: the most any file can contribute to stored text.
fn prefix_cap() -> usize {
scratch().limits().text * 2
}
#[test]
fn a_file_far_over_the_text_limit_reads_only_the_keepable_prefix() {
let body = vec![b'z'; prefix_cap() * 3];
let p = tmp("prefix", &body);
let out = extract(&p).unwrap();
assert_eq!(out.len(), prefix_cap(), "read past what can be kept");
std::fs::remove_file(&p).ok();
}
/// A read that cuts a multibyte character must not demote a UTF-8 file to
/// a legacy charset, which would decode the whole prefix as mojibake.
#[test]
fn a_character_cut_by_the_prefix_read_is_dropped_not_misdecoded() {
let mut body = vec![b'a'; prefix_cap() - 1];
body.extend_from_slice("é and more after the cut".as_bytes());
let p = tmp("cutchar", &body);
let out = extract(&p).unwrap();
assert_eq!(out.len(), prefix_cap() - 1, "the cut é's stub is dropped");
assert!(
out.bytes().all(|b| b == b'a'),
"decoded as UTF-8, not legacy"
);
std::fs::remove_file(&p).ok();
}
#[test]
fn a_file_under_the_cap_is_read_whole() {
let body = vec![b'y'; 4096];

60
crates/quicksearch-core/src/file_handling/batch.rs
View file

@ -14,17 +14,19 @@ use crate::db::repo;
/// with `slots[i]` saying where row `i`'s is, that it has none, or that its
/// compression failed (kept per row rather than failing the batch).
///
/// Everything here is reused across chunks: the encoder because a zstd
/// context per chunk is wasted CPU (`benches/index.rs`, `zstd_encode`), the
/// arena so the writer allocates nothing per document.
struct Bodies {
/// One per indexing run, reused by every batch and turn the writer takes: the
/// encoder because a zstd context per chunk is wasted CPU (`benches/index.rs`,
/// `zstd_encode`), the arena so the writer allocates nothing per document.
/// Measured against one per writer turn (`indexprobe cold`, 10k docs of
/// 9–20 KiB): VmHWM 60–65 → 44–50 MiB, CPU ~2% lower.
pub struct Bodies {
enc: repo::DocEncoder,
arena: Vec<u8>,
slots: Vec<Result<Option<std::ops::Range<usize>>, String>>,
}
impl Bodies {
fn new() -> Result<Bodies, String> {
pub fn new() -> Result<Bodies, String> {
Ok(Bodies {
enc: repo::DocEncoder::new()?,
arena: Vec::new(),
@ -34,14 +36,12 @@ impl Bodies {
/// Compress one chunk's bodies before the caller takes the connection, so
/// the lock covers only the SQL.
fn fill<'a>(&mut self, texts: impl Iterator<Item = Option<&'a str>>, config: &Config) {
fn fill<'a>(&mut self, texts: impl Iterator<Item = Option<&'a str>>) {
self.arena.clear();
self.slots.clear();
for text in texts {
let slot = match text {
Some(t) if config.processing.store_text_for_snippets && !t.is_empty() => {
self.enc.encode_into(t, &mut self.arena).map(Some)
}
Some(t) if !t.is_empty() => self.enc.encode_into(t, &mut self.arena).map(Some),
_ => Ok(None),
};
self.slots.push(slot);
@ -61,29 +61,25 @@ impl Bodies {
/// The skeleton behind [`process_batch_updates`] and
/// [`process_batch_inserts`]: compress each chunk's bodies outside the lock,
/// write its rows in one transaction through `write_row`, and store any
/// inline text on a fresh row.
///
/// `_fresh` is sound here because both row writers leave a row that provably
/// holds no content: see their comments at the call sites.
/// inline text on the row, which both row writers leave pending.
fn write_prepared_records(
conn_mutex: &Arc<Mutex<Connection>>,
records: &[OwnedNewFile],
stop_flag: &Arc<AtomicBool>,
config: &Config,
bodies: &mut Bodies,
write_row: impl Fn(&rusqlite::Transaction<'_>, &OwnedNewFile) -> Result<Option<i64>, String>,
) -> Result<(), String> {
// `.max(1)`: `chunks(0)` panics on the indexing thread, so a hand-edited
// `batch_size = 0` would wedge indexing while the UI reads "Running".
let chunk_size = config.processing.batch_size.max(1);
// One set of buffers for every chunk this call writes.
let mut bodies = Bodies::new()?;
for batch in records.chunks(chunk_size) {
if stop_flag.load(Ordering::Relaxed) {
return Ok(());
}
// Outside the lock; see `Bodies::fill`.
bodies.fill(batch.iter().map(|r| r.inline_text.as_deref()), config);
bodies.fill(batch.iter().map(|r| r.inline_text.as_deref()));
let conn = crate::lock_ok(conn_mutex);
let tx = conn
.unchecked_transaction()
@ -96,7 +92,9 @@ fn write_prepared_records(
let id = write_row(&tx, rec)?;
if let (Some(id), Some(text)) = (id, rec.inline_text.as_deref()) {
match bodies.get(i) {
Ok(zstd) => repo::set_content_done_fresh(&tx, id, text, zstd)?,
Ok(zstd) => {
repo::store_pending_text(&tx, id, text, zstd)?;
}
Err(e) => crate::log_warn!("compress text for {}: {}", rec.path(), e),
}
}
@ -117,19 +115,21 @@ pub fn process_batch_updates(
files_to_update: &[OwnedNewFile],
stop_flag: &Arc<AtomicBool>,
config: &Config,
bodies: &mut Bodies,
) -> Result<(), String> {
if files_to_update.is_empty() {
return Ok(());
}
// The row leaves this closure `_fresh`: `update_file_basic` cleared a
// DONE row's content in this same transaction, every other state
// provably holds none, and the insert fallback created the row outright.
// The row leaves this closure pending whenever it carries inline text:
// `update_file_basic` cleared a DONE row's content and reset its state
// in this same transaction, and the insert fallback created it pending.
write_prepared_records(
conn_mutex,
files_to_update,
stop_flag,
config,
bodies,
|tx, rec| {
let updated = repo::update_file_basic(tx, &rec.as_new_file()).map_err(|e| {
format!(
@ -162,18 +162,20 @@ pub fn process_batch_inserts(
files_to_insert: &[OwnedNewFile],
stop_flag: &Arc<AtomicBool>,
config: &Config,
bodies: &mut Bodies,
) -> Result<(), String> {
if files_to_insert.is_empty() {
return Ok(());
}
// The row leaves this closure `_fresh`: `insert_file` returned `Some`
// only by creating it, so it cannot carry content from anywhere.
// `insert_file` returned `Some` only by creating the row, pending when it
// needs content, so it cannot carry content from anywhere.
write_prepared_records(
conn_mutex,
files_to_insert,
stop_flag,
config,
bodies,
|tx, rec| {
repo::insert_file(tx, &rec.as_new_file())
.map_err(|e| format!("Failed to insert file record: {}", e))
@ -365,7 +367,8 @@ const STORE_CHUNK: usize = 32;
pub struct Stored {
/// Rows the caller must now drop from its buffer, written or not.
pub consumed: usize,
/// Rows whose write succeeded, so whose `content_state` moved.
/// Rows whose write succeeded, so whose `content_state` moved; a row
/// no longer pending by the time it is written is consumed, not written.
pub written: usize,
}
@ -379,12 +382,10 @@ pub fn store_extracted(
conn_mutex: &Arc<Mutex<Connection>>,
rows: &[crate::content::ExtractedRow],
stop_flag: &Arc<AtomicBool>,
config: &Config,
deadline: std::time::Instant,
bodies: &mut Bodies,
) -> Result<Stored, String> {
let mut done = Stored::default();
// One set of buffers for every chunk this turn writes.
let mut bodies = Bodies::new()?;
for chunk in rows.chunks(STORE_CHUNK) {
if stop_flag.load(Ordering::Relaxed) {
break;
@ -394,7 +395,6 @@ pub fn store_extracted(
chunk
.iter()
.map(|r| crate::file_handling::outcome_body(&r.outcome)),
config,
);
let conn = crate::lock_ok(conn_mutex);
let tx = conn
@ -406,8 +406,10 @@ pub fn store_extracted(
done.consumed += 1;
match bodies.get(i) {
Err(e) => crate::log_warn!("compress text for {}: {}", row.path(), e),
Ok(zstd) => match store_content_outcome(&tx, row.file_id, &row.outcome, zstd) {
Ok(()) => done.written += 1,
Ok(zstd) => match repo::complete_pending(&tx, row.file_id, &row.outcome, zstd) {
Ok(true) => done.written += 1,
// No longer pending: stored or deleted since it was paged.
Ok(false) => {}
Err(e) => crate::log_warn!("content indexing for {}: {}", row.path(), e),
},
}

11
crates/quicksearch-core/src/file_handling/count_and_extract_tests.rs
View file

@ -228,10 +228,17 @@ fn extract_scope_counts_only_files_an_extractor_claims() {
.collect();
let conn_mutex = Arc::new(Mutex::new(
crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap(),
crate::db::open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap(),
));
let stop = Arc::new(AtomicBool::new(false));
process_batch_inserts(&conn_mutex, &records, &stop, &config).unwrap();
process_batch_inserts(
&conn_mutex,
&records,
&stop,
&config,
&mut Bodies::new().unwrap(),
)
.unwrap();
let range = RootRange::for_root(root.to_str().unwrap());
let scope = extract_scope_prepare(&conn_mutex, &range, &config).unwrap();

9
crates/quicksearch-core/src/file_handling/mod.rs
View file

@ -16,8 +16,8 @@ pub(crate) use batch::extract_scope_prepare;
pub(crate) use batch::max_text_file_size;
pub use batch::{
cleanup_stale_index_entries, count_extract_scope, mark_oversize_pending_na,
process_batch_inserts, process_batch_updates, store_extracted, ExtractCursor, ExtractScope,
RootRange, Stored,
process_batch_inserts, process_batch_updates, store_extracted, Bodies, ExtractCursor,
ExtractScope, RootRange, Stored,
};
pub use counting::count_tree_entries_fast;
pub use paths::{
@ -27,10 +27,11 @@ pub(crate) use paths::{
dir_to_db_parent, index_file_set, normalize_root_string, path_to_db_string,
warn_if_unrepresentable, INDEX_SIDECAR_SUFFIXES,
};
pub(crate) use records::mtime_secs;
pub use records::{
classify_by_mtime, classify_for_indexing, content_extractable, decide_content,
extract_and_store, fts_begin_bulk_write, fts_begin_tombstone_burst, fts_end_tombstone_burst,
fts_finalize_after_text_indexing, get_file_hash, hash_failure_counts, outcome_body,
prepare_file_record, prepare_file_record_from_path, reset_run_warnings, store_content_outcome,
ContentOutcome, DirRows, FileIndexAction, OwnedNewFile, FTS_DELETEMERGE,
prepare_file_record, reset_run_warnings, ContentOutcome, DirRows, FileIndexAction,
OwnedNewFile, FTS_DELETEMERGE,
};

83
crates/quicksearch-core/src/file_handling/records.rs
View file

@ -43,13 +43,25 @@ pub fn classify_by_mtime(stored: Option<u64>, mtime: u64) -> FileIndexAction {
}
/// Truncate in place to at most `max_bytes`, backing up to a UTF-8 char
/// boundary.
/// boundary, and give back the capacity: the text goes on to wait in the
/// content pass's queues, where a 40 MiB log truncated without the shrink
/// would still hold 40 MiB per queued row.
pub(crate) fn safe_truncate(s: &mut String, max_bytes: usize) {
if s.len() > max_bytes {
s.truncate(s.floor_char_boundary(max_bytes));
s.shrink_to_fit();
}
}
/// A file's modified time in whole seconds, the unit `files.mtime` stores and
/// every unchanged-file check compares; `None` when the platform has none.
pub(crate) fn mtime_secs(meta: &std::fs::Metadata) -> Option<u64> {
meta.modified()
.ok()
.and_then(|t| t.duration_since(UNIX_EPOCH).ok())
.map(|d| d.as_secs())
}
/// A SHA-256 digest, held inline rather than as a `Vec` so hashing allocates
/// nothing per file.
pub type FileHash = [u8; 32];
@ -254,9 +266,10 @@ pub fn fts_finalize_after_text_indexing(conn: &Connection) {
/// An owned, fully-derived file record, produced by [`prepare_file_record`].
#[derive(Debug, Clone)]
pub struct OwnedNewFile {
pub name: String,
/// The containing directory, ending in the platform separator.
pub parent: String,
/// Parent and name in one buffer; the parent ends in the platform
/// separator. One allocation rather than two: `indexprobe cold`, 100k
/// files, 7.2 → 6.2 allocations per file.
pub path: repo::RowPath,
pub size: u64,
pub mtime: u64,
/// Borrowed from the static tables [`guess_mime_from_head`] answers from.
@ -275,14 +288,14 @@ pub struct OwnedNewFile {
}
impl OwnedNewFile {
pub fn path(&self) -> String {
format!("{}{}", self.parent, self.name)
pub fn path(&self) -> &str {
self.path.as_str()
}
pub fn as_new_file(&self) -> NewFile<'_> {
NewFile {
name: &self.name,
parent: &self.parent,
name: self.path.name(),
parent: self.path.parent(),
size: self.size,
mtime: self.mtime,
mime: self.mime,
@ -327,11 +340,7 @@ pub fn prepare_file_record(
}
let size = meta.len();
let mtime = meta
.modified()
.ok()
.and_then(|t| t.duration_since(UNIX_EPOCH).ok())
.map(|d| d.as_secs())?;
let mtime = mtime_secs(meta)?;
// A dehydrated cloud file is indexed from its metadata alone: reading
// even the first byte would block on downloading the whole file.
@ -362,7 +371,7 @@ pub fn prepare_file_record(
};
let (parent, name) = split_db_path(path)?;
let (parent, name) = (parent.to_string(), name.to_string());
let row_path = repo::RowPath::new(parent, name);
let head = scratch.head();
let mime = guess_mime_from_head(Path::new(path), head);
let ftype = mime.map(mime_to_type).unwrap_or(FileType::EMPTY);
@ -389,8 +398,7 @@ pub fn prepare_file_record(
});
Some(OwnedNewFile {
name,
parent,
path: row_path,
size,
mtime,
mime,
@ -401,26 +409,10 @@ pub fn prepare_file_record(
})
}
/// [`prepare_file_record`] for a path not yet resolved: the watcher path. One
/// file at a time, so it owns the scratch rather than being handed one.
pub fn prepare_file_record_from_path(
path: &Path,
config: &Config,
registry: &Registry,
) -> Option<OwnedNewFile> {
let canonical = path.canonicalize().ok()?;
if warn_if_unrepresentable(&canonical) {
return None;
}
let db_path = path_to_db_string(&canonical);
let meta = std::fs::metadata(&canonical).ok()?;
let mut scratch = Scratch::new(config);
prepare_file_record(&db_path, &meta, config, registry, &mut scratch)
}
/// Extract content for one file and record the outcome on its row. `mime` is
/// authoritative, including when it is `None`: the head was already sniffed
/// by [`prepare_file_record`], and re-sniffing gives the same answer.
/// Extract content for one pending file and record the outcome on its row
/// ([`repo::complete_pending`]). `mime` is authoritative, including when it
/// is `None`: the head was already sniffed by [`prepare_file_record`], and
/// re-sniffing gives the same answer.
pub fn extract_and_store(
tx: &rusqlite::Transaction<'_>,
file_id: i64,
@ -434,10 +426,10 @@ pub fn extract_and_store(
let mut scratch = Scratch::new(config);
let outcome = decide_content(path, mime, registry, config, &mut scratch);
let zstd = match outcome_body(&outcome) {
Some(text) => repo::encode_one(text, config.processing.store_text_for_snippets)?,
Some(text) => repo::encode_one(text)?,
None => None,
};
store_content_outcome(tx, file_id, &outcome, zstd.as_deref())
repo::complete_pending(tx, file_id, &outcome, zstd.as_deref()).map(|_| ())
}
/// What should be written for one file's content, decided without touching
@ -496,21 +488,6 @@ pub fn decide_content(
}
}
/// Apply a decision from [`decide_content`]: the cheap half, all that runs
/// with the connection held. `text_zstd`: see [`repo::set_content_done`].
pub fn store_content_outcome(
tx: &rusqlite::Transaction<'_>,
file_id: i64,
outcome: &ContentOutcome,
text_zstd: Option<&[u8]>,
) -> Result<(), String> {
match outcome {
ContentOutcome::Done { text } => repo::set_content_done(tx, file_id, text, text_zstd),
ContentOutcome::NotApplicable => repo::set_content_na(tx, file_id),
ContentOutcome::Failed(reason) => repo::set_content_failed(tx, file_id, reason),
}
}
/// The body a [`ContentOutcome`] would store, if any.
pub fn outcome_body(outcome: &ContentOutcome) -> Option<&str> {
match outcome {

26
crates/quicksearch-core/src/file_handling/tests.rs
View file

@ -576,7 +576,7 @@ fn a_symlinked_root_yields_no_directories_when_following_is_off() {
fn a_bulk_write_repairs_a_delete_merge_threshold_left_off() {
let dir = crate::testutil::scratch_dir("deletemerge-repair");
let db = dir.join("index.sqlite");
let conn = crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
let conn = crate::db::open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap();
// What a killed burst leaves behind.
fts_begin_tombstone_burst(&conn);
@ -626,9 +626,10 @@ fn seed_stale_corpus(
.expect("unique path");
ids.insert(name, id);
}
repo::set_content_done(&tx, ids["done-stale.txt"], "body one", None).unwrap();
repo::set_content_done(&tx, ids["done-kept.txt"], "body two", None).unwrap();
repo::set_content_failed(&tx, ids["failed-stale.txt"], "bad parse").unwrap();
repo::store_pending_text(&tx, ids["done-stale.txt"], "body one", None).unwrap();
repo::store_pending_text(&tx, ids["done-kept.txt"], "body two", None).unwrap();
let failed = crate::file_handling::ContentOutcome::Failed("bad parse".into());
repo::complete_pending(&tx, ids["failed-stale.txt"], &failed, None).unwrap();
tx.commit().unwrap();
ids
}
@ -646,7 +647,7 @@ fn stale_cleanup_deletes_exactly_the_stale_rows_and_their_postings() {
let dir = crate::testutil::scratch_dir("stale-cleanup");
let db = dir.join("index.sqlite");
let mut conn = crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
let mut conn = crate::db::open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap();
let ids = seed_stale_corpus(&mut conn);
// `vanished.txt` was never indexed: a stale list can legitimately name a
@ -675,14 +676,11 @@ fn stale_cleanup_deletes_exactly_the_stale_rows_and_their_postings() {
assert_eq!(count("SELECT COUNT(*) FROM searchabletext"), 1);
assert_eq!(count("SELECT COUNT(*) FROM documents_text"), 0);
assert_eq!(count("SELECT COUNT(*) FROM failed_files"), 0);
let hit: i64 = conn
.query_row(
"SELECT rowid FROM searchabletext WHERE searchabletext MATCH 'body'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(hit, ids["done-kept.txt"], "the survivor is still findable");
assert_eq!(
crate::testutil::fts_rowids(&conn, "body"),
vec![ids["done-kept.txt"]],
"the survivor is still findable"
);
assert_eq!(
deletemerge(&conn),
Some(i64::from(FTS_DELETEMERGE)),
@ -704,7 +702,7 @@ fn a_stopped_stale_cleanup_deletes_nothing_and_leaves_the_burst() {
let dir = crate::testutil::scratch_dir("stale-cleanup-stop");
let db = dir.join("index.sqlite");
let mut conn = crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
let mut conn = crate::db::open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap();
seed_stale_corpus(&mut conn);
let conn_mutex = Arc::new(Mutex::new(conn));

131
crates/quicksearch-core/src/incremental.rs
View file

@ -6,14 +6,14 @@ use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Instant;
use rusqlite::{Connection, OptionalExtension};
use rusqlite::Connection;
use crate::config::{Config, IgnoreSet};
use crate::db::repo;
use crate::extract::Registry;
use crate::extract::{Registry, Scratch};
use crate::file_handling::{
db_key_for_missing_path, extract_and_store, filtered_walk, prepare_file_record_from_path,
RootRange, UnreadableDirs,
db_key_for_missing_path, extract_and_store, filtered_walk, mtime_secs, path_to_db_string,
prepare_file_record, warn_if_unrepresentable, RootRange, UnreadableDirs,
};
use crate::platform::path_has_hidden_component_under;
use crate::watcher::FsEvent;
@ -90,8 +90,7 @@ fn upsert_path(
return Ok(Applied::Done);
}
// Checked before the followed `metadata` below: with following off the
// walk records no symlinks, and `prepare_file_record_from_path`
// canonicalizes, so a followed link targeting outside every root would
// walk records no symlinks, and `upsert_file` canonicalizes, so a followed link targeting outside every root would
// write a row no sweep range covers and only a rebuild clears.
if !config.indexing.follow_symlinks {
match std::fs::symlink_metadata(path) {
@ -143,7 +142,14 @@ fn upsert_file(
config: &Config,
registry: &Registry,
) -> Result<(), String> {
let Some(rec) = prepare_file_record_from_path(path, config, registry) else {
let Ok(canonical) = path.canonicalize() else {
return Ok(());
};
if warn_if_unrepresentable(&canonical) {
return Ok(());
}
let db_path = path_to_db_string(&canonical);
let Ok(meta) = std::fs::metadata(&canonical) else {
return Ok(());
};
@ -151,45 +157,43 @@ fn upsert_file(
.transaction()
.map_err(|e| format!("begin incremental tx: {}", e))?;
let existing: Option<(i64, i64)> = tx
.query_row(
"SELECT id, mtime FROM files WHERE parent = ?1 AND name = ?2",
rusqlite::params![rec.parent, rec.name],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.optional()
.map_err(|e| format!("lookup {}: {}", rec.path(), e))?;
let file_id = match existing {
Some((_, mtime)) if mtime.max(0) as u64 == rec.mtime => return Ok(()),
Some((id, _)) => {
repo::update_file_basic(&tx, &rec.as_new_file())?;
id
// Settled before the file is opened, as the full walk settles it: an
// unchanged file (a re-sent event, a `chmod`, the live view re-checking a
// row it shows) costs one lookup, never a read and a hash. Measured
// (`benches/index.rs`, `unchanged_event`, 500 small text files): 3.3 ms
// hashing and sniffing each against 1.4 ms.
let stored = repo::mtime_for_path(&tx, &db_path)?;
if stored.is_some() && stored == mtime_secs(&meta) {
return Ok(());
}
None => match repo::insert_file(&tx, &rec.as_new_file())? {
Some(id) => id,
// Lost a race with another writer; the row that won is current enough.
None => return Ok(()),
},
// One file at a time, so it owns the scratch rather than being handed one.
let mut scratch = Scratch::new(config);
let Some(rec) = prepare_file_record(&db_path, &meta, config, registry, &mut scratch) else {
return Ok(());
};
let written = match stored {
Some(_) => repo::update_file_basic(&tx, &rec.as_new_file())?,
None => repo::insert_file(&tx, &rec.as_new_file())?,
};
// `None`: lost a race with another writer; the row that won is current
// enough.
let Some(file_id) = written else {
return Ok(());
};
// The only size gate on this path: `decide_content` has none, and falling
// through would read a multi-gigabyte `.txt` whole into memory.
if !rec.needs_content {
repo::set_content_na(&tx, file_id)?;
} else if let Some(text) = rec.inline_text.as_deref() {
// `prepare_file_record_from_path` already read the whole file.
let zstd = repo::encode_one(text, config.processing.store_text_for_snippets)?;
repo::set_content_done(&tx, file_id, text, zstd.as_deref())?;
} else {
extract_and_store(
&tx,
file_id,
&rec.path(),
rec.mime,
registry,
config,
)?;
// A row that needs no content is already NA: both writers above set its
// state from `rec`. That gate is also the only size gate on this path:
// `decide_content` has none, and falling through would read a
// multi-gigabyte `.txt` whole into memory.
if rec.needs_content {
match rec.inline_text.as_deref() {
// `prepare_file_record` already read the whole file.
Some(text) => {
let zstd = repo::encode_one(text)?;
repo::store_pending_text(&tx, file_id, text, zstd.as_deref())?;
}
None => extract_and_store(&tx, file_id, rec.path(), rec.mime, registry, config)?,
}
}
tx.commit()
@ -236,6 +240,7 @@ pub fn remove_paths(
mod tests {
use super::*;
use crate::db::open_or_recreate;
use rusqlite::OptionalExtension;
use std::time::Duration;
struct Fixture {
@ -253,7 +258,7 @@ mod tests {
let dir = scratch.join("tree");
std::fs::create_dir_all(&dir).unwrap();
let db = scratch.join("index.sqlite");
let conn = open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
let conn = open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap();
let config = Config::default();
let ignore = IgnoreSet::compile(&config.indexing.ignore_patterns).unwrap();
Fixture {
@ -332,13 +337,7 @@ mod tests {
}
fn fts_hits(&self, term: &str) -> i64 {
self.conn
.query_row(
"SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH ?1",
rusqlite::params![format!("\"{}\"", term)],
|r| r.get(0),
)
.unwrap()
crate::testutil::fts_count(&self.conn, term)
}
}
@ -472,6 +471,36 @@ mod tests {
assert_eq!(f.counts(), (1, 1, 1), "still exactly one of everything");
}
/// An unchanged file is settled from its stored mtime, as the full walk
/// settles it, without being opened. Here it cannot be opened, so an
/// attempt would log a hash failure naming it.
#[cfg(unix)]
#[test]
fn an_unchanged_file_is_not_opened() {
use std::os::unix::fs::PermissionsExt;
let mut f = Fixture::new();
let p = f.write("sealed.txt", "sphinx of black quartz");
f.apply(&FsEvent::Create(p.clone()));
let key = f.canonical(&p);
let before = f.row(&key).expect("indexed");
std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o000)).unwrap();
// Root reads it anyway, and then there is nothing to prove.
let readable = std::fs::File::open(&p).is_ok();
f.apply(&FsEvent::Modify(p.clone()));
std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o644)).unwrap();
assert_eq!(f.row(&key), Some(before), "the row is untouched");
if !readable {
assert!(
!crate::log::snapshot()
.iter()
.any(|line| line.text.contains("cannot hash") && line.text.contains(&key)),
"the unchanged file was opened"
);
}
}
#[test]
fn remove_file_cleans_all_tables() {
let mut f = Fixture::new();

18
crates/quicksearch-core/src/indexing/config_check.rs
View file

@ -28,6 +28,11 @@ impl IndexingService {
("hash_algorithm", "size+head".to_string()),
("indexing_path", sorted_joined(roots)),
("tokenize", config.processing.tokenize.clone()),
// The effective layout, not the flag: see `fts_positions`.
(
"fts_positions",
config.processing.fts_positions().to_string(),
),
("include_hidden", config.indexing.include_hidden.to_string()),
(
"follow_symlinks",
@ -41,10 +46,6 @@ impl IndexingService {
"content_extensions",
sorted_joined(&config.indexing.content_extensions),
),
(
"store_text_for_snippets",
config.processing.store_text_for_snippets.to_string(),
),
]
}
@ -61,10 +62,11 @@ impl IndexingService {
}
/// Settings a difference in which cannot be reconciled: the FTS tokenizer
/// is part of the table definition, and hashes under a different length or
/// algorithm cannot be compared. Everything else reconciles; see
/// [`crate::scope`].
const REBUILD_KEYS: [&'static str; 3] = ["hash_length", "hash_algorithm", "tokenize"];
/// and position setting are part of the table definition, and hashes
/// under a different length or algorithm cannot be compared. Everything
/// else reconciles; see [`crate::scope`].
const REBUILD_KEYS: [&'static str; 4] =
["hash_length", "hash_algorithm", "tokenize", "fts_positions"];
/// The unreconcilable settings that no longer match, which a rebuild
/// prompt shows; `None` when none. A key absent (older index) never counts

68
crates/quicksearch-core/src/indexing/pipeline.rs
View file

@ -16,8 +16,8 @@ use crate::extract::Registry;
use crate::file_handling::{
cleanup_stale_index_entries, count_tree_entries_fast, fts_begin_bulk_write,
fts_finalize_after_text_indexing, mark_oversize_pending_na, normalize_root_string,
process_batch_inserts, process_batch_updates, store_extracted, ExtractCursor, ExtractScope,
FileIndexAction, OwnedNewFile, RootRange,
process_batch_inserts, process_batch_updates, store_extracted, Bodies, ExtractCursor,
ExtractScope, FileIndexAction, OwnedNewFile, RootRange,
};
use crate::walk::{thread_count_for, walk_indexable_files, ParallelWalk, TryNext, WalkEvent};
@ -149,12 +149,27 @@ pub(super) struct RootPipeline {
const IDLE_BACKOFF: Duration = Duration::from_millis(2);
/// A batch writer: [`process_batch_updates`] or [`process_batch_inserts`].
type WriteBatch =
fn(&Arc<Mutex<Connection>>, &[OwnedNewFile], &Arc<AtomicBool>, &Config) -> Result<(), String>;
type WriteBatch = fn(
&Arc<Mutex<Connection>>,
&[OwnedNewFile],
&Arc<AtomicBool>,
&Config,
&mut Bodies,
) -> Result<(), String>;
/// Write `rows` through `write`, then clear them; on an error they stay.
fn flush(cx: &RunCx<'_>, rows: &mut Vec<OwnedNewFile>, write: WriteBatch) -> Result<(), String> {
write(&cx.conn_mutex, rows, cx.stop_flag, cx.config)?;
fn flush(
cx: &mut RunCx<'_>,
rows: &mut Vec<OwnedNewFile>,
write: WriteBatch,
) -> Result<(), String> {
write(
&cx.conn_mutex,
rows,
cx.stop_flag,
cx.config,
&mut cx.bodies,
)?;
rows.clear();
Ok(())
}
@ -227,9 +242,7 @@ mod census {
let inline = |rows: &[crate::file_handling::OwnedNewFile]| -> u64 {
rows.iter()
.map(|r| {
(r.inline_text.as_ref().map_or(0, String::len)
+ r.name.len()
+ r.parent.len()) as u64
(r.inline_text.as_ref().map_or(0, String::len) + r.path().len()) as u64
})
.sum()
};
@ -500,7 +513,20 @@ impl RootPipeline {
if ready.is_empty() {
break;
}
let stored = store_extracted(&cx.conn_mutex, ready, cx.stop_flag, cx.config, deadline)?;
// Rowid order: FTS5 flushes its pending terms to a new segment
// whenever a write's rowid falls below the previous one, and the
// pool finishes rows in completion order, not id order. Measured
// (`indexprobe cold`, 10k docs of 9–20 KiB): 5% less CPU, 9% less
// WAL written; docs past FTS5's 1 MiB hash flush anyway, so a
// corpus of 8–200 KiB docs gained only the WAL figure (7%).
ready.sort_unstable_by_key(|row| row.file_id);
let stored = store_extracted(
&cx.conn_mutex,
ready,
cx.stop_flag,
deadline,
&mut cx.bodies,
)?;
if stored.consumed > 0 {
// The last row *written*, not the last fetched. The whole path,
// as the walk phase publishes: the hint must not change what it
@ -548,6 +574,8 @@ pub(super) struct RunCx<'a> {
/// Writer time one root's turn may take before the round moves on
/// ([`crate::config::ProcessingConfig::writer_turn_slice_ms`]).
pub(super) slice: Duration,
/// The run's one set of compression buffers, shared by every write.
pub(super) bodies: Bodies,
/// Rows with no file behind them, per-directory plus the vanished sweep.
pub(super) stale_candidates: Vec<String>,
/// Paths reached via symlink; their rows may live outside every root.
@ -562,8 +590,8 @@ impl<'a> RunCx<'a> {
db_path: &'a str,
stop_flag: &'a Arc<AtomicBool>,
status: Arc<Mutex<IndexingStatus>>,
) -> RunCx<'a> {
RunCx {
) -> Result<RunCx<'a>, String> {
Ok(RunCx {
conn_mutex,
config,
db_path,
@ -572,10 +600,11 @@ impl<'a> RunCx<'a> {
registry: Arc::new(Registry::default_set()),
batch: config.processing.batch_size.max(1),
slice: Duration::from_millis(config.processing.writer_turn_slice_ms),
bodies: Bodies::new()?,
stale_candidates: Vec::new(),
aliased_paths: HashSet::new(),
stale_cleanup_ok: true,
}
})
}
/// Publish `step` on the running status until the returned guard drops, so
@ -656,7 +685,14 @@ fn build_pipeline(
workers,
);
// A walked root needs no scan: its stored count is exact.
// A walked root needs no scan: its stored count is exact. A never-walked
// one gets a second, count-only traversal beside the walk, kept for the
// first run's progress figure. Measured (`indexprobe cold`, 50k files in
// 5k directories, alternating runs): on a virtiofs share it cost ~4% of
// wall time (2.29 s against 2.20 s without it) and ~7% of CPU, and no
// prefetch benefit to the walk showed; on warm local disk the difference
// was within noise. Its figure counts directories and ignored entries
// too, so it runs high.
let count_total = Arc::new(AtomicUsize::new(0));
match stored_count {
// `max(1)`: 0 is the "unknown" sentinel; an empty root must not store it.
@ -857,7 +893,7 @@ impl IndexingService {
) -> Result<Option<Connection>, String> {
// A large WAL recovery happens here; announced before attempted.
Self::set_prep_step(status, PrepStep::OpeningIndex);
let mut conn = db::open_or_recreate(db_path, &config.processing.tokenize)?;
let mut conn = db::open_or_recreate(db_path, &config.processing)?;
// Reconcile against the settings the index was last written under,
// *before* stamping the new ones: the old record is the only thing
@ -944,7 +980,7 @@ impl IndexingService {
let count_cancel = Arc::new(AtomicBool::new(false));
let _count_guard = CancelOnDrop(count_cancel.clone());
let mut cx = RunCx::new(conn_mutex, config, db_path, stop_flag, status.clone());
let mut cx = RunCx::new(conn_mutex, config, db_path, stop_flag, status.clone())?;
let stored_counts: Vec<Option<usize>> = {
let conn = crate::lock_ok(&cx.conn_mutex);

8
crates/quicksearch-core/src/indexing/tests.rs
View file

@ -124,7 +124,7 @@ fn worker_counts_follow_the_phase() {
let dir = tmp_dir("phase-workers");
std::fs::write(dir.join("a.txt"), "hello").unwrap();
let db_path = dir.join("index.db").to_string_lossy().into_owned();
drop(db::open_or_recreate(&db_path, "trigram").unwrap());
drop(db::open_or_recreate(&db_path, &Default::default()).unwrap());
let stop = Arc::new(AtomicBool::new(false));
let mut p = idle_pipeline(&dir, &db_path, 3, 2, &stop);
@ -150,7 +150,7 @@ fn an_extracting_turn_lands_its_leftovers_one_slice_at_a_time() {
let dir = tmp_dir("slice-leftovers");
let db_path = dir.join("index.db").to_string_lossy().into_owned();
let mut conn = db::open_or_recreate(&db_path, "trigram").unwrap();
let mut conn = db::open_or_recreate(&db_path, &Default::default()).unwrap();
let tree = dir.join("tree");
std::fs::create_dir_all(&tree).unwrap();
let mut ready: Vec<ExtractedRow> = Vec::new();
@ -204,7 +204,8 @@ fn an_extracting_turn_lands_its_leftovers_one_slice_at_a_time() {
&db_path,
&stop,
Arc::new(Mutex::new(IndexingStatus::Idle)),
);
)
.unwrap();
cx.slice = Duration::ZERO;
let mut turns = 0;
@ -488,6 +489,7 @@ fn cx_for<'a>(
stop,
status.clone(),
)
.unwrap()
}
fn running_status(maintenance: Option<MaintenanceStep>) -> Arc<Mutex<IndexingStatus>> {

25
crates/quicksearch-core/src/query/pattern.rs
View file

@ -263,22 +263,19 @@ impl TermPattern {
pub fn count_folded(&self, folded: &str) -> usize {
match self {
TermPattern::Empty => 0,
TermPattern::Literal(l) => snippet::count_occurrences(folded, &l.folded, true),
TermPattern::Literal(l) => snippet::count_occurrences(folded, &l.folded),
TermPattern::Wildcard(w) => w.search_ci.find_iter(folded).take(COUNT_CAP).count(),
}
}
/// Non-overlapping occurrence count. Wildcards stop at 1000, where the
/// Case-sensitive, non-overlapping occurrence count; the case-insensitive
/// one is [`TermPattern::count_folded`]. Wildcards stop at 1000, where the
/// cascade's `count_frac` saturates anyway.
pub fn count(&self, text: &str, case_insensitive: bool) -> usize {
pub fn count(&self, text: &str) -> usize {
match self {
TermPattern::Empty => 0,
TermPattern::Literal(l) => snippet::count_occurrences(text, &l.text, !case_insensitive),
TermPattern::Wildcard(w) => w
.search(case_insensitive)
.find_iter(text)
.take(COUNT_CAP)
.count(),
TermPattern::Literal(l) => snippet::count_occurrences(text, &l.text),
TermPattern::Wildcard(w) => w.search_cs.find_iter(text).take(COUNT_CAP).count(),
}
}
}
@ -428,7 +425,7 @@ mod tests {
assert!(p.is_empty(), "{:?}", parts);
assert!(!p.whole_match("anything", true));
assert!(p.find_first("anything", true).is_none());
assert_eq!(p.count("anything", true), 0);
assert_eq!(p.count("anything"), 0);
}
}
@ -516,9 +513,9 @@ mod tests {
#[test]
fn count_is_nonoverlapping_and_capped() {
let p = TermPattern::build(&[part("a*b", true)]).unwrap();
assert_eq!(p.count("ab ab ab", false), 3);
assert_eq!(p.count("ab ab ab"), 3);
let many = "ab ".repeat(2000);
assert_eq!(p.count(&many, false), 1000);
assert_eq!(p.count(&many), 1000);
}
#[test]
@ -530,8 +527,8 @@ mod tests {
assert_eq!(p.find_first("my Report.pdf", false), Some(3..9));
assert_eq!(p.find_first("my report.pdf", true), Some(3..9));
assert_eq!(p.find_first("my report.pdf", false), None);
assert_eq!(p.count("report Report", false), 1);
assert_eq!(p.count("report Report", true), 2);
assert_eq!(p.count("report Report"), 1);
assert_eq!(p.count_folded("report report"), 2);
}
#[test]

25
crates/quicksearch-core/src/scope.rs
View file

@ -418,8 +418,6 @@ struct PagePlan {
/// posting (see [`repo::delete_ids`]).
doomed: Vec<i64>,
doomed_with_postings: Vec<i64>,
/// Rows keeping their posting but losing the snippet source.
stale_text: Vec<i64>,
/// Surviving rows changing `content_state`, and across both, those whose
/// stored state holds a posting or a failure record to clear first.
to_pending: Vec<i64>,
@ -433,7 +431,6 @@ impl PagePlan {
for list in [
&mut self.doomed,
&mut self.doomed_with_postings,
&mut self.stale_text,
&mut self.to_pending,
&mut self.to_na,
&mut self.restated_with_postings,
@ -460,7 +457,6 @@ impl PagePlan {
/// Returns `(deleted, recontented)`.
fn write(&self, tx: &Transaction<'_>) -> Result<(usize, usize), String> {
let deleted = repo::delete_ids(tx, &self.doomed, &self.doomed_with_postings)?;
repo::drop_stored_text(tx, &self.stale_text)?;
repo::clear_content_for_ids(tx, &self.restated_with_postings)?;
repo::clear_failed_for_ids(tx, &self.restated_with_failures)?;
repo::set_content_state(tx, &self.to_pending, repo::STATE_PENDING)?;
@ -493,21 +489,14 @@ fn apply_page(
}
continue;
}
// Only a DONE row can have a `documents_text` body to drop.
if work.drop_text && row.content_state == repo::STATE_DONE {
plan.stale_text.push(row.id);
}
if work.reconcile_content || work.restore_text {
// The walker's decision, recomputed. Both directions run whenever
// either flag is set: a disagreeing row is wrong however it got that way.
if work.reconcile_content {
// The walker's decision, recomputed. Both directions: a
// disagreeing row is wrong however it got that way.
let wants = row.size <= config.processing.maximum_text_file_size
&& content_extractable(path, row.mime.as_deref(), config, registry);
if !wants && row.content_state != repo::STATE_NA {
plan.restate(row, false);
} else if wants
&& (row.content_state == repo::STATE_NA
|| (work.restore_text && row.content_state == repo::STATE_DONE))
{
} else if wants && row.content_state == repo::STATE_NA {
plan.restate(row, true);
}
}
@ -535,9 +524,9 @@ pub fn stored_config(conn: &Connection, config: &Config) -> Result<Config, Strin
"content_extensions" => stored.indexing.content_extensions = lines(&value),
"include_hidden" => stored.indexing.include_hidden = value == "true",
"follow_symlinks" => stored.indexing.follow_symlinks = value == "true",
"store_text_for_snippets" => {
stored.processing.store_text_for_snippets = value == "true"
}
// Recorded as the effective layout; a flag that reproduces it
// under the recorded tokenizer (see `fts_positions`).
"fts_positions" => stored.processing.compact_text_index = value == "false",
"hash_length" => match value.parse() {
Ok(n) => stored.processing.hash_length = n,
Err(e) => crate::log_warn!("stored hash_length {:?} unreadable: {}", value, e),

2
crates/quicksearch-core/src/scope_tests.rs
View file

@ -8,7 +8,7 @@ use std::sync::Arc;
fn empty_db(dir: &Path) -> PathBuf {
let db = dir.join("index.sqlite");
crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
crate::db::open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap();
db
}

16
crates/quicksearch-core/src/search/cascade.rs
View file

@ -34,7 +34,7 @@ use rusqlite::{Connection, OptionalExtension};
use crate::config::IgnoreSet;
use crate::query::pattern::clamp_match_range;
use crate::query::split::CascadeQuery;
use crate::query::translator::{escape_like, quote_phrase};
use crate::query::translator::escape_like;
use crate::snippet;
use super::fuzzy::{matcher_for, pigeonhole_chunks, Bitap};
@ -80,12 +80,7 @@ pub fn text_snippet_counted(
folded: &str,
) -> Option<(snippet::Snippet, usize)> {
let term = pattern.literal_folded()?;
Some(snippet::extract_folded(
text,
folded,
&[term],
&SNIPPET_OPTS,
))
Some(snippet::extract_folded(text, folded, term, &SNIPPET_OPTS))
}
/// The fuzzy full-text match in one document body: occurrence count within
@ -128,6 +123,7 @@ pub fn run(
conn,
query,
options,
positions: crate::db::repo::fts_positions(conn),
generation,
latest_gen,
ignore,
@ -310,6 +306,10 @@ struct Cx<'a> {
conn: &'a Connection,
query: &'a CascadeQuery,
options: &'a SearchOptions,
/// Whether the full-text index keeps positions, read from the index each
/// search: it decides how every `MATCH` is phrased
/// ([`prefilter::fts_literal`]).
positions: bool,
generation: u64,
latest_gen: &'a AtomicU64,
ignore: IgnoreSet,
@ -417,7 +417,7 @@ impl<'a> Cx<'a> {
stmt.query_row([file_id], |r| {
let blob = r.get_ref(0)?.as_blob()?;
// Strict UTF-8 loses nothing: bodies are written from `&str`
// (`repo::set_content_done`), and `stored_text` also treats
// (`repo::store_pending_text`), and `stored_text` also treats
// non-UTF-8 as absent.
Ok(doc.decode(blob).is_some_and(|t| re.is_match(t)))
})

95
crates/quicksearch-core/src/search/cascade/passes.rs
View file

@ -70,19 +70,17 @@ fn doc_scan_sql(filter_sql: &str) -> String {
)
}
/// The scan behind a `searchabletext MATCH` prefilter. `require_text` picks
/// the join to `documents_text`: `false` is [`Cx::pass_fulltext`]'s LEFT
/// JOIN, which keeps an FTS-matched row whose stored text was dropped.
fn fts_match_sql(filter_sql: &str, require_text: bool) -> String {
/// The scan behind a `searchabletext MATCH` prefilter. Every candidate is
/// verified against its stored text, and a row with a posting always has
/// some: text is stored whenever there is any.
fn fts_match_sql(filter_sql: &str) -> String {
format!(
"SELECT {}, dt.text_zstd \
FROM searchabletext \
JOIN files f ON f.id = searchabletext.rowid \
{} documents_text dt ON dt.file_id = f.id \
JOIN documents_text dt ON dt.file_id = f.id \
WHERE searchabletext MATCH ?{}",
HIT_COLUMNS,
if require_text { "JOIN" } else { "LEFT JOIN" },
filter_sql
HIT_COLUMNS, filter_sql
)
}
@ -109,12 +107,11 @@ fn anchor_segment(pattern: &crate::query::pattern::TermPattern) -> Option<&str>
impl<'a> Cx<'a> {
/// The SQL and parameters for a scan over document bodies: an FTS MATCH on
/// `expr` when there is one (`require_text` as in [`fts_match_sql`]),
/// else every stored document.
fn body_scan(&self, expr: Option<String>, require_text: bool) -> (String, Vec<Value>) {
/// `expr` when there is one, else every stored document.
fn body_scan(&self, expr: Option<String>) -> (String, Vec<Value>) {
match expr {
Some(expr) => (
fts_match_sql(&self.query.filter_sql, require_text),
fts_match_sql(&self.query.filter_sql),
self.params_with_filters(vec![Value::Text(expr)]),
),
None => (
@ -309,52 +306,46 @@ impl<'a> Cx<'a> {
// Below the trigram floor the MATCH can't return anything.
return Ok(true);
}
// The column filter goes inside the MATCH (`text: "..."`) so only
// bodies match. A literal term is one quoted phrase; a wildcard
// narrows with an AND of its segments of 3+ characters; with none
// (`ab*cd`) the pass scans every stored document. Every row is
// pattern-verified either way.
let match_expr = match pattern.literal() {
Some(term) => Some(format!("text: {}", quote_phrase(term))),
None => Some(
// A literal term is one segment; a wildcard narrows with an AND of
// its segments of 3+ characters, each asked for as the index allows
// ([`prefilter::fts_literal`]); with none (`ab*cd`) the pass scans
// every stored document. Every row is pattern-verified either way,
// which is what makes an index without positions exact.
let match_expr = Some(
pattern
.segments()
.iter()
.filter(|s| s.chars().count() >= prefilter::TRIGRAM_FLOOR)
.map(|s| format!("text: {}", quote_phrase(s)))
.filter_map(|s| prefilter::fts_literal(s, self.positions))
.collect::<Vec<_>>()
.join(" AND "),
)
.filter(|expr| !expr.is_empty()),
};
let narrowed = match_expr.is_some();
let (sql, params) = self.body_scan(match_expr, false);
.filter(|expr| !expr.is_empty());
let (sql, params) = self.body_scan(match_expr);
let mut doc = crate::db::repo::DocDecoder::new()?;
let mut lower = String::new();
// Decompression dominates: check cancellation every row.
self.scan_pass(&sql, params, 1, None, |cx, row, file_id, path, name| {
let text = stored_text(row, &mut doc)?;
let (rank, stage, snip) = match text {
Some(text) => {
// Fold once: count, first match and snippet all need it,
// even on rows about to be dropped.
// A body that will not decode cannot be verified.
let Some(text) = stored_text(row, &mut doc)? else {
return Ok(RowHit::Skip);
};
// Fold once: count, first match and snippet all need it, even on
// rows about to be dropped.
fold_into(&mut lower, text);
match super::text_snippet_counted(pattern, text, &lower) {
let (count, stage, snip) = match super::text_snippet_counted(pattern, text, &lower) {
Some((snip, count_ci)) => {
if count_ci == 0 {
return Ok(RowHit::Skip);
}
let count_cs = pattern.count(text, false);
let (count, stage) = if count_cs > 0 {
(count_cs, 5)
let count_cs = pattern.count(text);
if count_cs > 0 {
(count_cs, 5, Some(snip))
} else {
(count_ci, 6)
};
(stage as f64 + count_frac(count), stage as u8, Some(snip))
(count_ci, 6, Some(snip))
}
}
None => {
let count_cs = pattern.count(text, false);
let count_cs = pattern.count(text);
let (count, stage) = if count_cs > 0 {
(count_cs, 5)
} else {
@ -364,22 +355,11 @@ impl<'a> Cx<'a> {
}
(count_ci, 6)
};
let snip = super::text_snippet(pattern, text, &lower);
(stage as f64 + count_frac(count), stage as u8, snip)
}
}
}
// No stored text: FTS-narrowed rows are accepted at the bottom
// of rank 6 as count-unknown; on the full-scan fallback an
// unverifiable row is skipped.
None => {
if !narrowed {
return Ok(RowHit::Skip);
}
(6.0 + count_frac(1), 6, None)
(count, stage, super::text_snippet(pattern, text, &lower))
}
};
if !cx.regex_accepts(file_id, path, text)? {
let rank = stage as f64 + count_frac(count);
if !cx.regex_accepts(file_id, path, Some(text))? {
return Ok(RowHit::Skip);
}
@ -463,8 +443,8 @@ impl<'a> Cx<'a> {
.and_then(|chunks| {
prefilter::Required::new(chunks.iter().map(|c| c.to_string()).collect())
})
.and_then(|required| required.fts_expr());
let (sql, params) = self.body_scan(expr, true);
.and_then(|required| required.fts_expr(self.positions));
let (sql, params) = self.body_scan(expr);
// No fold buffer: the matcher folds in its mask table.
let mut doc = crate::db::repo::DocDecoder::new()?;
// Decompression dominates: check cancellation every row.
@ -541,7 +521,8 @@ impl<'a> Cx<'a> {
pub(super) fn pass_regex_content(&mut self) -> Result<bool, String> {
let query = self.query;
let re = query.regex.as_ref().expect("regex-only pass list");
let (sql, params) = self.body_scan(re.required().and_then(|r| r.fts_expr()), true);
let expr = re.required().and_then(|r| r.fts_expr(self.positions));
let (sql, params) = self.body_scan(expr);
let mut doc = crate::db::repo::DocDecoder::new()?;
// Decompression dominates: check cancellation every row.
self.scan_pass(&sql, params, 1, None, |_cx, row, file_id, path, name| {

2
crates/quicksearch-core/src/search/duplicates.rs
View file

@ -219,7 +219,7 @@ mod tests {
fn seed(dir: &str, rows: &[(&str, u64, Option<&[u8]>)]) -> (Scratch, String) {
let (scratch, p) = Scratch::db(dir);
let p = p.to_str().unwrap().to_string();
let mut conn = open_or_recreate(&p, "trigram").unwrap();
let mut conn = open_or_recreate(&p, &Default::default()).unwrap();
let tx = conn.transaction().unwrap();
for &(name, size, hash) in rows {
insert_file(

2
crates/quicksearch-core/src/search/mod.rs
View file

@ -55,7 +55,7 @@ pub struct SearchHit {
pub rank: f64,
pub stage: u8,
/// The matched span in context: name, path, or a window of the body per
/// stage (absent for full-text stages when no text is stored).
/// stage; absent only where no window could be cut.
pub snippet: Option<Snippet>,
}

89
crates/quicksearch-core/src/search/prefilter.rs
View file

@ -24,6 +24,35 @@ pub const TRIGRAM_FLOOR: usize = 3;
/// cheaper than the scan it replaces, and falling back is always correct.
const MAX_LITERALS: usize = 32;
/// A `searchabletext MATCH` expression every document containing `literal`
/// satisfies, for an index built with or without positions (see
/// [`crate::db::repo::fts_positions`]); `None` below [`TRIGRAM_FLOOR`]
/// characters, where the tokenizer indexes nothing to find.
///
/// With positions it is the literal as one phrase, exact up to the index's
/// case folding. Without them a phrase of several tokens is refused outright,
/// so it is the AND of the literal's distinct trigrams, each a one-token
/// phrase: a superset, since adjacency goes unchecked, which the full-text
/// pass's verification narrows back. Either way [`quote_phrase`] renders
/// FTS5 syntax in user text inert.
pub fn fts_literal(literal: &str, positions: bool) -> Option<String> {
let chars: Vec<char> = literal.chars().collect();
if chars.len() < TRIGRAM_FLOOR {
return None;
}
if positions {
return Some(format!("text: {}", quote_phrase(literal)));
}
let mut seen = std::collections::HashSet::new();
let trigrams: Vec<String> = chars
.windows(TRIGRAM_FLOOR)
.map(|w| w.iter().collect::<String>())
.filter(|t| seen.insert(t.clone()))
.map(|t| quote_phrase(&t))
.collect();
Some(format!("({})", trigrams.join(" AND ")))
}
/// Literals of which at least one occurs in anything the query can match.
#[derive(Debug, Clone)]
pub struct Required(Vec<String>);
@ -46,25 +75,20 @@ impl Required {
&self.0
}
/// A `searchabletext MATCH` expression: `(text: "a" OR text: "b" …)`.
/// A `searchabletext MATCH` expression: the OR of each literal's
/// [`fts_literal`].
///
/// `None` when any literal is shorter than [`TRIGRAM_FLOOR`] characters:
/// the tokenizer indexes character triples, so a byte-length test would
/// admit a phrase that matches no token. [`quote_phrase`] renders FTS5
/// syntax in user text inert. The index folds case and diacritics, so it
/// admit a phrase that matches no token. The index folds case, so it
/// matches more than the literal as written, the harmless direction.
pub fn fts_expr(&self) -> Option<String> {
if self.0.iter().any(|l| l.chars().count() < TRIGRAM_FLOOR) {
return None;
}
Some(format!(
"({})",
self.0
pub fn fts_expr(&self, positions: bool) -> Option<String> {
let each = self
.0
.iter()
.map(|l| format!("text: {}", quote_phrase(l)))
.collect::<Vec<_>>()
.join(" OR ")
))
.map(|l| fts_literal(l, positions))
.collect::<Option<Vec<_>>>()?;
Some(format!("({})", each.join(" OR ")))
}
/// A predicate over the `files` columns, plus the values it binds:
@ -114,23 +138,50 @@ mod tests {
#[test]
fn the_trigram_floor_counts_characters_not_bytes() {
assert!(req(&["café"]).unwrap().fts_expr().is_some());
assert!(req(&["日本"]).unwrap().fts_expr().is_none());
assert!(req(&["ab"]).unwrap().fts_expr().is_none());
for positions in [true, false] {
assert!(req(&["café"]).unwrap().fts_expr(positions).is_some());
assert!(req(&["日本"]).unwrap().fts_expr(positions).is_none());
assert!(req(&["ab"]).unwrap().fts_expr(positions).is_none());
assert!(
req(&["abc", "de"]).unwrap().fts_expr().is_none(),
req(&["abc", "de"]).unwrap().fts_expr(positions).is_none(),
"all of them"
);
}
}
#[test]
fn fts5_syntax_in_a_literal_is_quoted_inert() {
let expr = req(&["a\"b", "NEAR", "x*y"]).unwrap().fts_expr().unwrap();
let expr = req(&["a\"b", "NEAR", "x*y"])
.unwrap()
.fts_expr(true)
.unwrap();
// The embedded quote is doubled, which is FTS5's own escape.
assert!(expr.contains(r#""a""b""#), "{expr}");
assert!(expr.contains(r#""NEAR""#), "{expr}");
assert!(expr.contains(r#""x*y""#), "{expr}");
assert_eq!(expr.matches(" OR ").count(), 2);
// Each trigram is its own quoted phrase, so a quote inside one is
// escaped the same way.
let expr = req(&["a\"bc"]).unwrap().fts_expr(false).unwrap();
assert!(expr.contains(r#""a""b""#), "{expr}");
assert!(expr.contains(r#""""bc""#), "{expr}");
}
/// Without positions a literal is the AND of its distinct trigrams, the
/// literals still OR'ed: the only shape a `detail=none` table accepts.
#[test]
fn without_positions_a_literal_is_its_trigrams() {
assert_eq!(fts_literal("abcd", false).unwrap(), r#"("abc" AND "bcd")"#);
assert_eq!(
fts_literal("aaaa", false).unwrap(),
r#"("aaa")"#,
"a repeated trigram is asked for once"
);
assert_eq!(fts_literal("café", false).unwrap(), r#"("caf" AND "afé")"#);
assert_eq!(fts_literal("abcd", true).unwrap(), r#"text: "abcd""#);
let expr = req(&["abc", "xyz"]).unwrap().fts_expr(false).unwrap();
assert_eq!(expr, r#"(("abc") OR ("xyz"))"#);
}
#[test]

181
crates/quicksearch-core/src/snippet.rs
View file

@ -1,7 +1,7 @@
//! Snippet extraction for search results; contentless FTS5 has no
//! `snippet()`. Output is structural (window text plus byte ranges), never
//! markup. Matching folds ASCII only: `cafe` finds `café` via the tokenizer,
//! but the snippet does not mark the accented occurrence.
//! markup. Matching folds ASCII case only, as full-text verification does:
//! `cafe` neither finds nor marks `café`.
#[derive(Debug, Clone)]
pub struct Options {
@ -44,48 +44,34 @@ impl Snippet {
/// must be `text.to_ascii_lowercase()`: the fold preserves byte length, so
/// offsets found in it slice the original.
///
/// Returns the window and how many occurrences it found before touching
/// ranges were coalesced, the count the full-text pass ranks on. Blank terms
/// are dropped, but a padded term is searched as given, never trimmed.
pub fn extract_folded(
text: &str,
folded: &str,
terms: &[&str],
opts: &Options,
) -> (Snippet, usize) {
/// Returns the window and how many occurrences of `term` it found before
/// touching ranges were coalesced, the count the full-text pass ranks on. A
/// blank term matches nothing, but a padded term is searched as given, never
/// trimmed.
pub fn extract_folded(text: &str, folded: &str, term: &str, opts: &Options) -> (Snippet, usize) {
debug_assert_eq!(folded.len(), text.len(), "ASCII folding preserves length");
if text.is_empty() {
return (Snippet::empty(), 0);
}
let effective_terms: Vec<&str> = terms
.iter()
.copied()
.filter(|t| !t.trim().is_empty())
.collect();
if effective_terms.is_empty() {
if term.trim().is_empty() {
return (head_window(text, opts.approx_bytes), 0);
}
// `memmem`, not `str::match_indices`: measured faster on whole-body
// scans. The walk runs to the end (the count must see every occurrence),
// but matches past the window's right edge are not stored. Only a single
// term is bounded: with several, two terms can coalesce across the edge,
// which no single-pass bound sees.
let bounded = effective_terms.len() == 1;
let mut matches: Vec<(usize, usize)> = Vec::new();
let mut found = 0usize;
// The window's right edge once the first match fixes it, from the same
// [`window_bounds`] the window itself uses.
let mut keep_below: Option<usize> = None;
for term in &effective_terms {
// Borrow when the term is already folded, as every cascade call's is,
// to avoid an allocation per candidate row.
let pattern: std::borrow::Cow<'_, str> = if term.bytes().any(|b| b.is_ascii_uppercase()) {
std::borrow::Cow::Owned(term.to_ascii_lowercase())
} else {
std::borrow::Cow::Borrowed(*term)
std::borrow::Cow::Borrowed(term)
};
// `memmem`, not `str::match_indices`: measured faster on whole-body
// scans. The walk runs to the end (the count must see every occurrence),
// but matches past the window's right edge are not stored.
let mut matches: Vec<(usize, usize)> = Vec::new();
let mut found = 0usize;
// The window's right edge once the first match fixes it, from the same
// [`window_bounds`] the window itself uses.
let mut keep_below: Option<usize> = None;
// The end of the last kept match, so a chain of touching occurrences
// (`abab…` for `ab`) that starts inside the window and continues past
// it stays intact for `coalesce_overlapping` and the expansion step.
@ -94,25 +80,22 @@ pub fn extract_folded(
// Before any dropping: two abutting occurrences are two hits for
// ranking even though they paint as one highlight.
found += 1;
if bounded {
let bound =
*keep_below.get_or_insert_with(|| window_bounds(text, at, opts.approx_bytes).1);
let bound = *keep_below.get_or_insert_with(|| window_bounds(text, at, opts.approx_bytes).1);
if at >= bound && at > chain_end {
// Past the edge and not chained to anything kept: keep
// counting, stop storing.
// Past the edge and not chained to anything kept: keep counting,
// stop storing.
continue;
}
}
chain_end = at + pattern.len();
matches.push((at, chain_end));
}
}
if matches.is_empty() {
return (head_window(text, opts.approx_bytes), 0);
}
matches.sort_by_key(|(a, _)| *a);
// One term's occurrences arrive in order and never overlap; only
// abutting ones merge.
let matches = coalesce_overlapping(matches);
let (win_start, mut win_end) = window_bounds(text, matches[0].0, opts.approx_bytes);
@ -213,21 +196,13 @@ pub fn window_around(text: &str, range: (usize, usize), opts: &Options) -> Snipp
}
}
/// Count non-overlapping occurrences of `term` in `text`. Case-insensitive
/// counting folds ASCII only, matching the rest of the search pipeline.
pub fn count_occurrences(text: &str, term: &str, case_sensitive: bool) -> usize {
/// Count non-overlapping occurrences of `term` in `text`, byte for byte: a
/// case-insensitive count passes both already folded.
pub fn count_occurrences(text: &str, term: &str) -> usize {
if term.is_empty() || text.len() < term.len() {
return 0;
}
if case_sensitive {
memchr::memmem::find_iter(text.as_bytes(), term.as_bytes()).count()
} else {
memchr::memmem::find_iter(
text.to_ascii_lowercase().as_bytes(),
term.to_ascii_lowercase().as_bytes(),
)
.count()
}
}
/// The first `n` bytes of `text` (char-aligned) as a match-less window.
@ -264,12 +239,12 @@ fn coalesce_overlapping(v: Vec<(usize, usize)>) -> Vec<(usize, usize)> {
mod tests {
use super::*;
fn extract(text: &str, terms: &[&str], opts: &Options) -> Snippet {
extract_folded(text, &text.to_ascii_lowercase(), terms, opts).0
fn extract(text: &str, term: &str, opts: &Options) -> Snippet {
extract_folded(text, &text.to_ascii_lowercase(), term, opts).0
}
fn extract_counted(text: &str, terms: &[&str], opts: &Options) -> (Snippet, usize) {
extract_folded(text, &text.to_ascii_lowercase(), terms, opts)
fn extract_counted(text: &str, term: &str, opts: &Options) -> (Snippet, usize) {
extract_folded(text, &text.to_ascii_lowercase(), term, opts)
}
fn opts_small() -> Options {
@ -280,11 +255,11 @@ mod tests {
#[test]
fn extract_folded_counts_occurrences_not_ranges() {
let text = "abab and ab";
let (s, n) = extract_counted(text, &["ab"], &opts_small());
let (s, n) = extract_counted(text, "ab", &opts_small());
assert_eq!(n, 3, "three occurrences");
assert_eq!(
n,
count_occurrences(text, "ab", true),
count_occurrences(text, "ab"),
"must agree with count_occurrences"
);
assert_eq!(s.ranges.len(), 2, "the abutting pair paints as one range");
@ -304,10 +279,10 @@ mod tests {
];
for (text, term) in cases {
let folded = text.to_ascii_lowercase();
let (_, n) = extract_folded(text, &folded, &[term], &opts_small());
let (_, n) = extract_folded(text, &folded, term, &opts_small());
assert_eq!(
n,
count_occurrences(&folded, &term.to_ascii_lowercase(), true),
count_occurrences(&folded, &term.to_ascii_lowercase()),
"count mismatch for {:?} in {:?}",
term,
text
@ -321,8 +296,8 @@ mod tests {
fn extract_folded_needle_case_does_not_change_matches() {
let text = "The Needle and the needle";
let folded = text.to_ascii_lowercase();
let (upper, n_upper) = extract_folded(text, &folded, &["Needle"], &opts_small());
let (lower, n_lower) = extract_folded(text, &folded, &["needle"], &opts_small());
let (upper, n_upper) = extract_folded(text, &folded, "Needle", &opts_small());
let (lower, n_lower) = extract_folded(text, &folded, "needle", &opts_small());
assert_eq!(n_upper, 2);
assert_eq!(n_upper, n_lower);
assert_eq!(upper.ranges, lower.ranges);
@ -335,7 +310,7 @@ mod tests {
let text = "some text";
let folded = text.to_ascii_lowercase();
for term in ["", " ", "\t"] {
let (s, n) = extract_folded(text, &folded, &[term], &opts_small());
let (s, n) = extract_folded(text, &folded, term, &opts_small());
assert_eq!(n, 0, "blank term {:?} counted", term);
assert!(s.ranges.is_empty(), "blank term {:?} highlighted", term);
}
@ -346,9 +321,9 @@ mod tests {
fn extract_folded_does_not_trim_a_padded_term() {
let text = "needle needlework";
let folded = text.to_ascii_lowercase();
let (_, n) = extract_folded(text, &folded, &["needle "], &opts_small());
let (_, n) = extract_folded(text, &folded, "needle ", &opts_small());
assert_eq!(n, 1, "only the occurrence followed by a space");
assert_eq!(n, count_occurrences(&folded, "needle ", true));
assert_eq!(n, count_occurrences(&folded, "needle "));
}
/// The `Snippet::ranges` contract egui's LayoutJob sections rely on.
@ -374,7 +349,7 @@ mod tests {
fn occurrences_past_the_window_are_counted_but_not_kept() {
let unit = "needle filler filler ";
let text = unit.repeat(400);
let (snip, found) = extract_counted(&text, &["needle"], &opts_small());
let (snip, found) = extract_counted(&text, "needle", &opts_small());
assert_eq!(found, 400, "every occurrence must still be counted");
assert_ranges_valid(&snip);
@ -403,7 +378,7 @@ mod tests {
#[test]
fn a_coalescing_chain_is_not_cut_at_the_window_edge() {
let text = "ab".repeat(400);
let (snip, found) = extract_counted(&text, &["ab"], &opts_small());
let (snip, found) = extract_counted(&text, "ab", &opts_small());
assert_eq!(found, 400, "non-overlapping occurrences, all counted");
assert_ranges_valid(&snip);
@ -428,31 +403,16 @@ mod tests {
);
}
/// Several terms disable the bound; pin that the multi-term path still
/// produces the full, correct answer.
#[test]
fn several_terms_still_coalesce_across_the_edge() {
let text = format!("{}{}", "x".repeat(10), "abc".repeat(200));
let (snip, found) = extract_counted(&text, &["ab", "bc"], &opts_small());
assert_eq!(found, 400, "200 of each term");
assert_ranges_valid(&snip);
assert_eq!(
snip.ranges.len(),
1,
"the two terms interleave into one run"
);
}
#[test]
fn empty_text_returns_empty() {
let s = extract("", &["foo"], &Options::default());
let s = extract("", "foo", &Options::default());
assert_eq!(s, Snippet::empty());
}
#[test]
fn no_terms_returns_head_marked_truncated() {
fn a_blank_term_returns_head_marked_truncated() {
let long = "abcdefghijklmnop".repeat(10);
let s = extract(&long, &[], &opts_small());
let s = extract(&long, "", &opts_small());
assert!(s.truncated_end);
assert!(!s.truncated_start);
assert!(s.ranges.is_empty());
@ -460,40 +420,29 @@ mod tests {
}
#[test]
fn no_terms_untruncated() {
let s = extract("short text", &[], &opts_small());
fn a_blank_term_untruncated() {
let s = extract("short text", "", &opts_small());
assert_eq!(s.window, "short text");
assert!(!s.truncated_end && !s.truncated_start);
}
#[test]
fn simple_match_range() {
let s = extract("the quick brown fox", &["quick"], &opts_small());
let s = extract("the quick brown fox", "quick", &opts_small());
assert_eq!(marked(&s), vec!["quick"]);
assert_ranges_valid(&s);
}
#[test]
fn case_insensitive_match_reports_original_case() {
let s = extract("The QUICK brown fox", &["quick"], &opts_small());
let s = extract("The QUICK brown fox", "quick", &opts_small());
assert_eq!(marked(&s), vec!["QUICK"]);
}
#[test]
fn multiple_terms_both_marked() {
let s = extract(
"the quick brown fox jumps over the lazy dog",
&["quick", "lazy"],
&Options::default(),
);
assert_eq!(marked(&s), vec!["quick", "lazy"]);
assert_ranges_valid(&s);
}
#[test]
fn window_truncation_flags_on_both_sides() {
let text = "prefix ".repeat(20) + "MATCH in middle " + &"suffix ".repeat(20);
let s = extract(&text, &["MATCH"], &opts_small());
let s = extract(&text, "MATCH", &opts_small());
assert!(s.truncated_start);
assert!(s.truncated_end);
assert_eq!(marked(&s), vec!["MATCH"]);
@ -503,7 +452,7 @@ mod tests {
fn match_at_start_not_truncated_left() {
let s = extract(
"MATCH right at the start of this paragraph",
&["match"],
"match",
&opts_small(),
);
assert!(!s.truncated_start);
@ -513,22 +462,15 @@ mod tests {
#[test]
fn no_match_returns_head_without_ranges() {
let text = "alpha beta gamma delta epsilon zeta eta theta iota kappa";
let s = extract(text, &["nomatch"], &opts_small());
let s = extract(text, "nomatch", &opts_small());
assert!(s.ranges.is_empty());
assert!(s.window.starts_with("alpha"));
}
#[test]
fn overlapping_terms_coalesce() {
let s = extract("the RUSTC compiler", &["rust", "rustc"], &opts_small());
assert_eq!(marked(&s), vec!["RUSTC"]);
assert_ranges_valid(&s);
}
#[test]
fn utf8_boundaries_hold_with_multibyte_text() {
let text = "café café café café café café café café café café";
let s = extract(text, &["café"], &opts_small());
let s = extract(text, "café", &opts_small());
assert!(!s.ranges.is_empty());
assert_ranges_valid(&s);
for m in marked(&s) {
@ -540,16 +482,10 @@ mod tests {
fn match_near_right_edge_is_fully_included() {
let prefix = "x".repeat(30);
let text = format!("{}{}", prefix, "LONGMATCHTERMTEXT");
let s = extract(&text, &["LONGMATCHTERMTEXT"], &opts_small());
let s = extract(&text, "LONGMATCHTERMTEXT", &opts_small());
assert_eq!(marked(&s), vec!["LONGMATCHTERMTEXT"]);
}
#[test]
fn empty_query_term_ignored() {
let s = extract("hello world", &["", "world"], &opts_small());
assert_eq!(marked(&s), vec!["world"]);
}
#[test]
fn window_around_basic() {
let text = "prefix ".repeat(20) + "NEEDLE" + &" suffix".repeat(20);
@ -579,12 +515,11 @@ mod tests {
#[test]
fn count_occurrences_cases() {
assert_eq!(count_occurrences("aaaa", "aaa", true), 1, "non-overlapping");
assert_eq!(count_occurrences("abcABC", "abc", true), 1);
assert_eq!(count_occurrences("abcABC", "abc", false), 2);
assert_eq!(count_occurrences("", "x", true), 0);
assert_eq!(count_occurrences("xyz", "", true), 0);
assert_eq!(count_occurrences("no hits here", "zzz", false), 0);
assert_eq!(count_occurrences("ab ab ab", "ab", true), 3);
assert_eq!(count_occurrences("aaaa", "aaa"), 1, "non-overlapping");
assert_eq!(count_occurrences("abcABC", "abc"), 1, "byte for byte");
assert_eq!(count_occurrences("", "x"), 0);
assert_eq!(count_occurrences("xyz", ""), 0);
assert_eq!(count_occurrences("no hits here", "zzz"), 0);
assert_eq!(count_occurrences("ab ab ab", "ab"), 3);
}
}

36
crates/quicksearch-core/src/testutil.rs
View file

@ -7,9 +7,9 @@ use std::sync::atomic::{AtomicUsize, Ordering};
static NEXT: AtomicUsize = AtomicUsize::new(0);
/// The compressed body [`crate::db::repo::set_content_done`] wants.
/// The compressed body [`crate::db::repo::store_pending_text`] wants.
pub fn zstd_of(text: &str) -> Option<Vec<u8>> {
crate::db::repo::encode_one(text, true).expect("zstd encode")
crate::db::repo::encode_one(text).expect("zstd encode")
}
/// Old enough that a failure investigated the next morning has its tree.
@ -338,7 +338,7 @@ impl Default for SeedSpec {
/// Seed an index with synthetic rows, in one transaction. Shared by the
/// measurement harnesses so they all describe the same corpus.
pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) {
use crate::db::repo::{insert_file, set_content_done, NewFile};
use crate::db::repo::{insert_file, store_pending_text, NewFile};
use crate::mime::mime_to_type;
// Before the open: the profile decides how the file is created, and on a
@ -349,7 +349,7 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) {
if let Some(hmac) = spec.hmac {
crate::db::set_hmac_mode_override(hmac);
}
let conn = crate::db::open_or_recreate(path.to_str().unwrap(), "trigram").unwrap();
let conn = crate::db::open_or_recreate(path.to_str().unwrap(), &Default::default()).unwrap();
// Before the first row: `pgsz` decides how leaves are built, so setting it
// afterwards would only affect segments merged later.
if let Some(pgsz) = spec.pgsz {
@ -415,7 +415,7 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) {
})
});
// `needs_content` rows are born `STATE_PENDING`, so skipping the
// `set_content_done` below leaves one in the queue.
// `store_pending_text` below leaves one in the queue.
let needs_content = i % spec.content_every.max(1) == 0 && registry.supports(mime);
let extracted = needs_content && (spec.pending_every == 0 || i % spec.pending_every != 0);
let id = insert_file(
@ -444,7 +444,7 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) {
body[spec.body_words * 2 / 3] = BODY_TERM;
}
let body = body.join(" ");
set_content_done(&tx, id, &body, zstd_of(&body).as_deref()).unwrap();
store_pending_text(&tx, id, &body, zstd_of(&body).as_deref()).unwrap();
}
if spec.commit_every > 0 && (i + 1) % spec.commit_every == 0 {
tx.commit().unwrap();
@ -495,6 +495,30 @@ pub fn sniff(path: &std::path::Path, hash_length: usize) -> Option<&'static str>
crate::mime::guess_mime_from_head(path, &head)
}
/// The rows the full-text index matches for `term`, asked for as a search
/// would ask ([`crate::search::prefilter::fts_literal`], by the layout the
/// index records): a bare `MATCH 'word'` is a phrase of several trigrams,
/// which an index without positions refuses. Unverified, so on such an index
/// these are candidates, a superset of the documents containing `term`.
pub fn fts_rowids(conn: &rusqlite::Connection, term: &str) -> Vec<i64> {
let expr = crate::search::prefilter::fts_literal(term, crate::db::repo::fts_positions(conn))
.expect("a term of three characters or more");
let mut stmt = conn
.prepare("SELECT rowid FROM searchabletext WHERE searchabletext MATCH ?1")
.expect("prepare fts query");
let rows = stmt
.query_map([expr], |r| r.get(0))
.expect("fts query")
.collect::<Result<Vec<i64>, _>>()
.expect("fts rows");
rows
}
/// How many rows [`fts_rowids`] finds.
pub fn fts_count(conn: &rusqlite::Connection, term: &str) -> i64 {
fts_rowids(conn, term).len() as i64
}
/// `(leaf, overflow)` pages in `searchabletext_data`, from `dbstat`. An
/// overflow page is a second fetch and decrypt on every read of its leaf;
/// [`crate::db::schema::fts_pgsz_for`] keeps the count at zero.

47
crates/quicksearch-core/src/textenc.rs
View file

@ -77,13 +77,22 @@ pub fn looks_like_text(head: &[u8]) -> bool {
/// Takes the buffer by value so a UTF-8 file, the overwhelming majority,
/// becomes its `String` with no copy. Callers holding only a borrow want
/// [`decode_borrowed_text`].
pub fn decode_text(bytes: Vec<u8>, path: &Path) -> Result<String, String> {
///
/// `truncated` marks a prefix of the file rather than all of it: a multibyte
/// character cut by the read is dropped instead of demoting the file to
/// `Legacy`. The other encodings replace a cut tail with one U+FFFD.
pub fn decode_text(mut bytes: Vec<u8>, path: &Path, truncated: bool) -> Result<String, String> {
if bytes.is_empty() {
return Ok(String::new());
}
match classify(&bytes, false) {
// Cannot fail: classify ran strict validation with truncated=false.
TextClass::Utf8 => Ok(String::from_utf8(bytes).expect("classified as UTF-8")),
match classify(&bytes, truncated) {
TextClass::Utf8 => {
// `classify` forgave only a cut final character; drop its stub.
if let Err(e) = std::str::from_utf8(&bytes) {
bytes.truncate(e.valid_up_to());
}
Ok(String::from_utf8(bytes).expect("classified as UTF-8"))
}
_ => decode_borrowed_text(&bytes, path),
}
}
@ -157,7 +166,7 @@ mod tests {
let body = "plain ascii and café über 日本語".as_bytes().to_vec();
assert!(looks_like_text(&body));
assert_eq!(
decode_text(body, &p()).unwrap(),
decode_text(body, &p(), false).unwrap(),
"plain ascii and café über 日本語"
);
}
@ -167,7 +176,7 @@ mod tests {
let mut body = vec![0xEF, 0xBB, 0xBF];
body.extend_from_slice("hello".as_bytes());
assert!(looks_like_text(&body));
let text = decode_text(body, &p()).unwrap();
let text = decode_text(body, &p(), false).unwrap();
assert_eq!(text, "hello", "BOM must not survive into stored text");
}
@ -180,7 +189,7 @@ mod tests {
body.extend_from_slice(&unit.to_le_bytes());
}
assert!(looks_like_text(&body));
assert_eq!(decode_text(body, &p()).unwrap(), src);
assert_eq!(decode_text(body, &p(), false).unwrap(), src);
}
#[test]
@ -191,7 +200,7 @@ mod tests {
body.extend_from_slice(&unit.to_be_bytes());
}
assert!(looks_like_text(&body));
assert_eq!(decode_text(body, &p()).unwrap(), src);
assert_eq!(decode_text(body, &p(), false).unwrap(), src);
}
/// Legacy charsets decode, but do not *sniff*.
@ -203,7 +212,7 @@ mod tests {
"the sniff must not adopt a non-UTF-8 head on its own"
);
assert_eq!(
decode_text(body, &p()).unwrap(),
decode_text(body, &p(), false).unwrap(),
"Le café près de la fenêtre est agréable en été."
);
}
@ -217,7 +226,7 @@ mod tests {
body.extend_from_slice(head);
}
body.extend_from_slice(b"caf\xe9-tail-marker");
let out = decode_text(body, &p()).unwrap();
let out = decode_text(body, &p(), false).unwrap();
assert!(
out.ends_with("café-tail-marker"),
"the tail past the detection prefix must still be decoded"
@ -230,7 +239,7 @@ mod tests {
let body = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9.".to_vec();
assert!(body.len() < DETECT_PREFIX);
assert_eq!(
decode_text(body, &p()).unwrap(),
decode_text(body, &p(), false).unwrap(),
"Le café près de la fenêtre est agréable en été."
);
}
@ -242,14 +251,14 @@ mod tests {
assert!(!had_errors);
let body = encoded.into_owned();
assert!(!looks_like_text(&body));
assert_eq!(decode_text(body, &p()).unwrap(), src);
assert_eq!(decode_text(body, &p(), false).unwrap(), src);
}
#[test]
fn nul_bytes_are_binary() {
let body = b"looks like text until\x00it does not".to_vec();
assert!(!looks_like_text(&body));
let err = decode_text(body, &p()).unwrap_err();
let err = decode_text(body, &p(), false).unwrap_err();
assert!(err.contains("binary content"), "{err}");
assert!(
err.contains("textenc-test-file"),
@ -262,7 +271,7 @@ mod tests {
// 4 control bytes in 24 total = 16% > 10%.
let body = b"abcdefghijklmnopqrst\x01\x02\x03\x04".to_vec();
assert!(!looks_like_text(&body));
assert!(decode_text(body, &p()).is_err());
assert!(decode_text(body, &p(), false).is_err());
}
#[test]
@ -270,7 +279,7 @@ mod tests {
let body =
b"\x1b[31mERROR\x1b[0m something failed\n\x1b[33mWARN\x1b[0m retrying\n".to_vec();
assert!(looks_like_text(&body));
assert!(decode_text(body, &p()).is_ok());
assert!(decode_text(body, &p(), false).is_ok());
}
#[test]
@ -280,13 +289,13 @@ mod tests {
assert!(looks_like_text(&head));
// The same bytes as a complete file are not valid UTF-8, so the
// decoder treats them as legacy-encoded: still Ok, never a panic.
assert!(decode_text(head, &p()).is_ok());
assert!(decode_text(head, &p(), false).is_ok());
}
#[test]
fn empty_head_is_not_text_but_empty_file_decodes() {
assert!(!looks_like_text(b""));
assert_eq!(decode_text(Vec::new(), &p()).unwrap(), "");
assert_eq!(decode_text(Vec::new(), &p(), false).unwrap(), "");
}
/// UTF-16 without a BOM is out of scope by decision: its NULs trip the
@ -298,7 +307,7 @@ mod tests {
body.extend_from_slice(&unit.to_le_bytes());
}
assert!(!looks_like_text(&body));
assert!(decode_text(body, &p()).is_err());
assert!(decode_text(body, &p(), false).is_err());
}
/// Every head the sniff accepts must decode, which makes "sniffed as
@ -329,7 +338,7 @@ mod tests {
"sniff verdict changed for {head:?}"
);
assert!(
decode_text(head.clone(), &p()).is_ok(),
decode_text(head.clone(), &p(), false).is_ok(),
"head failed to decode: {head:?}"
);
}

32
crates/quicksearch-core/src/walk.rs
View file

@ -12,7 +12,6 @@ use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::{mpsc, Arc, Condvar, Mutex};
use std::thread::JoinHandle;
use std::time::UNIX_EPOCH;
use crate::config::{Config, IgnoreSet};
use crate::extract::Registry;
@ -170,8 +169,17 @@ fn read_directory(
}
};
// Every `continue` in the loop must be a genuine "not indexable", or the
// stale diff below deletes live rows.
let mut present: HashSet<String> = HashSet::new();
// stale diff below deletes live rows. Only names that have a row matter to
// that diff, so `present` borrows them from `rows`: no allocation per
// entry, and none at all for a directory with no rows yet. Measured
// (`indexprobe cold`, 100k files): 8.2 → 7.2 allocations per file,
// churn 42 → 35 MiB; time did not move, being `stat`-bound.
let mut present: HashSet<&str> = HashSet::new();
let mut mark_present = |name: &str| {
if let Some((known, _)) = rows.get_key_value(name) {
present.insert(known.as_str());
}
};
let mut unreadable_entry = false;
let mut files = Vec::new();
@ -263,15 +271,13 @@ fn read_directory(
}
}
Ok(_) => {
present.insert(name.to_string());
mark_present(name);
// `None` on Unix and on any reparse point.
let cached = crate::platform::entry_cached_metadata(|| entry.metadata().ok());
files.push(PendingFile { path, cached });
}
// Type unknown: mark it present so an existing row survives.
Err(_) => {
present.insert(name.to_string());
}
Err(_) => mark_present(name),
}
}
@ -363,12 +369,7 @@ fn prepare(
// "deleted".
return WalkedFile::skipped(db_path, aliased);
};
let Some(mtime) = meta
.modified()
.ok()
.and_then(|t| t.duration_since(UNIX_EPOCH).ok())
.map(|d| d.as_secs())
else {
let Some(mtime) = crate::file_handling::mtime_secs(&meta) else {
return WalkedFile::skipped(db_path, aliased);
};
@ -517,6 +518,11 @@ impl ParallelWalk {
/// Every canonical directory the walk queued, in `files.parent` spelling.
/// The vanished-directory sweep needs this: a directory deleted wholesale
/// is never read, so nothing reconciles the rows beneath it.
///
/// Measured and not adopted: moving the set out and reusing each path's
/// buffer, so it is never held twice, left VmHWM where it was
/// (`memprobe cold`, 200k directories: 84–105 MiB against 86–97 MiB). The
/// run's peak is not at the sweep.
pub fn seen_dirs(&self) -> HashSet<String> {
crate::lock_ok(&self.shared.queue)
.seen_dirs

8
crates/quicksearch-core/src/walk/tests.rs
View file

@ -1,5 +1,7 @@
use super::*;
use std::time::UNIX_EPOCH;
use crate::testutil::{touch, Scratch};
fn tmp_tree(tag: &str) -> Scratch {
@ -20,7 +22,7 @@ impl std::ops::Deref for TestDb {
fn db_with(tag: &str, rows: &[(String, u64)]) -> TestDb {
let (dir, p) = Scratch::db(tag);
let conn = crate::db::open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let conn = crate::db::open_or_recreate(p.to_str().unwrap(), &Default::default()).unwrap();
for (path, mtime) in rows {
// Split the same way the indexer does, so the seeded parent carries
// its trailing separator and the prefetcher's `parent = ?` finds it.
@ -789,7 +791,7 @@ fn the_index_and_its_sidecars_are_never_walked() {
touch(&root.join("ordinary.txt"), b"x");
let db = root.join("data").join("index.sqlite");
let conn = crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
let conn = crate::db::open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap();
// A write, so the WAL and SHM exist to be walked over.
conn.execute(
"INSERT INTO files (name, parent, size, mtime, type, content_state)
@ -822,7 +824,7 @@ fn the_index_and_its_sidecars_are_never_walked() {
fn a_stored_row_for_the_index_falls_to_the_stale_sweep() {
let root = crate::testutil::scratch_dir_canonical("walk-self-index-stale");
let db = root.join("data").join("index.sqlite");
let conn = crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
let conn = crate::db::open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap();
// A row for the database itself.
let stored = path_to_db_string(&db);

210
crates/quicksearch-core/tests/cascade.rs
View file

@ -5,8 +5,9 @@ use std::path::Path;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use quicksearch_core::config::ProcessingConfig;
use quicksearch_core::db::open_or_recreate;
use quicksearch_core::db::repo::{insert_file, set_content_done, NewFile};
use quicksearch_core::db::repo::{insert_file, store_pending_text, NewFile};
use quicksearch_core::mime::FileType;
use quicksearch_core::query::split::split_for_cascade;
use quicksearch_core::search::{
@ -16,14 +17,21 @@ use quicksearch_core::testutil::{zstd_of, Scratch};
struct Seeder {
conn: rusqlite::Connection,
store_text: bool,
}
impl Seeder {
fn new(path: &Path, store_text: bool) -> Seeder {
/// An index in the default layout: compact, without positions.
fn new(path: &Path) -> Seeder {
Seeder::with_positions(path, false)
}
fn with_positions(path: &Path, positions: bool) -> Seeder {
let processing = ProcessingConfig {
compact_text_index: !positions,
..Default::default()
};
Seeder {
conn: open_or_recreate(path.to_str().unwrap(), "trigram").unwrap(),
store_text,
conn: open_or_recreate(path.to_str().unwrap(), &processing).unwrap(),
}
}
@ -48,8 +56,7 @@ impl Seeder {
.unwrap()
.expect("unique path");
if let Some(text) = text {
let zstd = self.store_text.then(|| zstd_of(text)).flatten();
set_content_done(&tx, id, text, zstd.as_deref()).unwrap();
store_pending_text(&tx, id, text, zstd_of(text).as_deref()).unwrap();
}
tx.commit().unwrap();
id
@ -130,7 +137,7 @@ fn fuzzy_options_with_edits(max_edits: usize) -> SearchOptions {
#[test]
fn rank_classification_across_all_stages() {
let (_dir, p) = Scratch::db("ranks");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let rank1 = s.add("Report", "/a", 1, None);
let rank2 = s.add("report", "/b", 2, None);
let rank3 = s.add("Quarterly_Report.txt", "/c", 3, None);
@ -191,7 +198,7 @@ fn rank_classification_across_all_stages() {
#[test]
fn path_substring_tiers_split_by_case() {
let (_dir, p) = Scratch::db("pathcase");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let exact = s.add("a.bin", "/srv/Vacation/raw", 1, None);
let anycase = s.add("b.bin", "/vacation-2023", 2, None);
let _miss = s.add("c.bin", "/holiday", 3, None);
@ -216,7 +223,7 @@ fn path_substring_tiers_split_by_case() {
#[test]
fn path_match_is_deduped_against_a_better_name_match() {
let (_dir, p) = Scratch::db("pathdedup");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let star = s.add("Budget.txt", "/Budget/2024", 1, None);
let conn = s.done();
@ -230,7 +237,7 @@ fn path_match_is_deduped_against_a_better_name_match() {
#[test]
fn short_terms_respect_the_three_char_trigram_floor() {
let (_dir, p) = Scratch::db("pathfloor");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let dir_only = s.add("z.bin", "/abcdir", 1, None);
let name_hit = s.add("ab.txt", "/d", 2, None);
let _text_only = s.add("body.txt", "/d", 3, Some("ab ab ab"));
@ -255,7 +262,7 @@ fn short_terms_respect_the_three_char_trigram_floor() {
#[test]
fn term_with_separator_matches_across_the_path() {
let (_dir, p) = Scratch::db("pathsep");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let nested = s.add("report-final.txt", "/home/docs", 1, None);
let _elsewhere = s.add("report-final.txt", "/home/other", 2, None);
let conn = s.done();
@ -271,7 +278,7 @@ fn term_with_separator_matches_across_the_path() {
#[test]
fn fuzzy_path_tier_requires_the_fuzzy_flag() {
let (_dir, p) = Scratch::db("fuzzypath");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let typo_dir = s.add("gamma.bin", "/Reprot-archive", 1, None);
let conn = s.done();
@ -286,7 +293,7 @@ fn fuzzy_path_tier_requires_the_fuzzy_flag() {
#[test]
fn fuzzy_max_edits_widens_and_narrows_the_budget() {
let (_dir, p) = Scratch::db("fuzzybudget");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let two_edits = s.add("quartrely.txt", "/d", 1, None);
let three_edits = s.add("quxxxerly.txt", "/d", 2, None);
let conn = s.done();
@ -319,7 +326,7 @@ fn fuzzy_max_edits_widens_and_narrows_the_budget() {
#[test]
fn a_distant_fuzzy_filename_hit_stays_a_name_hit() {
let (_dir, p) = Scratch::db("fuzzystage");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
// Ten substitutions against a 30-character term, whose budget is ten.
let far = s.add("abcdefghijklmnopqrst##########", "/d", 1, None);
let conn = s.done();
@ -342,7 +349,7 @@ fn a_distant_fuzzy_filename_hit_stays_a_name_hit() {
#[test]
fn dedup_keeps_best_rank() {
let (_dir, p) = Scratch::db("dedup");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let star = s.add("Budget", "/a", 1, Some("Budget Budget Budget"));
let conn = s.done();
@ -356,7 +363,7 @@ fn dedup_keeps_best_rank() {
#[test]
fn occurrence_counts_order_within_rank() {
let (_dir, p) = Scratch::db("frac");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let one = s.add("one.txt", "/d", 1, Some("zebra"));
let three = s.add("three.txt", "/d", 2, Some("zebra zebra zebra"));
let thousand = s.add("thousand.txt", "/d", 3, Some(&"zebra ".repeat(1500)));
@ -376,7 +383,7 @@ fn occurrence_counts_order_within_rank() {
#[test]
fn like_metacharacters_are_literal() {
let (_dir, p) = Scratch::db("like");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let percent = s.add("100%.txt", "/d", 1, None);
let underscore = s.add("100_.txt", "/d", 2, None);
let contains = s.add("x100y.txt", "/d", 3, None);
@ -402,9 +409,10 @@ fn like_metacharacters_are_literal() {
#[test]
fn diacritic_fts_candidates_are_dropped() {
let (_dir, p) = Scratch::db("diacritic");
let mut s = Seeder::new(&p, true);
// trigram remove_diacritics 1 makes this an FTS candidate for "cafe",
// but the exact bytes never occur, so exact full-text must drop it.
let mut s = Seeder::new(&p);
// "cafe" never occurs in the accented body: not an FTS candidate, and
// exact full-text would drop it even if a diacritic-folding tokenizer
// were configured.
let _accented = s.add("menu.txt", "/d", 1, Some("le café est ouvert"));
let plain = s.add("plain.txt", "/d", 2, Some("the cafe is open"));
let conn = s.done();
@ -413,36 +421,82 @@ fn diacritic_fts_candidates_are_dropped() {
assert_eq!(ids(&hits), vec![plain]);
}
/// Without positions the index finds candidates by trigrams alone, adjacent
/// or not; the full-text pass must drop those whose text lacks the term.
#[test]
fn contentless_mode_degrades_to_unranked_stage6() {
let (_dir, p) = Scratch::db("notext");
let mut s = Seeder::new(&p, false); // store_text_for_snippets = false
let doc = s.add("doc.txt", "/d", 1, Some("walrus columns"));
fn a_compact_index_drops_candidates_whose_trigrams_are_not_adjacent() {
let (_dir, p) = Scratch::db("scattered");
let mut s = Seeder::new(&p);
// Holds `abc`, `bcd` and `cde`, never `abcde`.
let scattered = s.add("scattered.txt", "/d", 1, Some("abcx bcdy cdez"));
let whole = s.add("whole.txt", "/d", 2, Some("the abcde run"));
let conn = s.done();
let (hits, _) = run_collect(&conn, "walrus", &fuzzy_options());
assert_eq!(hits.len(), 1, "FTS still finds it");
let h = &hits[0];
assert_eq!(h.file_id, doc);
assert_eq!(h.stage, 6, "cannot case-verify without text");
assert!((h.rank - 6.999).abs() < 1e-9, "count-unknown fraction");
assert!(h.snippet.is_none());
assert!(!hits.iter().any(|h| h.stage == 8));
assert_eq!(
quicksearch_core::testutil::fts_rowids(&conn, "abcde").len(),
2,
"both are FTS candidates"
);
let (hits, _) = run_collect(&conn, "abcde", &SearchOptions::default());
assert_eq!(ids(&hits), vec![whole]);
assert!(!ids(&hits).contains(&scattered));
}
// Without stored text a floor-clearing wildcard can't be pattern-verified.
let (hits, _) = run_collect(&conn, "wal*rus", &SearchOptions::default());
assert_eq!(staged(&hits), vec![(doc, 6)]);
assert!(hits[0].snippet.is_none());
// The short-segment fallback has no FTS evidence, so it finds nothing.
let (none, _) = run_collect(&conn, "wa*us", &SearchOptions::default());
assert!(none.is_empty());
/// The layout changes how candidates are found, never what a search
/// returns: every query shape answers alike on either index.
#[test]
fn both_index_layouts_answer_every_query_shape_alike() {
let bodies = [
("ocean.txt", "the ocean forest at dawn"),
("reversed.txt", "a forest by the ocean"),
("report.txt", "Quarterly Report: revenue rose"),
("lower.txt", "the quarterly report is late"),
("wild.txt", "reprocessing the export"),
("typo.txt", "the mountian pass"),
("code.txt", "impl Iterator for Walker"),
("scattered.txt", "impl it; Iterator; for x"),
];
let queries: &[(&str, bool)] = &[
("ocean forest", false),
("Report", false),
("report", false),
("rep*rt", false),
("impl Iterator for", false),
("regex:revenue\\s+rose", false),
("mountain", true),
("iterator", true),
];
let answers: Vec<Vec<Vec<(i64, u8)>>> = [false, true]
.into_iter()
.map(|positions| {
let (_dir, p) = Scratch::db("layouts");
let mut s = Seeder::with_positions(&p, positions);
for (name, body) in bodies {
s.add(name, "/d", 1, Some(body));
}
let conn = s.done();
queries
.iter()
.map(|&(query, fuzzy)| {
let options = SearchOptions {
fuzzy,
..SearchOptions::default()
};
staged(&run_collect(&conn, query, &options).0)
})
.collect()
})
.collect();
for (i, (query, _)) in queries.iter().enumerate() {
assert_eq!(answers[0][i], answers[1][i], "{query}");
assert!(!answers[0][i].is_empty(), "{query} should find something");
}
}
#[test]
fn filters_apply_to_every_stage() {
let (_dir, p) = Scratch::db("filters");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let keep_name = s.add("alpha.txt", "/keep", 1, None);
let _skip_name = s.add("alpha.txt", "/skip", 1, None);
let keep_text = s.add("k.txt", "/keep", 2, Some("alpha body"));
@ -491,7 +545,7 @@ fn the_limit_truncates_flags_and_keeps_the_best_hits() {
for ((tag, subs, exact_last, limit), (len, limited, first)) in cases {
let (_dir, p) = Scratch::db(tag);
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
for i in 0..subs {
s.add(&format!("zz-match-{:02}.txt", i), "/d", 1, None);
}
@ -515,7 +569,7 @@ fn the_limit_truncates_flags_and_keeps_the_best_hits() {
#[test]
fn session_ignores_hide_hits_before_the_cap() {
let (_dir, p) = Scratch::db("ignores");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let keep = s.add("keep-match.txt", "/d", 1, None);
let _log = s.add("match.log", "/d", 2, None);
let _sub = s.add("match.txt", "/d/logs", 3, None);
@ -535,7 +589,7 @@ fn session_ignores_hide_hits_before_the_cap() {
#[test]
fn empty_and_filter_only_terms_return_nothing() {
let (_dir, p) = Scratch::db("empty");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
s.add("anything.txt", "/d", 1, Some("anything"));
let conn = s.done();
@ -549,7 +603,7 @@ fn empty_and_filter_only_terms_return_nothing() {
#[test]
fn hostile_terms_are_inert() {
let (_dir, p) = Scratch::db("hostile");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
s.add("innocent.txt", "/d", 1, Some("innocent content"));
let conn = s.done();
@ -585,7 +639,7 @@ fn hostile_terms_are_inert() {
#[test]
fn wildcard_name_ranks_through_the_same_tiers() {
let (_dir, p) = Scratch::db("wildranks");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
// `report*` anchors the whole name, so these are tiers 1 and 2 …
let whole_cs = s.add("report2024.pdf", "/a", 1, None);
let whole_ci = s.add("Report2024.pdf", "/b", 2, None);
@ -620,7 +674,7 @@ fn wildcard_name_ranks_through_the_same_tiers() {
#[test]
fn extension_glob_whole_matches_every_such_file() {
let (_dir, p) = Scratch::db("extglob");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let a = s.add("alpha.txt", "/d", 1, None);
let b = s.add("beta.txt", "/d", 2, None);
let upper = s.add("GAMMA.TXT", "/d", 3, None);
@ -649,7 +703,7 @@ fn extension_glob_whole_matches_every_such_file() {
#[test]
fn wildcard_fulltext_narrows_with_fts_and_verifies_order() {
let (_dir, p) = Scratch::db("wildtext");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let ordered = s.add("a.txt", "/d", 1, Some("a wondrous world indeed"));
// FTS AND-of-segments finds this too (both trigram runs occur), but the
// pattern requires "wond" before "world", so verification drops it.
@ -674,7 +728,7 @@ fn wildcard_fulltext_narrows_with_fts_and_verifies_order() {
#[test]
fn wildcard_with_short_segments_falls_back_to_a_full_scan() {
let (_dir, p) = Scratch::db("wildshort");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let hit = s.add("doc.txt", "/d", 1, Some("zz abXcd zz"));
let _miss = s.add("other.txt", "/d", 2, Some("cd before ab"));
let conn = s.done();
@ -691,7 +745,7 @@ fn wildcard_with_short_segments_falls_back_to_a_full_scan() {
#[test]
fn a_wildcard_prefilter_never_loses_a_hit_the_full_scan_finds() {
let (_dir, p) = Scratch::db("wildprefilter");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let in_name = s.add("report-q3.txt", "/data", 1, None);
let in_parent = s.add("a.bin", "/reports/q3", 2, None);
// The pattern's `%` spans the parent/name boundary.
@ -751,7 +805,7 @@ fn a_fuzzy_prefilter_never_loses_a_hit_the_full_scan_finds() {
use quicksearch_core::search::fuzzy::{edit_budget, Bitap};
let (_dir, p) = Scratch::db("fuzzyprefilter");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let rows = [
("report.txt", "/data"), // exact
("REPORT.TXT", "/upper"), // ASCII case is free for LIKE and bitap both
@ -807,7 +861,7 @@ fn a_fuzzy_prefilter_never_loses_a_hit_the_full_scan_finds() {
#[test]
fn a_fuzzy_term_with_a_separator_still_scans_and_matches() {
let (_dir, p) = Scratch::db("fuzzysep");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let hit = s.add("orts.txt", "/a/rep", 1, None);
let _miss = s.add("plans.txt", "/a/sum", 2, None);
let conn = s.done();
@ -827,7 +881,7 @@ fn a_fuzzy_term_with_a_separator_still_scans_and_matches() {
#[test]
fn a_wildcard_with_only_separator_segments_still_scans_and_matches() {
let (_dir, p) = Scratch::db("wildnoanchor");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let hit = s.add("q3.txt", "/a/reports", 1, None);
let _miss = s.add("q3.txt", "/a/summaries", 2, None);
let conn = s.done();
@ -839,7 +893,7 @@ fn a_wildcard_with_only_separator_segments_still_scans_and_matches() {
#[test]
fn wildcard_path_tier_and_filters() {
let (_dir, p) = Scratch::db("wildpath");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let dir_hit = s.add("a.bin", "/Vacation-2024", 1, None);
let filtered = s.add("b.bin", "/elsewhere/Vacation-2023", 2, None);
let conn = s.done();
@ -855,7 +909,7 @@ fn wildcard_path_tier_and_filters() {
#[test]
fn wildcard_terms_skip_the_fuzzy_stages() {
let (_dir, p) = Scratch::db("wildfuzzy");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let real = s.add("report.txt", "/d", 1, None);
// A 2-edit typo: fuzzy would admit it for a literal term.
let _typo = s.add("Reprot.txt", "/d", 2, None);
@ -873,7 +927,7 @@ fn wildcard_terms_skip_the_fuzzy_stages() {
#[test]
fn regex_only_query_hits_name_content_and_path() {
let (_dir, p) = Scratch::db("regexonly");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let by_name = s.add("qz42.txt", "/d", 1, None);
let by_content = s.add("notes.txt", "/d", 2, Some("ref qz7 in the body"));
let by_path = s.add("b.bin", "/qz99-dir", 3, None);
@ -901,7 +955,7 @@ fn regex_only_query_hits_name_content_and_path() {
#[test]
fn regex_is_case_insensitive_by_default_and_respects_filters() {
let (_dir, p) = Scratch::db("regexci");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let keep = s.add("QZ1.txt", "/keep", 1, None);
let _skip = s.add("qz2.txt", "/skip", 2, None);
let conn = s.done();
@ -916,7 +970,7 @@ fn regex_is_case_insensitive_by_default_and_respects_filters() {
#[test]
fn regex_alongside_a_term_is_an_accept_predicate() {
let (_dir, p) = Scratch::db("regexpred");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
// Kept via the lazy content fetch: the regex is nowhere in its name/path.
let kept = s.add("budget-a.txt", "/d", 1, Some("code acme7 inside"));
let _dropped = s.add("budget-b.txt", "/d", 2, None);
@ -933,7 +987,7 @@ fn regex_alongside_a_term_is_an_accept_predicate() {
#[test]
fn hostile_regexes_complete_quickly() {
let (_dir, p) = Scratch::db("regexhostile");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
s.add("aaa.txt", "/d", 1, Some(&"a".repeat(50_000)));
let conn = s.done();
@ -950,7 +1004,7 @@ fn hostile_regexes_complete_quickly() {
#[test]
fn service_surfaces_invalid_regex_as_an_error() {
let (_dir, p) = Scratch::db("regexerr");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
s.add("anything.txt", "/d", 1, None);
drop(s.done());
@ -964,7 +1018,7 @@ fn service_surfaces_invalid_regex_as_an_error() {
#[test]
fn generation_bump_cancels_mid_stream() {
let (_dir, p) = Scratch::db("cancel");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
for i in 0..500 {
s.add(&format!("bulk-{:04}.txt", i), "/d", 1, None);
}
@ -994,7 +1048,7 @@ fn generation_bump_cancels_mid_stream() {
#[test]
fn service_rapid_fire_completes_only_the_last_generation() {
let (_dir, p) = Scratch::db("service");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
for i in 0..2000 {
s.add(
&format!("file-{:04}.txt", i),
@ -1082,7 +1136,7 @@ fn search_names(
#[test]
fn repeated_searches_on_one_service_stay_correct() {
let (_dir, p) = Scratch::db("warmrepeat");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
s.add("alpha.txt", "/d", 1, Some("alpha body"));
s.add("beta.txt", "/d", 1, Some("beta body"));
drop(s.done());
@ -1107,7 +1161,7 @@ fn repeated_searches_on_one_service_stay_correct() {
#[test]
fn a_rebuilt_index_at_the_same_path_is_picked_up() {
let (_dir, p) = Scratch::db("warmrebuild");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
s.add("before.txt", "/d", 1, Some("shared body"));
drop(s.done());
@ -1124,7 +1178,7 @@ fn a_rebuilt_index_at_the_same_path_is_picked_up() {
for suffix in ["", "-wal", "-shm"] {
std::fs::remove_file(format!("{}{}", p.display(), suffix)).ok();
}
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
s.add("after.txt", "/d", 1, Some("shared body"));
drop(s.done());
@ -1139,12 +1193,12 @@ fn a_rebuilt_index_at_the_same_path_is_picked_up() {
#[test]
fn set_db_path_repoints_a_held_connection() {
let (_dir1, first) = Scratch::db("warmpath1");
let mut s = Seeder::new(&first, true);
let mut s = Seeder::new(&first);
s.add("infirst.txt", "/d", 1, Some("shared body"));
drop(s.done());
let (_dir2, second) = Scratch::db("warmpath2");
let mut s = Seeder::new(&second, true);
let mut s = Seeder::new(&second);
s.add("insecond.txt", "/d", 1, Some("shared body"));
drop(s.done());
@ -1170,7 +1224,7 @@ fn a_failed_search_does_not_wedge_the_next_one() {
search_names(&service, &updates, "anything").expect_err("no index yet");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
s.add("arrived.txt", "/d", 1, Some("shared body"));
drop(s.done());
@ -1190,7 +1244,7 @@ fn a_failed_search_does_not_wedge_the_next_one() {
#[test]
fn the_connection_is_released_once_searching_stops() {
let (_dir, p) = Scratch::db("warmrelease");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
s.add("held.txt", "/d", 1, Some("shared body"));
drop(s.done());
@ -1242,7 +1296,7 @@ fn the_connection_is_released_once_searching_stops() {
#[test]
fn a_release_supersedes_a_search_queued_before_it() {
let (_dir, p) = Scratch::db("relcancel");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
s.add("held.txt", "/d", 1, Some("shared body"));
drop(s.done());
@ -1277,7 +1331,7 @@ fn a_release_supersedes_a_search_queued_before_it() {
#[test]
fn a_pass_hands_hits_over_before_the_scan_reaches_the_end() {
let (_dir, p) = Scratch::db("stream");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let early_worse = s.add("my_zebra_file.txt", "/aaa", 1, None); // rank 3
for i in 0..400 {
s.add(&format!("filler{:04}.txt", i), "/mmm", i as u64 + 2, None);
@ -1329,7 +1383,7 @@ fn a_pass_hands_hits_over_before_the_scan_reaches_the_end() {
#[test]
fn a_sparse_match_still_streams_before_the_scan_ends() {
let (_dir, p) = Scratch::db("sparse");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
for i in 0..3000 {
let name = if i % 1000 == 500 {
format!("needle{:04}.txt", i)
@ -1365,7 +1419,7 @@ fn a_sparse_match_still_streams_before_the_scan_ends() {
#[test]
fn fuzzy_name_and_path_snippets_carry_the_whole_field() {
let (_dir, p) = Scratch::db("fuzzy-whole-field");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
// The match sits in the last third of the name, where a windowed snippet
// would start past zero.
s.add(
@ -1400,7 +1454,7 @@ fn fuzzy_name_and_path_snippets_carry_the_whole_field() {
#[test]
fn a_fuzzy_mark_covers_the_matched_text_and_nothing_else() {
let (_dir, p) = Scratch::db("fuzzy-mark-span");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
// A start taken as `end - term.len()` would reach back over the leading
// digit. In the body too, for the full-text tier.
s.add("1Reporter.txt", "/home/me/docs", 1, None);
@ -1449,7 +1503,7 @@ fn a_fuzzy_mark_covers_the_matched_text_and_nothing_else() {
#[test]
fn a_fuzzy_mark_is_not_truncated_to_a_leading_part_of_the_term() {
let (_dir, p) = Scratch::db("fuzzy-mark-full");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
// The name must not match at all: a row the filename tier claims never
// reaches the full-text tier. And the body must hold a fuzzy variant: the
// term verbatim would be an exact content match, stage 5 or 6.
@ -1484,7 +1538,7 @@ fn a_fuzzy_mark_is_not_truncated_to_a_leading_part_of_the_term() {
#[test]
fn a_wildcard_spanning_the_directory_boundary_is_still_found() {
let (_dir, p) = Scratch::db("pathstraddle");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let straddling = s.add("q3.txt", "/x/docs", 1, None);
// Same two pieces, both inside the name: found either way, so it proves
// the query ran rather than that the fallback was reached.
@ -1510,7 +1564,7 @@ fn a_wildcard_spanning_the_directory_boundary_is_still_found() {
#[test]
fn the_folder_filter_covers_the_folder_itself_and_its_subtree() {
let (_dir, p) = Scratch::db("folderself");
let mut s = Seeder::new(&p, true);
let mut s = Seeder::new(&p);
let own = s.add("top.txt", "/srv/data", 1, None);
let nested = s.add("deep.txt", "/srv/data/2024", 2, None);
let _sibling = s.add("other.txt", "/srv/data-archive", 3, None);

7
crates/quicksearch-core/tests/encrypted.rs
View file

@ -32,12 +32,7 @@ fn header(db_path: &Path) -> [u8; 16] {
fn match_count(db_path: &Path, term: &str) -> i64 {
let conn = db::open_existing(&db_path.to_string_lossy(), false).unwrap();
conn.query_row(
"SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH ?1",
[term],
|r| r.get(0),
)
.unwrap()
quicksearch_core::testutil::fts_count(&conn, term)
}
#[test]

58
crates/quicksearch-core/tests/full_index.rs
View file

@ -67,16 +67,9 @@ fn count(db: &Path, sql: &str) -> i64 {
.unwrap()
}
/// FTS rows matching the phrase `word`.
/// FTS rows matching `word`.
fn hits(db: &Path, word: &str) -> i64 {
rusqlite::Connection::open(db)
.unwrap()
.query_row(
"SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH ?1",
[format!("\"{}\"", word)],
|r| r.get(0),
)
.unwrap()
quicksearch_core::testutil::fts_count(&rusqlite::Connection::open(db).unwrap(), word)
}
/// Bytes in `db`'s write-ahead log; 0 when there is none.
@ -1065,26 +1058,32 @@ fn the_content_extension_filter_still_excludes_small_text_files() {
);
}
/// Both layouts index the same files the same way; only the FTS table
/// differs, and every file with text keeps its body for verification.
#[test]
fn contentless_mode_still_indexes_inlined_files_without_storing_bodies() {
let root = Scratch::dir("inline-contentless");
let (_db_dir, db) = Scratch::db("inline-contentless-db");
fn either_index_layout_stores_and_finds_the_same_files() {
for compact in [true, false] {
let root = Scratch::dir("layout");
let (_db_dir, db) = Scratch::db("layout-db");
let mut config = Config::default();
config.processing.store_text_for_snippets = false;
config.processing.compact_text_index = compact;
touch(&root.join("small.txt"), b"searchable xylophone body");
touch(&root.join("other.txt"), b"nothing to see");
index_once(&root, &db, &config);
assert_eq!(
count(&db, "SELECT COUNT(*) FROM documents_text"),
0,
"contentless mode stores no bodies"
2,
"every body is stored (compact = {compact})"
);
assert_eq!(
hits(&db, "xylophone"),
1,
"an inlined file is still searchable in contentless mode"
assert_eq!(hits(&db, "xylophone"), 1, "compact = {compact}");
let detail_none = count(
&db,
"SELECT COUNT(*) FROM sqlite_master \
WHERE name = 'searchabletext' AND sql LIKE '%detail=none%'",
);
assert_eq!(detail_none == 1, compact);
}
}
// ---------------------------------------------------------------------------
@ -1251,11 +1250,13 @@ const MIN_ROWS_SAMPLED: usize = 4;
/// Removes the fixture before asserting, unlike the rest of this file: it is
/// generated and identical every run, and leaving tens of MB in a RAM-backed
/// /tmp behind a failure is itself a reason to fail.
#[allow(clippy::too_many_arguments)]
fn watch_heavy_beside_light(
tag: &str,
heavy_files: usize,
heavy_dirs: usize,
body: &[u8],
ext: &str,
workers: usize,
mut config: Config,
) -> Overlap {
@ -1263,7 +1264,7 @@ fn watch_heavy_beside_light(
let heavy = Scratch::dir(&heavy_tag);
for i in 0..heavy_files {
touch(
&heavy.join(format!("d{}/big{:04}.txt", i % heavy_dirs, i)),
&heavy.join(format!("d{}/big{:04}.{}", i % heavy_dirs, i, ext)),
body,
);
}
@ -1336,9 +1337,14 @@ fn a_heavy_root_does_not_stall_a_light_one() {
// with the row count, while what the light root drains does not. Sized
// for the guards: enough that the rows arrive over separate writer
// rounds, cheap enough that a loaded runner still finishes in seconds.
// RTF, because its extractor reads and parses the whole file, where
// plaintext stops at twice `maximum_text_size`.
const HEAVY_FILES: usize = 12;
// ~2 MiB each, 24 MB total.
let body = "sphinx of black quartz judge my vow ".repeat(58_000);
let body = format!(
"{{\\rtf1\\ansi {}}}",
"sphinx of black quartz judge my vow ".repeat(58_000)
);
let mut config = Config::default();
config.processing.maximum_text_size = 1024;
@ -1347,7 +1353,7 @@ fn a_heavy_root_does_not_stall_a_light_one() {
config.processing.maximum_text_file_size = 4 * 1024 * 1024;
// One extraction thread: with the default four the pass is four times
// shorter, and so is the margin.
let seen = watch_heavy_beside_light("stall", HEAVY_FILES, 4, body.as_bytes(), 1, config);
let seen = watch_heavy_beside_light("stall", HEAVY_FILES, 4, body.as_bytes(), "rtf", 1, config);
eprintln!(
"light files drained while the heavy root extracted: {} against {} heavy \
@ -1392,6 +1398,7 @@ fn a_heavy_root_does_not_stall_a_light_one_at_the_writer() {
HEAVY_FILES,
8,
body.as_bytes(),
"txt",
4,
Config::default(),
);
@ -1631,7 +1638,10 @@ fn the_wal_stays_bounded_through_the_tail_of_a_warm_reindex() {
let root = seed_text_tree("wal-tail");
let (db_dir, db) = Scratch::db("wal-tail-db");
let dir_key = db_dir.to_string_lossy().into_owned();
let config = Config::default();
// With positions: the log is measured against the index's size, and
// position lists make this fixture's index several times larger.
let mut config = Config::default();
config.processing.compact_text_index = false;
// An empty root first, purely to bring the index into existence so the
// reader below can be opened before the run that matters.

2
crates/quicksearch-core/tests/page_size.rs
View file

@ -272,7 +272,7 @@ fn check_previous_profile(previous: Profile, keyed: bool, stale_version: bool) {
);
// The rebuild lands at the current layout, encrypted if it was.
let conn = db::open_or_recreate(&db_path, "trigram")
let conn = db::open_or_recreate(&db_path, &Default::default())
.unwrap_or_else(|e| panic!("{}: rebuild failed: {}", what, e));
assert_eq!(
page_size_of(&conn),

14
crates/quicksearch-core/tests/prefilter_fuzz.rs
View file

@ -51,8 +51,8 @@ const DOCS: usize = 60;
/// the corpus goes straight into the term and the FTS chunks: FTS5
/// metacharacters (must be quoted into inertness by
/// `translator::quote_phrase`), multi-byte characters at 2, 3 and 4 bytes,
/// diacritics (folded by the trigram index but not by bitap, so the prefilter
/// must stay a superset across that), mixed case, and a degenerate
/// diacritics (a fuzzy match may span an accented character, which the
/// prefilter must survive), mixed case, and a degenerate
/// repeated-character shape.
fn bodies() -> Vec<String> {
let mut out = Vec::new();
@ -586,7 +586,8 @@ fn seed_files(path: &std::path::Path, rows: &[(String, String)]) -> Vec<i64> {
use quicksearch_core::mime::FileType;
let mut conn =
quicksearch_core::db::open_or_recreate(path.to_str().unwrap(), "trigram").unwrap();
quicksearch_core::db::open_or_recreate(path.to_str().unwrap(), &Default::default())
.unwrap();
let tx = conn.transaction().unwrap();
let mut ids = Vec::with_capacity(rows.len());
for (parent, name) in rows {
@ -838,12 +839,13 @@ fn a_regex_finds_exactly_the_documents_it_matches() {
}
fn seed(path: &std::path::Path, bodies: &[String]) -> Vec<i64> {
use quicksearch_core::db::repo::{insert_file, set_content_done, NewFile};
use quicksearch_core::db::repo::{insert_file, store_pending_text, NewFile};
use quicksearch_core::mime::FileType;
use quicksearch_core::testutil::zstd_of;
let mut conn =
quicksearch_core::db::open_or_recreate(path.to_str().unwrap(), "trigram").unwrap();
quicksearch_core::db::open_or_recreate(path.to_str().unwrap(), &Default::default())
.unwrap();
let tx = conn.transaction().unwrap();
let mut ids = Vec::with_capacity(bodies.len());
for (i, body) in bodies.iter().enumerate() {
@ -864,7 +866,7 @@ fn seed(path: &std::path::Path, bodies: &[String]) -> Vec<i64> {
)
.unwrap()
.expect("unique path");
set_content_done(&tx, id, body, zstd_of(body).as_deref()).unwrap();
store_pending_text(&tx, id, body, zstd_of(body).as_deref()).unwrap();
ids.push(id);
}
tx.commit().unwrap();

50
crates/quicksearch-core/tests/reconcile.rs
View file

@ -87,6 +87,11 @@ fn names(db: &Path) -> Vec<String> {
.collect()
}
/// FTS rows matching `term`, asked for as the index's layout allows.
fn fts_count(db: &Path, term: &str) -> i64 {
quicksearch_core::testutil::fts_count(&conn(db), term)
}
fn count(db: &Path, sql: &str) -> i64 {
conn(db).query_row(sql, [], |r| r.get(0)).unwrap()
}
@ -177,10 +182,7 @@ fn adding_an_ignore_pattern_prunes_instead_of_rebuilding() {
);
assert_eq!(orphans(&db), 0);
assert_eq!(
count(
&db,
"SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'echo'"
),
fts_count(&db, "echo"),
0,
"the ignored file's content is no longer findable"
);
@ -304,13 +306,7 @@ fn narrowing_content_extensions_keeps_the_file_findable_by_name() {
assert_eq!(recontented, 1);
assert_eq!(paths(&db).len(), 2, "both rows are still there");
assert_eq!(residue(&db).0, 1, "only the .txt keeps its postings");
assert_eq!(
count(
&db,
"SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'markdown'"
),
0
);
assert_eq!(fts_count(&db, "markdown"), 0);
assert_eq!(
count(&db, "SELECT COUNT(*) FROM files WHERE content_state = 3"),
1,
@ -330,38 +326,6 @@ fn narrowing_content_extensions_keeps_the_file_findable_by_name() {
assert_eq!(created_at(&db), born, "the index was never rebuilt");
}
/// On again re-extracts: the text of files already indexed was never kept.
#[test]
fn store_text_toggles_without_losing_full_text_search() {
let root = Scratch::canonical("store-text");
let db_dir = Scratch::canonical("store-text-db");
let db = db_dir.join("index.sqlite");
touch(&root.join("a.txt"), b"alpha unique-token");
let on = base_config(&root, &db);
index_once(&db, &on);
let born = created_at(&db);
assert_eq!(residue(&db).1, 1, "text stored");
let mut off = on.clone();
off.processing.store_text_for_snippets = false;
reconcile(&db, &on, &off);
assert_eq!(residue(&db).1, 0, "blobs dropped");
assert_eq!(
count(
&db,
"SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'unique'"
),
1,
"full-text search is unaffected"
);
reconcile(&db, &off, &on);
index_once(&db, &on);
assert_eq!(residue(&db).1, 1, "text re-extracted");
assert_eq!(created_at(&db), born, "the index was never rebuilt");
}
#[test]
fn turning_hidden_files_off_prunes_whole_hidden_subtrees() {
let root = Scratch::canonical("hidden");

5
crates/quicksearch-gui/src/manage_tab.rs
View file

@ -531,9 +531,8 @@ fn db_size_tooltip(ui: &mut egui::Ui) {
"Remove indexed folders you do not need (Indexed folders, below).",
"Narrow the full-text extension whitelist to the file types whose text \
you search.",
"Turn off \"Store text for snippets\" on the Settings tab. Full-text search \
keeps working, without previews, occurrence ranking or fuzzy matching \
inside file contents.",
"Keep \"Compact text index\" on, on the Settings tab: it makes the \
full-text part of the index several times smaller.",
"Lower \"Max text file size\" and \"Max stored text\", both on the Settings tab.",
] {
ui.label(format!("• {}", lever));

2
crates/quicksearch-gui/src/manage_tab/tests.rs
View file

@ -916,7 +916,7 @@ fn hovering_the_size_explains_how_to_shrink_the_index() {
"ignore filters",
"Indexed folders",
"whitelist",
"Store text for snippets",
"Compact text index",
"Settings tab",
] {
assert!(text.contains(lever), "tooltip never mentions {}", lever);

6
crates/quicksearch-gui/src/search_tab.rs
View file

@ -1281,9 +1281,11 @@ impl SearchTab {
) {
self.live_armed = self.live_wanted.clone();
actions.live_targets = Some(self.live_wanted.clone());
} else if settled && !armed_already {
} else if self.live_enabled && settled && !armed_already {
// Once settled nothing else asks for frames, so a bare `Instant`
// deadline would never come due.
// deadline would never come due. Only while arming can happen:
// with live results off the set is never armed, and past the
// delay this asked for a frame every frame, forever.
if let Some(changed) = self.live_changed_at {
let waited = now.duration_since(changed);
ui.ctx()

33
crates/quicksearch-gui/src/search_tab/tests.rs
View file

@ -2067,6 +2067,39 @@ fn should_arm_waits_for_the_results_to_settle_and_hold_still() {
);
}
/// An idle table asks for no frames. With live results off nothing is ever
/// armed, and past the arm delay the wait for it asked for a frame every
/// frame, forever; the "more below" hint must not animate either. On a
/// software-rendered desktop the two cost a third of a core while idle.
#[test]
fn an_idle_table_with_more_rows_than_fit_asks_for_no_frames() {
let ctx = crate::test_ui::ctx();
let mut tab = tab_with_results(200);
tab.live_enabled = false;
for _ in 0..200 {
run_frame(&ctx, &mut tab, vec![]);
if tab.fade_settled() {
break;
}
}
assert!(tab.fade_settled(), "the reveal never finished");
// The visible rows have stood still for longer than the arm delay.
tab.live_changed_at = Some(Instant::now() - LIVE_ARM_DELAY * 2);
// egui's own layout passes ask for a few frames of their own first.
for _ in 0..8 {
run_frame(&ctx, &mut tab, vec![]);
}
let out = run_frame(&ctx, &mut tab, vec![]);
let delay = out
.viewport_output
.values()
.map(|v| v.repaint_delay)
.min()
.expect("a viewport");
assert_eq!(delay, std::time::Duration::MAX, "an idle table repaints");
}
/// What counts as "already watching these rows": the paths and whether each
/// needs its body re-read, and nothing else.
#[test]

14
crates/quicksearch-gui/src/settings_tab.rs
View file

@ -322,11 +322,13 @@ impl SettingsTab {
// with advanced settings shown.
ui.label(hint(if form.advanced {
"Narrowing a filter removes the entries it excludes; widening \
one reindexes to find what it now allows. Only the tokenizer \
and hash sample size require a full rebuild."
one reindexes to find what it now allows. Only the tokenizer, \
the compact text index and the hash sample size require a \
full rebuild."
} else {
"Narrowing a filter removes the entries it excludes; widening \
one reindexes to find what it now allows."
one reindexes to find what it now allows. Changing the \
compact text index rebuilds the index."
}));
});
crate::ui_util::more_below_hint(ui, &scroll);
@ -880,9 +882,9 @@ fn config_editor_ui(
form.row(
Level::Everyday,
ui,
"Store text for snippets",
&tips::STORE_TEXT,
|ui| ui.checkbox(&mut config.processing.store_text_for_snippets, ""),
"Compact text index",
&tips::COMPACT_INDEX,
|ui| ui.checkbox(&mut config.processing.compact_text_index, ""),
);
});
}

2
crates/quicksearch-gui/src/settings_tab/tests.rs
View file

@ -246,7 +246,7 @@ const ROWS: &[(Section, &str, &tips::Tip, Level)] = &[
(Section::Processing, "Max text file size (bytes)", &tips::MAX_TEXT_FILE_SIZE, Level::Advanced),
(Section::Processing, "Batch size", &tips::BATCH_SIZE, Level::Advanced),
(Section::Processing, "Max WAL size (bytes)", &tips::MAX_WAL_SIZE, Level::Advanced),
(Section::Processing, "Store text for snippets", &tips::STORE_TEXT, Level::Everyday),
(Section::Processing, "Compact text index", &tips::COMPACT_INDEX, Level::Everyday),
(Section::Search, "Fuzzy search ON by default", &tips::FUZZY_DEFAULT, Level::Everyday),
(Section::Search, "Fuzzy edit distance", &tips::FUZZY_EDITS, Level::Advanced),
(Section::Search, "Display limit", &tips::DISPLAY_LIMIT, Level::Advanced),

30
crates/quicksearch-gui/src/tips.rs
View file

@ -235,21 +235,23 @@ pub static MAX_WAL_SIZE: Tip = Tip {
caution: None,
};
pub static STORE_TEXT: Tip = Tip {
title: "Store text for snippets",
body: "Keeps the text read from your files, not just the search index \
built from it. This is what shows a preview line under each \
result.\n\n\
Off, searching inside files still works, but there are no \
previews, no ranking by how often a word appears, no telling \
Report from report, and no typo tolerance inside file contents. \
The index shrinks considerably.\n\n\
Turning it off discards the stored text at once; turning it back \
on reads your files again.",
pub static COMPACT_INDEX: Tip = Tip {
title: "Compact text index",
body: "Builds the index of your files' text without recording where in \
each file every word sits, which makes it several times smaller \
and quicker to build.\n\n\
Searches find exactly the same files either way: QuickSearch checks \
each possible match against the text it keeps, and without \
positions there are more possible matches to check. Searching file \
contents for a word can take up to twice as long, and for a phrase \
of common words, such as \"impl Iterator for\", up to three times.\n\n\
Only the trigram tokenizer can use it. Other tokenizers always \
record positions.",
examples: &[
"off when the index has grown larger than you want and you can do without previews.",
"on, the default: 1.1 GB of source code made a 219 MB index instead of 1.3 GB, built a third faster.",
"off when disk space is plentiful and you often search for phrases of common words.",
],
caution: None,
caution: Some("Changing this rebuilds the index from scratch."),
};
// --- Settings: Search -----------------------------------------------------
@ -643,7 +645,7 @@ mod tests {
&MAX_TEXT_FILE_SIZE,
&BATCH_SIZE,
&MAX_WAL_SIZE,
&STORE_TEXT,
&COMPACT_INDEX,
&FUZZY_DEFAULT,
&FUZZY_EDITS,
&DISPLAY_LIMIT,

12
crates/quicksearch-gui/src/ui_util.rs
View file

@ -197,6 +197,11 @@ pub fn middle_elide<'a>(
/// A semitransparent down-arrow while more content lies below the fold.
/// Painter-only, so it never swallows clicks; the fonts have no ▼ glyph.
///
/// Static on purpose. It used to pulse, which needed a repaint every 50 ms
/// for as long as it showed: on an idle Search tab with more results than
/// rows, that re-laid-out every visible row 20 times a second, measured at
/// 11.8 s of CPU per idle 30 s (100k-file index, 1000 results).
pub fn more_below_hint<R>(ui: &egui::Ui, out: &egui::scroll_area::ScrollAreaOutput<R>) {
let more_below = out.state.offset.y + out.inner_rect.height() < out.content_size.y - 1.0;
if !more_below {
@ -206,12 +211,7 @@ pub fn more_below_hint<R>(ui: &egui::Ui, out: &egui::scroll_area::ScrollAreaOutp
let cx = out.inner_rect.center().x;
let tip = out.inner_rect.bottom() - 5.0;
let (half_width, height) = (7.0, 6.0);
// A slow opacity pulse; repaints only while visible, at a lazy cadence.
let t = ui.ctx().input(|i| i.time);
let pulse = 0.35 + 0.10 * ((t * std::f64::consts::TAU / 2.5).sin() as f32);
let color = ui.visuals().strong_text_color().gamma_multiply(pulse);
ui.ctx()
.request_repaint_after(std::time::Duration::from_millis(50));
let color = ui.visuals().strong_text_color().gamma_multiply(0.35);
painter.add(egui::Shape::convex_polygon(
vec![
egui::pos2(cx - half_width, tip - height),