Removal of positions index from FTS for dramatically reduced index size and improved indexing speed at the cost of slightly slower search of common terms.

DESIGN.md

@@ -21,12 +21,16 @@ memory, memory churn, and indexing and search speed.
 
 The index is a single SQLite database: file metadata, an FTS5 full-text index
 over document contents, and the extracted text itself, stored zstd-compressed
-for snippets and ranking. The database runs in WAL mode with exactly one
+for snippets, ranking and verification. The database runs in WAL mode with exactly one
 writer at a time, so read-only searches are never blocked by indexing. Schema
 changes wipe and rebuild the index rather than migrating: the index is a cache
 of the filesystem, so rebuilding is always safe, and only the indexer may do
 it. FTS5 runs in trigram mode for full-text search and also speeds up regex
-and fuzzy searches. Tuning this database is central to QuickSearch's
+and fuzzy searches. By default it records no word positions: every candidate
+is checked against the stored text anyway, so positions only narrow the
+candidates, and dropping them makes the index several times smaller and
+faster to build, for content searches up to two or three times slower. A
+setting trades the other way. Tuning this database is central to QuickSearch's
 performance.
 
 ## The search path never stalls

config_example.toml

@@ -94,17 +94,20 @@ writer_turn_slice_ms = 100
 # searches that run alongside indexing and lengthens crash recovery. The
 # default favours faster runs.
 maximum_wal_size = 2147483648
-# FTS5 tokenizer: 'trigram' (substring matching, the default; gets
+# FTS5 tokenizer: 'trigram' (substring matching, the default), 'unicode61',
-# remove_diacritics 1 appended), 'unicode61', 'porter', or a full FTS5 option
+# 'porter', or a full FTS5 option string. See
-# string. See https://www.sqlite.org/fts5.html#tokenizers. Changing it
+# https://www.sqlite.org/fts5.html#tokenizers. Matching is exact apart from
-# rebuilds the index.
+# ASCII case: "cafe" does not find "café". Changing it rebuilds the index.
 tokenize = "trigram"
-# Store extracted text (zstd-compressed) alongside the FTS index. Off, the
+# Build the full-text index without word positions. Several times smaller
-# index shrinks considerably, but search loses snippet previews, occurrence
+# and faster to build (1.1 GB of source code: a 219 MiB index instead of
-# ranking, case verification and fuzzy full-text. Turning it off discards
+# 1.3 GB, built a third faster), and searches return exactly the same files,
-# the stored text at once; turning it on re-extracts, because the text of
+# because every candidate is checked against the stored text. The cost is
-# files already indexed was never kept.
+# more candidates to check: searching contents for a word can take up to
-store_text_for_snippets = true
+# twice as long, for a phrase of common words up to three times. Only the
+# trigram tokenizer can use it; other tokenizers always record positions.
+# Changing it rebuilds the index.
+compact_text_index = true
 
 [security]
 # Encrypt the index with a password (SQLCipher). The password is asked for at

crates/quicksearch-core/benches/index.rs

@@ -84,7 +84,7 @@ mod text_pipeline {
         let path = Path::new("/tmp/bench/notes.txt");
         bencher.bench(|| {
             let looks = textenc::looks_like_text(divan::black_box(head));
-            let text = textenc::decode_text(head.to_vec(), path).unwrap();
+            let text = textenc::decode_text(head.to_vec(), path, false).unwrap();
             (looks, text.len())
         });
     }
@@ -101,7 +101,7 @@ mod text_pipeline {
     fn decode_only(bencher: Bencher) {
         let head = corpus::text_head();
         let path = Path::new("/tmp/bench/notes.txt");
-        bencher.bench(|| textenc::decode_text(divan::black_box(head).to_vec(), path));
+        bencher.bench(|| textenc::decode_text(divan::black_box(head).to_vec(), path, false));
     }
 }
 
@@ -167,7 +167,8 @@ mod update_narrowing {
     fn seeded_na(tag: &str) -> (Scratch, Connection) {
         let (dir, p) = Scratch::db(tag);
         let mut conn =
-            quicksearch_core::db::open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
+            quicksearch_core::db::open_or_recreate(p.to_str().unwrap(), &Default::default())
+                .unwrap();
         let tx = conn.transaction().unwrap();
         let names: Vec<String> = (0..ROWS).map(|i| format!("f{:04}.txt", i)).collect();
         for name in &names {
@@ -233,3 +234,54 @@ mod update_narrowing {
         });
     }
 }
+
+/// A watcher event for a directory whose files are all indexed and unchanged:
+/// a re-sent `Create`, a `chmod -R`, the live view re-checking what it shows.
+/// Nothing is written, so the cost is whatever it reads to learn that; the
+/// full walk's answer is one `stat` per file.
+mod unchanged_event {
+    use super::*;
+    use quicksearch_core::config::{Config, IgnoreSet};
+    use quicksearch_core::extract::Registry;
+    use quicksearch_core::incremental::{apply_fs_event, Budget};
+    use quicksearch_core::testutil::Scratch;
+    use quicksearch_core::watcher::FsEvent;
+    use std::sync::atomic::AtomicBool;
+    use std::time::{Duration, Instant};
+
+    const FILES: usize = 500;
+
+    #[divan::bench]
+    fn directory_create(bencher: Bencher) {
+        let (dir, db) = Scratch::db("bench-unchanged-event");
+        let tree = db.with_file_name("tree");
+        std::fs::create_dir_all(&tree).unwrap();
+        for i in 0..FILES {
+            std::fs::write(
+                tree.join(format!("f{:04}.txt", i)),
+                corpus::text(1 << 10, 4),
+            )
+            .unwrap();
+        }
+        let mut conn =
+            quicksearch_core::db::open_or_recreate(db.to_str().unwrap(), &Default::default())
+                .unwrap();
+        let config = Config::default();
+        let ignore = IgnoreSet::compile(&config.indexing.ignore_patterns).unwrap();
+        let registry = Registry::default_set();
+        let event = FsEvent::Create(tree.clone());
+        let cancel = AtomicBool::new(false);
+        let mut apply = move || {
+            let budget = Budget {
+                deadline: Instant::now() + Duration::from_secs(3600),
+                cancel: &cancel,
+                resume_from: 0,
+            };
+            apply_fs_event(&mut conn, &event, &config, &ignore, &registry, &budget).unwrap()
+        };
+        // Indexed once; every timed run finds it unchanged.
+        apply();
+        bencher.bench_local(apply);
+        drop(dir);
+    }
+}

crates/quicksearch-core/benches/index_alloc.rs

@@ -183,7 +183,7 @@ fn compression_traffic_per_chunk(config: &Config) {
 
     let per_row = measure(|| {
         (0..CHUNK)
-            .map(|_| repo::encode_one(&doc, true).expect("encode"))
+            .map(|_| repo::encode_one(&doc).expect("encode"))
             .collect::<Vec<_>>()
     });
     // What `Bodies::fill` does: one encoder and one arena for the chunk.

crates/quicksearch-core/benches/search.rs

@@ -177,10 +177,10 @@ mod substring {
             let (s, b) = snippet::extract_folded(
                 divan::black_box(text),
                 divan::black_box(folded),
-                &[corpus::NEEDLE],
+                corpus::NEEDLE,
                 &opts,
             );
-            let a = pattern.count(text, false);
+            let a = pattern.count(text);
             (a, b, s.ranges.len())
         });
     }
@@ -194,9 +194,9 @@ mod substring {
         let folded = corpus::text_folded(size, 4);
         let opts = snippet::Options { approx_bytes: 600 };
         bencher.bench(|| {
-            let a = pattern.count(divan::black_box(text), false);
+            let a = pattern.count(divan::black_box(text));
             let b = pattern.count_folded(divan::black_box(folded));
-            let (s, _) = snippet::extract_folded(text, folded, &[corpus::NEEDLE], &opts);
+            let (s, _) = snippet::extract_folded(text, folded, corpus::NEEDLE, &opts);
             (a, b, s.ranges.len())
         });
     }
@@ -216,7 +216,7 @@ mod snippet_extract {
             snippet::extract_folded(
                 divan::black_box(text),
                 divan::black_box(folded),
-                &[corpus::NEEDLE],
+                corpus::NEEDLE,
                 &opts,
             )
         });

crates/quicksearch-core/examples/contentprobe.rs

@@ -397,9 +397,7 @@ fn run_stage_with(
             continue;
         }
 
-        // `scope::apply_page`'s decision. `restore_text` is false because this
+        // `scope::apply_page`'s decision.
-        // probe only narrows the filter, so the second arm reduces to "wants
-        // content but has none".
         let at = Instant::now();
         pending.clear();
         for row in &rows {
@@ -463,9 +461,8 @@ fn write_page(tx: &Connection, stage: Stage, pending: &Pending, t: &mut Timing)
         (&pending.all, &pending.all)
     };
 
-    // The per-row clears are `repo::remove_content_for_id` followed by
+    // The per-row clears are `repo::remove_content_for_id` followed by a
-    // `set_state_clearing_failure`'s `failed_files` delete, one bound
+    // `failed_files` delete, one bound statement at a time. They prepare through the connection's cache, as
-    // statement at a time. They prepare through the connection's cache, as
     // `repo::exec` does; preparing afresh per row would price a mistake the
     // product does not make.
     if stage.clears() {

crates/quicksearch-core/examples/pruneprobe.rs

@@ -674,10 +674,10 @@ fn main() {
 
         // The page cache. `PRAGMAS_INCREMENTAL` gives the reconcile a flat
         // 4 MiB for a pass across the whole index. A delete also maintains
-        // `idx_files_mtime`, `idx_files_mime` and `idx_files_hash`, three
+        // `idx_files_hash`, a b-tree in an order uncorrelated with
-        // b-trees in orders uncorrelated with `(parent, name)`, so every
+        // `(parent, name)`, so every doomed row scatters a page touch that a
-        // doomed row scatters ~3 page touches that a 512-page cache cannot
+        // 512-page cache cannot hold (three, before `idx_files_mtime` and
-        // hold. Read the `misses` column. Both commit policies are swept,
+        // `idx_files_mime` were dropped). Read the `misses` column. Both commit policies are swept,
         // because a cache that holds the working set may make commit cadence
         // stop mattering.
         println!("\n  page cache, at +fts(all)  [{} = shipped]:", SHIPPED_MIB);

crates/quicksearch-core/examples/searchtime.rs

@@ -5,8 +5,12 @@
 //! ```text
 //! cargo build -p quicksearch-core --example searchtime --release
 //! ./target/release/examples/searchtime /path/to/index.db
+//! ./target/release/examples/searchtime /path/to/index.db serialize "fn main" ~recieve
 //! ```
 //!
+//! Queries after the path replace the built-in set, for a corpus whose
+//! vocabulary the defaults do not hit; a leading `~` runs one fuzzy.
+//!
 //! Queries run against one held connection, as the worker holds one; each
 //! is timed best-of-N so a scheduling hiccup is not the headline.
 
@@ -21,9 +25,14 @@ use quicksearch_core::testutil;
 const RUNS: u32 = 5;
 
 /// The content queries go through however many segments the write side
-/// left behind; the filename query is the control and must not move.
+/// left behind; the filename query is the control and must not move. The
+/// filter rows price `modified:` and `mime:` against that control: neither
+/// column has an index, so they must cost no more than the scan it already is.
 const QUERIES: &[(&str, bool, &str)] = &[
     ("filename (control)", false, "doc42"),
+    ("filename + modified", false, "doc42 modified:>=2025-06-01"),
+    ("filename + one day", false, "doc42 modified:2025-06-01"),
+    ("filename + mime", false, "doc42 mime:application/pdf"),
     ("content, common", false, "mountain"),
     ("content, rare", false, "quartzite"),
     ("content, two words", false, "ocean forest"),
@@ -55,8 +64,24 @@ fn main() {
     );
     println!("{:<22} {:>10} {:>8}", "query", "best", "hits");
 
+    let custom: Vec<(String, bool, String)> = std::env::args()
+        .skip(2)
+        .map(|q| match q.strip_prefix('~') {
+            Some(term) => (q.clone(), true, term.to_string()),
+            None => (q.clone(), false, q),
+        })
+        .collect();
+    let queries: Vec<(&str, bool, &str)> = if custom.is_empty() {
+        QUERIES.to_vec()
+    } else {
+        custom
+            .iter()
+            .map(|(label, fuzzy, query)| (label.as_str(), *fuzzy, query.as_str()))
+            .collect()
+    };
+
     let mut total = Duration::ZERO;
-    for (label, fuzzy, query) in QUERIES {
+    for &(label, fuzzy, query) in &queries {
         // The display limit makes this unfair: `scan_pass` stops when the
         // limit fills, and candidates stream in insertion order, so which
         // documents drew low ids changes bytes read 13x. A `QSB_LIMIT` past
@@ -66,7 +91,7 @@ fn main() {
             .and_then(|v| v.parse().ok())
             .unwrap_or(1000);
         let options = SearchOptions {
-            fuzzy: *fuzzy,
+            fuzzy,
             limit,
             ..SearchOptions::default()
         };

crates/quicksearch-core/examples/walkprobe.rs

@@ -40,7 +40,7 @@ fn main() {
         std::process::id()
     ));
     let _ = std::fs::remove_file(&db);
-    quicksearch_core::db::open_or_recreate(db.to_str().unwrap(), &config.processing.tokenize)
+    quicksearch_core::db::open_or_recreate(db.to_str().unwrap(), &config.processing)
         .expect("scratch index");
     let existing = DirRows::new();
 

crates/quicksearch-core/src/cli.rs

@@ -197,7 +197,8 @@ pub fn clear_path(db_path: &str, path: &str) -> Result<bool, String> {
 mod tests {
     use super::*;
     use crate::db::open_or_recreate;
-    use crate::db::repo::{insert_file, set_content_done, set_content_failed, NewFile};
+    use crate::db::repo::{complete_pending, insert_file, store_pending_text, NewFile};
+    use crate::file_handling::ContentOutcome;
     use crate::mime::FileType;
     use crate::testutil::{zstd_of, Scratch};
 
@@ -226,13 +227,15 @@ mod tests {
     }
 
     fn seed_fixture(db_path: &str) -> (i64, i64) {
-        let mut conn = open_or_recreate(db_path, "trigram").unwrap();
+        let mut conn = open_or_recreate(db_path, &Default::default()).unwrap();
         let (a, b) = {
             let tx = conn.transaction().unwrap();
             let a = row(&tx, "a.txt", true);
-            set_content_done(&tx, a, "hello", zstd_of("hello").as_deref()).unwrap();
+            store_pending_text(&tx, a, "hello", zstd_of("hello").as_deref()).unwrap();
-            let b = row(&tx, "b.bin", false);
+            // Pending, not NA: only an extraction that ran can fail.
-            set_content_failed(&tx, b, "bad extract").unwrap();
+            let b = row(&tx, "b.bin", true);
+            let failed = ContentOutcome::Failed("bad extract".into());
+            assert!(complete_pending(&tx, b, &failed, None).unwrap());
             tx.commit().unwrap();
             (a, b)
         };
@@ -279,7 +282,7 @@ mod tests {
         let _ = seed_fixture(dbp);
         assert_eq!(pending_content_count(dbp).unwrap(), 0);
 
-        let mut conn = open_or_recreate(dbp, "trigram").unwrap();
+        let mut conn = open_or_recreate(dbp, &Default::default()).unwrap();
         {
             let tx = conn.transaction().unwrap();
             // Claimed by an extractor, text not read yet: the backlog.
@@ -321,12 +324,12 @@ mod tests {
     fn documents_text_ratio_reports_savings_on_compressible_prose() {
         // Guards against silently swapping compression for a pass-through.
         let (_dir, p) = Scratch::db("cli");
-        let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
+        let mut conn = open_or_recreate(p.to_str().unwrap(), &Default::default()).unwrap();
         {
             let tx = conn.transaction().unwrap();
             let id = row(&tx, "big.txt", true);
             let prose = "the quick brown fox jumps over the lazy dog. ".repeat(500);
-            set_content_done(&tx, id, &prose, zstd_of(&prose).as_deref()).unwrap();
+            store_pending_text(&tx, id, &prose, zstd_of(&prose).as_deref()).unwrap();
             tx.commit().unwrap();
         }
         drop(conn);
@@ -359,7 +362,11 @@ mod tests {
         let (_dir, p) = Scratch::db("cli");
         let dbp = p.to_str().unwrap();
         {
-            let mut conn = open_or_recreate(dbp, "unicode61").unwrap();
+            let unicode61 = crate::config::ProcessingConfig {
+                tokenize: "unicode61".into(),
+                ..Default::default()
+            };
+            let mut conn = open_or_recreate(dbp, &unicode61).unwrap();
             let tx = conn.transaction().unwrap();
             for name in ["a.txt", "b.txt"] {
                 row(&tx, name, true);

crates/quicksearch-core/src/config/diff.rs

@@ -30,11 +30,6 @@ pub struct IndexWork {
     /// back to pending; newly excluded ones lose text and FTS but keep their
     /// row.
     pub reconcile_content: bool,
-    /// `store_text_for_snippets` turned on; rows extracted under the old
-    /// setting kept no text, so they must run again.
-    pub restore_text: bool,
-    /// `store_text_for_snippets` turned off.
-    pub drop_text: bool,
     /// Newly in-scope files exist only on disk, so only a walk can find them.
     pub reindex: bool,
 }
@@ -56,8 +51,6 @@ impl IndexWork {
         self.drop_aliases |= other.drop_aliases;
         self.prune_scope |= other.prune_scope;
         self.reconcile_content |= other.reconcile_content;
-        self.restore_text |= other.restore_text;
-        self.drop_text |= other.drop_text;
         self.reindex |= other.reindex;
     }
 
@@ -67,14 +60,12 @@ impl IndexWork {
             || self.drop_aliases
             || self.prune_scope
             || self.reconcile_content
-            || self.restore_text
-            || self.drop_text
     }
 
     /// Whether applying this scans rows under each surviving root, instead
     /// of only deleting whole ranges.
     pub fn scans_rows(&self) -> bool {
-        self.prune_scope || self.reconcile_content || self.restore_text || self.drop_text
+        self.prune_scope || self.reconcile_content
     }
 
     /// The plan in one line for the log, naming what changed.
@@ -88,8 +79,6 @@ impl IndexWork {
                 self.reconcile_content,
                 "changed which files have their text read",
             ),
-            (self.restore_text, "snippet text turned on"),
-            (self.drop_text, "snippet text turned off"),
         ]
         .into_iter()
         .filter_map(|(changed, part)| changed.then_some(part))
@@ -174,12 +163,13 @@ fn ignore_set(c: &Config) -> BTreeSet<&str> {
 pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions {
     let search_db_changed = old.paths.database_path != new.paths.database_path;
 
-    // The only three: a different key makes the file unreadable, the
+    // The only four: a different key makes the file unreadable, the
-    // tokenizer is baked into the FTS table definition, and `hash_length`
+    // tokenizer and the position setting are baked into the FTS table
-    // decides what bytes a stored hash covers. (`use_keychain` changes only
+    // definition, and `hash_length` decides what bytes a stored hash covers.
-    // where the key is remembered.)
+    // (`use_keychain` changes only where the key is remembered.)
     let requires_rebuild = old.processing.hash_length != new.processing.hash_length
         || old.processing.tokenize != new.processing.tokenize
+        || old.processing.fts_positions() != new.processing.fts_positions()
         || old.security.password_protected != new.security.password_protected
         || old.security.salt != new.security.salt;
     if requires_rebuild {
@@ -200,9 +190,6 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions {
     let new_content = content_filter_set(&new_idx.content_extensions);
     let old_cap = old.processing.maximum_text_file_size;
     let new_cap = new.processing.maximum_text_file_size;
-    let old_store = old.processing.store_text_for_snippets;
-    let new_store = new.processing.store_text_for_snippets;
-    let restore_text = !old_store && new_store;
 
     let work = IndexWork {
         drop_roots: old_roots.difference(&new_roots).cloned().collect(),
@@ -215,8 +202,6 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions {
         drop_aliases: old_idx.follow_symlinks && !new_idx.follow_symlinks,
         // See the field: the size cap decides what the extension filter does.
         reconcile_content: old_content != new_content || old_cap != new_cap,
-        restore_text,
-        drop_text: old_store && !new_store,
         // Widening only adds files, and a file absent from the index is not
         // findable from it: only a walk produces those rows. Re-extraction
         // needs a run too, since the content pass runs as part of one; a
@@ -226,8 +211,7 @@ pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions {
             || (!old_idx.include_hidden && new_idx.include_hidden)
             || (!old_idx.follow_symlinks && new_idx.follow_symlinks)
             || filter_widened(&old_content, &new_content)
-            || new_cap > old_cap
+            || new_cap > old_cap,
-            || restore_text,
     };
 
     ConfigActions {

crates/quicksearch-core/src/config/mod.rs

@@ -93,11 +93,30 @@ pub struct ProcessingConfig {
     /// User-facing guidance is in `config_example.toml`.
     pub maximum_wal_size: u64,
     pub tokenize: String,
-    /// When `true` (default), extracted text is stored zstd-compressed in
+    /// When `true` (default), the full-text index keeps no word positions
-    /// `documents_text` for snippet previews. When `false` queries return the
+    /// (FTS5 `detail=none`): a term is looked up as the AND of its trigrams
-    /// same hits, but results carry no snippets, can't be case-verified or
+    /// and the full-text pass drops the documents where they are not
-    /// occurrence-ranked, and fuzzy full-text search is unavailable.
+    /// adjacent, against the stored text. Several times smaller and faster
-    pub store_text_for_snippets: bool,
+    /// to build, at the cost of checking more candidates per search; see
+    /// [`crate::db::schema::fts_create_sql`] for the measurements. Only
+    /// `trigram` can be queried that way; see [`ProcessingConfig::fts_positions`].
+    pub compact_text_index: bool,
+}
+
+impl ProcessingConfig {
+    /// Whether the full-text index is built with word positions. Everything
+    /// that decides the index's layout asks this, never
+    /// [`ProcessingConfig::compact_text_index`] directly: a word tokenizer
+    /// keeps positions whatever that says, because a multi-word phrase has no
+    /// trigram form to fall back on, and toggling it there changes nothing.
+    pub fn fts_positions(&self) -> bool {
+        let trigram = self
+            .tokenize
+            .split_whitespace()
+            .next()
+            .is_some_and(|name| name.eq_ignore_ascii_case("trigram"));
+        !(self.compact_text_index && trigram)
+    }
 }
 
 /// Floor on a non-zero [`ProcessingConfig::maximum_wal_size`]: a checkpoint
@@ -231,7 +250,7 @@ impl Default for ProcessingConfig {
             writer_turn_slice_ms: 100,
             maximum_wal_size: 1024 * 1024 * 1024 * 2,
             tokenize: "trigram".to_string(),
-            store_text_for_snippets: true,
+            compact_text_index: true,
         }
     }
 }

crates/quicksearch-core/src/config/tests.rs

@@ -671,16 +671,41 @@ fn turning_symlinks_off_drops_aliases_and_on_walks() {
     );
 }
 
-/// Stored text: on re-extracts, off drops the blobs; never a rebuild.
+/// The position setting is part of the FTS table definition, so a change
+/// rebuilds; but only a change of the layout it produces, and a word
+/// tokenizer keeps positions either way.
 #[test]
-fn stored_text_on_re_extracts_and_off_drops_it() {
+fn the_compact_index_flag_rebuilds_only_when_the_layout_changes() {
     let base = wide_base();
-    let off = edited(&base, |c| c.processing.store_text_for_snippets = false);
+    let full = edited(&base, |c| c.processing.compact_text_index = false);
-    let on = edited(&base, |c| c.processing.store_text_for_snippets = true);
+    assert!(diff_actions(&base, &full).requires_rebuild);
-    let a = diff_actions(&on, &off);
+    assert!(diff_actions(&full, &base).requires_rebuild);
-    assert!(a.work.drop_text && !a.work.restore_text && !a.work.reindex);
+
-    let a = diff_actions(&off, &on);
+    let words = edited(&base, |c| c.processing.tokenize = "unicode61".into());
-    assert!(a.work.restore_text && !a.work.drop_text && a.work.reindex);
+    let words_full = edited(&words, |c| c.processing.compact_text_index = false);
+    assert_eq!(
+        diff_actions(&words, &words_full),
+        ConfigActions::default(),
+        "positions either way: nothing to do"
+    );
+}
+
+#[test]
+fn only_trigram_can_drop_positions() {
+    let mut p = ProcessingConfig::default();
+    assert!(p.compact_text_index, "compact by default");
+    assert!(!p.fts_positions());
+    p.tokenize = "trigram case_sensitive 1".into();
+    assert!(!p.fts_positions(), "trigram with options is still trigram");
+    p.tokenize = "  TRIGRAM ".into();
+    assert!(!p.fts_positions());
+    for word in ["unicode61", "porter", "porter unicode61", "trigrammer"] {
+        p.tokenize = word.into();
+        assert!(p.fts_positions(), "{word} cannot be queried by trigrams");
+    }
+    p.tokenize = "trigram".into();
+    p.compact_text_index = false;
+    assert!(p.fts_positions());
 }
 
 #[test]
@@ -713,7 +738,7 @@ fn merging_two_plans_loses_nothing() {
     let first = IndexWork {
         drop_roots: vec!["/gone".into(), "/shared".into()],
         prune_scope: true,
-        drop_text: true,
+        drop_aliases: true,
         ..IndexWork::default()
     };
     let second = IndexWork {
@@ -730,7 +755,7 @@ fn merging_two_plans_loses_nothing() {
         vec!["/shared", "/also-gone", "/gone"],
         "every root from both, each once"
     );
-    assert!(merged.prune_scope && merged.drop_text);
+    assert!(merged.prune_scope && merged.drop_aliases);
     assert!(merged.reconcile_content && merged.reindex);
 
     // Empty merge is a no-op and self-merge is the identity; both make a

crates/quicksearch-core/src/content.rs

@@ -331,12 +331,18 @@ mod tests {
 
     use crate::db::open_or_recreate;
     use crate::db::repo::{self, insert_file, NewFile};
-    use crate::file_handling::{extract_scope_prepare, store_extracted, ExtractScope, Stored};
+    use crate::file_handling::{
+        extract_scope_prepare, store_extracted, Bodies, ExtractScope, Stored,
+    };
     use crate::mime::FileType;
     use crate::testutil::Scratch;
     use std::path::PathBuf;
     use std::time::{Duration, Instant};
 
+    fn bodies() -> Bodies {
+        Bodies::new().unwrap()
+    }
+
     fn tree(s: &Scratch) -> PathBuf {
         s.join("tree")
     }
@@ -349,7 +355,7 @@ mod tests {
     /// a pending row for each. The guard owns both.
     fn seed(tag: &str, dirs: &[(&str, usize)]) -> Scratch {
         let s = Scratch::dir(tag);
-        let mut conn = open_or_recreate(db(&s).to_str().unwrap(), "trigram").unwrap();
+        let mut conn = open_or_recreate(db(&s).to_str().unwrap(), &Default::default()).unwrap();
         let tx = conn.transaction().unwrap();
         for (dir, n) in dirs {
             let d = tree(&s).join(dir);
@@ -385,7 +391,7 @@ mod tests {
 
     fn writer(s: &Scratch) -> Arc<Mutex<rusqlite::Connection>> {
         Arc::new(Mutex::new(
-            open_or_recreate(db(s).to_str().unwrap(), "trigram").unwrap(),
+            open_or_recreate(db(s).to_str().unwrap(), &Default::default()).unwrap(),
         ))
     }
 
@@ -422,7 +428,7 @@ mod tests {
         assert!(pass.finish());
         let stop = Arc::new(AtomicBool::new(false));
         let far = Instant::now() + Duration::from_secs(60);
-        store_extracted(conn, &rows, &stop, &Config::default(), far).unwrap()
+        store_extracted(conn, &rows, &stop, far, &mut bodies()).unwrap()
     }
 
     #[test]
@@ -479,15 +485,7 @@ mod tests {
             "out-of-range row untouched"
         );
 
-        let hits: i64 = conn_mutex
+        let hits = crate::testutil::fts_count(&conn_mutex.lock().unwrap(), "sphinx");
-            .lock()
-            .unwrap()
-            .query_row(
-                "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH '\"sphinx\"'",
-                [],
-                |r| r.get(0),
-            )
-            .unwrap();
         assert_eq!(hits, 3);
     }
 
@@ -545,7 +543,6 @@ mod tests {
     fn store_extracted_honours_its_deadline_but_always_makes_progress() {
         let s = seed("deadline", &[("r1", 5)]);
         let conn_mutex = writer(&s);
-        let config = Config::default();
         let mut pass = pass_for(&s, "r1", 2);
         let rows = drain(&mut pass);
         assert!(pass.finish());
@@ -555,7 +552,7 @@ mod tests {
         // A deadline already gone by: one row, then out.
         let past = Instant::now() - Duration::from_secs(1);
         assert_eq!(
-            store_extracted(&conn_mutex, &rows, &stop, &config, past).unwrap(),
+            store_extracted(&conn_mutex, &rows, &stop, past, &mut bodies()).unwrap(),
             Stored {
                 consumed: 1,
                 written: 1
@@ -563,7 +560,7 @@ mod tests {
         );
         let far = Instant::now() + Duration::from_secs(60);
         assert_eq!(
-            store_extracted(&conn_mutex, &rows[1..], &stop, &config, far).unwrap(),
+            store_extracted(&conn_mutex, &rows[1..], &stop, far, &mut bodies()).unwrap(),
             Stored {
                 consumed: 4,
                 written: 4
@@ -583,7 +580,7 @@ mod tests {
         // Stopped before it starts: nothing consumed.
         stop.store(true, std::sync::atomic::Ordering::Relaxed);
         assert_eq!(
-            store_extracted(&conn_mutex, &rows, &stop, &config, far).unwrap(),
+            store_extracted(&conn_mutex, &rows, &stop, far, &mut bodies()).unwrap(),
             Stored::default()
         );
     }

crates/quicksearch-core/src/db/open.rs

@@ -7,10 +7,11 @@ use std::path::Path;
 use rusqlite::{params, Connection, OpenFlags, OptionalExtension};
 
 use super::schema::{
-    effective_tokenizer, fts_create_sql, fts_set_pgsz, pragmas_search,
+    fts_create_sql, fts_set_pgsz, pragmas_search, recommended_search_cache_mib, Profile,
-    recommended_search_cache_mib, Profile, PRAGMAS_FAST, PRAGMAS_INCREMENTAL, PRAGMAS_MAINTENANCE,
+    PRAGMAS_FAST, PRAGMAS_INCREMENTAL, PRAGMAS_MAINTENANCE, PRAGMAS_READONLY, PRAGMAS_WALK_READER,
-    PRAGMAS_READONLY, PRAGMAS_WALK_READER, SCHEMA_CURRENT,
+    SCHEMA_CURRENT,
 };
+use crate::config::ProcessingConfig;
 use crate::security::IndexKey;
 
 /// Prefix tagging every "the key does not fit this file" error, so a wrong
@@ -26,18 +27,25 @@ pub const KEY_MISMATCH_PREFIX: &str = "KEY_MISMATCH: ";
 /// table. Keyed indexes are rebuilt anyway via the profile retry in
 /// [`open_probed`]; the bump rebuilds unprotected ones too.
 ///
+/// v12 dropped the `mtime` and `mime` indexes (see [`schema::SCHEMA_CURRENT`]).
+///
 /// [`schema::HMAC_MODE`]: super::schema::HMAC_MODE
-pub const CURRENT_SCHEMA_VERSION: u32 = 11;
+/// [`schema::SCHEMA_CURRENT`]: super::schema::SCHEMA_CURRENT
+pub const CURRENT_SCHEMA_VERSION: u32 = 12;
 
-/// Open `db_path`; on any schema or tokenizer mismatch, delete the file and
+/// Open `db_path`; on a schema mismatch, or a full-text layout other than
+/// `processing` asks for (tokenizer, positions), delete the file and
 /// recreate it empty, for the caller to re-index.
-pub fn open_or_recreate(db_path: &str, tokenizer: &str) -> Result<Connection, String> {
+pub fn open_or_recreate(
-    open_or_recreate_keyed(db_path, tokenizer, super::key::process_key().as_ref())
+    db_path: &str,
+    processing: &ProcessingConfig,
+) -> Result<Connection, String> {
+    open_or_recreate_keyed(db_path, processing, super::key::process_key().as_ref())
 }
 
 pub(crate) fn open_or_recreate_keyed(
     db_path: &str,
-    tokenizer: &str,
+    processing: &ProcessingConfig,
     key: Option<&IndexKey>,
 ) -> Result<Connection, String> {
     let path = Path::new(db_path).to_path_buf();
@@ -67,7 +75,7 @@ pub(crate) fn open_or_recreate_keyed(
         ProfileMatch::Previous(found) => {
             format!("was built with {} and this build uses {}", found, want)
         }
-        ProfileMatch::Current if db_matches_current(&conn, tokenizer)? => return Ok(conn),
+        ProfileMatch::Current if db_matches_current(&conn, processing)? => return Ok(conn),
         ProfileMatch::Current => "does not match current schema".to_string(),
     };
     crate::log_warn!(
@@ -76,7 +84,7 @@ pub(crate) fn open_or_recreate_keyed(
         why
     );
     let conn = wipe_and_reopen(conn, &path, key, want)?;
-    apply_current_schema(&conn, tokenizer, key, want)?;
+    apply_current_schema(&conn, processing, key, want)?;
     Ok(conn)
 }
 
@@ -491,9 +499,9 @@ fn schema_version_current(conn: &Connection) -> Result<bool, String> {
     Ok(version.as_deref() == Some(&CURRENT_SCHEMA_VERSION.to_string()))
 }
 
-/// True iff the DB has the current schema version *and* the
+/// True iff the DB has the current schema version *and* the full-text layout
-/// effective-tokenizer string this caller asked for.
+/// this caller asked for.
-fn db_matches_current(conn: &Connection, tokenizer: &str) -> Result<bool, String> {
+fn db_matches_current(conn: &Connection, processing: &ProcessingConfig) -> Result<bool, String> {
     if !schema_version_current(conn)? {
         // Refuse rather than wipe unless the file is recognisably ours:
         // `database_path` is free text with no confirmation, and a typo
@@ -504,8 +512,11 @@ fn db_matches_current(conn: &Connection, tokenizer: &str) -> Result<bool, String
         }
         return Ok(false);
     }
-    let want_tokenize = effective_tokenizer(tokenizer);
+    Ok(
-    Ok(stored_info(conn, "tokenize")?.as_deref() == Some(&*want_tokenize))
+        stored_info(conn, "tokenize")?.as_deref() == Some(processing.tokenize.trim())
+            && stored_info(conn, "fts_positions")?.as_deref()
+                == Some(&*processing.fts_positions().to_string()),
+    )
 }
 
 /// Delete the index at `path` and its SQLite sidecars (`-wal`, `-shm`,
@@ -557,13 +568,14 @@ fn wipe_and_reopen(
 
 fn apply_current_schema(
     conn: &Connection,
-    tokenizer: &str,
+    processing: &ProcessingConfig,
     key: Option<&IndexKey>,
     profile: Profile,
 ) -> Result<(), String> {
     conn.execute_batch(SCHEMA_CURRENT)
         .map_err(|e| format!("Failed to create current schema tables: {}", e))?;
-    let fts = fts_create_sql(tokenizer);
+    let positions = processing.fts_positions();
+    let fts = fts_create_sql(&processing.tokenize, positions);
     conn.execute_batch(&fts)
         .map_err(|e| format!("Failed to create searchabletext: {}", e))?;
     // Only here: FTS5's leaf size must suit this file's page size and
@@ -573,13 +585,16 @@ fn apply_current_schema(
         .map_err(|e| format!("Failed to set searchabletext pgsz: {}", e))?;
 
     let now = crate::log::now_unix();
-    let effective = effective_tokenizer(tokenizer);
+    // `fts_positions` is read back by every search (`repo::fts_positions`),
+    // which must phrase its queries for the table as it was built.
     conn.execute(
-        "INSERT INTO schema_info(key, value) VALUES ('version', ?1), ('created_at', ?2), ('tokenize', ?3)",
+        "INSERT INTO schema_info(key, value) VALUES ('version', ?1), ('created_at', ?2), \
+         ('tokenize', ?3), ('fts_positions', ?4)",
         params![
             CURRENT_SCHEMA_VERSION.to_string(),
             now.to_string(),
-            effective
+            processing.tokenize.trim(),
+            positions.to_string()
         ],
     )
     .map_err(|e| format!("Failed to seed schema_info: {}", e))?;

crates/quicksearch-core/src/db/open_tests.rs

@@ -15,6 +15,24 @@ fn info(conn: &Connection, key: &str) -> String {
     .unwrap()
 }
 
+/// Processing settings asking for `tokenize`, the rest default.
+fn with_tokenizer(tokenize: &str) -> ProcessingConfig {
+    ProcessingConfig {
+        tokenize: tokenize.into(),
+        ..Default::default()
+    }
+}
+
+/// The DDL the full-text table was created with.
+fn fts_sql(conn: &Connection) -> String {
+    conn.query_row(
+        "SELECT sql FROM sqlite_master WHERE name = 'searchabletext'",
+        [],
+        |r| r.get(0),
+    )
+    .unwrap()
+}
+
 fn file_count(conn: &Connection) -> i64 {
     conn.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
         .unwrap()
@@ -42,15 +60,15 @@ fn write_v1_index(p: &str, name: &str) {
 #[test]
 fn fresh_db_gets_current_version() {
     let (_dir, p) = db();
-    let conn = open_or_recreate(&p, "trigram").unwrap();
+    let conn = open_or_recreate(&p, &Default::default()).unwrap();
     assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string());
 }
 
 #[test]
 fn reopen_is_idempotent() {
     let (_dir, p) = db();
-    drop(open_or_recreate(&p, "trigram").unwrap());
+    drop(open_or_recreate(&p, &Default::default()).unwrap());
-    let conn = open_or_recreate(&p, "trigram").unwrap();
+    let conn = open_or_recreate(&p, &Default::default()).unwrap();
     assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string());
 }
 
@@ -58,7 +76,7 @@ fn reopen_is_idempotent() {
 fn older_versioned_db_is_wiped_and_recreated() {
     let (_dir, p) = db();
     write_v1_index(&p, "a.txt");
-    let conn = open_or_recreate(&p, "trigram").unwrap();
+    let conn = open_or_recreate(&p, &Default::default()).unwrap();
     assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string());
     assert_eq!(file_count(&conn), 0, "old rows should be wiped");
 }
@@ -79,7 +97,7 @@ fn legacy_layout_db_is_wiped_and_recreated() {
         )
         .unwrap();
     }
-    let conn = open_or_recreate(&p, "trigram").unwrap();
+    let conn = open_or_recreate(&p, &Default::default()).unwrap();
     assert_eq!(file_count(&conn), 0);
     // An unknown column name fails to prepare.
     conn.prepare("SELECT content_state, type, mime FROM files LIMIT 0")
@@ -90,7 +108,7 @@ fn legacy_layout_db_is_wiped_and_recreated() {
 fn tokenizer_drift_wipes_db() {
     let (_dir, p) = db();
     let first_effective = {
-        let conn = open_or_recreate(&p, "trigram").unwrap();
+        let conn = open_or_recreate(&p, &Default::default()).unwrap();
         conn.execute(
             "INSERT INTO files (name, parent, size, mtime) \
              VALUES ('x', '/', 0, 0)",
@@ -99,16 +117,77 @@ fn tokenizer_drift_wipes_db() {
         .unwrap();
         info(&conn, "tokenize")
     };
-    let conn = open_or_recreate(&p, "unicode61").unwrap();
+    let conn = open_or_recreate(&p, &with_tokenizer("unicode61")).unwrap();
     assert_eq!(file_count(&conn), 0, "tokenizer drift should wipe rows");
     assert_ne!(first_effective, info(&conn, "tokenize"));
 }
 
+/// The position setting is part of the table definition, like the
+/// tokenizer: a change wipes, and the layout built is the one recorded,
+/// which is what every search reads back.
+#[test]
+fn position_drift_wipes_db_and_the_layout_is_recorded() {
+    let (_dir, p) = db();
+    {
+        let conn = open_or_recreate(&p, &Default::default()).unwrap();
+        assert!(fts_sql(&conn).contains("detail=none"), "compact by default");
+        assert_eq!(info(&conn, "fts_positions"), "false");
+        assert!(!crate::db::repo::fts_positions(&conn));
+        conn.execute(
+            "INSERT INTO files (name, parent, size, mtime) VALUES ('x', '/', 0, 0)",
+            [],
+        )
+        .unwrap();
+    }
+    let full = ProcessingConfig {
+        compact_text_index: false,
+        ..Default::default()
+    };
+    let conn = open_or_recreate(&p, &full).unwrap();
+    assert_eq!(file_count(&conn), 0, "a layout change wipes the rows");
+    assert!(!fts_sql(&conn).contains("detail"));
+    assert_eq!(info(&conn, "fts_positions"), "true");
+    assert!(crate::db::repo::fts_positions(&conn));
+}
+
+/// A word tokenizer cannot be queried by trigrams, so it keeps positions
+/// whatever the flag says, and the flag alone then changes nothing.
+#[test]
+fn a_word_tokenizer_keeps_positions_and_ignores_the_flag() {
+    let (_dir, p) = db();
+    {
+        let conn = open_or_recreate(&p, &with_tokenizer("unicode61")).unwrap();
+        assert!(!fts_sql(&conn).contains("detail"));
+        assert_eq!(info(&conn, "fts_positions"), "true");
+        conn.execute(
+            "INSERT INTO files (name, parent, size, mtime) VALUES ('x', '/', 0, 0)",
+            [],
+        )
+        .unwrap();
+    }
+    let flag_off = ProcessingConfig {
+        compact_text_index: false,
+        ..with_tokenizer("unicode61")
+    };
+    let conn = open_or_recreate(&p, &flag_off).unwrap();
+    assert_eq!(file_count(&conn), 1, "the same layout: nothing to rebuild");
+}
+
+/// An index from before the layout was recorded kept positions.
+#[test]
+fn an_unrecorded_layout_reads_as_positions() {
+    let (_dir, p) = db();
+    let conn = open_or_recreate(&p, &Default::default()).unwrap();
+    conn.execute("DELETE FROM schema_info WHERE key = 'fts_positions'", [])
+        .unwrap();
+    assert!(crate::db::repo::fts_positions(&conn));
+}
+
 #[test]
 fn open_existing_reads_nondefault_tokenizer_without_wiping() {
     let (_dir, p) = db();
     {
-        let conn = open_or_recreate(&p, "unicode61").unwrap();
+        let conn = open_or_recreate(&p, &with_tokenizer("unicode61")).unwrap();
         conn.execute(
             "INSERT INTO files (name, parent, size, mtime) \
              VALUES ('note', '/', 0, 0)",
@@ -129,6 +208,7 @@ fn open_existing_reads_nondefault_tokenizer_without_wiping() {
         1,
         "open_existing must not wipe a non-default-tokenizer DB"
     );
+    // A word tokenizer: `hello` is one token, so a bare MATCH is exact.
     let hits: i64 = conn
         .query_row(
             "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'hello'",
@@ -145,7 +225,7 @@ fn open_existing_reads_nondefault_tokenizer_without_wiping() {
 fn open_or_recreate_creates_missing_parent_dirs() {
     let dir = Scratch::dir("mkdir");
     let db = dir.join("nested/deeper/index.sqlite");
-    drop(open_or_recreate(db.to_str().unwrap(), "trigram").unwrap());
+    drop(open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap());
     assert!(db.exists());
 }
 
@@ -156,7 +236,7 @@ fn writable_opens_use_wal_and_it_persists() {
         conn.query_row("PRAGMA journal_mode", [], |r| r.get(0))
             .unwrap()
     };
-    let conn = open_or_recreate(&p, "trigram").unwrap();
+    let conn = open_or_recreate(&p, &Default::default()).unwrap();
     assert_eq!(mode(&conn).to_lowercase(), "wal");
     drop(conn);
     // WAL is persistent in the file: a later read-only consumer sees it.
@@ -173,7 +253,7 @@ fn maintenance_opens_keep_temporaries_on_disk() {
         conn.query_row("PRAGMA temp_store", [], |r| r.get(0))
             .unwrap()
     };
-    let conn = open_or_recreate(&p, "trigram").unwrap();
+    let conn = open_or_recreate(&p, &Default::default()).unwrap();
     assert_eq!(temp_store(&conn), 2, "the indexer's own profile is MEMORY");
     drop(conn);
 
@@ -211,13 +291,13 @@ fn index_needs_rebuild_only_when_the_schema_really_differs() {
     let cases: [(Fixture, bool, &str); 5] = [
         (|_| {}, false, "no file yet is a fresh install, not a reset"),
         (
-            |p| drop(open_or_recreate(p, "trigram").unwrap()),
+            |p| drop(open_or_recreate(p, &Default::default()).unwrap()),
             false,
             "a current index is not going to be rebuilt",
         ),
         (
             |p| {
-                drop(open_or_recreate(p, "trigram").unwrap());
+                drop(open_or_recreate(p, &Default::default()).unwrap());
                 open_existing(p, true)
                     .unwrap()
                     .execute(
@@ -289,7 +369,7 @@ fn keyed_create_reopen_and_header_is_encrypted() {
     let (_dir, p) = db();
     let key = test_key(0xa1);
     {
-        let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap();
+        let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&key)).unwrap();
         conn.execute(
             "INSERT INTO files (name, parent, size, mtime) \
              VALUES ('secret', '/', 0, 0)",
@@ -300,7 +380,7 @@ fn keyed_create_reopen_and_header_is_encrypted() {
     let head = &file_bytes(&p)[..16];
     assert_ne!(head, b"SQLite format 3\0", "file must not be plaintext");
 
-    let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap();
+    let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&key)).unwrap();
     assert_eq!(
         file_count(&conn),
         1,
@@ -315,7 +395,7 @@ fn keyed_create_reopen_and_header_is_encrypted() {
 fn wrong_key_errors_without_wiping() {
     let (_dir, p) = db();
     {
-        let conn = open_or_recreate_keyed(&p, "trigram", Some(&test_key(0xa1))).unwrap();
+        let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&test_key(0xa1))).unwrap();
         conn.execute(
             "INSERT INTO files (name, parent, size, mtime) \
              VALUES ('x', '/', 0, 0)",
@@ -329,7 +409,7 @@ fn wrong_key_errors_without_wiping() {
         assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}");
     }
     // A wrong key is never a "schema mismatch" to answer with a wipe.
-    let err = open_or_recreate_keyed(&p, "trigram", Some(&test_key(0xb2))).unwrap_err();
+    let err = open_or_recreate_keyed(&p, &Default::default(), Some(&test_key(0xb2))).unwrap_err();
     assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}");
     assert_eq!(before, file_bytes(&p), "file must be byte-identical");
 }
@@ -337,12 +417,12 @@ fn wrong_key_errors_without_wiping() {
 #[test]
 fn missing_key_on_encrypted_db_errors_without_wiping() {
     let (_dir, p) = db();
-    drop(open_or_recreate_keyed(&p, "trigram", Some(&test_key(0xa1))).unwrap());
+    drop(open_or_recreate_keyed(&p, &Default::default(), Some(&test_key(0xa1))).unwrap());
     let before = file_bytes(&p);
     let err = open_existing_keyed(&p, false, None).unwrap_err();
     assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}");
     assert!(err.contains("password-protected"), "got: {err}");
-    let err = open_or_recreate_keyed(&p, "trigram", None).unwrap_err();
+    let err = open_or_recreate_keyed(&p, &Default::default(), None).unwrap_err();
     assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}");
     assert_eq!(before, file_bytes(&p));
 }
@@ -350,13 +430,13 @@ fn missing_key_on_encrypted_db_errors_without_wiping() {
 #[test]
 fn key_on_plaintext_db_errors_without_wiping() {
     let (_dir, p) = db();
-    drop(open_or_recreate(&p, "trigram").unwrap());
+    drop(open_or_recreate(&p, &Default::default()).unwrap());
     let before = file_bytes(&p);
     let err = open_existing_keyed(&p, false, Some(&test_key(0xa1))).unwrap_err();
     assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}");
     // The sniffed plaintext header yields the precise diagnosis.
     assert!(err.contains("not encrypted"), "got: {err}");
-    let err = open_or_recreate_keyed(&p, "trigram", Some(&test_key(0xa1))).unwrap_err();
+    let err = open_or_recreate_keyed(&p, &Default::default(), Some(&test_key(0xa1))).unwrap_err();
     assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}");
     assert_eq!(before, file_bytes(&p));
 }
@@ -377,7 +457,7 @@ fn schema_mismatch_under_key_wipes_and_recreates_encrypted() {
         key_and_probe(&conn, &p, Some(&key), previous).unwrap();
         write_v1_schema_info(&conn);
     }
-    let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap();
+    let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&key)).unwrap();
     assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string());
     drop(conn);
     let head = &file_bytes(&p)[..16];
@@ -401,7 +481,7 @@ fn garbage_file_with_key_reports_mismatch_not_corruption() {
 fn open_existing_rw_allows_delete() {
     let (_dir, p) = db();
     {
-        let conn = open_or_recreate(&p, "trigram").unwrap();
+        let conn = open_or_recreate(&p, &Default::default()).unwrap();
         conn.execute(
             "INSERT INTO files (name, parent, size, mtime) \
              VALUES ('a', '/', 0, 0)",
@@ -428,7 +508,7 @@ fn a_fresh_index_and_its_sidecars_are_owner_only() {
     // test: an existing directory keeps whatever mode its owner chose.
     let dir = Scratch::dir("open");
     let p = dir.join("data").join("index.sqlite");
-    let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
+    let conn = open_or_recreate(p.to_str().unwrap(), &Default::default()).unwrap();
     conn.execute(
         "INSERT INTO files (name, parent, size, mtime) \
          VALUES ('a', '/', 0, 0)",
@@ -462,7 +542,7 @@ fn maintain_reads_its_pragmas_on_a_keyed_index() {
     let (_dir, p) = db();
     let key = test_key(0xc3);
     {
-        let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap();
+        let conn = open_or_recreate_keyed(&p, &Default::default(), Some(&key)).unwrap();
         conn.execute(
             "INSERT INTO files (name, parent, size, mtime) VALUES ('x', '/', 0, 0)",
             [],
@@ -484,6 +564,16 @@ fn maintain_reads_its_pragmas_on_a_keyed_index() {
     );
 }
 
+/// A layout with positions: `pgsz` caps a leaf the same in either layout,
+/// and position lists fill leaves soonest, so the geometry tests below need
+/// the smallest corpus this way.
+fn positional() -> ProcessingConfig {
+    ProcessingConfig {
+        compact_text_index: false,
+        ..Default::default()
+    }
+}
+
 /// Write enough text that FTS5 emits several *full* leaves. A corpus of tiny
 /// documents fits in one part-filled leaf, never reaches the inline-payload
 /// limit, and would let the tests below pass on a broken geometry.
@@ -565,7 +655,7 @@ fn assert_leaves_inline(conn: &Connection, keyed: bool) {
 fn keyed_fts_leaves_stay_inline() {
     let (_dir, p) = db();
     let key = test_key(0xd4);
-    let conn = open_or_recreate_keyed(&p, "trigram", Some(&key)).unwrap();
+    let conn = open_or_recreate_keyed(&p, &positional(), Some(&key)).unwrap();
     assert_eq!(
         fts_stored_pgsz(&conn),
         Some(crate::db::schema::fts_pgsz_for(
@@ -638,7 +728,7 @@ fn derived_fts_pgsz_fits_inline_at_every_swept_page_size() {
 fn a_plain_index_gets_the_derived_pgsz_for_its_page_size() {
     use crate::db::schema::{fts_pgsz_for, PROFILE};
     let (_dir, p) = db();
-    let conn = open_or_recreate(&p, "trigram").unwrap();
+    let conn = open_or_recreate(&p, &positional()).unwrap();
     assert_eq!(
         fts_stored_pgsz(&conn),
         Some(fts_pgsz_for(PROFILE, false)),
@@ -672,7 +762,7 @@ fn a_foreign_database_is_refused_not_wiped() {
         )
         .unwrap();
     }
-    let err = open_or_recreate(&p, "trigram").unwrap_err();
+    let err = open_or_recreate(&p, &Default::default()).unwrap_err();
     assert!(err.starts_with(FOREIGN_DB_PREFIX), "got: {err}");
     assert!(
         err.contains("moz_places"),
@@ -692,7 +782,7 @@ fn an_empty_database_file_is_still_ours_to_build() {
     let (_dir, p) = db();
     // An empty but real SQLite file, header and all.
     drop(Connection::open(&p).unwrap());
-    let conn = open_or_recreate(&p, "trigram").unwrap();
+    let conn = open_or_recreate(&p, &Default::default()).unwrap();
     assert_eq!(info(&conn, "version"), CURRENT_SCHEMA_VERSION.to_string());
 }
 
@@ -706,7 +796,7 @@ fn a_legacy_table_marks_a_file_as_ours() {
             .unwrap()
             .execute(&format!("CREATE TABLE {} (x INTEGER)", table), [])
             .unwrap();
-        open_or_recreate(&p, "trigram")
+        open_or_recreate(&p, &Default::default())
             .unwrap_or_else(|e| panic!("{table} should read as ours, got: {e}"));
     }
 }
@@ -723,7 +813,7 @@ fn a_schema_info_without_a_version_row_is_still_ours() {
             [],
         )
         .unwrap();
-    open_or_recreate(&p, "trigram").expect("half-built index is ours");
+    open_or_recreate(&p, &Default::default()).expect("half-built index is ours");
 }
 
 /// A table that merely *shares the name* `schema_info` is not ours.
@@ -737,7 +827,7 @@ fn a_foreign_schema_info_is_refused() {
         conn.execute("INSERT INTO schema_info(revision) VALUES(3)", [])
             .unwrap();
     }
-    let err = open_or_recreate(&p, "trigram").unwrap_err();
+    let err = open_or_recreate(&p, &Default::default()).unwrap_err();
     assert!(err.starts_with(FOREIGN_DB_PREFIX), "got: {err}");
 
     let conn = Connection::open(&p).unwrap();

crates/quicksearch-core/src/db/repo.rs

@@ -3,7 +3,7 @@
 
 use rusqlite::{params, params_from_iter, Connection, OptionalExtension, Transaction};
 
-use crate::file_handling::{ExtractCursor, RootRange};
+use crate::file_handling::{ContentOutcome, ExtractCursor, RootRange};
 use crate::mime::FileType;
 
 pub const STATE_PENDING: i64 = 0;
@@ -27,30 +27,6 @@ fn exec(
         .map_err(|e| format!("{}: {}", what(), e))
 }
 
-/// Set a file's content state and clear any failure record with it:
-/// `list-failed` reads `failed_files` directly, so a stale entry keeps
-/// reporting a file broken. [`set_content_failed`] does not use this.
-fn set_state_clearing_failure(
-    tx: &Transaction<'_>,
-    file_id: i64,
-    state: i64,
-    transition: &'static str,
-) -> Result<(), String> {
-    exec(
-        tx,
-        "UPDATE files SET content_state = ?1 WHERE id = ?2",
-        params![state, file_id],
-        || format!("{} content_state {}", transition, file_id),
-    )?;
-    exec(
-        tx,
-        "DELETE FROM failed_files WHERE file_id = ?1",
-        params![file_id],
-        || format!("clear failed_files {}", file_id),
-    )?;
-    Ok(())
-}
-
 #[derive(Debug, Clone)]
 pub struct NewFile<'a> {
     pub name: &'a str,
@@ -154,9 +130,8 @@ pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option
     }
     let new_state = initial_content_state(f);
     if old_state != new_state {
-        // A plain assignment, not `set_state_clearing_failure`: the FAILED
+        // The FAILED arm above already took the failure record, and no
-        // arm above already took the failure record, and no other state can
+        // other state can hold one.
-        // hold one.
         exec(
             tx,
             "UPDATE files SET content_state = ?1 WHERE id = ?2",
@@ -167,29 +142,64 @@ pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option
     Ok(Some(id))
 }
 
-/// Mark a file's content indexing as complete and write the extracted text
+/// Move a [`STATE_PENDING`] row to `state`; `false`, writing nothing, when
-/// atomically. `text_zstd` is the pre-compressed `documents_text` sidecar
+/// the row is no longer pending (or no longer exists).
-/// body, or `None` for no sidecar. Compression is the caller's job: this runs
+fn claim_pending(tx: &Transaction<'_>, file_id: i64, state: i64) -> Result<bool, String> {
-/// inside the writer's transaction, so compress before taking the lock.
+    let changed = exec(
-pub fn set_content_done(
+        tx,
-    tx: &Transaction<'_>,
+        "UPDATE files SET content_state = ?1 WHERE id = ?2 AND content_state = ?3",
-    file_id: i64,
+        params![state, file_id, STATE_PENDING],
-    text: &str,
+        || format!("set content_state {} on {}", state, file_id),
-    text_zstd: Option<&[u8]>,
+    )?;
-) -> Result<(), String> {
+    Ok(changed > 0)
-    remove_content_for_id(tx, file_id)?;
-    set_content_done_fresh(tx, file_id, text, text_zstd)
 }
 
-/// [`set_content_done`] for a row that **provably holds no content yet**,
+/// Record the outcome of extracting a pending row: the one way content state
-/// skipping the pre-delete. Use it only where the same transaction has
+/// leaves [`STATE_PENDING`]. `text_zstd` is the pre-compressed
-/// established that; a wrong call leaves a duplicate FTS entry, not an error.
+/// `documents_text` body for a `Done` outcome, or `None` for no sidecar;
-pub fn set_content_done_fresh(
+/// compress before taking the writer's lock. Returns whether the row was
+/// still pending, and so whether anything was written.
+///
+/// Pending is also what makes it cheap: a pending row provably holds no
+/// posting, stored body or failure record (see [`STATE_DONE`] and
+/// [`STATE_FAILED`]), so nothing is cleared first. A row that stopped being
+/// pending while its extraction was in flight (stored by another route, or
+/// deleted by stale cleanup) is left alone: writing it anyway would leave a
+/// posting with no `files` row behind it, which a reused rowid inherits.
+pub fn complete_pending(
+    tx: &Transaction<'_>,
+    file_id: i64,
+    outcome: &ContentOutcome,
+    text_zstd: Option<&[u8]>,
+) -> Result<bool, String> {
+    match outcome {
+        ContentOutcome::Done { text } => store_pending_text(tx, file_id, text, text_zstd),
+        ContentOutcome::NotApplicable => claim_pending(tx, file_id, STATE_NA),
+        ContentOutcome::Failed(reason) => {
+            if !claim_pending(tx, file_id, STATE_FAILED)? {
+                return Ok(false);
+            }
+            exec(
+                tx,
+                "INSERT INTO failed_files(file_id, reason, ts) VALUES (?1, ?2, ?3)",
+                params![file_id, reason, crate::log::now_unix() as i64],
+                || format!("insert failed_files {}", file_id),
+            )?;
+            Ok(true)
+        }
+    }
+}
+
+/// [`complete_pending`] with extracted text, for callers holding it borrowed.
+pub fn store_pending_text(
     tx: &Transaction<'_>,
     file_id: i64,
     text: &str,
     text_zstd: Option<&[u8]>,
-) -> Result<(), String> {
+) -> Result<bool, String> {
+    if !claim_pending(tx, file_id, STATE_DONE)? {
+        return Ok(false);
+    }
     // Contentless FTS5 takes the text on INSERT for the tokenizer and does
     // not store it.
     exec(
@@ -198,7 +208,6 @@ pub fn set_content_done_fresh(
         params![file_id, text],
         || format!("insert FTS row {}", file_id),
     )?;
-
     if let Some(compressed) = text_zstd {
         exec(
             tx,
@@ -207,8 +216,7 @@ pub fn set_content_done_fresh(
             || format!("insert documents_text {}", file_id),
         )?;
     }
-
+    Ok(true)
-    set_state_clearing_failure(tx, file_id, STATE_DONE, "update DONE")
 }
 
 /// Reusable decode buffer and context for the readers of `documents_text`;
@@ -313,9 +321,10 @@ impl DocEncoder {
     }
 }
 
-/// Compress one body, for the writers that handle a single row.
+/// Compress one body, for the writers that handle a single row; `None` for
-pub fn encode_one(text: &str, store_text: bool) -> Result<Option<Vec<u8>>, String> {
+/// an empty one, which gets no `documents_text` row.
-    if !store_text || text.is_empty() {
+pub fn encode_one(text: &str) -> Result<Option<Vec<u8>>, String> {
+    if text.is_empty() {
         return Ok(None);
     }
     DocEncoder::new()?.encode(text).map(Some)
@@ -327,38 +336,6 @@ pub fn raw_text_len(blob: &[u8]) -> Option<u64> {
     zstd::zstd_safe::get_frame_content_size(blob).ok().flatten()
 }
 
-/// Mark a file's content extraction as failed, keeping the row.
-///
-/// Clears any posting and stored body first, so text from an earlier version
-/// of the file stops matching and [`STATE_DONE`]'s invariant holds. Callers
-/// usually reach here on a row that is already clear, so the deletes are
-/// normally no-ops.
-pub fn set_content_failed(tx: &Transaction<'_>, file_id: i64, reason: &str) -> Result<(), String> {
-    remove_content_for_id(tx, file_id)?;
-    let now = crate::log::now_unix() as i64;
-    exec(
-        tx,
-        "UPDATE files SET content_state = ?1 WHERE id = ?2",
-        params![STATE_FAILED, file_id],
-        || format!("update content_state FAILED {}", file_id),
-    )?;
-    exec(
-        tx,
-        "INSERT OR REPLACE INTO failed_files(file_id, reason, ts) VALUES (?1, ?2, ?3)",
-        params![file_id, reason, now],
-        || format!("insert failed_files {}", file_id),
-    )?;
-    Ok(())
-}
-
-/// Mark content extraction as not applicable; the row still serves filename
-/// search. Clears any posting and stored body first, as
-/// [`set_content_failed`] does.
-pub fn set_content_na(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> {
-    remove_content_for_id(tx, file_id)?;
-    set_state_clearing_failure(tx, file_id, STATE_NA, "update NA")
-}
-
 /// Delete a file row by path, keeping FTS in sync; returns whether a row went.
 pub fn delete_file_by_path(tx: &Transaction<'_>, path: &str) -> Result<bool, String> {
     let Some((parent, name)) = crate::file_handling::split_db_path(path) else {
@@ -504,10 +481,9 @@ pub fn delete_ids(
 
 /// Set `content_state` on many rows at once, leaving everything else alone.
 ///
-/// The batch form of [`set_state_clearing_failure`]'s `UPDATE`, without its
+/// No `failed_files` sweep: a caller that knows the stored states clears the
-/// `failed_files` sweep: a caller that knows the stored states clears the few
+/// few [`STATE_FAILED`] rows with [`clear_failed_for_ids`], and their content
-/// [`STATE_FAILED`] rows with [`clear_failed_for_ids`]. A caller that does not
+/// with [`clear_content_for_ids`].
-/// know them must use the per-row helpers.
 pub fn set_content_state(tx: &Transaction<'_>, ids: &[i64], state: i64) -> Result<usize, String> {
     let mut changed = 0;
     for chunk in ids.chunks(DELETE_IDS_CHUNK) {
@@ -551,9 +527,8 @@ pub fn clear_content_for_ids(tx: &Transaction<'_>, ids: &[i64]) -> Result<(), St
     Ok(())
 }
 
-/// Forget the failure records of many rows at once: the batch half of
+/// Forget the failure records of many rows at once. Pass only
-/// [`set_state_clearing_failure`]. Pass only [`STATE_FAILED`] ids; nothing
+/// [`STATE_FAILED`] ids; nothing else holds a record.
-/// else holds a record.
 pub fn clear_failed_for_ids(tx: &Transaction<'_>, ids: &[i64]) -> Result<(), String> {
     exec_chunked(
         tx,
@@ -586,7 +561,7 @@ pub fn dir_rows(
 /// A stored path that remembers where its `name` begins: `parent + name` in
 /// one buffer instead of two or three strings. The content pass moves one per
 /// file from the feeder to the writer.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct RowPath {
     path: String,
     name_at: usize,
@@ -612,6 +587,11 @@ impl RowPath {
         // `name_at` is the parent's length, so it is a char boundary.
         &self.path[self.name_at..]
     }
+
+    /// The `files.parent` half, trailing separator included.
+    pub fn parent(&self) -> &str {
+        &self.path[..self.name_at]
+    }
 }
 
 /// One row the content pass has yet to extract.
@@ -757,18 +737,6 @@ pub fn rows_in_range_page(
         .map_err(|e| format!("read range page row: {}", e))
 }
 
-/// Drop the stored text of the given file ids, leaving their FTS row and
-/// `files` row intact: full-text search keeps working, only the snippet
-/// source goes away.
-pub fn drop_stored_text(tx: &Transaction<'_>, ids: &[i64]) -> Result<usize, String> {
-    exec_chunked(
-        tx,
-        "DELETE FROM documents_text WHERE file_id IN (",
-        ids,
-        |n| format!("drop stored text for {} ids", n),
-    )
-}
-
 /// Reset every failed file to pending and drop the failure records, so the
 /// coming run re-attempts them. Returns how many rows were reset.
 pub fn retry_failed_files(tx: &Transaction<'_>) -> Result<usize, String> {
@@ -1092,6 +1060,15 @@ fn set_info(conn: &Connection, key: &str, value: &str) -> rusqlite::Result<usize
     )
 }
 
+/// Whether the full-text index was built with positions, which decides how
+/// a search may phrase its `MATCH` (`search::prefilter::fts_literal`). Read
+/// from the index rather than the config, which may already ask for the
+/// other layout before the rebuild that brings it. An index that never
+/// recorded it predates the compact layout, so kept positions.
+pub fn fts_positions(conn: &Connection) -> bool {
+    get_info(conn, "fts_positions").is_none_or(|v| v != "false")
+}
+
 /// Unix seconds of the last *successful* full indexing run; `None` means
 /// "never".
 pub fn get_last_full_index(conn: &Connection) -> Option<u64> {

crates/quicksearch-core/src/db/repo_tests.rs

@@ -9,7 +9,7 @@ use crate::testutil::{zstd_of, Scratch};
 fn fresh() -> (Scratch, String, Connection) {
     let (dir, path) = Scratch::db("repo");
     let path = path.to_str().unwrap().to_string();
-    let conn = open_or_recreate(&path, "trigram").unwrap();
+    let conn = open_or_recreate(&path, &Default::default()).unwrap();
     (dir, path, conn)
 }
 
@@ -76,14 +76,9 @@ fn fts_rows(conn: &Connection) -> i64 {
     count(conn, "SELECT COUNT(*) FROM searchabletext")
 }
 
-/// Rows whose body matches the FTS query `term`.
+/// Rows whose body the FTS index matches for `term`.
 fn matches(conn: &Connection, term: &str) -> i64 {
-    conn.query_row(
+    crate::testutil::fts_count(conn, term)
-        "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH ?1",
-        [term],
-        |r| r.get(0),
-    )
-    .unwrap()
 }
 
 fn state(conn: &Connection, id: i64) -> i64 {
@@ -93,6 +88,22 @@ fn state(conn: &Connection, id: i64) -> i64 {
     .unwrap()
 }
 
+/// Store `text` on a pending row, which must still be pending.
+fn done(tx: &Transaction<'_>, id: i64, text: &str) {
+    assert!(store_pending_text(tx, id, text, zstd_of(text).as_deref()).unwrap());
+}
+
+/// Fail a row as a later run would: a changed file clears it back to pending
+/// (the state `update_file_basic` leaves), then its extraction fails.
+fn fail(tx: &Transaction<'_>, id: i64, reason: &str) {
+    if state(tx, id) == STATE_DONE {
+        clear_content_for_ids(tx, &[id]).unwrap();
+    }
+    set_content_state(tx, &[id], STATE_PENDING).unwrap();
+    let failed = ContentOutcome::Failed(reason.to_string());
+    assert!(complete_pending(tx, id, &failed, None).unwrap());
+}
+
 /// Every stored path, in `(parent, name)` order.
 fn survivors(conn: &Connection) -> Vec<String> {
     let mut stmt = conn
@@ -114,18 +125,18 @@ fn seeded(conn: &mut Connection, paths: &[&str]) -> std::collections::HashMap<St
             .iter()
             .map(|path| {
                 let id = insert_at(tx, path, true);
-                set_content_done(tx, id, "body text", zstd_of("body text").as_deref()).unwrap();
+                done(tx, id, "body text");
                 (path.to_string(), id)
             })
             .collect()
     })
 }
 
-/// `set_content_done_fresh` skips the pre-delete, so its safety is the
+/// [`complete_pending`] clears nothing first, so its safety is that only a
-/// caller's claim that the row is clean. A duplicate `searchabletext` row
+/// pending row is written. A duplicate `searchabletext` row shows up as a file
-/// shows up as a file appearing twice in results, never as an error.
+/// appearing twice in results, never as an error.
 #[test]
-fn the_fresh_content_write_leaves_exactly_one_fts_row() {
+fn a_pending_write_leaves_exactly_one_fts_row() {
     let row = |mtime| NewFile {
         size: 10,
         mtime,
@@ -135,13 +146,12 @@ fn the_fresh_content_write_leaves_exactly_one_fts_row() {
 
     let id = with_tx(&mut conn, |tx| {
         let id = insert_file(tx, &row(1)).unwrap().unwrap();
-        set_content_done_fresh(tx, id, "sphinx quartz", zstd_of("sphinx quartz").as_deref())
+        done(tx, id, "sphinx quartz");
-            .unwrap();
 
         // The update path: the sequence `process_batch_updates` performs.
         let same = update_file_basic(tx, &row(2)).unwrap().unwrap();
         assert_eq!(same, id, "the same row");
-        set_content_done_fresh(tx, id, "sphinx onyx", zstd_of("sphinx onyx").as_deref()).unwrap();
+        done(tx, id, "sphinx onyx");
         id
     });
 
@@ -150,16 +160,43 @@ fn the_fresh_content_write_leaves_exactly_one_fts_row() {
     assert_eq!(matches(&conn, "onyx"), 1, "the new body is searchable");
     assert_eq!(matches(&conn, "quartz"), 0, "the old body is gone");
 
-    // The idempotent entry point, which the content pass and the watcher use,
+    // A second write to a row that is no longer pending (the content pass
-    // still repairs a row that really does hold content.
+    // holding a row the walk's inline text already stored) writes nothing.
-    with_tx(&mut conn, |tx| {
+    let stored = with_tx(&mut conn, |tx| {
-        set_content_done(tx, id, "sphinx jasper", zstd_of("sphinx jasper").as_deref()).unwrap()
+        store_pending_text(tx, id, "sphinx jasper", zstd_of("sphinx jasper").as_deref()).unwrap()
     });
-
+    assert!(!stored);
     assert_eq!(fts_rows(&conn), 1);
     assert_eq!(count(&conn, "SELECT COUNT(*) FROM documents_text"), 1);
-    assert_eq!(matches(&conn, "jasper"), 1);
+    assert_eq!(matches(&conn, "jasper"), 0);
-    assert_eq!(matches(&conn, "onyx"), 0);
+    assert_eq!(matches(&conn, "onyx"), 1);
+}
+
+/// Stale cleanup can delete a pending row while its extraction is in flight.
+/// Writing the outcome anyway would leave a posting with no `files` row,
+/// matching searches forever and inherited by whichever file reuses the id.
+#[test]
+fn an_outcome_for_a_row_deleted_in_flight_writes_nothing() {
+    let (_dir, _path, mut conn) = fresh();
+    let (text_id, failed_id) = with_tx(&mut conn, |tx| {
+        let a = insert_at(tx, "/gone/a.txt", true);
+        let b = insert_at(tx, "/gone/b.txt", true);
+        delete_ids(tx, &[a, b], &[]).unwrap();
+        (a, b)
+    });
+    with_tx(&mut conn, |tx| {
+        let text = ContentOutcome::Done {
+            text: "sphinx quartz".into(),
+        };
+        assert!(
+            !complete_pending(tx, text_id, &text, zstd_of("sphinx quartz").as_deref()).unwrap()
+        );
+        let failed = ContentOutcome::Failed("boom".into());
+        assert!(!complete_pending(tx, failed_id, &failed, None).unwrap());
+    });
+    assert_eq!(fts_rows(&conn), 0, "no orphan posting");
+    assert_eq!(count(&conn, "SELECT COUNT(*) FROM documents_text"), 0);
+    assert_eq!(count(&conn, "SELECT COUNT(*) FROM failed_files"), 0);
 }
 
 /// The size report reads each body's length out of its zstd frame header,
@@ -252,17 +289,10 @@ fn insert_update_delete_round_trip() {
             ..text_row("/tmp/", "a.txt")
         };
         let id = insert_file(tx, &row).unwrap().expect("unique path");
-        set_content_done(tx, id, "hello world", zstd_of("hello world").as_deref()).unwrap();
+        done(tx, id, "hello world");
     });
 
-    let hit: i64 = conn
+    assert_eq!(matches(&conn, "hello"), 1);
-        .query_row(
-            "SELECT rowid FROM searchabletext WHERE searchabletext MATCH 'hello'",
-            [],
-            |r| r.get(0),
-        )
-        .unwrap();
-    assert!(hit > 0);
 
     with_tx(&mut conn, |tx| {
         assert!(delete_file_by_path(tx, "/tmp/a.txt").unwrap())
@@ -294,11 +324,11 @@ fn update_writes_content_state_from_needs_content() {
     let (_dir, _path, mut conn) = fresh();
     let row = NewFile {
         size: 10,
-        ..na_row("/tmp/", "a.txt")
+        ..text_row("/tmp/", "a.txt")
     };
     let id = with_tx(&mut conn, |tx| {
         let id = insert_file(tx, &row).unwrap().expect("unique path");
-        set_content_done(tx, id, "old text", zstd_of("old text").as_deref()).unwrap();
+        done(tx, id, "old text");
         id
     });
 
@@ -365,7 +395,7 @@ fn delete_subtree_clears_every_dependent_table() {
         ],
     );
     with_tx(&mut conn, |tx| {
-        set_content_failed(tx, ids["/tree/deep/c.txt"], "bad parse").unwrap()
+        fail(tx, ids["/tree/deep/c.txt"], "bad parse")
     });
 
     let range = RootRange::for_root("/tree");
@@ -441,9 +471,7 @@ fn delete_ids_clears_every_dependent_table() {
         &mut conn,
         &["/t/a.txt", "/t/b.log", "/t/deep/c.log", "/t/keep.txt"],
     );
-    with_tx(&mut conn, |tx| {
+    with_tx(&mut conn, |tx| fail(tx, ids["/t/deep/c.log"], "bad parse"));
-        set_content_failed(tx, ids["/t/deep/c.log"], "bad parse").unwrap()
-    });
 
     let doomed = vec![ids["/t/b.log"], ids["/t/deep/c.log"]];
     // `c.log` is FAILED and so holds no posting: the distinction the second
@@ -499,7 +527,7 @@ fn updating_clears_content_exactly_for_the_states_that_hold_it() {
     let ids = seeded(&mut conn, &["/t/done.txt", "/t/failed.txt", "/t/keep.txt"]);
     let (na_id, pending_id) = with_tx(&mut conn, |tx| {
         // Clears failed.txt's posting too; the narrowing rests on that.
-        set_content_failed(tx, ids["/t/failed.txt"], "bad parse").unwrap();
+        fail(tx, ids["/t/failed.txt"], "bad parse");
         let na = insert_file(tx, &na_row("/t/", "na.bin"))
             .unwrap()
             .expect("unique path");
@@ -589,25 +617,6 @@ fn id_and_state_for_path_reads_the_stored_state() {
     );
 }
 
-/// Dropping stored text must cost the file its snippets and nothing else.
-#[test]
-fn drop_stored_text_keeps_the_file_searchable() {
-    let (_dir, _path, mut conn) = fresh();
-    let ids = seeded(&mut conn, &["/t/a.txt", "/t/b.txt"]);
-    with_tx(&mut conn, |tx| {
-        drop_stored_text(tx, &[ids["/t/a.txt"]]).unwrap()
-    });
-
-    assert_eq!(count(&conn, "SELECT COUNT(*) FROM documents_text"), 1);
-    assert_eq!(count(&conn, "SELECT COUNT(*) FROM files"), 2);
-    assert_eq!(fts_rows(&conn), 2);
-    assert_eq!(
-        matches(&conn, "body"),
-        2,
-        "both files still match on content"
-    );
-}
-
 /// The reconciliation scan must serve every row in the range exactly once,
 /// in order, and stop at the range bound rather than at a name prefix.
 #[test]
@@ -742,7 +751,7 @@ fn seed_rows(conn: &mut Connection, range: std::ops::Range<usize>) {
             let id = insert_file(tx, &text_row("/tmp/bulk/", &name))
                 .unwrap()
                 .unwrap();
-            set_content_done(tx, id, &body, zstd_of(&body).as_deref()).unwrap();
+            done(tx, id, &body);
         }
     });
 }
@@ -788,7 +797,15 @@ fn checkpoint_truncate_reports_an_incomplete_checkpoint() {
 /// [`crate::config::MINIMUM_WAL_SIZE`]; this shows the limit works.
 #[test]
 fn journal_size_limit_gives_the_space_back_after_a_blocked_truncate() {
-    let (_dir, p, mut writer) = fresh();
+    // With positions: the test needs more than 16 MiB of log, and position
+    // lists write it from the fewest rows.
+    let (_dir, p) = Scratch::db("repo");
+    let p = p.to_str().unwrap().to_string();
+    let positional = crate::config::ProcessingConfig {
+        compact_text_index: false,
+        ..Default::default()
+    };
+    let mut writer = open_or_recreate(&p, &positional).unwrap();
     seed_rows(&mut writer, 0..10);
     writer
         .busy_timeout(std::time::Duration::from_millis(100))
@@ -864,7 +881,7 @@ fn a_busy_reader_defeats_the_autocheckpoint_but_not_a_forced_one() {
     }
 
     fn run(p: &str, force_every: usize) -> u64 {
-        let mut conn = open_or_recreate(p, "trigram").unwrap();
+        let mut conn = open_or_recreate(p, &Default::default()).unwrap();
         seed_bare(&mut conn, 0..500);
 
         // Stands in for a walk prefetcher: short reads, no gaps.
@@ -1006,15 +1023,11 @@ fn maintain_skips_vacuum_on_a_tight_file() {
 }
 
 #[test]
-fn set_content_failed_writes_failed_table() {
+fn a_failed_extraction_records_its_reason() {
     let (_dir, _path, mut conn) = fresh();
     let id = with_tx(&mut conn, |tx| {
-        let row = NewFile {
+        let id = insert_at(tx, "/tmp/oops.txt", true);
-            size: 0,
+        fail(tx, id, "bad parse");
-            ..na_row("/tmp/", "oops.bin")
-        };
-        let id = insert_file(tx, &row).unwrap().expect("unique path");
-        set_content_failed(tx, id, "bad parse").unwrap();
         id
     });
 
@@ -1029,11 +1042,8 @@ fn set_content_failed_writes_failed_table() {
     assert_eq!(state(&conn, id), STATE_FAILED);
 }
 
-/// A way out of `STATE_DONE`, applied to the row with the given id.
+/// Every way a changed file leaves [`STATE_DONE`] takes the posting and the
-type Transition = fn(&Transaction<'_>, i64);
+/// stored body with it, so its invariant holds however a row got where it is.
-
-/// Every transition out of [`STATE_DONE`] takes the posting and the stored
-/// body with it, so its invariant holds however a row got where it is.
 ///
 /// `count_root_counts_the_fts_rows_it_says_it_does` checks rows that were
 /// never DONE, which passes even for a transition that leaves a stale posting
@@ -1044,31 +1054,22 @@ type Transition = fn(&Transaction<'_>, i64);
 fn leaving_done_always_takes_the_posting_with_it() {
     let (_dir, _path, mut conn) = fresh();
 
-    // Each row starts DONE, with a posting and a stored body, and then takes
+    // Each row starts DONE, with a posting and a stored body, and is then
-    // one of the three ways out.
+    // rewritten in place: still text (back to pending), or no longer
-    let leave: [(&str, Transition); 3] = [
+    // anything an extractor claims (NA).
-        ("/t/failed.txt", |tx, id| {
+    for (name, needs_content) in [("rewritten.txt", true), ("now-binary.txt", false)] {
-            set_content_failed(tx, id, "bad parse").unwrap()
+        with_tx(&mut conn, |tx| {
-        }),
+            let id = insert_at(tx, &format!("/t/{}", name), true);
-        ("/t/na.txt", |tx, id| set_content_na(tx, id).unwrap()),
+            done(tx, id, "body text");
-        ("/t/rewritten.txt", |tx, _id| {
-            // The shape a changed file takes: metadata updated in place.
             let row = NewFile {
                 size: 99,
                 mtime: 99,
-                ..text_row("/t/", "rewritten.txt")
+                needs_content,
+                ..text_row("/t/", name)
             };
             update_file_basic(tx, &row)
                 .unwrap()
                 .expect("the row is there");
-        }),
-    ];
-
-    for (path, transition) in leave {
-        with_tx(&mut conn, |tx| {
-            let id = insert_at(tx, path, true);
-            set_content_done(tx, id, "body text", zstd_of("body text").as_deref()).unwrap();
-            transition(tx, id);
         });
     }
 
@@ -1094,12 +1095,12 @@ fn count_root_counts_the_fts_rows_it_says_it_does() {
         // Two searchable, and one of each way a row can fail to be.
         for path in ["/tree/a.txt", "/tree/b.txt"] {
             let id = insert_at(tx, path, true);
-            set_content_done(tx, id, "body text", zstd_of("body text").as_deref()).unwrap();
+            done(tx, id, "body text");
         }
         let failed = insert_at(tx, "/tree/c.bin", true);
-        set_content_failed(tx, failed, "bad parse").unwrap();
+        fail(tx, failed, "bad parse");
         let na = insert_at(tx, "/tree/d.iso", true);
-        set_content_na(tx, na).unwrap();
+        assert!(complete_pending(tx, na, &ContentOutcome::NotApplicable, None).unwrap());
         insert_at(tx, "/tree/e.txt", true); // still pending
     });
 
@@ -1116,7 +1117,7 @@ fn count_root_counts_the_fts_rows_it_says_it_does() {
     // root's figure.
     with_tx(&mut conn, |tx| {
         let id = insert_at(tx, "/elsewhere/f.txt", true);
-        set_content_done(tx, id, "body text", zstd_of("body text").as_deref()).unwrap();
+        done(tx, id, "body text");
     });
     assert_eq!(fts_rows(&conn), 3);
     assert_eq!(
@@ -1226,11 +1227,10 @@ fn deleting_a_file_row_cascades_the_fk_tables() {
     let (_dir, _path, mut conn) = fresh();
     seeded(&mut conn, &["/casc/a.txt", "/casc/b.txt"]);
     // The failure goes on a row that never extracted, the only way a run
-    // reaches `set_content_failed`, so the FTS figures below stay about the
+    // reaches a failure, so the FTS figures below stay about the cascade.
-    // cascade.
     with_tx(&mut conn, |tx| {
         let never = insert_at(tx, "/casc/c.bin", true);
-        set_content_failed(tx, never, "boom").unwrap();
+        fail(tx, never, "boom");
     });
     assert_eq!(count(&conn, "SELECT COUNT(*) FROM documents_text"), 2);
     assert_eq!(count(&conn, "SELECT COUNT(*) FROM failed_files"), 1);
@@ -1260,7 +1260,7 @@ fn retry_failed_files_resets_state_and_clears_records() {
     let (_dir, _path, mut conn) = fresh();
     let ids = seeded(&mut conn, &["/retry/bad.txt", "/retry/good.txt"]);
     with_tx(&mut conn, |tx| {
-        set_content_failed(tx, ids["/retry/bad.txt"], "parser choked").unwrap()
+        fail(tx, ids["/retry/bad.txt"], "parser choked")
     });
 
     assert_eq!(with_tx(&mut conn, |tx| retry_failed_files(tx).unwrap()), 1);
@@ -1281,8 +1281,8 @@ fn a_changed_file_clears_its_failure_record() {
     let (_dir, _path, mut conn) = fresh();
     let ids = seeded(&mut conn, &["/chg/a.txt", "/chg/b.txt"]);
     with_tx(&mut conn, |tx| {
-        set_content_failed(tx, ids["/chg/a.txt"], "boom").unwrap();
+        fail(tx, ids["/chg/a.txt"], "boom");
-        set_content_failed(tx, ids["/chg/b.txt"], "boom").unwrap();
+        fail(tx, ids["/chg/b.txt"], "boom");
     });
 
     let changed = |name: &'static str, needs_content: bool| NewFile {

crates/quicksearch-core/src/db/schema.rs

@@ -170,6 +170,13 @@ pub const PRAGMAS_WALK_READER: &str = "
 /// Measured: an index cost ~10% on inserts and 805 pages at 300k rows, pages
 /// that compete with the ones search wants resident.
 ///
+/// Nor do `mtime` and `mime`, which only the `modified:` and `mime:` filters
+/// could seek, while every insert, update and delete paid for both. Measured
+/// (`indexprobe cold`, 100k tiny files): index 23.0 → 19.5 MiB, wall time
+/// 0.47 → 0.41 s. The filters now cost what the `files` scan behind every
+/// filename query already does: `searchtime` at 300k rows, 25 ms with or
+/// without a filter, where the indexes had made a one-day filter 0.06 ms.
+///
 /// `documents_text` holds a row only when the extractor produced text (an
 /// audio file with empty tags is done but has none), so `files.content_state`
 /// is the authority on "content indexed". The uncompressed length is not
@@ -196,8 +203,6 @@ CREATE TABLE files (
 );
 
 CREATE UNIQUE INDEX idx_files_parent ON files(parent, name);
-CREATE INDEX idx_files_mtime  ON files(mtime);
-CREATE INDEX idx_files_mime   ON files(mime);
 CREATE INDEX idx_files_hash   ON files(hash);
 CREATE INDEX idx_files_content_pending ON files(id) WHERE content_state = 0;
 
@@ -481,31 +486,44 @@ pub const FTS5_DEFAULT_PGSZ: i64 = 4050;
 ///
 /// One column: only document bodies are ever MATCHed, and a `name` column
 /// would cost (len − 2) postings per file.
-pub fn fts_create_sql(tokenizer: &str) -> String {
+///
-    let effective = effective_tokenizer(tokenizer);
+/// `positions` is [`crate::config::ProcessingConfig::fts_positions`]. Without
+/// them (`detail=none`, the default through `compact_text_index`) the table
+/// answers only single-token phrases, so the search side asks for a term as
+/// the AND of its trigrams (`search::prefilter::fts_literal`) and the
+/// full-text pass drops the documents where they are not adjacent, which it
+/// can because text is always stored. Measured on 1.1 GB of crate sources
+/// (`indexprobe cold`, then `searchtime`), identical results either way:
+///
+/// | | positions | none |
+/// |---|---|---|
+/// | index | 1291 MiB | **219 MiB** |
+/// | build | 37.4 s | **24.6 s** |
+/// | written through the WAL | 4.1 GB | **0.7 GB** |
+/// | candidates per true hit | 1 | 1.0–2x a word, 8–13x a phrase |
+/// | warm search time | 1 | 0.9–1.8x a word, 2.8x a phrase |
+///
+/// The slowest was a phrase of common words (`impl Iterator for`, 24 → 67
+/// ms), whose trigrams occur in many documents that lack the phrase.
+///
+/// The configured tokenizer is applied as written. Plain `trigram` once got
+/// `remove_diacritics 1` appended, but full-text verification folds ASCII
+/// only (`tests/cascade.rs`, `diacritic_fts_candidates_are_dropped`), so the
+/// extra candidates it matched were always dropped again. Dropping it measured
+/// neutral on indexing time (`indexprobe cold`, 5k accented documents).
+pub fn fts_create_sql(tokenizer: &str, positions: bool) -> String {
     format!(
         "CREATE VIRTUAL TABLE searchabletext USING fts5(\
             text, \
             tokenize='{}', \
             content='', \
-            contentless_delete=1\
+            contentless_delete=1{}\
         );",
-        effective.replace('\'', "''")
+        tokenizer.trim().replace('\'', "''"),
+        if positions { "" } else { ", detail=none" }
     )
 }
 
-/// Map a user-facing tokenizer name to the FTS5 option string applied: plain
-/// `trigram` gets `remove_diacritics 1` (so `cafe` matches `café`); any
-/// explicit option string is used verbatim.
-pub fn effective_tokenizer(tokenizer: &str) -> String {
-    let trimmed = tokenizer.trim();
-    if trimmed.eq_ignore_ascii_case("trigram") {
-        "trigram remove_diacritics 1".to_string()
-    } else {
-        trimmed.to_string()
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -630,24 +648,19 @@ mod tests {
     }
 
     #[test]
-    fn plain_trigram_gets_accent_stripping() {
+    fn the_tokenizer_is_applied_as_written() {
-        assert_eq!(
+        assert!(fts_create_sql("  trigram  ", true).contains("tokenize='trigram',"));
-            effective_tokenizer("trigram"),
+        assert!(fts_create_sql("unicode61", true).contains("tokenize='unicode61',"));
-            "trigram remove_diacritics 1"
+        assert!(
-        );
+            fts_create_sql("trigram remove_diacritics 1", true)
-        assert_eq!(
+                .contains("tokenize='trigram remove_diacritics 1',"),
-            effective_tokenizer("  trigram  "),
+            "an explicit option string still passes through"
-            "trigram remove_diacritics 1"
         );
     }
 
     #[test]
-    fn explicit_tokenizers_pass_through() {
+    fn positions_decide_the_detail_option() {
-        assert_eq!(
+        assert!(!fts_create_sql("trigram", true).contains("detail"));
-            effective_tokenizer("trigram remove_diacritics 0"),
+        assert!(fts_create_sql("trigram", false).contains(", detail=none)"));
-            "trigram remove_diacritics 0"
-        );
-        assert_eq!(effective_tokenizer("porter"), "porter");
-        assert_eq!(effective_tokenizer("unicode61"), "unicode61");
     }
 }

crates/quicksearch-core/src/extract/pdf.rs

@@ -51,17 +51,19 @@ impl Extractor for PdfExtractor {
     /// has been measured holding tens of megabytes. Input and output are
     /// bounded by the limits, but the middle is `pdf_extract`'s with no
     /// scratch to reuse, so peak for PDFs alone is `workers × amplification`,
-    /// one pool per root.
+    /// one pool per root. Pages stop being rendered once the text reaches
+    /// `limits().text`.
     fn extract(
         &self,
         path: &Path,
         out: &mut String,
-        _scratch: &mut Scratch,
+        scratch: &mut Scratch,
     ) -> Result<(), ExtractError> {
+        let limit = scratch.limits().text;
         // Loading is inside the guard too: a panic outside it takes the thread.
         install_quiet_panic_hook();
         SUPPRESS_PANIC_PRINT.with(|flag| flag.set(true));
-        let result = std::panic::catch_unwind(|| extract_one_pass(path));
+        let result = std::panic::catch_unwind(|| extract_one_pass(path, limit));
         SUPPRESS_PANIC_PRINT.with(|flag| flag.set(false));
         let text = result
             .map_err(|panic| format!("pdf_extract panicked: {}", panic_message(&*panic)))??;
@@ -70,8 +72,36 @@ impl Extractor for PdfExtractor {
     }
 }
 
-/// `pdf_extract::extract_text`'s body (load, decrypt, `output_doc`) spelled out.
+/// The text sink, full at `limit` bytes. Once the text reaches what the
-fn extract_one_pass(path: &Path) -> Result<String, ExtractError> {
+/// caller keeps, the next write fails, which ends `output_doc` rather than
+/// rendering pages whose text would only be truncated away. Measured
+/// (`extractprobe`, text-only PDFs of 684 and ~340 pages at the default
+/// 256 KiB limit): 1.7 s → 0.8 s of CPU.
+struct Capped<'a> {
+    text: &'a mut String,
+    limit: usize,
+}
+
+impl std::fmt::Write for Capped<'_> {
+    fn write_str(&mut self, s: &str) -> std::fmt::Result {
+        if self.text.len() >= self.limit {
+            return Err(std::fmt::Error);
+        }
+        self.text.push_str(s);
+        Ok(())
+    }
+}
+
+impl pdf_extract::ConvertToFmt for Capped<'_> {
+    type Writer = Self;
+    fn convert(self) -> Self {
+        self
+    }
+}
+
+/// `pdf_extract::extract_text`'s body (load, decrypt, `output_doc`) spelled
+/// out, stopping once `limit` bytes of text are in.
+fn extract_one_pass(path: &Path, limit: usize) -> Result<String, ExtractError> {
     let mut doc = Document::load(path).map_err(|e| format!("pdf_extract: {}", e))?;
     // Empty-password decrypt only; nothing on this path can prompt for a real one.
     if doc.is_encrypted() {
@@ -79,11 +109,19 @@ fn extract_one_pass(path: &Path) -> Result<String, ExtractError> {
     }
 
     let mut text = String::new();
-    {
+    let rendered = {
-        let mut sink = PlainTextOutput::new(&mut text);
+        let mut sink = PlainTextOutput::new(Capped {
-        pdf_extract::output_doc(&doc, &mut sink).map_err(|e| format!("pdf_extract: {}", e))?;
+            text: &mut text,
+            limit,
+        });
+        pdf_extract::output_doc(&doc, &mut sink)
+    };
+    match rendered {
+        Ok(()) => Ok(text),
+        // The sink filled: the stop is ours, not a failure.
+        Err(_) if text.len() >= limit => Ok(text),
+        Err(e) => Err(format!("pdf_extract: {}", e)),
     }
-    Ok(text)
 }
 
 #[cfg(test)]
@@ -195,6 +233,72 @@ mod tests {
         assert!(out.contains("Body Only"));
     }
 
+    /// Rendering stops once the text limit is reached: a later page's text
+    /// is never produced, not produced and cut.
+    #[test]
+    fn extraction_stops_at_the_text_limit() {
+        let mut doc = Document::with_version("1.5");
+        let font = doc.add_object(dictionary! {
+            "Type" => "Font",
+            "Subtype" => "Type1",
+            "BaseFont" => "Helvetica",
+        });
+        let resources = doc.add_object(dictionary! {
+            "Font" => dictionary! { "F1" => font },
+        });
+        let pages_id = doc.new_object_id();
+        let kids: Vec<Object> = ["alphapage", "bravopage", "charliepage", "deltapage"]
+            .iter()
+            .map(|word| {
+                let content = format!("BT /F1 24 Tf 72 720 Td ({}) Tj ET", word);
+                let contents = doc.add_object(Stream::new(dictionary! {}, content.into_bytes()));
+                doc.add_object(dictionary! {
+                    "Type" => "Page",
+                    "Parent" => pages_id,
+                    "Contents" => contents,
+                    "MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
+                })
+                .into()
+            })
+            .collect();
+        doc.objects.insert(
+            pages_id,
+            Object::Dictionary(dictionary! {
+                "Type" => "Pages",
+                "Count" => kids.len() as i64,
+                "Kids" => kids,
+                "Resources" => resources,
+            }),
+        );
+        let catalog = doc.add_object(dictionary! {
+            "Type" => "Catalog",
+            "Pages" => pages_id,
+        });
+        doc.trailer.set("Root", catalog);
+        let path = crate::testutil::scratch_dir("pdf-limit").join("pages.pdf");
+        doc.save(&path).expect("write fixture pdf");
+
+        let whole = extract(&path).expect("extract");
+        assert!(
+            whole.contains("deltapage"),
+            "every page renders: {:?}",
+            whole
+        );
+
+        let mut config = crate::config::Config::default();
+        config.processing.maximum_text_size = 12;
+        let mut out = String::new();
+        PdfExtractor
+            .extract(&path, &mut out, &mut Scratch::new(&config))
+            .expect("a stop at the limit is not a failure");
+        assert!(out.contains("alphapage"), "got {:?}", out);
+        assert!(
+            !out.contains("deltapage"),
+            "rendered past the limit: {:?}",
+            out
+        );
+    }
+
     #[test]
     fn malformed_pdf_fails_without_panicking_the_process() {
         let path = crate::testutil::scratch_dir("pdf-malformed").join("broken.pdf");

crates/quicksearch-core/src/extract/plaintext.rs

@@ -38,11 +38,17 @@ pub(crate) const EXTRA_TEXT_MIMES: &[&str] = &[
     "message/rfc822",
 ];
 
-/// Decode `bytes` into `out`. Valid UTF-8 is moved, not copied, which is why
+/// Decode `bytes`, a prefix of the file when `truncated`, into `out`. Valid
-/// plaintext reads into a fresh buffer rather than the worker's scratch:
+/// UTF-8 is moved, not copied, which is why plaintext reads into a fresh
-/// reusing one would trade an allocation for a full-length `memcpy` per file.
+/// buffer rather than the worker's scratch: reusing one would trade an
-fn decode_into(bytes: Vec<u8>, path: &Path, out: &mut String) -> Result<(), ExtractError> {
+/// allocation for a full-length `memcpy` per file.
-    super::append(out, crate::textenc::decode_text(bytes, path)?);
+fn decode_into(
+    bytes: Vec<u8>,
+    path: &Path,
+    truncated: bool,
+    out: &mut String,
+) -> Result<(), ExtractError> {
+    super::append(out, crate::textenc::decode_text(bytes, path, truncated)?);
     Ok(())
 }
 
@@ -79,9 +85,12 @@ impl Extractor for PlaintextExtractor {
         out: &mut String,
         scratch: &mut Scratch,
     ) -> Result<(), ExtractError> {
-        // The pass never offers a file above `limits().read`, so this cap only
+        // Only the prefix that can survive truncation to `limits().text` is
-        // bites for a file that grew since the walk or reports a false size.
+        // read: UTF-16 is the widest input per byte of UTF-8 output (2:1), so
-        let cap = scratch.limits().read;
+        // no file contributes more than twice the limit. Reading a 40 MiB log
+        // whole to keep 256 KiB of it was the content pass's peak memory.
+        let limits = scratch.limits();
+        let cap = limits.read.min(limits.text.saturating_mul(2));
         let mut f =
             File::open(path).map_err(|e| format!("plaintext read {}: {}", path.display(), e))?;
         let size = f
@@ -96,10 +105,12 @@ impl Extractor for PlaintextExtractor {
             f.take(cap as u64)
                 .read_to_end(&mut buf)
                 .map_err(|e| format!("plaintext read {}: {}", path.display(), e))?;
-            return decode_into(buf, path, out);
+            let truncated = buf.len() == cap;
+            return decode_into(buf, path, truncated, out);
         }
 
-        decode_into(read_sized(&mut f, size, cap, path)?, path, out)
+        let buf = read_sized(&mut f, size, cap, path)?;
+        decode_into(buf, path, size > cap, out)
     }
 
     /// The head belongs to the walk worker and is reused for the next file,
@@ -301,6 +312,36 @@ mod tests {
         std::fs::remove_file(&p).ok();
     }
 
+    /// Twice the text limit: the most any file can contribute to stored text.
+    fn prefix_cap() -> usize {
+        scratch().limits().text * 2
+    }
+
+    #[test]
+    fn a_file_far_over_the_text_limit_reads_only_the_keepable_prefix() {
+        let body = vec![b'z'; prefix_cap() * 3];
+        let p = tmp("prefix", &body);
+        let out = extract(&p).unwrap();
+        assert_eq!(out.len(), prefix_cap(), "read past what can be kept");
+        std::fs::remove_file(&p).ok();
+    }
+
+    /// A read that cuts a multibyte character must not demote a UTF-8 file to
+    /// a legacy charset, which would decode the whole prefix as mojibake.
+    #[test]
+    fn a_character_cut_by_the_prefix_read_is_dropped_not_misdecoded() {
+        let mut body = vec![b'a'; prefix_cap() - 1];
+        body.extend_from_slice("é and more after the cut".as_bytes());
+        let p = tmp("cutchar", &body);
+        let out = extract(&p).unwrap();
+        assert_eq!(out.len(), prefix_cap() - 1, "the cut é's stub is dropped");
+        assert!(
+            out.bytes().all(|b| b == b'a'),
+            "decoded as UTF-8, not legacy"
+        );
+        std::fs::remove_file(&p).ok();
+    }
+
     #[test]
     fn a_file_under_the_cap_is_read_whole() {
         let body = vec![b'y'; 4096];

crates/quicksearch-core/src/file_handling/batch.rs

@@ -14,17 +14,19 @@ use crate::db::repo;
 /// with `slots[i]` saying where row `i`'s is, that it has none, or that its
 /// compression failed (kept per row rather than failing the batch).
 ///
-/// Everything here is reused across chunks: the encoder because a zstd
+/// One per indexing run, reused by every batch and turn the writer takes: the
-/// context per chunk is wasted CPU (`benches/index.rs`, `zstd_encode`), the
+/// encoder because a zstd context per chunk is wasted CPU (`benches/index.rs`,
-/// arena so the writer allocates nothing per document.
+/// `zstd_encode`), the arena so the writer allocates nothing per document.
-struct Bodies {
+/// Measured against one per writer turn (`indexprobe cold`, 10k docs of
+/// 9–20 KiB): VmHWM 60–65 → 44–50 MiB, CPU ~2% lower.
+pub struct Bodies {
     enc: repo::DocEncoder,
     arena: Vec<u8>,
     slots: Vec<Result<Option<std::ops::Range<usize>>, String>>,
 }
 
 impl Bodies {
-    fn new() -> Result<Bodies, String> {
+    pub fn new() -> Result<Bodies, String> {
         Ok(Bodies {
             enc: repo::DocEncoder::new()?,
             arena: Vec::new(),
@@ -34,14 +36,12 @@ impl Bodies {
 
     /// Compress one chunk's bodies before the caller takes the connection, so
     /// the lock covers only the SQL.
-    fn fill<'a>(&mut self, texts: impl Iterator<Item = Option<&'a str>>, config: &Config) {
+    fn fill<'a>(&mut self, texts: impl Iterator<Item = Option<&'a str>>) {
         self.arena.clear();
         self.slots.clear();
         for text in texts {
             let slot = match text {
-                Some(t) if config.processing.store_text_for_snippets && !t.is_empty() => {
+                Some(t) if !t.is_empty() => self.enc.encode_into(t, &mut self.arena).map(Some),
-                    self.enc.encode_into(t, &mut self.arena).map(Some)
-                }
                 _ => Ok(None),
             };
             self.slots.push(slot);
@@ -61,29 +61,25 @@ impl Bodies {
 /// The skeleton behind [`process_batch_updates`] and
 /// [`process_batch_inserts`]: compress each chunk's bodies outside the lock,
 /// write its rows in one transaction through `write_row`, and store any
-/// inline text on a fresh row.
+/// inline text on the row, which both row writers leave pending.
-///
-/// `_fresh` is sound here because both row writers leave a row that provably
-/// holds no content: see their comments at the call sites.
 fn write_prepared_records(
     conn_mutex: &Arc<Mutex<Connection>>,
     records: &[OwnedNewFile],
     stop_flag: &Arc<AtomicBool>,
     config: &Config,
+    bodies: &mut Bodies,
     write_row: impl Fn(&rusqlite::Transaction<'_>, &OwnedNewFile) -> Result<Option<i64>, String>,
 ) -> Result<(), String> {
     // `.max(1)`: `chunks(0)` panics on the indexing thread, so a hand-edited
     // `batch_size = 0` would wedge indexing while the UI reads "Running".
     let chunk_size = config.processing.batch_size.max(1);
-    // One set of buffers for every chunk this call writes.
-    let mut bodies = Bodies::new()?;
     for batch in records.chunks(chunk_size) {
         if stop_flag.load(Ordering::Relaxed) {
             return Ok(());
         }
 
         // Outside the lock; see `Bodies::fill`.
-        bodies.fill(batch.iter().map(|r| r.inline_text.as_deref()), config);
+        bodies.fill(batch.iter().map(|r| r.inline_text.as_deref()));
         let conn = crate::lock_ok(conn_mutex);
         let tx = conn
             .unchecked_transaction()
@@ -96,7 +92,9 @@ fn write_prepared_records(
             let id = write_row(&tx, rec)?;
             if let (Some(id), Some(text)) = (id, rec.inline_text.as_deref()) {
                 match bodies.get(i) {
-                    Ok(zstd) => repo::set_content_done_fresh(&tx, id, text, zstd)?,
+                    Ok(zstd) => {
+                        repo::store_pending_text(&tx, id, text, zstd)?;
+                    }
                     Err(e) => crate::log_warn!("compress text for {}: {}", rec.path(), e),
                 }
             }
@@ -117,19 +115,21 @@ pub fn process_batch_updates(
     files_to_update: &[OwnedNewFile],
     stop_flag: &Arc<AtomicBool>,
     config: &Config,
+    bodies: &mut Bodies,
 ) -> Result<(), String> {
     if files_to_update.is_empty() {
         return Ok(());
     }
 
-    // The row leaves this closure `_fresh`: `update_file_basic` cleared a
+    // The row leaves this closure pending whenever it carries inline text:
-    // DONE row's content in this same transaction, every other state
+    // `update_file_basic` cleared a DONE row's content and reset its state
-    // provably holds none, and the insert fallback created the row outright.
+    // in this same transaction, and the insert fallback created it pending.
     write_prepared_records(
         conn_mutex,
         files_to_update,
         stop_flag,
         config,
+        bodies,
         |tx, rec| {
             let updated = repo::update_file_basic(tx, &rec.as_new_file()).map_err(|e| {
                 format!(
@@ -162,18 +162,20 @@ pub fn process_batch_inserts(
     files_to_insert: &[OwnedNewFile],
     stop_flag: &Arc<AtomicBool>,
     config: &Config,
+    bodies: &mut Bodies,
 ) -> Result<(), String> {
     if files_to_insert.is_empty() {
         return Ok(());
     }
 
-    // The row leaves this closure `_fresh`: `insert_file` returned `Some`
+    // `insert_file` returned `Some` only by creating the row, pending when it
-    // only by creating it, so it cannot carry content from anywhere.
+    // needs content, so it cannot carry content from anywhere.
     write_prepared_records(
         conn_mutex,
         files_to_insert,
         stop_flag,
         config,
+        bodies,
         |tx, rec| {
             repo::insert_file(tx, &rec.as_new_file())
                 .map_err(|e| format!("Failed to insert file record: {}", e))
@@ -365,7 +367,8 @@ const STORE_CHUNK: usize = 32;
 pub struct Stored {
     /// Rows the caller must now drop from its buffer, written or not.
     pub consumed: usize,
-    /// Rows whose write succeeded, so whose `content_state` moved.
+    /// Rows whose write succeeded, so whose `content_state` moved; a row
+    /// no longer pending by the time it is written is consumed, not written.
     pub written: usize,
 }
 
@@ -379,12 +382,10 @@ pub fn store_extracted(
     conn_mutex: &Arc<Mutex<Connection>>,
     rows: &[crate::content::ExtractedRow],
     stop_flag: &Arc<AtomicBool>,
-    config: &Config,
     deadline: std::time::Instant,
+    bodies: &mut Bodies,
 ) -> Result<Stored, String> {
     let mut done = Stored::default();
-    // One set of buffers for every chunk this turn writes.
-    let mut bodies = Bodies::new()?;
     for chunk in rows.chunks(STORE_CHUNK) {
         if stop_flag.load(Ordering::Relaxed) {
             break;
@@ -394,7 +395,6 @@ pub fn store_extracted(
             chunk
                 .iter()
                 .map(|r| crate::file_handling::outcome_body(&r.outcome)),
-            config,
         );
         let conn = crate::lock_ok(conn_mutex);
         let tx = conn
@@ -406,8 +406,10 @@ pub fn store_extracted(
             done.consumed += 1;
             match bodies.get(i) {
                 Err(e) => crate::log_warn!("compress text for {}: {}", row.path(), e),
-                Ok(zstd) => match store_content_outcome(&tx, row.file_id, &row.outcome, zstd) {
+                Ok(zstd) => match repo::complete_pending(&tx, row.file_id, &row.outcome, zstd) {
-                    Ok(()) => done.written += 1,
+                    Ok(true) => done.written += 1,
+                    // No longer pending: stored or deleted since it was paged.
+                    Ok(false) => {}
                     Err(e) => crate::log_warn!("content indexing for {}: {}", row.path(), e),
                 },
             }

crates/quicksearch-core/src/file_handling/count_and_extract_tests.rs

@@ -228,10 +228,17 @@ fn extract_scope_counts_only_files_an_extractor_claims() {
         .collect();
 
     let conn_mutex = Arc::new(Mutex::new(
-        crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap(),
+        crate::db::open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap(),
     ));
     let stop = Arc::new(AtomicBool::new(false));
-    process_batch_inserts(&conn_mutex, &records, &stop, &config).unwrap();
+    process_batch_inserts(
+        &conn_mutex,
+        &records,
+        &stop,
+        &config,
+        &mut Bodies::new().unwrap(),
+    )
+    .unwrap();
 
     let range = RootRange::for_root(root.to_str().unwrap());
     let scope = extract_scope_prepare(&conn_mutex, &range, &config).unwrap();

crates/quicksearch-core/src/file_handling/mod.rs

@@ -16,8 +16,8 @@ pub(crate) use batch::extract_scope_prepare;
 pub(crate) use batch::max_text_file_size;
 pub use batch::{
     cleanup_stale_index_entries, count_extract_scope, mark_oversize_pending_na,
-    process_batch_inserts, process_batch_updates, store_extracted, ExtractCursor, ExtractScope,
+    process_batch_inserts, process_batch_updates, store_extracted, Bodies, ExtractCursor,
-    RootRange, Stored,
+    ExtractScope, RootRange, Stored,
 };
 pub use counting::count_tree_entries_fast;
 pub use paths::{
@@ -27,10 +27,11 @@ pub(crate) use paths::{
     dir_to_db_parent, index_file_set, normalize_root_string, path_to_db_string,
     warn_if_unrepresentable, INDEX_SIDECAR_SUFFIXES,
 };
+pub(crate) use records::mtime_secs;
 pub use records::{
     classify_by_mtime, classify_for_indexing, content_extractable, decide_content,
     extract_and_store, fts_begin_bulk_write, fts_begin_tombstone_burst, fts_end_tombstone_burst,
     fts_finalize_after_text_indexing, get_file_hash, hash_failure_counts, outcome_body,
-    prepare_file_record, prepare_file_record_from_path, reset_run_warnings, store_content_outcome,
+    prepare_file_record, reset_run_warnings, ContentOutcome, DirRows, FileIndexAction,
-    ContentOutcome, DirRows, FileIndexAction, OwnedNewFile, FTS_DELETEMERGE,
+    OwnedNewFile, FTS_DELETEMERGE,
 };

crates/quicksearch-core/src/file_handling/records.rs

@@ -43,13 +43,25 @@ pub fn classify_by_mtime(stored: Option<u64>, mtime: u64) -> FileIndexAction {
 }
 
 /// Truncate in place to at most `max_bytes`, backing up to a UTF-8 char
-/// boundary.
+/// boundary, and give back the capacity: the text goes on to wait in the
+/// content pass's queues, where a 40 MiB log truncated without the shrink
+/// would still hold 40 MiB per queued row.
 pub(crate) fn safe_truncate(s: &mut String, max_bytes: usize) {
     if s.len() > max_bytes {
         s.truncate(s.floor_char_boundary(max_bytes));
+        s.shrink_to_fit();
     }
 }
 
+/// A file's modified time in whole seconds, the unit `files.mtime` stores and
+/// every unchanged-file check compares; `None` when the platform has none.
+pub(crate) fn mtime_secs(meta: &std::fs::Metadata) -> Option<u64> {
+    meta.modified()
+        .ok()
+        .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
+        .map(|d| d.as_secs())
+}
+
 /// A SHA-256 digest, held inline rather than as a `Vec` so hashing allocates
 /// nothing per file.
 pub type FileHash = [u8; 32];
@@ -254,9 +266,10 @@ pub fn fts_finalize_after_text_indexing(conn: &Connection) {
 /// An owned, fully-derived file record, produced by [`prepare_file_record`].
 #[derive(Debug, Clone)]
 pub struct OwnedNewFile {
-    pub name: String,
+    /// Parent and name in one buffer; the parent ends in the platform
-    /// The containing directory, ending in the platform separator.
+    /// separator. One allocation rather than two: `indexprobe cold`, 100k
-    pub parent: String,
+    /// files, 7.2 → 6.2 allocations per file.
+    pub path: repo::RowPath,
     pub size: u64,
     pub mtime: u64,
     /// Borrowed from the static tables [`guess_mime_from_head`] answers from.
@@ -275,14 +288,14 @@ pub struct OwnedNewFile {
 }
 
 impl OwnedNewFile {
-    pub fn path(&self) -> String {
+    pub fn path(&self) -> &str {
-        format!("{}{}", self.parent, self.name)
+        self.path.as_str()
     }
 
     pub fn as_new_file(&self) -> NewFile<'_> {
         NewFile {
-            name: &self.name,
+            name: self.path.name(),
-            parent: &self.parent,
+            parent: self.path.parent(),
             size: self.size,
             mtime: self.mtime,
             mime: self.mime,
@@ -327,11 +340,7 @@ pub fn prepare_file_record(
     }
 
     let size = meta.len();
-    let mtime = meta
+    let mtime = mtime_secs(meta)?;
-        .modified()
-        .ok()
-        .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
-        .map(|d| d.as_secs())?;
 
     // A dehydrated cloud file is indexed from its metadata alone: reading
     // even the first byte would block on downloading the whole file.
@@ -362,7 +371,7 @@ pub fn prepare_file_record(
     };
 
     let (parent, name) = split_db_path(path)?;
-    let (parent, name) = (parent.to_string(), name.to_string());
+    let row_path = repo::RowPath::new(parent, name);
     let head = scratch.head();
     let mime = guess_mime_from_head(Path::new(path), head);
     let ftype = mime.map(mime_to_type).unwrap_or(FileType::EMPTY);
@@ -389,8 +398,7 @@ pub fn prepare_file_record(
         });
 
     Some(OwnedNewFile {
-        name,
+        path: row_path,
-        parent,
         size,
         mtime,
         mime,
@@ -401,26 +409,10 @@ pub fn prepare_file_record(
     })
 }
 
-/// [`prepare_file_record`] for a path not yet resolved: the watcher path. One
+/// Extract content for one pending file and record the outcome on its row
-/// file at a time, so it owns the scratch rather than being handed one.
+/// ([`repo::complete_pending`]). `mime` is authoritative, including when it
-pub fn prepare_file_record_from_path(
+/// is `None`: the head was already sniffed by [`prepare_file_record`], and
-    path: &Path,
+/// re-sniffing gives the same answer.
-    config: &Config,
-    registry: &Registry,
-) -> Option<OwnedNewFile> {
-    let canonical = path.canonicalize().ok()?;
-    if warn_if_unrepresentable(&canonical) {
-        return None;
-    }
-    let db_path = path_to_db_string(&canonical);
-    let meta = std::fs::metadata(&canonical).ok()?;
-    let mut scratch = Scratch::new(config);
-    prepare_file_record(&db_path, &meta, config, registry, &mut scratch)
-}
-
-/// Extract content for one file and record the outcome on its row. `mime` is
-/// authoritative, including when it is `None`: the head was already sniffed
-/// by [`prepare_file_record`], and re-sniffing gives the same answer.
 pub fn extract_and_store(
     tx: &rusqlite::Transaction<'_>,
     file_id: i64,
@@ -434,10 +426,10 @@ pub fn extract_and_store(
     let mut scratch = Scratch::new(config);
     let outcome = decide_content(path, mime, registry, config, &mut scratch);
     let zstd = match outcome_body(&outcome) {
-        Some(text) => repo::encode_one(text, config.processing.store_text_for_snippets)?,
+        Some(text) => repo::encode_one(text)?,
         None => None,
     };
-    store_content_outcome(tx, file_id, &outcome, zstd.as_deref())
+    repo::complete_pending(tx, file_id, &outcome, zstd.as_deref()).map(|_| ())
 }
 
 /// What should be written for one file's content, decided without touching
@@ -496,21 +488,6 @@ pub fn decide_content(
     }
 }
 
-/// Apply a decision from [`decide_content`]: the cheap half, all that runs
-/// with the connection held. `text_zstd`: see [`repo::set_content_done`].
-pub fn store_content_outcome(
-    tx: &rusqlite::Transaction<'_>,
-    file_id: i64,
-    outcome: &ContentOutcome,
-    text_zstd: Option<&[u8]>,
-) -> Result<(), String> {
-    match outcome {
-        ContentOutcome::Done { text } => repo::set_content_done(tx, file_id, text, text_zstd),
-        ContentOutcome::NotApplicable => repo::set_content_na(tx, file_id),
-        ContentOutcome::Failed(reason) => repo::set_content_failed(tx, file_id, reason),
-    }
-}
-
 /// The body a [`ContentOutcome`] would store, if any.
 pub fn outcome_body(outcome: &ContentOutcome) -> Option<&str> {
     match outcome {

crates/quicksearch-core/src/file_handling/tests.rs

@@ -576,7 +576,7 @@ fn a_symlinked_root_yields_no_directories_when_following_is_off() {
 fn a_bulk_write_repairs_a_delete_merge_threshold_left_off() {
     let dir = crate::testutil::scratch_dir("deletemerge-repair");
     let db = dir.join("index.sqlite");
-    let conn = crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
+    let conn = crate::db::open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap();
 
     // What a killed burst leaves behind.
     fts_begin_tombstone_burst(&conn);
@@ -626,9 +626,10 @@ fn seed_stale_corpus(
         .expect("unique path");
         ids.insert(name, id);
     }
-    repo::set_content_done(&tx, ids["done-stale.txt"], "body one", None).unwrap();
+    repo::store_pending_text(&tx, ids["done-stale.txt"], "body one", None).unwrap();
-    repo::set_content_done(&tx, ids["done-kept.txt"], "body two", None).unwrap();
+    repo::store_pending_text(&tx, ids["done-kept.txt"], "body two", None).unwrap();
-    repo::set_content_failed(&tx, ids["failed-stale.txt"], "bad parse").unwrap();
+    let failed = crate::file_handling::ContentOutcome::Failed("bad parse".into());
+    repo::complete_pending(&tx, ids["failed-stale.txt"], &failed, None).unwrap();
     tx.commit().unwrap();
     ids
 }
@@ -646,7 +647,7 @@ fn stale_cleanup_deletes_exactly_the_stale_rows_and_their_postings() {
 
     let dir = crate::testutil::scratch_dir("stale-cleanup");
     let db = dir.join("index.sqlite");
-    let mut conn = crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
+    let mut conn = crate::db::open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap();
     let ids = seed_stale_corpus(&mut conn);
 
     // `vanished.txt` was never indexed: a stale list can legitimately name a
@@ -675,14 +676,11 @@ fn stale_cleanup_deletes_exactly_the_stale_rows_and_their_postings() {
     assert_eq!(count("SELECT COUNT(*) FROM searchabletext"), 1);
     assert_eq!(count("SELECT COUNT(*) FROM documents_text"), 0);
     assert_eq!(count("SELECT COUNT(*) FROM failed_files"), 0);
-    let hit: i64 = conn
+    assert_eq!(
-        .query_row(
+        crate::testutil::fts_rowids(&conn, "body"),
-            "SELECT rowid FROM searchabletext WHERE searchabletext MATCH 'body'",
+        vec![ids["done-kept.txt"]],
-            [],
+        "the survivor is still findable"
-            |r| r.get(0),
+    );
-        )
-        .unwrap();
-    assert_eq!(hit, ids["done-kept.txt"], "the survivor is still findable");
     assert_eq!(
         deletemerge(&conn),
         Some(i64::from(FTS_DELETEMERGE)),
@@ -704,7 +702,7 @@ fn a_stopped_stale_cleanup_deletes_nothing_and_leaves_the_burst() {
 
     let dir = crate::testutil::scratch_dir("stale-cleanup-stop");
     let db = dir.join("index.sqlite");
-    let mut conn = crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
+    let mut conn = crate::db::open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap();
     seed_stale_corpus(&mut conn);
 
     let conn_mutex = Arc::new(Mutex::new(conn));

crates/quicksearch-core/src/incremental.rs

@@ -6,14 +6,14 @@ use std::path::Path;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::time::Instant;
 
-use rusqlite::{Connection, OptionalExtension};
+use rusqlite::Connection;
 
 use crate::config::{Config, IgnoreSet};
 use crate::db::repo;
-use crate::extract::Registry;
+use crate::extract::{Registry, Scratch};
 use crate::file_handling::{
-    db_key_for_missing_path, extract_and_store, filtered_walk, prepare_file_record_from_path,
+    db_key_for_missing_path, extract_and_store, filtered_walk, mtime_secs, path_to_db_string,
-    RootRange, UnreadableDirs,
+    prepare_file_record, warn_if_unrepresentable, RootRange, UnreadableDirs,
 };
 use crate::platform::path_has_hidden_component_under;
 use crate::watcher::FsEvent;
@@ -90,8 +90,7 @@ fn upsert_path(
         return Ok(Applied::Done);
     }
     // Checked before the followed `metadata` below: with following off the
-    // walk records no symlinks, and `prepare_file_record_from_path`
+    // walk records no symlinks, and `upsert_file` canonicalizes, so a followed link targeting outside every root would
-    // canonicalizes, so a followed link targeting outside every root would
     // write a row no sweep range covers and only a rebuild clears.
     if !config.indexing.follow_symlinks {
         match std::fs::symlink_metadata(path) {
@@ -143,7 +142,14 @@ fn upsert_file(
     config: &Config,
     registry: &Registry,
 ) -> Result<(), String> {
-    let Some(rec) = prepare_file_record_from_path(path, config, registry) else {
+    let Ok(canonical) = path.canonicalize() else {
+        return Ok(());
+    };
+    if warn_if_unrepresentable(&canonical) {
+        return Ok(());
+    }
+    let db_path = path_to_db_string(&canonical);
+    let Ok(meta) = std::fs::metadata(&canonical) else {
         return Ok(());
     };
 
@@ -151,45 +157,43 @@ fn upsert_file(
         .transaction()
         .map_err(|e| format!("begin incremental tx: {}", e))?;
 
-    let existing: Option<(i64, i64)> = tx
+    // Settled before the file is opened, as the full walk settles it: an
-        .query_row(
+    // unchanged file (a re-sent event, a `chmod`, the live view re-checking a
-            "SELECT id, mtime FROM files WHERE parent = ?1 AND name = ?2",
+    // row it shows) costs one lookup, never a read and a hash. Measured
-            rusqlite::params![rec.parent, rec.name],
+    // (`benches/index.rs`, `unchanged_event`, 500 small text files): 3.3 ms
-            |r| Ok((r.get(0)?, r.get(1)?)),
+    // hashing and sniffing each against 1.4 ms.
-        )
+    let stored = repo::mtime_for_path(&tx, &db_path)?;
-        .optional()
+    if stored.is_some() && stored == mtime_secs(&meta) {
-        .map_err(|e| format!("lookup {}: {}", rec.path(), e))?;
+        return Ok(());
-
+    }
-    let file_id = match existing {
+    // One file at a time, so it owns the scratch rather than being handed one.
-        Some((_, mtime)) if mtime.max(0) as u64 == rec.mtime => return Ok(()),
+    let mut scratch = Scratch::new(config);
-        Some((id, _)) => {
+    let Some(rec) = prepare_file_record(&db_path, &meta, config, registry, &mut scratch) else {
-            repo::update_file_basic(&tx, &rec.as_new_file())?;
+        return Ok(());
-            id
+    };
-        }
+    let written = match stored {
-        None => match repo::insert_file(&tx, &rec.as_new_file())? {
+        Some(_) => repo::update_file_basic(&tx, &rec.as_new_file())?,
-            Some(id) => id,
+        None => repo::insert_file(&tx, &rec.as_new_file())?,
-            // Lost a race with another writer; the row that won is current enough.
+    };
-            None => return Ok(()),
+    // `None`: lost a race with another writer; the row that won is current
-        },
+    // enough.
+    let Some(file_id) = written else {
+        return Ok(());
     };
 
-    // The only size gate on this path: `decide_content` has none, and falling
+    // A row that needs no content is already NA: both writers above set its
-    // through would read a multi-gigabyte `.txt` whole into memory.
+    // state from `rec`. That gate is also the only size gate on this path:
-    if !rec.needs_content {
+    // `decide_content` has none, and falling through would read a
-        repo::set_content_na(&tx, file_id)?;
+    // multi-gigabyte `.txt` whole into memory.
-    } else if let Some(text) = rec.inline_text.as_deref() {
+    if rec.needs_content {
-        // `prepare_file_record_from_path` already read the whole file.
+        match rec.inline_text.as_deref() {
-        let zstd = repo::encode_one(text, config.processing.store_text_for_snippets)?;
+            // `prepare_file_record` already read the whole file.
-        repo::set_content_done(&tx, file_id, text, zstd.as_deref())?;
+            Some(text) => {
-    } else {
+                let zstd = repo::encode_one(text)?;
-        extract_and_store(
+                repo::store_pending_text(&tx, file_id, text, zstd.as_deref())?;
-            &tx,
+            }
-            file_id,
+            None => extract_and_store(&tx, file_id, rec.path(), rec.mime, registry, config)?,
-            &rec.path(),
+        }
-            rec.mime,
-            registry,
-            config,
-        )?;
     }
 
     tx.commit()
@@ -236,6 +240,7 @@ pub fn remove_paths(
 mod tests {
     use super::*;
     use crate::db::open_or_recreate;
+    use rusqlite::OptionalExtension;
     use std::time::Duration;
 
     struct Fixture {
@@ -253,7 +258,7 @@ mod tests {
             let dir = scratch.join("tree");
             std::fs::create_dir_all(&dir).unwrap();
             let db = scratch.join("index.sqlite");
-            let conn = open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
+            let conn = open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap();
             let config = Config::default();
             let ignore = IgnoreSet::compile(&config.indexing.ignore_patterns).unwrap();
             Fixture {
@@ -332,13 +337,7 @@ mod tests {
         }
 
         fn fts_hits(&self, term: &str) -> i64 {
-            self.conn
+            crate::testutil::fts_count(&self.conn, term)
-                .query_row(
-                    "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH ?1",
-                    rusqlite::params![format!("\"{}\"", term)],
-                    |r| r.get(0),
-                )
-                .unwrap()
         }
     }
 
@@ -472,6 +471,36 @@ mod tests {
         assert_eq!(f.counts(), (1, 1, 1), "still exactly one of everything");
     }
 
+    /// An unchanged file is settled from its stored mtime, as the full walk
+    /// settles it, without being opened. Here it cannot be opened, so an
+    /// attempt would log a hash failure naming it.
+    #[cfg(unix)]
+    #[test]
+    fn an_unchanged_file_is_not_opened() {
+        use std::os::unix::fs::PermissionsExt;
+        let mut f = Fixture::new();
+        let p = f.write("sealed.txt", "sphinx of black quartz");
+        f.apply(&FsEvent::Create(p.clone()));
+        let key = f.canonical(&p);
+        let before = f.row(&key).expect("indexed");
+
+        std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o000)).unwrap();
+        // Root reads it anyway, and then there is nothing to prove.
+        let readable = std::fs::File::open(&p).is_ok();
+        f.apply(&FsEvent::Modify(p.clone()));
+        std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o644)).unwrap();
+
+        assert_eq!(f.row(&key), Some(before), "the row is untouched");
+        if !readable {
+            assert!(
+                !crate::log::snapshot()
+                    .iter()
+                    .any(|line| line.text.contains("cannot hash") && line.text.contains(&key)),
+                "the unchanged file was opened"
+            );
+        }
+    }
+
     #[test]
     fn remove_file_cleans_all_tables() {
         let mut f = Fixture::new();

crates/quicksearch-core/src/indexing/config_check.rs

@@ -28,6 +28,11 @@ impl IndexingService {
             ("hash_algorithm", "size+head".to_string()),
             ("indexing_path", sorted_joined(roots)),
             ("tokenize", config.processing.tokenize.clone()),
+            // The effective layout, not the flag: see `fts_positions`.
+            (
+                "fts_positions",
+                config.processing.fts_positions().to_string(),
+            ),
             ("include_hidden", config.indexing.include_hidden.to_string()),
             (
                 "follow_symlinks",
@@ -41,10 +46,6 @@ impl IndexingService {
                 "content_extensions",
                 sorted_joined(&config.indexing.content_extensions),
             ),
-            (
-                "store_text_for_snippets",
-                config.processing.store_text_for_snippets.to_string(),
-            ),
         ]
     }
 
@@ -61,10 +62,11 @@ impl IndexingService {
     }
 
     /// Settings a difference in which cannot be reconciled: the FTS tokenizer
-    /// is part of the table definition, and hashes under a different length or
+    /// and position setting are part of the table definition, and hashes
-    /// algorithm cannot be compared. Everything else reconciles; see
+    /// under a different length or algorithm cannot be compared. Everything
-    /// [`crate::scope`].
+    /// else reconciles; see [`crate::scope`].
-    const REBUILD_KEYS: [&'static str; 3] = ["hash_length", "hash_algorithm", "tokenize"];
+    const REBUILD_KEYS: [&'static str; 4] =
+        ["hash_length", "hash_algorithm", "tokenize", "fts_positions"];
 
     /// The unreconcilable settings that no longer match, which a rebuild
     /// prompt shows; `None` when none. A key absent (older index) never counts

crates/quicksearch-core/src/indexing/pipeline.rs

@@ -16,8 +16,8 @@ use crate::extract::Registry;
 use crate::file_handling::{
     cleanup_stale_index_entries, count_tree_entries_fast, fts_begin_bulk_write,
     fts_finalize_after_text_indexing, mark_oversize_pending_na, normalize_root_string,
-    process_batch_inserts, process_batch_updates, store_extracted, ExtractCursor, ExtractScope,
+    process_batch_inserts, process_batch_updates, store_extracted, Bodies, ExtractCursor,
-    FileIndexAction, OwnedNewFile, RootRange,
+    ExtractScope, FileIndexAction, OwnedNewFile, RootRange,
 };
 use crate::walk::{thread_count_for, walk_indexable_files, ParallelWalk, TryNext, WalkEvent};
 
@@ -149,12 +149,27 @@ pub(super) struct RootPipeline {
 const IDLE_BACKOFF: Duration = Duration::from_millis(2);
 
 /// A batch writer: [`process_batch_updates`] or [`process_batch_inserts`].
-type WriteBatch =
+type WriteBatch = fn(
-    fn(&Arc<Mutex<Connection>>, &[OwnedNewFile], &Arc<AtomicBool>, &Config) -> Result<(), String>;
+    &Arc<Mutex<Connection>>,
+    &[OwnedNewFile],
+    &Arc<AtomicBool>,
+    &Config,
+    &mut Bodies,
+) -> Result<(), String>;
 
 /// Write `rows` through `write`, then clear them; on an error they stay.
-fn flush(cx: &RunCx<'_>, rows: &mut Vec<OwnedNewFile>, write: WriteBatch) -> Result<(), String> {
+fn flush(
-    write(&cx.conn_mutex, rows, cx.stop_flag, cx.config)?;
+    cx: &mut RunCx<'_>,
+    rows: &mut Vec<OwnedNewFile>,
+    write: WriteBatch,
+) -> Result<(), String> {
+    write(
+        &cx.conn_mutex,
+        rows,
+        cx.stop_flag,
+        cx.config,
+        &mut cx.bodies,
+    )?;
     rows.clear();
     Ok(())
 }
@@ -227,9 +242,7 @@ mod census {
             let inline = |rows: &[crate::file_handling::OwnedNewFile]| -> u64 {
                 rows.iter()
                     .map(|r| {
-                        (r.inline_text.as_ref().map_or(0, String::len)
+                        (r.inline_text.as_ref().map_or(0, String::len) + r.path().len()) as u64
-                            + r.name.len()
-                            + r.parent.len()) as u64
                     })
                     .sum()
             };
@@ -500,7 +513,20 @@ impl RootPipeline {
             if ready.is_empty() {
                 break;
             }
-            let stored = store_extracted(&cx.conn_mutex, ready, cx.stop_flag, cx.config, deadline)?;
+            // Rowid order: FTS5 flushes its pending terms to a new segment
+            // whenever a write's rowid falls below the previous one, and the
+            // pool finishes rows in completion order, not id order. Measured
+            // (`indexprobe cold`, 10k docs of 9–20 KiB): 5% less CPU, 9% less
+            // WAL written; docs past FTS5's 1 MiB hash flush anyway, so a
+            // corpus of 8–200 KiB docs gained only the WAL figure (7%).
+            ready.sort_unstable_by_key(|row| row.file_id);
+            let stored = store_extracted(
+                &cx.conn_mutex,
+                ready,
+                cx.stop_flag,
+                deadline,
+                &mut cx.bodies,
+            )?;
             if stored.consumed > 0 {
                 // The last row *written*, not the last fetched. The whole path,
                 // as the walk phase publishes: the hint must not change what it
@@ -548,6 +574,8 @@ pub(super) struct RunCx<'a> {
     /// Writer time one root's turn may take before the round moves on
     /// ([`crate::config::ProcessingConfig::writer_turn_slice_ms`]).
     pub(super) slice: Duration,
+    /// The run's one set of compression buffers, shared by every write.
+    pub(super) bodies: Bodies,
     /// Rows with no file behind them, per-directory plus the vanished sweep.
     pub(super) stale_candidates: Vec<String>,
     /// Paths reached via symlink; their rows may live outside every root.
@@ -562,8 +590,8 @@ impl<'a> RunCx<'a> {
         db_path: &'a str,
         stop_flag: &'a Arc<AtomicBool>,
         status: Arc<Mutex<IndexingStatus>>,
-    ) -> RunCx<'a> {
+    ) -> Result<RunCx<'a>, String> {
-        RunCx {
+        Ok(RunCx {
             conn_mutex,
             config,
             db_path,
@@ -572,10 +600,11 @@ impl<'a> RunCx<'a> {
             registry: Arc::new(Registry::default_set()),
             batch: config.processing.batch_size.max(1),
             slice: Duration::from_millis(config.processing.writer_turn_slice_ms),
+            bodies: Bodies::new()?,
             stale_candidates: Vec::new(),
             aliased_paths: HashSet::new(),
             stale_cleanup_ok: true,
-        }
+        })
     }
 
     /// Publish `step` on the running status until the returned guard drops, so
@@ -656,7 +685,14 @@ fn build_pipeline(
         workers,
     );
 
-    // A walked root needs no scan: its stored count is exact.
+    // A walked root needs no scan: its stored count is exact. A never-walked
+    // one gets a second, count-only traversal beside the walk, kept for the
+    // first run's progress figure. Measured (`indexprobe cold`, 50k files in
+    // 5k directories, alternating runs): on a virtiofs share it cost ~4% of
+    // wall time (2.29 s against 2.20 s without it) and ~7% of CPU, and no
+    // prefetch benefit to the walk showed; on warm local disk the difference
+    // was within noise. Its figure counts directories and ignored entries
+    // too, so it runs high.
     let count_total = Arc::new(AtomicUsize::new(0));
     match stored_count {
         // `max(1)`: 0 is the "unknown" sentinel; an empty root must not store it.
@@ -857,7 +893,7 @@ impl IndexingService {
     ) -> Result<Option<Connection>, String> {
         // A large WAL recovery happens here; announced before attempted.
         Self::set_prep_step(status, PrepStep::OpeningIndex);
-        let mut conn = db::open_or_recreate(db_path, &config.processing.tokenize)?;
+        let mut conn = db::open_or_recreate(db_path, &config.processing)?;
 
         // Reconcile against the settings the index was last written under,
         // *before* stamping the new ones: the old record is the only thing
@@ -944,7 +980,7 @@ impl IndexingService {
         let count_cancel = Arc::new(AtomicBool::new(false));
         let _count_guard = CancelOnDrop(count_cancel.clone());
 
-        let mut cx = RunCx::new(conn_mutex, config, db_path, stop_flag, status.clone());
+        let mut cx = RunCx::new(conn_mutex, config, db_path, stop_flag, status.clone())?;
 
         let stored_counts: Vec<Option<usize>> = {
             let conn = crate::lock_ok(&cx.conn_mutex);

crates/quicksearch-core/src/indexing/tests.rs

@@ -124,7 +124,7 @@ fn worker_counts_follow_the_phase() {
     let dir = tmp_dir("phase-workers");
     std::fs::write(dir.join("a.txt"), "hello").unwrap();
     let db_path = dir.join("index.db").to_string_lossy().into_owned();
-    drop(db::open_or_recreate(&db_path, "trigram").unwrap());
+    drop(db::open_or_recreate(&db_path, &Default::default()).unwrap());
 
     let stop = Arc::new(AtomicBool::new(false));
     let mut p = idle_pipeline(&dir, &db_path, 3, 2, &stop);
@@ -150,7 +150,7 @@ fn an_extracting_turn_lands_its_leftovers_one_slice_at_a_time() {
 
     let dir = tmp_dir("slice-leftovers");
     let db_path = dir.join("index.db").to_string_lossy().into_owned();
-    let mut conn = db::open_or_recreate(&db_path, "trigram").unwrap();
+    let mut conn = db::open_or_recreate(&db_path, &Default::default()).unwrap();
     let tree = dir.join("tree");
     std::fs::create_dir_all(&tree).unwrap();
     let mut ready: Vec<ExtractedRow> = Vec::new();
@@ -204,7 +204,8 @@ fn an_extracting_turn_lands_its_leftovers_one_slice_at_a_time() {
         &db_path,
         &stop,
         Arc::new(Mutex::new(IndexingStatus::Idle)),
-    );
+    )
+    .unwrap();
     cx.slice = Duration::ZERO;
 
     let mut turns = 0;
@@ -488,6 +489,7 @@ fn cx_for<'a>(
         stop,
         status.clone(),
     )
+    .unwrap()
 }
 
 fn running_status(maintenance: Option<MaintenanceStep>) -> Arc<Mutex<IndexingStatus>> {

crates/quicksearch-core/src/query/pattern.rs

@@ -263,22 +263,19 @@ impl TermPattern {
     pub fn count_folded(&self, folded: &str) -> usize {
         match self {
             TermPattern::Empty => 0,
-            TermPattern::Literal(l) => snippet::count_occurrences(folded, &l.folded, true),
+            TermPattern::Literal(l) => snippet::count_occurrences(folded, &l.folded),
             TermPattern::Wildcard(w) => w.search_ci.find_iter(folded).take(COUNT_CAP).count(),
         }
     }
 
-    /// Non-overlapping occurrence count. Wildcards stop at 1000, where the
+    /// Case-sensitive, non-overlapping occurrence count; the case-insensitive
+    /// one is [`TermPattern::count_folded`]. Wildcards stop at 1000, where the
     /// cascade's `count_frac` saturates anyway.
-    pub fn count(&self, text: &str, case_insensitive: bool) -> usize {
+    pub fn count(&self, text: &str) -> usize {
         match self {
             TermPattern::Empty => 0,
-            TermPattern::Literal(l) => snippet::count_occurrences(text, &l.text, !case_insensitive),
+            TermPattern::Literal(l) => snippet::count_occurrences(text, &l.text),
-            TermPattern::Wildcard(w) => w
+            TermPattern::Wildcard(w) => w.search_cs.find_iter(text).take(COUNT_CAP).count(),
-                .search(case_insensitive)
-                .find_iter(text)
-                .take(COUNT_CAP)
-                .count(),
         }
     }
 }
@@ -428,7 +425,7 @@ mod tests {
             assert!(p.is_empty(), "{:?}", parts);
             assert!(!p.whole_match("anything", true));
             assert!(p.find_first("anything", true).is_none());
-            assert_eq!(p.count("anything", true), 0);
+            assert_eq!(p.count("anything"), 0);
         }
     }
 
@@ -516,9 +513,9 @@ mod tests {
     #[test]
     fn count_is_nonoverlapping_and_capped() {
         let p = TermPattern::build(&[part("a*b", true)]).unwrap();
-        assert_eq!(p.count("ab ab ab", false), 3);
+        assert_eq!(p.count("ab ab ab"), 3);
         let many = "ab ".repeat(2000);
-        assert_eq!(p.count(&many, false), 1000);
+        assert_eq!(p.count(&many), 1000);
     }
 
     #[test]
@@ -530,8 +527,8 @@ mod tests {
         assert_eq!(p.find_first("my Report.pdf", false), Some(3..9));
         assert_eq!(p.find_first("my report.pdf", true), Some(3..9));
         assert_eq!(p.find_first("my report.pdf", false), None);
-        assert_eq!(p.count("report Report", false), 1);
+        assert_eq!(p.count("report Report"), 1);
-        assert_eq!(p.count("report Report", true), 2);
+        assert_eq!(p.count_folded("report report"), 2);
     }
 
     #[test]

crates/quicksearch-core/src/scope.rs

@@ -418,8 +418,6 @@ struct PagePlan {
     /// posting (see [`repo::delete_ids`]).
     doomed: Vec<i64>,
     doomed_with_postings: Vec<i64>,
-    /// Rows keeping their posting but losing the snippet source.
-    stale_text: Vec<i64>,
     /// Surviving rows changing `content_state`, and across both, those whose
     /// stored state holds a posting or a failure record to clear first.
     to_pending: Vec<i64>,
@@ -433,7 +431,6 @@ impl PagePlan {
         for list in [
             &mut self.doomed,
             &mut self.doomed_with_postings,
-            &mut self.stale_text,
             &mut self.to_pending,
             &mut self.to_na,
             &mut self.restated_with_postings,
@@ -460,7 +457,6 @@ impl PagePlan {
     /// Returns `(deleted, recontented)`.
     fn write(&self, tx: &Transaction<'_>) -> Result<(usize, usize), String> {
         let deleted = repo::delete_ids(tx, &self.doomed, &self.doomed_with_postings)?;
-        repo::drop_stored_text(tx, &self.stale_text)?;
         repo::clear_content_for_ids(tx, &self.restated_with_postings)?;
         repo::clear_failed_for_ids(tx, &self.restated_with_failures)?;
         repo::set_content_state(tx, &self.to_pending, repo::STATE_PENDING)?;
@@ -493,21 +489,14 @@ fn apply_page(
             }
             continue;
         }
-        // Only a DONE row can have a `documents_text` body to drop.
+        if work.reconcile_content {
-        if work.drop_text && row.content_state == repo::STATE_DONE {
+            // The walker's decision, recomputed. Both directions: a
-            plan.stale_text.push(row.id);
+            // disagreeing row is wrong however it got that way.
-        }
-        if work.reconcile_content || work.restore_text {
-            // The walker's decision, recomputed. Both directions run whenever
-            // either flag is set: a disagreeing row is wrong however it got that way.
             let wants = row.size <= config.processing.maximum_text_file_size
                 && content_extractable(path, row.mime.as_deref(), config, registry);
             if !wants && row.content_state != repo::STATE_NA {
                 plan.restate(row, false);
-            } else if wants
+            } else if wants && row.content_state == repo::STATE_NA {
-                && (row.content_state == repo::STATE_NA
-                    || (work.restore_text && row.content_state == repo::STATE_DONE))
-            {
                 plan.restate(row, true);
             }
         }
@@ -535,9 +524,9 @@ pub fn stored_config(conn: &Connection, config: &Config) -> Result<Config, Strin
             "content_extensions" => stored.indexing.content_extensions = lines(&value),
             "include_hidden" => stored.indexing.include_hidden = value == "true",
             "follow_symlinks" => stored.indexing.follow_symlinks = value == "true",
-            "store_text_for_snippets" => {
+            // Recorded as the effective layout; a flag that reproduces it
-                stored.processing.store_text_for_snippets = value == "true"
+            // under the recorded tokenizer (see `fts_positions`).
-            }
+            "fts_positions" => stored.processing.compact_text_index = value == "false",
             "hash_length" => match value.parse() {
                 Ok(n) => stored.processing.hash_length = n,
                 Err(e) => crate::log_warn!("stored hash_length {:?} unreadable: {}", value, e),

crates/quicksearch-core/src/scope_tests.rs

@@ -8,7 +8,7 @@ use std::sync::Arc;
 
 fn empty_db(dir: &Path) -> PathBuf {
     let db = dir.join("index.sqlite");
-    crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
+    crate::db::open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap();
     db
 }
 

crates/quicksearch-core/src/search/cascade.rs

@@ -34,7 +34,7 @@ use rusqlite::{Connection, OptionalExtension};
 use crate::config::IgnoreSet;
 use crate::query::pattern::clamp_match_range;
 use crate::query::split::CascadeQuery;
-use crate::query::translator::{escape_like, quote_phrase};
+use crate::query::translator::escape_like;
 use crate::snippet;
 
 use super::fuzzy::{matcher_for, pigeonhole_chunks, Bitap};
@@ -80,12 +80,7 @@ pub fn text_snippet_counted(
     folded: &str,
 ) -> Option<(snippet::Snippet, usize)> {
     let term = pattern.literal_folded()?;
-    Some(snippet::extract_folded(
+    Some(snippet::extract_folded(text, folded, term, &SNIPPET_OPTS))
-        text,
-        folded,
-        &[term],
-        &SNIPPET_OPTS,
-    ))
 }
 
 /// The fuzzy full-text match in one document body: occurrence count within
@@ -128,6 +123,7 @@ pub fn run(
         conn,
         query,
         options,
+        positions: crate::db::repo::fts_positions(conn),
         generation,
         latest_gen,
         ignore,
@@ -310,6 +306,10 @@ struct Cx<'a> {
     conn: &'a Connection,
     query: &'a CascadeQuery,
     options: &'a SearchOptions,
+    /// Whether the full-text index keeps positions, read from the index each
+    /// search: it decides how every `MATCH` is phrased
+    /// ([`prefilter::fts_literal`]).
+    positions: bool,
     generation: u64,
     latest_gen: &'a AtomicU64,
     ignore: IgnoreSet,
@@ -417,7 +417,7 @@ impl<'a> Cx<'a> {
                 stmt.query_row([file_id], |r| {
                     let blob = r.get_ref(0)?.as_blob()?;
                     // Strict UTF-8 loses nothing: bodies are written from `&str`
-                    // (`repo::set_content_done`), and `stored_text` also treats
+                    // (`repo::store_pending_text`), and `stored_text` also treats
                     // non-UTF-8 as absent.
                     Ok(doc.decode(blob).is_some_and(|t| re.is_match(t)))
                 })

crates/quicksearch-core/src/search/cascade/passes.rs

@@ -70,19 +70,17 @@ fn doc_scan_sql(filter_sql: &str) -> String {
     )
 }
 
-/// The scan behind a `searchabletext MATCH` prefilter. `require_text` picks
+/// The scan behind a `searchabletext MATCH` prefilter. Every candidate is
-/// the join to `documents_text`: `false` is [`Cx::pass_fulltext`]'s LEFT
+/// verified against its stored text, and a row with a posting always has
-/// JOIN, which keeps an FTS-matched row whose stored text was dropped.
+/// some: text is stored whenever there is any.
-fn fts_match_sql(filter_sql: &str, require_text: bool) -> String {
+fn fts_match_sql(filter_sql: &str) -> String {
     format!(
         "SELECT {}, dt.text_zstd \
          FROM searchabletext \
          JOIN files f ON f.id = searchabletext.rowid \
-         {} documents_text dt ON dt.file_id = f.id \
+         JOIN documents_text dt ON dt.file_id = f.id \
          WHERE searchabletext MATCH ?{}",
-        HIT_COLUMNS,
+        HIT_COLUMNS, filter_sql
-        if require_text { "JOIN" } else { "LEFT JOIN" },
-        filter_sql
     )
 }
 
@@ -109,12 +107,11 @@ fn anchor_segment(pattern: &crate::query::pattern::TermPattern) -> Option<&str>
 
 impl<'a> Cx<'a> {
     /// The SQL and parameters for a scan over document bodies: an FTS MATCH on
-    /// `expr` when there is one (`require_text` as in [`fts_match_sql`]),
+    /// `expr` when there is one, else every stored document.
-    /// else every stored document.
+    fn body_scan(&self, expr: Option<String>) -> (String, Vec<Value>) {
-    fn body_scan(&self, expr: Option<String>, require_text: bool) -> (String, Vec<Value>) {
         match expr {
             Some(expr) => (
-                fts_match_sql(&self.query.filter_sql, require_text),
+                fts_match_sql(&self.query.filter_sql),
                 self.params_with_filters(vec![Value::Text(expr)]),
             ),
             None => (
@@ -309,77 +306,60 @@ impl<'a> Cx<'a> {
             // Below the trigram floor the MATCH can't return anything.
             return Ok(true);
         }
-        // The column filter goes inside the MATCH (`text: "..."`) so only
+        // A literal term is one segment; a wildcard narrows with an AND of
-        // bodies match. A literal term is one quoted phrase; a wildcard
+        // its segments of 3+ characters, each asked for as the index allows
-        // narrows with an AND of its segments of 3+ characters; with none
+        // ([`prefilter::fts_literal`]); with none (`ab*cd`) the pass scans
-        // (`ab*cd`) the pass scans every stored document. Every row is
+        // every stored document. Every row is pattern-verified either way,
-        // pattern-verified either way.
+        // which is what makes an index without positions exact.
-        let match_expr = match pattern.literal() {
+        let match_expr = Some(
-            Some(term) => Some(format!("text: {}", quote_phrase(term))),
+            pattern
-            None => Some(
+                .segments()
-                pattern
+                .iter()
-                    .segments()
+                .filter_map(|s| prefilter::fts_literal(s, self.positions))
-                    .iter()
+                .collect::<Vec<_>>()
-                    .filter(|s| s.chars().count() >= prefilter::TRIGRAM_FLOOR)
+                .join(" AND "),
-                    .map(|s| format!("text: {}", quote_phrase(s)))
+        )
-                    .collect::<Vec<_>>()
+        .filter(|expr| !expr.is_empty());
-                    .join(" AND "),
+        let (sql, params) = self.body_scan(match_expr);
-            )
-            .filter(|expr| !expr.is_empty()),
-        };
-        let narrowed = match_expr.is_some();
-        let (sql, params) = self.body_scan(match_expr, false);
         let mut doc = crate::db::repo::DocDecoder::new()?;
         let mut lower = String::new();
         // Decompression dominates: check cancellation every row.
         self.scan_pass(&sql, params, 1, None, |cx, row, file_id, path, name| {
-            let text = stored_text(row, &mut doc)?;
+            // A body that will not decode cannot be verified.
-
+            let Some(text) = stored_text(row, &mut doc)? else {
-            let (rank, stage, snip) = match text {
+                return Ok(RowHit::Skip);
-                Some(text) => {
+            };
-                    // Fold once: count, first match and snippet all need it,
+            // Fold once: count, first match and snippet all need it, even on
-                    // even on rows about to be dropped.
+            // rows about to be dropped.
-                    fold_into(&mut lower, text);
+            fold_into(&mut lower, text);
-                    match super::text_snippet_counted(pattern, text, &lower) {
+            let (count, stage, snip) = match super::text_snippet_counted(pattern, text, &lower) {
-                        Some((snip, count_ci)) => {
+                Some((snip, count_ci)) => {
-                            if count_ci == 0 {
+                    if count_ci == 0 {
-                                return Ok(RowHit::Skip);
-                            }
-                            let count_cs = pattern.count(text, false);
-                            let (count, stage) = if count_cs > 0 {
-                                (count_cs, 5)
-                            } else {
-                                (count_ci, 6)
-                            };
-                            (stage as f64 + count_frac(count), stage as u8, Some(snip))
-                        }
-                        None => {
-                            let count_cs = pattern.count(text, false);
-                            let (count, stage) = if count_cs > 0 {
-                                (count_cs, 5)
-                            } else {
-                                let count_ci = pattern.count_folded(&lower);
-                                if count_ci == 0 {
-                                    return Ok(RowHit::Skip);
-                                }
-                                (count_ci, 6)
-                            };
-                            let snip = super::text_snippet(pattern, text, &lower);
-                            (stage as f64 + count_frac(count), stage as u8, snip)
-                        }
-                    }
-                }
-                // No stored text: FTS-narrowed rows are accepted at the bottom
-                // of rank 6 as count-unknown; on the full-scan fallback an
-                // unverifiable row is skipped.
-                None => {
-                    if !narrowed {
                         return Ok(RowHit::Skip);
                     }
-                    (6.0 + count_frac(1), 6, None)
+                    let count_cs = pattern.count(text);
+                    if count_cs > 0 {
+                        (count_cs, 5, Some(snip))
+                    } else {
+                        (count_ci, 6, Some(snip))
+                    }
+                }
+                None => {
+                    let count_cs = pattern.count(text);
+                    let (count, stage) = if count_cs > 0 {
+                        (count_cs, 5)
+                    } else {
+                        let count_ci = pattern.count_folded(&lower);
+                        if count_ci == 0 {
+                            return Ok(RowHit::Skip);
+                        }
+                        (count_ci, 6)
+                    };
+                    (count, stage, super::text_snippet(pattern, text, &lower))
                 }
             };
-            if !cx.regex_accepts(file_id, path, text)? {
+            let rank = stage as f64 + count_frac(count);
+            if !cx.regex_accepts(file_id, path, Some(text))? {
                 return Ok(RowHit::Skip);
             }
 
@@ -463,8 +443,8 @@ impl<'a> Cx<'a> {
             .and_then(|chunks| {
                 prefilter::Required::new(chunks.iter().map(|c| c.to_string()).collect())
             })
-            .and_then(|required| required.fts_expr());
+            .and_then(|required| required.fts_expr(self.positions));
-        let (sql, params) = self.body_scan(expr, true);
+        let (sql, params) = self.body_scan(expr);
         // No fold buffer: the matcher folds in its mask table.
         let mut doc = crate::db::repo::DocDecoder::new()?;
         // Decompression dominates: check cancellation every row.
@@ -541,7 +521,8 @@ impl<'a> Cx<'a> {
     pub(super) fn pass_regex_content(&mut self) -> Result<bool, String> {
         let query = self.query;
         let re = query.regex.as_ref().expect("regex-only pass list");
-        let (sql, params) = self.body_scan(re.required().and_then(|r| r.fts_expr()), true);
+        let expr = re.required().and_then(|r| r.fts_expr(self.positions));
+        let (sql, params) = self.body_scan(expr);
         let mut doc = crate::db::repo::DocDecoder::new()?;
         // Decompression dominates: check cancellation every row.
         self.scan_pass(&sql, params, 1, None, |_cx, row, file_id, path, name| {

crates/quicksearch-core/src/search/duplicates.rs

@@ -219,7 +219,7 @@ mod tests {
     fn seed(dir: &str, rows: &[(&str, u64, Option<&[u8]>)]) -> (Scratch, String) {
         let (scratch, p) = Scratch::db(dir);
         let p = p.to_str().unwrap().to_string();
-        let mut conn = open_or_recreate(&p, "trigram").unwrap();
+        let mut conn = open_or_recreate(&p, &Default::default()).unwrap();
         let tx = conn.transaction().unwrap();
         for &(name, size, hash) in rows {
             insert_file(

crates/quicksearch-core/src/search/mod.rs

@@ -55,7 +55,7 @@ pub struct SearchHit {
     pub rank: f64,
     pub stage: u8,
     /// The matched span in context: name, path, or a window of the body per
-    /// stage (absent for full-text stages when no text is stored).
+    /// stage; absent only where no window could be cut.
     pub snippet: Option<Snippet>,
 }
 

crates/quicksearch-core/src/search/prefilter.rs

@@ -24,6 +24,35 @@ pub const TRIGRAM_FLOOR: usize = 3;
 /// cheaper than the scan it replaces, and falling back is always correct.
 const MAX_LITERALS: usize = 32;
 
+/// A `searchabletext MATCH` expression every document containing `literal`
+/// satisfies, for an index built with or without positions (see
+/// [`crate::db::repo::fts_positions`]); `None` below [`TRIGRAM_FLOOR`]
+/// characters, where the tokenizer indexes nothing to find.
+///
+/// With positions it is the literal as one phrase, exact up to the index's
+/// case folding. Without them a phrase of several tokens is refused outright,
+/// so it is the AND of the literal's distinct trigrams, each a one-token
+/// phrase: a superset, since adjacency goes unchecked, which the full-text
+/// pass's verification narrows back. Either way [`quote_phrase`] renders
+/// FTS5 syntax in user text inert.
+pub fn fts_literal(literal: &str, positions: bool) -> Option<String> {
+    let chars: Vec<char> = literal.chars().collect();
+    if chars.len() < TRIGRAM_FLOOR {
+        return None;
+    }
+    if positions {
+        return Some(format!("text: {}", quote_phrase(literal)));
+    }
+    let mut seen = std::collections::HashSet::new();
+    let trigrams: Vec<String> = chars
+        .windows(TRIGRAM_FLOOR)
+        .map(|w| w.iter().collect::<String>())
+        .filter(|t| seen.insert(t.clone()))
+        .map(|t| quote_phrase(&t))
+        .collect();
+    Some(format!("({})", trigrams.join(" AND ")))
+}
+
 /// Literals of which at least one occurs in anything the query can match.
 #[derive(Debug, Clone)]
 pub struct Required(Vec<String>);
@@ -46,25 +75,20 @@ impl Required {
         &self.0
     }
 
-    /// A `searchabletext MATCH` expression: `(text: "a" OR text: "b" …)`.
+    /// A `searchabletext MATCH` expression: the OR of each literal's
+    /// [`fts_literal`].
     ///
     /// `None` when any literal is shorter than [`TRIGRAM_FLOOR`] characters:
     /// the tokenizer indexes character triples, so a byte-length test would
-    /// admit a phrase that matches no token. [`quote_phrase`] renders FTS5
+    /// admit a phrase that matches no token. The index folds case, so it
-    /// syntax in user text inert. The index folds case and diacritics, so it
     /// matches more than the literal as written, the harmless direction.
-    pub fn fts_expr(&self) -> Option<String> {
+    pub fn fts_expr(&self, positions: bool) -> Option<String> {
-        if self.0.iter().any(|l| l.chars().count() < TRIGRAM_FLOOR) {
+        let each = self
-            return None;
+            .0
-        }
+            .iter()
-        Some(format!(
+            .map(|l| fts_literal(l, positions))
-            "({})",
+            .collect::<Option<Vec<_>>>()?;
-            self.0
+        Some(format!("({})", each.join(" OR ")))
-                .iter()
-                .map(|l| format!("text: {}", quote_phrase(l)))
-                .collect::<Vec<_>>()
-                .join(" OR ")
-        ))
     }
 
     /// A predicate over the `files` columns, plus the values it binds:
@@ -114,23 +138,50 @@ mod tests {
 
     #[test]
     fn the_trigram_floor_counts_characters_not_bytes() {
-        assert!(req(&["café"]).unwrap().fts_expr().is_some());
+        for positions in [true, false] {
-        assert!(req(&["日本"]).unwrap().fts_expr().is_none());
+            assert!(req(&["café"]).unwrap().fts_expr(positions).is_some());
-        assert!(req(&["ab"]).unwrap().fts_expr().is_none());
+            assert!(req(&["日本"]).unwrap().fts_expr(positions).is_none());
-        assert!(
+            assert!(req(&["ab"]).unwrap().fts_expr(positions).is_none());
-            req(&["abc", "de"]).unwrap().fts_expr().is_none(),
+            assert!(
-            "all of them"
+                req(&["abc", "de"]).unwrap().fts_expr(positions).is_none(),
-        );
+                "all of them"
+            );
+        }
     }
 
     #[test]
     fn fts5_syntax_in_a_literal_is_quoted_inert() {
-        let expr = req(&["a\"b", "NEAR", "x*y"]).unwrap().fts_expr().unwrap();
+        let expr = req(&["a\"b", "NEAR", "x*y"])
+            .unwrap()
+            .fts_expr(true)
+            .unwrap();
         // The embedded quote is doubled, which is FTS5's own escape.
         assert!(expr.contains(r#""a""b""#), "{expr}");
         assert!(expr.contains(r#""NEAR""#), "{expr}");
         assert!(expr.contains(r#""x*y""#), "{expr}");
         assert_eq!(expr.matches(" OR ").count(), 2);
+
+        // Each trigram is its own quoted phrase, so a quote inside one is
+        // escaped the same way.
+        let expr = req(&["a\"bc"]).unwrap().fts_expr(false).unwrap();
+        assert!(expr.contains(r#""a""b""#), "{expr}");
+        assert!(expr.contains(r#""""bc""#), "{expr}");
+    }
+
+    /// Without positions a literal is the AND of its distinct trigrams, the
+    /// literals still OR'ed: the only shape a `detail=none` table accepts.
+    #[test]
+    fn without_positions_a_literal_is_its_trigrams() {
+        assert_eq!(fts_literal("abcd", false).unwrap(), r#"("abc" AND "bcd")"#);
+        assert_eq!(
+            fts_literal("aaaa", false).unwrap(),
+            r#"("aaa")"#,
+            "a repeated trigram is asked for once"
+        );
+        assert_eq!(fts_literal("café", false).unwrap(), r#"("caf" AND "afé")"#);
+        assert_eq!(fts_literal("abcd", true).unwrap(), r#"text: "abcd""#);
+        let expr = req(&["abc", "xyz"]).unwrap().fts_expr(false).unwrap();
+        assert_eq!(expr, r#"(("abc") OR ("xyz"))"#);
     }
 
     #[test]

crates/quicksearch-core/src/snippet.rs

@@ -1,7 +1,7 @@
 //! Snippet extraction for search results; contentless FTS5 has no
 //! `snippet()`. Output is structural (window text plus byte ranges), never
-//! markup. Matching folds ASCII only: `cafe` finds `café` via the tokenizer,
+//! markup. Matching folds ASCII case only, as full-text verification does:
-//! but the snippet does not mark the accented occurrence.
+//! `cafe` neither finds nor marks `café`.
 
 #[derive(Debug, Clone)]
 pub struct Options {
@@ -44,75 +44,58 @@ impl Snippet {
 /// must be `text.to_ascii_lowercase()`: the fold preserves byte length, so
 /// offsets found in it slice the original.
 ///
-/// Returns the window and how many occurrences it found before touching
+/// Returns the window and how many occurrences of `term` it found before
-/// ranges were coalesced, the count the full-text pass ranks on. Blank terms
+/// touching ranges were coalesced, the count the full-text pass ranks on. A
-/// are dropped, but a padded term is searched as given, never trimmed.
+/// blank term matches nothing, but a padded term is searched as given, never
-pub fn extract_folded(
+/// trimmed.
-    text: &str,
+pub fn extract_folded(text: &str, folded: &str, term: &str, opts: &Options) -> (Snippet, usize) {
-    folded: &str,
-    terms: &[&str],
-    opts: &Options,
-) -> (Snippet, usize) {
     debug_assert_eq!(folded.len(), text.len(), "ASCII folding preserves length");
     if text.is_empty() {
         return (Snippet::empty(), 0);
     }
-    let effective_terms: Vec<&str> = terms
+    if term.trim().is_empty() {
-        .iter()
-        .copied()
-        .filter(|t| !t.trim().is_empty())
-        .collect();
-    if effective_terms.is_empty() {
         return (head_window(text, opts.approx_bytes), 0);
     }
 
+    // Borrow when the term is already folded, as every cascade call's is,
+    // to avoid an allocation per candidate row.
+    let pattern: std::borrow::Cow<'_, str> = if term.bytes().any(|b| b.is_ascii_uppercase()) {
+        std::borrow::Cow::Owned(term.to_ascii_lowercase())
+    } else {
+        std::borrow::Cow::Borrowed(term)
+    };
     // `memmem`, not `str::match_indices`: measured faster on whole-body
     // scans. The walk runs to the end (the count must see every occurrence),
-    // but matches past the window's right edge are not stored. Only a single
+    // but matches past the window's right edge are not stored.
-    // term is bounded: with several, two terms can coalesce across the edge,
-    // which no single-pass bound sees.
-    let bounded = effective_terms.len() == 1;
     let mut matches: Vec<(usize, usize)> = Vec::new();
     let mut found = 0usize;
     // The window's right edge once the first match fixes it, from the same
     // [`window_bounds`] the window itself uses.
     let mut keep_below: Option<usize> = None;
-
+    // The end of the last kept match, so a chain of touching occurrences
-    for term in &effective_terms {
+    // (`abab…` for `ab`) that starts inside the window and continues past
-        // Borrow when the term is already folded, as every cascade call's is,
+    // it stays intact for `coalesce_overlapping` and the expansion step.
-        // to avoid an allocation per candidate row.
+    let mut chain_end = 0usize;
-        let pattern: std::borrow::Cow<'_, str> = if term.bytes().any(|b| b.is_ascii_uppercase()) {
+    for at in memchr::memmem::find_iter(folded.as_bytes(), pattern.as_bytes()) {
-            std::borrow::Cow::Owned(term.to_ascii_lowercase())
+        // Before any dropping: two abutting occurrences are two hits for
-        } else {
+        // ranking even though they paint as one highlight.
-            std::borrow::Cow::Borrowed(*term)
+        found += 1;
-        };
+        let bound = *keep_below.get_or_insert_with(|| window_bounds(text, at, opts.approx_bytes).1);
-        // The end of the last kept match, so a chain of touching occurrences
+        if at >= bound && at > chain_end {
-        // (`abab…` for `ab`) that starts inside the window and continues past
+            // Past the edge and not chained to anything kept: keep counting,
-        // it stays intact for `coalesce_overlapping` and the expansion step.
+            // stop storing.
-        let mut chain_end = 0usize;
+            continue;
-        for at in memchr::memmem::find_iter(folded.as_bytes(), pattern.as_bytes()) {
-            // Before any dropping: two abutting occurrences are two hits for
-            // ranking even though they paint as one highlight.
-            found += 1;
-            if bounded {
-                let bound =
-                    *keep_below.get_or_insert_with(|| window_bounds(text, at, opts.approx_bytes).1);
-                if at >= bound && at > chain_end {
-                    // Past the edge and not chained to anything kept: keep
-                    // counting, stop storing.
-                    continue;
-                }
-            }
-            chain_end = at + pattern.len();
-            matches.push((at, chain_end));
         }
+        chain_end = at + pattern.len();
+        matches.push((at, chain_end));
     }
 
     if matches.is_empty() {
         return (head_window(text, opts.approx_bytes), 0);
     }
 
-    matches.sort_by_key(|(a, _)| *a);
+    // One term's occurrences arrive in order and never overlap; only
+    // abutting ones merge.
     let matches = coalesce_overlapping(matches);
 
     let (win_start, mut win_end) = window_bounds(text, matches[0].0, opts.approx_bytes);
@@ -213,21 +196,13 @@ pub fn window_around(text: &str, range: (usize, usize), opts: &Options) -> Snipp
     }
 }
 
-/// Count non-overlapping occurrences of `term` in `text`. Case-insensitive
+/// Count non-overlapping occurrences of `term` in `text`, byte for byte: a
-/// counting folds ASCII only, matching the rest of the search pipeline.
+/// case-insensitive count passes both already folded.
-pub fn count_occurrences(text: &str, term: &str, case_sensitive: bool) -> usize {
+pub fn count_occurrences(text: &str, term: &str) -> usize {
     if term.is_empty() || text.len() < term.len() {
         return 0;
     }
-    if case_sensitive {
+    memchr::memmem::find_iter(text.as_bytes(), term.as_bytes()).count()
-        memchr::memmem::find_iter(text.as_bytes(), term.as_bytes()).count()
-    } else {
-        memchr::memmem::find_iter(
-            text.to_ascii_lowercase().as_bytes(),
-            term.to_ascii_lowercase().as_bytes(),
-        )
-        .count()
-    }
 }
 
 /// The first `n` bytes of `text` (char-aligned) as a match-less window.
@@ -264,12 +239,12 @@ fn coalesce_overlapping(v: Vec<(usize, usize)>) -> Vec<(usize, usize)> {
 mod tests {
     use super::*;
 
-    fn extract(text: &str, terms: &[&str], opts: &Options) -> Snippet {
+    fn extract(text: &str, term: &str, opts: &Options) -> Snippet {
-        extract_folded(text, &text.to_ascii_lowercase(), terms, opts).0
+        extract_folded(text, &text.to_ascii_lowercase(), term, opts).0
     }
 
-    fn extract_counted(text: &str, terms: &[&str], opts: &Options) -> (Snippet, usize) {
+    fn extract_counted(text: &str, term: &str, opts: &Options) -> (Snippet, usize) {
-        extract_folded(text, &text.to_ascii_lowercase(), terms, opts)
+        extract_folded(text, &text.to_ascii_lowercase(), term, opts)
     }
 
     fn opts_small() -> Options {
@@ -280,11 +255,11 @@ mod tests {
     #[test]
     fn extract_folded_counts_occurrences_not_ranges() {
         let text = "abab and ab";
-        let (s, n) = extract_counted(text, &["ab"], &opts_small());
+        let (s, n) = extract_counted(text, "ab", &opts_small());
         assert_eq!(n, 3, "three occurrences");
         assert_eq!(
             n,
-            count_occurrences(text, "ab", true),
+            count_occurrences(text, "ab"),
             "must agree with count_occurrences"
         );
         assert_eq!(s.ranges.len(), 2, "the abutting pair paints as one range");
@@ -304,10 +279,10 @@ mod tests {
         ];
         for (text, term) in cases {
             let folded = text.to_ascii_lowercase();
-            let (_, n) = extract_folded(text, &folded, &[term], &opts_small());
+            let (_, n) = extract_folded(text, &folded, term, &opts_small());
             assert_eq!(
                 n,
-                count_occurrences(&folded, &term.to_ascii_lowercase(), true),
+                count_occurrences(&folded, &term.to_ascii_lowercase()),
                 "count mismatch for {:?} in {:?}",
                 term,
                 text
@@ -321,8 +296,8 @@ mod tests {
     fn extract_folded_needle_case_does_not_change_matches() {
         let text = "The Needle and the needle";
         let folded = text.to_ascii_lowercase();
-        let (upper, n_upper) = extract_folded(text, &folded, &["Needle"], &opts_small());
+        let (upper, n_upper) = extract_folded(text, &folded, "Needle", &opts_small());
-        let (lower, n_lower) = extract_folded(text, &folded, &["needle"], &opts_small());
+        let (lower, n_lower) = extract_folded(text, &folded, "needle", &opts_small());
         assert_eq!(n_upper, 2);
         assert_eq!(n_upper, n_lower);
         assert_eq!(upper.ranges, lower.ranges);
@@ -335,7 +310,7 @@ mod tests {
         let text = "some text";
         let folded = text.to_ascii_lowercase();
         for term in ["", "   ", "\t"] {
-            let (s, n) = extract_folded(text, &folded, &[term], &opts_small());
+            let (s, n) = extract_folded(text, &folded, term, &opts_small());
             assert_eq!(n, 0, "blank term {:?} counted", term);
             assert!(s.ranges.is_empty(), "blank term {:?} highlighted", term);
         }
@@ -346,9 +321,9 @@ mod tests {
     fn extract_folded_does_not_trim_a_padded_term() {
         let text = "needle needlework";
         let folded = text.to_ascii_lowercase();
-        let (_, n) = extract_folded(text, &folded, &["needle "], &opts_small());
+        let (_, n) = extract_folded(text, &folded, "needle ", &opts_small());
         assert_eq!(n, 1, "only the occurrence followed by a space");
-        assert_eq!(n, count_occurrences(&folded, "needle ", true));
+        assert_eq!(n, count_occurrences(&folded, "needle "));
     }
 
     /// The `Snippet::ranges` contract egui's LayoutJob sections rely on.
@@ -374,7 +349,7 @@ mod tests {
     fn occurrences_past_the_window_are_counted_but_not_kept() {
         let unit = "needle filler filler ";
         let text = unit.repeat(400);
-        let (snip, found) = extract_counted(&text, &["needle"], &opts_small());
+        let (snip, found) = extract_counted(&text, "needle", &opts_small());
 
         assert_eq!(found, 400, "every occurrence must still be counted");
         assert_ranges_valid(&snip);
@@ -403,7 +378,7 @@ mod tests {
     #[test]
     fn a_coalescing_chain_is_not_cut_at_the_window_edge() {
         let text = "ab".repeat(400);
-        let (snip, found) = extract_counted(&text, &["ab"], &opts_small());
+        let (snip, found) = extract_counted(&text, "ab", &opts_small());
 
         assert_eq!(found, 400, "non-overlapping occurrences, all counted");
         assert_ranges_valid(&snip);
@@ -428,31 +403,16 @@ mod tests {
         );
     }
 
-    /// Several terms disable the bound; pin that the multi-term path still
-    /// produces the full, correct answer.
-    #[test]
-    fn several_terms_still_coalesce_across_the_edge() {
-        let text = format!("{}{}", "x".repeat(10), "abc".repeat(200));
-        let (snip, found) = extract_counted(&text, &["ab", "bc"], &opts_small());
-        assert_eq!(found, 400, "200 of each term");
-        assert_ranges_valid(&snip);
-        assert_eq!(
-            snip.ranges.len(),
-            1,
-            "the two terms interleave into one run"
-        );
-    }
-
     #[test]
     fn empty_text_returns_empty() {
-        let s = extract("", &["foo"], &Options::default());
+        let s = extract("", "foo", &Options::default());
         assert_eq!(s, Snippet::empty());
     }
 
     #[test]
-    fn no_terms_returns_head_marked_truncated() {
+    fn a_blank_term_returns_head_marked_truncated() {
         let long = "abcdefghijklmnop".repeat(10);
-        let s = extract(&long, &[], &opts_small());
+        let s = extract(&long, "", &opts_small());
         assert!(s.truncated_end);
         assert!(!s.truncated_start);
         assert!(s.ranges.is_empty());
@@ -460,40 +420,29 @@ mod tests {
     }
 
     #[test]
-    fn no_terms_untruncated() {
+    fn a_blank_term_untruncated() {
-        let s = extract("short text", &[], &opts_small());
+        let s = extract("short text", "", &opts_small());
         assert_eq!(s.window, "short text");
         assert!(!s.truncated_end && !s.truncated_start);
     }
 
     #[test]
     fn simple_match_range() {
-        let s = extract("the quick brown fox", &["quick"], &opts_small());
+        let s = extract("the quick brown fox", "quick", &opts_small());
         assert_eq!(marked(&s), vec!["quick"]);
         assert_ranges_valid(&s);
     }
 
     #[test]
     fn case_insensitive_match_reports_original_case() {
-        let s = extract("The QUICK brown fox", &["quick"], &opts_small());
+        let s = extract("The QUICK brown fox", "quick", &opts_small());
         assert_eq!(marked(&s), vec!["QUICK"]);
     }
 
-    #[test]
-    fn multiple_terms_both_marked() {
-        let s = extract(
-            "the quick brown fox jumps over the lazy dog",
-            &["quick", "lazy"],
-            &Options::default(),
-        );
-        assert_eq!(marked(&s), vec!["quick", "lazy"]);
-        assert_ranges_valid(&s);
-    }
-
     #[test]
     fn window_truncation_flags_on_both_sides() {
         let text = "prefix ".repeat(20) + "MATCH in middle " + &"suffix ".repeat(20);
-        let s = extract(&text, &["MATCH"], &opts_small());
+        let s = extract(&text, "MATCH", &opts_small());
         assert!(s.truncated_start);
         assert!(s.truncated_end);
         assert_eq!(marked(&s), vec!["MATCH"]);
@@ -503,7 +452,7 @@ mod tests {
     fn match_at_start_not_truncated_left() {
         let s = extract(
             "MATCH right at the start of this paragraph",
-            &["match"],
+            "match",
             &opts_small(),
         );
         assert!(!s.truncated_start);
@@ -513,22 +462,15 @@ mod tests {
     #[test]
     fn no_match_returns_head_without_ranges() {
         let text = "alpha beta gamma delta epsilon zeta eta theta iota kappa";
-        let s = extract(text, &["nomatch"], &opts_small());
+        let s = extract(text, "nomatch", &opts_small());
         assert!(s.ranges.is_empty());
         assert!(s.window.starts_with("alpha"));
     }
 
-    #[test]
-    fn overlapping_terms_coalesce() {
-        let s = extract("the RUSTC compiler", &["rust", "rustc"], &opts_small());
-        assert_eq!(marked(&s), vec!["RUSTC"]);
-        assert_ranges_valid(&s);
-    }
-
     #[test]
     fn utf8_boundaries_hold_with_multibyte_text() {
         let text = "café café café café café café café café café café";
-        let s = extract(text, &["café"], &opts_small());
+        let s = extract(text, "café", &opts_small());
         assert!(!s.ranges.is_empty());
         assert_ranges_valid(&s);
         for m in marked(&s) {
@@ -540,16 +482,10 @@ mod tests {
     fn match_near_right_edge_is_fully_included() {
         let prefix = "x".repeat(30);
         let text = format!("{}{}", prefix, "LONGMATCHTERMTEXT");
-        let s = extract(&text, &["LONGMATCHTERMTEXT"], &opts_small());
+        let s = extract(&text, "LONGMATCHTERMTEXT", &opts_small());
         assert_eq!(marked(&s), vec!["LONGMATCHTERMTEXT"]);
     }
 
-    #[test]
-    fn empty_query_term_ignored() {
-        let s = extract("hello world", &["", "world"], &opts_small());
-        assert_eq!(marked(&s), vec!["world"]);
-    }
-
     #[test]
     fn window_around_basic() {
         let text = "prefix ".repeat(20) + "NEEDLE" + &" suffix".repeat(20);
@@ -579,12 +515,11 @@ mod tests {
 
     #[test]
     fn count_occurrences_cases() {
-        assert_eq!(count_occurrences("aaaa", "aaa", true), 1, "non-overlapping");
+        assert_eq!(count_occurrences("aaaa", "aaa"), 1, "non-overlapping");
-        assert_eq!(count_occurrences("abcABC", "abc", true), 1);
+        assert_eq!(count_occurrences("abcABC", "abc"), 1, "byte for byte");
-        assert_eq!(count_occurrences("abcABC", "abc", false), 2);
+        assert_eq!(count_occurrences("", "x"), 0);
-        assert_eq!(count_occurrences("", "x", true), 0);
+        assert_eq!(count_occurrences("xyz", ""), 0);
-        assert_eq!(count_occurrences("xyz", "", true), 0);
+        assert_eq!(count_occurrences("no hits here", "zzz"), 0);
-        assert_eq!(count_occurrences("no hits here", "zzz", false), 0);
+        assert_eq!(count_occurrences("ab ab ab", "ab"), 3);
-        assert_eq!(count_occurrences("ab ab ab", "ab", true), 3);
     }
 }

crates/quicksearch-core/src/testutil.rs

@@ -7,9 +7,9 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 
 static NEXT: AtomicUsize = AtomicUsize::new(0);
 
-/// The compressed body [`crate::db::repo::set_content_done`] wants.
+/// The compressed body [`crate::db::repo::store_pending_text`] wants.
 pub fn zstd_of(text: &str) -> Option<Vec<u8>> {
-    crate::db::repo::encode_one(text, true).expect("zstd encode")
+    crate::db::repo::encode_one(text).expect("zstd encode")
 }
 
 /// Old enough that a failure investigated the next morning has its tree.
@@ -338,7 +338,7 @@ impl Default for SeedSpec {
 /// Seed an index with synthetic rows, in one transaction. Shared by the
 /// measurement harnesses so they all describe the same corpus.
 pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) {
-    use crate::db::repo::{insert_file, set_content_done, NewFile};
+    use crate::db::repo::{insert_file, store_pending_text, NewFile};
     use crate::mime::mime_to_type;
 
     // Before the open: the profile decides how the file is created, and on a
@@ -349,7 +349,7 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) {
     if let Some(hmac) = spec.hmac {
         crate::db::set_hmac_mode_override(hmac);
     }
-    let conn = crate::db::open_or_recreate(path.to_str().unwrap(), "trigram").unwrap();
+    let conn = crate::db::open_or_recreate(path.to_str().unwrap(), &Default::default()).unwrap();
     // Before the first row: `pgsz` decides how leaves are built, so setting it
     // afterwards would only affect segments merged later.
     if let Some(pgsz) = spec.pgsz {
@@ -415,7 +415,7 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) {
             })
         });
         // `needs_content` rows are born `STATE_PENDING`, so skipping the
-        // `set_content_done` below leaves one in the queue.
+        // `store_pending_text` below leaves one in the queue.
         let needs_content = i % spec.content_every.max(1) == 0 && registry.supports(mime);
         let extracted = needs_content && (spec.pending_every == 0 || i % spec.pending_every != 0);
         let id = insert_file(
@@ -444,7 +444,7 @@ pub fn seed_index(path: &std::path::Path, spec: &SeedSpec) {
                 body[spec.body_words * 2 / 3] = BODY_TERM;
             }
             let body = body.join(" ");
-            set_content_done(&tx, id, &body, zstd_of(&body).as_deref()).unwrap();
+            store_pending_text(&tx, id, &body, zstd_of(&body).as_deref()).unwrap();
         }
         if spec.commit_every > 0 && (i + 1) % spec.commit_every == 0 {
             tx.commit().unwrap();
@@ -495,6 +495,30 @@ pub fn sniff(path: &std::path::Path, hash_length: usize) -> Option<&'static str>
     crate::mime::guess_mime_from_head(path, &head)
 }
 
+/// The rows the full-text index matches for `term`, asked for as a search
+/// would ask ([`crate::search::prefilter::fts_literal`], by the layout the
+/// index records): a bare `MATCH 'word'` is a phrase of several trigrams,
+/// which an index without positions refuses. Unverified, so on such an index
+/// these are candidates, a superset of the documents containing `term`.
+pub fn fts_rowids(conn: &rusqlite::Connection, term: &str) -> Vec<i64> {
+    let expr = crate::search::prefilter::fts_literal(term, crate::db::repo::fts_positions(conn))
+        .expect("a term of three characters or more");
+    let mut stmt = conn
+        .prepare("SELECT rowid FROM searchabletext WHERE searchabletext MATCH ?1")
+        .expect("prepare fts query");
+    let rows = stmt
+        .query_map([expr], |r| r.get(0))
+        .expect("fts query")
+        .collect::<Result<Vec<i64>, _>>()
+        .expect("fts rows");
+    rows
+}
+
+/// How many rows [`fts_rowids`] finds.
+pub fn fts_count(conn: &rusqlite::Connection, term: &str) -> i64 {
+    fts_rowids(conn, term).len() as i64
+}
+
 /// `(leaf, overflow)` pages in `searchabletext_data`, from `dbstat`. An
 /// overflow page is a second fetch and decrypt on every read of its leaf;
 /// [`crate::db::schema::fts_pgsz_for`] keeps the count at zero.

crates/quicksearch-core/src/textenc.rs

@@ -77,13 +77,22 @@ pub fn looks_like_text(head: &[u8]) -> bool {
 /// Takes the buffer by value so a UTF-8 file, the overwhelming majority,
 /// becomes its `String` with no copy. Callers holding only a borrow want
 /// [`decode_borrowed_text`].
-pub fn decode_text(bytes: Vec<u8>, path: &Path) -> Result<String, String> {
+///
+/// `truncated` marks a prefix of the file rather than all of it: a multibyte
+/// character cut by the read is dropped instead of demoting the file to
+/// `Legacy`. The other encodings replace a cut tail with one U+FFFD.
+pub fn decode_text(mut bytes: Vec<u8>, path: &Path, truncated: bool) -> Result<String, String> {
     if bytes.is_empty() {
         return Ok(String::new());
     }
-    match classify(&bytes, false) {
+    match classify(&bytes, truncated) {
-        // Cannot fail: classify ran strict validation with truncated=false.
+        TextClass::Utf8 => {
-        TextClass::Utf8 => Ok(String::from_utf8(bytes).expect("classified as UTF-8")),
+            // `classify` forgave only a cut final character; drop its stub.
+            if let Err(e) = std::str::from_utf8(&bytes) {
+                bytes.truncate(e.valid_up_to());
+            }
+            Ok(String::from_utf8(bytes).expect("classified as UTF-8"))
+        }
         _ => decode_borrowed_text(&bytes, path),
     }
 }
@@ -157,7 +166,7 @@ mod tests {
         let body = "plain ascii and café über 日本語".as_bytes().to_vec();
         assert!(looks_like_text(&body));
         assert_eq!(
-            decode_text(body, &p()).unwrap(),
+            decode_text(body, &p(), false).unwrap(),
             "plain ascii and café über 日本語"
         );
     }
@@ -167,7 +176,7 @@ mod tests {
         let mut body = vec![0xEF, 0xBB, 0xBF];
         body.extend_from_slice("hello".as_bytes());
         assert!(looks_like_text(&body));
-        let text = decode_text(body, &p()).unwrap();
+        let text = decode_text(body, &p(), false).unwrap();
         assert_eq!(text, "hello", "BOM must not survive into stored text");
     }
 
@@ -180,7 +189,7 @@ mod tests {
             body.extend_from_slice(&unit.to_le_bytes());
         }
         assert!(looks_like_text(&body));
-        assert_eq!(decode_text(body, &p()).unwrap(), src);
+        assert_eq!(decode_text(body, &p(), false).unwrap(), src);
     }
 
     #[test]
@@ -191,7 +200,7 @@ mod tests {
             body.extend_from_slice(&unit.to_be_bytes());
         }
         assert!(looks_like_text(&body));
-        assert_eq!(decode_text(body, &p()).unwrap(), src);
+        assert_eq!(decode_text(body, &p(), false).unwrap(), src);
     }
 
     /// Legacy charsets decode, but do not *sniff*.
@@ -203,7 +212,7 @@ mod tests {
             "the sniff must not adopt a non-UTF-8 head on its own"
         );
         assert_eq!(
-            decode_text(body, &p()).unwrap(),
+            decode_text(body, &p(), false).unwrap(),
             "Le café près de la fenêtre est agréable en été."
         );
     }
@@ -217,7 +226,7 @@ mod tests {
             body.extend_from_slice(head);
         }
         body.extend_from_slice(b"caf\xe9-tail-marker");
-        let out = decode_text(body, &p()).unwrap();
+        let out = decode_text(body, &p(), false).unwrap();
         assert!(
             out.ends_with("café-tail-marker"),
             "the tail past the detection prefix must still be decoded"
@@ -230,7 +239,7 @@ mod tests {
         let body = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9.".to_vec();
         assert!(body.len() < DETECT_PREFIX);
         assert_eq!(
-            decode_text(body, &p()).unwrap(),
+            decode_text(body, &p(), false).unwrap(),
             "Le café près de la fenêtre est agréable en été."
         );
     }
@@ -242,14 +251,14 @@ mod tests {
         assert!(!had_errors);
         let body = encoded.into_owned();
         assert!(!looks_like_text(&body));
-        assert_eq!(decode_text(body, &p()).unwrap(), src);
+        assert_eq!(decode_text(body, &p(), false).unwrap(), src);
     }
 
     #[test]
     fn nul_bytes_are_binary() {
         let body = b"looks like text until\x00it does not".to_vec();
         assert!(!looks_like_text(&body));
-        let err = decode_text(body, &p()).unwrap_err();
+        let err = decode_text(body, &p(), false).unwrap_err();
         assert!(err.contains("binary content"), "{err}");
         assert!(
             err.contains("textenc-test-file"),
@@ -262,7 +271,7 @@ mod tests {
         // 4 control bytes in 24 total = 16% > 10%.
         let body = b"abcdefghijklmnopqrst\x01\x02\x03\x04".to_vec();
         assert!(!looks_like_text(&body));
-        assert!(decode_text(body, &p()).is_err());
+        assert!(decode_text(body, &p(), false).is_err());
     }
 
     #[test]
@@ -270,7 +279,7 @@ mod tests {
         let body =
             b"\x1b[31mERROR\x1b[0m something failed\n\x1b[33mWARN\x1b[0m retrying\n".to_vec();
         assert!(looks_like_text(&body));
-        assert!(decode_text(body, &p()).is_ok());
+        assert!(decode_text(body, &p(), false).is_ok());
     }
 
     #[test]
@@ -280,13 +289,13 @@ mod tests {
         assert!(looks_like_text(&head));
         // The same bytes as a complete file are not valid UTF-8, so the
         // decoder treats them as legacy-encoded: still Ok, never a panic.
-        assert!(decode_text(head, &p()).is_ok());
+        assert!(decode_text(head, &p(), false).is_ok());
     }
 
     #[test]
     fn empty_head_is_not_text_but_empty_file_decodes() {
         assert!(!looks_like_text(b""));
-        assert_eq!(decode_text(Vec::new(), &p()).unwrap(), "");
+        assert_eq!(decode_text(Vec::new(), &p(), false).unwrap(), "");
     }
 
     /// UTF-16 without a BOM is out of scope by decision: its NULs trip the
@@ -298,7 +307,7 @@ mod tests {
             body.extend_from_slice(&unit.to_le_bytes());
         }
         assert!(!looks_like_text(&body));
-        assert!(decode_text(body, &p()).is_err());
+        assert!(decode_text(body, &p(), false).is_err());
     }
 
     /// Every head the sniff accepts must decode, which makes "sniffed as
@@ -329,7 +338,7 @@ mod tests {
                 "sniff verdict changed for {head:?}"
             );
             assert!(
-                decode_text(head.clone(), &p()).is_ok(),
+                decode_text(head.clone(), &p(), false).is_ok(),
                 "head failed to decode: {head:?}"
             );
         }

crates/quicksearch-core/src/walk.rs

@@ -12,7 +12,6 @@ use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
 use std::sync::{mpsc, Arc, Condvar, Mutex};
 use std::thread::JoinHandle;
-use std::time::UNIX_EPOCH;
 
 use crate::config::{Config, IgnoreSet};
 use crate::extract::Registry;
@@ -170,8 +169,17 @@ fn read_directory(
         }
     };
     // Every `continue` in the loop must be a genuine "not indexable", or the
-    // stale diff below deletes live rows.
+    // stale diff below deletes live rows. Only names that have a row matter to
-    let mut present: HashSet<String> = HashSet::new();
+    // that diff, so `present` borrows them from `rows`: no allocation per
+    // entry, and none at all for a directory with no rows yet. Measured
+    // (`indexprobe cold`, 100k files): 8.2 → 7.2 allocations per file,
+    // churn 42 → 35 MiB; time did not move, being `stat`-bound.
+    let mut present: HashSet<&str> = HashSet::new();
+    let mut mark_present = |name: &str| {
+        if let Some((known, _)) = rows.get_key_value(name) {
+            present.insert(known.as_str());
+        }
+    };
     let mut unreadable_entry = false;
 
     let mut files = Vec::new();
@@ -263,15 +271,13 @@ fn read_directory(
                 }
             }
             Ok(_) => {
-                present.insert(name.to_string());
+                mark_present(name);
                 // `None` on Unix and on any reparse point.
                 let cached = crate::platform::entry_cached_metadata(|| entry.metadata().ok());
                 files.push(PendingFile { path, cached });
             }
             // Type unknown: mark it present so an existing row survives.
-            Err(_) => {
+            Err(_) => mark_present(name),
-                present.insert(name.to_string());
-            }
         }
     }
 
@@ -363,12 +369,7 @@ fn prepare(
         // "deleted".
         return WalkedFile::skipped(db_path, aliased);
     };
-    let Some(mtime) = meta
+    let Some(mtime) = crate::file_handling::mtime_secs(&meta) else {
-        .modified()
-        .ok()
-        .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
-        .map(|d| d.as_secs())
-    else {
         return WalkedFile::skipped(db_path, aliased);
     };
 
@@ -517,6 +518,11 @@ impl ParallelWalk {
     /// Every canonical directory the walk queued, in `files.parent` spelling.
     /// The vanished-directory sweep needs this: a directory deleted wholesale
     /// is never read, so nothing reconciles the rows beneath it.
+    ///
+    /// Measured and not adopted: moving the set out and reusing each path's
+    /// buffer, so it is never held twice, left VmHWM where it was
+    /// (`memprobe cold`, 200k directories: 84–105 MiB against 86–97 MiB). The
+    /// run's peak is not at the sweep.
     pub fn seen_dirs(&self) -> HashSet<String> {
         crate::lock_ok(&self.shared.queue)
             .seen_dirs

crates/quicksearch-core/src/walk/tests.rs

@@ -1,5 +1,7 @@
 use super::*;
 
+use std::time::UNIX_EPOCH;
+
 use crate::testutil::{touch, Scratch};
 
 fn tmp_tree(tag: &str) -> Scratch {
@@ -20,7 +22,7 @@ impl std::ops::Deref for TestDb {
 
 fn db_with(tag: &str, rows: &[(String, u64)]) -> TestDb {
     let (dir, p) = Scratch::db(tag);
-    let conn = crate::db::open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
+    let conn = crate::db::open_or_recreate(p.to_str().unwrap(), &Default::default()).unwrap();
     for (path, mtime) in rows {
         // Split the same way the indexer does, so the seeded parent carries
         // its trailing separator and the prefetcher's `parent = ?` finds it.
@@ -789,7 +791,7 @@ fn the_index_and_its_sidecars_are_never_walked() {
     touch(&root.join("ordinary.txt"), b"x");
 
     let db = root.join("data").join("index.sqlite");
-    let conn = crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
+    let conn = crate::db::open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap();
     // A write, so the WAL and SHM exist to be walked over.
     conn.execute(
         "INSERT INTO files (name, parent, size, mtime, type, content_state)
@@ -822,7 +824,7 @@ fn the_index_and_its_sidecars_are_never_walked() {
 fn a_stored_row_for_the_index_falls_to_the_stale_sweep() {
     let root = crate::testutil::scratch_dir_canonical("walk-self-index-stale");
     let db = root.join("data").join("index.sqlite");
-    let conn = crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
+    let conn = crate::db::open_or_recreate(db.to_str().unwrap(), &Default::default()).unwrap();
 
     // A row for the database itself.
     let stored = path_to_db_string(&db);

crates/quicksearch-core/tests/cascade.rs

@@ -5,8 +5,9 @@ use std::path::Path;
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::Arc;
 
+use quicksearch_core::config::ProcessingConfig;
 use quicksearch_core::db::open_or_recreate;
-use quicksearch_core::db::repo::{insert_file, set_content_done, NewFile};
+use quicksearch_core::db::repo::{insert_file, store_pending_text, NewFile};
 use quicksearch_core::mime::FileType;
 use quicksearch_core::query::split::split_for_cascade;
 use quicksearch_core::search::{
@@ -16,14 +17,21 @@ use quicksearch_core::testutil::{zstd_of, Scratch};
 
 struct Seeder {
     conn: rusqlite::Connection,
-    store_text: bool,
 }
 
 impl Seeder {
-    fn new(path: &Path, store_text: bool) -> Seeder {
+    /// An index in the default layout: compact, without positions.
+    fn new(path: &Path) -> Seeder {
+        Seeder::with_positions(path, false)
+    }
+
+    fn with_positions(path: &Path, positions: bool) -> Seeder {
+        let processing = ProcessingConfig {
+            compact_text_index: !positions,
+            ..Default::default()
+        };
         Seeder {
-            conn: open_or_recreate(path.to_str().unwrap(), "trigram").unwrap(),
+            conn: open_or_recreate(path.to_str().unwrap(), &processing).unwrap(),
-            store_text,
         }
     }
 
@@ -48,8 +56,7 @@ impl Seeder {
         .unwrap()
         .expect("unique path");
         if let Some(text) = text {
-            let zstd = self.store_text.then(|| zstd_of(text)).flatten();
+            store_pending_text(&tx, id, text, zstd_of(text).as_deref()).unwrap();
-            set_content_done(&tx, id, text, zstd.as_deref()).unwrap();
         }
         tx.commit().unwrap();
         id
@@ -130,7 +137,7 @@ fn fuzzy_options_with_edits(max_edits: usize) -> SearchOptions {
 #[test]
 fn rank_classification_across_all_stages() {
     let (_dir, p) = Scratch::db("ranks");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let rank1 = s.add("Report", "/a", 1, None);
     let rank2 = s.add("report", "/b", 2, None);
     let rank3 = s.add("Quarterly_Report.txt", "/c", 3, None);
@@ -191,7 +198,7 @@ fn rank_classification_across_all_stages() {
 #[test]
 fn path_substring_tiers_split_by_case() {
     let (_dir, p) = Scratch::db("pathcase");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let exact = s.add("a.bin", "/srv/Vacation/raw", 1, None);
     let anycase = s.add("b.bin", "/vacation-2023", 2, None);
     let _miss = s.add("c.bin", "/holiday", 3, None);
@@ -216,7 +223,7 @@ fn path_substring_tiers_split_by_case() {
 #[test]
 fn path_match_is_deduped_against_a_better_name_match() {
     let (_dir, p) = Scratch::db("pathdedup");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let star = s.add("Budget.txt", "/Budget/2024", 1, None);
     let conn = s.done();
 
@@ -230,7 +237,7 @@ fn path_match_is_deduped_against_a_better_name_match() {
 #[test]
 fn short_terms_respect_the_three_char_trigram_floor() {
     let (_dir, p) = Scratch::db("pathfloor");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let dir_only = s.add("z.bin", "/abcdir", 1, None);
     let name_hit = s.add("ab.txt", "/d", 2, None);
     let _text_only = s.add("body.txt", "/d", 3, Some("ab ab ab"));
@@ -255,7 +262,7 @@ fn short_terms_respect_the_three_char_trigram_floor() {
 #[test]
 fn term_with_separator_matches_across_the_path() {
     let (_dir, p) = Scratch::db("pathsep");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let nested = s.add("report-final.txt", "/home/docs", 1, None);
     let _elsewhere = s.add("report-final.txt", "/home/other", 2, None);
     let conn = s.done();
@@ -271,7 +278,7 @@ fn term_with_separator_matches_across_the_path() {
 #[test]
 fn fuzzy_path_tier_requires_the_fuzzy_flag() {
     let (_dir, p) = Scratch::db("fuzzypath");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let typo_dir = s.add("gamma.bin", "/Reprot-archive", 1, None);
     let conn = s.done();
 
@@ -286,7 +293,7 @@ fn fuzzy_path_tier_requires_the_fuzzy_flag() {
 #[test]
 fn fuzzy_max_edits_widens_and_narrows_the_budget() {
     let (_dir, p) = Scratch::db("fuzzybudget");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let two_edits = s.add("quartrely.txt", "/d", 1, None);
     let three_edits = s.add("quxxxerly.txt", "/d", 2, None);
     let conn = s.done();
@@ -319,7 +326,7 @@ fn fuzzy_max_edits_widens_and_narrows_the_budget() {
 #[test]
 fn a_distant_fuzzy_filename_hit_stays_a_name_hit() {
     let (_dir, p) = Scratch::db("fuzzystage");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     // Ten substitutions against a 30-character term, whose budget is ten.
     let far = s.add("abcdefghijklmnopqrst##########", "/d", 1, None);
     let conn = s.done();
@@ -342,7 +349,7 @@ fn a_distant_fuzzy_filename_hit_stays_a_name_hit() {
 #[test]
 fn dedup_keeps_best_rank() {
     let (_dir, p) = Scratch::db("dedup");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let star = s.add("Budget", "/a", 1, Some("Budget Budget Budget"));
     let conn = s.done();
 
@@ -356,7 +363,7 @@ fn dedup_keeps_best_rank() {
 #[test]
 fn occurrence_counts_order_within_rank() {
     let (_dir, p) = Scratch::db("frac");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let one = s.add("one.txt", "/d", 1, Some("zebra"));
     let three = s.add("three.txt", "/d", 2, Some("zebra zebra zebra"));
     let thousand = s.add("thousand.txt", "/d", 3, Some(&"zebra ".repeat(1500)));
@@ -376,7 +383,7 @@ fn occurrence_counts_order_within_rank() {
 #[test]
 fn like_metacharacters_are_literal() {
     let (_dir, p) = Scratch::db("like");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let percent = s.add("100%.txt", "/d", 1, None);
     let underscore = s.add("100_.txt", "/d", 2, None);
     let contains = s.add("x100y.txt", "/d", 3, None);
@@ -402,9 +409,10 @@ fn like_metacharacters_are_literal() {
 #[test]
 fn diacritic_fts_candidates_are_dropped() {
     let (_dir, p) = Scratch::db("diacritic");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
-    // trigram remove_diacritics 1 makes this an FTS candidate for "cafe",
+    // "cafe" never occurs in the accented body: not an FTS candidate, and
-    // but the exact bytes never occur, so exact full-text must drop it.
+    // exact full-text would drop it even if a diacritic-folding tokenizer
+    // were configured.
     let _accented = s.add("menu.txt", "/d", 1, Some("le café est ouvert"));
     let plain = s.add("plain.txt", "/d", 2, Some("the cafe is open"));
     let conn = s.done();
@@ -413,36 +421,82 @@ fn diacritic_fts_candidates_are_dropped() {
     assert_eq!(ids(&hits), vec![plain]);
 }
 
+/// Without positions the index finds candidates by trigrams alone, adjacent
+/// or not; the full-text pass must drop those whose text lacks the term.
 #[test]
-fn contentless_mode_degrades_to_unranked_stage6() {
+fn a_compact_index_drops_candidates_whose_trigrams_are_not_adjacent() {
-    let (_dir, p) = Scratch::db("notext");
+    let (_dir, p) = Scratch::db("scattered");
-    let mut s = Seeder::new(&p, false); // store_text_for_snippets = false
+    let mut s = Seeder::new(&p);
-    let doc = s.add("doc.txt", "/d", 1, Some("walrus columns"));
+    // Holds `abc`, `bcd` and `cde`, never `abcde`.
+    let scattered = s.add("scattered.txt", "/d", 1, Some("abcx bcdy cdez"));
+    let whole = s.add("whole.txt", "/d", 2, Some("the abcde run"));
     let conn = s.done();
 
-    let (hits, _) = run_collect(&conn, "walrus", &fuzzy_options());
+    assert_eq!(
-    assert_eq!(hits.len(), 1, "FTS still finds it");
+        quicksearch_core::testutil::fts_rowids(&conn, "abcde").len(),
-    let h = &hits[0];
+        2,
-    assert_eq!(h.file_id, doc);
+        "both are FTS candidates"
-    assert_eq!(h.stage, 6, "cannot case-verify without text");
+    );
-    assert!((h.rank - 6.999).abs() < 1e-9, "count-unknown fraction");
+    let (hits, _) = run_collect(&conn, "abcde", &SearchOptions::default());
-    assert!(h.snippet.is_none());
+    assert_eq!(ids(&hits), vec![whole]);
-    assert!(!hits.iter().any(|h| h.stage == 8));
+    assert!(!ids(&hits).contains(&scattered));
+}
 
-    // Without stored text a floor-clearing wildcard can't be pattern-verified.
+/// The layout changes how candidates are found, never what a search
-    let (hits, _) = run_collect(&conn, "wal*rus", &SearchOptions::default());
+/// returns: every query shape answers alike on either index.
-    assert_eq!(staged(&hits), vec![(doc, 6)]);
+#[test]
-    assert!(hits[0].snippet.is_none());
+fn both_index_layouts_answer_every_query_shape_alike() {
-
+    let bodies = [
-    // The short-segment fallback has no FTS evidence, so it finds nothing.
+        ("ocean.txt", "the ocean forest at dawn"),
-    let (none, _) = run_collect(&conn, "wa*us", &SearchOptions::default());
+        ("reversed.txt", "a forest by the ocean"),
-    assert!(none.is_empty());
+        ("report.txt", "Quarterly Report: revenue rose"),
+        ("lower.txt", "the quarterly report is late"),
+        ("wild.txt", "reprocessing the export"),
+        ("typo.txt", "the mountian pass"),
+        ("code.txt", "impl Iterator for Walker"),
+        ("scattered.txt", "impl it; Iterator; for x"),
+    ];
+    let queries: &[(&str, bool)] = &[
+        ("ocean forest", false),
+        ("Report", false),
+        ("report", false),
+        ("rep*rt", false),
+        ("impl Iterator for", false),
+        ("regex:revenue\\s+rose", false),
+        ("mountain", true),
+        ("iterator", true),
+    ];
+    let answers: Vec<Vec<Vec<(i64, u8)>>> = [false, true]
+        .into_iter()
+        .map(|positions| {
+            let (_dir, p) = Scratch::db("layouts");
+            let mut s = Seeder::with_positions(&p, positions);
+            for (name, body) in bodies {
+                s.add(name, "/d", 1, Some(body));
+            }
+            let conn = s.done();
+            queries
+                .iter()
+                .map(|&(query, fuzzy)| {
+                    let options = SearchOptions {
+                        fuzzy,
+                        ..SearchOptions::default()
+                    };
+                    staged(&run_collect(&conn, query, &options).0)
+                })
+                .collect()
+        })
+        .collect();
+    for (i, (query, _)) in queries.iter().enumerate() {
+        assert_eq!(answers[0][i], answers[1][i], "{query}");
+        assert!(!answers[0][i].is_empty(), "{query} should find something");
+    }
 }
 
 #[test]
 fn filters_apply_to_every_stage() {
     let (_dir, p) = Scratch::db("filters");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let keep_name = s.add("alpha.txt", "/keep", 1, None);
     let _skip_name = s.add("alpha.txt", "/skip", 1, None);
     let keep_text = s.add("k.txt", "/keep", 2, Some("alpha body"));
@@ -491,7 +545,7 @@ fn the_limit_truncates_flags_and_keeps_the_best_hits() {
 
     for ((tag, subs, exact_last, limit), (len, limited, first)) in cases {
         let (_dir, p) = Scratch::db(tag);
-        let mut s = Seeder::new(&p, true);
+        let mut s = Seeder::new(&p);
         for i in 0..subs {
             s.add(&format!("zz-match-{:02}.txt", i), "/d", 1, None);
         }
@@ -515,7 +569,7 @@ fn the_limit_truncates_flags_and_keeps_the_best_hits() {
 #[test]
 fn session_ignores_hide_hits_before_the_cap() {
     let (_dir, p) = Scratch::db("ignores");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let keep = s.add("keep-match.txt", "/d", 1, None);
     let _log = s.add("match.log", "/d", 2, None);
     let _sub = s.add("match.txt", "/d/logs", 3, None);
@@ -535,7 +589,7 @@ fn session_ignores_hide_hits_before_the_cap() {
 #[test]
 fn empty_and_filter_only_terms_return_nothing() {
     let (_dir, p) = Scratch::db("empty");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     s.add("anything.txt", "/d", 1, Some("anything"));
     let conn = s.done();
 
@@ -549,7 +603,7 @@ fn empty_and_filter_only_terms_return_nothing() {
 #[test]
 fn hostile_terms_are_inert() {
     let (_dir, p) = Scratch::db("hostile");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     s.add("innocent.txt", "/d", 1, Some("innocent content"));
     let conn = s.done();
 
@@ -585,7 +639,7 @@ fn hostile_terms_are_inert() {
 #[test]
 fn wildcard_name_ranks_through_the_same_tiers() {
     let (_dir, p) = Scratch::db("wildranks");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     // `report*` anchors the whole name, so these are tiers 1 and 2 …
     let whole_cs = s.add("report2024.pdf", "/a", 1, None);
     let whole_ci = s.add("Report2024.pdf", "/b", 2, None);
@@ -620,7 +674,7 @@ fn wildcard_name_ranks_through_the_same_tiers() {
 #[test]
 fn extension_glob_whole_matches_every_such_file() {
     let (_dir, p) = Scratch::db("extglob");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let a = s.add("alpha.txt", "/d", 1, None);
     let b = s.add("beta.txt", "/d", 2, None);
     let upper = s.add("GAMMA.TXT", "/d", 3, None);
@@ -649,7 +703,7 @@ fn extension_glob_whole_matches_every_such_file() {
 #[test]
 fn wildcard_fulltext_narrows_with_fts_and_verifies_order() {
     let (_dir, p) = Scratch::db("wildtext");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let ordered = s.add("a.txt", "/d", 1, Some("a wondrous world indeed"));
     // FTS AND-of-segments finds this too (both trigram runs occur), but the
     // pattern requires "wond" before "world", so verification drops it.
@@ -674,7 +728,7 @@ fn wildcard_fulltext_narrows_with_fts_and_verifies_order() {
 #[test]
 fn wildcard_with_short_segments_falls_back_to_a_full_scan() {
     let (_dir, p) = Scratch::db("wildshort");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let hit = s.add("doc.txt", "/d", 1, Some("zz abXcd zz"));
     let _miss = s.add("other.txt", "/d", 2, Some("cd before ab"));
     let conn = s.done();
@@ -691,7 +745,7 @@ fn wildcard_with_short_segments_falls_back_to_a_full_scan() {
 #[test]
 fn a_wildcard_prefilter_never_loses_a_hit_the_full_scan_finds() {
     let (_dir, p) = Scratch::db("wildprefilter");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let in_name = s.add("report-q3.txt", "/data", 1, None);
     let in_parent = s.add("a.bin", "/reports/q3", 2, None);
     // The pattern's `%` spans the parent/name boundary.
@@ -751,7 +805,7 @@ fn a_fuzzy_prefilter_never_loses_a_hit_the_full_scan_finds() {
     use quicksearch_core::search::fuzzy::{edit_budget, Bitap};
 
     let (_dir, p) = Scratch::db("fuzzyprefilter");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let rows = [
         ("report.txt", "/data"),  // exact
         ("REPORT.TXT", "/upper"), // ASCII case is free for LIKE and bitap both
@@ -807,7 +861,7 @@ fn a_fuzzy_prefilter_never_loses_a_hit_the_full_scan_finds() {
 #[test]
 fn a_fuzzy_term_with_a_separator_still_scans_and_matches() {
     let (_dir, p) = Scratch::db("fuzzysep");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let hit = s.add("orts.txt", "/a/rep", 1, None);
     let _miss = s.add("plans.txt", "/a/sum", 2, None);
     let conn = s.done();
@@ -827,7 +881,7 @@ fn a_fuzzy_term_with_a_separator_still_scans_and_matches() {
 #[test]
 fn a_wildcard_with_only_separator_segments_still_scans_and_matches() {
     let (_dir, p) = Scratch::db("wildnoanchor");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let hit = s.add("q3.txt", "/a/reports", 1, None);
     let _miss = s.add("q3.txt", "/a/summaries", 2, None);
     let conn = s.done();
@@ -839,7 +893,7 @@ fn a_wildcard_with_only_separator_segments_still_scans_and_matches() {
 #[test]
 fn wildcard_path_tier_and_filters() {
     let (_dir, p) = Scratch::db("wildpath");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let dir_hit = s.add("a.bin", "/Vacation-2024", 1, None);
     let filtered = s.add("b.bin", "/elsewhere/Vacation-2023", 2, None);
     let conn = s.done();
@@ -855,7 +909,7 @@ fn wildcard_path_tier_and_filters() {
 #[test]
 fn wildcard_terms_skip_the_fuzzy_stages() {
     let (_dir, p) = Scratch::db("wildfuzzy");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let real = s.add("report.txt", "/d", 1, None);
     // A 2-edit typo: fuzzy would admit it for a literal term.
     let _typo = s.add("Reprot.txt", "/d", 2, None);
@@ -873,7 +927,7 @@ fn wildcard_terms_skip_the_fuzzy_stages() {
 #[test]
 fn regex_only_query_hits_name_content_and_path() {
     let (_dir, p) = Scratch::db("regexonly");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let by_name = s.add("qz42.txt", "/d", 1, None);
     let by_content = s.add("notes.txt", "/d", 2, Some("ref qz7 in the body"));
     let by_path = s.add("b.bin", "/qz99-dir", 3, None);
@@ -901,7 +955,7 @@ fn regex_only_query_hits_name_content_and_path() {
 #[test]
 fn regex_is_case_insensitive_by_default_and_respects_filters() {
     let (_dir, p) = Scratch::db("regexci");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let keep = s.add("QZ1.txt", "/keep", 1, None);
     let _skip = s.add("qz2.txt", "/skip", 2, None);
     let conn = s.done();
@@ -916,7 +970,7 @@ fn regex_is_case_insensitive_by_default_and_respects_filters() {
 #[test]
 fn regex_alongside_a_term_is_an_accept_predicate() {
     let (_dir, p) = Scratch::db("regexpred");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     // Kept via the lazy content fetch: the regex is nowhere in its name/path.
     let kept = s.add("budget-a.txt", "/d", 1, Some("code acme7 inside"));
     let _dropped = s.add("budget-b.txt", "/d", 2, None);
@@ -933,7 +987,7 @@ fn regex_alongside_a_term_is_an_accept_predicate() {
 #[test]
 fn hostile_regexes_complete_quickly() {
     let (_dir, p) = Scratch::db("regexhostile");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     s.add("aaa.txt", "/d", 1, Some(&"a".repeat(50_000)));
     let conn = s.done();
 
@@ -950,7 +1004,7 @@ fn hostile_regexes_complete_quickly() {
 #[test]
 fn service_surfaces_invalid_regex_as_an_error() {
     let (_dir, p) = Scratch::db("regexerr");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     s.add("anything.txt", "/d", 1, None);
     drop(s.done());
 
@@ -964,7 +1018,7 @@ fn service_surfaces_invalid_regex_as_an_error() {
 #[test]
 fn generation_bump_cancels_mid_stream() {
     let (_dir, p) = Scratch::db("cancel");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     for i in 0..500 {
         s.add(&format!("bulk-{:04}.txt", i), "/d", 1, None);
     }
@@ -994,7 +1048,7 @@ fn generation_bump_cancels_mid_stream() {
 #[test]
 fn service_rapid_fire_completes_only_the_last_generation() {
     let (_dir, p) = Scratch::db("service");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     for i in 0..2000 {
         s.add(
             &format!("file-{:04}.txt", i),
@@ -1082,7 +1136,7 @@ fn search_names(
 #[test]
 fn repeated_searches_on_one_service_stay_correct() {
     let (_dir, p) = Scratch::db("warmrepeat");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     s.add("alpha.txt", "/d", 1, Some("alpha body"));
     s.add("beta.txt", "/d", 1, Some("beta body"));
     drop(s.done());
@@ -1107,7 +1161,7 @@ fn repeated_searches_on_one_service_stay_correct() {
 #[test]
 fn a_rebuilt_index_at_the_same_path_is_picked_up() {
     let (_dir, p) = Scratch::db("warmrebuild");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     s.add("before.txt", "/d", 1, Some("shared body"));
     drop(s.done());
 
@@ -1124,7 +1178,7 @@ fn a_rebuilt_index_at_the_same_path_is_picked_up() {
     for suffix in ["", "-wal", "-shm"] {
         std::fs::remove_file(format!("{}{}", p.display(), suffix)).ok();
     }
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     s.add("after.txt", "/d", 1, Some("shared body"));
     drop(s.done());
 
@@ -1139,12 +1193,12 @@ fn a_rebuilt_index_at_the_same_path_is_picked_up() {
 #[test]
 fn set_db_path_repoints_a_held_connection() {
     let (_dir1, first) = Scratch::db("warmpath1");
-    let mut s = Seeder::new(&first, true);
+    let mut s = Seeder::new(&first);
     s.add("infirst.txt", "/d", 1, Some("shared body"));
     drop(s.done());
 
     let (_dir2, second) = Scratch::db("warmpath2");
-    let mut s = Seeder::new(&second, true);
+    let mut s = Seeder::new(&second);
     s.add("insecond.txt", "/d", 1, Some("shared body"));
     drop(s.done());
 
@@ -1170,7 +1224,7 @@ fn a_failed_search_does_not_wedge_the_next_one() {
 
     search_names(&service, &updates, "anything").expect_err("no index yet");
 
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     s.add("arrived.txt", "/d", 1, Some("shared body"));
     drop(s.done());
 
@@ -1190,7 +1244,7 @@ fn a_failed_search_does_not_wedge_the_next_one() {
 #[test]
 fn the_connection_is_released_once_searching_stops() {
     let (_dir, p) = Scratch::db("warmrelease");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     s.add("held.txt", "/d", 1, Some("shared body"));
     drop(s.done());
 
@@ -1242,7 +1296,7 @@ fn the_connection_is_released_once_searching_stops() {
 #[test]
 fn a_release_supersedes_a_search_queued_before_it() {
     let (_dir, p) = Scratch::db("relcancel");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     s.add("held.txt", "/d", 1, Some("shared body"));
     drop(s.done());
 
@@ -1277,7 +1331,7 @@ fn a_release_supersedes_a_search_queued_before_it() {
 #[test]
 fn a_pass_hands_hits_over_before_the_scan_reaches_the_end() {
     let (_dir, p) = Scratch::db("stream");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let early_worse = s.add("my_zebra_file.txt", "/aaa", 1, None); // rank 3
     for i in 0..400 {
         s.add(&format!("filler{:04}.txt", i), "/mmm", i as u64 + 2, None);
@@ -1329,7 +1383,7 @@ fn a_pass_hands_hits_over_before_the_scan_reaches_the_end() {
 #[test]
 fn a_sparse_match_still_streams_before_the_scan_ends() {
     let (_dir, p) = Scratch::db("sparse");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     for i in 0..3000 {
         let name = if i % 1000 == 500 {
             format!("needle{:04}.txt", i)
@@ -1365,7 +1419,7 @@ fn a_sparse_match_still_streams_before_the_scan_ends() {
 #[test]
 fn fuzzy_name_and_path_snippets_carry_the_whole_field() {
     let (_dir, p) = Scratch::db("fuzzy-whole-field");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     // The match sits in the last third of the name, where a windowed snippet
     // would start past zero.
     s.add(
@@ -1400,7 +1454,7 @@ fn fuzzy_name_and_path_snippets_carry_the_whole_field() {
 #[test]
 fn a_fuzzy_mark_covers_the_matched_text_and_nothing_else() {
     let (_dir, p) = Scratch::db("fuzzy-mark-span");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     // A start taken as `end - term.len()` would reach back over the leading
     // digit. In the body too, for the full-text tier.
     s.add("1Reporter.txt", "/home/me/docs", 1, None);
@@ -1449,7 +1503,7 @@ fn a_fuzzy_mark_covers_the_matched_text_and_nothing_else() {
 #[test]
 fn a_fuzzy_mark_is_not_truncated_to_a_leading_part_of_the_term() {
     let (_dir, p) = Scratch::db("fuzzy-mark-full");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     // The name must not match at all: a row the filename tier claims never
     // reaches the full-text tier. And the body must hold a fuzzy variant: the
     // term verbatim would be an exact content match, stage 5 or 6.
@@ -1484,7 +1538,7 @@ fn a_fuzzy_mark_is_not_truncated_to_a_leading_part_of_the_term() {
 #[test]
 fn a_wildcard_spanning_the_directory_boundary_is_still_found() {
     let (_dir, p) = Scratch::db("pathstraddle");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let straddling = s.add("q3.txt", "/x/docs", 1, None);
     // Same two pieces, both inside the name: found either way, so it proves
     // the query ran rather than that the fallback was reached.
@@ -1510,7 +1564,7 @@ fn a_wildcard_spanning_the_directory_boundary_is_still_found() {
 #[test]
 fn the_folder_filter_covers_the_folder_itself_and_its_subtree() {
     let (_dir, p) = Scratch::db("folderself");
-    let mut s = Seeder::new(&p, true);
+    let mut s = Seeder::new(&p);
     let own = s.add("top.txt", "/srv/data", 1, None);
     let nested = s.add("deep.txt", "/srv/data/2024", 2, None);
     let _sibling = s.add("other.txt", "/srv/data-archive", 3, None);

crates/quicksearch-core/tests/encrypted.rs

@@ -32,12 +32,7 @@ fn header(db_path: &Path) -> [u8; 16] {
 
 fn match_count(db_path: &Path, term: &str) -> i64 {
     let conn = db::open_existing(&db_path.to_string_lossy(), false).unwrap();
-    conn.query_row(
+    quicksearch_core::testutil::fts_count(&conn, term)
-        "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH ?1",
-        [term],
-        |r| r.get(0),
-    )
-    .unwrap()
 }
 
 #[test]

crates/quicksearch-core/tests/full_index.rs

@@ -67,16 +67,9 @@ fn count(db: &Path, sql: &str) -> i64 {
         .unwrap()
 }
 
-/// FTS rows matching the phrase `word`.
+/// FTS rows matching `word`.
 fn hits(db: &Path, word: &str) -> i64 {
-    rusqlite::Connection::open(db)
+    quicksearch_core::testutil::fts_count(&rusqlite::Connection::open(db).unwrap(), word)
-        .unwrap()
-        .query_row(
-            "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH ?1",
-            [format!("\"{}\"", word)],
-            |r| r.get(0),
-        )
-        .unwrap()
 }
 
 /// Bytes in `db`'s write-ahead log; 0 when there is none.
@@ -1065,26 +1058,32 @@ fn the_content_extension_filter_still_excludes_small_text_files() {
     );
 }
 
+/// Both layouts index the same files the same way; only the FTS table
+/// differs, and every file with text keeps its body for verification.
 #[test]
-fn contentless_mode_still_indexes_inlined_files_without_storing_bodies() {
+fn either_index_layout_stores_and_finds_the_same_files() {
-    let root = Scratch::dir("inline-contentless");
+    for compact in [true, false] {
-    let (_db_dir, db) = Scratch::db("inline-contentless-db");
+        let root = Scratch::dir("layout");
+        let (_db_dir, db) = Scratch::db("layout-db");
+        let mut config = Config::default();
+        config.processing.compact_text_index = compact;
+        touch(&root.join("small.txt"), b"searchable xylophone body");
+        touch(&root.join("other.txt"), b"nothing to see");
+        index_once(&root, &db, &config);
 
-    let mut config = Config::default();
+        assert_eq!(
-    config.processing.store_text_for_snippets = false;
+            count(&db, "SELECT COUNT(*) FROM documents_text"),
-    touch(&root.join("small.txt"), b"searchable xylophone body");
+            2,
-    index_once(&root, &db, &config);
+            "every body is stored (compact = {compact})"
-
+        );
-    assert_eq!(
+        assert_eq!(hits(&db, "xylophone"), 1, "compact = {compact}");
-        count(&db, "SELECT COUNT(*) FROM documents_text"),
+        let detail_none = count(
-        0,
+            &db,
-        "contentless mode stores no bodies"
+            "SELECT COUNT(*) FROM sqlite_master \
-    );
+             WHERE name = 'searchabletext' AND sql LIKE '%detail=none%'",
-    assert_eq!(
+        );
-        hits(&db, "xylophone"),
+        assert_eq!(detail_none == 1, compact);
-        1,
+    }
-        "an inlined file is still searchable in contentless mode"
-    );
 }
 
 // ---------------------------------------------------------------------------
@@ -1251,11 +1250,13 @@ const MIN_ROWS_SAMPLED: usize = 4;
 /// Removes the fixture before asserting, unlike the rest of this file: it is
 /// generated and identical every run, and leaving tens of MB in a RAM-backed
 /// /tmp behind a failure is itself a reason to fail.
+#[allow(clippy::too_many_arguments)]
 fn watch_heavy_beside_light(
     tag: &str,
     heavy_files: usize,
     heavy_dirs: usize,
     body: &[u8],
+    ext: &str,
     workers: usize,
     mut config: Config,
 ) -> Overlap {
@@ -1263,7 +1264,7 @@ fn watch_heavy_beside_light(
     let heavy = Scratch::dir(&heavy_tag);
     for i in 0..heavy_files {
         touch(
-            &heavy.join(format!("d{}/big{:04}.txt", i % heavy_dirs, i)),
+            &heavy.join(format!("d{}/big{:04}.{}", i % heavy_dirs, i, ext)),
             body,
         );
     }
@@ -1336,9 +1337,14 @@ fn a_heavy_root_does_not_stall_a_light_one() {
     // with the row count, while what the light root drains does not. Sized
     // for the guards: enough that the rows arrive over separate writer
     // rounds, cheap enough that a loaded runner still finishes in seconds.
+    // RTF, because its extractor reads and parses the whole file, where
+    // plaintext stops at twice `maximum_text_size`.
     const HEAVY_FILES: usize = 12;
     // ~2 MiB each, 24 MB total.
-    let body = "sphinx of black quartz judge my vow ".repeat(58_000);
+    let body = format!(
+        "{{\\rtf1\\ansi {}}}",
+        "sphinx of black quartz judge my vow ".repeat(58_000)
+    );
 
     let mut config = Config::default();
     config.processing.maximum_text_size = 1024;
@@ -1347,7 +1353,7 @@ fn a_heavy_root_does_not_stall_a_light_one() {
     config.processing.maximum_text_file_size = 4 * 1024 * 1024;
     // One extraction thread: with the default four the pass is four times
     // shorter, and so is the margin.
-    let seen = watch_heavy_beside_light("stall", HEAVY_FILES, 4, body.as_bytes(), 1, config);
+    let seen = watch_heavy_beside_light("stall", HEAVY_FILES, 4, body.as_bytes(), "rtf", 1, config);
 
     eprintln!(
         "light files drained while the heavy root extracted: {} against {} heavy \
@@ -1392,6 +1398,7 @@ fn a_heavy_root_does_not_stall_a_light_one_at_the_writer() {
         HEAVY_FILES,
         8,
         body.as_bytes(),
+        "txt",
         4,
         Config::default(),
     );
@@ -1631,7 +1638,10 @@ fn the_wal_stays_bounded_through_the_tail_of_a_warm_reindex() {
     let root = seed_text_tree("wal-tail");
     let (db_dir, db) = Scratch::db("wal-tail-db");
     let dir_key = db_dir.to_string_lossy().into_owned();
-    let config = Config::default();
+    // With positions: the log is measured against the index's size, and
+    // position lists make this fixture's index several times larger.
+    let mut config = Config::default();
+    config.processing.compact_text_index = false;
 
     // An empty root first, purely to bring the index into existence so the
     // reader below can be opened before the run that matters.

crates/quicksearch-core/tests/page_size.rs

@@ -272,7 +272,7 @@ fn check_previous_profile(previous: Profile, keyed: bool, stale_version: bool) {
     );
 
     // The rebuild lands at the current layout, encrypted if it was.
-    let conn = db::open_or_recreate(&db_path, "trigram")
+    let conn = db::open_or_recreate(&db_path, &Default::default())
         .unwrap_or_else(|e| panic!("{}: rebuild failed: {}", what, e));
     assert_eq!(
         page_size_of(&conn),

crates/quicksearch-core/tests/prefilter_fuzz.rs

@@ -51,8 +51,8 @@ const DOCS: usize = 60;
 /// the corpus goes straight into the term and the FTS chunks: FTS5
 /// metacharacters (must be quoted into inertness by
 /// `translator::quote_phrase`), multi-byte characters at 2, 3 and 4 bytes,
-/// diacritics (folded by the trigram index but not by bitap, so the prefilter
+/// diacritics (a fuzzy match may span an accented character, which the
-/// must stay a superset across that), mixed case, and a degenerate
+/// prefilter must survive), mixed case, and a degenerate
 /// repeated-character shape.
 fn bodies() -> Vec<String> {
     let mut out = Vec::new();
@@ -586,7 +586,8 @@ fn seed_files(path: &std::path::Path, rows: &[(String, String)]) -> Vec<i64> {
     use quicksearch_core::mime::FileType;
 
     let mut conn =
-        quicksearch_core::db::open_or_recreate(path.to_str().unwrap(), "trigram").unwrap();
+        quicksearch_core::db::open_or_recreate(path.to_str().unwrap(), &Default::default())
+            .unwrap();
     let tx = conn.transaction().unwrap();
     let mut ids = Vec::with_capacity(rows.len());
     for (parent, name) in rows {
@@ -838,12 +839,13 @@ fn a_regex_finds_exactly_the_documents_it_matches() {
 }
 
 fn seed(path: &std::path::Path, bodies: &[String]) -> Vec<i64> {
-    use quicksearch_core::db::repo::{insert_file, set_content_done, NewFile};
+    use quicksearch_core::db::repo::{insert_file, store_pending_text, NewFile};
     use quicksearch_core::mime::FileType;
     use quicksearch_core::testutil::zstd_of;
 
     let mut conn =
-        quicksearch_core::db::open_or_recreate(path.to_str().unwrap(), "trigram").unwrap();
+        quicksearch_core::db::open_or_recreate(path.to_str().unwrap(), &Default::default())
+            .unwrap();
     let tx = conn.transaction().unwrap();
     let mut ids = Vec::with_capacity(bodies.len());
     for (i, body) in bodies.iter().enumerate() {
@@ -864,7 +866,7 @@ fn seed(path: &std::path::Path, bodies: &[String]) -> Vec<i64> {
         )
         .unwrap()
         .expect("unique path");
-        set_content_done(&tx, id, body, zstd_of(body).as_deref()).unwrap();
+        store_pending_text(&tx, id, body, zstd_of(body).as_deref()).unwrap();
         ids.push(id);
     }
     tx.commit().unwrap();

crates/quicksearch-core/tests/reconcile.rs

@@ -87,6 +87,11 @@ fn names(db: &Path) -> Vec<String> {
         .collect()
 }
 
+/// FTS rows matching `term`, asked for as the index's layout allows.
+fn fts_count(db: &Path, term: &str) -> i64 {
+    quicksearch_core::testutil::fts_count(&conn(db), term)
+}
+
 fn count(db: &Path, sql: &str) -> i64 {
     conn(db).query_row(sql, [], |r| r.get(0)).unwrap()
 }
@@ -177,10 +182,7 @@ fn adding_an_ignore_pattern_prunes_instead_of_rebuilding() {
     );
     assert_eq!(orphans(&db), 0);
     assert_eq!(
-        count(
+        fts_count(&db, "echo"),
-            &db,
-            "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'echo'"
-        ),
         0,
         "the ignored file's content is no longer findable"
     );
@@ -304,13 +306,7 @@ fn narrowing_content_extensions_keeps_the_file_findable_by_name() {
     assert_eq!(recontented, 1);
     assert_eq!(paths(&db).len(), 2, "both rows are still there");
     assert_eq!(residue(&db).0, 1, "only the .txt keeps its postings");
-    assert_eq!(
+    assert_eq!(fts_count(&db, "markdown"), 0);
-        count(
-            &db,
-            "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'markdown'"
-        ),
-        0
-    );
     assert_eq!(
         count(&db, "SELECT COUNT(*) FROM files WHERE content_state = 3"),
         1,
@@ -330,38 +326,6 @@ fn narrowing_content_extensions_keeps_the_file_findable_by_name() {
     assert_eq!(created_at(&db), born, "the index was never rebuilt");
 }
 
-/// On again re-extracts: the text of files already indexed was never kept.
-#[test]
-fn store_text_toggles_without_losing_full_text_search() {
-    let root = Scratch::canonical("store-text");
-    let db_dir = Scratch::canonical("store-text-db");
-    let db = db_dir.join("index.sqlite");
-    touch(&root.join("a.txt"), b"alpha unique-token");
-
-    let on = base_config(&root, &db);
-    index_once(&db, &on);
-    let born = created_at(&db);
-    assert_eq!(residue(&db).1, 1, "text stored");
-
-    let mut off = on.clone();
-    off.processing.store_text_for_snippets = false;
-    reconcile(&db, &on, &off);
-    assert_eq!(residue(&db).1, 0, "blobs dropped");
-    assert_eq!(
-        count(
-            &db,
-            "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'unique'"
-        ),
-        1,
-        "full-text search is unaffected"
-    );
-
-    reconcile(&db, &off, &on);
-    index_once(&db, &on);
-    assert_eq!(residue(&db).1, 1, "text re-extracted");
-    assert_eq!(created_at(&db), born, "the index was never rebuilt");
-}
-
 #[test]
 fn turning_hidden_files_off_prunes_whole_hidden_subtrees() {
     let root = Scratch::canonical("hidden");

crates/quicksearch-gui/src/manage_tab.rs

@@ -531,9 +531,8 @@ fn db_size_tooltip(ui: &mut egui::Ui) {
         "Remove indexed folders you do not need (Indexed folders, below).",
         "Narrow the full-text extension whitelist to the file types whose text \
          you search.",
-        "Turn off \"Store text for snippets\" on the Settings tab. Full-text search \
+        "Keep \"Compact text index\" on, on the Settings tab: it makes the \
-         keeps working, without previews, occurrence ranking or fuzzy matching \
+         full-text part of the index several times smaller.",
-         inside file contents.",
         "Lower \"Max text file size\" and \"Max stored text\", both on the Settings tab.",
     ] {
         ui.label(format!("•  {}", lever));

crates/quicksearch-gui/src/manage_tab/tests.rs

@@ -916,7 +916,7 @@ fn hovering_the_size_explains_how_to_shrink_the_index() {
         "ignore filters",
         "Indexed folders",
         "whitelist",
-        "Store text for snippets",
+        "Compact text index",
         "Settings tab",
     ] {
         assert!(text.contains(lever), "tooltip never mentions {}", lever);

crates/quicksearch-gui/src/search_tab.rs

@@ -1281,9 +1281,11 @@ impl SearchTab {
         ) {
             self.live_armed = self.live_wanted.clone();
             actions.live_targets = Some(self.live_wanted.clone());
-        } else if settled && !armed_already {
+        } else if self.live_enabled && settled && !armed_already {
             // Once settled nothing else asks for frames, so a bare `Instant`
-            // deadline would never come due.
+            // deadline would never come due. Only while arming can happen:
+            // with live results off the set is never armed, and past the
+            // delay this asked for a frame every frame, forever.
             if let Some(changed) = self.live_changed_at {
                 let waited = now.duration_since(changed);
                 ui.ctx()

crates/quicksearch-gui/src/search_tab/tests.rs

@@ -2067,6 +2067,39 @@ fn should_arm_waits_for_the_results_to_settle_and_hold_still() {
     );
 }
 
+/// An idle table asks for no frames. With live results off nothing is ever
+/// armed, and past the arm delay the wait for it asked for a frame every
+/// frame, forever; the "more below" hint must not animate either. On a
+/// software-rendered desktop the two cost a third of a core while idle.
+#[test]
+fn an_idle_table_with_more_rows_than_fit_asks_for_no_frames() {
+    let ctx = crate::test_ui::ctx();
+    let mut tab = tab_with_results(200);
+    tab.live_enabled = false;
+    for _ in 0..200 {
+        run_frame(&ctx, &mut tab, vec![]);
+        if tab.fade_settled() {
+            break;
+        }
+    }
+    assert!(tab.fade_settled(), "the reveal never finished");
+    // The visible rows have stood still for longer than the arm delay.
+    tab.live_changed_at = Some(Instant::now() - LIVE_ARM_DELAY * 2);
+
+    // egui's own layout passes ask for a few frames of their own first.
+    for _ in 0..8 {
+        run_frame(&ctx, &mut tab, vec![]);
+    }
+    let out = run_frame(&ctx, &mut tab, vec![]);
+    let delay = out
+        .viewport_output
+        .values()
+        .map(|v| v.repaint_delay)
+        .min()
+        .expect("a viewport");
+    assert_eq!(delay, std::time::Duration::MAX, "an idle table repaints");
+}
+
 /// What counts as "already watching these rows": the paths and whether each
 /// needs its body re-read, and nothing else.
 #[test]

crates/quicksearch-gui/src/settings_tab.rs

@@ -322,11 +322,13 @@ impl SettingsTab {
                 // with advanced settings shown.
                 ui.label(hint(if form.advanced {
                     "Narrowing a filter removes the entries it excludes; widening \
-                     one reindexes to find what it now allows. Only the tokenizer \
+                     one reindexes to find what it now allows. Only the tokenizer, \
-                     and hash sample size require a full rebuild."
+                     the compact text index and the hash sample size require a \
+                     full rebuild."
                 } else {
                     "Narrowing a filter removes the entries it excludes; widening \
-                     one reindexes to find what it now allows."
+                     one reindexes to find what it now allows. Changing the \
+                     compact text index rebuilds the index."
                 }));
             });
         crate::ui_util::more_below_hint(ui, &scroll);
@@ -880,9 +882,9 @@ fn config_editor_ui(
                     form.row(
                         Level::Everyday,
                         ui,
-                        "Store text for snippets",
+                        "Compact text index",
-                        &tips::STORE_TEXT,
+                        &tips::COMPACT_INDEX,
-                        |ui| ui.checkbox(&mut config.processing.store_text_for_snippets, ""),
+                        |ui| ui.checkbox(&mut config.processing.compact_text_index, ""),
                     );
                 });
         }

crates/quicksearch-gui/src/settings_tab/tests.rs

@@ -246,7 +246,7 @@ const ROWS: &[(Section, &str, &tips::Tip, Level)] = &[
     (Section::Processing, "Max text file size (bytes)", &tips::MAX_TEXT_FILE_SIZE, Level::Advanced),
     (Section::Processing, "Batch size", &tips::BATCH_SIZE, Level::Advanced),
     (Section::Processing, "Max WAL size (bytes)", &tips::MAX_WAL_SIZE, Level::Advanced),
-    (Section::Processing, "Store text for snippets", &tips::STORE_TEXT, Level::Everyday),
+    (Section::Processing, "Compact text index", &tips::COMPACT_INDEX, Level::Everyday),
     (Section::Search, "Fuzzy search ON by default", &tips::FUZZY_DEFAULT, Level::Everyday),
     (Section::Search, "Fuzzy edit distance", &tips::FUZZY_EDITS, Level::Advanced),
     (Section::Search, "Display limit", &tips::DISPLAY_LIMIT, Level::Advanced),

crates/quicksearch-gui/src/tips.rs

@@ -235,21 +235,23 @@ pub static MAX_WAL_SIZE: Tip = Tip {
     caution: None,
 };
 
-pub static STORE_TEXT: Tip = Tip {
+pub static COMPACT_INDEX: Tip = Tip {
-    title: "Store text for snippets",
+    title: "Compact text index",
-    body: "Keeps the text read from your files, not just the search index \
+    body: "Builds the index of your files' text without recording where in \
-           built from it. This is what shows a preview line under each \
+           each file every word sits, which makes it several times smaller \
-           result.\n\n\
+           and quicker to build.\n\n\
-           Off, searching inside files still works, but there are no \
+           Searches find exactly the same files either way: QuickSearch checks \
-           previews, no ranking by how often a word appears, no telling \
+           each possible match against the text it keeps, and without \
-           Report from report, and no typo tolerance inside file contents. \
+           positions there are more possible matches to check. Searching file \
-           The index shrinks considerably.\n\n\
+           contents for a word can take up to twice as long, and for a phrase \
-           Turning it off discards the stored text at once; turning it back \
+           of common words, such as \"impl Iterator for\", up to three times.\n\n\
-           on reads your files again.",
+           Only the trigram tokenizer can use it. Other tokenizers always \
+           record positions.",
     examples: &[
-        "off when the index has grown larger than you want and you can do without previews.",
+        "on, the default: 1.1 GB of source code made a 219 MB index instead of 1.3 GB, built a third faster.",
+        "off when disk space is plentiful and you often search for phrases of common words.",
     ],
-    caution: None,
+    caution: Some("Changing this rebuilds the index from scratch."),
 };
 
 // --- Settings: Search -----------------------------------------------------
@@ -643,7 +645,7 @@ mod tests {
         &MAX_TEXT_FILE_SIZE,
         &BATCH_SIZE,
         &MAX_WAL_SIZE,
-        &STORE_TEXT,
+        &COMPACT_INDEX,
         &FUZZY_DEFAULT,
         &FUZZY_EDITS,
         &DISPLAY_LIMIT,

crates/quicksearch-gui/src/ui_util.rs

@@ -197,6 +197,11 @@ pub fn middle_elide<'a>(
 
 /// A semitransparent down-arrow while more content lies below the fold.
 /// Painter-only, so it never swallows clicks; the fonts have no ▼ glyph.
+///
+/// Static on purpose. It used to pulse, which needed a repaint every 50 ms
+/// for as long as it showed: on an idle Search tab with more results than
+/// rows, that re-laid-out every visible row 20 times a second, measured at
+/// 11.8 s of CPU per idle 30 s (100k-file index, 1000 results).
 pub fn more_below_hint<R>(ui: &egui::Ui, out: &egui::scroll_area::ScrollAreaOutput<R>) {
     let more_below = out.state.offset.y + out.inner_rect.height() < out.content_size.y - 1.0;
     if !more_below {
@@ -206,12 +211,7 @@ pub fn more_below_hint<R>(ui: &egui::Ui, out: &egui::scroll_area::ScrollAreaOutp
     let cx = out.inner_rect.center().x;
     let tip = out.inner_rect.bottom() - 5.0;
     let (half_width, height) = (7.0, 6.0);
-    // A slow opacity pulse; repaints only while visible, at a lazy cadence.
+    let color = ui.visuals().strong_text_color().gamma_multiply(0.35);
-    let t = ui.ctx().input(|i| i.time);
-    let pulse = 0.35 + 0.10 * ((t * std::f64::consts::TAU / 2.5).sin() as f32);
-    let color = ui.visuals().strong_text_color().gamma_multiply(pulse);
-    ui.ctx()
-        .request_repaint_after(std::time::Duration::from_millis(50));
     painter.add(egui::Shape::convex_polygon(
         vec![
             egui::pos2(cx - half_width, tip - height),