From 5f55d1984901f8384daa26dfb7bba0773d549706 Mon Sep 17 00:00:00 2001 From: = <=> Date: Sun, 9 Aug 2026 18:36:47 -0400 Subject: [PATCH] Significant performance pass. Reduced memory churn massively. ~1.5-3x search performance improvements. Minor improvements to warm indexing time. --- Cargo.lock | 85 +++- Cargo.toml | 21 +- README.md | 73 +++- crates/quicksearch-core/Cargo.toml | 25 ++ crates/quicksearch-core/benches/corpus/mod.rs | 188 +++++++++ crates/quicksearch-core/benches/index.rs | 190 +++++++++ crates/quicksearch-core/benches/search.rs | 363 ++++++++++++++++++ .../quicksearch-core/examples/indexprobe.rs | 8 +- crates/quicksearch-core/src/cli.rs | 5 +- crates/quicksearch-core/src/coordinator.rs | 20 + .../quicksearch-core/src/coordinator/inner.rs | 50 ++- crates/quicksearch-core/src/db/repo.rs | 193 +++++++++- crates/quicksearch-core/src/db/repo_tests.rs | 205 +++++++++- .../src/file_handling/batch.rs | 90 ++++- .../quicksearch-core/src/file_handling/mod.rs | 8 +- .../src/file_handling/records.rs | 31 +- crates/quicksearch-core/src/incremental.rs | 3 +- .../quicksearch-core/src/indexing/pipeline.rs | 22 +- crates/quicksearch-core/src/indexing/tests.rs | 92 +++++ crates/quicksearch-core/src/query/pattern.rs | 28 +- .../src/search/cascade/passes.rs | 173 +++++++-- crates/quicksearch-core/src/search/mod.rs | 15 +- crates/quicksearch-core/src/snippet.rs | 19 +- crates/quicksearch-core/src/testutil.rs | 10 + crates/quicksearch-core/tests/cascade.rs | 4 +- crates/quicksearch-core/tests/search_perf.rs | 17 +- crates/quicksearch-gui/src/logs_tab.rs | 58 ++- crates/quicksearch-gui/src/manage_tab.rs | 27 ++ .../quicksearch-gui/src/manage_tab/tests.rs | 75 ++++ crates/quicksearch-gui/src/tips.rs | 15 + 30 files changed, 1991 insertions(+), 122 deletions(-) create mode 100644 crates/quicksearch-core/benches/corpus/mod.rs create mode 100644 crates/quicksearch-core/benches/index.rs create mode 100644 crates/quicksearch-core/benches/search.rs diff --git a/Cargo.lock b/Cargo.lock index 6632ec6..9c4fab1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,6 +111,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + [[package]] name = "arboard" version = "3.6.1" @@ -658,6 +664,32 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstyle", + "clap_lex", + "terminal_size", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "clipboard-win" version = "5.4.1" @@ -697,6 +729,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "condtype" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" + [[package]] name = "core-foundation" version = "0.9.4" @@ -890,6 +928,31 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "divan" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a405457ec78b8fe08b0e32b4a3570ab5dff6dd16eb9e76a5ee0a9d9cbd898933" +dependencies = [ + "cfg-if", + "clap", + "condtype", + "divan-macros", + "libc", + "regex-lite", +] + +[[package]] +name = "divan-macros" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9556bc800956545d6420a640173e5ba7dfa82f38d3ea5a167eb555bc69ac3323" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "dlib" version = "0.5.3" @@ -3125,12 +3188,13 @@ dependencies = [ [[package]] name = "quicksearch-core" -version = "1.0.5" +version = "1.0.6" dependencies = [ "argon2", "cfb", "chardetng", "ctrlc", + "divan", "encoding_rs", "getrandom 0.2.15", "globset", @@ -3138,6 +3202,7 @@ dependencies = [ "kamadak-exif", "libc", "lofty", + "memchr", "mime_guess", "notify", "pdf-extract", @@ -3157,7 +3222,7 @@ dependencies = [ [[package]] name = "quicksearch-gui" -version = "1.0.5" +version = "1.0.6" dependencies = [ "ashpd", "chrono", @@ -3323,6 +3388,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.4" @@ -3824,6 +3895,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix 1.1.4", + "windows-sys 0.61.0", +] + [[package]] name = "thiserror" version = "1.0.61" diff --git a/Cargo.toml b/Cargo.toml index 4e4c7d3..085e79a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,8 +5,27 @@ members = [ "crates/quicksearch-gui", ] +# No `[profile.release]` on purpose. Cross-crate inlining looks like it should +# pay here — the cascade calls into `snippet`/`query`, both into `memchr` and +# `zstd`, everything into `rusqlite`'s FFI wrappers — so it was measured, and +# it does not. Against an incremental rebuild of the GUI after touching core +# (10.2 s at the defaults): +# +# lto=thin 67 s search cold 29.9 ms warm best 14.77 ms index cold 459 ms +# lto=thin,cgu=1 152 s (build cost alone ruled it out) +# lto=fat,cgu=1 200 s search cold 29.2 ms warm best 14.57 ms index cold 479 ms +# (defaults) 10 s search cold 30.0 ms warm best 14.92 ms index cold 478 ms +# +# Every runtime column moves 0-4% and index-cold is not even monotonic, so the +# gain is at the noise floor while the build is 6.6-19.5x longer — on a CI that +# builds three targets. Re-measure before concluding otherwise. +# +# `panic = "abort"` is separately unavailable: `extract/pdf.rs` runs +# `pdf-extract` inside `catch_unwind`, so aborting would turn a malformed PDF +# into a killed process instead of one skipped file. + [workspace.package] -version = "1.0.5" +version = "1.0.6" edition = "2021" license = "GPL-3.0-or-later" authors = ["Jeremy "] diff --git a/README.md b/README.md index 3160073..c2f5508 100644 --- a/README.md +++ b/README.md @@ -237,7 +237,13 @@ inside that folder. restarts until you return to automatic. The index size beside the status heading totals the database and its `-wal`/`-shm` sidecars, refreshed every ten seconds; hovering it lists the ways to make it - smaller. + smaller. Each folder in the list carries what it holds — files + indexed, and how many of those had text extracted — counted once as + each indexing run finishes and stored with the index, so they are + there the moment the app opens rather than costing a scan to show. A + folder nothing has finished indexing reads "not yet indexed" rather + than zero, and because the figures come from completed runs they do + not move as live updates apply single changes in between. - **Duplicates**: files sharing a content hash, grouped. - **Logs**: the lines the app would have printed to a terminal — warnings from indexing, folder watching and opening files, newest last, with a @@ -458,7 +464,14 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. then held — the coordinator's writer, before it learned to let go when idle — keeps that memory for the life of the process. Search is the one deliberately large one, because it is the only cache reused often enough to pay for - itself, and it is released once searching stops. + itself, and it is released once searching stops. Dropping the connection is + only half of releasing it: glibc hands the freed pages back to its own arena + rather than to the kernel, so the search worker calls + `platform::release_free_heap` after it lets go, exactly as + `coordinator::go_idle` does for the writer. Without that call one typing + session left the process 42 MiB heavier for as long as it ran — measured on a + 77k-file index, where an idle GUI sat at 76 MiB `RssAnon` and stayed there, + against 34 MiB before the first search and 42 MiB once the trim runs. - **Indexing** (`indexing.rs`, `file_handling.rs`): full runs walk each root (`filtered_walk` prunes hidden/ignored subtrees before descending), classify files by mtime into insert/update/skip, batch-write metadata, @@ -467,9 +480,15 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. OLE2 streams are read in `extract/ole.rs` — PDF, audio tags, EXIF; see `extract/`) for FTS. PDFs are parsed once, with the text and the `Info` dictionary taken off the same document: the two-parse version that preceded - it was a run's largest single memory consumer, and it was what pulled a - second copy of `lopdf` — and with it rayon's never-torn-down thread pool — - into the build. Files whose extension no MIME + it was the largest single memory consumer of a run over a PDF-heavy tree, and + it was what pulled a second copy of `lopdf` — and with it rayon's + never-torn-down thread pool — into the build. That is a claim about PDFs + rather than about runs in general, and it is worth knowing which tree a + number came from: on one with almost no PDFs, a cold run peaks at 130 MiB + against 27 MiB for the same walk with content extraction switched off, and + switching off `store_text_for_snippets` moves that peak not at all — so what + is left is the extraction workers and FTS5's own index build, not any single + parser and not the stored text. Files whose extension no MIME table knows — including extensionless ones like `README` or `Makefile` — are sniffed from their head bytes and indexed as text only when that head is provably text: valid UTF-8, or BOM-marked (`mime.rs`, `textenc.rs`). @@ -554,6 +573,31 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. flush last, so weaker matches only ever append. All SQL is parameterized; structured filters from the query language (`query/`) are ANDed onto every pass. + + The passes that read document text share one `zstd::bulk::Decompressor` and + one output buffer per scan (`DocDecoder` in `search/cascade/passes.rs`), + and the row's path is borrowed from the statement rather than copied — only + rows that become hits own one. Both matter more than they look: peak memory + during a search never exceeded 14 MiB even before any of this, but a *single* + fuzzy query moved 31 GiB through `malloc`, and resident-set sampling is blind + to that by construction, because a buffer allocated and freed inside one loop + iteration never moves RSS. `DocDecoder` must therefore never fall back to a + per-row allocating decode, and there is a trap waiting there: + `zstd::encode_all`, which the indexer writes with, is *stream*-based and so + records no content size in the frame header, meaning + `get_frame_content_size` returns `None` for every row this ever sees. The + "cannot happen" branch is the only branch. Sizing the buffer from the header + and handing the `None` case to `zstd::decode_all` looks obviously right and + costs ~2.4 MiB per document, because `decode_all` builds a streaming decoder + per call — 27 of the 30 GiB a fuzzy search moved. Growing this buffer and + keeping it is what makes decoding a row allocate nothing at all. + Measured over the same 77k-file index, per query: + `cascade` 582 → 14 MiB, `function` 6.0 GiB → 29 MiB, `--fuzzy cascade` + 31 GiB → 57 MiB, `regex:` 31 GiB → 39 MiB, each a little faster rather than + slower. `TermPattern::find_first` folds nothing either: its + case-insensitive literal branch used to allocate a lowercased copy of its + haystack, which the filename pass asked for twice per row of a full-table + scan. - **Baloo compatibility** (`cli.rs`, `mime.rs`): the read API this repo's parent consumes — `status_for_path`, `list_failed`, `index_size_breakdown`, `pending_content_count`, `clear_path` — plus a @@ -612,6 +656,15 @@ microseconds regardless of row count. synthesizing clicks and reading back the painted text (`test_ui.rs`) — over the search and manage tabs, the options editor, the unlock gate, the logs and duplicates tabs, and query highlighting. +- `cargo bench -p quicksearch-core --bench search` and `--bench index`: divan + microbenchmarks over the two hot paths. Each group runs *what the code does + today* against *the change being considered*, in one process on one corpus, + so the comparison is a measurement rather than an estimate — read the losing + arm as documentation of something already tried. Sizes sweep 1 KiB, 16 KiB + and 256 KiB, the last being `maximum_text_size` and so the worst a full-text + row can present; `benches/corpus/` builds all of it from a fixed seed. This + is the harness to extend when a hot path is in question, because it is the + only one here that can A/B a single function. - `QSB_SNIPPET_PERF=1 cargo test --release -p quicksearch-core --test snippet_perf -- --nocapture`: snippet pipeline benchmark. - `QSB_SEARCH_PERF=1 cargo test --release -p quicksearch-core --test @@ -632,6 +685,16 @@ microseconds regardless of row count. distinguishable from live data. It reads another process's `/proc`, so it measures a build made without knowing it would be measured. `indexprobe` and `walkprobe` answer "how fast" rather than "how much". + + All of these read the resident set, and none of them can see allocator + *churn*: a buffer allocated and freed within one loop iteration never moves + RSS, so a search whose peak is a flat 14 MiB can still be pushing tens of + gigabytes a query through `malloc`. Both search-side regressions found so far + were invisible to every probe listed above and showed up only under an + interposed `malloc` that counted calls and bytes. Until a probe here reports + allocation counts, measure that separately before concluding a path is cheap, + and measure the GUI rather than a one-shot `quicksearch-cli` run — a + short-lived process cannot show what a typing session retains. - `.forgejo/workflows/ci.yml`: builds both platforms on every push to `master` and every pull request. To cut a release, bump `[workspace.package] version` in `Cargo.toml` and push the commit on a branch named `Release...`; CI runs diff --git a/crates/quicksearch-core/Cargo.toml b/crates/quicksearch-core/Cargo.toml index 026d209..90fa28e 100644 --- a/crates/quicksearch-core/Cargo.toml +++ b/crates/quicksearch-core/Cargo.toml @@ -62,6 +62,13 @@ ctrlc = "3.4" zstd = "0.13" globset = "0.4" regex = "1" +# SIMD substring search for the full-text passes. `str::match_indices` uses +# std's Two-Way searcher, which has no vector prefilter: measured against a +# 256 KiB body (`benches/search.rs`, group `substring`) it runs 111 µs where +# `memmem` runs 2.4 µs, and the full-text passes scan a body per candidate +# row. Already in the lockfile transitively (regex, globset, chardetng), so +# naming it directly compiles nothing new. +memchr = "2" # `nice()`, for dropping indexing threads to background scheduling priority. # Linux schedules per task, so it affects only the calling thread. Already in @@ -91,3 +98,21 @@ windows-sys = { version = "0.52", features = [ "Win32_System_Threading", "Win32_System_WindowsProgramming", ] } + +# Microbenchmark harness for `benches/`. Criterion is the more common choice +# and was rejected: it pulls rayon (a global thread pool that never tears +# down, the same reason `lopdf` is not named directly above) plus plotters and +# clap, some forty crates. Divan's mandatory set is a handful and it spawns no +# threads of its own. +[dev-dependencies] +divan = "0.1" + +# `harness = false` on both: divan supplies its own `main` via `divan::main()`, +# so libtest must not also link one in. +[[bench]] +name = "search" +harness = false + +[[bench]] +name = "index" +harness = false diff --git a/crates/quicksearch-core/benches/corpus/mod.rs b/crates/quicksearch-core/benches/corpus/mod.rs new file mode 100644 index 0000000..feaf9de --- /dev/null +++ b/crates/quicksearch-core/benches/corpus/mod.rs @@ -0,0 +1,188 @@ +//! Shared fixtures for the `search` and `index` benchmarks. +//! +//! Everything here is deterministic — the same LCG `tests/search_perf.rs` and +//! `examples/indexprobe.rs` use, seeded identically. A fixed corpus is what +//! makes two runs comparable, so a number that moved is a real change rather +//! than a different document. +//! +//! Corpora are built once per process and shared by every benchmark that +//! wants them. Divan re-runs a benchmarked closure thousands of times; paying +//! 256 KiB of text generation inside that loop would measure the generator. + +// Both bench binaries compile the whole module but each uses only part of it. +#![allow(dead_code)] + +use std::collections::HashMap; +use std::sync::LazyLock; + +/// Deterministic pseudo-random word picker. +pub struct Lcg(u64); + +impl Lcg { + pub fn new(seed: u64) -> Lcg { + Lcg(seed) + } + + pub fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1); + self.0 >> 33 + } +} + +/// Filler vocabulary. Deliberately excludes [`NEEDLE`] and every prefix of it +/// past two characters, so the only occurrences in a document are the ones +/// planted on purpose and a "zero hits" corpus really has zero. +const WORDS: &[&str] = &[ + "alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", "iota", "kappa", + "lambda", "sigma", "report", "summary", "meeting", "invoice", "contract", "budget", "revenue", + "quarter", "planning", "review", "draft", "final", "notes", "appendix", "figure", +]; + +/// The term every search benchmark looks for. Nine bytes, no overlap with +/// [`WORDS`], and long enough to clear the trigram floor the full-text pass +/// applies. +pub const NEEDLE: &str = "quartzite"; + +/// The three document sizes every size-swept benchmark uses, in bytes. +/// +/// 1 KiB is a short note, 16 KiB a typical source file or README, and 256 KiB +/// is `maximum_text_size` — the ceiling `config.processing` lets into the +/// index, and so the worst case a full-text pass has to survive per row. +pub const SIZES: [usize; 3] = [1 << 10, 16 << 10, 256 << 10]; + +/// Occurrence counts to sweep. Zero is the important one: a full-text pass +/// verifies far more candidate rows than it accepts, because the trigram +/// index matches on character triples rather than the whole term. +pub const HITS: [usize; 3] = [0, 4, 64]; + +/// Word text of about `size` bytes with [`NEEDLE`] planted `hits` times at +/// even spacing. +/// +/// Even spacing matters: it means a count has to scan to the end of the +/// document, and a snippet window never contains every match. Both are what +/// the cascade actually does. +pub fn document(size: usize, hits: usize) -> String { + let mut lcg = Lcg::new(0x5eed); + let mut out = String::with_capacity(size + 16); + let stride = if hits == 0 { usize::MAX } else { size / hits }; + let mut next_plant = stride; + while out.len() < size { + if out.len() >= next_plant { + out.push_str(NEEDLE); + out.push(' '); + next_plant = next_plant.saturating_add(stride); + continue; + } + out.push_str(WORDS[lcg.next() as usize % WORDS.len()]); + out.push(' '); + } + out +} + +/// [`document`], but with the planted term capitalised so a case-sensitive +/// scan misses and only the folded one hits. +/// +/// This is the stage-6 row and the tier-4 row — the case the cascade pays a +/// fold for, and the one a corpus of all-lowercase text would never produce. +pub fn document_mixed_case(size: usize, hits: usize) -> String { + let mut needle = NEEDLE.to_string(); + needle.replace_range(0..1, &NEEDLE[0..1].to_uppercase()); + document(size, hits).replace(NEEDLE, &needle) +} + +type Corpus = HashMap<(usize, usize), String>; + +fn build(f: fn(usize, usize) -> String) -> Corpus { + let mut m = HashMap::new(); + for size in SIZES { + for hits in HITS { + m.insert((size, hits), f(size, hits)); + } + } + m +} + +static LOWER: LazyLock = LazyLock::new(|| build(document)); +static MIXED: LazyLock = LazyLock::new(|| build(document_mixed_case)); +static FOLDED: LazyLock = LazyLock::new(|| { + MIXED + .iter() + .map(|(k, v)| (*k, v.to_ascii_lowercase())) + .collect() +}); +static BLOBS: LazyLock>> = LazyLock::new(|| { + LOWER + .iter() + .map(|(k, v)| (*k, zstd::encode_all(v.as_bytes(), 3).expect("encode"))) + .collect() +}); + +/// An all-lowercase document: a case-sensitive scan finds every planted term. +pub fn text(size: usize, hits: usize) -> &'static str { + &LOWER[&(size, hits)] +} + +/// A document whose planted terms are capitalised: case-sensitive misses, +/// folded hits. +pub fn text_mixed(size: usize, hits: usize) -> &'static str { + &MIXED[&(size, hits)] +} + +/// [`text_mixed`] pre-folded, for the benchmarks that measure a search +/// against a haystack the caller already lowered. +pub fn text_folded(size: usize, hits: usize) -> &'static str { + &FOLDED[&(size, hits)] +} + +/// [`text`] as stored: zstd level 3, exactly what `db/repo.rs` writes. +pub fn blob(size: usize, hits: usize) -> &'static [u8] { + &BLOBS[&(size, hits)] +} + +/// Realistic file names and full paths, for the benchmarks that model the +/// filename pass rather than the full-text one. +/// +/// The name pass scans the whole `files` table, so what matters is the +/// per-row cost on a *miss* — most rows match the SQL `LIKE` on some +/// directory component and then fail every name tier. +pub struct Row { + pub name: String, + pub path: String, +} + +static ROWS: LazyLock> = LazyLock::new(|| { + let mut lcg = Lcg::new(0xd00d); + (0..2000) + .map(|i| { + let w1 = WORDS[lcg.next() as usize % WORDS.len()]; + let w2 = WORDS[lcg.next() as usize % WORDS.len()]; + let name = format!("{}-{}-{:05}.txt", w1, w2, i); + // Mixed case in the directory portion, so the folded tiers are + // the ones that resolve — the common shape on real trees. + let path = format!("/home/user/Documents/Quartzite/{:03}/{}", i % 40, name); + Row { name, path } + }) + .collect() +}); + +pub fn rows() -> &'static [Row] { + &ROWS +} + +/// The head of a plain-text file, as the walk reads it: `hash_length` +/// (8 KiB) bytes or the whole file, whichever is smaller. +pub fn text_head() -> &'static [u8] { + static HEAD: LazyLock> = + LazyLock::new(|| document(8 << 10, 2).into_bytes()[..8 << 10].to_vec()); + &HEAD +} + +/// A binary head that no extractor claims — the control group for the MIME +/// sniff, which has to reject it by scanning. +pub fn binary_head() -> &'static [u8] { + static HEAD: LazyLock> = LazyLock::new(|| { + let mut lcg = Lcg::new(0xbeef); + (0..8 << 10).map(|_| (lcg.next() & 0xff) as u8).collect() + }); + &HEAD +} diff --git a/crates/quicksearch-core/benches/index.rs b/crates/quicksearch-core/benches/index.rs new file mode 100644 index 0000000..feda835 --- /dev/null +++ b/crates/quicksearch-core/benches/index.rs @@ -0,0 +1,190 @@ +//! Indexing-path microbenchmarks. +//! +//! Same convention as `benches/search.rs`: each group pairs two ways of doing +//! the same work, measured together, so a choice is justified rather than +//! asserted — and a losing arm records something already tried. +//! +//! ```text +//! cargo bench -p quicksearch-core --bench index +//! ``` + +mod corpus; + +use divan::Bencher; +use quicksearch_core::{mime, textenc, walk}; + +fn main() { + divan::main(); +} + +/// Compressing an extracted document, which `repo::set_content_done` used to +/// do itself — on the single writer thread, inside the transaction, and so +/// inside the `conn_mutex` hold `store_extracted` takes across a whole chunk. +/// +/// `encode_all` allocates and tears down a fresh `ZSTD_CCtx` — window, hash +/// and chain tables — per document, and for the small documents that dominate +/// a real tree that setup costs more than the compression. `batch_*` below +/// measures a whole chunk of it, which is what `compress_bodies` now does +/// before taking the lock. +mod zstd_encode { + use super::*; + + /// Level 3, matching `db/repo.rs`'s `ZSTD_LEVEL`. Not a variable here: + /// the level is argued in place and this measures the machinery around + /// it, not the level. + const LEVEL: i32 = 3; + + #[divan::bench(args = corpus::SIZES)] + fn encode_all(bencher: Bencher, size: usize) { + let text = corpus::text(size, 4).as_bytes(); + bencher.bench(|| zstd::encode_all(divan::black_box(text), LEVEL).unwrap()); + } + + #[divan::bench(args = corpus::SIZES)] + fn bulk_reused(bencher: Bencher, size: usize) { + let text = corpus::text(size, 4).as_bytes(); + let mut enc = zstd::bulk::Compressor::new(LEVEL).unwrap(); + bencher.bench_local(move || enc.compress(divan::black_box(text)).unwrap()); + } + + /// One writer chunk: `processing.batch_size` documents of the size most + /// documents are. + /// + /// The `encode_all` figure is how much pure CPU used to sit inside the + /// `conn_mutex` hold. That lock serializes the indexer against itself — + /// not against search, which holds its own connection and reads through + /// WAL — so this is a contention figure, not a throughput one. An + /// end-to-end cold index of an 80 MiB tree does not move measurably: FTS5 + /// trigram tokenization dominates it, and all compression together is + /// under 1% of the run. + const BATCH: usize = 500; + + #[divan::bench] + fn batch_encode_all(bencher: Bencher) { + let text = corpus::text(1 << 10, 4).as_bytes(); + bencher.bench(|| { + (0..BATCH) + .map(|_| { + zstd::encode_all(divan::black_box(text), LEVEL) + .unwrap() + .len() + }) + .sum::() + }); + } + + #[divan::bench] + fn batch_bulk_reused(bencher: Bencher) { + let text = corpus::text(1 << 10, 4).as_bytes(); + let mut enc = zstd::bulk::Compressor::new(LEVEL).unwrap(); + bencher.bench_local(move || { + (0..BATCH) + .map(|_| enc.compress(divan::black_box(text)).unwrap().len()) + .sum::() + }); + } +} + +/// What a small text file costs between the MIME sniff and the decode. +/// +/// The walk reads an 8 KiB head, sniffs it (`looks_like_text` → `classify`, a +/// control-byte scan plus a UTF-8 validation), and then — for a file the head +/// covers entirely — decodes it inline (`decode_text` → `classify` again, on +/// the same bytes). So `classify` runs twice on identical bytes, and +/// `sniff_only` says the second pass is about half of `sniff_then_decode`. +/// +/// Left alone deliberately. The two calls pass different `truncated` flags and +/// genuinely disagree on a file ending mid-multibyte-sequence, so sharing a +/// verdict means threading `TextClass` through `mime.rs`, the `Extractor` +/// trait and `plaintext.rs`. Against a cold `indexprobe` run that whole +/// pipeline is under 1% of wall time — the redundancy is real and still not +/// worth the coupling. Re-measure here before deciding otherwise. +mod text_pipeline { + use super::*; + use std::path::Path; + + #[divan::bench] + fn sniff_then_decode(bencher: Bencher) { + let head = corpus::text_head(); + let path = Path::new("/tmp/bench/notes.txt"); + bencher.bench(|| { + let looks = textenc::looks_like_text(divan::black_box(head)); + let text = textenc::decode_text(head.to_vec(), path).unwrap(); + (looks, text.len()) + }); + } + + #[divan::bench] + fn sniff_only(bencher: Bencher) { + let head = corpus::text_head(); + bencher.bench(|| textenc::looks_like_text(divan::black_box(head))); + } + + /// Includes the `head.to_vec()` at `extract/plaintext.rs:113` — a full + /// copy of the head made only because `decode_text` takes ownership. + #[divan::bench] + fn decode_only(bencher: Bencher) { + let head = corpus::text_head(); + let path = Path::new("/tmp/bench/notes.txt"); + bencher.bench(|| textenc::decode_text(divan::black_box(head).to_vec(), path)); + } +} + +/// The MIME sniff itself, once per new or changed file. +/// +/// It carries several `to_ascii_lowercase` allocations for extension and MIME +/// comparisons that `eq_ignore_ascii_case` would do without allocating — and +/// removing them is not worth doing: the whole sniff is ~180 ns on a text head, +/// against ~2.2 µs for the `classify` next to it. This group exists to keep +/// that ratio visible. +mod mime_sniff { + use super::*; + use std::path::Path; + + #[divan::bench] + fn text_head(bencher: Bencher) { + let head = corpus::text_head(); + let path = Path::new("/tmp/bench/notes.txt"); + bencher.bench(|| mime::guess_mime_from_head(path, divan::black_box(head))); + } + + #[divan::bench] + fn binary_head(bencher: Bencher) { + let head = corpus::binary_head(); + let path = Path::new("/tmp/bench/blob.bin"); + bencher.bench(|| mime::guess_mime_from_head(path, divan::black_box(head))); + } + + /// No extension to go on, so the sniff falls all the way through to the + /// magic-byte scan and the text classifier. + #[divan::bench] + fn no_extension(bencher: Bencher) { + let head = corpus::text_head(); + let path = Path::new("/tmp/bench/LICENSE"); + bencher.bench(|| mime::guess_mime_from_head(path, divan::black_box(head))); + } +} + +/// A SHA-256 over the path string, per file, on every run — including the +/// unchanged steady state, where it is the only compute a file costs beyond +/// its `statx`. +/// +/// Reference measurement only. The plan does not propose changing it: its +/// rationale is collision resistance against adversarial filenames on shared +/// volumes, and this is here to confirm the cost is small enough that the +/// argument stands unchallenged. +mod path_digest { + use super::*; + + #[divan::bench] + fn per_path(bencher: Bencher) { + let rows = corpus::rows(); + bencher.bench(|| { + let mut acc = 0u128; + for row in divan::black_box(rows) { + acc ^= walk::path_digest(&row.path); + } + acc + }); + } +} diff --git a/crates/quicksearch-core/benches/search.rs b/crates/quicksearch-core/benches/search.rs new file mode 100644 index 0000000..15e2248 --- /dev/null +++ b/crates/quicksearch-core/benches/search.rs @@ -0,0 +1,363 @@ +//! Search-path microbenchmarks. +//! +//! Every group pairs two ways of doing the same work in one run, so the delta +//! is a measurement rather than an estimate. Some pairs justify a choice the +//! code has already made; others record something tried and rejected. Both are +//! worth keeping — a losing arm is the cheapest documentation there is that an +//! obvious-looking idea was measured and did not pay. +//! +//! Run with: +//! +//! ```text +//! cargo bench -p quicksearch-core --bench search +//! ``` +//! +//! Sizes come from `corpus::SIZES` — 1 KiB, 16 KiB and 256 KiB, the last +//! being `maximum_text_size`, the largest document the index will hold and so +//! the worst case a full-text row can present. + +mod corpus; + +use divan::Bencher; +use quicksearch_core::query::pattern::{TermPart, TermPattern}; +use quicksearch_core::search::fuzzy::Bitap; +use quicksearch_core::snippet; + +fn main() { + divan::main(); +} + +fn literal(term: &str) -> TermPattern { + TermPattern::build(&[TermPart { + text: term.to_string(), + glob: false, + }]) + .expect("literal patterns always compile") +} + +/// Decompressing the stored document body — the first thing every full-text +/// row does, at `search/cascade/passes.rs:241`, `:425` and `:528`. +/// +/// `decode_all` builds a fresh `ZSTD_DCtx` and a ~131 KB `BufReader` per call, +/// then grows an unsized `Vec` as it goes; the other arm reuses one context and +/// sizes the output up front. Measured at 8.4/16.0/102 µs against +/// 1.6/8.0/89 µs — a 4.4x gap at 1 KiB, which is the size most documents are. +/// This is why `DocDecoder` exists. +mod zstd_decode { + use super::*; + + #[divan::bench(args = corpus::SIZES)] + fn decode_all(bencher: Bencher, size: usize) { + let blob = corpus::blob(size, 4); + bencher.bench(|| zstd::decode_all(divan::black_box(blob)).unwrap()); + } + + #[divan::bench(args = corpus::SIZES)] + fn bulk_reused(bencher: Bencher, size: usize) { + let blob = corpus::blob(size, 4); + let capacity = corpus::text(size, 4).len(); + let mut dec = zstd::bulk::Decompressor::new().unwrap(); + bencher.bench_local(move || dec.decompress(divan::black_box(blob), capacity).unwrap()); + } +} + +/// Turning decompressed bytes into a `&str`. +/// +/// `from_utf8_lossy(..).into_owned()` copies the whole document even when the +/// bytes are already valid UTF-8 — and they always are, since +/// `textenc::decode_text` is the only thing that writes them. It is also far +/// slower than it looks: its validation is a scanning loop, where +/// `String::from_utf8` uses the vectorized one and *moves* the buffer it +/// validates. 230/6360/52200 ns against 15/227/3200 ns, a 16-28x gap that is +/// mostly validation rather than the copy. `DocDecoder` borrows instead. +mod utf8 { + use super::*; + + #[divan::bench(args = corpus::SIZES)] + fn lossy_into_owned(bencher: Bencher, size: usize) { + let raw = corpus::text(size, 4).as_bytes(); + bencher.bench(|| String::from_utf8_lossy(divan::black_box(raw)).into_owned()); + } + + #[divan::bench(args = corpus::SIZES)] + fn lossy_borrowed(bencher: Bencher, size: usize) { + let raw = corpus::text(size, 4).as_bytes(); + bencher.bench(|| { + let cow = String::from_utf8_lossy(divan::black_box(raw)); + cow.len() + }); + } + + /// The decompressor hands back an owned `Vec` that nothing else + /// references, so `String::from_utf8` can validate and *move* it rather + /// than validate and copy. Falling back to `from_utf8_lossy` on error + /// keeps the current behaviour for a corrupt row exactly. + #[divan::bench(args = corpus::SIZES)] + fn from_utf8_move(bencher: Bencher, size: usize) { + let raw = corpus::text(size, 4).as_bytes(); + bencher + .with_inputs(|| raw.to_vec()) + .bench_values(|owned| match String::from_utf8(owned) { + Ok(s) => s, + Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(), + }); + } +} + +/// ASCII-folding the document, which every full-text row needs for the +/// case-insensitive count and the snippet. +/// +/// A result worth keeping visible: folding into a reused buffer is *not* +/// faster. `to_ascii_lowercase` allocates and folds in one pass, where +/// clear + `push_str` + `make_ascii_lowercase` walks the bytes twice, and at +/// 256 KiB the reused buffer measures slightly behind. `fold_into` is chosen +/// for what it does to the allocator, not to the clock — do not "optimize" the +/// other direction on the assumption that removing an allocation must win. +mod fold { + use super::*; + + #[divan::bench(args = corpus::SIZES)] + fn to_ascii_lowercase(bencher: Bencher, size: usize) { + let text = corpus::text_mixed(size, 4); + bencher.bench(|| divan::black_box(text).to_ascii_lowercase()); + } + + #[divan::bench(args = corpus::SIZES)] + fn into_reused_buffer(bencher: Bencher, size: usize) { + let text = corpus::text_mixed(size, 4); + let mut buf = String::new(); + bencher.bench_local(move || { + buf.clear(); + buf.push_str(divan::black_box(text)); + // SAFETY-free equivalent of the in-place fold: `make_ascii_lowercase` + // is byte-length preserving, which is the same invariant the + // cascade already relies on for folded offsets. + buf.make_ascii_lowercase(); + buf.len() + }); + } +} + +/// Substring search over a document body: `str::match_indices` (std's Two-Way +/// searcher) against `memchr::memmem` (Two-Way plus a SIMD prefilter). +/// +/// The miss case matters most. The trigram index matches on character triples, +/// so a full-text pass verifies far more rows than it accepts, and a miss scans +/// the whole document before giving up. At 256 KiB that is 111 µs against +/// 2.4 µs — the measurement `snippet.rs` uses `memmem` for. `match_indices` +/// stays here as the regression guard: if these two ever converge, the SIMD +/// path has stopped being selected. +mod substring { + use super::*; + + #[divan::bench(args = corpus::SIZES)] + fn match_indices_miss(bencher: Bencher, size: usize) { + let text = corpus::text(size, 0); + bencher.bench(|| divan::black_box(text).match_indices(corpus::NEEDLE).count()); + } + + #[divan::bench(args = corpus::SIZES)] + fn memmem_miss(bencher: Bencher, size: usize) { + let text = corpus::text(size, 0).as_bytes(); + let finder = memchr::memmem::Finder::new(corpus::NEEDLE); + bencher.bench(|| finder.find_iter(divan::black_box(text)).count()); + } + + #[divan::bench(args = corpus::SIZES)] + fn match_indices_hits(bencher: Bencher, size: usize) { + let text = corpus::text(size, 64); + bencher.bench(|| divan::black_box(text).match_indices(corpus::NEEDLE).count()); + } + + #[divan::bench(args = corpus::SIZES)] + fn memmem_hits(bencher: Bencher, size: usize) { + let text = corpus::text(size, 64).as_bytes(); + let finder = memchr::memmem::Finder::new(corpus::NEEDLE); + bencher.bench(|| finder.find_iter(divan::black_box(text)).count()); + } + + /// What `pass_fulltext` actually runs per row, through the real crate + /// entry points: a case-sensitive count, then a folded count, then the + /// snippet extraction. Three sweeps of the same document. + #[divan::bench(args = corpus::SIZES)] + fn cascade_row_sweeps(bencher: Bencher, size: usize) { + let pattern = literal(corpus::NEEDLE); + let text = corpus::text_mixed(size, 4); + let folded = corpus::text_folded(size, 4); + let opts = snippet::Options { approx_chars: 600 }; + bencher.bench(|| { + let a = pattern.count(divan::black_box(text), false); + let b = pattern.count_folded(divan::black_box(folded)); + let s = snippet::extract_folded(text, folded, &[corpus::NEEDLE], &opts); + (a, b, s.ranges.len()) + }); + } +} + +/// Snippet extraction against a pre-folded haystack, the third of those +/// sweeps. Also carries a per-call `term.to_ascii_lowercase()` at +/// `snippet.rs:81` for a needle the caller already holds folded. +mod snippet_extract { + use super::*; + + #[divan::bench(args = corpus::SIZES)] + fn extract_folded(bencher: Bencher, size: usize) { + let text = corpus::text_mixed(size, 64); + let folded = corpus::text_folded(size, 64); + let opts = snippet::Options { approx_chars: 600 }; + bencher.bench(|| { + snippet::extract_folded( + divan::black_box(text), + divan::black_box(folded), + &[corpus::NEEDLE], + &opts, + ) + }); + } +} + +/// The filename pass's per-row ladder, over 2000 realistic name/path rows. +/// +/// `pass_filename` scans the whole `files` table — its `LIKE '%term%'` +/// predicate can use no index — and tiers 4 and 10 both run a +/// case-insensitive find, so a row matching on its directory portion pays +/// twice. +/// +/// The instructive part is that the two obvious fixes each make it *worse* +/// alone: a reused fold buffer measures ~2x slower than folding into a fresh +/// allocation, and a prebuilt `memmem::Finder` is slower than `str::find` on +/// haystacks this short. Only together do they win, and only by ~1.2x. Short +/// strings do not behave like document bodies; measure them separately. +mod filename_ladder { + use super::*; + + #[divan::bench] + fn find_first_ci_current(bencher: Bencher) { + let pattern = literal("quartzite"); + let rows = corpus::rows(); + bencher.bench(|| { + let mut found = 0usize; + for row in divan::black_box(rows) { + if pattern.find_first(&row.name, true).is_some() + || pattern.find_first(&row.path, true).is_some() + { + found += 1; + } + } + found + }); + } + + #[divan::bench] + fn find_first_ci_scratch(bencher: Bencher) { + let pattern = literal("quartzite"); + let rows = corpus::rows(); + let mut scratch = String::new(); + bencher.bench_local(move || { + let mut found = 0usize; + for row in divan::black_box(rows) { + scratch.clear(); + scratch.push_str(&row.name); + scratch.make_ascii_lowercase(); + if pattern.find_first_folded(&scratch).is_some() { + found += 1; + continue; + } + scratch.clear(); + scratch.push_str(&row.path); + scratch.make_ascii_lowercase(); + if pattern.find_first_folded(&scratch).is_some() { + found += 1; + } + } + found + }); + } + + /// Fold as today, but search the folded copy with a `Finder` built once + /// per query instead of `str::find`'s Two-Way. Isolates the searcher from + /// the allocation: if this wins and `find_first_ci_scratch` does not, the + /// fold was never the problem. + #[divan::bench] + fn find_first_ci_memmem(bencher: Bencher) { + let finder = memchr::memmem::Finder::new("quartzite"); + let rows = corpus::rows(); + bencher.bench(|| { + let mut found = 0usize; + for row in divan::black_box(rows) { + if finder + .find(row.name.to_ascii_lowercase().as_bytes()) + .is_some() + || finder + .find(row.path.to_ascii_lowercase().as_bytes()) + .is_some() + { + found += 1; + } + } + found + }); + } + + /// Both at once: one reused fold buffer and a prebuilt `Finder`. + #[divan::bench] + fn find_first_ci_scratch_memmem(bencher: Bencher) { + let finder = memchr::memmem::Finder::new("quartzite"); + let rows = corpus::rows(); + let mut scratch = String::new(); + bencher.bench_local(move || { + let mut found = 0usize; + for row in divan::black_box(rows) { + scratch.clear(); + scratch.push_str(&row.name); + scratch.make_ascii_lowercase(); + if finder.find(scratch.as_bytes()).is_some() { + found += 1; + continue; + } + scratch.clear(); + scratch.push_str(&row.path); + scratch.make_ascii_lowercase(); + if finder.find(scratch.as_bytes()).is_some() { + found += 1; + } + } + found + }); + } +} + +/// Bitap, the fuzzy passes' inner loop. Both fuzzy passes are whole-table +/// scans, so this runs over every row in the index when fuzzy is on. +/// +/// `step` still takes `&mut [u64]` rather than `&mut [u64; MAX_REGISTERS]`, +/// so the register indices are bounds-checked and the trip count is opaque +/// to the optimizer. +mod bitap { + use super::*; + + #[divan::bench(args = corpus::SIZES)] + fn count_and_first_k2(bencher: Bencher, size: usize) { + let bitap = Bitap::new(corpus::NEEDLE.as_bytes(), 2).unwrap(); + let hay = corpus::text(size, 4).as_bytes(); + bencher.bench(|| bitap.count_and_first(divan::black_box(hay))); + } + + /// The filename pass's shape: many short haystacks rather than one long + /// one, with the per-call 176-byte register memset amortized over very + /// little work. + #[divan::bench] + fn best_distance_over_names_k2(bencher: Bencher) { + let bitap = Bitap::new(b"quartzite", 2).unwrap(); + let rows = corpus::rows(); + bencher.bench(|| { + let mut hits = 0usize; + for row in divan::black_box(rows) { + if bitap.best_distance_and_first(row.name.as_bytes()).is_some() { + hits += 1; + } + } + hits + }); + } +} diff --git a/crates/quicksearch-core/examples/indexprobe.rs b/crates/quicksearch-core/examples/indexprobe.rs index 1320c6d..89beeea 100644 --- a/crates/quicksearch-core/examples/indexprobe.rs +++ b/crates/quicksearch-core/examples/indexprobe.rs @@ -251,7 +251,13 @@ fn run(mode: &str, tree: &Path, db: &Path) { assert!(done, "indexing did not finish within the timeout"); service.stop_indexing().expect("stop"); - let total = SMALL_TEXT + LARGE_TEXT + BINARY; + // Count what was actually indexed rather than assuming `gen`'s tree. + // The constants describe the tree this probe builds; pointing it at any + // other one made the rate a fiction. + let total = rusqlite::Connection::open(db) + .ok() + .and_then(|c| quicksearch_core::db::repo::row_count(&c).ok()) + .unwrap_or(0); eprintln!( "{}: {:?} ({:.0} files/sec over {} files)", mode, diff --git a/crates/quicksearch-core/src/cli.rs b/crates/quicksearch-core/src/cli.rs index d8746a7..a3b0e36 100644 --- a/crates/quicksearch-core/src/cli.rs +++ b/crates/quicksearch-core/src/cli.rs @@ -221,6 +221,7 @@ mod tests { use crate::db::open_or_recreate; use crate::db::repo::{insert_file, set_content_done, set_content_failed, NewFile}; use crate::mime::FileType; + use crate::testutil::zstd_of; fn tmp_path() -> std::path::PathBuf { crate::testutil::scratch_dir("cli").join("index.sqlite") @@ -248,7 +249,7 @@ mod tests { ) .unwrap() .expect("unique path"); - set_content_done(&tx, a, "a.txt", "hello", &[], true).unwrap(); + set_content_done(&tx, a, "a.txt", "hello", &[], zstd_of("hello").as_deref()).unwrap(); let b = insert_file( &tx, &NewFile { @@ -427,7 +428,7 @@ mod tests { .unwrap() .expect("unique path"); let prose = "the quick brown fox jumps over the lazy dog. ".repeat(500); - set_content_done(&tx, id, "big.txt", &prose, &[], true).unwrap(); + set_content_done(&tx, id, "big.txt", &prose, &[], zstd_of(&prose).as_deref()).unwrap(); tx.commit().unwrap(); } drop(conn); diff --git a/crates/quicksearch-core/src/coordinator.rs b/crates/quicksearch-core/src/coordinator.rs index f8aaf1d..6543c5e 100644 --- a/crates/quicksearch-core/src/coordinator.rs +++ b/crates/quicksearch-core/src/coordinator.rs @@ -104,6 +104,23 @@ pub struct IndexerState { /// A run's *own* reconciliation is not here — it reads as /// [`IndexingStatus::Preparing`] with a [`PrepStep::Reconciling`]. pub reconcile: Option, + /// What each configured root held when indexing last completed. Roots + /// never indexed to completion are absent rather than zero. + /// + /// `Arc` because `state()` is called more than once per frame and this + /// changes only when a run ends. + pub root_counts: Arc>, +} + +/// One configured root's stored figures, keyed the way the caller spells it. +#[derive(Debug, Clone)] +pub struct RootCount { + /// The root exactly as `paths.indexing_paths` gives it, so a frontend can + /// match it against the string it already draws. The `schema_info` key + /// behind it is the canonicalized spelling, so re-spelling a root in the + /// config keeps its figures. + pub root: String, + pub counts: db::repo::RootCounts, } #[allow(clippy::large_enum_variant)] @@ -161,6 +178,7 @@ struct Shared { queued_events: usize, watcher: WatcherStatus, reconcile: Option, + root_counts: Arc>, } impl IndexCoordinator { @@ -192,6 +210,7 @@ impl IndexCoordinator { queued_events: 0, watcher: WatcherStatus::Off, reconcile: None, + root_counts: Arc::new(Vec::new()), })); let reconcile_stop = Arc::new(ReconcileStop::default()); @@ -251,6 +270,7 @@ impl IndexCoordinator { queued_events: shared.queued_events, watcher: shared.watcher.clone(), reconcile: shared.reconcile, + root_counts: shared.root_counts.clone(), } } diff --git a/crates/quicksearch-core/src/coordinator/inner.rs b/crates/quicksearch-core/src/coordinator/inner.rs index d83f5d3..64d6282 100644 --- a/crates/quicksearch-core/src/coordinator/inner.rs +++ b/crates/quicksearch-core/src/coordinator/inner.rs @@ -116,6 +116,9 @@ impl Inner { // restart is cheap and unconditional beats a diff here. self.start_watcher(); } + // The root list, its spellings, or the database behind it may + // all have moved; re-pair them with what is stored. + self.refresh_last_full_index(); } CoordCmd::RebuildIndex => { let db = self.db_path(); @@ -127,6 +130,7 @@ impl Inner { if let Err(e) = self.indexing.delete_index_for_rebuild(&db) { crate::log_warn!("coordinator: rebuild: {}", e); } + self.clear_root_counts(); self.start_full_run(); if self.mode != IndexMode::Auto { self.mode = IndexMode::ManualRunning; @@ -148,6 +152,9 @@ impl Inner { // Zero, not `None`: nothing will rebuild this index, so no // later read corrects a stale figure. shared.files = Some(0); + // Per root the empty list reads as "not yet indexed", which is + // what every folder now is. + shared.root_counts = Arc::new(Vec::new()); drop(shared); self.files_at = None; } @@ -717,7 +724,13 @@ impl Inner { } } - /// Re-read the stamp the last completed full run left behind. + /// Re-read what the last completed full run left behind: its stamp, and + /// the per-root figures the folder list shows. + /// + /// Both off one connection because they are wanted at the same moments — + /// startup, a run finishing, a config change. Neither is a scan: the stamp + /// and each root's counts are single `schema_info` key lookups, the work of + /// counting having been done by the run that stored them. /// /// A failed open is *not* published as `None`: `periodic_due` reads `None` /// as "never indexed" and would start a fresh run every tick for as long @@ -726,12 +739,45 @@ impl Inner { match db::open_existing(&self.db_path(), false) { Ok(conn) => { let last = db::repo::get_last_full_index(&conn); - crate::lock_ok(&self.shared).last_full_index = last; + let counts = self.read_root_counts(&conn); + let mut shared = crate::lock_ok(&self.shared); + shared.last_full_index = last; + shared.root_counts = Arc::new(counts); } Err(e) => crate::log_warn!("coordinator: last-full-index unreadable: {}", e), } } + /// Pair every configured root with its stored figures, keyed by the + /// spelling the config uses so a frontend can match what it draws. + /// + /// The `schema_info` keys are canonicalized, which is what makes writing + /// `~/docs` where the config said `/home/me/docs` keep the figures — the + /// same re-keying `indexing::resolved_root_workers` does in the other + /// direction. + fn read_root_counts(&self, conn: &Connection) -> Vec { + self.config + .paths + .indexing_paths + .iter() + .zip(self.config.resolved_indexing_paths()) + .filter_map(|(raw, resolved)| { + let root = crate::file_handling::normalize_root_string(&resolved.to_string_lossy()); + let counts = db::repo::get_root_counts(conn, &root)?; + Some(RootCount { + root: raw.clone(), + counts, + }) + }) + .collect() + } + + /// Forget the published figures: the index behind them is gone, and + /// nothing will correct them until a run rebuilds it. + fn clear_root_counts(&self) { + crate::lock_ok(&self.shared).root_counts = Arc::new(Vec::new()); + } + fn publish(&mut self) { let reconcile = match &self.pending_work { Some(cursor) => Some(ReconcileState::Running(cursor.progress())), diff --git a/crates/quicksearch-core/src/db/repo.rs b/crates/quicksearch-core/src/db/repo.rs index 102ba2c..48fa991 100644 --- a/crates/quicksearch-core/src/db/repo.rs +++ b/crates/quicksearch-core/src/db/repo.rs @@ -166,13 +166,22 @@ pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result