Significant performance pass. Reduced memory churn massively. ~1.5-3x search performance improvements. Minor improvements to warm indexing time.
This commit is contained in:
=
parent
5714b5f969
commit
5f55d19849
30 changed files with 1991 additions and 122 deletions
85
Cargo.lock
generated
85
Cargo.lock
generated
|
|
@ -111,6 +111,12 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
||||
|
||||
[[package]]
|
||||
name = "arboard"
|
||||
version = "3.6.1"
|
||||
|
|
@ -658,6 +664,32 @@ dependencies = [
|
|||
"inout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
"terminal_size",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
|
||||
|
||||
[[package]]
|
||||
name = "clipboard-win"
|
||||
version = "5.4.1"
|
||||
|
|
@ -697,6 +729,12 @@ dependencies = [
|
|||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "condtype"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af"
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation"
|
||||
version = "0.9.4"
|
||||
|
|
@ -890,6 +928,31 @@ dependencies = [
|
|||
"syn 2.0.66",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "divan"
|
||||
version = "0.1.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a405457ec78b8fe08b0e32b4a3570ab5dff6dd16eb9e76a5ee0a9d9cbd898933"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"clap",
|
||||
"condtype",
|
||||
"divan-macros",
|
||||
"libc",
|
||||
"regex-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "divan-macros"
|
||||
version = "0.1.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9556bc800956545d6420a640173e5ba7dfa82f38d3ea5a167eb555bc69ac3323"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.66",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dlib"
|
||||
version = "0.5.3"
|
||||
|
|
@ -3125,12 +3188,13 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "quicksearch-core"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"argon2",
|
||||
"cfb",
|
||||
"chardetng",
|
||||
"ctrlc",
|
||||
"divan",
|
||||
"encoding_rs",
|
||||
"getrandom 0.2.15",
|
||||
"globset",
|
||||
|
|
@ -3138,6 +3202,7 @@ dependencies = [
|
|||
"kamadak-exif",
|
||||
"libc",
|
||||
"lofty",
|
||||
"memchr",
|
||||
"mime_guess",
|
||||
"notify",
|
||||
"pdf-extract",
|
||||
|
|
@ -3157,7 +3222,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "quicksearch-gui"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"ashpd",
|
||||
"chrono",
|
||||
|
|
@ -3323,6 +3388,12 @@ dependencies = [
|
|||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-lite"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.4"
|
||||
|
|
@ -3824,6 +3895,16 @@ dependencies = [
|
|||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "terminal_size"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874"
|
||||
dependencies = [
|
||||
"rustix 1.1.4",
|
||||
"windows-sys 0.61.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "1.0.61"
|
||||
|
|
|
|||
21
Cargo.toml
21
Cargo.toml
|
|
@ -5,8 +5,27 @@ members = [
|
|||
"crates/quicksearch-gui",
|
||||
]
|
||||
|
||||
# No `[profile.release]` on purpose. Cross-crate inlining looks like it should
|
||||
# pay here — the cascade calls into `snippet`/`query`, both into `memchr` and
|
||||
# `zstd`, everything into `rusqlite`'s FFI wrappers — so it was measured, and
|
||||
# it does not. Against an incremental rebuild of the GUI after touching core
|
||||
# (10.2 s at the defaults):
|
||||
#
|
||||
# lto=thin 67 s search cold 29.9 ms warm best 14.77 ms index cold 459 ms
|
||||
# lto=thin,cgu=1 152 s (build cost alone ruled it out)
|
||||
# lto=fat,cgu=1 200 s search cold 29.2 ms warm best 14.57 ms index cold 479 ms
|
||||
# (defaults) 10 s search cold 30.0 ms warm best 14.92 ms index cold 478 ms
|
||||
#
|
||||
# Every runtime column moves 0-4% and index-cold is not even monotonic, so the
|
||||
# gain is at the noise floor while the build is 6.6-19.5x longer — on a CI that
|
||||
# builds three targets. Re-measure before concluding otherwise.
|
||||
#
|
||||
# `panic = "abort"` is separately unavailable: `extract/pdf.rs` runs
|
||||
# `pdf-extract` inside `catch_unwind`, so aborting would turn a malformed PDF
|
||||
# into a killed process instead of one skipped file.
|
||||
|
||||
[workspace.package]
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
edition = "2021"
|
||||
license = "GPL-3.0-or-later"
|
||||
authors = ["Jeremy <jeremy@karsttech.com>"]
|
||||
|
|
|
|||
73
README.md
73
README.md
|
|
@ -237,7 +237,13 @@ inside that folder.
|
|||
restarts until you return to automatic. The index size beside the
|
||||
status heading totals the database and its `-wal`/`-shm` sidecars,
|
||||
refreshed every ten seconds; hovering it lists the ways to make it
|
||||
smaller.
|
||||
smaller. Each folder in the list carries what it holds — files
|
||||
indexed, and how many of those had text extracted — counted once as
|
||||
each indexing run finishes and stored with the index, so they are
|
||||
there the moment the app opens rather than costing a scan to show. A
|
||||
folder nothing has finished indexing reads "not yet indexed" rather
|
||||
than zero, and because the figures come from completed runs they do
|
||||
not move as live updates apply single changes in between.
|
||||
- **Duplicates**: files sharing a content hash, grouped.
|
||||
- **Logs**: the lines the app would have printed to a terminal — warnings
|
||||
from indexing, folder watching and opening files, newest last, with a
|
||||
|
|
@ -458,7 +464,14 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
|
|||
then held — the coordinator's writer, before it learned to let go when idle —
|
||||
keeps that memory for the life of the process. Search is the one deliberately
|
||||
large one, because it is the only cache reused often enough to pay for
|
||||
itself, and it is released once searching stops.
|
||||
itself, and it is released once searching stops. Dropping the connection is
|
||||
only half of releasing it: glibc hands the freed pages back to its own arena
|
||||
rather than to the kernel, so the search worker calls
|
||||
`platform::release_free_heap` after it lets go, exactly as
|
||||
`coordinator::go_idle` does for the writer. Without that call one typing
|
||||
session left the process 42 MiB heavier for as long as it ran — measured on a
|
||||
77k-file index, where an idle GUI sat at 76 MiB `RssAnon` and stayed there,
|
||||
against 34 MiB before the first search and 42 MiB once the trim runs.
|
||||
- **Indexing** (`indexing.rs`, `file_handling.rs`): full runs walk each
|
||||
root (`filtered_walk` prunes hidden/ignored subtrees before descending),
|
||||
classify files by mtime into insert/update/skip, batch-write metadata,
|
||||
|
|
@ -467,9 +480,15 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
|
|||
OLE2 streams are read in `extract/ole.rs` — PDF, audio tags, EXIF; see
|
||||
`extract/`) for FTS. PDFs are parsed once, with the text and the `Info`
|
||||
dictionary taken off the same document: the two-parse version that preceded
|
||||
it was a run's largest single memory consumer, and it was what pulled a
|
||||
second copy of `lopdf` — and with it rayon's never-torn-down thread pool —
|
||||
into the build. Files whose extension no MIME
|
||||
it was the largest single memory consumer of a run over a PDF-heavy tree, and
|
||||
it was what pulled a second copy of `lopdf` — and with it rayon's
|
||||
never-torn-down thread pool — into the build. That is a claim about PDFs
|
||||
rather than about runs in general, and it is worth knowing which tree a
|
||||
number came from: on one with almost no PDFs, a cold run peaks at 130 MiB
|
||||
against 27 MiB for the same walk with content extraction switched off, and
|
||||
switching off `store_text_for_snippets` moves that peak not at all — so what
|
||||
is left is the extraction workers and FTS5's own index build, not any single
|
||||
parser and not the stored text. Files whose extension no MIME
|
||||
table knows — including extensionless ones like `README` or `Makefile` —
|
||||
are sniffed from their head bytes and indexed as text only when that head
|
||||
is provably text: valid UTF-8, or BOM-marked (`mime.rs`, `textenc.rs`).
|
||||
|
|
@ -554,6 +573,31 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
|
|||
flush last, so weaker matches only ever append. All SQL is
|
||||
parameterized; structured filters from the query language (`query/`)
|
||||
are ANDed onto every pass.
|
||||
|
||||
The passes that read document text share one `zstd::bulk::Decompressor` and
|
||||
one output buffer per scan (`DocDecoder` in `search/cascade/passes.rs`),
|
||||
and the row's path is borrowed from the statement rather than copied — only
|
||||
rows that become hits own one. Both matter more than they look: peak memory
|
||||
during a search never exceeded 14 MiB even before any of this, but a *single*
|
||||
fuzzy query moved 31 GiB through `malloc`, and resident-set sampling is blind
|
||||
to that by construction, because a buffer allocated and freed inside one loop
|
||||
iteration never moves RSS. `DocDecoder` must therefore never fall back to a
|
||||
per-row allocating decode, and there is a trap waiting there:
|
||||
`zstd::encode_all`, which the indexer writes with, is *stream*-based and so
|
||||
records no content size in the frame header, meaning
|
||||
`get_frame_content_size` returns `None` for every row this ever sees. The
|
||||
"cannot happen" branch is the only branch. Sizing the buffer from the header
|
||||
and handing the `None` case to `zstd::decode_all` looks obviously right and
|
||||
costs ~2.4 MiB per document, because `decode_all` builds a streaming decoder
|
||||
per call — 27 of the 30 GiB a fuzzy search moved. Growing this buffer and
|
||||
keeping it is what makes decoding a row allocate nothing at all.
|
||||
Measured over the same 77k-file index, per query:
|
||||
`cascade` 582 → 14 MiB, `function` 6.0 GiB → 29 MiB, `--fuzzy cascade`
|
||||
31 GiB → 57 MiB, `regex:` 31 GiB → 39 MiB, each a little faster rather than
|
||||
slower. `TermPattern::find_first` folds nothing either: its
|
||||
case-insensitive literal branch used to allocate a lowercased copy of its
|
||||
haystack, which the filename pass asked for twice per row of a full-table
|
||||
scan.
|
||||
- **Baloo compatibility** (`cli.rs`, `mime.rs`): the read API this repo's
|
||||
parent consumes — `status_for_path`, `list_failed`,
|
||||
`index_size_breakdown`, `pending_content_count`, `clear_path` — plus a
|
||||
|
|
@ -612,6 +656,15 @@ microseconds regardless of row count.
|
|||
synthesizing clicks and reading back the painted text (`test_ui.rs`) — over
|
||||
the search and manage tabs, the options editor, the unlock gate, the logs
|
||||
and duplicates tabs, and query highlighting.
|
||||
- `cargo bench -p quicksearch-core --bench search` and `--bench index`: divan
|
||||
microbenchmarks over the two hot paths. Each group runs *what the code does
|
||||
today* against *the change being considered*, in one process on one corpus,
|
||||
so the comparison is a measurement rather than an estimate — read the losing
|
||||
arm as documentation of something already tried. Sizes sweep 1 KiB, 16 KiB
|
||||
and 256 KiB, the last being `maximum_text_size` and so the worst a full-text
|
||||
row can present; `benches/corpus/` builds all of it from a fixed seed. This
|
||||
is the harness to extend when a hot path is in question, because it is the
|
||||
only one here that can A/B a single function.
|
||||
- `QSB_SNIPPET_PERF=1 cargo test --release -p quicksearch-core --test
|
||||
snippet_perf -- --nocapture`: snippet pipeline benchmark.
|
||||
- `QSB_SEARCH_PERF=1 cargo test --release -p quicksearch-core --test
|
||||
|
|
@ -632,6 +685,16 @@ microseconds regardless of row count.
|
|||
distinguishable from live data. It reads another process's `/proc`, so it
|
||||
measures a build made without knowing it would be measured.
|
||||
`indexprobe` and `walkprobe` answer "how fast" rather than "how much".
|
||||
|
||||
All of these read the resident set, and none of them can see allocator
|
||||
*churn*: a buffer allocated and freed within one loop iteration never moves
|
||||
RSS, so a search whose peak is a flat 14 MiB can still be pushing tens of
|
||||
gigabytes a query through `malloc`. Both search-side regressions found so far
|
||||
were invisible to every probe listed above and showed up only under an
|
||||
interposed `malloc` that counted calls and bytes. Until a probe here reports
|
||||
allocation counts, measure that separately before concluding a path is cheap,
|
||||
and measure the GUI rather than a one-shot `quicksearch-cli` run — a
|
||||
short-lived process cannot show what a typing session retains.
|
||||
- `.forgejo/workflows/ci.yml`: builds both platforms on every push to `master`
|
||||
and every pull request. To cut a release, bump `[workspace.package] version`
|
||||
in `Cargo.toml` and push the commit on a branch named `Release...`; CI runs
|
||||
|
|
|
|||
|
|
@ -62,6 +62,13 @@ ctrlc = "3.4"
|
|||
zstd = "0.13"
|
||||
globset = "0.4"
|
||||
regex = "1"
|
||||
# SIMD substring search for the full-text passes. `str::match_indices` uses
|
||||
# std's Two-Way searcher, which has no vector prefilter: measured against a
|
||||
# 256 KiB body (`benches/search.rs`, group `substring`) it runs 111 µs where
|
||||
# `memmem` runs 2.4 µs, and the full-text passes scan a body per candidate
|
||||
# row. Already in the lockfile transitively (regex, globset, chardetng), so
|
||||
# naming it directly compiles nothing new.
|
||||
memchr = "2"
|
||||
|
||||
# `nice()`, for dropping indexing threads to background scheduling priority.
|
||||
# Linux schedules per task, so it affects only the calling thread. Already in
|
||||
|
|
@ -91,3 +98,21 @@ windows-sys = { version = "0.52", features = [
|
|||
"Win32_System_Threading",
|
||||
"Win32_System_WindowsProgramming",
|
||||
] }
|
||||
|
||||
# Microbenchmark harness for `benches/`. Criterion is the more common choice
|
||||
# and was rejected: it pulls rayon (a global thread pool that never tears
|
||||
# down, the same reason `lopdf` is not named directly above) plus plotters and
|
||||
# clap, some forty crates. Divan's mandatory set is a handful and it spawns no
|
||||
# threads of its own.
|
||||
[dev-dependencies]
|
||||
divan = "0.1"
|
||||
|
||||
# `harness = false` on both: divan supplies its own `main` via `divan::main()`,
|
||||
# so libtest must not also link one in.
|
||||
[[bench]]
|
||||
name = "search"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "index"
|
||||
harness = false
|
||||
|
|
|
|||
188
crates/quicksearch-core/benches/corpus/mod.rs
Normal file
188
crates/quicksearch-core/benches/corpus/mod.rs
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
//! Shared fixtures for the `search` and `index` benchmarks.
|
||||
//!
|
||||
//! Everything here is deterministic — the same LCG `tests/search_perf.rs` and
|
||||
//! `examples/indexprobe.rs` use, seeded identically. A fixed corpus is what
|
||||
//! makes two runs comparable, so a number that moved is a real change rather
|
||||
//! than a different document.
|
||||
//!
|
||||
//! Corpora are built once per process and shared by every benchmark that
|
||||
//! wants them. Divan re-runs a benchmarked closure thousands of times; paying
|
||||
//! 256 KiB of text generation inside that loop would measure the generator.
|
||||
|
||||
// Both bench binaries compile the whole module but each uses only part of it.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
/// Deterministic pseudo-random word picker.
|
||||
pub struct Lcg(u64);
|
||||
|
||||
impl Lcg {
|
||||
pub fn new(seed: u64) -> Lcg {
|
||||
Lcg(seed)
|
||||
}
|
||||
|
||||
pub fn next(&mut self) -> u64 {
|
||||
self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1);
|
||||
self.0 >> 33
|
||||
}
|
||||
}
|
||||
|
||||
/// Filler vocabulary. Deliberately excludes [`NEEDLE`] and every prefix of it
|
||||
/// past two characters, so the only occurrences in a document are the ones
|
||||
/// planted on purpose and a "zero hits" corpus really has zero.
|
||||
const WORDS: &[&str] = &[
|
||||
"alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", "iota", "kappa",
|
||||
"lambda", "sigma", "report", "summary", "meeting", "invoice", "contract", "budget", "revenue",
|
||||
"quarter", "planning", "review", "draft", "final", "notes", "appendix", "figure",
|
||||
];
|
||||
|
||||
/// The term every search benchmark looks for. Nine bytes, no overlap with
|
||||
/// [`WORDS`], and long enough to clear the trigram floor the full-text pass
|
||||
/// applies.
|
||||
pub const NEEDLE: &str = "quartzite";
|
||||
|
||||
/// The three document sizes every size-swept benchmark uses, in bytes.
|
||||
///
|
||||
/// 1 KiB is a short note, 16 KiB a typical source file or README, and 256 KiB
|
||||
/// is `maximum_text_size` — the ceiling `config.processing` lets into the
|
||||
/// index, and so the worst case a full-text pass has to survive per row.
|
||||
pub const SIZES: [usize; 3] = [1 << 10, 16 << 10, 256 << 10];
|
||||
|
||||
/// Occurrence counts to sweep. Zero is the important one: a full-text pass
|
||||
/// verifies far more candidate rows than it accepts, because the trigram
|
||||
/// index matches on character triples rather than the whole term.
|
||||
pub const HITS: [usize; 3] = [0, 4, 64];
|
||||
|
||||
/// Word text of about `size` bytes with [`NEEDLE`] planted `hits` times at
|
||||
/// even spacing.
|
||||
///
|
||||
/// Even spacing matters: it means a count has to scan to the end of the
|
||||
/// document, and a snippet window never contains every match. Both are what
|
||||
/// the cascade actually does.
|
||||
pub fn document(size: usize, hits: usize) -> String {
|
||||
let mut lcg = Lcg::new(0x5eed);
|
||||
let mut out = String::with_capacity(size + 16);
|
||||
let stride = if hits == 0 { usize::MAX } else { size / hits };
|
||||
let mut next_plant = stride;
|
||||
while out.len() < size {
|
||||
if out.len() >= next_plant {
|
||||
out.push_str(NEEDLE);
|
||||
out.push(' ');
|
||||
next_plant = next_plant.saturating_add(stride);
|
||||
continue;
|
||||
}
|
||||
out.push_str(WORDS[lcg.next() as usize % WORDS.len()]);
|
||||
out.push(' ');
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// [`document`], but with the planted term capitalised so a case-sensitive
|
||||
/// scan misses and only the folded one hits.
|
||||
///
|
||||
/// This is the stage-6 row and the tier-4 row — the case the cascade pays a
|
||||
/// fold for, and the one a corpus of all-lowercase text would never produce.
|
||||
pub fn document_mixed_case(size: usize, hits: usize) -> String {
|
||||
let mut needle = NEEDLE.to_string();
|
||||
needle.replace_range(0..1, &NEEDLE[0..1].to_uppercase());
|
||||
document(size, hits).replace(NEEDLE, &needle)
|
||||
}
|
||||
|
||||
type Corpus = HashMap<(usize, usize), String>;
|
||||
|
||||
fn build(f: fn(usize, usize) -> String) -> Corpus {
|
||||
let mut m = HashMap::new();
|
||||
for size in SIZES {
|
||||
for hits in HITS {
|
||||
m.insert((size, hits), f(size, hits));
|
||||
}
|
||||
}
|
||||
m
|
||||
}
|
||||
|
||||
static LOWER: LazyLock<Corpus> = LazyLock::new(|| build(document));
|
||||
static MIXED: LazyLock<Corpus> = LazyLock::new(|| build(document_mixed_case));
|
||||
static FOLDED: LazyLock<Corpus> = LazyLock::new(|| {
|
||||
MIXED
|
||||
.iter()
|
||||
.map(|(k, v)| (*k, v.to_ascii_lowercase()))
|
||||
.collect()
|
||||
});
|
||||
static BLOBS: LazyLock<HashMap<(usize, usize), Vec<u8>>> = LazyLock::new(|| {
|
||||
LOWER
|
||||
.iter()
|
||||
.map(|(k, v)| (*k, zstd::encode_all(v.as_bytes(), 3).expect("encode")))
|
||||
.collect()
|
||||
});
|
||||
|
||||
/// An all-lowercase document: a case-sensitive scan finds every planted term.
|
||||
pub fn text(size: usize, hits: usize) -> &'static str {
|
||||
&LOWER[&(size, hits)]
|
||||
}
|
||||
|
||||
/// A document whose planted terms are capitalised: case-sensitive misses,
|
||||
/// folded hits.
|
||||
pub fn text_mixed(size: usize, hits: usize) -> &'static str {
|
||||
&MIXED[&(size, hits)]
|
||||
}
|
||||
|
||||
/// [`text_mixed`] pre-folded, for the benchmarks that measure a search
|
||||
/// against a haystack the caller already lowered.
|
||||
pub fn text_folded(size: usize, hits: usize) -> &'static str {
|
||||
&FOLDED[&(size, hits)]
|
||||
}
|
||||
|
||||
/// [`text`] as stored: zstd level 3, exactly what `db/repo.rs` writes.
|
||||
pub fn blob(size: usize, hits: usize) -> &'static [u8] {
|
||||
&BLOBS[&(size, hits)]
|
||||
}
|
||||
|
||||
/// Realistic file names and full paths, for the benchmarks that model the
|
||||
/// filename pass rather than the full-text one.
|
||||
///
|
||||
/// The name pass scans the whole `files` table, so what matters is the
|
||||
/// per-row cost on a *miss* — most rows match the SQL `LIKE` on some
|
||||
/// directory component and then fail every name tier.
|
||||
pub struct Row {
|
||||
pub name: String,
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
static ROWS: LazyLock<Vec<Row>> = LazyLock::new(|| {
|
||||
let mut lcg = Lcg::new(0xd00d);
|
||||
(0..2000)
|
||||
.map(|i| {
|
||||
let w1 = WORDS[lcg.next() as usize % WORDS.len()];
|
||||
let w2 = WORDS[lcg.next() as usize % WORDS.len()];
|
||||
let name = format!("{}-{}-{:05}.txt", w1, w2, i);
|
||||
// Mixed case in the directory portion, so the folded tiers are
|
||||
// the ones that resolve — the common shape on real trees.
|
||||
let path = format!("/home/user/Documents/Quartzite/{:03}/{}", i % 40, name);
|
||||
Row { name, path }
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
|
||||
pub fn rows() -> &'static [Row] {
|
||||
&ROWS
|
||||
}
|
||||
|
||||
/// The head of a plain-text file, as the walk reads it: `hash_length`
|
||||
/// (8 KiB) bytes or the whole file, whichever is smaller.
|
||||
pub fn text_head() -> &'static [u8] {
|
||||
static HEAD: LazyLock<Vec<u8>> =
|
||||
LazyLock::new(|| document(8 << 10, 2).into_bytes()[..8 << 10].to_vec());
|
||||
&HEAD
|
||||
}
|
||||
|
||||
/// A binary head that no extractor claims — the control group for the MIME
|
||||
/// sniff, which has to reject it by scanning.
|
||||
pub fn binary_head() -> &'static [u8] {
|
||||
static HEAD: LazyLock<Vec<u8>> = LazyLock::new(|| {
|
||||
let mut lcg = Lcg::new(0xbeef);
|
||||
(0..8 << 10).map(|_| (lcg.next() & 0xff) as u8).collect()
|
||||
});
|
||||
&HEAD
|
||||
}
|
||||
190
crates/quicksearch-core/benches/index.rs
Normal file
190
crates/quicksearch-core/benches/index.rs
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
//! Indexing-path microbenchmarks.
|
||||
//!
|
||||
//! Same convention as `benches/search.rs`: each group pairs two ways of doing
|
||||
//! the same work, measured together, so a choice is justified rather than
|
||||
//! asserted — and a losing arm records something already tried.
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo bench -p quicksearch-core --bench index
|
||||
//! ```
|
||||
|
||||
mod corpus;
|
||||
|
||||
use divan::Bencher;
|
||||
use quicksearch_core::{mime, textenc, walk};
|
||||
|
||||
fn main() {
|
||||
divan::main();
|
||||
}
|
||||
|
||||
/// Compressing an extracted document, which `repo::set_content_done` used to
|
||||
/// do itself — on the single writer thread, inside the transaction, and so
|
||||
/// inside the `conn_mutex` hold `store_extracted` takes across a whole chunk.
|
||||
///
|
||||
/// `encode_all` allocates and tears down a fresh `ZSTD_CCtx` — window, hash
|
||||
/// and chain tables — per document, and for the small documents that dominate
|
||||
/// a real tree that setup costs more than the compression. `batch_*` below
|
||||
/// measures a whole chunk of it, which is what `compress_bodies` now does
|
||||
/// before taking the lock.
|
||||
mod zstd_encode {
|
||||
use super::*;
|
||||
|
||||
/// Level 3, matching `db/repo.rs`'s `ZSTD_LEVEL`. Not a variable here:
|
||||
/// the level is argued in place and this measures the machinery around
|
||||
/// it, not the level.
|
||||
const LEVEL: i32 = 3;
|
||||
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn encode_all(bencher: Bencher, size: usize) {
|
||||
let text = corpus::text(size, 4).as_bytes();
|
||||
bencher.bench(|| zstd::encode_all(divan::black_box(text), LEVEL).unwrap());
|
||||
}
|
||||
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn bulk_reused(bencher: Bencher, size: usize) {
|
||||
let text = corpus::text(size, 4).as_bytes();
|
||||
let mut enc = zstd::bulk::Compressor::new(LEVEL).unwrap();
|
||||
bencher.bench_local(move || enc.compress(divan::black_box(text)).unwrap());
|
||||
}
|
||||
|
||||
/// One writer chunk: `processing.batch_size` documents of the size most
|
||||
/// documents are.
|
||||
///
|
||||
/// The `encode_all` figure is how much pure CPU used to sit inside the
|
||||
/// `conn_mutex` hold. That lock serializes the indexer against itself —
|
||||
/// not against search, which holds its own connection and reads through
|
||||
/// WAL — so this is a contention figure, not a throughput one. An
|
||||
/// end-to-end cold index of an 80 MiB tree does not move measurably: FTS5
|
||||
/// trigram tokenization dominates it, and all compression together is
|
||||
/// under 1% of the run.
|
||||
const BATCH: usize = 500;
|
||||
|
||||
#[divan::bench]
|
||||
fn batch_encode_all(bencher: Bencher) {
|
||||
let text = corpus::text(1 << 10, 4).as_bytes();
|
||||
bencher.bench(|| {
|
||||
(0..BATCH)
|
||||
.map(|_| {
|
||||
zstd::encode_all(divan::black_box(text), LEVEL)
|
||||
.unwrap()
|
||||
.len()
|
||||
})
|
||||
.sum::<usize>()
|
||||
});
|
||||
}
|
||||
|
||||
#[divan::bench]
|
||||
fn batch_bulk_reused(bencher: Bencher) {
|
||||
let text = corpus::text(1 << 10, 4).as_bytes();
|
||||
let mut enc = zstd::bulk::Compressor::new(LEVEL).unwrap();
|
||||
bencher.bench_local(move || {
|
||||
(0..BATCH)
|
||||
.map(|_| enc.compress(divan::black_box(text)).unwrap().len())
|
||||
.sum::<usize>()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// What a small text file costs between the MIME sniff and the decode.
|
||||
///
|
||||
/// The walk reads an 8 KiB head, sniffs it (`looks_like_text` → `classify`, a
|
||||
/// control-byte scan plus a UTF-8 validation), and then — for a file the head
|
||||
/// covers entirely — decodes it inline (`decode_text` → `classify` again, on
|
||||
/// the same bytes). So `classify` runs twice on identical bytes, and
|
||||
/// `sniff_only` says the second pass is about half of `sniff_then_decode`.
|
||||
///
|
||||
/// Left alone deliberately. The two calls pass different `truncated` flags and
|
||||
/// genuinely disagree on a file ending mid-multibyte-sequence, so sharing a
|
||||
/// verdict means threading `TextClass` through `mime.rs`, the `Extractor`
|
||||
/// trait and `plaintext.rs`. Against a cold `indexprobe` run that whole
|
||||
/// pipeline is under 1% of wall time — the redundancy is real and still not
|
||||
/// worth the coupling. Re-measure here before deciding otherwise.
|
||||
mod text_pipeline {
|
||||
use super::*;
|
||||
use std::path::Path;
|
||||
|
||||
#[divan::bench]
|
||||
fn sniff_then_decode(bencher: Bencher) {
|
||||
let head = corpus::text_head();
|
||||
let path = Path::new("/tmp/bench/notes.txt");
|
||||
bencher.bench(|| {
|
||||
let looks = textenc::looks_like_text(divan::black_box(head));
|
||||
let text = textenc::decode_text(head.to_vec(), path).unwrap();
|
||||
(looks, text.len())
|
||||
});
|
||||
}
|
||||
|
||||
#[divan::bench]
|
||||
fn sniff_only(bencher: Bencher) {
|
||||
let head = corpus::text_head();
|
||||
bencher.bench(|| textenc::looks_like_text(divan::black_box(head)));
|
||||
}
|
||||
|
||||
/// Includes the `head.to_vec()` at `extract/plaintext.rs:113` — a full
|
||||
/// copy of the head made only because `decode_text` takes ownership.
|
||||
#[divan::bench]
|
||||
fn decode_only(bencher: Bencher) {
|
||||
let head = corpus::text_head();
|
||||
let path = Path::new("/tmp/bench/notes.txt");
|
||||
bencher.bench(|| textenc::decode_text(divan::black_box(head).to_vec(), path));
|
||||
}
|
||||
}
|
||||
|
||||
/// The MIME sniff itself, once per new or changed file.
|
||||
///
|
||||
/// It carries several `to_ascii_lowercase` allocations for extension and MIME
|
||||
/// comparisons that `eq_ignore_ascii_case` would do without allocating — and
|
||||
/// removing them is not worth doing: the whole sniff is ~180 ns on a text head,
|
||||
/// against ~2.2 µs for the `classify` next to it. This group exists to keep
|
||||
/// that ratio visible.
|
||||
mod mime_sniff {
|
||||
use super::*;
|
||||
use std::path::Path;
|
||||
|
||||
#[divan::bench]
|
||||
fn text_head(bencher: Bencher) {
|
||||
let head = corpus::text_head();
|
||||
let path = Path::new("/tmp/bench/notes.txt");
|
||||
bencher.bench(|| mime::guess_mime_from_head(path, divan::black_box(head)));
|
||||
}
|
||||
|
||||
#[divan::bench]
|
||||
fn binary_head(bencher: Bencher) {
|
||||
let head = corpus::binary_head();
|
||||
let path = Path::new("/tmp/bench/blob.bin");
|
||||
bencher.bench(|| mime::guess_mime_from_head(path, divan::black_box(head)));
|
||||
}
|
||||
|
||||
/// No extension to go on, so the sniff falls all the way through to the
|
||||
/// magic-byte scan and the text classifier.
|
||||
#[divan::bench]
|
||||
fn no_extension(bencher: Bencher) {
|
||||
let head = corpus::text_head();
|
||||
let path = Path::new("/tmp/bench/LICENSE");
|
||||
bencher.bench(|| mime::guess_mime_from_head(path, divan::black_box(head)));
|
||||
}
|
||||
}
|
||||
|
||||
/// A SHA-256 over the path string, per file, on every run — including the
|
||||
/// unchanged steady state, where it is the only compute a file costs beyond
|
||||
/// its `statx`.
|
||||
///
|
||||
/// Reference measurement only. The plan does not propose changing it: its
|
||||
/// rationale is collision resistance against adversarial filenames on shared
|
||||
/// volumes, and this is here to confirm the cost is small enough that the
|
||||
/// argument stands unchallenged.
|
||||
mod path_digest {
|
||||
use super::*;
|
||||
|
||||
#[divan::bench]
|
||||
fn per_path(bencher: Bencher) {
|
||||
let rows = corpus::rows();
|
||||
bencher.bench(|| {
|
||||
let mut acc = 0u128;
|
||||
for row in divan::black_box(rows) {
|
||||
acc ^= walk::path_digest(&row.path);
|
||||
}
|
||||
acc
|
||||
});
|
||||
}
|
||||
}
|
||||
363
crates/quicksearch-core/benches/search.rs
Normal file
363
crates/quicksearch-core/benches/search.rs
Normal file
|
|
@ -0,0 +1,363 @@
|
|||
//! Search-path microbenchmarks.
|
||||
//!
|
||||
//! Every group pairs two ways of doing the same work in one run, so the delta
|
||||
//! is a measurement rather than an estimate. Some pairs justify a choice the
|
||||
//! code has already made; others record something tried and rejected. Both are
|
||||
//! worth keeping — a losing arm is the cheapest documentation there is that an
|
||||
//! obvious-looking idea was measured and did not pay.
|
||||
//!
|
||||
//! Run with:
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo bench -p quicksearch-core --bench search
|
||||
//! ```
|
||||
//!
|
||||
//! Sizes come from `corpus::SIZES` — 1 KiB, 16 KiB and 256 KiB, the last
|
||||
//! being `maximum_text_size`, the largest document the index will hold and so
|
||||
//! the worst case a full-text row can present.
|
||||
|
||||
mod corpus;
|
||||
|
||||
use divan::Bencher;
|
||||
use quicksearch_core::query::pattern::{TermPart, TermPattern};
|
||||
use quicksearch_core::search::fuzzy::Bitap;
|
||||
use quicksearch_core::snippet;
|
||||
|
||||
fn main() {
|
||||
divan::main();
|
||||
}
|
||||
|
||||
fn literal(term: &str) -> TermPattern {
|
||||
TermPattern::build(&[TermPart {
|
||||
text: term.to_string(),
|
||||
glob: false,
|
||||
}])
|
||||
.expect("literal patterns always compile")
|
||||
}
|
||||
|
||||
/// Decompressing the stored document body — the first thing every full-text
|
||||
/// row does, at `search/cascade/passes.rs:241`, `:425` and `:528`.
|
||||
///
|
||||
/// `decode_all` builds a fresh `ZSTD_DCtx` and a ~131 KB `BufReader` per call,
|
||||
/// then grows an unsized `Vec` as it goes; the other arm reuses one context and
|
||||
/// sizes the output up front. Measured at 8.4/16.0/102 µs against
|
||||
/// 1.6/8.0/89 µs — a 4.4x gap at 1 KiB, which is the size most documents are.
|
||||
/// This is why `DocDecoder` exists.
|
||||
mod zstd_decode {
|
||||
use super::*;
|
||||
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn decode_all(bencher: Bencher, size: usize) {
|
||||
let blob = corpus::blob(size, 4);
|
||||
bencher.bench(|| zstd::decode_all(divan::black_box(blob)).unwrap());
|
||||
}
|
||||
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn bulk_reused(bencher: Bencher, size: usize) {
|
||||
let blob = corpus::blob(size, 4);
|
||||
let capacity = corpus::text(size, 4).len();
|
||||
let mut dec = zstd::bulk::Decompressor::new().unwrap();
|
||||
bencher.bench_local(move || dec.decompress(divan::black_box(blob), capacity).unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
/// Turning decompressed bytes into a `&str`.
|
||||
///
|
||||
/// `from_utf8_lossy(..).into_owned()` copies the whole document even when the
|
||||
/// bytes are already valid UTF-8 — and they always are, since
|
||||
/// `textenc::decode_text` is the only thing that writes them. It is also far
|
||||
/// slower than it looks: its validation is a scanning loop, where
|
||||
/// `String::from_utf8` uses the vectorized one and *moves* the buffer it
|
||||
/// validates. 230/6360/52200 ns against 15/227/3200 ns, a 16-28x gap that is
|
||||
/// mostly validation rather than the copy. `DocDecoder` borrows instead.
|
||||
mod utf8 {
|
||||
use super::*;
|
||||
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn lossy_into_owned(bencher: Bencher, size: usize) {
|
||||
let raw = corpus::text(size, 4).as_bytes();
|
||||
bencher.bench(|| String::from_utf8_lossy(divan::black_box(raw)).into_owned());
|
||||
}
|
||||
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn lossy_borrowed(bencher: Bencher, size: usize) {
|
||||
let raw = corpus::text(size, 4).as_bytes();
|
||||
bencher.bench(|| {
|
||||
let cow = String::from_utf8_lossy(divan::black_box(raw));
|
||||
cow.len()
|
||||
});
|
||||
}
|
||||
|
||||
/// The decompressor hands back an owned `Vec<u8>` that nothing else
|
||||
/// references, so `String::from_utf8` can validate and *move* it rather
|
||||
/// than validate and copy. Falling back to `from_utf8_lossy` on error
|
||||
/// keeps the current behaviour for a corrupt row exactly.
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn from_utf8_move(bencher: Bencher, size: usize) {
|
||||
let raw = corpus::text(size, 4).as_bytes();
|
||||
bencher
|
||||
.with_inputs(|| raw.to_vec())
|
||||
.bench_values(|owned| match String::from_utf8(owned) {
|
||||
Ok(s) => s,
|
||||
Err(e) => String::from_utf8_lossy(e.as_bytes()).into_owned(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// ASCII-folding the document, which every full-text row needs for the
|
||||
/// case-insensitive count and the snippet.
|
||||
///
|
||||
/// A result worth keeping visible: folding into a reused buffer is *not*
|
||||
/// faster. `to_ascii_lowercase` allocates and folds in one pass, where
|
||||
/// clear + `push_str` + `make_ascii_lowercase` walks the bytes twice, and at
|
||||
/// 256 KiB the reused buffer measures slightly behind. `fold_into` is chosen
|
||||
/// for what it does to the allocator, not to the clock — do not "optimize" the
|
||||
/// other direction on the assumption that removing an allocation must win.
|
||||
mod fold {
|
||||
use super::*;
|
||||
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn to_ascii_lowercase(bencher: Bencher, size: usize) {
|
||||
let text = corpus::text_mixed(size, 4);
|
||||
bencher.bench(|| divan::black_box(text).to_ascii_lowercase());
|
||||
}
|
||||
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn into_reused_buffer(bencher: Bencher, size: usize) {
|
||||
let text = corpus::text_mixed(size, 4);
|
||||
let mut buf = String::new();
|
||||
bencher.bench_local(move || {
|
||||
buf.clear();
|
||||
buf.push_str(divan::black_box(text));
|
||||
// SAFETY-free equivalent of the in-place fold: `make_ascii_lowercase`
|
||||
// is byte-length preserving, which is the same invariant the
|
||||
// cascade already relies on for folded offsets.
|
||||
buf.make_ascii_lowercase();
|
||||
buf.len()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Substring search over a document body: `str::match_indices` (std's Two-Way
|
||||
/// searcher) against `memchr::memmem` (Two-Way plus a SIMD prefilter).
|
||||
///
|
||||
/// The miss case matters most. The trigram index matches on character triples,
|
||||
/// so a full-text pass verifies far more rows than it accepts, and a miss scans
|
||||
/// the whole document before giving up. At 256 KiB that is 111 µs against
|
||||
/// 2.4 µs — the measurement `snippet.rs` uses `memmem` for. `match_indices`
|
||||
/// stays here as the regression guard: if these two ever converge, the SIMD
|
||||
/// path has stopped being selected.
|
||||
mod substring {
|
||||
use super::*;
|
||||
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn match_indices_miss(bencher: Bencher, size: usize) {
|
||||
let text = corpus::text(size, 0);
|
||||
bencher.bench(|| divan::black_box(text).match_indices(corpus::NEEDLE).count());
|
||||
}
|
||||
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn memmem_miss(bencher: Bencher, size: usize) {
|
||||
let text = corpus::text(size, 0).as_bytes();
|
||||
let finder = memchr::memmem::Finder::new(corpus::NEEDLE);
|
||||
bencher.bench(|| finder.find_iter(divan::black_box(text)).count());
|
||||
}
|
||||
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn match_indices_hits(bencher: Bencher, size: usize) {
|
||||
let text = corpus::text(size, 64);
|
||||
bencher.bench(|| divan::black_box(text).match_indices(corpus::NEEDLE).count());
|
||||
}
|
||||
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn memmem_hits(bencher: Bencher, size: usize) {
|
||||
let text = corpus::text(size, 64).as_bytes();
|
||||
let finder = memchr::memmem::Finder::new(corpus::NEEDLE);
|
||||
bencher.bench(|| finder.find_iter(divan::black_box(text)).count());
|
||||
}
|
||||
|
||||
/// What `pass_fulltext` actually runs per row, through the real crate
|
||||
/// entry points: a case-sensitive count, then a folded count, then the
|
||||
/// snippet extraction. Three sweeps of the same document.
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn cascade_row_sweeps(bencher: Bencher, size: usize) {
|
||||
let pattern = literal(corpus::NEEDLE);
|
||||
let text = corpus::text_mixed(size, 4);
|
||||
let folded = corpus::text_folded(size, 4);
|
||||
let opts = snippet::Options { approx_chars: 600 };
|
||||
bencher.bench(|| {
|
||||
let a = pattern.count(divan::black_box(text), false);
|
||||
let b = pattern.count_folded(divan::black_box(folded));
|
||||
let s = snippet::extract_folded(text, folded, &[corpus::NEEDLE], &opts);
|
||||
(a, b, s.ranges.len())
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Snippet extraction against a pre-folded haystack, the third of those
|
||||
/// sweeps. Also carries a per-call `term.to_ascii_lowercase()` at
|
||||
/// `snippet.rs:81` for a needle the caller already holds folded.
|
||||
mod snippet_extract {
|
||||
use super::*;
|
||||
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn extract_folded(bencher: Bencher, size: usize) {
|
||||
let text = corpus::text_mixed(size, 64);
|
||||
let folded = corpus::text_folded(size, 64);
|
||||
let opts = snippet::Options { approx_chars: 600 };
|
||||
bencher.bench(|| {
|
||||
snippet::extract_folded(
|
||||
divan::black_box(text),
|
||||
divan::black_box(folded),
|
||||
&[corpus::NEEDLE],
|
||||
&opts,
|
||||
)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// The filename pass's per-row ladder, over 2000 realistic name/path rows.
|
||||
///
|
||||
/// `pass_filename` scans the whole `files` table — its `LIKE '%term%'`
|
||||
/// predicate can use no index — and tiers 4 and 10 both run a
|
||||
/// case-insensitive find, so a row matching on its directory portion pays
|
||||
/// twice.
|
||||
///
|
||||
/// The instructive part is that the two obvious fixes each make it *worse*
|
||||
/// alone: a reused fold buffer measures ~2x slower than folding into a fresh
|
||||
/// allocation, and a prebuilt `memmem::Finder` is slower than `str::find` on
|
||||
/// haystacks this short. Only together do they win, and only by ~1.2x. Short
|
||||
/// strings do not behave like document bodies; measure them separately.
|
||||
mod filename_ladder {
|
||||
use super::*;
|
||||
|
||||
#[divan::bench]
|
||||
fn find_first_ci_current(bencher: Bencher) {
|
||||
let pattern = literal("quartzite");
|
||||
let rows = corpus::rows();
|
||||
bencher.bench(|| {
|
||||
let mut found = 0usize;
|
||||
for row in divan::black_box(rows) {
|
||||
if pattern.find_first(&row.name, true).is_some()
|
||||
|| pattern.find_first(&row.path, true).is_some()
|
||||
{
|
||||
found += 1;
|
||||
}
|
||||
}
|
||||
found
|
||||
});
|
||||
}
|
||||
|
||||
#[divan::bench]
|
||||
fn find_first_ci_scratch(bencher: Bencher) {
|
||||
let pattern = literal("quartzite");
|
||||
let rows = corpus::rows();
|
||||
let mut scratch = String::new();
|
||||
bencher.bench_local(move || {
|
||||
let mut found = 0usize;
|
||||
for row in divan::black_box(rows) {
|
||||
scratch.clear();
|
||||
scratch.push_str(&row.name);
|
||||
scratch.make_ascii_lowercase();
|
||||
if pattern.find_first_folded(&scratch).is_some() {
|
||||
found += 1;
|
||||
continue;
|
||||
}
|
||||
scratch.clear();
|
||||
scratch.push_str(&row.path);
|
||||
scratch.make_ascii_lowercase();
|
||||
if pattern.find_first_folded(&scratch).is_some() {
|
||||
found += 1;
|
||||
}
|
||||
}
|
||||
found
|
||||
});
|
||||
}
|
||||
|
||||
/// Fold as today, but search the folded copy with a `Finder` built once
|
||||
/// per query instead of `str::find`'s Two-Way. Isolates the searcher from
|
||||
/// the allocation: if this wins and `find_first_ci_scratch` does not, the
|
||||
/// fold was never the problem.
|
||||
#[divan::bench]
|
||||
fn find_first_ci_memmem(bencher: Bencher) {
|
||||
let finder = memchr::memmem::Finder::new("quartzite");
|
||||
let rows = corpus::rows();
|
||||
bencher.bench(|| {
|
||||
let mut found = 0usize;
|
||||
for row in divan::black_box(rows) {
|
||||
if finder
|
||||
.find(row.name.to_ascii_lowercase().as_bytes())
|
||||
.is_some()
|
||||
|| finder
|
||||
.find(row.path.to_ascii_lowercase().as_bytes())
|
||||
.is_some()
|
||||
{
|
||||
found += 1;
|
||||
}
|
||||
}
|
||||
found
|
||||
});
|
||||
}
|
||||
|
||||
/// Both at once: one reused fold buffer and a prebuilt `Finder`.
|
||||
#[divan::bench]
|
||||
fn find_first_ci_scratch_memmem(bencher: Bencher) {
|
||||
let finder = memchr::memmem::Finder::new("quartzite");
|
||||
let rows = corpus::rows();
|
||||
let mut scratch = String::new();
|
||||
bencher.bench_local(move || {
|
||||
let mut found = 0usize;
|
||||
for row in divan::black_box(rows) {
|
||||
scratch.clear();
|
||||
scratch.push_str(&row.name);
|
||||
scratch.make_ascii_lowercase();
|
||||
if finder.find(scratch.as_bytes()).is_some() {
|
||||
found += 1;
|
||||
continue;
|
||||
}
|
||||
scratch.clear();
|
||||
scratch.push_str(&row.path);
|
||||
scratch.make_ascii_lowercase();
|
||||
if finder.find(scratch.as_bytes()).is_some() {
|
||||
found += 1;
|
||||
}
|
||||
}
|
||||
found
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Bitap, the fuzzy passes' inner loop. Both fuzzy passes are whole-table
|
||||
/// scans, so this runs over every row in the index when fuzzy is on.
|
||||
///
|
||||
/// `step` still takes `&mut [u64]` rather than `&mut [u64; MAX_REGISTERS]`,
|
||||
/// so the register indices are bounds-checked and the trip count is opaque
|
||||
/// to the optimizer.
|
||||
mod bitap {
|
||||
use super::*;
|
||||
|
||||
#[divan::bench(args = corpus::SIZES)]
|
||||
fn count_and_first_k2(bencher: Bencher, size: usize) {
|
||||
let bitap = Bitap::new(corpus::NEEDLE.as_bytes(), 2).unwrap();
|
||||
let hay = corpus::text(size, 4).as_bytes();
|
||||
bencher.bench(|| bitap.count_and_first(divan::black_box(hay)));
|
||||
}
|
||||
|
||||
/// The filename pass's shape: many short haystacks rather than one long
|
||||
/// one, with the per-call 176-byte register memset amortized over very
|
||||
/// little work.
|
||||
#[divan::bench]
|
||||
fn best_distance_over_names_k2(bencher: Bencher) {
|
||||
let bitap = Bitap::new(b"quartzite", 2).unwrap();
|
||||
let rows = corpus::rows();
|
||||
bencher.bench(|| {
|
||||
let mut hits = 0usize;
|
||||
for row in divan::black_box(rows) {
|
||||
if bitap.best_distance_and_first(row.name.as_bytes()).is_some() {
|
||||
hits += 1;
|
||||
}
|
||||
}
|
||||
hits
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -251,7 +251,13 @@ fn run(mode: &str, tree: &Path, db: &Path) {
|
|||
assert!(done, "indexing did not finish within the timeout");
|
||||
service.stop_indexing().expect("stop");
|
||||
|
||||
let total = SMALL_TEXT + LARGE_TEXT + BINARY;
|
||||
// Count what was actually indexed rather than assuming `gen`'s tree.
|
||||
// The constants describe the tree this probe builds; pointing it at any
|
||||
// other one made the rate a fiction.
|
||||
let total = rusqlite::Connection::open(db)
|
||||
.ok()
|
||||
.and_then(|c| quicksearch_core::db::repo::row_count(&c).ok())
|
||||
.unwrap_or(0);
|
||||
eprintln!(
|
||||
"{}: {:?} ({:.0} files/sec over {} files)",
|
||||
mode,
|
||||
|
|
|
|||
|
|
@ -221,6 +221,7 @@ mod tests {
|
|||
use crate::db::open_or_recreate;
|
||||
use crate::db::repo::{insert_file, set_content_done, set_content_failed, NewFile};
|
||||
use crate::mime::FileType;
|
||||
use crate::testutil::zstd_of;
|
||||
|
||||
fn tmp_path() -> std::path::PathBuf {
|
||||
crate::testutil::scratch_dir("cli").join("index.sqlite")
|
||||
|
|
@ -248,7 +249,7 @@ mod tests {
|
|||
)
|
||||
.unwrap()
|
||||
.expect("unique path");
|
||||
set_content_done(&tx, a, "a.txt", "hello", &[], true).unwrap();
|
||||
set_content_done(&tx, a, "a.txt", "hello", &[], zstd_of("hello").as_deref()).unwrap();
|
||||
let b = insert_file(
|
||||
&tx,
|
||||
&NewFile {
|
||||
|
|
@ -427,7 +428,7 @@ mod tests {
|
|||
.unwrap()
|
||||
.expect("unique path");
|
||||
let prose = "the quick brown fox jumps over the lazy dog. ".repeat(500);
|
||||
set_content_done(&tx, id, "big.txt", &prose, &[], true).unwrap();
|
||||
set_content_done(&tx, id, "big.txt", &prose, &[], zstd_of(&prose).as_deref()).unwrap();
|
||||
tx.commit().unwrap();
|
||||
}
|
||||
drop(conn);
|
||||
|
|
|
|||
|
|
@ -104,6 +104,23 @@ pub struct IndexerState {
|
|||
/// A run's *own* reconciliation is not here — it reads as
|
||||
/// [`IndexingStatus::Preparing`] with a [`PrepStep::Reconciling`].
|
||||
pub reconcile: Option<ReconcileState>,
|
||||
/// What each configured root held when indexing last completed. Roots
|
||||
/// never indexed to completion are absent rather than zero.
|
||||
///
|
||||
/// `Arc` because `state()` is called more than once per frame and this
|
||||
/// changes only when a run ends.
|
||||
pub root_counts: Arc<Vec<RootCount>>,
|
||||
}
|
||||
|
||||
/// One configured root's stored figures, keyed the way the caller spells it.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RootCount {
|
||||
/// The root exactly as `paths.indexing_paths` gives it, so a frontend can
|
||||
/// match it against the string it already draws. The `schema_info` key
|
||||
/// behind it is the canonicalized spelling, so re-spelling a root in the
|
||||
/// config keeps its figures.
|
||||
pub root: String,
|
||||
pub counts: db::repo::RootCounts,
|
||||
}
|
||||
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
|
|
@ -161,6 +178,7 @@ struct Shared {
|
|||
queued_events: usize,
|
||||
watcher: WatcherStatus,
|
||||
reconcile: Option<ReconcileState>,
|
||||
root_counts: Arc<Vec<RootCount>>,
|
||||
}
|
||||
|
||||
impl IndexCoordinator {
|
||||
|
|
@ -192,6 +210,7 @@ impl IndexCoordinator {
|
|||
queued_events: 0,
|
||||
watcher: WatcherStatus::Off,
|
||||
reconcile: None,
|
||||
root_counts: Arc::new(Vec::new()),
|
||||
}));
|
||||
|
||||
let reconcile_stop = Arc::new(ReconcileStop::default());
|
||||
|
|
@ -251,6 +270,7 @@ impl IndexCoordinator {
|
|||
queued_events: shared.queued_events,
|
||||
watcher: shared.watcher.clone(),
|
||||
reconcile: shared.reconcile,
|
||||
root_counts: shared.root_counts.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -116,6 +116,9 @@ impl Inner {
|
|||
// restart is cheap and unconditional beats a diff here.
|
||||
self.start_watcher();
|
||||
}
|
||||
// The root list, its spellings, or the database behind it may
|
||||
// all have moved; re-pair them with what is stored.
|
||||
self.refresh_last_full_index();
|
||||
}
|
||||
CoordCmd::RebuildIndex => {
|
||||
let db = self.db_path();
|
||||
|
|
@ -127,6 +130,7 @@ impl Inner {
|
|||
if let Err(e) = self.indexing.delete_index_for_rebuild(&db) {
|
||||
crate::log_warn!("coordinator: rebuild: {}", e);
|
||||
}
|
||||
self.clear_root_counts();
|
||||
self.start_full_run();
|
||||
if self.mode != IndexMode::Auto {
|
||||
self.mode = IndexMode::ManualRunning;
|
||||
|
|
@ -148,6 +152,9 @@ impl Inner {
|
|||
// Zero, not `None`: nothing will rebuild this index, so no
|
||||
// later read corrects a stale figure.
|
||||
shared.files = Some(0);
|
||||
// Per root the empty list reads as "not yet indexed", which is
|
||||
// what every folder now is.
|
||||
shared.root_counts = Arc::new(Vec::new());
|
||||
drop(shared);
|
||||
self.files_at = None;
|
||||
}
|
||||
|
|
@ -717,7 +724,13 @@ impl Inner {
|
|||
}
|
||||
}
|
||||
|
||||
/// Re-read the stamp the last completed full run left behind.
|
||||
/// Re-read what the last completed full run left behind: its stamp, and
|
||||
/// the per-root figures the folder list shows.
|
||||
///
|
||||
/// Both off one connection because they are wanted at the same moments —
|
||||
/// startup, a run finishing, a config change. Neither is a scan: the stamp
|
||||
/// and each root's counts are single `schema_info` key lookups, the work of
|
||||
/// counting having been done by the run that stored them.
|
||||
///
|
||||
/// A failed open is *not* published as `None`: `periodic_due` reads `None`
|
||||
/// as "never indexed" and would start a fresh run every tick for as long
|
||||
|
|
@ -726,12 +739,45 @@ impl Inner {
|
|||
match db::open_existing(&self.db_path(), false) {
|
||||
Ok(conn) => {
|
||||
let last = db::repo::get_last_full_index(&conn);
|
||||
crate::lock_ok(&self.shared).last_full_index = last;
|
||||
let counts = self.read_root_counts(&conn);
|
||||
let mut shared = crate::lock_ok(&self.shared);
|
||||
shared.last_full_index = last;
|
||||
shared.root_counts = Arc::new(counts);
|
||||
}
|
||||
Err(e) => crate::log_warn!("coordinator: last-full-index unreadable: {}", e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Pair every configured root with its stored figures, keyed by the
|
||||
/// spelling the config uses so a frontend can match what it draws.
|
||||
///
|
||||
/// The `schema_info` keys are canonicalized, which is what makes writing
|
||||
/// `~/docs` where the config said `/home/me/docs` keep the figures — the
|
||||
/// same re-keying `indexing::resolved_root_workers` does in the other
|
||||
/// direction.
|
||||
fn read_root_counts(&self, conn: &Connection) -> Vec<RootCount> {
|
||||
self.config
|
||||
.paths
|
||||
.indexing_paths
|
||||
.iter()
|
||||
.zip(self.config.resolved_indexing_paths())
|
||||
.filter_map(|(raw, resolved)| {
|
||||
let root = crate::file_handling::normalize_root_string(&resolved.to_string_lossy());
|
||||
let counts = db::repo::get_root_counts(conn, &root)?;
|
||||
Some(RootCount {
|
||||
root: raw.clone(),
|
||||
counts,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Forget the published figures: the index behind them is gone, and
|
||||
/// nothing will correct them until a run rebuilds it.
|
||||
fn clear_root_counts(&self) {
|
||||
crate::lock_ok(&self.shared).root_counts = Arc::new(Vec::new());
|
||||
}
|
||||
|
||||
fn publish(&mut self) {
|
||||
let reconcile = match &self.pending_work {
|
||||
Some(cursor) => Some(ReconcileState::Running(cursor.progress())),
|
||||
|
|
|
|||
|
|
@ -166,13 +166,22 @@ pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option
|
|||
/// result rows can't render snippets). `properties` are stored both as a
|
||||
/// structured side-table (exact retrieval) and concatenated into the FTS
|
||||
/// `properties` column (MATCH).
|
||||
/// `text_zstd` is the already-compressed body for the `documents_text`
|
||||
/// sidecar, or `None` to write no sidecar at all (an empty body, or
|
||||
/// `store_text_for_snippets` off).
|
||||
///
|
||||
/// Compression is the caller's job, and deliberately so: it is the expensive
|
||||
/// half of a content write, and this runs inside the writer's transaction
|
||||
/// with the shared connection held. Callers on the indexing path compress a
|
||||
/// whole batch through one [`DocEncoder`] *before* taking the lock, so the
|
||||
/// transaction only binds finished blobs.
|
||||
pub fn set_content_done(
|
||||
tx: &Transaction<'_>,
|
||||
file_id: i64,
|
||||
name: &str,
|
||||
text: &str,
|
||||
properties: &[(String, String)],
|
||||
store_text: bool,
|
||||
text_zstd: Option<&[u8]>,
|
||||
) -> Result<(), String> {
|
||||
remove_content_for_id(tx, file_id)?;
|
||||
|
||||
|
|
@ -195,10 +204,8 @@ pub fn set_content_done(
|
|||
)?;
|
||||
|
||||
// No sidecar row for empty body text (e.g. an image whose extractor
|
||||
// returned only EXIF properties).
|
||||
if store_text && !text.is_empty() {
|
||||
let compressed = zstd::encode_all(text.as_bytes(), ZSTD_LEVEL)
|
||||
.map_err(|e| format!("zstd encode for file {}: {}", file_id, e))?;
|
||||
// returned only EXIF properties) — the caller passes `None` for that.
|
||||
if let Some(compressed) = text_zstd {
|
||||
exec(
|
||||
tx,
|
||||
"INSERT INTO documents_text(file_id, text_zstd, text_len) VALUES (?1, ?2, ?3)",
|
||||
|
|
@ -215,6 +222,45 @@ pub fn set_content_done(
|
|||
/// readers decompress far faster than writers compress.
|
||||
const ZSTD_LEVEL: i32 = 3;
|
||||
|
||||
/// Reusable compression context for the `documents_text` sidecar — the write
|
||||
/// side's mirror of the cascade's `DocDecoder`.
|
||||
///
|
||||
/// `zstd::encode_all` builds and tears down a `ZSTD_CCtx` — window, hash and
|
||||
/// chain tables — on every call, and the writer calls it once per extracted
|
||||
/// document. At 1 KiB, the size most documents actually are, that setup costs
|
||||
/// more than the compression: 16.4 µs against 3.4 µs for the same bytes
|
||||
/// through a context that already exists (`benches/index.rs`, group
|
||||
/// `zstd_encode`). One encoder per batch makes it a per-batch cost.
|
||||
pub struct DocEncoder(zstd::bulk::Compressor<'static>);
|
||||
|
||||
impl DocEncoder {
|
||||
pub fn new() -> Result<DocEncoder, String> {
|
||||
zstd::bulk::Compressor::new(ZSTD_LEVEL)
|
||||
.map(DocEncoder)
|
||||
.map_err(|e| format!("zstd encoder: {}", e))
|
||||
}
|
||||
|
||||
/// Compress `text` for [`set_content_done`]'s `text_zstd` argument.
|
||||
pub fn encode(&mut self, text: &str) -> Result<Vec<u8>, String> {
|
||||
self.0
|
||||
.compress(text.as_bytes())
|
||||
.map_err(|e| format!("zstd encode: {}", e))
|
||||
}
|
||||
}
|
||||
|
||||
/// Compress one body, for the writers that handle a single row.
|
||||
///
|
||||
/// The batch writers reuse one [`DocEncoder`] across a chunk and run it
|
||||
/// outside the connection lock. The single-row paths — the watcher, and a
|
||||
/// walk-time inline body — write one row per transaction, so there is no
|
||||
/// batch to amortize a context over and this builds one for the document.
|
||||
pub fn encode_one(text: &str, store_text: bool) -> Result<Option<Vec<u8>>, String> {
|
||||
if !store_text || text.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
DocEncoder::new()?.encode(text).map(Some)
|
||||
}
|
||||
|
||||
/// Mark a file's content extraction as failed. Keeps the basic row in place.
|
||||
pub fn set_content_failed(tx: &Transaction<'_>, file_id: i64, reason: &str) -> Result<(), String> {
|
||||
let now = crate::log::now_unix() as i64;
|
||||
|
|
@ -442,6 +488,44 @@ pub fn row_count(conn: &Connection) -> Result<usize, String> {
|
|||
.map_err(|e| format!("count indexed files: {}", e))
|
||||
}
|
||||
|
||||
/// What one root holds: rows under it, and how many of those are searchable
|
||||
/// by content.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct RootCounts {
|
||||
pub files: i64,
|
||||
/// Rows carrying a `searchabletext` entry.
|
||||
pub fts: i64,
|
||||
}
|
||||
|
||||
/// Count the rows in the half-open path range `[lo, hi)` and, in the same
|
||||
/// pass, how many of them have a full-text row.
|
||||
///
|
||||
/// `content_state = STATE_DONE` *is* "has a `searchabletext` row":
|
||||
/// [`set_content_done`] holds the only insert into that table and is what
|
||||
/// writes the state, and [`remove_content_for_id`] clears the two together.
|
||||
/// Asking `files` is what makes both figures one statement — the FTS table is
|
||||
/// contentless and keyed by `rowid`, so it has no path to range-scan on.
|
||||
///
|
||||
/// One statement, but not a cheap one: `content_state` is not carried by the
|
||||
/// `UNIQUE(files.path)` index the range seeks on, so every row in the range is
|
||||
/// fetched. Call it where a run has just read those rows anyway, not on a
|
||||
/// cadence.
|
||||
pub fn count_root(conn: &Connection, lo: &str, hi: &str) -> Result<RootCounts, String> {
|
||||
conn.prepare_cached(
|
||||
"SELECT COUNT(*), COALESCE(SUM(content_state = ?3), 0) FROM files
|
||||
WHERE path >= ?1 AND path < ?2",
|
||||
)
|
||||
.and_then(|mut stmt| {
|
||||
stmt.query_row(params![lo, hi, STATE_DONE], |r| {
|
||||
Ok(RootCounts {
|
||||
files: r.get(0)?,
|
||||
fts: r.get(1)?,
|
||||
})
|
||||
})
|
||||
})
|
||||
.map_err(|e| format!("count root {}: {}", lo, e))
|
||||
}
|
||||
|
||||
/// One page of rows whose path is `> after` and `< hi`, in path order.
|
||||
///
|
||||
/// Keyset on `path`: every page is an index walk with no sort step, and a row
|
||||
|
|
@ -554,14 +638,24 @@ pub fn paths_in_dir(conn: &Connection, parent: &str) -> Result<Vec<String>, Stri
|
|||
pub fn remove_content_for_id(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> {
|
||||
// `contentless_delete=1` on the FTS5 table makes this work without
|
||||
// re-supplying the old column values (it tombstones the rowid).
|
||||
for (table, key) in [
|
||||
("searchabletext", "rowid"),
|
||||
("documents_text", "file_id"),
|
||||
("properties", "file_id"),
|
||||
//
|
||||
// Spelled out rather than built from a (table, key) table: this runs for
|
||||
// every extracted document and every changed file, and `format!`ing three
|
||||
// constant strings per call also handed `prepare_cached` three freshly
|
||||
// allocated keys to hash.
|
||||
for (what, sql) in [
|
||||
(
|
||||
"searchabletext",
|
||||
"DELETE FROM searchabletext WHERE rowid = ?1",
|
||||
),
|
||||
(
|
||||
"documents_text",
|
||||
"DELETE FROM documents_text WHERE file_id = ?1",
|
||||
),
|
||||
("properties", "DELETE FROM properties WHERE file_id = ?1"),
|
||||
] {
|
||||
let sql = format!("DELETE FROM {} WHERE {} = ?1", table, key);
|
||||
exec(tx, &sql, params![file_id], || {
|
||||
format!("delete {} for {}", table, file_id)
|
||||
exec(tx, sql, params![file_id], || {
|
||||
format!("delete {} for {}", what, file_id)
|
||||
})?;
|
||||
}
|
||||
Ok(())
|
||||
|
|
@ -705,9 +799,24 @@ pub fn set_last_full_index(conn: &Connection, ts: u64) -> Result<(), String> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// The `schema_info` key prefixes holding per-root figures. Every one of them
|
||||
/// is swept by [`prune_root_stats`], so a new prefix belongs in this list or a
|
||||
/// de-configured root leaves it behind forever.
|
||||
const ROOT_STAT_PREFIXES: [&str; 2] = ["walk_count:", "counts:"];
|
||||
|
||||
/// `schema_info` key holding one root's figure of the given kind.
|
||||
fn root_key(prefix: &str, root: &str) -> String {
|
||||
format!("{}{}", prefix, root)
|
||||
}
|
||||
|
||||
/// `schema_info` key holding one root's last known file count.
|
||||
fn walk_count_key(root: &str) -> String {
|
||||
format!("walk_count:{}", root)
|
||||
root_key(ROOT_STAT_PREFIXES[0], root)
|
||||
}
|
||||
|
||||
/// `schema_info` key holding one root's last completed run's [`RootCounts`].
|
||||
fn counts_key(root: &str) -> String {
|
||||
root_key(ROOT_STAT_PREFIXES[1], root)
|
||||
}
|
||||
|
||||
/// How many files the last clean walk of `root` reported — the progress bar's
|
||||
|
|
@ -741,22 +850,64 @@ pub fn set_root_walk_count(conn: &Connection, root: &str, n: usize) -> Result<()
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the stored counts of roots that are no longer configured, so a
|
||||
/// root removed and later re-added does not start from a stale count.
|
||||
pub fn prune_root_walk_counts(conn: &Connection, keep: &[String]) -> Result<(), String> {
|
||||
let keep: std::collections::HashSet<String> = keep.iter().map(|r| walk_count_key(r)).collect();
|
||||
/// What the last completed run counted under `root`, if one has finished
|
||||
/// since the root was configured. Absent — never indexed, cleared, or a
|
||||
/// value this build cannot parse — reads as `None`, like the walk count.
|
||||
pub fn get_root_counts(conn: &Connection, root: &str) -> Option<RootCounts> {
|
||||
let stored: String = conn
|
||||
.query_row(
|
||||
"SELECT value FROM schema_info WHERE key = ?1",
|
||||
params![counts_key(root)],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.optional()
|
||||
.ok()
|
||||
.flatten()?;
|
||||
let (files, fts) = stored.split_once(',')?;
|
||||
Some(RootCounts {
|
||||
files: files.parse().ok()?,
|
||||
fts: fts.parse().ok()?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Record what `root` holds, for the folder list to show once the run that
|
||||
/// counted it is over.
|
||||
///
|
||||
/// Written only at the end of a run that completed: a stopped one has counted
|
||||
/// part of a tree it was still changing, and the previous figure is closer to
|
||||
/// the truth than that.
|
||||
pub fn set_root_counts(conn: &Connection, root: &str, counts: RootCounts) -> Result<(), String> {
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO schema_info(key, value) VALUES (?1, ?2)",
|
||||
params![counts_key(root), format!("{},{}", counts.files, counts.fts)],
|
||||
)
|
||||
.map_err(|e| format!("write counts for {}: {}", root, e))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the stored figures of roots that are no longer configured, so a
|
||||
/// root removed and later re-added does not start from stale ones.
|
||||
pub fn prune_root_stats(conn: &Connection, keep: &[String]) -> Result<(), String> {
|
||||
let keep: std::collections::HashSet<String> = ROOT_STAT_PREFIXES
|
||||
.iter()
|
||||
.flat_map(|prefix| keep.iter().map(move |r| root_key(prefix, r)))
|
||||
.collect();
|
||||
// Filtered here rather than with a `LIKE` per prefix: `schema_info` holds
|
||||
// a handful of keys plus these, and a SQL pattern list would be a second
|
||||
// spelling of `ROOT_STAT_PREFIXES` to keep in step with the first.
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT key FROM schema_info WHERE key LIKE 'walk_count:%'")
|
||||
.map_err(|e| format!("read walk counts: {}", e))?;
|
||||
.prepare("SELECT key FROM schema_info")
|
||||
.map_err(|e| format!("read root stats: {}", e))?;
|
||||
let stored: Vec<String> = stmt
|
||||
.query_map([], |r| r.get::<_, String>(0))
|
||||
.map_err(|e| format!("read walk counts: {}", e))?
|
||||
.map_err(|e| format!("read root stats: {}", e))?
|
||||
.filter_map(|r| r.ok())
|
||||
.filter(|k| ROOT_STAT_PREFIXES.iter().any(|p| k.starts_with(p)))
|
||||
.collect();
|
||||
drop(stmt);
|
||||
for key in stored.iter().filter(|k| !keep.contains(*k)) {
|
||||
conn.execute("DELETE FROM schema_info WHERE key = ?1", params![key])
|
||||
.map_err(|e| format!("drop walk count {}: {}", key, e))?;
|
||||
.map_err(|e| format!("drop root stat {}: {}", key, e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ use std::sync::Arc;
|
|||
|
||||
use super::*;
|
||||
use crate::db::open_or_recreate;
|
||||
use crate::testutil::zstd_of;
|
||||
|
||||
fn tmp_path() -> std::path::PathBuf {
|
||||
crate::testutil::scratch_dir("repo").join("index.sqlite")
|
||||
|
|
@ -38,7 +39,7 @@ fn insert_update_delete_round_trip() {
|
|||
"a.txt",
|
||||
"hello world",
|
||||
&[("title".to_string(), "hi".to_string())],
|
||||
true,
|
||||
zstd_of("hello world").as_deref(),
|
||||
)
|
||||
.unwrap();
|
||||
tx.commit().unwrap();
|
||||
|
|
@ -136,7 +137,15 @@ fn update_writes_content_state_from_needs_content() {
|
|||
let id = {
|
||||
let tx = conn.transaction().unwrap();
|
||||
let id = insert_file(&tx, &row).unwrap().expect("unique path");
|
||||
set_content_done(&tx, id, "a.txt", "old text", &[], true).unwrap();
|
||||
set_content_done(
|
||||
&tx,
|
||||
id,
|
||||
"a.txt",
|
||||
"old text",
|
||||
&[],
|
||||
zstd_of("old text").as_deref(),
|
||||
)
|
||||
.unwrap();
|
||||
tx.commit().unwrap();
|
||||
id
|
||||
};
|
||||
|
|
@ -265,7 +274,15 @@ fn delete_subtree_clears_every_dependent_table() {
|
|||
)
|
||||
.unwrap()
|
||||
.expect("unique path");
|
||||
set_content_done(tx, id, name, "body text", &[("k".into(), "v".into())], true).unwrap();
|
||||
set_content_done(
|
||||
tx,
|
||||
id,
|
||||
name,
|
||||
"body text",
|
||||
&[("k".into(), "v".into())],
|
||||
zstd_of("body text").as_deref(),
|
||||
)
|
||||
.unwrap();
|
||||
id
|
||||
};
|
||||
|
||||
|
|
@ -351,7 +368,7 @@ fn seeded(conn: &mut Connection, paths: &[&str]) -> std::collections::HashMap<St
|
|||
name,
|
||||
"body text",
|
||||
&[("k".into(), "v".into())],
|
||||
true,
|
||||
zstd_of("body text").as_deref(),
|
||||
)
|
||||
.unwrap();
|
||||
ids.insert((*path).to_string(), id);
|
||||
|
|
@ -765,7 +782,7 @@ fn seed_rows(conn: &mut Connection, range: std::ops::Range<usize>) {
|
|||
&name,
|
||||
&"lorem ipsum dolor sit amet ".repeat(64),
|
||||
&[],
|
||||
true,
|
||||
zstd_of(&"lorem ipsum dolor sit amet ".repeat(64)).as_deref(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
|
@ -1038,3 +1055,181 @@ fn set_content_failed_writes_failed_table() {
|
|||
drop(conn);
|
||||
std::fs::remove_file(&p).ok();
|
||||
}
|
||||
|
||||
/// Insert one row under `path`, born pending when `needs_content`.
|
||||
fn insert_at(tx: &Transaction<'_>, path: &str, needs_content: bool) -> i64 {
|
||||
let name = path.rsplit('/').next().unwrap();
|
||||
let parent = &path[..path.rfind('/').unwrap()];
|
||||
insert_file(
|
||||
tx,
|
||||
&NewFile {
|
||||
name,
|
||||
path,
|
||||
parent,
|
||||
size: 1,
|
||||
mtime: 1,
|
||||
inode: None,
|
||||
device_id: None,
|
||||
mime: Some("text/plain"),
|
||||
ftype: FileType::TEXT,
|
||||
hash: None,
|
||||
needs_content,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.expect("unique path")
|
||||
}
|
||||
|
||||
fn fts_rows(conn: &Connection) -> i64 {
|
||||
conn.query_row("SELECT COUNT(*) FROM searchabletext", [], |r| r.get(0))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
/// The whole premise of answering both figures from `files`: a row reads
|
||||
/// `content_state = DONE` exactly when it has a `searchabletext` row, so the
|
||||
/// conditional sum *is* a count of the FTS table restricted to a path range.
|
||||
///
|
||||
/// Pinned against the FTS table itself rather than against the states that
|
||||
/// were written, because the equivalence is what would break if some future
|
||||
/// transition wrote one without the other.
|
||||
#[test]
|
||||
fn count_root_counts_the_fts_rows_it_says_it_does() {
|
||||
let p = tmp_path();
|
||||
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
|
||||
{
|
||||
let tx = conn.transaction().unwrap();
|
||||
// Two searchable, and one of each way a row can fail to be.
|
||||
for path in ["/tree/a.txt", "/tree/b.txt"] {
|
||||
let id = insert_at(&tx, path, true);
|
||||
set_content_done(
|
||||
&tx,
|
||||
id,
|
||||
"n",
|
||||
"body text",
|
||||
&[],
|
||||
zstd_of("body text").as_deref(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let failed = insert_at(&tx, "/tree/c.bin", true);
|
||||
set_content_failed(&tx, failed, "bad parse").unwrap();
|
||||
let na = insert_at(&tx, "/tree/d.iso", true);
|
||||
set_content_na(&tx, na).unwrap();
|
||||
insert_at(&tx, "/tree/e.txt", true); // still pending
|
||||
tx.commit().unwrap();
|
||||
}
|
||||
|
||||
let counts = count_root(&conn, "/tree/", "/tree0").unwrap();
|
||||
assert_eq!(counts.files, 5, "every row under the root");
|
||||
assert_eq!(
|
||||
counts.fts,
|
||||
fts_rows(&conn),
|
||||
"the root holds everything, so its FTS figure is the whole table"
|
||||
);
|
||||
assert_eq!(counts.fts, 2);
|
||||
|
||||
// A searchable row outside the range moves the table's total and not the
|
||||
// root's figure — otherwise the assertion above would hold for a count
|
||||
// that ignored its bounds.
|
||||
{
|
||||
let tx = conn.transaction().unwrap();
|
||||
let id = insert_at(&tx, "/elsewhere/f.txt", true);
|
||||
set_content_done(
|
||||
&tx,
|
||||
id,
|
||||
"n",
|
||||
"body text",
|
||||
&[],
|
||||
zstd_of("body text").as_deref(),
|
||||
)
|
||||
.unwrap();
|
||||
tx.commit().unwrap();
|
||||
}
|
||||
assert_eq!(fts_rows(&conn), 3);
|
||||
assert_eq!(
|
||||
count_root(&conn, "/tree/", "/tree0").unwrap(),
|
||||
counts,
|
||||
"a row outside the range belongs to no root's figures"
|
||||
);
|
||||
|
||||
drop(conn);
|
||||
std::fs::remove_file(&p).ok();
|
||||
}
|
||||
|
||||
/// An empty range is 0/0, not an error: a configured root nothing has been
|
||||
/// walked into yet is a normal state, and `SUM` over no rows is NULL.
|
||||
#[test]
|
||||
fn count_root_reports_zero_for_an_empty_range() {
|
||||
let p = tmp_path();
|
||||
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
|
||||
assert_eq!(
|
||||
count_root(&conn, "/nothing/", "/nothing0").unwrap(),
|
||||
RootCounts { files: 0, fts: 0 }
|
||||
);
|
||||
drop(conn);
|
||||
std::fs::remove_file(&p).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn root_counts_round_trip() {
|
||||
let p = tmp_path();
|
||||
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
|
||||
assert_eq!(get_root_counts(&conn, "/tree"), None, "never counted");
|
||||
|
||||
set_root_counts(&conn, "/tree", RootCounts { files: 12, fts: 5 }).unwrap();
|
||||
assert_eq!(
|
||||
get_root_counts(&conn, "/tree"),
|
||||
Some(RootCounts { files: 12, fts: 5 })
|
||||
);
|
||||
// Overwrite, not accumulate.
|
||||
set_root_counts(&conn, "/tree", RootCounts { files: 20, fts: 9 }).unwrap();
|
||||
assert_eq!(
|
||||
get_root_counts(&conn, "/tree"),
|
||||
Some(RootCounts { files: 20, fts: 9 })
|
||||
);
|
||||
// Roots do not read each other's figures.
|
||||
assert_eq!(get_root_counts(&conn, "/other"), None);
|
||||
|
||||
// A value this build cannot parse reads as absent, like a missing one:
|
||||
// the folder list says "not yet indexed" rather than showing a number
|
||||
// that is a guess.
|
||||
for bad in ["", "12", "12,", "a,b", "12,5,3"] {
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO schema_info(key, value) VALUES ('counts:/tree', ?1)",
|
||||
params![bad],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(get_root_counts(&conn, "/tree"), None, "parsed {:?}", bad);
|
||||
}
|
||||
|
||||
drop(conn);
|
||||
std::fs::remove_file(&p).ok();
|
||||
}
|
||||
|
||||
/// Both kinds of per-root figure are swept together, so a root removed and
|
||||
/// later re-added starts from neither a stale denominator nor a stale count.
|
||||
#[test]
|
||||
fn prune_root_stats_drops_every_figure_of_a_dropped_root() {
|
||||
let p = tmp_path();
|
||||
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
|
||||
for root in ["/kept", "/dropped"] {
|
||||
set_root_walk_count(&conn, root, 100).unwrap();
|
||||
set_root_counts(&conn, root, RootCounts { files: 90, fts: 40 }).unwrap();
|
||||
}
|
||||
set_last_full_index(&conn, 1_700_000_000).unwrap();
|
||||
|
||||
prune_root_stats(&conn, &["/kept".to_string()]).unwrap();
|
||||
|
||||
assert_eq!(get_root_walk_count(&conn, "/kept"), Some(100));
|
||||
assert_eq!(
|
||||
get_root_counts(&conn, "/kept"),
|
||||
Some(RootCounts { files: 90, fts: 40 })
|
||||
);
|
||||
assert_eq!(get_root_walk_count(&conn, "/dropped"), None);
|
||||
assert_eq!(get_root_counts(&conn, "/dropped"), None);
|
||||
// The sweep reads every `schema_info` key; unrelated ones must survive it.
|
||||
assert_eq!(get_last_full_index(&conn), Some(1_700_000_000));
|
||||
|
||||
drop(conn);
|
||||
std::fs::remove_file(&p).ok();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,58 @@ use crate::config::Config;
|
|||
use crate::db::repo::{self};
|
||||
use crate::indexing::should_abort;
|
||||
|
||||
/// The compressed sidecar for one row, or `None` where there is none to write
|
||||
/// — an empty body, or `store_text_for_snippets` turned off.
|
||||
///
|
||||
/// `Err` is kept per row rather than failing the batch, because every caller
|
||||
/// here already logs and skips a row whose write fails.
|
||||
type Body = Result<Option<Vec<u8>>, String>;
|
||||
|
||||
/// Compress a batch's bodies through one context, before the caller takes the
|
||||
/// connection.
|
||||
///
|
||||
/// Compression used to run inside the transaction, so a chunk's worth of it —
|
||||
/// measured at ~8 ms per 500 documents — sat inside the `conn_mutex` hold as
|
||||
/// pure CPU. Hoisting it here leaves the lock covering only the SQL, and
|
||||
/// reusing one [`repo::DocEncoder`] across the batch cuts the compression
|
||||
/// itself by ~4.7x (`benches/index.rs`, group `zstd_encode`).
|
||||
///
|
||||
/// What that lock does *not* gate, so the benefit is not overclaimed: search
|
||||
/// holds its own connection (`db::open::open_search_reader`) and the database
|
||||
/// is WAL, where a reader never blocks on a writer. `conn_mutex` serializes
|
||||
/// the indexer against itself — one root's content stores against another's
|
||||
/// walk inserts, the scope reconciler's slices, and WAL checkpointing. A
|
||||
/// whole-tree wall-clock run is dominated by FTS5 trigram tokenization and
|
||||
/// does not move measurably from this change; it is contention that improves,
|
||||
/// not throughput.
|
||||
fn compress_bodies<'a>(
|
||||
texts: impl Iterator<Item = Option<&'a str>>,
|
||||
config: &Config,
|
||||
) -> Result<Vec<Body>, String> {
|
||||
let mut enc = repo::DocEncoder::new()?;
|
||||
Ok(texts
|
||||
.map(|text| match text {
|
||||
Some(t) if config.processing.store_text_for_snippets && !t.is_empty() => {
|
||||
enc.encode(t).map(Some)
|
||||
}
|
||||
_ => Ok(None),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// The sidecar blob for row `i`, or a logged skip if its compression failed.
|
||||
macro_rules! body_or_skip {
|
||||
($bodies:expr, $i:expr, $what:expr) => {
|
||||
match &$bodies[$i] {
|
||||
Ok(b) => b.as_deref(),
|
||||
Err(e) => {
|
||||
crate::log_warn!("compress text for {}: {}", $what, e);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// Write already-prepared records for files whose content changed.
|
||||
///
|
||||
/// The records arrive fully built (see [`prepare_file_record`]), so this does
|
||||
|
|
@ -35,12 +87,14 @@ pub fn process_batch_updates(
|
|||
return Ok(());
|
||||
}
|
||||
|
||||
// Outside the lock — see `compress_bodies`.
|
||||
let bodies = compress_bodies(batch.iter().map(|r| r.inline_text.as_deref()), config)?;
|
||||
let conn = crate::lock_ok(conn_mutex);
|
||||
let tx = conn
|
||||
.unchecked_transaction()
|
||||
.map_err(|e| format!("Failed to begin transaction: {}", e))?;
|
||||
|
||||
for rec in batch.iter() {
|
||||
for (i, rec) in batch.iter().enumerate() {
|
||||
if stop_flag.load(Ordering::Relaxed) {
|
||||
drop(tx);
|
||||
drop(conn);
|
||||
|
|
@ -70,7 +124,8 @@ pub fn process_batch_updates(
|
|||
};
|
||||
|
||||
if let (Some(id), Some(text)) = (id, rec.inline_text.as_deref()) {
|
||||
store_inline_text(&tx, id, rec, text, config)?;
|
||||
let zstd = body_or_skip!(bodies, i, rec.path);
|
||||
store_inline_text(&tx, id, rec, text, zstd)?;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -89,16 +144,9 @@ pub(crate) fn store_inline_text(
|
|||
file_id: i64,
|
||||
rec: &OwnedNewFile,
|
||||
text: &str,
|
||||
config: &Config,
|
||||
text_zstd: Option<&[u8]>,
|
||||
) -> Result<(), String> {
|
||||
repo::set_content_done(
|
||||
tx,
|
||||
file_id,
|
||||
&rec.name,
|
||||
text,
|
||||
&[],
|
||||
config.processing.store_text_for_snippets,
|
||||
)
|
||||
repo::set_content_done(tx, file_id, &rec.name, text, &[], text_zstd)
|
||||
}
|
||||
|
||||
/// Write already-prepared records for newly discovered files. Silent, like
|
||||
|
|
@ -119,12 +167,14 @@ pub fn process_batch_inserts(
|
|||
return Ok(());
|
||||
}
|
||||
|
||||
// Outside the lock — see `compress_bodies`.
|
||||
let bodies = compress_bodies(batch.iter().map(|r| r.inline_text.as_deref()), config)?;
|
||||
let conn = crate::lock_ok(conn_mutex);
|
||||
let tx = conn
|
||||
.unchecked_transaction()
|
||||
.map_err(|e| format!("Failed to begin transaction: {}", e))?;
|
||||
|
||||
for rec in batch.iter() {
|
||||
for (i, rec) in batch.iter().enumerate() {
|
||||
if stop_flag.load(Ordering::Relaxed) {
|
||||
drop(tx);
|
||||
drop(conn);
|
||||
|
|
@ -133,7 +183,8 @@ pub fn process_batch_inserts(
|
|||
let id = repo::insert_file(&tx, &rec.as_new_file())
|
||||
.map_err(|e| format!("Failed to insert file record: {}", e))?;
|
||||
if let (Some(id), Some(text)) = (id, rec.inline_text.as_deref()) {
|
||||
store_inline_text(&tx, id, rec, text, config)?;
|
||||
let zstd = body_or_skip!(bodies, i, rec.path);
|
||||
store_inline_text(&tx, id, rec, text, zstd)?;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -307,13 +358,20 @@ pub fn store_extracted(
|
|||
if stop_flag.load(Ordering::Relaxed) {
|
||||
return Ok(written);
|
||||
}
|
||||
// Outside the lock — see `compress_bodies`.
|
||||
let bodies = compress_bodies(
|
||||
batch
|
||||
.iter()
|
||||
.map(|r| crate::file_handling::outcome_body(&r.outcome)),
|
||||
config,
|
||||
)?;
|
||||
let conn = crate::lock_ok(conn_mutex);
|
||||
let tx = conn
|
||||
.unchecked_transaction()
|
||||
.map_err(|e| format!("Failed to begin transaction: {}", e))?;
|
||||
for row in batch {
|
||||
if let Err(e) = store_content_outcome(&tx, row.file_id, &row.name, &row.outcome, config)
|
||||
{
|
||||
for (i, row) in batch.iter().enumerate() {
|
||||
let zstd = body_or_skip!(bodies, i, row.name);
|
||||
if let Err(e) = store_content_outcome(&tx, row.file_id, &row.name, &row.outcome, zstd) {
|
||||
crate::log_warn!("content indexing for {}: {}", row.name, e);
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,17 +11,17 @@ mod count_and_extract_tests;
|
|||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub(crate) use batch::store_inline_text;
|
||||
pub use batch::{
|
||||
cleanup_stale_index_entries, extract_scope_prepare, process_batch_inserts,
|
||||
process_batch_updates, store_extracted, ExtractCursor, ExtractScope,
|
||||
};
|
||||
pub(crate) use batch::store_inline_text;
|
||||
pub use counting::count_tree_entries_fast;
|
||||
pub use paths::{db_key_for_missing_path, filtered_dirs, filtered_walk, UnreadableDirs};
|
||||
pub(crate) use paths::{normalize_root_string, path_to_db_string, warn_if_unrepresentable};
|
||||
pub use records::{
|
||||
classify_by_mtime, classify_for_indexing, content_extractable, decide_content,
|
||||
extract_and_store, fts_finalize_after_text_indexing, hash_failure_counts,
|
||||
prepare_file_record, prepare_file_record_from_path, reset_run_warnings,
|
||||
store_content_outcome, ContentOutcome, DirRows, FileIndexAction, OwnedNewFile,
|
||||
extract_and_store, fts_finalize_after_text_indexing, hash_failure_counts, outcome_body,
|
||||
prepare_file_record, prepare_file_record_from_path, reset_run_warnings, store_content_outcome,
|
||||
ContentOutcome, DirRows, FileIndexAction, OwnedNewFile,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -302,7 +302,11 @@ pub fn extract_and_store(
|
|||
config: &Config,
|
||||
) -> Result<(), String> {
|
||||
let outcome = decide_content(path, mime, registry, config);
|
||||
store_content_outcome(tx, file_id, name, &outcome, config)
|
||||
let zstd = match outcome_body(&outcome) {
|
||||
Some(text) => repo::encode_one(text, config.processing.store_text_for_snippets)?,
|
||||
None => None,
|
||||
};
|
||||
store_content_outcome(tx, file_id, name, &outcome, zstd.as_deref())
|
||||
}
|
||||
|
||||
/// What should be written for one file's content, decided without touching
|
||||
|
|
@ -375,23 +379,30 @@ pub fn decide_content(
|
|||
|
||||
/// Apply a decision from [`decide_content`]. The cheap half: pure database
|
||||
/// writes, so this is all that runs with the connection held.
|
||||
///
|
||||
/// `text_zstd` is the compressed body for a `Done` outcome, prepared by the
|
||||
/// caller before it took the lock — see [`repo::set_content_done`].
|
||||
pub fn store_content_outcome(
|
||||
tx: &rusqlite::Transaction<'_>,
|
||||
file_id: i64,
|
||||
name: &str,
|
||||
outcome: &ContentOutcome,
|
||||
config: &Config,
|
||||
text_zstd: Option<&[u8]>,
|
||||
) -> Result<(), String> {
|
||||
match outcome {
|
||||
ContentOutcome::Done { text, properties } => repo::set_content_done(
|
||||
tx,
|
||||
file_id,
|
||||
name,
|
||||
text,
|
||||
properties,
|
||||
config.processing.store_text_for_snippets,
|
||||
),
|
||||
ContentOutcome::Done { text, properties } => {
|
||||
repo::set_content_done(tx, file_id, name, text, properties, text_zstd)
|
||||
}
|
||||
ContentOutcome::NotApplicable => repo::set_content_na(tx, file_id),
|
||||
ContentOutcome::Failed(reason) => repo::set_content_failed(tx, file_id, reason),
|
||||
}
|
||||
}
|
||||
|
||||
/// The body a [`ContentOutcome`] would store, if any — what the caller feeds
|
||||
/// to its [`repo::DocEncoder`] ahead of the lock.
|
||||
pub fn outcome_body(outcome: &ContentOutcome) -> Option<&str> {
|
||||
match outcome {
|
||||
ContentOutcome::Done { text, .. } => Some(text),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -137,7 +137,8 @@ fn upsert_file(
|
|||
repo::set_content_na(&tx, file_id)?;
|
||||
} else if let Some(text) = rec.inline_text.as_deref() {
|
||||
// `prepare_file_record_from_path` already read the whole file.
|
||||
store_inline_text(&tx, file_id, &rec, text, config)?;
|
||||
let zstd = repo::encode_one(text, config.processing.store_text_for_snippets)?;
|
||||
store_inline_text(&tx, file_id, &rec, text, zstd.as_deref())?;
|
||||
} else {
|
||||
extract_and_store(
|
||||
&tx,
|
||||
|
|
|
|||
|
|
@ -655,7 +655,7 @@ impl IndexingService {
|
|||
// compete for the connection.
|
||||
let stored_counts: Vec<Option<usize>> = {
|
||||
let conn = crate::lock_ok(&cx.conn_mutex);
|
||||
let _ = crate::db::repo::prune_root_walk_counts(&conn, &roots);
|
||||
let _ = crate::db::repo::prune_root_stats(&conn, &roots);
|
||||
roots
|
||||
.iter()
|
||||
.map(|r| crate::db::repo::get_root_walk_count(&conn, r))
|
||||
|
|
@ -805,6 +805,26 @@ impl IndexingService {
|
|||
crate::log_warn!("{}", e);
|
||||
}
|
||||
|
||||
// What each root holds, for the folder list to show once these
|
||||
// pipelines and their `RootProgress` rows are gone. Here rather than on
|
||||
// a cadence: `count_root` reads every row in the range, and this run
|
||||
// has just written them, so the pages are as warm as they will ever be.
|
||||
// Under the interrupt guard because it is still a scan per root, and
|
||||
// quitting should not wait out one of them; a root whose count fails
|
||||
// keeps the figure it had.
|
||||
let _guard = db::InterruptGuard::arm(interrupt, &conn);
|
||||
for root in &roots {
|
||||
let range = ExtractCursor::for_root(root);
|
||||
match repo::count_root(&conn, &range.lo, &range.hi) {
|
||||
Ok(counts) => {
|
||||
if let Err(e) = repo::set_root_counts(&conn, root, counts) {
|
||||
crate::log_warn!("{}", e);
|
||||
}
|
||||
}
|
||||
Err(e) => crate::log_warn!("counts for {} unavailable: {}", root, e),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -163,6 +163,12 @@ fn stored_walk_count(db_path: &str, root: &str) -> Option<usize> {
|
|||
crate::db::repo::get_root_walk_count(&conn, root)
|
||||
}
|
||||
|
||||
/// What the last completed run counted under `root`, if any.
|
||||
fn stored_root_counts(db_path: &str, root: &str) -> Option<crate::db::repo::RootCounts> {
|
||||
let conn = db::open_existing(db_path, false).unwrap();
|
||||
crate::db::repo::get_root_counts(&conn, root)
|
||||
}
|
||||
|
||||
/// A walk that could not read part of its tree must not record its count:
|
||||
/// the figure is the next run's progress denominator, and nothing ever
|
||||
/// re-derives it.
|
||||
|
|
@ -234,6 +240,92 @@ fn a_stopped_run_keeps_the_walk_count_unrecorded() {
|
|||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
/// A completed run records what each root holds, so the folder list can show
|
||||
/// it once the run's own per-root progress rows are gone.
|
||||
///
|
||||
/// The extension whitelist is narrowed to `txt` so the split between the two
|
||||
/// figures is the test's to decide rather than the default list's.
|
||||
#[test]
|
||||
fn a_completed_run_records_what_each_root_holds() {
|
||||
let dir = tmp_dir("root-counts");
|
||||
// A subdirectory, so the index and its WAL sidecars are not themselves
|
||||
// files under the root being counted.
|
||||
let tree = dir.join("tree");
|
||||
std::fs::create_dir_all(&tree).unwrap();
|
||||
std::fs::write(tree.join("a.txt"), "alpha body").unwrap();
|
||||
std::fs::write(tree.join("b.txt"), "beta body").unwrap();
|
||||
std::fs::write(tree.join("c.log"), "outside the whitelist").unwrap();
|
||||
|
||||
let db_path = dir.join("index.db").to_string_lossy().into_owned();
|
||||
let mut config = Config::default();
|
||||
config.paths.indexing_paths = vec![tree.to_string_lossy().into_owned()];
|
||||
config.paths.database_path = db_path.clone();
|
||||
config.indexing.content_extensions = vec!["txt".into()];
|
||||
let root = normalize_root_string(&tree.to_string_lossy());
|
||||
|
||||
run_with(&config, &db_path, &Arc::new(AtomicBool::new(false))).unwrap();
|
||||
|
||||
let stored = stored_root_counts(&db_path, &root).expect("a completed run records them");
|
||||
assert_eq!(
|
||||
stored,
|
||||
crate::db::repo::RootCounts { files: 3, fts: 2 },
|
||||
"every file under the root, and the two the whitelist let through"
|
||||
);
|
||||
|
||||
// And they describe the index rather than the walk: the same two numbers
|
||||
// read straight off the tables the folder list is standing in for.
|
||||
let conn = db::open_existing(&db_path, false).unwrap();
|
||||
let files: i64 = conn
|
||||
.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
let fts: i64 = conn
|
||||
.query_row("SELECT COUNT(*) FROM searchabletext", [], |r| r.get(0))
|
||||
.unwrap();
|
||||
assert_eq!((stored.files, stored.fts), (files, fts));
|
||||
drop(conn);
|
||||
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
/// A stopped run counted part of a tree it was still changing, so the figures
|
||||
/// it would store are worse than the ones already there. Pinned in both
|
||||
/// directions: a guard that simply never stored anything would satisfy the
|
||||
/// negative half on its own.
|
||||
#[test]
|
||||
fn a_stopped_run_keeps_the_recorded_counts() {
|
||||
let dir = tmp_dir("stopped-root-counts");
|
||||
let tree = dir.join("tree");
|
||||
std::fs::create_dir_all(&tree).unwrap();
|
||||
std::fs::write(tree.join("a.txt"), "alpha body").unwrap();
|
||||
|
||||
let db_path = dir.join("index.db").to_string_lossy().into_owned();
|
||||
let mut config = Config::default();
|
||||
config.paths.indexing_paths = vec![tree.to_string_lossy().into_owned()];
|
||||
config.paths.database_path = db_path.clone();
|
||||
let root = normalize_root_string(&tree.to_string_lossy());
|
||||
|
||||
run_with(&config, &db_path, &Arc::new(AtomicBool::new(false))).unwrap();
|
||||
let after_run = stored_root_counts(&db_path, &root).expect("recorded");
|
||||
assert_eq!(after_run.files, 1);
|
||||
|
||||
// Two more files, then a run that stops at its first check — the
|
||||
// deterministic stand-in for a shutdown part-way through.
|
||||
std::fs::write(tree.join("b.txt"), "beta body").unwrap();
|
||||
std::fs::write(tree.join("c.txt"), "gamma body").unwrap();
|
||||
run_with(&config, &db_path, &Arc::new(AtomicBool::new(true))).unwrap();
|
||||
assert_eq!(
|
||||
stored_root_counts(&db_path, &root),
|
||||
Some(after_run),
|
||||
"a stopped run leaves the last completed run's figures alone"
|
||||
);
|
||||
|
||||
// The same tree, run to completion, does move them.
|
||||
run_with(&config, &db_path, &Arc::new(AtomicBool::new(false))).unwrap();
|
||||
assert_eq!(stored_root_counts(&db_path, &root).unwrap().files, 3);
|
||||
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
/// A reconcile the stop flag cut short must leave the stored fingerprint
|
||||
/// alone: stamping it would tell every later run the index already matches,
|
||||
/// and nothing would ever revisit the rows the scan had not reached.
|
||||
|
|
|
|||
|
|
@ -211,6 +211,32 @@ impl TermPattern {
|
|||
}
|
||||
}
|
||||
|
||||
/// ASCII-case-insensitive [`str::find`], without folding the haystack.
|
||||
///
|
||||
/// `needle` must already be `to_ascii_lowercase`d — `LiteralPattern`
|
||||
/// stores it that way. This exists because the folding version allocated
|
||||
/// a lowercase copy of its haystack on every call, and the cascade's
|
||||
/// filename pass calls it twice — once on the name, once on the path —
|
||||
/// for every row of a full-table scan.
|
||||
///
|
||||
/// The candidate index is always a `char` boundary: a `&str`'s first byte
|
||||
/// is either ASCII or a UTF-8 lead byte, never a continuation byte, so a
|
||||
/// first-byte hit cannot land mid-character. Both sides being valid UTF-8
|
||||
/// that differ only in ASCII case then makes the end a boundary too, which
|
||||
/// is what keeps the returned range valid in the unfolded original.
|
||||
fn find_ascii_ci(hay: &str, needle: &str) -> Option<usize> {
|
||||
if needle.is_empty() {
|
||||
return Some(0);
|
||||
}
|
||||
let (h, n) = (hay.as_bytes(), needle.as_bytes());
|
||||
let first = n[0];
|
||||
h.len().checked_sub(n.len()).and_then(|last| {
|
||||
(0..=last).find(|&i| {
|
||||
h[i].to_ascii_lowercase() == first && h[i..i + n.len()].eq_ignore_ascii_case(n)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Leftmost match as a byte range. Literal folding is ASCII-only and
|
||||
/// byte-length preserving, so folded offsets are valid in the original —
|
||||
/// the same invariant the cascade has always relied on.
|
||||
|
|
@ -219,7 +245,7 @@ impl TermPattern {
|
|||
TermPattern::Empty => None,
|
||||
TermPattern::Literal(l) => {
|
||||
let pos = if case_insensitive {
|
||||
text.to_ascii_lowercase().find(&l.folded)?
|
||||
Self::find_ascii_ci(text, &l.folded)?
|
||||
} else {
|
||||
text.find(&l.text)?
|
||||
};
|
||||
|
|
|
|||
|
|
@ -10,6 +10,92 @@ enum RowHit {
|
|||
Defer(SearchHit),
|
||||
}
|
||||
|
||||
/// Reusable decode buffer and decompression context for the passes that read
|
||||
/// document text.
|
||||
///
|
||||
/// `zstd::decode_all` builds and tears down a `ZSTD_DCtx` *and* allocates a
|
||||
/// fresh output `Vec` on every call, and it is called once per candidate row.
|
||||
/// One context and one buffer, reused across a whole scan, make that a
|
||||
/// per-scan cost instead of a per-row one.
|
||||
struct DocDecoder {
|
||||
dctx: zstd::bulk::Decompressor<'static>,
|
||||
buf: Vec<u8>,
|
||||
}
|
||||
|
||||
/// Where [`DocDecoder::decode`]'s buffer starts before it has seen a document.
|
||||
/// Most extracted text is well under this, so the doubling below rarely runs.
|
||||
const INITIAL_DOC_CAPACITY: usize = 64 * 1024;
|
||||
|
||||
/// Where the doubling stops. Stored text is capped at
|
||||
/// `processing.maximum_text_size` (256 KiB by default), so this is far above
|
||||
/// any legitimate document even if that setting is raised — past it, a failure
|
||||
/// is a corrupt frame rather than a buffer that is too small.
|
||||
const MAX_DOC_CAPACITY: usize = 64 * 1024 * 1024;
|
||||
|
||||
impl DocDecoder {
|
||||
fn new() -> Result<Self, String> {
|
||||
Ok(DocDecoder {
|
||||
dctx: zstd::bulk::Decompressor::new().map_err(|e| e.to_string())?,
|
||||
buf: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Decompress `blob` and borrow the result as text.
|
||||
///
|
||||
/// Returns `None` for a corrupt frame or non-UTF-8 content. Nothing is
|
||||
/// copied: the indexer stores UTF-8, so the bytes are borrowed in place
|
||||
/// rather than run through `String::from_utf8_lossy(..).into_owned()`,
|
||||
/// which duplicated the whole document even when it was already valid.
|
||||
fn decode(&mut self, blob: &[u8]) -> Option<&str> {
|
||||
self.buf.clear();
|
||||
// `decompress_to_buffer` writes into spare capacity and fails rather
|
||||
// than growing, so the room has to be there first.
|
||||
//
|
||||
// The frame header would say how much is needed, but the indexer
|
||||
// writes with `zstd::encode_all`, which is *stream*-based and so
|
||||
// records no content size — `get_frame_content_size` says `None` for
|
||||
// every row this ever sees. Falling back to `zstd::decode_all` there
|
||||
// looked harmless and was not: it builds a streaming decoder per call,
|
||||
// which measured as one ~2.4 MiB allocation per document and 27 of the
|
||||
// 30 GiB a fuzzy search moved through the allocator.
|
||||
//
|
||||
// So grow this buffer instead and keep reusing it. It settles at the
|
||||
// largest document in the scan within the first few rows, after which
|
||||
// decoding a row allocates nothing at all.
|
||||
if let Ok(Some(size)) = zstd::zstd_safe::get_frame_content_size(blob) {
|
||||
self.buf.reserve(usize::try_from(size).ok()?);
|
||||
}
|
||||
loop {
|
||||
if self.buf.capacity() == 0 {
|
||||
self.buf.reserve(INITIAL_DOC_CAPACITY);
|
||||
}
|
||||
match self.dctx.decompress_to_buffer(blob, &mut self.buf) {
|
||||
Ok(_) => break,
|
||||
// Too small, or corrupt — the bulk API cannot tell us which.
|
||||
// Growing is only worth trying while the buffer is still
|
||||
// smaller than any document could legitimately be.
|
||||
Err(_) if self.buf.capacity() < MAX_DOC_CAPACITY => {
|
||||
let bigger = self.buf.capacity().saturating_mul(2);
|
||||
self.buf.clear();
|
||||
self.buf.reserve(bigger);
|
||||
}
|
||||
Err(_) => return None,
|
||||
}
|
||||
}
|
||||
std::str::from_utf8(&self.buf).ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// Fold `text` into `dst` in place, reusing its allocation.
|
||||
///
|
||||
/// The ASCII fold is byte-length preserving, which is what lets the cascade
|
||||
/// use offsets found in the folded copy against the unfolded original.
|
||||
fn fold_into(dst: &mut String, text: &str) {
|
||||
dst.clear();
|
||||
dst.push_str(text);
|
||||
dst.make_ascii_lowercase();
|
||||
}
|
||||
|
||||
/// Which [`Deferred`] buffer a scan's held-back hits go to.
|
||||
enum DeferSlot {
|
||||
/// Ranks 9–10, flushed by [`Pass::Path`]. Shared by passes A and E,
|
||||
|
|
@ -39,7 +125,10 @@ impl<'a> Cx<'a> {
|
|||
mut classify: impl FnMut(&mut Self, &rusqlite::Row<'_>, i64, &str) -> Result<RowHit, String>,
|
||||
) -> Result<bool, String> {
|
||||
let conn = self.conn;
|
||||
let mut stmt = conn.prepare(sql).map_err(|e| e.to_string())?;
|
||||
// Cached: a search re-runs the same six statements on every keystroke,
|
||||
// and only the bound term changes between them — the filter SQL each
|
||||
// one interpolates is fixed for the life of the query.
|
||||
let mut stmt = conn.prepare_cached(sql).map_err(|e| e.to_string())?;
|
||||
let mut rows = stmt
|
||||
.query(rusqlite::params_from_iter(params))
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
|
@ -55,11 +144,18 @@ impl<'a> Cx<'a> {
|
|||
return Ok(false);
|
||||
}
|
||||
let file_id: i64 = col(row, 0)?;
|
||||
let path: String = col(row, 2)?;
|
||||
if self.skip(file_id, &path) {
|
||||
// Borrowed from the statement rather than `col::<String>`: this
|
||||
// runs for every *scanned* row — a full-table scan on the filename
|
||||
// pass — while only the few that become hits need an owned copy.
|
||||
let path = row
|
||||
.get_ref(2)
|
||||
.map_err(|e| e.to_string())?
|
||||
.as_str()
|
||||
.map_err(|e| e.to_string())?;
|
||||
if self.skip(file_id, path) {
|
||||
continue;
|
||||
}
|
||||
match classify(self, row, file_id, &path)? {
|
||||
match classify(self, row, file_id, path)? {
|
||||
RowHit::Skip => {}
|
||||
RowHit::Emit(hit) => {
|
||||
buf.push(hit);
|
||||
|
|
@ -234,26 +330,33 @@ impl<'a> Cx<'a> {
|
|||
let snippet_opts = snippet::Options {
|
||||
approx_chars: SNIPPET_WINDOW_CHARS,
|
||||
};
|
||||
// One decoder and one fold buffer for the whole scan; both are reused
|
||||
// per row rather than reallocated.
|
||||
let mut doc = DocDecoder::new()?;
|
||||
let mut lower = String::new();
|
||||
// Decompression dominates: check cancellation every row.
|
||||
self.scan_pass(&sql, params, 1, None, |cx, row, file_id, path| {
|
||||
let blob: Option<Vec<u8>> = col(row, 5)?;
|
||||
let text = blob
|
||||
.and_then(|b| zstd::decode_all(b.as_slice()).ok())
|
||||
.map(|raw| String::from_utf8_lossy(&raw).into_owned());
|
||||
let blob: Option<&[u8]> = row
|
||||
.get_ref(5)
|
||||
.map_err(|e| e.to_string())?
|
||||
.as_blob_or_null()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let text = blob.and_then(|b| doc.decode(b));
|
||||
|
||||
let (rank, stage, snip) = match &text {
|
||||
let (rank, stage, snip) = match text {
|
||||
Some(text) => {
|
||||
// Fold once: the case-insensitive count, the first-match
|
||||
// search and the snippet extraction all need it, and
|
||||
// nearly every candidate takes this path.
|
||||
let mut folded: Option<String> = None;
|
||||
let mut folded = false;
|
||||
let (count, stage) = {
|
||||
let count_cs = pattern.count(text, false);
|
||||
if count_cs > 0 {
|
||||
(count_cs, 5)
|
||||
} else {
|
||||
let lower = folded.insert(text.to_ascii_lowercase());
|
||||
let count_ci = pattern.count_folded(lower);
|
||||
fold_into(&mut lower, text);
|
||||
folded = true;
|
||||
let count_ci = pattern.count_folded(&lower);
|
||||
if count_ci > 0 {
|
||||
(count_ci, 6)
|
||||
} else {
|
||||
|
|
@ -263,9 +366,11 @@ impl<'a> Cx<'a> {
|
|||
}
|
||||
}
|
||||
};
|
||||
if !folded {
|
||||
fold_into(&mut lower, text);
|
||||
}
|
||||
// Literal terms keep the richer multi-occurrence
|
||||
// extract; a wildcard match marks its own first range.
|
||||
let lower = folded.unwrap_or_else(|| text.to_ascii_lowercase());
|
||||
let snip = match pattern.literal() {
|
||||
Some(term) => Some(snippet::extract_folded(
|
||||
text,
|
||||
|
|
@ -291,7 +396,7 @@ impl<'a> Cx<'a> {
|
|||
(6.0 + count_frac(1), 6, None)
|
||||
}
|
||||
};
|
||||
if !cx.regex_accepts(file_id, path, text.as_deref())? {
|
||||
if !cx.regex_accepts(file_id, path, text)? {
|
||||
return Ok(RowHit::Skip);
|
||||
}
|
||||
|
||||
|
|
@ -416,27 +521,30 @@ impl<'a> Cx<'a> {
|
|||
let snippet_opts = snippet::Options {
|
||||
approx_chars: SNIPPET_WINDOW_CHARS,
|
||||
};
|
||||
// One decoder and one fold buffer for the whole scan, reused per row.
|
||||
let mut doc = DocDecoder::new()?;
|
||||
let mut folded = String::new();
|
||||
// Decompression dominates: check cancellation every row.
|
||||
self.scan_pass(&sql, params, 1, None, |cx, row, file_id, path| {
|
||||
let blob: Option<Vec<u8>> = col(row, 5)?;
|
||||
let Some(blob) = blob else {
|
||||
let blob: Option<&[u8]> = row
|
||||
.get_ref(5)
|
||||
.map_err(|e| e.to_string())?
|
||||
.as_blob_or_null()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let Some(text) = blob.and_then(|b| doc.decode(b)) else {
|
||||
return Ok(RowHit::Skip);
|
||||
};
|
||||
let Ok(raw) = zstd::decode_all(blob.as_slice()) else {
|
||||
return Ok(RowHit::Skip);
|
||||
};
|
||||
let text = String::from_utf8_lossy(&raw).into_owned();
|
||||
// ASCII folding is byte-length preserving, so ranges found in
|
||||
// the folded buffer are valid in the original.
|
||||
let folded = text.to_ascii_lowercase();
|
||||
fold_into(&mut folded, text);
|
||||
let (count, first) = bitap.count_and_first(folded.as_bytes());
|
||||
if count == 0 {
|
||||
return Ok(RowHit::Skip);
|
||||
}
|
||||
if !cx.regex_accepts(file_id, path, Some(&text))? {
|
||||
if !cx.regex_accepts(file_id, path, Some(text))? {
|
||||
return Ok(RowHit::Skip);
|
||||
}
|
||||
let snip = first.map(|range| snippet::window_around(&text, range, &snippet_opts));
|
||||
let snip = first.map(|range| snippet::window_around(text, range, &snippet_opts));
|
||||
let (size, mtime) = size_and_mtime(row)?;
|
||||
Ok(RowHit::Emit(SearchHit {
|
||||
file_id,
|
||||
|
|
@ -522,22 +630,27 @@ impl<'a> Cx<'a> {
|
|||
let snippet_opts = snippet::Options {
|
||||
approx_chars: SNIPPET_WINDOW_CHARS,
|
||||
};
|
||||
// One decoder for the whole scan, reused per row.
|
||||
let mut doc = DocDecoder::new()?;
|
||||
// Decompression dominates: check cancellation every row.
|
||||
self.scan_pass(&sql, params, 1, None, |_cx, row, file_id, path| {
|
||||
let blob: Option<Vec<u8>> = col(row, 5)?;
|
||||
let Some(raw) = blob.and_then(|b| zstd::decode_all(b.as_slice()).ok()) else {
|
||||
let blob: Option<&[u8]> = row
|
||||
.get_ref(5)
|
||||
.map_err(|e| e.to_string())?
|
||||
.as_blob_or_null()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let Some(text) = blob.and_then(|b| doc.decode(b)) else {
|
||||
return Ok(RowHit::Skip);
|
||||
};
|
||||
let text = String::from_utf8_lossy(&raw).into_owned();
|
||||
let count = re.count(&text);
|
||||
let count = re.count(text);
|
||||
if count == 0 {
|
||||
return Ok(RowHit::Skip);
|
||||
}
|
||||
// A greedy user regex can match megabytes; clamp the range
|
||||
// before the snippet window is cut.
|
||||
let snip = re.find_first(&text).map(|r| {
|
||||
let r = clamp_match_range(&text, r, SNIPPET_WINDOW_CHARS);
|
||||
snippet::window_around(&text, (r.start, r.end), &snippet_opts)
|
||||
let snip = re.find_first(text).map(|r| {
|
||||
let r = clamp_match_range(text, r, SNIPPET_WINDOW_CHARS);
|
||||
snippet::window_around(text, (r.start, r.end), &snippet_opts)
|
||||
});
|
||||
let (size, mtime) = size_and_mtime(row)?;
|
||||
Ok(RowHit::Emit(SearchHit {
|
||||
|
|
|
|||
|
|
@ -298,7 +298,20 @@ impl Worker {
|
|||
let first = match self.req_rx.recv_timeout(self.idle_release) {
|
||||
Ok(req) => req,
|
||||
Err(mpsc::RecvTimeoutError::Timeout) => {
|
||||
self.open = None;
|
||||
// Dropping the connection frees `PRAGMAS_SEARCH`'s 32 MiB
|
||||
// page cache to glibc, which parks it in an arena rather
|
||||
// than returning it to the kernel. Without the trim a
|
||||
// single typing session raises the process floor by ~42 MiB
|
||||
// for as long as it runs — measured on a 77k-file index,
|
||||
// where an idle GUI sat at 76 MiB `RssAnon` instead of 34.
|
||||
// Gated on there having *been* a connection so this is once
|
||||
// per session→idle transition, never a repeating tick:
|
||||
// `malloc_trim` walks every arena and costs milliseconds.
|
||||
// The coordinator's writer settles the same way in
|
||||
// `go_idle`.
|
||||
if self.open.take().is_some() {
|
||||
crate::platform::release_free_heap();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
Err(mpsc::RecvTimeoutError::Disconnected) => return,
|
||||
|
|
|
|||
|
|
@ -76,13 +76,16 @@ pub fn extract_folded(text: &str, folded: &str, terms: &[&str], opts: &Options)
|
|||
return head_window(text, opts.approx_chars);
|
||||
}
|
||||
|
||||
// `memmem` rather than `str::match_indices`: both find non-overlapping
|
||||
// occurrences, but std's Two-Way searcher has no vector prefilter and a
|
||||
// full-text row scans a whole document body. See `benches/search.rs`,
|
||||
// group `substring`.
|
||||
let mut matches: Vec<(usize, usize)> = Vec::new();
|
||||
for term in &effective_terms {
|
||||
let pattern = term.to_ascii_lowercase();
|
||||
matches.extend(
|
||||
folded
|
||||
.match_indices(&pattern)
|
||||
.map(|(at, _)| (at, at + pattern.len())),
|
||||
memchr::memmem::find_iter(folded.as_bytes(), pattern.as_bytes())
|
||||
.map(|at| (at, at + pattern.len())),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -185,11 +188,13 @@ pub fn count_occurrences(text: &str, term: &str, case_sensitive: bool) -> usize
|
|||
return 0;
|
||||
}
|
||||
if case_sensitive {
|
||||
text.match_indices(term).count()
|
||||
memchr::memmem::find_iter(text.as_bytes(), term.as_bytes()).count()
|
||||
} else {
|
||||
text.to_ascii_lowercase()
|
||||
.match_indices(&term.to_ascii_lowercase())
|
||||
.count()
|
||||
memchr::memmem::find_iter(
|
||||
text.to_ascii_lowercase().as_bytes(),
|
||||
term.to_ascii_lowercase().as_bytes(),
|
||||
)
|
||||
.count()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,6 +11,16 @@ use std::sync::atomic::{AtomicUsize, Ordering};
|
|||
/// lets two tests in the same millisecond collide.
|
||||
static NEXT: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
/// The compressed body [`crate::db::repo::set_content_done`] wants, for tests
|
||||
/// that only care that a sidecar row gets written.
|
||||
///
|
||||
/// Production callers compress a whole batch through one
|
||||
/// [`crate::db::repo::DocEncoder`] before taking the connection lock; a test
|
||||
/// writing one row has nothing to amortize and wants the one-liner.
|
||||
pub fn zstd_of(text: &str) -> Option<Vec<u8>> {
|
||||
crate::db::repo::encode_one(text, true).expect("zstd encode")
|
||||
}
|
||||
|
||||
/// A fresh, empty directory under the system temp dir, named for `tag`.
|
||||
/// Not cleaned up on drop: when a test fails, the tree it built is most of
|
||||
/// the evidence. Panics — a test that cannot create a directory has nothing
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ use quicksearch_core::db::repo::{insert_file, set_content_done, NewFile};
|
|||
use quicksearch_core::mime::FileType;
|
||||
use quicksearch_core::query::split::split_for_cascade;
|
||||
use quicksearch_core::search::{cascade, SearchHit, SearchOptions, SearchService, SearchUpdate};
|
||||
use quicksearch_core::testutil::zstd_of;
|
||||
|
||||
mod common;
|
||||
use common::scratch_db as tmp_db;
|
||||
|
|
@ -50,7 +51,8 @@ impl Seeder {
|
|||
.unwrap()
|
||||
.expect("unique path");
|
||||
if let Some(text) = text {
|
||||
set_content_done(&tx, id, name, text, &[], self.store_text).unwrap();
|
||||
let zstd = self.store_text.then(|| zstd_of(text)).flatten();
|
||||
set_content_done(&tx, id, name, text, &[], zstd.as_deref()).unwrap();
|
||||
}
|
||||
tx.commit().unwrap();
|
||||
id
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ use quicksearch_core::mime::FileType;
|
|||
use quicksearch_core::query::split::split_for_cascade;
|
||||
use quicksearch_core::search::{cascade, SearchHit, SearchOptions};
|
||||
use quicksearch_core::security::IndexKey;
|
||||
use quicksearch_core::testutil::zstd_of;
|
||||
use rusqlite::Connection;
|
||||
|
||||
mod common;
|
||||
|
|
@ -137,7 +138,8 @@ fn seed(path: &std::path::Path) {
|
|||
let body: Vec<&str> = (0..60)
|
||||
.map(|_| WORDS[(rng.next() as usize) % WORDS.len()])
|
||||
.collect();
|
||||
set_content_done(&tx, id, &name, &body.join(" "), &[], true).unwrap();
|
||||
let body = body.join(" ");
|
||||
set_content_done(&tx, id, &name, &body, &[], zstd_of(&body).as_deref()).unwrap();
|
||||
}
|
||||
}
|
||||
tx.commit().unwrap();
|
||||
|
|
@ -203,6 +205,15 @@ fn run_matrix(label: &str, path: &std::path::Path) {
|
|||
}
|
||||
}
|
||||
|
||||
/// Serializes the two matrices below.
|
||||
///
|
||||
/// `set_process_key` is process-global, and libtest runs `#[test]` functions
|
||||
/// on concurrent threads within one process — so without this the encrypted
|
||||
/// run's key is set while the unencrypted run opens its plain index, and that
|
||||
/// open fails with `NotADatabase`. Being separate tests is not enough on its
|
||||
/// own; they have to not overlap.
|
||||
static SERIAL: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
||||
|
||||
/// The headline comparison, printed rather than asserted.
|
||||
///
|
||||
/// Deliberately not a pass/fail threshold: timings on a shared CI box are not
|
||||
|
|
@ -215,6 +226,7 @@ fn cache_size_against_search_latency() {
|
|||
eprintln!("skipping: set QSB_SEARCH_PERF=1 to run");
|
||||
return;
|
||||
}
|
||||
let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner());
|
||||
|
||||
let plain = scratch_db("searchperf-plain");
|
||||
let seeded = Instant::now();
|
||||
|
|
@ -232,13 +244,14 @@ fn cache_size_against_search_latency() {
|
|||
///
|
||||
/// Separate test, and separate process-wide key, because
|
||||
/// [`set_process_key`] is global: running both in one test would have the
|
||||
/// plain index opened with a key set.
|
||||
/// plain index opened with a key set. [`SERIAL`] keeps them from overlapping.
|
||||
#[test]
|
||||
fn cache_size_against_search_latency_encrypted() {
|
||||
if !enabled() {
|
||||
eprintln!("skipping: set QSB_SEARCH_PERF=1 to run");
|
||||
return;
|
||||
}
|
||||
let _serial = SERIAL.lock().unwrap_or_else(|e| e.into_inner());
|
||||
|
||||
set_process_key(Some(
|
||||
IndexKey::from_hex(&"42".repeat(32)).expect("valid 32-byte key"),
|
||||
|
|
|
|||
|
|
@ -26,6 +26,16 @@ pub struct LogsTab {
|
|||
/// (egui unsticks a scroll area the user moves, and re-sticks it when
|
||||
/// they return to the bottom); unticking this stops it following at all.
|
||||
follow: bool,
|
||||
/// Indices into `lines` that pass both filters.
|
||||
///
|
||||
/// Cached rather than rebuilt per frame: `keep` lowercases each line to
|
||||
/// compare it, so with a filter typed this was up to [`log::CAPACITY`]
|
||||
/// (5,000) string allocations every frame, on a tab that repaints twice a
|
||||
/// second by itself and on every input frame besides.
|
||||
shown: Vec<usize>,
|
||||
/// What `shown` was computed from — refresh counter, filter text,
|
||||
/// warnings-only — so it can be rebuilt exactly when one of them moves.
|
||||
shown_key: (u64, String, bool),
|
||||
}
|
||||
|
||||
impl LogsTab {
|
||||
|
|
@ -37,9 +47,36 @@ impl LogsTab {
|
|||
filter: String::new(),
|
||||
warnings_only: false,
|
||||
follow: true,
|
||||
shown: Vec::new(),
|
||||
// `u64::MAX` so the first frame always counts as stale, whatever
|
||||
// the ring's counter happens to be.
|
||||
shown_key: (u64::MAX, String::new(), false),
|
||||
}
|
||||
}
|
||||
|
||||
/// Rebuild [`LogsTab::shown`] if any of its inputs moved.
|
||||
fn resync_shown(&mut self) {
|
||||
if self.shown_key.0 == self.seen
|
||||
&& self.shown_key.2 == self.warnings_only
|
||||
&& self.shown_key.1 == self.filter
|
||||
{
|
||||
return;
|
||||
}
|
||||
let needle = self.filter.to_lowercase();
|
||||
let warnings_only = self.warnings_only;
|
||||
let mut shown = std::mem::take(&mut self.shown);
|
||||
shown.clear();
|
||||
shown.extend(
|
||||
self.lines
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, l)| keep(l, &needle, warnings_only))
|
||||
.map(|(i, _)| i),
|
||||
);
|
||||
self.shown = shown;
|
||||
self.shown_key = (self.seen, self.filter.clone(), warnings_only);
|
||||
}
|
||||
|
||||
fn refresh(&mut self) {
|
||||
self.lines = log::snapshot();
|
||||
self.seen = log::recorded();
|
||||
|
|
@ -53,14 +90,11 @@ impl LogsTab {
|
|||
ui.ctx()
|
||||
.request_repaint_after(std::time::Duration::from_millis(REFRESH_MS));
|
||||
|
||||
let needle = self.filter.to_lowercase();
|
||||
let shown: Vec<usize> = self
|
||||
.lines
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, l)| keep(l, &needle, self.warnings_only))
|
||||
.map(|(i, _)| i)
|
||||
.collect();
|
||||
// The filter and warnings-only widgets are drawn below, so a change
|
||||
// lands on the next frame — the same one-frame lag this always had,
|
||||
// and the tab repaints immediately anyway.
|
||||
self.resync_shown();
|
||||
let shown = std::mem::take(&mut self.shown);
|
||||
|
||||
let mut cleared = false;
|
||||
ui.horizontal(|ui| {
|
||||
|
|
@ -106,7 +140,10 @@ impl LogsTab {
|
|||
});
|
||||
});
|
||||
if cleared {
|
||||
// `shown` indexes lines that no longer exist.
|
||||
// `shown` indexes lines that no longer exist. Handing the buffer
|
||||
// back keeps its capacity; `refresh` moves `seen`, which is what
|
||||
// makes the next frame rebuild the contents.
|
||||
self.shown = shown;
|
||||
self.refresh();
|
||||
return;
|
||||
}
|
||||
|
|
@ -131,10 +168,12 @@ impl LogsTab {
|
|||
)
|
||||
.weak(),
|
||||
);
|
||||
self.shown = shown;
|
||||
return;
|
||||
}
|
||||
if shown.is_empty() {
|
||||
ui.label(egui::RichText::new("No lines match the filter.").weak());
|
||||
self.shown = shown;
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -163,6 +202,7 @@ impl LogsTab {
|
|||
}
|
||||
});
|
||||
crate::ui_util::more_below_hint(ui, &scroll);
|
||||
self.shown = shown;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -295,6 +295,15 @@ impl ManageTab {
|
|||
}
|
||||
ui.label(hint("workers:"));
|
||||
|
||||
// Unconditional, placeholder and all: egui names a
|
||||
// widget by how many precede it, so a label that
|
||||
// came and went would rename the field above and
|
||||
// cost it any edit in progress. After that field
|
||||
// for the same reason — in this right-to-left
|
||||
// layout "after" is to its left.
|
||||
ui.label(hint(root_counts_text(state, root)))
|
||||
.tip(&tips::ROOT_COUNTS);
|
||||
|
||||
ui.with_layout(
|
||||
egui::Layout::left_to_right(egui::Align::Center),
|
||||
|ui| {
|
||||
|
|
@ -549,6 +558,24 @@ fn db_size_tooltip(ui: &mut egui::Ui) {
|
|||
));
|
||||
}
|
||||
|
||||
/// What `root` held when indexing last completed, worded as the live
|
||||
/// per-root rows word it (see [`root_row`]) so the list does not rename the
|
||||
/// same two figures once the run that produced them is over.
|
||||
///
|
||||
/// A root the coordinator has no figures for — never indexed to completion,
|
||||
/// staged in the draft but not yet applied, or an index that was cleared —
|
||||
/// says so rather than claiming zero.
|
||||
fn root_counts_text(state: &IndexerState, root: &str) -> String {
|
||||
match state.root_counts.iter().find(|c| c.root == root) {
|
||||
Some(c) => format!(
|
||||
"indexed {} · extracted {}",
|
||||
group_thousands(c.counts.files.max(0) as u64),
|
||||
group_thousands(c.counts.fts.max(0) as u64)
|
||||
),
|
||||
None => "not yet indexed".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a root to the draft unless it would duplicate or nest with an
|
||||
/// existing one; the rejection reason lands in `error`.
|
||||
fn try_add_root(draft: &mut Config, candidate: String, error: &mut Option<String>) -> bool {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
use super::*;
|
||||
|
||||
use std::cell::RefCell;
|
||||
use std::sync::Arc;
|
||||
|
||||
use quicksearch_core::coordinator::RootCount;
|
||||
use quicksearch_core::db::repo::RootCounts;
|
||||
|
||||
// The widgets under test report themselves here so their identity can be
|
||||
// checked across frames.
|
||||
|
|
@ -32,6 +36,19 @@ fn idle_state() -> IndexerState {
|
|||
queued_events: 0,
|
||||
watcher: WatcherStatus::Active { dirs: 10 },
|
||||
reconcile: None,
|
||||
root_counts: Arc::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// An idle state carrying figures for `/data`, the root `cfg_with_root`
|
||||
/// configures.
|
||||
fn counted_state(files: i64, fts: i64) -> IndexerState {
|
||||
IndexerState {
|
||||
root_counts: Arc::new(vec![RootCount {
|
||||
root: "/data".into(),
|
||||
counts: RootCounts { files, fts },
|
||||
}]),
|
||||
..idle_state()
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -171,6 +188,9 @@ fn the_worker_field_keeps_its_identity_as_the_status_changes() {
|
|||
running_state(&["/data"], Some("/data/file")),
|
||||
running_state(&["/data", "/other"], None),
|
||||
idle_state(),
|
||||
// The per-root figures appear and disappear on the same row as the
|
||||
// field, which is the case a conditionally-drawn label would break.
|
||||
counted_state(1_234_567, 456_789),
|
||||
IndexerState {
|
||||
watcher: WatcherStatus::Off,
|
||||
..idle_state()
|
||||
|
|
@ -315,6 +335,61 @@ fn a_finished_root_reports_its_exact_count_not_the_estimate() {
|
|||
);
|
||||
}
|
||||
|
||||
/// The folder list carries the last completed run's figures, so what a root
|
||||
/// holds survives the run that counted it.
|
||||
#[test]
|
||||
fn a_configured_root_shows_what_the_last_run_counted() {
|
||||
let ctx = egui::Context::default();
|
||||
let mut tab = ManageTab::new();
|
||||
|
||||
let text = frame_text(&ctx, &mut tab, &counted_state(1_234_567, 456_789)).join(" | ");
|
||||
assert!(
|
||||
text.contains("indexed 1,234,567 · extracted 456,789"),
|
||||
"folder row: {}",
|
||||
text
|
||||
);
|
||||
}
|
||||
|
||||
/// A root with no stored figures says so. Zero would be a claim — that the
|
||||
/// folder is empty — where the truth is that nothing has counted it yet.
|
||||
#[test]
|
||||
fn a_root_the_index_has_never_counted_says_so() {
|
||||
let ctx = egui::Context::default();
|
||||
let mut tab = ManageTab::new();
|
||||
|
||||
let text = frame_text(&ctx, &mut tab, &idle_state()).join(" | ");
|
||||
assert!(text.contains("not yet indexed"), "folder row: {}", text);
|
||||
assert!(
|
||||
!text.contains("indexed 0 · extracted 0"),
|
||||
"an uncounted root must not read as an empty one: {}",
|
||||
text
|
||||
);
|
||||
}
|
||||
|
||||
/// Figures are matched to the root by the spelling the config uses, so a
|
||||
/// root the coordinator has not published anything for keeps the placeholder
|
||||
/// rather than borrowing another root's numbers.
|
||||
#[test]
|
||||
fn figures_belong_to_the_root_they_were_counted_for() {
|
||||
let ctx = egui::Context::default();
|
||||
let mut tab = ManageTab::new();
|
||||
let state = IndexerState {
|
||||
root_counts: Arc::new(vec![RootCount {
|
||||
root: "/somewhere-else".into(),
|
||||
counts: RootCounts { files: 99, fts: 9 },
|
||||
}]),
|
||||
..idle_state()
|
||||
};
|
||||
|
||||
let text = frame_text(&ctx, &mut tab, &state).join(" | ");
|
||||
assert!(text.contains("not yet indexed"), "folder row: {}", text);
|
||||
assert!(
|
||||
!text.contains("99"),
|
||||
"borrowed another root's count: {}",
|
||||
text
|
||||
);
|
||||
}
|
||||
|
||||
/// The estimate is shown while the walk runs — but never below what has
|
||||
/// already been walked, or the row would read as a hang at 100%.
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -517,6 +517,20 @@ pub static ROOT_WORKERS: Tip = Tip {
|
|||
caution: None,
|
||||
};
|
||||
|
||||
pub static ROOT_COUNTS: Tip = Tip {
|
||||
title: "What this folder holds",
|
||||
body: "How many files under this folder are in the index, and how many of \
|
||||
those had their text extracted and so can be found by their \
|
||||
contents rather than only by name. The gap between the two is the \
|
||||
files nothing could read text from (images, videos, archives, \
|
||||
program binaries) plus anything the extension whitelist \
|
||||
excludes.\n\n\
|
||||
Both are counted when an indexing run finishes, so they do not \
|
||||
move as live updates apply single changes in between.",
|
||||
examples: &[],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
// --- Manage Index tab: content filters -----------------------------------
|
||||
|
||||
pub static EXT_WHITELIST: Tip = Tip {
|
||||
|
|
@ -610,6 +624,7 @@ mod tests {
|
|||
&ADD_ROOT,
|
||||
&REMOVE_ROOT,
|
||||
&ROOT_WORKERS,
|
||||
&ROOT_COUNTS,
|
||||
&EXT_WHITELIST,
|
||||
&IGNORE_PATTERNS,
|
||||
&APPLY_SAVE,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue