From 208cc900fceb935ac0e86b09994ce70072f70e48 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 17 Aug 2026 22:30:43 -0400 Subject: [PATCH] Fix a flakey test that fails in CI, pruned unused SQLITE table entries from old work. Trimmed binary size by removing unneeded emoji and other EGUI items. LTO performance improvements. --- Cargo.lock | 155 ++-------------- Cargo.toml | 101 +++++++++-- README.md | 25 ++- config_example.toml | 6 + crates/quicksearch-core/Cargo.toml | 10 +- crates/quicksearch-core/src/cli.rs | 61 ++++--- crates/quicksearch-core/src/config/diff.rs | 4 +- crates/quicksearch-core/src/config/mod.rs | 17 ++ crates/quicksearch-core/src/content.rs | 2 - crates/quicksearch-core/src/db/open.rs | 2 +- crates/quicksearch-core/src/db/open_tests.rs | 6 +- crates/quicksearch-core/src/db/repo.rs | 129 ++++++-------- crates/quicksearch-core/src/db/repo_tests.rs | 131 ++++---------- crates/quicksearch-core/src/db/schema.rs | 30 ++-- crates/quicksearch-core/src/extract/audio.rs | 166 ++++++++++++------ crates/quicksearch-core/src/extract/mod.rs | 92 +++++----- crates/quicksearch-core/src/extract/pdf.rs | 166 +++++------------- .../quicksearch-core/src/extract/plaintext.rs | 1 - .../src/file_handling/batch.rs | 33 +--- .../quicksearch-core/src/file_handling/mod.rs | 2 +- .../src/file_handling/paths.rs | 14 -- .../src/file_handling/records.rs | 38 ++-- crates/quicksearch-core/src/incremental.rs | 5 +- .../quicksearch-core/src/indexing/pipeline.rs | 24 +-- crates/quicksearch-core/src/indexing/tests.rs | 3 - crates/quicksearch-core/src/scope.rs | 4 +- crates/quicksearch-core/src/scope_tests.rs | 2 - .../quicksearch-core/src/search/duplicates.rs | 2 - crates/quicksearch-core/src/walk/tests.rs | 5 +- crates/quicksearch-core/tests/cascade.rs | 4 +- crates/quicksearch-core/tests/encrypted.rs | 4 +- crates/quicksearch-core/tests/full_index.rs | 130 +++++++++++--- crates/quicksearch-core/tests/reconcile.rs | 1 - crates/quicksearch-core/tests/search_perf.rs | 4 +- crates/quicksearch-core/tests/snippet_perf.rs | 17 +- crates/quicksearch-gui/Cargo.toml | 15 +- .../assets/fonts/Hack-Regular.ttf | Bin 0 -> 309408 bytes .../assets/fonts/Hack-Regular.txt | 31 ++++ crates/quicksearch-gui/assets/fonts/UFL.txt | 96 ++++++++++ .../assets/fonts/Ubuntu-Light.ttf | Bin 0 -> 361676 bytes crates/quicksearch-gui/src/app/modals.rs | 10 +- .../quicksearch-gui/src/app/security_tests.rs | 10 +- .../quicksearch-gui/src/app/verify_tests.rs | 12 +- .../src/duplicates_tab/tests.rs | 11 +- crates/quicksearch-gui/src/fonts.rs | 119 +++++++++++++ crates/quicksearch-gui/src/help_tab.rs | 23 +++ crates/quicksearch-gui/src/main.rs | 6 + crates/quicksearch-gui/src/manage_tab.rs | 2 +- .../quicksearch-gui/src/manage_tab/tests.rs | 43 ++--- crates/quicksearch-gui/src/search_tab.rs | 4 +- .../quicksearch-gui/src/search_tab/tests.rs | 88 +++++----- .../quicksearch-gui/src/settings_tab/tests.rs | 39 ++-- crates/quicksearch-gui/src/test_ui.rs | 49 +++++- crates/quicksearch-gui/src/tips.rs | 6 +- crates/quicksearch-gui/src/tutorial.rs | 12 +- crates/quicksearch-gui/src/unlock.rs | 6 +- packaging/copyright | 35 +++- 57 files changed, 1155 insertions(+), 858 deletions(-) create mode 100644 crates/quicksearch-gui/assets/fonts/Hack-Regular.ttf create mode 100644 crates/quicksearch-gui/assets/fonts/Hack-Regular.txt create mode 100644 crates/quicksearch-gui/assets/fonts/UFL.txt create mode 100644 crates/quicksearch-gui/assets/fonts/Ubuntu-Light.ttf create mode 100644 crates/quicksearch-gui/src/fonts.rs diff --git a/Cargo.lock b/Cargo.lock index 81b979d..4592138 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1112,7 +1112,6 @@ dependencies = [ "egui", "enum-map", "log", - "mime_guess2", "profiling", ] @@ -1222,7 +1221,6 @@ dependencies = [ "bytemuck", "ecolor", "emath", - "epaint_default_fonts", "log", "nohash-hasher", "parking_lot", @@ -1230,12 +1228,6 @@ dependencies = [ "serde", ] -[[package]] -name = "epaint_default_fonts" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1537accc50c9cab5a272c39300bdd0dd5dca210f6e5e8d70be048df9596e7ca2" - [[package]] name = "equivalent" version = "1.0.1" @@ -2078,15 +2070,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "kamadak-exif" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4fc70d0ab7e5b6bafa30216a6b48705ea964cdfc29c050f2412295eba58077" -dependencies = [ - "mutate_once", -] - [[package]] name = "keyboard-types" version = "0.7.0" @@ -2337,18 +2320,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "mime_guess2" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1706dc14a2e140dec0a7a07109d9a3d5890b81e85bd6c60b906b249a77adf0ca" -dependencies = [ - "mime", - "phf", - "phf_shared", - "unicase", -] - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2381,12 +2352,6 @@ dependencies = [ "pxfm", ] -[[package]] -name = "mutate_once" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af" - [[package]] name = "naga" version = "25.0.1" @@ -2975,50 +2940,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "phf" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" -dependencies = [ - "phf_macros", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" -dependencies = [ - "phf_shared", - "rand 0.8.5", -] - -[[package]] -name = "phf_macros" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" -dependencies = [ - "phf_generator", - "phf_shared", - "proc-macro2", - "quote", - "syn 2.0.66", - "unicase", -] - -[[package]] -name = "phf_shared" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" -dependencies = [ - "siphasher", - "unicase", -] - [[package]] name = "pin-project" version = "1.1.10" @@ -3138,7 +3059,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.13+spec-1.1.0", + "toml_edit", ] [[package]] @@ -3199,7 +3120,6 @@ dependencies = [ "getrandom 0.2.15", "globset", "infer", - "kamadak-exif", "libc", "lofty", "memchr", @@ -3265,15 +3185,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.5" @@ -3621,11 +3532,11 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.6" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -3676,12 +3587,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" -[[package]] -name = "siphasher" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" - [[package]] name = "slab" version = "0.4.9" @@ -3986,23 +3891,17 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "toml" -version = "0.8.2" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "185d8ab0dfbb35cf1399a6344d8484209c088f75f8f68230da55d48d95d43e3d" +checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" dependencies = [ - "serde", + "indexmap", + "serde_core", "serde_spanned", - "toml_datetime 0.6.3", - "toml_edit 0.20.2", -] - -[[package]] -name = "toml_datetime" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" -dependencies = [ - "serde", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow 1.0.4", ] [[package]] @@ -4014,19 +3913,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "toml_edit" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" -dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime 0.6.3", - "winnow 0.5.40", -] - [[package]] name = "toml_edit" version = "0.25.13+spec-1.1.0" @@ -4034,7 +3920,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ "indexmap", - "toml_datetime 1.1.1+spec-1.1.0", + "toml_datetime", "toml_parser", "winnow 1.0.4", ] @@ -4048,6 +3934,12 @@ dependencies = [ "winnow 1.0.4", ] +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + [[package]] name = "tracing" version = "0.1.40" @@ -5008,15 +4900,6 @@ dependencies = [ "xkbcommon-dl", ] -[[package]] -name = "winnow" -version = "0.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" -dependencies = [ - "memchr", -] - [[package]] name = "winnow" version = "0.7.15" diff --git a/Cargo.toml b/Cargo.toml index 9ddd46b..b8252d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,28 +5,91 @@ members = [ "crates/quicksearch-gui", ] -# No `[profile.release]` on purpose. Cross-crate inlining looks like it should -# pay here — the cascade calls into `snippet`/`query`, both into `memchr` and -# `zstd`, everything into `rusqlite`'s FFI wrappers — so it was measured, and -# it does not. Against an incremental rebuild of the GUI after touching core -# (10.2 s at the defaults): -# -# lto=thin 67 s search cold 29.9 ms warm best 14.77 ms index cold 459 ms -# lto=thin,cgu=1 152 s (build cost alone ruled it out) -# lto=fat,cgu=1 200 s search cold 29.2 ms warm best 14.57 ms index cold 479 ms -# (defaults) 10 s search cold 30.0 ms warm best 14.92 ms index cold 478 ms -# -# Every runtime column moves 0-4% and index-cold is not even monotonic, so the -# gain is at the noise floor while the build is 6.6-19.5x longer — on a CI that -# builds three targets. Re-measure before concluding otherwise. -# -# `panic = "abort"` is separately unavailable: `extract/pdf.rs` runs -# `pdf-extract` inside `catch_unwind`, so aborting would turn a malformed PDF -# into a killed process instead of one skipped file. - [workspace.package] version = "1.1.0" edition = "2021" license = "GPL-3.0-or-later" authors = ["Jeremy "] repository = "https://code.karsttech.com/jeremy/quick_search.git" + +# Fat LTO, one codegen unit. This reverses an earlier decision, so both +# measurements are kept: the old one was right about what it measured, and it +# measured the wrong axis on the wrong kind of build. +# +# WHAT WAS MEASURED BEFORE, against an *incremental* rebuild of the GUI after +# touching core (10.2 s at the defaults): +# +# lto=thin 67 s search cold 29.9 ms warm best 14.77 ms index cold 459 ms +# lto=thin,cgu=1 152 s (build cost alone ruled it out) +# lto=fat,cgu=1 200 s search cold 29.2 ms warm best 14.57 ms index cold 479 ms +# (defaults) 10 s search cold 30.0 ms warm best 14.92 ms index cold 478 ms +# +# It concluded that the runtime gain was at the noise floor against a 6.6-19.5x +# longer build. Two things were wrong with that as a decision: +# +# * It never weighed SIZE, which turns out to be where the effect is. +# * A 10 s incremental rebuild is the worst possible denominator for a +# link-time optimization, because the LTO link is nearly the whole cost and +# there is no compile phase to amortize it against. CI never builds that +# way; it builds clean. +# +# RE-MEASURED CLEAN, on 6 cores, stripped `target/*/quicksearch`: +# +# stripped vs def search cold warm best index cold +# (defaults) 28,248,272 - 28.2 ms 14.7 ms 445 ms +# cgu=1 25,909,616 -8.3% 28.6 ms 14.8 ms 452 ms +# lto=thin 28,233,296 -0.1% (not run: no size effect) +# lto=thin,cgu=1 25,991,536 -8.0% (not run: worse than cgu=1 alone) +# lto=fat,cgu=1 24,284,912 -14.0% 27.3 ms 14.1 ms 443 ms +# +# Search is best of three runs; the spreads overlap, so read fat as "3-4% +# faster or even", never as a regression. It is both the fastest and by far the +# smallest, which is why it wins outright. +# +# Two surprises worth keeping. `codegen-units = 1` ALONE is worth 2.3 MB - the +# old table never isolated it, because every cgu=1 row there also carried LTO. +# And `lto = "thin"` alone is worth nothing at all (0.1%), while thin+cgu=1 is +# slightly *worse* than cgu=1 by itself. Thin is not a cheaper fat here; it is +# a different, useless thing. +# +# BUILD COST, clean, 6 cores. The second column is what CI actually pays on top +# of the binaries, since `cargo test --release --workspace` links every test, +# example and bin target - `--release` is `--profile=release`, so this profile +# is theirs too: +# +# bins + all test targets total +# (defaults) 143 s 28 s 171 s +# cgu=1 158 s 41 s 199 s +# lto=fat,cgu=1 198 s 100 s 298 s +# +# So +127 s on a clean CI run, not the tens of minutes the incremental figure +# above implies. Test targets link cheaply because they are small; it is the +# 320-crate dependency compile that dominates, and cgu=1 spreads across cores. +# +# WHAT IS DELIBERATELY NOT SET: +# +# `strip` would save nothing shipped. The packaging scripts and ci.yml strip the +# staged *copy*, which keeps `target/release/` symbolised for `perf` and for the +# RUST_BACKTRACE=1 release-test backtraces CI prints, and keeps `--no-strip` on +# build-deb.sh and build-appimage.sh meaning something. Setting it here would +# move the same 6.5 MB saving to a place where it costs debuggability. +# +# `panic = "abort"` is the largest win left - it would delete `.gcc_except_table` +# (428 KB after LTO) and most of `.eh_frame` (1.33 MB) - and is unavailable for +# two independent reasons. `extract/pdf.rs` runs `pdf-extract` inside +# `catch_unwind`, so aborting would turn a malformed PDF into a killed process +# instead of one skipped file. And cargo forces every dependency to rebuild with +# unwind when building tests under an abort profile, so `cargo test --release` +# would compile the whole graph a second time. +# +# Non-PIE would remove most of `.rela.dyn` (1.37 MB) and is rejected on +# hardening grounds: this program parses arbitrary user PDFs with a crate known +# to panic on malformed input, which is the last place to give up ASLR. +# +# The vendored OpenSSL (~1.3 MB of `.text`, via SQLCipher) is unreachable by any +# profile knob - `openssl-src` pins its own `-O2` - and dropping it would add a +# runtime `libcrypto` dependency, which is exactly what the AppImage's +# bundles-no-libraries invariant forbids. +[profile.release] +lto = "fat" +codegen-units = 1 diff --git a/README.md b/README.md index a54c462..2c2d043 100644 --- a/README.md +++ b/README.md @@ -492,11 +492,15 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. log, because SQLite's own autocheckpoint can only reset the log at an instant no reader holds it — and a run keeps a reader per root querying throughout, so left alone the log grows for the whole run. `files` holds - metadata (name, path, size, mtime, hash, MIME/type bitmask, per-row - index state); `searchabletext` is a *contentless* FTS5 table (postings - only, configurable tokenizer, trigram by default); canonical extracted + metadata (name, path, size, mtime, hash, MIME/type bitmask, content + state); `searchabletext` is a *contentless* FTS5 table over one column, + the document body (postings only, configurable tokenizer, trigram by + default) — filename ranks come from scanning `files.name`, so a `name` + column there would only index the same strings twice; canonical extracted text lives zstd-compressed in `documents_text`, which powers snippets, - occurrence ranking, and fuzzy full-text search. Schema changes wipe and + occurrence ranking, and fuzzy full-text search, and whose uncompressed + length is read back from the zstd frame header rather than stored beside + it. Schema changes wipe and rebuild by policy; the indexer (`open_or_recreate`) is the only code allowed to do that; every consumer uses `open_existing`, which treats drift as an error, never data loss. With password protection on, every @@ -523,11 +527,14 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. classify files by mtime into insert/update/skip, batch-write metadata, sweep stale rows, then extract content (plaintext, RTF, Office — both the OOXML/ODF zip formats and the pre-2007 binary `.doc`/`.xls`/`.ppt`, whose - OLE2 streams are read in `extract/ole.rs` — PDF, audio tags, EXIF; see - `extract/`) for FTS. PDFs are parsed once, with the text and the `Info` - dictionary taken off the same document: the two-parse version that preceded - it was the largest single memory consumer of a run over a PDF-heavy tree, and - it was what pulled a second copy of `lopdf` — and with it rayon's + OLE2 streams are read in `extract/ole.rs` — PDF, audio tags; see + `extract/`) for FTS. Images are claimed by no extractor: the EXIF reader + produced structured properties and never text, and with properties parked + (see `extract::ExtractedContent`) leaving `image/*` unclaimed is what keeps + the content pass from opening every image on disk. PDFs are parsed once: + the two-parse version that preceded it was the largest single memory + consumer of a run over a PDF-heavy tree, and it was what pulled a second + copy of `lopdf` — and with it rayon's never-torn-down thread pool — into the build. That is a claim about PDFs rather than about runs in general, and it is worth knowing which tree a number came from: on one with almost no PDFs, a cold run peaks at 130 MiB diff --git a/config_example.toml b/config_example.toml index 26a34e7..41efad9 100644 --- a/config_example.toml +++ b/config_example.toml @@ -109,6 +109,12 @@ maximum_text_size = 262144 maximum_text_file_size = 2097152 # Files per batch during walks / inserts / extraction. batch_size = 500 +# Writer time one indexing root's turn may take before the round-robin moves +# on (milliseconds). The time half of the knob whose row half is batch_size: +# it bounds how long one root can hold up the others, so a root extracting +# large documents cannot leave another root's walkers parked behind it. 0 +# gives each turn one batch_size quantum and no more. +writer_turn_slice_ms = 100 # Files per transaction for incremental FTS updates. fts_update_batch_size = 1000 # How large the write-ahead log (index.sqlite-wal) may grow during an diff --git a/crates/quicksearch-core/Cargo.toml b/crates/quicksearch-core/Cargo.toml index 90fa28e..69d0947 100644 --- a/crates/quicksearch-core/Cargo.toml +++ b/crates/quicksearch-core/Cargo.toml @@ -35,7 +35,12 @@ quick-xml = "0.31" # nothing new. cfb = "0.7" serde = { version = "1.0", features = ["derive"] } -toml = "0.8" +# 1.x, not 0.8: 0.8 pulls `toml_edit`, the whole format-preserving document +# model, for what is only `from_str` and `to_string_pretty` here. 1.x parses and +# writes as a stream instead - no `toml_edit`, no `winnow` 0.5 alongside the 1.0 +# already in the tree - which is 252 KB of `.text` this crate was paying for a +# document API it never touches. +toml = "1" mime_guess = "2.0" infer = "0.15" # Charset decoding for non-UTF-8 text (UTF-16 .reg exports, legacy @@ -56,7 +61,8 @@ rtf-parser = { version = "0.4", default-features = false } # global thread pool is never torn down), chrono, time, md5 and a second nom. pdf-extract = "0.12" lofty = "0.19" -kamadak-exif = "0.5" +# Parked with `extract::image` — see `extract::ExtractedContent`. +# kamadak-exif = "0.5" notify = "6.1" ctrlc = "3.4" zstd = "0.13" diff --git a/crates/quicksearch-core/src/cli.rs b/crates/quicksearch-core/src/cli.rs index a3b0e36..77c420e 100644 --- a/crates/quicksearch-core/src/cli.rs +++ b/crates/quicksearch-core/src/cli.rs @@ -77,7 +77,6 @@ pub struct FailedEntry { pub struct SizeReport { pub file_size_bytes: u64, pub files_row_count: i64, - pub properties_row_count: i64, pub failed_files_row_count: i64, pub searchabletext_row_count: i64, pub documents_text_row_count: i64, @@ -98,13 +97,20 @@ impl SizeReport { /// Query the per-file indexing status. Returns `FileStatus` with /// `basic == NotIndexed` if the path isn't in the database. +/// +/// There is no stored basic state: a `files` row exists only once its +/// metadata has been read, so the row *is* the basic-indexed state. The +/// failure reason comes from `failed_files`, the one place it is written. pub fn status_for_path(db_path: &str, path: &str) -> Result { let conn = open_existing(db_path, false)?; - let row: Option<(i64, i64, Option)> = conn + let row: Option<(i64, Option)> = conn .query_row( - "SELECT basic_state, content_state, failure_msg FROM files WHERE path = ?1", + "SELECT f.content_state, ff.reason \ + FROM files f \ + LEFT JOIN failed_files ff ON ff.file_id = f.id \ + WHERE f.path = ?1", params![path], - |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + |r| Ok((r.get(0)?, r.get(1)?)), ) .optional() .map_err(|e| format!("status_for_path({}): {}", path, e))?; @@ -115,9 +121,9 @@ pub fn status_for_path(db_path: &str, path: &str) -> Result content: IndexState::NotIndexed, failure_reason: None, }, - Some((basic, content, reason)) => FileStatus { + Some((content, reason)) => FileStatus { path: path.to_string(), - basic: IndexState::from(basic), + basic: IndexState::Done, content: IndexState::from(content), failure_reason: reason, }, @@ -164,18 +170,29 @@ pub fn index_size_breakdown(db_path: &str) -> Result { .map_err(|e| format!("count {}: {}", table, e)) }; let dt_row_count: i64 = count("documents_text")?; - let (dt_raw, dt_compressed): (i64, i64) = conn - .query_row( - "SELECT COALESCE(SUM(text_len), 0), COALESCE(SUM(LENGTH(text_zstd)), 0) FROM documents_text", - [], - |r| Ok((r.get(0)?, r.get(1)?)), - ) + // The uncompressed length is not a column: zstd records it in each frame's + // header, so this reads it back (see `repo::raw_text_len`). Only the + // header is wanted, and 18 bytes is the most one can occupy — projecting + // the prefix keeps this off the document bodies themselves. + let mut stmt = conn + .prepare("SELECT substr(text_zstd, 1, 18), LENGTH(text_zstd) FROM documents_text") + .map_err(|e| format!("documents_text size sum prepare: {}", e))?; + let rows = stmt + .query_map([], |r| Ok((r.get::<_, Vec>(0)?, r.get::<_, i64>(1)?))) .map_err(|e| format!("documents_text size sum: {}", e))?; + let (mut dt_raw, mut dt_compressed) = (0i64, 0i64); + for row in rows { + let (header, compressed) = row.map_err(|e| format!("documents_text size row: {}", e))?; + // A frame with no recorded content size contributes nothing rather + // than skewing the ratio with a guess. + dt_raw += crate::db::repo::raw_text_len(&header).unwrap_or(0) as i64; + dt_compressed += compressed; + } + drop(stmt); Ok(SizeReport { file_size_bytes, files_row_count: count("files")?, - properties_row_count: count("properties")?, failed_files_row_count: count("failed_files")?, searchabletext_row_count: count("searchabletext")?, documents_text_row_count: dt_row_count, @@ -203,7 +220,7 @@ pub fn pending_content_count(db_path: &str) -> Result { } /// Remove a single file from the index. Returns whether a row was deleted. -/// Keeps FTS/documents/properties in sync via the repo helpers. +/// Keeps FTS and `documents_text` in sync via the repo helpers. pub fn clear_path(db_path: &str, path: &str) -> Result { let mut conn = open_existing(db_path, true)?; let tx = conn @@ -239,8 +256,6 @@ mod tests { parent: "/tmp", size: 1, mtime: 1, - inode: None, - device_id: None, mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, @@ -249,7 +264,7 @@ mod tests { ) .unwrap() .expect("unique path"); - set_content_done(&tx, a, "a.txt", "hello", &[], zstd_of("hello").as_deref()).unwrap(); + set_content_done(&tx, a, "hello", zstd_of("hello").as_deref()).unwrap(); let b = insert_file( &tx, &NewFile { @@ -258,8 +273,6 @@ mod tests { parent: "/tmp", size: 1, mtime: 1, - inode: None, - device_id: None, mime: None, ftype: FileType::EMPTY, hash: None, @@ -334,8 +347,6 @@ mod tests { parent: "/tmp", size: 1, mtime: 1, - inode: None, - device_id: None, mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, @@ -354,8 +365,6 @@ mod tests { parent: "/tmp", size: 1, mtime: 1, - inode: None, - device_id: None, mime: None, ftype: FileType::EMPTY, hash: None, @@ -417,8 +426,6 @@ mod tests { parent: "/tmp", size: 1, mtime: 1, - inode: None, - device_id: None, mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, @@ -428,7 +435,7 @@ mod tests { .unwrap() .expect("unique path"); let prose = "the quick brown fox jumps over the lazy dog. ".repeat(500); - set_content_done(&tx, id, "big.txt", &prose, &[], zstd_of(&prose).as_deref()).unwrap(); + set_content_done(&tx, id, &prose, zstd_of(&prose).as_deref()).unwrap(); tx.commit().unwrap(); } drop(conn); @@ -479,8 +486,6 @@ mod tests { parent: "/tmp", size: 1, mtime: 1, - inode: None, - device_id: None, mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, diff --git a/crates/quicksearch-core/src/config/diff.rs b/crates/quicksearch-core/src/config/diff.rs index 5e800fe..e495cac 100644 --- a/crates/quicksearch-core/src/config/diff.rs +++ b/crates/quicksearch-core/src/config/diff.rs @@ -31,8 +31,8 @@ pub struct IndexWork { pub drop_aliases: bool, /// The `content_extensions` filter changed. Kept rows are re-tested /// against it in both directions: newly-included files go back to - /// pending, newly-excluded ones give up their text, properties and FTS - /// row but keep the name/path row that filename search needs. + /// pending, newly-excluded ones give up their text and FTS row but keep + /// the name/path row that filename search needs. pub reconcile_content: bool, /// `store_text_for_snippets` turned on. Rows that finished extraction /// under the old setting kept no text, so they must run again. diff --git a/crates/quicksearch-core/src/config/mod.rs b/crates/quicksearch-core/src/config/mod.rs index e2bb513..b2bb4b7 100644 --- a/crates/quicksearch-core/src/config/mod.rs +++ b/crates/quicksearch-core/src/config/mod.rs @@ -116,6 +116,22 @@ pub struct ProcessingConfig { pub maximum_text_size: usize, pub maximum_text_file_size: u64, pub batch_size: usize, + /// Writer time one root's turn may take before the round moves on, in + /// milliseconds. The time half of the round-robin whose row half is + /// `batch_size`, and so the bound on how long any one root can hold up + /// the others. + /// + /// Before there was one, an extraction turn ran to the end of whatever + /// was ready — half a second to two seconds of FTS5 trigram tokenization + /// for a batch of large documents — while a walking root's rows sat in + /// its channel and its walkers parked behind them. Reads as "4/4 workers + /// busy, no progress". + /// + /// `0` gives each turn one `batch_size` quantum and no more, which is + /// the finest the round-robin goes; the tests that count work per round + /// use small values here so a phase cannot begin and end between two + /// status snapshots. + pub writer_turn_slice_ms: u64, pub fts_update_batch_size: usize, /// How large the write-ahead log may grow during a run before the indexer /// forces a checkpoint, in bytes. `0` disables forced checkpoints; @@ -258,6 +274,7 @@ impl Default for ProcessingConfig { maximum_text_size: 1024 * 256, maximum_text_file_size: 1024 * 1024 * 2, batch_size: 500, + writer_turn_slice_ms: 100, fts_update_batch_size: 1000, maximum_wal_size: 1024 * 1024 * 512, tokenize: "trigram".to_string(), diff --git a/crates/quicksearch-core/src/content.rs b/crates/quicksearch-core/src/content.rs index 009d086..7cf908a 100644 --- a/crates/quicksearch-core/src/content.rs +++ b/crates/quicksearch-core/src/content.rs @@ -400,8 +400,6 @@ mod tests { parent: d.to_str().unwrap(), size: std::fs::metadata(&f).unwrap().len(), mtime: 1, - inode: None, - device_id: None, mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, diff --git a/crates/quicksearch-core/src/db/open.rs b/crates/quicksearch-core/src/db/open.rs index 17002fa..0e7ca46 100644 --- a/crates/quicksearch-core/src/db/open.rs +++ b/crates/quicksearch-core/src/db/open.rs @@ -25,7 +25,7 @@ pub const KEY_MISMATCH_PREFIX: &str = "KEY_MISMATCH: "; /// values go stale: `files.mime`, `files.type` and `content_state` are /// computed at walk time and never re-derived for unchanged files, so a /// classification change needs the wipe to apply everywhere. -pub const CURRENT_SCHEMA_VERSION: u32 = 6; +pub const CURRENT_SCHEMA_VERSION: u32 = 7; /// Open `db_path` and ensure the on-disk schema matches this build; if it /// doesn't (including a changed `tokenizer`), delete the file and recreate it diff --git a/crates/quicksearch-core/src/db/open_tests.rs b/crates/quicksearch-core/src/db/open_tests.rs index a533cc5..d68e1c2 100644 --- a/crates/quicksearch-core/src/db/open_tests.rs +++ b/crates/quicksearch-core/src/db/open_tests.rs @@ -102,7 +102,7 @@ fn legacy_layout_db_is_wiped_and_recreated() { // New columns should exist (just prepare the SELECT — an // unknown column name would parse-error here). conn.query_row( - "SELECT basic_state, content_state, type, mime FROM files LIMIT 0", + "SELECT content_state, type, mime FROM files LIMIT 0", [], |_| Ok(()), ) @@ -172,8 +172,8 @@ fn open_existing_reads_nondefault_tokenizer_without_wiping() { // Seed the FTS index (rowid = the files row we just inserted) so a // MATCH query can be exercised against the on-disk tokenizer. conn.execute( - "INSERT INTO searchabletext (rowid, name, text, properties) \ - VALUES (last_insert_rowid(), 'note', 'hello world', '')", + "INSERT INTO searchabletext (rowid, text) \ + VALUES (last_insert_rowid(), 'hello world')", [], ) .unwrap(); diff --git a/crates/quicksearch-core/src/db/repo.rs b/crates/quicksearch-core/src/db/repo.rs index 165703f..ff8feb5 100644 --- a/crates/quicksearch-core/src/db/repo.rs +++ b/crates/quicksearch-core/src/db/repo.rs @@ -1,7 +1,7 @@ //! Row-level write helpers that keep the FTS5 contentless table in sync with -//! `files`/`documents`/`properties`. +//! `files`/`documents_text`. //! -//! States (mirrors `basic_state` / `content_state` columns): +//! States (mirrors the `content_state` column): //! //! | value | meaning | //! |------:|---------| @@ -45,7 +45,7 @@ fn set_state_clearing_failure( ) -> Result<(), String> { exec( tx, - "UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2", + "UPDATE files SET content_state = ?1 WHERE id = ?2", params![state, file_id], || format!("{} content_state {}", transition, file_id), )?; @@ -66,8 +66,6 @@ pub struct NewFile<'a> { pub parent: &'a str, pub size: u64, pub mtime: u64, - pub inode: Option, - pub device_id: Option, pub mime: Option<&'a str>, pub ftype: FileType, pub hash: Option<&'a [u8]>, @@ -76,17 +74,16 @@ pub struct NewFile<'a> { pub needs_content: bool, } -/// Insert a new file row, returning its id. `basic_state` is set to DONE -/// (the row existing *is* the basic-index state); `content_state` comes from -/// `needs_content`. `INSERT OR IGNORE`: a UNIQUE(path) collision returns -/// `None` rather than aborting the batch. +/// Insert a new file row, returning its id. `content_state` comes from +/// `needs_content`; there is no separate basic state, because the row +/// existing *is* the basic-index state. `INSERT OR IGNORE`: a UNIQUE(path) +/// collision returns `None` rather than aborting the batch. pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result, String> { let rows = tx .prepare_cached( "INSERT OR IGNORE INTO files ( - name, path, parent, size, mtime, inode, device_id, - mime, type, basic_state, content_state, hash - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", + name, path, parent, size, mtime, mime, type, content_state, hash + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", ) .and_then(|mut stmt| { stmt.execute(params![ @@ -95,11 +92,8 @@ pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result, f.parent, f.size as i64, f.mtime as i64, - f.inode.map(|x| x as i64), - f.device_id.map(|x| x as i64), f.mime, f.ftype.bits() as i64, - STATE_DONE, initial_content_state(f), f.hash, ]) @@ -125,14 +119,14 @@ fn initial_content_state(f: &NewFile<'_>) -> i64 { /// Update a file's metadata in place (same path, changed size/mtime/hash) and /// reset its content state from `f.needs_content`, clearing any extracted /// content so the text-indexing pass re-processes it. Writes `size`, `mtime`, -/// `hash`, `mime`, `type`, `content_state` and `failure_msg` — and only -/// those; `name`, `parent`, `inode` and `device_id` are not refreshed here. +/// `hash`, `mime`, `type` and `content_state` — and only those; `name` and +/// `parent` are not refreshed here. pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result, String> { let id: Option = tx .prepare_cached( "UPDATE files SET size = ?1, mtime = ?2, hash = ?3, mime = ?4, type = ?5, - content_state = ?6, failure_msg = NULL + content_state = ?6 WHERE path = ?7 RETURNING id", ) @@ -159,13 +153,11 @@ pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result