diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 41db656..3fb8887 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -63,12 +63,6 @@ jobs: HOME: /root # The highest libc6 version the .deb is allowed to require. MAX_GLIBC: '2.35' - # full_index.rs asserts a heavy indexing root cannot stall a light one, and - # measures that as wall-clock stall. The 100 ms default is calibrated on a - # developer machine; this runner measured 188 ms for the same correct - # behaviour. 600 ms keeps the check meaningful - the regression it exists to - # catch is ~6x the healthy figure, so it would land near 1.2 s here. - QSB_STALL_BUDGET_MS: '600' steps: - uses: actions/checkout@v4 @@ -99,6 +93,10 @@ jobs: # for before it will run. # libcap2-bin provides capsh, which the Test step uses to drop the two # DAC capabilities so root obeys permission bits. + # jq parses the tags API for the releasable check below. Named rather + # than assumed: the release job gets it from its image, and a base + # image change that dropped it would turn that check into a silent + # pass rather than a failure. # zsync and appstream are for build-appimage.sh: appimagetool shells # out to zsyncmake rather than bundling it, and reports success while # writing nothing when it is absent, so the script checks for it up @@ -107,7 +105,44 @@ jobs: apt-get install -y --no-install-recommends \ build-essential perl pkg-config \ binutils dpkg-dev desktop-file-utils gzip libcap2-bin \ - zsync appstream + zsync appstream jq + + - name: Check the release tag is free + # The other half of the guard above, for the path that actually cuts + # most releases: pushing a Release* branch. There the tag comes from + # [workspace.package] rather than the ref, so the mistake is not a + # mismatched tag but a *forgotten bump* — the version still points at a + # release that already shipped. + # + # The release job checks this too and remains the authority; it just + # cannot check it until both build jobs are green, so forgetting the + # bump used to cost two full release builds, packaging and an artifact + # upload before anything said so. This says so in seconds. + # + # Deliberately as lenient as the release job: a tag at *this* commit is + # a re-run and fine, and an unreachable API reads as "not found" and + # lets the build proceed rather than failing on a network hiccup. An + # early check that blocks a good release is worse than one that misses + # a bad one, because the late check still catches it. + if: >- + startsWith(github.ref, 'refs/heads/Release') + || startsWith(github.ref, 'refs/heads/release') + env: + RELEASE_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + version=$(sed -n '/^\[workspace\.package\]/,/^\[/{ s/^version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p }' Cargo.toml) + [ -n "$version" ] || { echo "could not read the version from Cargo.toml" >&2; exit 1; } + tag="v$version" + api="${GITHUB_API_URL:-$GITHUB_SERVER_URL/api/v1}" + at=$(curl -sS -H "Authorization: token $RELEASE_TOKEN" \ + "$api/repos/$GITHUB_REPOSITORY/tags/$tag" | jq -r '.commit.sha // empty') + if [ -n "$at" ] && [ "$at" != "$GITHUB_SHA" ]; then + echo "ERROR: $tag already exists at $at, not $GITHUB_SHA." >&2 + echo "Bump [workspace.package] version in Cargo.toml, refresh Cargo.lock" >&2 + echo "with 'cargo update -w', commit and push again." >&2 + exit 1 + fi + echo "OK: $tag is free (or already at this commit)" - name: Trust the workspace # checkout writes as root into a directory git then considers dubiously @@ -168,7 +203,23 @@ jobs: echo "capabilities in test shell: $(grep CapEff /proc/self/status | tr -d "\t")" cargo test --release --locked --workspace' + # Everything from here down produces release assets, so it runs only + # where a release can actually come out: a v* tag or a Release* branch. + # On master and pull requests the job stops after Build and Test, which + # is what those runs are for — the packaging that used to follow built a + # .deb, an AppImage, a tarball and a 14-day artifact upload that nothing + # would ever download, because the release job is skipped there anyway. + # + # The condition is repeated rather than hoisted into an env var: Actions + # has no workflow-level expression alias, YAML anchors are not supported, + # and `env.X` inside `if:` would fail *closed* on a runner that did not + # populate it — silently skipping packaging on a real release. Spelled + # out, it is the same form the release job's own gate uses. - name: Build the .deb + if: >- + startsWith(github.ref, 'refs/tags/v') + || startsWith(github.ref, 'refs/heads/Release') + || startsWith(github.ref, 'refs/heads/release') # --no-build reuses the binaries from the Build step rather than # recompiling. SOURCE_DATE_EPOCH pins the generated changelog date so # repeat builds of the same commit are byte-identical. @@ -177,8 +228,14 @@ jobs: ./packaging/build-deb.sh --no-build - name: Check the glibc floor + if: >- + startsWith(github.ref, 'refs/tags/v') + || startsWith(github.ref, 'refs/heads/Release') + || startsWith(github.ref, 'refs/heads/release') # The whole point of pinning the container. If someone bumps the image, # this fails loudly instead of quietly shipping an uninstallable package. + # Gated with the .deb it inspects — there is no package to read without + # the step above. run: | deb=$(ls dist/*.deb) depends=$(dpkg-deb -f "$deb" Depends) @@ -194,6 +251,10 @@ jobs: echo "OK: glibc floor $floor <= $MAX_GLIBC" - name: Build the AppImage + if: >- + startsWith(github.ref, 'refs/tags/v') + || startsWith(github.ref, 'refs/heads/Release') + || startsWith(github.ref, 'refs/heads/release') # After the glibc gate, so the cheaper check still fails first. --no-build # reuses the binaries from the Build step, as the .deb step does, and # SOURCE_DATE_EPOCH pins the date substituted into the AppStream release @@ -207,6 +268,10 @@ jobs: ./packaging/build-appimage.sh --no-build - name: Package the binaries + if: >- + startsWith(github.ref, 'refs/tags/v') + || startsWith(github.ref, 'refs/heads/Release') + || startsWith(github.ref, 'refs/heads/release') # A tarball for anyone not installing the .deb, stripped to match what # build-deb.sh ships. run: | @@ -231,6 +296,12 @@ jobs: # node16, which current Forgejo runner images no longer ship; the # -node20 tags are Forgejo's builds for precisely this combination. - uses: actions/upload-artifact@v3-node20 + # `if-no-files-found: error` below would fail every master push once the + # packaging steps above are gated, so this carries the same gate. + if: >- + startsWith(github.ref, 'refs/tags/v') + || startsWith(github.ref, 'refs/heads/Release') + || startsWith(github.ref, 'refs/heads/release') with: name: linux-x86_64 # The .zsync is not optional: the update URL baked into every AppImage @@ -322,7 +393,16 @@ jobs: fi done + # As in the linux job, the asset-producing steps run only where a release + # can come out. The DLL check above deliberately stays ungated: it + # validates the .exe itself rather than packaging it, costs an objdump, + # and is exactly the kind of regression worth catching on master rather + # than at release time. - name: Build the installer + if: >- + startsWith(github.ref, 'refs/tags/v') + || startsWith(github.ref, 'refs/heads/Release') + || startsWith(github.ref, 'refs/heads/release') # --no-build reuses the binaries from the Build step rather than # cross-compiling them a second time. The installer and the .zip below # are alternatives, not a two-step download: the installer puts the app @@ -332,6 +412,10 @@ jobs: run: ./packaging/build-installer.sh --no-build - name: Package the binaries + if: >- + startsWith(github.ref, 'refs/tags/v') + || startsWith(github.ref, 'refs/heads/Release') + || startsWith(github.ref, 'refs/heads/release') run: | version=$(sed -n '/^\[workspace\.package\]/,/^\[/{ s/^version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p }' Cargo.toml) [ -n "$version" ] || { echo "could not read the version from Cargo.toml" >&2; exit 1; } @@ -354,6 +438,12 @@ jobs: # node16, which current Forgejo runner images no longer ship; the # -node20 tags are Forgejo's builds for precisely this combination. - uses: actions/upload-artifact@v3-node20 + # Same gate as the packaging steps, for the same `if-no-files-found` + # reason as the linux job. + if: >- + startsWith(github.ref, 'refs/tags/v') + || startsWith(github.ref, 'refs/heads/Release') + || startsWith(github.ref, 'refs/heads/release') with: name: windows-x86_64 path: | diff --git a/Cargo.lock b/Cargo.lock index 9c4fab1..db2027a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1112,7 +1112,6 @@ dependencies = [ "egui", "enum-map", "log", - "mime_guess2", "profiling", ] @@ -1222,7 +1221,6 @@ dependencies = [ "bytemuck", "ecolor", "emath", - "epaint_default_fonts", "log", "nohash-hasher", "parking_lot", @@ -1230,12 +1228,6 @@ dependencies = [ "serde", ] -[[package]] -name = "epaint_default_fonts" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1537accc50c9cab5a272c39300bdd0dd5dca210f6e5e8d70be048df9596e7ca2" - [[package]] name = "equivalent" version = "1.0.1" @@ -2078,15 +2070,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "kamadak-exif" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4fc70d0ab7e5b6bafa30216a6b48705ea964cdfc29c050f2412295eba58077" -dependencies = [ - "mutate_once", -] - [[package]] name = "keyboard-types" version = "0.7.0" @@ -2337,18 +2320,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "mime_guess2" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1706dc14a2e140dec0a7a07109d9a3d5890b81e85bd6c60b906b249a77adf0ca" -dependencies = [ - "mime", - "phf", - "phf_shared", - "unicase", -] - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2381,12 +2352,6 @@ dependencies = [ "pxfm", ] -[[package]] -name = "mutate_once" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13d2233c9842d08cfe13f9eac96e207ca6a2ea10b80259ebe8ad0268be27d2af" - [[package]] name = "naga" version = "25.0.1" @@ -2955,8 +2920,6 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pdf-extract" version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "417e8fdc940f1d5bc62c5f89864c3a2255f74f69aa353c98509213d67df61e73" dependencies = [ "adobe-cmap-parser", "cff-parser", @@ -2975,50 +2938,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "phf" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" -dependencies = [ - "phf_macros", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" -dependencies = [ - "phf_shared", - "rand 0.8.5", -] - -[[package]] -name = "phf_macros" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" -dependencies = [ - "phf_generator", - "phf_shared", - "proc-macro2", - "quote", - "syn 2.0.66", - "unicase", -] - -[[package]] -name = "phf_shared" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" -dependencies = [ - "siphasher", - "unicase", -] - [[package]] name = "pin-project" version = "1.1.10" @@ -3138,7 +3057,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.13+spec-1.1.0", + "toml_edit", ] [[package]] @@ -3188,7 +3107,7 @@ dependencies = [ [[package]] name = "quicksearch-core" -version = "1.0.6" +version = "1.1.0" dependencies = [ "argon2", "cfb", @@ -3199,7 +3118,6 @@ dependencies = [ "getrandom 0.2.15", "globset", "infer", - "kamadak-exif", "libc", "lofty", "memchr", @@ -3222,7 +3140,7 @@ dependencies = [ [[package]] name = "quicksearch-gui" -version = "1.0.6" +version = "1.1.0" dependencies = [ "ashpd", "chrono", @@ -3265,15 +3183,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.5" @@ -3621,11 +3530,11 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.6" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -3676,12 +3585,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" -[[package]] -name = "siphasher" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" - [[package]] name = "slab" version = "0.4.9" @@ -3986,23 +3889,17 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "toml" -version = "0.8.2" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "185d8ab0dfbb35cf1399a6344d8484209c088f75f8f68230da55d48d95d43e3d" +checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" dependencies = [ - "serde", + "indexmap", + "serde_core", "serde_spanned", - "toml_datetime 0.6.3", - "toml_edit 0.20.2", -] - -[[package]] -name = "toml_datetime" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" -dependencies = [ - "serde", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow 1.0.4", ] [[package]] @@ -4014,19 +3911,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "toml_edit" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "396e4d48bbb2b7554c944bde63101b5ae446cff6ec4a24227428f15eb72ef338" -dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime 0.6.3", - "winnow 0.5.40", -] - [[package]] name = "toml_edit" version = "0.25.13+spec-1.1.0" @@ -4034,7 +3918,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ "indexmap", - "toml_datetime 1.1.1+spec-1.1.0", + "toml_datetime", "toml_parser", "winnow 1.0.4", ] @@ -4048,6 +3932,12 @@ dependencies = [ "winnow 1.0.4", ] +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + [[package]] name = "tracing" version = "0.1.40" @@ -5008,15 +4898,6 @@ dependencies = [ "xkbcommon-dl", ] -[[package]] -name = "winnow" -version = "0.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" -dependencies = [ - "memchr", -] - [[package]] name = "winnow" version = "0.7.15" diff --git a/Cargo.toml b/Cargo.toml index 085e79a..9b2677c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,29 +4,117 @@ members = [ "crates/quicksearch-core", "crates/quicksearch-gui", ] +# `vendor/pdf-extract` is deliberately NOT a member: it is a third-party crate +# carried here for one patch, not part of this workspace's lints, tests or +# release profile. `[patch.crates-io]` below is what makes the dependency graph +# resolve to it. +exclude = ["vendor/pdf-extract"] -# No `[profile.release]` on purpose. Cross-crate inlining looks like it should -# pay here — the cascade calls into `snippet`/`query`, both into `memchr` and -# `zstd`, everything into `rusqlite`'s FFI wrappers — so it was measured, and -# it does not. Against an incremental rebuild of the GUI after touching core -# (10.2 s at the defaults): +# pdf-extract 0.12.0, with the two unbounded recursions in it bounded. +# +# `get_inherited` follows `/Parent` and `process_stream`'s `Do` arm follows +# Form XObjects, neither with a depth counter or a visited set. A page whose +# `/Parent` is itself, or an XObject whose content stream draws itself, walks +# the stack until it hits the guard page — and a stack overflow is not a panic +# that `catch_unwind` can contain (`extract/pdf.rs` has one, for the parser's +# ordinary panics): Rust's handler calls `abort()`, so a ~600-byte file kills +# the process. It recurs on every run, because the row keeps +# `content_state = 0` and the feeder selects exactly those; and the live +# watcher re-extracts on-screen rows on the GUI thread, so such a file crashes +# the app when it merely appears in a result list. +# +# Vendored rather than forked-by-URL so the build stays offline, `--locked` +# keeps meaning what it means, and the cross-compile job needs no new host. +# The patch is marked LOCAL PATCH in the source and is upstreamable; the crate +# is MIT and the copy is recorded in `packaging/copyright`. +[patch.crates-io] +pdf-extract = { path = "vendor/pdf-extract" } + +[workspace.package] +version = "1.1.1" +edition = "2021" +license = "GPL-3.0-or-later" +authors = ["Jeremy "] +repository = "https://code.karsttech.com/jeremy/quick_search.git" + +# Fat LTO, one codegen unit. This reverses an earlier decision, so both +# measurements are kept: the old one was right about what it measured, and it +# measured the wrong axis on the wrong kind of build. +# +# WHAT WAS MEASURED BEFORE, against an *incremental* rebuild of the GUI after +# touching core (10.2 s at the defaults): # # lto=thin 67 s search cold 29.9 ms warm best 14.77 ms index cold 459 ms # lto=thin,cgu=1 152 s (build cost alone ruled it out) # lto=fat,cgu=1 200 s search cold 29.2 ms warm best 14.57 ms index cold 479 ms # (defaults) 10 s search cold 30.0 ms warm best 14.92 ms index cold 478 ms # -# Every runtime column moves 0-4% and index-cold is not even monotonic, so the -# gain is at the noise floor while the build is 6.6-19.5x longer — on a CI that -# builds three targets. Re-measure before concluding otherwise. +# It concluded that the runtime gain was at the noise floor against a 6.6-19.5x +# longer build. Two things were wrong with that as a decision: # -# `panic = "abort"` is separately unavailable: `extract/pdf.rs` runs -# `pdf-extract` inside `catch_unwind`, so aborting would turn a malformed PDF -# into a killed process instead of one skipped file. - -[workspace.package] -version = "1.0.6" -edition = "2021" -license = "GPL-3.0-or-later" -authors = ["Jeremy "] -repository = "https://code.karsttech.com/jeremy/quick_search.git" +# * It never weighed SIZE, which turns out to be where the effect is. +# * A 10 s incremental rebuild is the worst possible denominator for a +# link-time optimization, because the LTO link is nearly the whole cost and +# there is no compile phase to amortize it against. CI never builds that +# way; it builds clean. +# +# RE-MEASURED CLEAN, on 6 cores, stripped `target/*/quicksearch`: +# +# stripped vs def search cold warm best index cold +# (defaults) 28,248,272 - 28.2 ms 14.7 ms 445 ms +# cgu=1 25,909,616 -8.3% 28.6 ms 14.8 ms 452 ms +# lto=thin 28,233,296 -0.1% (not run: no size effect) +# lto=thin,cgu=1 25,991,536 -8.0% (not run: worse than cgu=1 alone) +# lto=fat,cgu=1 24,284,912 -14.0% 27.3 ms 14.1 ms 443 ms +# +# Search is best of three runs; the spreads overlap, so read fat as "3-4% +# faster or even", never as a regression. It is both the fastest and by far the +# smallest, which is why it wins outright. +# +# Two surprises worth keeping. `codegen-units = 1` ALONE is worth 2.3 MB - the +# old table never isolated it, because every cgu=1 row there also carried LTO. +# And `lto = "thin"` alone is worth nothing at all (0.1%), while thin+cgu=1 is +# slightly *worse* than cgu=1 by itself. Thin is not a cheaper fat here; it is +# a different, useless thing. +# +# BUILD COST, clean, 6 cores. The second column is what CI actually pays on top +# of the binaries, since `cargo test --release --workspace` links every test, +# example and bin target - `--release` is `--profile=release`, so this profile +# is theirs too: +# +# bins + all test targets total +# (defaults) 143 s 28 s 171 s +# cgu=1 158 s 41 s 199 s +# lto=fat,cgu=1 198 s 100 s 298 s +# +# So +127 s on a clean CI run, not the tens of minutes the incremental figure +# above implies. Test targets link cheaply because they are small; it is the +# 320-crate dependency compile that dominates, and cgu=1 spreads across cores. +# +# WHAT IS DELIBERATELY NOT SET: +# +# `strip` would save nothing shipped. The packaging scripts and ci.yml strip the +# staged *copy*, which keeps `target/release/` symbolised for `perf` and for the +# RUST_BACKTRACE=1 release-test backtraces CI prints, and keeps `--no-strip` on +# build-deb.sh and build-appimage.sh meaning something. Setting it here would +# move the same 6.5 MB saving to a place where it costs debuggability. +# +# `panic = "abort"` is the largest win left - it would delete `.gcc_except_table` +# (428 KB after LTO) and most of `.eh_frame` (1.33 MB) - and is unavailable for +# two independent reasons. `extract/pdf.rs` runs `pdf-extract` inside +# `catch_unwind`, so aborting would turn a malformed PDF into a killed process +# instead of one skipped file. And cargo forces every dependency to rebuild with +# unwind when building tests under an abort profile, so `cargo test --release` +# would compile the whole graph a second time. +# +# Non-PIE would remove most of `.rela.dyn` (1.37 MB) and is rejected on +# hardening grounds: this program parses arbitrary user PDFs with a crate known +# to panic on malformed input, which is the last place to give up ASLR. +# +# The vendored OpenSSL (~1.3 MB of `.text`, via SQLCipher) is unreachable by any +# profile knob - `openssl-src` pins its own `-O2` - and dropping it would add a +# runtime `libcrypto` dependency, which is exactly what the AppImage's +# bundles-no-libraries invariant forbids. +[profile.release] +lto = "fat" +codegen-units = 1 diff --git a/README.md b/README.md index c2f5508..2c2d043 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,9 @@ The install is per-machine and asks for elevation. Into | `uninstall.exe` | written by the installer; Add/Remove Programs runs it | The components page offers a Start menu shortcut (on) and a desktop shortcut -(off); both are created for all users. No `config.toml` is installed, for the +(off); both are created for all users. The final page lists what was installed +and where — the install itself takes about a second, which without saying so +reads as a failure — and offers to start QuickSearch, ticked. No `config.toml` is installed, for the same reason the `.deb` ships none — one next to the binaries is portable mode (see [Configuration](#configuration)) and would override the personal config of every account. The app writes `%APPDATA%\quicksearch\config.toml` on first @@ -224,12 +226,32 @@ inside that folder. `quicksearch` with no query arguments opens the app: - **Search**: results appear as you type; every keystroke cancels the - previous search. One checkbox enables the two fuzzy passes. Sort by - rank, name, path, size, or modified. Double-click a result to open it; + previous search. One checkbox enables the two fuzzy passes, and once a + search has finished a button inside the right of the search box re-runs + it. Click a column heading to sort by it; **right-click any heading to + choose which columns are shown** — the path is always there, and size + and modified date start hidden, which is what buys the width the path + and the match get instead. The choice is saved (`[search.columns]`, + also in Settings → Search) and applies immediately. Sorting by a column + you then hide falls back to rank. Double-click a result to open it; right-click it to reveal it in the file manager, open it, copy its path, or build an ignore filter from it (session-only by default, optionally persisted to the config). Result text can be selected and - copied in place. Matches in file contents show highlighted snippets. + copied in place. A match in a file's **name** or **path** is + highlighted in that column; a match in its **contents** shows a + highlighted snippet in the Content Match column, with more of the + surrounding text on hover. Rows matched on name or path show a dash + there instead. With `[search] live_results` on (the default) the rows + actually on screen are watched, and what they show is read from the + files themselves: a rename, a deletion or an edit lands within a + second, whether or not indexing is running. The rows coming on screen + are also checked against the disk as they are watched, so one the index + was already out of date about corrects itself; the index is then + brought back in line for those files alone. Over a network share, where + the system reports no events, that check is all you get — the row is + right when it comes on screen and then holds still. Nothing is ever + added, removed or re-ordered underneath you; a file that disappears is + struck through where it sits. Editing the query drops every watch. - **Manage Index**: full indexing status, Start/Stop/Automatic controls, indexed folder list, full-text extension filters, ignore patterns, and the indexing options. Stopping switches to manual mode and saves that @@ -244,25 +266,45 @@ inside that folder. folder nothing has finished indexing reads "not yet indexed" rather than zero, and because the figures come from completed runs they do not move as live updates apply single changes in between. -- **Duplicates**: files sharing a content hash, grouped. +- **Duplicates**: files sharing a content hash, grouped. That hash covers + each file's size and its first `processing.hash_length` bytes and nothing + else, which is the whole reason indexing is affordable — and the reason a + group is a strong suspicion rather than a fact. Right-click a group, or any + file in one, to settle it: every member is read through and compared byte + for byte, with progress and a Cancel button in a modal that then names each + file as identical, differing at a given byte, a different size, or + unreadable. Nothing is deleted or changed either way; the point is to know + before you delete something yourself. - **Logs**: the lines the app would have printed to a terminal — warnings from indexing, folder watching and opening files, newest last, with a filter box and Copy button. Launched from a desktop launcher (or on Windows, where the app has no console at all) this is the only place they are visible. - **Help**: an in-app quickstart — first indexing run, example queries, - what each tab does — pointing here for everything technical. + what each tab does — pointing here for everything technical. A brand-new + installation is shown a short click-through introduction covering the + same ground on its first launch; the Help tab brings it back. Upgrading + into this version does not raise it (see `[ui] tutorial_seen`). +- **Settings**: every configuration control in one place — the database + path, indexing and processing limits, search behaviour, the interface + (scale, shortcut, color scheme) and password protection. Each row + explains itself on hover. Edits are staged and applied together by + **Apply & Save**; leaving the tab with unapplied edits asks first. The + column choices and the password controls are the exceptions, acting the + moment they are used, since the Search tab's own header menu writes the + same settings. The indexed folder list and the indexing mode live on + Manage Index instead, next to the controls that act on them. **Ctrl+Shift+F from anywhere** brings QuickSearch to the front, restoring it if it was minimized, and puts the cursor in the search box with the previous search selected, so the next thing you type is the new one. The -Options window's Interface section rebinds it — click the button and press +Settings tab's Interface section rebinds it — click the button and press the keys — or switches it off. It is a system-wide shortcut, registered with Windows or with the X server, so it works while another application has focus. Wayland does not let an application claim a key, so there the shortcut is registered with your desktop through the XDG desktop portal instead; your desktop then has the final say over which key it is, and its -own keyboard settings are where to change it. The Options window says which +own keyboard settings are where to change it. The Settings tab says which key it settled on. Wayland likewise gives no application a way to put itself in front of what you are doing, so under it the shortcut selects the Search tab and the search box but leaves raising the window to the desktop; on X11 @@ -301,7 +343,7 @@ processing; Windows Terminal has it, and older consoles get plain text. The index contains the names and (by default) the full text of everything it indexes — for most setups, your entire home directory. That is a lot of -concentrated risk in one file. **Options → Security → Enable password +concentrated risk in one file. **Settings → Security → Enable password protection** encrypts the index on disk with SQLCipher; from then on QuickSearch asks for the password every time it starts, in the GUI (an unlock screen before anything opens the index) and in the terminal (a @@ -315,6 +357,10 @@ rebuilds the index — there is no in-place conversion. in the OS keychain — Secret Service/KWallet on Linux, Credential Manager on Windows — and skips the prompt. Without a keychain daemon the option quietly falls back to prompting. +- **Show database key** asks for the password, then shows the raw SQLCipher + key as `0x…` (64 hex digits) with a copy button, for opening the index in + other SQLCipher tools. That key alone reads the index, so treat a copy of + it as carefully as the password. - Scripts can set `QUICKSEARCH_PASSWORD` for non-interactive terminal search. Environment variables are readable by other processes of the same user (`/proc//environ`) — prefer the keychain. @@ -397,13 +443,13 @@ containing the binary, its config, and its index can be moved wholesale. The GUI edits the config live; external edits apply on next start. `[ui] search_hotkey` is the system-wide search shortcut, written the way -the Options window prints it (`Ctrl+Shift+F`): Ctrl, Alt and Shift in any +the Settings tab prints it (`Ctrl+Shift+F`): Ctrl, Alt and Shift in any combination, plus one key, joined with `+`. An empty string switches it off. A value that is not a shortcut is not a config error — the app loads, -says so in the Options window, and runs without one. +says so on the Settings tab, and runs without one. -`[ui] color_scheme` is `dark` (the default) or `light`, changeable in the -Options window and applied without a restart. It does not follow the +`[ui] color_scheme` is `dark` (the default) or `light`, changeable on the +Settings tab and applied without a restart. It does not follow the desktop's own light/dark setting: on Linux nothing in the window system reports that, so the only way to know is to connect to the session message bus and subscribe to the user's settings feed — more of your session than a @@ -446,11 +492,15 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. log, because SQLite's own autocheckpoint can only reset the log at an instant no reader holds it — and a run keeps a reader per root querying throughout, so left alone the log grows for the whole run. `files` holds - metadata (name, path, size, mtime, hash, MIME/type bitmask, per-row - index state); `searchabletext` is a *contentless* FTS5 table (postings - only, configurable tokenizer, trigram by default); canonical extracted + metadata (name, path, size, mtime, hash, MIME/type bitmask, content + state); `searchabletext` is a *contentless* FTS5 table over one column, + the document body (postings only, configurable tokenizer, trigram by + default) — filename ranks come from scanning `files.name`, so a `name` + column there would only index the same strings twice; canonical extracted text lives zstd-compressed in `documents_text`, which powers snippets, - occurrence ranking, and fuzzy full-text search. Schema changes wipe and + occurrence ranking, and fuzzy full-text search, and whose uncompressed + length is read back from the zstd frame header rather than stored beside + it. Schema changes wipe and rebuild by policy; the indexer (`open_or_recreate`) is the only code allowed to do that; every consumer uses `open_existing`, which treats drift as an error, never data loss. With password protection on, every @@ -477,11 +527,14 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. classify files by mtime into insert/update/skip, batch-write metadata, sweep stale rows, then extract content (plaintext, RTF, Office — both the OOXML/ODF zip formats and the pre-2007 binary `.doc`/`.xls`/`.ppt`, whose - OLE2 streams are read in `extract/ole.rs` — PDF, audio tags, EXIF; see - `extract/`) for FTS. PDFs are parsed once, with the text and the `Info` - dictionary taken off the same document: the two-parse version that preceded - it was the largest single memory consumer of a run over a PDF-heavy tree, and - it was what pulled a second copy of `lopdf` — and with it rayon's + OLE2 streams are read in `extract/ole.rs` — PDF, audio tags; see + `extract/`) for FTS. Images are claimed by no extractor: the EXIF reader + produced structured properties and never text, and with properties parked + (see `extract::ExtractedContent`) leaving `image/*` unclaimed is what keeps + the content pass from opening every image on disk. PDFs are parsed once: + the two-parse version that preceded it was the largest single memory + consumer of a run over a PDF-heavy tree, and it was what pulled a second + copy of `lopdf` — and with it rayon's never-torn-down thread pool — into the build. That is a claim about PDFs rather than about runs in general, and it is worth knowing which tree a number came from: on one with almost no PDFs, a cold run peaks at 130 MiB @@ -499,7 +552,25 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. bigger index — `indexing.content_extensions` remains the throttle. Files no larger than `processing.hash_length` skip that second pass entirely: the head the walk reads to hash them is already their whole content, so a plaintext body is extracted in the same `read` and - stored complete. Every run ends — whether + stored complete. Every root runs its own pipeline — its own walker pool + and, once the walk ends, its own extraction pool — but every root's + *writes* go through one thread and one connection + (`indexing/pipeline.rs`), because that is what a single SQLite file + allows. That thread is where FTS5 tokenizes, up to `maximum_text_size` + of text per document inside the insert, and it is the run's dominant + cost. So its loop is scheduled around the walk, the disk-bound phase and + the one whose stall shows: each round serves every walking root first, + then one extracting root, and no turn runs past a 100 ms slice — an + extraction turn commits at the slice and carries the rows it did not + reach to its next turn. A walk therefore waits at most one slice per + round, which its walkers' channel absorbs, so a root walking a large tree + runs at its own rate while another root tokenizes big documents beside + it. What a root has left to extract is counted by its own content pass, + on that pass's read connection, rather than on the writer: on a large + root the count is seconds, and seconds of writer time is every other + root's walk standing still. Total write throughput is what one connection + tokenizing can do; the scheduling shares it fairly and keeps the walk + first, it does not raise it. Every run ends — whether it completed or was stopped — with an optimize pass on its own connection: checkpoint, VACUUM if the file has at least 10% slack to reclaim, `PRAGMA optimize`, checkpoint again. Progress streams through a polled @@ -555,6 +626,26 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. handle and takes a single watch per root, filtering the events instead. Either way a tree too large to watch degrades to periodic reindexing rather than going silently stale. +- **Live results** (`live.rs`): a second, much smaller watcher, owned by the + frontend rather than the coordinator, pointed at the parent directories of + the result rows *currently on screen* once they have held still for a + moment. It watches directories, not the result files: editors save by + writing a temporary file and renaming it over the target, so the event + lands on the directory and a watch on the file is left holding an orphaned + inode. What a row shows is read from the **file**, never from the index — + metadata from `stat`, and for a content match the same MIME sniffing and + extractors the indexer uses, re-cut through the same `cascade::text_snippet` + the search itself does. That is what makes it work with indexing stopped. + Arming also sweeps each target once against the size and modified time the + row is displaying, which on a fresh result is what the index said: so + bringing a row on screen *is* a check of the index against the disk, and it + is the only thing that reports anything where the platform sends no events. + It still writes nothing itself; the paths it has just read go to + `IndexCoordinator::update_paths`, which applies them on the coordinator's + own thread — in any mode, so a stopped index does not drift from the screen + — leaving the single-writer rule intact. Caps at 64 directories and 256 + rows, rate-limited per path, and dropped wholesale the moment the query is + edited. - **Search** (`search/`): `SearchService` runs one worker thread; each query is a *generation*. New queries interrupt the in-flight SQLite statement (`InterruptHandle`) and stale generations stop cooperatively, @@ -598,6 +689,21 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. case-insensitive literal branch used to allocate a lowercased copy of its haystack, which the filename pass asked for twice per row of a full-table scan. +- **Duplicate verification** (`verify.rs`): the second opinion on a group from + `search/duplicates.rs`, which groups by `sha256(size ‖ head)` and so cannot + tell two pre-allocated disk images apart — same size, same zeroes at the + front, everything that distinguishes them in a footer. One lockstep pass: + open every member, drop the ones whose length already disagrees without + reading them, then read a chunk from the first that opened and the same + span from each of the others, reporting the offset of the first byte that + differs and dropping that file from the walk. Deliberately not a hash — + "the same digest" is a probabilistic answer, and a probabilistic answer is + what the head hash already gave. The reference is the first member that + *opens*, so one unreadable file costs its own verdict and nobody else's, + and termination follows what that file actually reads rather than the + length it claimed, so a file truncated mid-run degrades to a short + comparison. The read buffers share a fixed 8 MiB between them however many + members a group has, because a hardlink farm's group runs to thousands. - **Baloo compatibility** (`cli.rs`, `mime.rs`): the read API this repo's parent consumes — `status_for_path`, `list_failed`, `index_size_breakdown`, `pending_content_count`, `clear_path` — plus a @@ -630,14 +736,17 @@ core threads ─────────────▶ ctx.request_repaint() (w ``` Modules map one-to-one onto what you see: `app.rs` (shell and config -routing, with `app/` submodules for the status bar, the security flow and -the confirmation modals), `search_tab.rs` (query strip and virtualized +routing, with `app/` submodules for the status bar, the security flow, the +confirmation modals and the duplicate-verification modal — the one place a +worker's progress is shown in a window rather than the status bar), +`search_tab.rs` (query strip and virtualized results table; snippet rendering via `LayoutJob` byte ranges, the ignore dialog and the syntax help live in `search_tab/`), `manage_tab.rs` (status detail + `tracker.rs` rate estimation, roots and filter editors), `duplicates_tab.rs`, `logs_tab.rs` (a virtualized view of the core log -ring), `options.rs` (draft-based settings editor shared between the window -and the Manage tab), `platform.rs` (open / reveal-in-file-manager, and the +ring), `settings_tab.rs` (the draft-based config editor, the second of the +two tabs that stage their edits behind an Apply & Save), `platform.rs` +(open / reveal-in-file-manager, and the Windows stdio setup a window-subsystem process needs before anything prints), `hotkey/` (the system-wide search shortcut: one key table feeding both a `RegisterHotKey` / `XGrabKey` registration and, on Wayland, an XDG @@ -650,12 +759,26 @@ microseconds regardless of row count. - `cargo test -p quicksearch-core`: unit + integration suites (cascade ranking, cancellation, incremental indexing, coordinator modes, config - resolution, fuzzy matcher vs. brute-force oracle). + resolution, fuzzy matcher vs. brute-force oracle, `verify.rs`'s byte-for-byte + comparison — the shared-head-different-tail case the head hash cannot see, an + unreadable first member, a difference past the first chunk, cancellation — + and `live.rs`'s event + classification, where the platform-specific rename and atomic-save shapes are + synthesized rather than provoked, so they are checked on every platform). - `cargo test -p quicksearch-gui`: formatter/tracker/CLI-parsing units plus headless egui tests that drive the real widgets — building an input frame, synthesizing clicks and reading back the painted text (`test_ui.rs`) — over - the search and manage tabs, the options editor, the unlock gate, the logs - and duplicates tabs, and query highlighting. + the search, manage and settings tabs, the unlock gate, the logs + and duplicates tabs, the first-start tour, and query highlighting. The search + tab's cover the column picker (including that the path column survives all + 32 combinations of the others), which column a match is highlighted in, and + that the repeat-search button appearing inside the query box does not cost it + keyboard focus. The duplicates tab's open the real context menus and click + the entries inside them, so "the verification asks for the whole group, from + either menu, and not at all while one is running" is checked rather than + assumed; the verification modal is rendered in each of its states, and the + tour's footer is probed for where its three buttons actually landed rather + than for the numbers they were expected to land on. - `cargo bench -p quicksearch-core --bench search` and `--bench index`: divan microbenchmarks over the two hot paths. Each group runs *what the code does today* against *the change being considered*, in one process on one corpus, @@ -695,13 +818,26 @@ microseconds regardless of row count. allocation counts, measure that separately before concluding a path is cheap, and measure the GUI rather than a one-shot `quicksearch-cli` run — a short-lived process cannot show what a typing session retains. -- `.forgejo/workflows/ci.yml`: builds both platforms on every push to `master` - and every pull request. To cut a release, bump `[workspace.package] version` +- `.forgejo/workflows/ci.yml`: builds and tests both platforms on every push to + `master` and every pull request. Those runs stop there — packaging (the + `.deb`, the AppImage, the tarball, the Windows installer and `.zip`, and the + artifact upload) runs only where a release can actually come out, which is a + `v*` tag or a `Release...` branch. The Windows non-system-DLL check is + deliberately not gated that way: it validates the `.exe` rather than + packaging it, so it runs everywhere and catches a regression on `master` + rather than at release time. + + To cut a release, bump `[workspace.package] version` in `Cargo.toml` and push the commit on a branch named `Release...`; CI runs `cargo update -w` first, so a lockfile still pinning the old member versions is not something you have to remember. That only re-resolves the workspace crates, so the `--locked` build after it still fails on a dependency added or - bumped without committing `Cargo.lock`. Once both + bumped without committing `Cargo.lock`. Forgetting the version bump is caught + in seconds rather than after two full release builds: each release path gets + a guard before anything is compiled — a `v*` tag is checked against the + workspace version, and a `Release...` branch is checked against the tags that + already exist. Both are accelerators, not the authority; the release job + re-checks and remains the thing that actually refuses. Once both build jobs are green, CI tags that commit `v` and publishes a release with the `.deb`, an AppImage and its `.zsync` sidecar, a Linux tarball, the Windows installer and a Windows zip attached; pushing a `v*` tag by hand does diff --git a/config_example.toml b/config_example.toml index d4fa338..41efad9 100644 --- a/config_example.toml +++ b/config_example.toml @@ -109,6 +109,12 @@ maximum_text_size = 262144 maximum_text_file_size = 2097152 # Files per batch during walks / inserts / extraction. batch_size = 500 +# Writer time one indexing root's turn may take before the round-robin moves +# on (milliseconds). The time half of the knob whose row half is batch_size: +# it bounds how long one root can hold up the others, so a root extracting +# large documents cannot leave another root's walkers parked behind it. 0 +# gives each turn one batch_size quantum and no more. +writer_turn_slice_ms = 100 # Files per transaction for incremental FTS updates. fts_update_batch_size = 1000 # How large the write-ahead log (index.sqlite-wal) may grow during an @@ -133,7 +139,7 @@ store_text_for_snippets = true [security] # Encrypt the index with a password (SQLCipher). The password is asked # for every time QuickSearch starts; turning this on or off deletes and -# rebuilds the index. Change it from the GUI (Options → Security), not by +# rebuilds the index. Change it from the GUI (Settings → Security), not by # hand: enabling protection also generates the KDF salt below. password_protected = false # Store the derived key in the OS keychain (Secret Service / KWallet on @@ -163,12 +169,18 @@ watch_cap_warned_roots = [] # registered with your desktop, which may assign a different key and lets # you change it in its own keyboard settings. search_hotkey = "Ctrl+Shift+F" -# 'dark' or 'light'. Applied as soon as it is changed in the Options -# window. Your desktop's own light/dark setting is not consulted: reading +# 'dark' or 'light'. Applied as soon as it is changed on the Settings +# tab. Your desktop's own light/dark setting is not consulted: reading # it would mean connecting to your session's message bus and subscribing to # your settings, which is more than a search tool should ask for. Anything # other than 'light' is dark. color_scheme = "dark" +# Written by QuickSearch, not by you: whether the short introduction shown +# on a brand-new installation has been dismissed. Absent means this config +# predates that introduction - an installation that upgraded into this +# version, which is not offered it. The Help tab can show it again at any +# time. +tutorial_seen = false [search] # Start with the fuzzy passes enabled. @@ -185,3 +197,32 @@ display_limit = 1000 results_per_page = 100 # How long the GUI waits after the last keystroke before searching (ms). debounce_ms = 150 +# Watch the search results on screen and show renames, deletions and +# content changes as they happen. Only the rows actually visible are +# watched, and editing the query drops the watches. What a row shows is +# read from the file itself, so this works whether or not indexing is +# running — and the files it reads are then brought up to date in the +# index, so what is stored cannot drift from what you are looking at. +# Rows are also checked against the disk as they come on screen, which is +# all you get over a network share, where the system does not report other +# machines' writes. Nothing is ever added, removed or re-ordered while you +# read. +live_results = true + +# Which columns the Search tab shows. The same choices are on the +# right-click menu of any column header, and in Settings → Search; both +# write here immediately, without an Apply. +# +# There is deliberately no 'path' key: the path is always shown, because +# it is the only column that identifies a result on its own. +[search.columns] +name = true +# The excerpt of a file's contents around the match. Rows that matched on +# their name or path show a dash there instead. +content_match = true +# Off by default: the width these take is usually better spent on the path +# and the matched text. Turning one on also makes it available to sort by; +# sorting by a column that is hidden falls back to sorting by rank. +size = false +modified = false +rank = true diff --git a/crates/quicksearch-core/Cargo.toml b/crates/quicksearch-core/Cargo.toml index 90fa28e..69d0947 100644 --- a/crates/quicksearch-core/Cargo.toml +++ b/crates/quicksearch-core/Cargo.toml @@ -35,7 +35,12 @@ quick-xml = "0.31" # nothing new. cfb = "0.7" serde = { version = "1.0", features = ["derive"] } -toml = "0.8" +# 1.x, not 0.8: 0.8 pulls `toml_edit`, the whole format-preserving document +# model, for what is only `from_str` and `to_string_pretty` here. 1.x parses and +# writes as a stream instead - no `toml_edit`, no `winnow` 0.5 alongside the 1.0 +# already in the tree - which is 252 KB of `.text` this crate was paying for a +# document API it never touches. +toml = "1" mime_guess = "2.0" infer = "0.15" # Charset decoding for non-UTF-8 text (UTF-16 .reg exports, legacy @@ -56,7 +61,8 @@ rtf-parser = { version = "0.4", default-features = false } # global thread pool is never torn down), chrono, time, md5 and a second nom. pdf-extract = "0.12" lofty = "0.19" -kamadak-exif = "0.5" +# Parked with `extract::image` — see `extract::ExtractedContent`. +# kamadak-exif = "0.5" notify = "6.1" ctrlc = "3.4" zstd = "0.13" diff --git a/crates/quicksearch-core/examples/walkprobe.rs b/crates/quicksearch-core/examples/walkprobe.rs index 2c193ff..d85541c 100644 --- a/crates/quicksearch-core/examples/walkprobe.rs +++ b/crates/quicksearch-core/examples/walkprobe.rs @@ -122,7 +122,6 @@ fn parallel(root: &str, config: &Config, db_path: &str) -> (usize, usize) { config.clone(), Arc::new(Registry::default_set()), Arc::new(AtomicBool::new(false)), - Arc::new(AtomicBool::new(false)), 4, ) { let WalkEvent::File(file) = event else { diff --git a/crates/quicksearch-core/src/cli.rs b/crates/quicksearch-core/src/cli.rs index a3b0e36..77c420e 100644 --- a/crates/quicksearch-core/src/cli.rs +++ b/crates/quicksearch-core/src/cli.rs @@ -77,7 +77,6 @@ pub struct FailedEntry { pub struct SizeReport { pub file_size_bytes: u64, pub files_row_count: i64, - pub properties_row_count: i64, pub failed_files_row_count: i64, pub searchabletext_row_count: i64, pub documents_text_row_count: i64, @@ -98,13 +97,20 @@ impl SizeReport { /// Query the per-file indexing status. Returns `FileStatus` with /// `basic == NotIndexed` if the path isn't in the database. +/// +/// There is no stored basic state: a `files` row exists only once its +/// metadata has been read, so the row *is* the basic-indexed state. The +/// failure reason comes from `failed_files`, the one place it is written. pub fn status_for_path(db_path: &str, path: &str) -> Result { let conn = open_existing(db_path, false)?; - let row: Option<(i64, i64, Option)> = conn + let row: Option<(i64, Option)> = conn .query_row( - "SELECT basic_state, content_state, failure_msg FROM files WHERE path = ?1", + "SELECT f.content_state, ff.reason \ + FROM files f \ + LEFT JOIN failed_files ff ON ff.file_id = f.id \ + WHERE f.path = ?1", params![path], - |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + |r| Ok((r.get(0)?, r.get(1)?)), ) .optional() .map_err(|e| format!("status_for_path({}): {}", path, e))?; @@ -115,9 +121,9 @@ pub fn status_for_path(db_path: &str, path: &str) -> Result content: IndexState::NotIndexed, failure_reason: None, }, - Some((basic, content, reason)) => FileStatus { + Some((content, reason)) => FileStatus { path: path.to_string(), - basic: IndexState::from(basic), + basic: IndexState::Done, content: IndexState::from(content), failure_reason: reason, }, @@ -164,18 +170,29 @@ pub fn index_size_breakdown(db_path: &str) -> Result { .map_err(|e| format!("count {}: {}", table, e)) }; let dt_row_count: i64 = count("documents_text")?; - let (dt_raw, dt_compressed): (i64, i64) = conn - .query_row( - "SELECT COALESCE(SUM(text_len), 0), COALESCE(SUM(LENGTH(text_zstd)), 0) FROM documents_text", - [], - |r| Ok((r.get(0)?, r.get(1)?)), - ) + // The uncompressed length is not a column: zstd records it in each frame's + // header, so this reads it back (see `repo::raw_text_len`). Only the + // header is wanted, and 18 bytes is the most one can occupy — projecting + // the prefix keeps this off the document bodies themselves. + let mut stmt = conn + .prepare("SELECT substr(text_zstd, 1, 18), LENGTH(text_zstd) FROM documents_text") + .map_err(|e| format!("documents_text size sum prepare: {}", e))?; + let rows = stmt + .query_map([], |r| Ok((r.get::<_, Vec>(0)?, r.get::<_, i64>(1)?))) .map_err(|e| format!("documents_text size sum: {}", e))?; + let (mut dt_raw, mut dt_compressed) = (0i64, 0i64); + for row in rows { + let (header, compressed) = row.map_err(|e| format!("documents_text size row: {}", e))?; + // A frame with no recorded content size contributes nothing rather + // than skewing the ratio with a guess. + dt_raw += crate::db::repo::raw_text_len(&header).unwrap_or(0) as i64; + dt_compressed += compressed; + } + drop(stmt); Ok(SizeReport { file_size_bytes, files_row_count: count("files")?, - properties_row_count: count("properties")?, failed_files_row_count: count("failed_files")?, searchabletext_row_count: count("searchabletext")?, documents_text_row_count: dt_row_count, @@ -203,7 +220,7 @@ pub fn pending_content_count(db_path: &str) -> Result { } /// Remove a single file from the index. Returns whether a row was deleted. -/// Keeps FTS/documents/properties in sync via the repo helpers. +/// Keeps FTS and `documents_text` in sync via the repo helpers. pub fn clear_path(db_path: &str, path: &str) -> Result { let mut conn = open_existing(db_path, true)?; let tx = conn @@ -239,8 +256,6 @@ mod tests { parent: "/tmp", size: 1, mtime: 1, - inode: None, - device_id: None, mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, @@ -249,7 +264,7 @@ mod tests { ) .unwrap() .expect("unique path"); - set_content_done(&tx, a, "a.txt", "hello", &[], zstd_of("hello").as_deref()).unwrap(); + set_content_done(&tx, a, "hello", zstd_of("hello").as_deref()).unwrap(); let b = insert_file( &tx, &NewFile { @@ -258,8 +273,6 @@ mod tests { parent: "/tmp", size: 1, mtime: 1, - inode: None, - device_id: None, mime: None, ftype: FileType::EMPTY, hash: None, @@ -334,8 +347,6 @@ mod tests { parent: "/tmp", size: 1, mtime: 1, - inode: None, - device_id: None, mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, @@ -354,8 +365,6 @@ mod tests { parent: "/tmp", size: 1, mtime: 1, - inode: None, - device_id: None, mime: None, ftype: FileType::EMPTY, hash: None, @@ -417,8 +426,6 @@ mod tests { parent: "/tmp", size: 1, mtime: 1, - inode: None, - device_id: None, mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, @@ -428,7 +435,7 @@ mod tests { .unwrap() .expect("unique path"); let prose = "the quick brown fox jumps over the lazy dog. ".repeat(500); - set_content_done(&tx, id, "big.txt", &prose, &[], zstd_of(&prose).as_deref()).unwrap(); + set_content_done(&tx, id, &prose, zstd_of(&prose).as_deref()).unwrap(); tx.commit().unwrap(); } drop(conn); @@ -479,8 +486,6 @@ mod tests { parent: "/tmp", size: 1, mtime: 1, - inode: None, - device_id: None, mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, diff --git a/crates/quicksearch-core/src/config/diff.rs b/crates/quicksearch-core/src/config/diff.rs index 5e800fe..e495cac 100644 --- a/crates/quicksearch-core/src/config/diff.rs +++ b/crates/quicksearch-core/src/config/diff.rs @@ -31,8 +31,8 @@ pub struct IndexWork { pub drop_aliases: bool, /// The `content_extensions` filter changed. Kept rows are re-tested /// against it in both directions: newly-included files go back to - /// pending, newly-excluded ones give up their text, properties and FTS - /// row but keep the name/path row that filename search needs. + /// pending, newly-excluded ones give up their text and FTS row but keep + /// the name/path row that filename search needs. pub reconcile_content: bool, /// `store_text_for_snippets` turned on. Rows that finished extraction /// under the old setting kept no text, so they must run again. diff --git a/crates/quicksearch-core/src/config/mod.rs b/crates/quicksearch-core/src/config/mod.rs index 98d9ef8..8184a6e 100644 --- a/crates/quicksearch-core/src/config/mod.rs +++ b/crates/quicksearch-core/src/config/mod.rs @@ -116,6 +116,22 @@ pub struct ProcessingConfig { pub maximum_text_size: usize, pub maximum_text_file_size: u64, pub batch_size: usize, + /// Writer time one root's turn may take before the round moves on, in + /// milliseconds. The time half of the round-robin whose row half is + /// `batch_size`, and so the bound on how long any one root can hold up + /// the others. + /// + /// Before there was one, an extraction turn ran to the end of whatever + /// was ready — half a second to two seconds of FTS5 trigram tokenization + /// for a batch of large documents — while a walking root's rows sat in + /// its channel and its walkers parked behind them. Reads as "4/4 workers + /// busy, no progress". + /// + /// `0` gives each turn one `batch_size` quantum and no more, which is + /// the finest the round-robin goes; the tests that count work per round + /// use small values here so a phase cannot begin and end between two + /// status snapshots. + pub writer_turn_slice_ms: u64, pub fts_update_batch_size: usize, /// How large the write-ahead log may grow during a run before the indexer /// forces a checkpoint, in bytes. `0` disables forced checkpoints; @@ -169,6 +185,47 @@ pub struct SearchConfig { pub results_per_page: usize, /// How long the GUI waits after the last keystroke before searching. pub debounce_ms: u64, + /// Watch the search results currently on screen and show renames, + /// deletions and content changes as they happen. Only the rows actually + /// visible are watched, and any edit to the query drops the watches. + /// What a row shows is read from the file itself, so this holds whether + /// or not indexing is running; the files it reads are then brought up to + /// date in the index, so what is stored cannot drift from what is on + /// screen. See [`crate::live`]. + pub live_results: bool, + /// Which columns the Search tab shows. + pub columns: ColumnsConfig, +} + +/// Which columns the Search tab shows, as picked from the right-click menu on +/// any column header or from the Settings tab. +/// +/// The path column is deliberately not represented: it is always shown, so +/// "no columns at all" is not a state this can hold. Size and modified are off +/// by default — the width they cost is better spent on the path and the match, +/// and both are one click away. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(default)] +pub struct ColumnsConfig { + pub name: bool, + /// The matched excerpt from a file's contents. Rows that matched on their + /// name or path instead show a dash there. + pub content_match: bool, + pub size: bool, + pub modified: bool, + pub rank: bool, +} + +impl Default for ColumnsConfig { + fn default() -> Self { + ColumnsConfig { + name: true, + content_match: true, + size: false, + modified: false, + rank: true, + } + } } impl SearchConfig { @@ -217,6 +274,7 @@ impl Default for ProcessingConfig { maximum_text_size: 1024 * 256, maximum_text_file_size: 1024 * 1024 * 2, batch_size: 500, + writer_turn_slice_ms: 100, fts_update_batch_size: 1000, maximum_wal_size: 1024 * 1024 * 512, tokenize: "trigram".to_string(), @@ -233,6 +291,8 @@ impl Default for SearchConfig { display_limit: 1000, results_per_page: 100, debounce_ms: 150, + live_results: true, + columns: ColumnsConfig::default(), } } } @@ -302,6 +362,21 @@ pub struct UiConfig { /// recognises falls back to dark, where a typed-out enum would fail to /// deserialize and take the whole config file down with it. pub color_scheme: String, + /// Whether the first-start tour has been dismissed. + /// + /// Three-valued on purpose. `None` means the key predates the tour — an + /// installation that upgraded into this version, which has already found + /// its way around — so only a config file this version *created* (which + /// gets `Some(false)` from [`UiConfig::default`]) is ever offered the tour. + /// A plain `bool` could not tell those apart. + /// + /// The field-level `default` is load-bearing and not redundant with the + /// `#[serde(default)]` on the struct: that one fills a missing field from + /// `UiConfig::default()`, which says `Some(false)` — and would hand every + /// upgrading installation the tour. This one fills it from + /// `Option::default()`, which is `None`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tutorial_seen: Option, } impl Default for UiConfig { @@ -311,6 +386,10 @@ impl Default for UiConfig { watch_cap_warned_roots: Vec::new(), search_hotkey: "Ctrl+Shift+F".to_string(), color_scheme: "dark".to_string(), + // Not `None`: a config built from these defaults is a config being + // written for the first time, and that is exactly who the tour is + // for. `None` is reserved for a file that predates the key. + tutorial_seen: Some(false), } } } @@ -470,16 +549,31 @@ impl Config { /// Write back to the file this config was loaded from (or the default /// location), creating parent directories as needed. Raw values are /// written verbatim — relative paths in a portable config stay relative. + /// + /// Atomic: see the comment on the rename below. pub fn save(&self) -> Result<(), String> { let path = self.source.clone().unwrap_or_else(Self::config_path); if let Some(dir) = path.parent() { - fs::create_dir_all(dir) + crate::platform::create_dir_private(dir) .map_err(|e| format!("Failed to create config dir {}: {}", dir.display(), e))?; } let content = toml::to_string_pretty(self) .map_err(|e| format!("Failed to serialize config: {}", e))?; - fs::write(&path, content) - .map_err(|e| format!("Failed to write config file {}: {}", path.display(), e))?; + // Written beside the target and renamed over it, rather than + // truncate-then-write. `[security].salt` exists *only* in this file: + // it is not derivable from the index and not stored anywhere else, so + // a config truncated by a crash, a full disk or a power cut in the + // middle of `write` is an encrypted index that no password can ever + // open again. `rename` is atomic on both platforms, and the `sync_all` + // before it means the bytes are on the disk before the name points at + // them. + let tmp = path.with_extension("toml.tmp"); + write_private(&tmp, content.as_bytes()) + .map_err(|e| format!("Failed to write config file {}: {}", tmp.display(), e))?; + fs::rename(&tmp, &path).map_err(|e| { + let _ = fs::remove_file(&tmp); + format!("Failed to replace config file {}: {}", path.display(), e) + })?; Ok(()) } @@ -537,6 +631,32 @@ impl Config { } } +/// Write `bytes` to `path`, owner-readable only, and flush them to the disk +/// before returning. +/// +/// `O_NOFOLLOW` on Unix: the config directory is not always somewhere only +/// this user can write — a portable install can sit in a shared or removable +/// directory — and a symlink left at the config's name would otherwise +/// redirect this write onto whatever it points at. +fn write_private(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + use std::io::Write; + + let mut opts = fs::OpenOptions::new(); + opts.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.custom_flags(libc::O_NOFOLLOW); + opts.mode(0o600); + } + let mut f = opts.open(path)?; + f.write_all(bytes)?; + // The rename that follows is atomic with respect to the *directory*, not + // to the file's contents: without this, a crash can leave the new name + // pointing at a block of zeroes. + f.sync_all() +} + /// Reserved `content_extensions` entry standing for "files with no /// extension". Matched case-insensitively, and it cannot collide with a real /// extension because the parentheses are not part of one. diff --git a/crates/quicksearch-core/src/config/tests.rs b/crates/quicksearch-core/src/config/tests.rs index 49b0c2a..fcdc4cc 100644 --- a/crates/quicksearch-core/src/config/tests.rs +++ b/crates/quicksearch-core/src/config/tests.rs @@ -985,3 +985,61 @@ fn fuzzy_edits_warning_only_above_the_threshold() { assert!(msg.contains(&FUZZY_EDITS_WARN_ABOVE.to_string())); } } + +/// `config_example.toml` is the documentation for every setting, so a key +/// renamed in the struct and not here would silently ship a config file that +/// does nothing. Parsing it also proves the `[search.columns]` sub-table is +/// spelled the way serde expects. +#[test] +fn the_documented_example_config_parses_to_the_defaults() { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../config_example.toml") + .canonicalize() + .expect("config_example.toml sits at the repository root"); + let text = std::fs::read_to_string(&path).expect("readable"); + let parsed: Config = toml::from_str(&text).expect("config_example.toml parses"); + + // The example documents the shipped defaults for everything that has one + // that does not depend on the machine (paths and the hotkey do). + let d = Config::default(); + assert_eq!( + parsed.search, d.search, + "[search] drifted from the defaults" + ); + assert_eq!(parsed.processing, d.processing); + assert_eq!(parsed.ui.scale, d.ui.scale); + assert_eq!(parsed.ui.color_scheme, d.ui.color_scheme); + assert_eq!(parsed.ui.tutorial_seen, Some(false)); +} + +/// The tour is offered to an installation this version created, and to no +/// other. A config written before the key existed reads as `None`, which is +/// how "already found their way around" is distinguished from "brand new". +#[test] +fn only_a_freshly_written_config_asks_for_the_tour() { + assert_eq!(UiConfig::default().tutorial_seen, Some(false)); + + let older: Config = toml::from_str("[ui]\nscale = 1.1\n").expect("parses"); + assert_eq!( + older.ui.tutorial_seen, None, + "a config predating the tour must not be offered it" + ); + + let dismissed: Config = toml::from_str("[ui]\ntutorial_seen = true\n").expect("parses"); + assert_eq!(dismissed.ui.tutorial_seen, Some(true)); +} + +/// Size and modified cost more width than they earn for most searches. +#[test] +fn the_search_table_ships_without_size_or_modified() { + let cols = ColumnsConfig::default(); + assert!(cols.name && cols.content_match && cols.rank); + assert!(!cols.size, "the size column is on by default"); + assert!(!cols.modified, "the modified column is on by default"); + + // A `[search]` block written before the picker existed still gets them. + let older: Config = toml::from_str("[search]\ndisplay_limit = 500\n").expect("parses"); + assert_eq!(older.search.columns, cols); + assert_eq!(older.search.display_limit, 500); + assert!(older.search.live_results, "live results default to on"); +} diff --git a/crates/quicksearch-core/src/content.rs b/crates/quicksearch-core/src/content.rs index 8b7d5a4..7cf908a 100644 --- a/crates/quicksearch-core/src/content.rs +++ b/crates/quicksearch-core/src/content.rs @@ -1,22 +1,24 @@ //! Parallel content extraction for one indexing root. //! //! The second half of a root's pipeline, and the sibling of [`crate::walk`]: -//! a pool of worker threads produces finished work over a bounded channel, and -//! the single writer drains it round-robin against every other root. +//! a pool of worker threads produces finished work over a bounded channel, +//! and the single writer drains it in time-bounded turns. Walking roots are +//! served first and only one extracting root per round, so a pass that is +//! producing faster than the writer can tokenize waits rather than holding up +//! anyone's walk — see `indexing::pipeline`. //! //! **One feeder thread owns the only database connection**, paging through //! the root's pending rows, while N workers do nothing but filesystem work. A //! connection per worker would multiply SQLite's page cache by the pool size //! (see [`crate::db::schema::PRAGMAS_WALK_READER`]). -use std::sync::atomic::AtomicBool; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{mpsc, Arc, Condvar, Mutex}; use std::thread::JoinHandle; use crate::config::Config; use crate::extract::Registry; -use crate::file_handling::{decide_content, ContentOutcome, ExtractCursor}; -use crate::indexing::should_abort; +use crate::file_handling::{decide_content, ContentOutcome, ExtractCursor, ExtractScope}; use crate::walk::{try_recv_next, TryNext, WorkerStats}; /// Finished rows waiting for the writer. @@ -69,6 +71,11 @@ struct Queue { struct Shared { queue: Mutex, idle: Condvar, + /// What the range held when the pass began: rows still to extract and + /// rows already done. Set by the feeder before it pages anything, so + /// `already_done + rows written this pass` stays exact; never set if the + /// feeder could not count. + totals: std::sync::OnceLock, } impl Shared { @@ -163,6 +170,16 @@ impl ContentPass { self.stats.clone() } + /// The range's pending and already-done counts as they stood when the + /// pass began. + /// + /// `None` until the feeder has counted — a scan that takes seconds on a + /// large root, which is why it happens here on the pass's own connection + /// and not on the indexer's writer — and forever if it could not. + pub fn totals(&self) -> Option { + self.shared.totals.get().copied() + } + /// Join the workers and report whether every one finished cleanly. /// See [`crate::walk::ParallelWalk::finish`]. pub fn finish(&mut self) -> bool { @@ -197,7 +214,7 @@ impl Drop for ContentPass { /// A failed query ends the pass rather than retrying: the rows stay /// `content_state = 0` and the next run picks them up, which is the same /// outcome as being interrupted. -fn feeder(shared: &Shared, db_path: &str, mut cursor: ExtractCursor, max_size: i64) { +fn feeder(shared: &Shared, db_path: &str, mut cursor: ExtractCursor, config: &Config) { let conn = match crate::db::open::open_walk_reader(db_path) { Ok(conn) => conn, Err(e) => { @@ -207,6 +224,18 @@ fn feeder(shared: &Shared, db_path: &str, mut cursor: ExtractCursor, max_size: i } }; + // Before the first page, so nothing this pass writes is inside the count. + // The workers cannot run ahead of this: they block in `take` until the + // first page lands. A failure here costs the progress figure, not the + // pass. + match crate::file_handling::count_extract_scope(&conn, &cursor, config) { + Ok(totals) => { + let _ = shared.totals.set(totals); + } + Err(e) => crate::log_warn!("content reader: {}", e), + } + + let max_size = crate::file_handling::max_text_file_size(config); while shared.take_feed_slot().is_some() { let page = match crate::db::repo::pending_content_page(&conn, &cursor, max_size, FEED_PAGE as i64) @@ -244,14 +273,13 @@ fn worker( registry: &Registry, config: &Config, stop_flag: &Arc, - suspend_flag: &Arc, stats: &WorkerStats, ) { while let Some(row) = shared.take() { // Held for the whole of `decide_content`; that is the work the // progress line reports. let _busy = stats.enter(); - if should_abort(stop_flag, suspend_flag) { + if stop_flag.load(Ordering::Relaxed) { shared.shutdown(); return; } @@ -279,14 +307,13 @@ pub fn extract_content( registry: Arc, config: Config, stop_flag: Arc, - suspend_flag: Arc, workers: usize, ) -> ContentPass { let shared = Arc::new(Shared { queue: Mutex::new(Queue::default()), idle: Condvar::new(), + totals: std::sync::OnceLock::new(), }); - let max_size = i64::try_from(config.processing.maximum_text_file_size).unwrap_or(i64::MAX); let (tx, rx) = mpsc::sync_channel(READY_CAP); let stats = WorkerStats::new(workers.clamp(1, 64)); @@ -294,19 +321,11 @@ pub fn extract_content( .map(|_| { let (shared, tx) = (shared.clone(), tx.clone()); let (registry, config) = (registry.clone(), config.clone()); - let (stop_flag, suspend_flag) = (stop_flag.clone(), suspend_flag.clone()); + let stop_flag = stop_flag.clone(); let stats = stats.clone(); crate::platform::spawn_worker("qs-extract", move || { crate::platform::set_background_priority(); - worker( - &shared, - &tx, - ®istry, - &config, - &stop_flag, - &suspend_flag, - &stats, - ) + worker(&shared, &tx, ®istry, &config, &stop_flag, &stats) }) }) .collect(); @@ -318,7 +337,7 @@ pub fn extract_content( let (shared, db_path, cursor) = (shared.clone(), db_path.to_string(), cursor.clone()); crate::platform::spawn_worker("qs-feeder", move || { crate::platform::set_background_priority(); - feeder(&shared, &db_path, cursor, max_size) + feeder(&shared, &db_path, cursor, &config) }) }; @@ -338,9 +357,23 @@ mod tests { use crate::db::open_or_recreate; use crate::db::repo::{self, insert_file, NewFile}; - use crate::file_handling::{extract_scope_prepare, store_extracted}; + use crate::file_handling::{store_extracted, ExtractScope, Stored}; use crate::mime::FileType; use std::path::{Path, PathBuf}; + use std::time::{Duration, Instant}; + + /// The removed `extract_scope_prepare`: the sweep on the writer, then the + /// count the content pass now does on its own connection. Composed here + /// because these tests want both halves in one call. + fn extract_scope_prepare( + conn_mutex: &Arc>, + cursor: &ExtractCursor, + config: &Config, + ) -> Result { + let conn = crate::lock_ok(conn_mutex); + crate::file_handling::mark_oversize_pending_na(&conn, cursor, config)?; + crate::file_handling::count_extract_scope(&conn, cursor, config) + } /// A path that does not exist yet — the caller builds the tree under it. fn tmp(tag: &str) -> PathBuf { crate::testutil::scratch_dir(tag).join("tree") @@ -367,8 +400,6 @@ mod tests { parent: d.to_str().unwrap(), size: std::fs::metadata(&f).unwrap().len(), mtime: 1, - inode: None, - device_id: None, mime: Some("text/plain"), ftype: FileType::TEXT, hash: None, @@ -391,7 +422,6 @@ mod tests { Arc::new(Registry::default_set()), Config::default(), Arc::new(AtomicBool::new(false)), - Arc::new(AtomicBool::new(false)), workers, ) } @@ -450,9 +480,13 @@ mod tests { assert_eq!(rows.len(), 3); let stop = Arc::new(AtomicBool::new(false)); + let far = Instant::now() + Duration::from_secs(60); assert_eq!( - store_extracted(&conn_mutex, &rows, &stop, &config).unwrap(), - 3 + store_extracted(&conn_mutex, &rows, &stop, &config, far).unwrap(), + Stored { + consumed: 3, + written: 3 + } ); let state = |p: &Path| -> i64 { @@ -501,6 +535,93 @@ mod tests { let mut pass = pass_for(&tree, &db, "nonexistent", 4); assert!(drain(&mut pass).is_empty()); assert!(pass.finish()); + // The pass still counted: an empty range is a known zero, not an + // unknown. + assert_eq!( + pass.totals(), + Some(ExtractScope { + pending: 0, + already_done: 0 + }) + ); + std::fs::remove_dir_all(&tree).ok(); + std::fs::remove_file(&db).ok(); + } + + /// The pass counts its range on its own connection, before it pages — + /// which is what lets the writer thread stop doing it. The count is what + /// stood at the start: rows this pass writes are not inside it. + #[test] + fn the_pass_counts_its_range_before_it_starts() { + let (tree, db) = seed("totals", &[("r1", 3), ("r2", 2)]); + let mut pass = pass_for(&tree, &db, "r1", 2); + let rows = drain(&mut pass); + assert!(pass.finish()); + assert_eq!(rows.len(), 3); + assert_eq!( + pass.totals(), + Some(ExtractScope { + pending: 3, + already_done: 0 + }), + "only r1's rows, all of them pending when the pass began" + ); + std::fs::remove_dir_all(&tree).ok(); + std::fs::remove_file(&db).ok(); + } + + /// The writer's turn is bounded by time, not by batch: `store_extracted` + /// stops at its deadline, tells the caller how far it got, and always + /// gets at least one row down so a caller looping on it cannot spin. + #[test] + fn store_extracted_honours_its_deadline_but_always_makes_progress() { + let (tree, db) = seed("deadline", &[("r1", 5)]); + let conn_mutex = Arc::new(Mutex::new( + open_or_recreate(db.to_str().unwrap(), "trigram").unwrap(), + )); + let config = Config::default(); + let mut pass = pass_for(&tree, &db, "r1", 2); + let rows = drain(&mut pass); + assert!(pass.finish()); + assert_eq!(rows.len(), 5); + let stop = Arc::new(AtomicBool::new(false)); + + // A deadline already gone by: one row, then out. + let past = Instant::now() - Duration::from_secs(1); + assert_eq!( + store_extracted(&conn_mutex, &rows, &stop, &config, past).unwrap(), + Stored { + consumed: 1, + written: 1 + } + ); + // Plenty of time: the rest, in one call. + let far = Instant::now() + Duration::from_secs(60); + assert_eq!( + store_extracted(&conn_mutex, &rows[1..], &stop, &config, far).unwrap(), + Stored { + consumed: 4, + written: 4 + } + ); + let done: i64 = conn_mutex + .lock() + .unwrap() + .query_row( + "SELECT COUNT(*) FROM files WHERE content_state = 1", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(done, 5, "every row landed across the two calls"); + + // Stopped before it starts: nothing consumed, and the caller can tell. + stop.store(true, std::sync::atomic::Ordering::Relaxed); + assert_eq!( + store_extracted(&conn_mutex, &rows, &stop, &config, far).unwrap(), + Stored::default() + ); + std::fs::remove_dir_all(&tree).ok(); std::fs::remove_file(&db).ok(); } @@ -514,7 +635,6 @@ mod tests { Arc::new(Registry::default_set()), Config::default(), Arc::new(AtomicBool::new(true)), - Arc::new(AtomicBool::new(false)), 4, ); assert!(drain(&mut pass).len() < 400); diff --git a/crates/quicksearch-core/src/coordinator.rs b/crates/quicksearch-core/src/coordinator.rs index 6543c5e..59cfb24 100644 --- a/crates/quicksearch-core/src/coordinator.rs +++ b/crates/quicksearch-core/src/coordinator.rs @@ -36,7 +36,7 @@ use rusqlite::Connection; use crate::config::{diff_actions, Config, IgnoreSet, IndexWork}; use crate::db; use crate::extract::Registry; -use crate::incremental::apply_fs_event; +use crate::incremental::{apply_fs_event, Applied, Budget}; use crate::indexing::{ConfigChange, IndexingService, IndexingStatus, PrepStep, ReconcileProgress}; use crate::scope::WorkCursor; use crate::watcher::{FsEvent, WatchError, WatchFilters, Watcher, WatcherConfig}; @@ -130,6 +130,7 @@ enum CoordCmd { ConfigChanged(Config), RebuildIndex, ClearIndex, + UpdatePaths(Vec), Shutdown, } @@ -228,6 +229,8 @@ impl IndexCoordinator { watcher_rx: None, watcher_gen: 0, pending: HashMap::new(), + targeted: HashMap::new(), + resume_from: HashMap::new(), last_event_at: None, pending_since: None, needs_full_run: false, @@ -303,6 +306,26 @@ impl IndexCoordinator { let _ = self.cmd_tx.send(CoordCmd::ClearIndex); } + /// Bring the index up to date for these paths and nothing else. + /// + /// For [`crate::live`]: a frontend that has just read a displayed file + /// from disk hands the path here so the index agrees with what the user + /// is looking at. Deliberately **not** gated on [`IndexMode`] — the whole + /// point is that the rows on screen stay honest with indexing stopped — + /// but still applied on the coordinator's own thread, so the + /// single-writer rule holds and a full run is never raced. + /// + /// Each path is re-read and rewritten only if its modified time has moved + /// (see [`crate::incremental::apply_fs_event`]), so submitting a path that + /// is already current costs a `stat` and a row lookup. A path that no + /// longer exists is removed from the index. + pub fn update_paths(&self, paths: Vec) { + if paths.is_empty() { + return; + } + let _ = self.cmd_tx.send(CoordCmd::UpdatePaths(paths)); + } + /// Compare `config` against what the index was built with. Read-only. pub fn check_config_validation( &self, @@ -356,6 +379,35 @@ impl Drop for IndexCoordinator { } } +/// The verb a path submitted through [`IndexCoordinator::update_paths`] +/// deserves, or `None` to leave the index alone. +/// +/// The caller knows a file changed, not what it changed into. `is_file()` is +/// the fast answer and almost always the right one — one `stat`, and this runs +/// while results are on screen — but it folds every stat error into `false`, +/// and a `Remove` costs the row *and everything beneath it*. So the negative +/// answer, and only it, is confirmed with a second `stat` that can tell "gone" +/// from "cannot see it just now": a share that dropped, a drive pulled while +/// its rows were displayed, a parent another process chmod'd. +/// +/// In doubt the index wins. A stale row is a wrong line on screen until the +/// next full run; a deleted live one is data no run brings back until the file +/// is walked again — and if the reason it could not be read was that its whole +/// tree went away, that walk will not reach it either. +fn verb_for(path: PathBuf) -> Option { + if path.is_file() { + return Some(FsEvent::Modify(path)); + } + // `metadata`, not `symlink_metadata`: it has to agree with `is_file()` + // above about following links, or the two can disagree about the verb. + match std::fs::metadata(&path) { + // There, but no longer something the walk would index. + Ok(_) => Some(FsEvent::Remove(path)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Some(FsEvent::Remove(path)), + Err(_) => None, + } +} + /// Fold `event` into the last-event-wins pending map. Renames split into /// their halves so downstream application never needs pair handling. fn enqueue(pending: &mut HashMap, event: FsEvent) { diff --git a/crates/quicksearch-core/src/coordinator/inner.rs b/crates/quicksearch-core/src/coordinator/inner.rs index 64d6282..8a37e9f 100644 --- a/crates/quicksearch-core/src/coordinator/inner.rs +++ b/crates/quicksearch-core/src/coordinator/inner.rs @@ -22,6 +22,17 @@ pub(super) struct Inner { pub(super) watcher_rx: Option)>>, pub(super) watcher_gen: u64, pub(super) pending: HashMap, + /// Paths a frontend asked for by name — see + /// [`IndexCoordinator::update_paths`]. Kept apart from [`Inner::pending`] + /// on purpose: this queue survives [`Inner::clear_pending`] and is applied + /// in manual mode, because it exists to keep the rows a user is *reading* + /// in step with the disk however the indexer is configured. + pub(super) targeted: HashMap, + /// How far a directory event got before its turn's budget ran out, so the + /// next turn resumes rather than re-walking what it already applied. Keyed + /// by the same path as the queue the event went back into, and removed + /// when the event completes or is dropped. + pub(super) resume_from: HashMap, /// When the most recent event arrived; the burst is over once this is /// `pending_settle` old. pub(super) last_event_at: Option, @@ -158,6 +169,32 @@ impl Inner { drop(shared); self.files_at = None; } + CoordCmd::UpdatePaths(paths) => { + // Only paths under an indexed root: the watcher never + // delivers anything else, so nothing downstream checks, and + // a file renamed *out* of every root would otherwise be + // written into the index at its new home. Roots in the same + // spelling `files.path` uses — the caller's paths are. + let prefixes: Vec = self + .config + .normalized_indexing_paths() + .iter() + .map(|root| crate::file_handling::ExtractCursor::for_root(root).lo) + .collect(); + for path in paths { + let spelled = path.to_string_lossy(); + if !prefixes.iter().any(|lo| spelled.starts_with(lo.as_str())) { + continue; + } + // Existence decides the verb; `verb_for` also decides when + // it cannot be decided at all, and says so with `None`. + let Some(event) = verb_for(path) else { + continue; + }; + enqueue(&mut self.targeted, event); + } + self.was_busy = true; + } CoordCmd::Shutdown => unreachable!("handled in run()"), } } @@ -197,6 +234,14 @@ impl Inner { self.refresh_file_count(); + // Ahead of both the reconcile and the mode gate, and ahead of the + // settle window the watcher queue waits out: these are rows a user is + // looking at right now, there are at most a screenful, and a stopped + // indexer is exactly when the frontend most needs them to be current. + if !self.targeted.is_empty() { + self.apply_targeted(); + } + // Ahead of the mode gate: a config edit is reconciled in manual mode // too. if self.pending_work.is_some() { @@ -377,6 +422,11 @@ impl Inner { // `clear` keeps the map's capacity — up to 100k slots after a storm; // shrinking is the point. self.pending.shrink_to_fit(); + // Resume points describe events that no longer exist. Entries for + // `targeted` events survive, which is why this filters rather than + // clearing: that queue deliberately outlives this call. + self.resume_from + .retain(|p, _| self.targeted.contains_key(p)); self.last_event_at = None; self.pending_since = None; } @@ -446,13 +496,35 @@ impl Inner { let Some(ev) = self.pending.remove(&path) else { continue; }; - if let Err(e) = - apply_fs_event(&mut conn, &ev, &self.config, &self.ignore, &self.registry) - { - // As above: the event is out of `pending`, so only a full - // run still picks the file up. - crate::log_warn!("coordinator: apply {:?}: {}; scheduling full run", ev, e); - self.needs_full_run = true; + match apply_fs_event( + &mut conn, + &ev, + &self.config, + &self.ignore, + &self.registry, + &Budget { + deadline, + cancel: &self.reconcile_stop.cancel, + resume_from: self.resume_from.remove(&path).unwrap_or(0), + }, + ) { + // A directory event can cover a whole moved-in tree. Put + // it back, with a note of how far it got, and let the next + // tick continue it — so one `mv` cannot hold this loop, or + // the shutdown queued behind it, for as long as the tree + // takes. + Ok(Applied::Unfinished { done }) => { + self.resume_from.insert(path.clone(), done); + self.pending.insert(path, ev); + break; + } + Ok(Applied::Done) => {} + Err(e) => { + // As above: the event is out of `pending`, so only a + // full run still picks the file up. + crate::log_warn!("coordinator: apply {:?}: {}; scheduling full run", ev, e); + self.needs_full_run = true; + } } if Instant::now() >= deadline { break; @@ -469,6 +541,88 @@ impl Inner { } } + /// Apply the by-name queue: the paths a frontend is displaying. + /// + /// Shaped like [`Inner::apply_pending`] — removals first, same budget — + /// but it never escalates to [`Inner::needs_full_run`]. A frontend reads + /// what it shows from the file itself, so a failure here leaves the screen + /// correct and only the index behind; reindexing the world over that would + /// be wildly out of proportion. + fn apply_targeted(&mut self) { + self.was_busy = true; + let mut conn = match self.ensure_write_conn() { + Ok(conn) => conn, + Err(e) => { + crate::log_warn!("coordinator: targeted update unavailable: {}", e); + self.targeted.clear(); + return; + } + }; + let deadline = Instant::now() + APPLY_BUDGET; + let chunk = self.config.processing.batch_size.max(1); + + // Removals lead for the same reason they do in `apply_pending`: the + // queue is an unordered map, and a rename enqueues both halves. + let removals: Vec = self + .targeted + .iter() + .filter(|(_, ev)| is_removal(ev)) + .map(|(p, _)| p.clone()) + .collect(); + for batch in removals.chunks(chunk) { + if let Err(e) = crate::incremental::remove_paths(&mut conn, batch, chunk) { + crate::log_warn!("coordinator: targeted remove: {}", e); + } + for path in batch { + self.targeted.remove(path); + } + if Instant::now() >= deadline { + break; + } + } + + if Instant::now() < deadline { + let upserts: Vec = self + .targeted + .iter() + .filter(|(_, ev)| !is_removal(ev)) + .map(|(p, _)| p.clone()) + .collect(); + for path in upserts { + let Some(ev) = self.targeted.remove(&path) else { + continue; + }; + match apply_fs_event( + &mut conn, + &ev, + &self.config, + &self.ignore, + &self.registry, + &Budget { + deadline, + cancel: &self.reconcile_stop.cancel, + resume_from: self.resume_from.remove(&path).unwrap_or(0), + }, + ) { + Ok(Applied::Unfinished { done }) => { + self.resume_from.insert(path.clone(), done); + self.targeted.insert(path, ev); + break; + } + Ok(Applied::Done) => {} + Err(e) => { + crate::log_warn!("coordinator: targeted apply {:?}: {}", ev, e); + } + } + if Instant::now() >= deadline { + break; + } + } + } + + self.write_conn = Some(conn); + } + fn ensure_write_conn(&mut self) -> Result { if let Some(conn) = self.write_conn.take() { return Ok(conn); @@ -799,7 +953,7 @@ impl Inner { ); let mut shared = crate::lock_ok(&self.shared); shared.mode = self.mode; - shared.queued_events = self.pending.len(); + shared.queued_events = self.pending.len() + self.targeted.len(); shared.reconcile = reconcile; drop(shared); diff --git a/crates/quicksearch-core/src/coordinator/tests.rs b/crates/quicksearch-core/src/coordinator/tests.rs index 7d0282a..2879cd8 100644 --- a/crates/quicksearch-core/src/coordinator/tests.rs +++ b/crates/quicksearch-core/src/coordinator/tests.rs @@ -505,6 +505,149 @@ fn a_run_it_schedules_itself_wakes_the_frontend() { coord.shutdown(); } +// --- targeted updates (see `IndexCoordinator::update_paths`) -------------- + +impl Fixture { + /// The `mtime` the index holds for one path, or `None` if it has no row. + fn stored_mtime(&self, path: &std::path::Path) -> Option { + let conn = db::open_existing(&self.db.to_string_lossy(), false).ok()?; + conn.query_row( + "SELECT mtime FROM files WHERE path = ?1", + [path.to_string_lossy().as_ref()], + |r| r.get(0), + ) + .ok() + } +} + +/// The point of the whole thing: the frontend has just read a file the user is +/// looking at, and the index catches up even though indexing is stopped — with +/// no watcher running and no full run scheduled. +#[test] +fn update_paths_indexes_one_file_with_indexing_stopped() { + let f = Fixture::new(false); + std::fs::write(f.dir.join("seed.txt"), "initial content").unwrap(); + f.seed_index(); + assert_eq!(f.file_count(), 1); + + let coord = start_coord(f.config.clone()); + let added = f.dir.join("added.txt"); + std::fs::write(&added, "written while indexing was stopped").unwrap(); + coord.update_paths(vec![added.clone()]); + + wait_for("targeted insert", Duration::from_secs(20), || { + f.stored_mtime(&added).is_some() + }); + assert_eq!( + coord.state().mode, + IndexMode::ManualStopped, + "a targeted update started a run" + ); + assert!( + coord.state().last_full_index.is_some(), + "the seed stamp was disturbed" + ); + coord.shutdown(); +} + +/// The same call is how a row is *validated*: submitting a path the index +/// already agrees with must not rewrite it, which is what makes it cheap +/// enough for the frontend to submit whatever it just looked at. +#[test] +fn update_paths_leaves_a_row_that_already_agrees_alone() { + let f = Fixture::new(false); + let file = f.dir.join("steady.txt"); + std::fs::write(&file, "unchanged").unwrap(); + f.seed_index(); + let before = f.stored_mtime(&file).expect("seeded"); + + let coord = start_coord(f.config.clone()); + coord.update_paths(vec![file.clone()]); + // No state change to wait on, so wait out a few ticks instead. + std::thread::sleep(Duration::from_secs(3)); + + assert_eq!(f.stored_mtime(&file), Some(before)); + assert_eq!(f.file_count(), 1); + coord.shutdown(); +} + +/// A path outside every indexed root is not the index's to hold, however it +/// was submitted: a result renamed into an un-indexed folder must not follow +/// the row into the index at its new home. +#[test] +fn update_paths_ignores_a_path_outside_every_root() { + let f = Fixture::new(false); + std::fs::write(f.dir.join("seed.txt"), "initial content").unwrap(); + f.seed_index(); + assert_eq!(f.file_count(), 1); + + // A sibling of the indexed tree, under the same scratch parent. + let outside = f.dir.parent().unwrap().join("elsewhere"); + std::fs::create_dir_all(&outside).unwrap(); + let stray = outside.join("moved-here.txt"); + std::fs::write(&stray, "renamed out of the index").unwrap(); + + let coord = start_coord(f.config.clone()); + coord.update_paths(vec![stray.clone()]); + std::thread::sleep(Duration::from_secs(3)); + + assert_eq!( + f.stored_mtime(&stray), + None, + "an un-indexed folder gained a row" + ); + assert_eq!(f.file_count(), 1); + coord.shutdown(); + std::fs::remove_dir_all(&outside).ok(); +} + +/// A row whose file has gone leaves the index too — the frontend hands over +/// the path, not a verb, so the coordinator decides from what is on disk. +#[test] +fn update_paths_removes_a_row_whose_file_is_gone() { + let f = Fixture::new(false); + let file = f.dir.join("doomed.txt"); + std::fs::write(&file, "not for long").unwrap(); + f.seed_index(); + assert!(f.stored_mtime(&file).is_some()); + + let coord = start_coord(f.config.clone()); + std::fs::remove_file(&file).unwrap(); + coord.update_paths(vec![file.clone()]); + + wait_for("targeted remove", Duration::from_secs(20), || { + f.stored_mtime(&file).is_none() + }); + coord.shutdown(); +} + +/// The single-writer rule still holds: a targeted update submitted while a +/// full run owns the database waits for it rather than opening a second +/// writer beside it. +#[test] +fn update_paths_waits_for_a_full_run_rather_than_racing_it() { + let f = Fixture::new(false); + for i in 0..400 { + std::fs::write(f.dir.join(format!("f{i}.txt")), "body").unwrap(); + } + let coord = start_coord(f.config.clone()); + coord.reindex_now(); + + let added = f.dir.join("late.txt"); + std::fs::write(&added, "submitted mid-run").unwrap(); + coord.update_paths(vec![added.clone()]); + + wait_for("run finished", Duration::from_secs(60), || { + coord.state().last_full_index.is_some() + }); + wait_for( + "targeted insert after the run", + Duration::from_secs(20), + || f.stored_mtime(&added).is_some(), + ); + coord.shutdown(); +} + #[test] fn auto_mode_runs_initial_index_and_applies_watcher_events() { let f = Fixture::new(true); @@ -805,6 +948,53 @@ fn a_deletion_during_a_full_run_is_queued_then_applied() { coord.shutdown(); } +/// A path that cannot be read is not a path that is gone. +/// +/// `update_paths` is fed by the frontend from the rows it is displaying, and +/// its `Remove` verb takes the row *and its whole subtree*. `is_file()` cannot +/// tell "not a regular file" from "I could not look", so an unreadable file — +/// a network share that dropped, a removable drive unplugged with its results +/// on screen, a directory another process just chmod'd — used to read as a +/// deletion. The next full run cannot undo it: an unreachable root is recorded +/// unreadable rather than re-walked. +#[test] +fn an_unreadable_path_is_not_treated_as_deleted() { + let dir = crate::testutil::scratch_dir("verb"); + let present = dir.join("here.txt"); + std::fs::write(&present, b"x").unwrap(); + assert!( + matches!(verb_for(present.clone()), Some(FsEvent::Modify(_))), + "a readable file is a Modify" + ); + + let gone = dir.join("never-existed.txt"); + assert!( + matches!(verb_for(gone), Some(FsEvent::Remove(_))), + "a genuinely absent file is a Remove" + ); + + // A directory is not something the walk indexes, so it is still a Remove. + assert!(matches!(verb_for(dir.clone()), Some(FsEvent::Remove(_)))); + + // The case that matters: the file is there, and unreadable. + let locked = dir.join("locked"); + std::fs::create_dir_all(&locked).unwrap(); + let hidden = locked.join("file.txt"); + std::fs::write(&hidden, b"x").unwrap(); + if crate::platform::deny_read(&locked).is_ok() { + // Skipped when the test runs with rights that ignore the mode — CI + // drops CAP_DAC_OVERRIDE with capsh for exactly this reason. + if std::fs::metadata(&hidden).is_err() { + assert!( + verb_for(hidden).is_none(), + "an unreadable file must leave the index alone" + ); + } + let _ = crate::platform::restore_read(&locked); + } + std::fs::remove_file(&present).ok(); +} + #[test] fn enqueue_last_wins_and_rename_splits() { let mut pending = HashMap::new(); diff --git a/crates/quicksearch-core/src/db/open.rs b/crates/quicksearch-core/src/db/open.rs index 17002fa..311d93d 100644 --- a/crates/quicksearch-core/src/db/open.rs +++ b/crates/quicksearch-core/src/db/open.rs @@ -25,7 +25,7 @@ pub const KEY_MISMATCH_PREFIX: &str = "KEY_MISMATCH: "; /// values go stale: `files.mime`, `files.type` and `content_state` are /// computed at walk time and never re-derived for unchanged files, so a /// classification change needs the wipe to apply everywhere. -pub const CURRENT_SCHEMA_VERSION: u32 = 6; +pub const CURRENT_SCHEMA_VERSION: u32 = 7; /// Open `db_path` and ensure the on-disk schema matches this build; if it /// doesn't (including a changed `tokenizer`), delete the file and recreate it @@ -42,12 +42,18 @@ pub(crate) fn open_or_recreate_keyed( let path = Path::new(db_path).to_path_buf(); if let Some(dir) = path.parent() { if !dir.as_os_str().is_empty() { - std::fs::create_dir_all(dir) + crate::platform::create_dir_private(dir) .map_err(|e| format!("Failed to create database dir {}: {}", dir.display(), e))?; } } let conn = Connection::open(db_path) .map_err(|e| format!("Failed to open database at {}: {}", db_path, e))?; + // Before a single row is written. SQLite creates the file 0644 and hands + // that mode on to `-wal` and `-shm`, so on a default umask every other + // user on the machine could read the index — which holds the names and + // full text of everything under the configured roots, including files + // whose own permissions are 0600. + crate::platform::restrict_to_owner(&path); key_and_probe(&conn, db_path, key)?; conn.execute_batch(PRAGMAS_FAST) .map_err(|e| format!("Failed to apply pragmas: {}", e))?; @@ -345,6 +351,9 @@ fn wipe_and_reopen( } let conn = Connection::open(path) .map_err(|e| format!("Failed to reopen database after rebuild: {}", e))?; + // A rebuild creates the file afresh, so it needs narrowing again for the + // same reason the first open does. + crate::platform::restrict_to_owner(path); key_and_probe(&conn, &path.to_string_lossy(), key)?; conn.execute_batch(PRAGMAS_FAST) .map_err(|e| format!("Failed to apply pragmas after rebuild: {}", e))?; diff --git a/crates/quicksearch-core/src/db/open_tests.rs b/crates/quicksearch-core/src/db/open_tests.rs index a533cc5..ab6081b 100644 --- a/crates/quicksearch-core/src/db/open_tests.rs +++ b/crates/quicksearch-core/src/db/open_tests.rs @@ -102,7 +102,7 @@ fn legacy_layout_db_is_wiped_and_recreated() { // New columns should exist (just prepare the SELECT — an // unknown column name would parse-error here). conn.query_row( - "SELECT basic_state, content_state, type, mime FROM files LIMIT 0", + "SELECT content_state, type, mime FROM files LIMIT 0", [], |_| Ok(()), ) @@ -172,8 +172,8 @@ fn open_existing_reads_nondefault_tokenizer_without_wiping() { // Seed the FTS index (rowid = the files row we just inserted) so a // MATCH query can be exercised against the on-disk tokenizer. conn.execute( - "INSERT INTO searchabletext (rowid, name, text, properties) \ - VALUES (last_insert_rowid(), 'note', 'hello world', '')", + "INSERT INTO searchabletext (rowid, text) \ + VALUES (last_insert_rowid(), 'hello world')", [], ) .unwrap(); @@ -546,3 +546,55 @@ fn open_existing_rw_allows_delete() { drop(conn); std::fs::remove_file(&p).ok(); } + +/// The index, and the WAL and SHM it hands its mode to, must not be readable +/// by other users on the machine. +/// +/// SQLite creates its database file 0644 and copies that mode to `-wal` and +/// `-shm`; with the near-universal umask 022 that leaves the names and full +/// text of everything under the configured roots — including documents whose +/// own files are 0600 — readable by every account on a shared machine. The +/// README's carve-out is about other *processes of the same user*, not other +/// users, so nothing else covers this. +#[cfg(unix)] +#[test] +fn a_fresh_index_and_its_sidecars_are_owner_only() { + use std::os::unix::fs::PermissionsExt; + + // A directory that does not exist yet, so the creation path is the one + // under test: an existing directory keeps whatever mode its owner chose. + let p = tmp_db_path() + .parent() + .unwrap() + .join("data") + .join("index.sqlite"); + let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + // A write, so the WAL and SHM exist to be checked. + conn.execute( + "INSERT INTO files (name, path, parent, size, mtime) \ + VALUES ('a', '/perm-a', '/', 0, 0)", + [], + ) + .unwrap(); + + let mode_of = |path: &std::path::Path| { + std::fs::metadata(path) + .unwrap_or_else(|e| panic!("stat {}: {e}", path.display())) + .permissions() + .mode() + & 0o777 + }; + assert_eq!(mode_of(&p), 0o600, "index at {}", p.display()); + for suffix in ["-wal", "-shm"] { + let sidecar = std::path::PathBuf::from(format!("{}{}", p.display(), suffix)); + if sidecar.exists() { + assert_eq!(mode_of(&sidecar), 0o600, "sidecar {}", sidecar.display()); + } + } + // And the directory created for it, which would otherwise take the umask + // and let any account list what is indexed. + assert_eq!(mode_of(p.parent().unwrap()), 0o700); + + drop(conn); + std::fs::remove_file(&p).ok(); +} diff --git a/crates/quicksearch-core/src/db/repo.rs b/crates/quicksearch-core/src/db/repo.rs index 48fa991..95c11a7 100644 --- a/crates/quicksearch-core/src/db/repo.rs +++ b/crates/quicksearch-core/src/db/repo.rs @@ -1,7 +1,7 @@ //! Row-level write helpers that keep the FTS5 contentless table in sync with -//! `files`/`documents`/`properties`. +//! `files`/`documents_text`. //! -//! States (mirrors `basic_state` / `content_state` columns): +//! States (mirrors the `content_state` column): //! //! | value | meaning | //! |------:|---------| @@ -45,7 +45,7 @@ fn set_state_clearing_failure( ) -> Result<(), String> { exec( tx, - "UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2", + "UPDATE files SET content_state = ?1 WHERE id = ?2", params![state, file_id], || format!("{} content_state {}", transition, file_id), )?; @@ -66,8 +66,6 @@ pub struct NewFile<'a> { pub parent: &'a str, pub size: u64, pub mtime: u64, - pub inode: Option, - pub device_id: Option, pub mime: Option<&'a str>, pub ftype: FileType, pub hash: Option<&'a [u8]>, @@ -76,17 +74,16 @@ pub struct NewFile<'a> { pub needs_content: bool, } -/// Insert a new file row, returning its id. `basic_state` is set to DONE -/// (the row existing *is* the basic-index state); `content_state` comes from -/// `needs_content`. `INSERT OR IGNORE`: a UNIQUE(path) collision returns -/// `None` rather than aborting the batch. +/// Insert a new file row, returning its id. `content_state` comes from +/// `needs_content`; there is no separate basic state, because the row +/// existing *is* the basic-index state. `INSERT OR IGNORE`: a UNIQUE(path) +/// collision returns `None` rather than aborting the batch. pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result, String> { let rows = tx .prepare_cached( "INSERT OR IGNORE INTO files ( - name, path, parent, size, mtime, inode, device_id, - mime, type, basic_state, content_state, hash - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)", + name, path, parent, size, mtime, mime, type, content_state, hash + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", ) .and_then(|mut stmt| { stmt.execute(params![ @@ -95,11 +92,8 @@ pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result, f.parent, f.size as i64, f.mtime as i64, - f.inode.map(|x| x as i64), - f.device_id.map(|x| x as i64), f.mime, f.ftype.bits() as i64, - STATE_DONE, initial_content_state(f), f.hash, ]) @@ -125,14 +119,14 @@ fn initial_content_state(f: &NewFile<'_>) -> i64 { /// Update a file's metadata in place (same path, changed size/mtime/hash) and /// reset its content state from `f.needs_content`, clearing any extracted /// content so the text-indexing pass re-processes it. Writes `size`, `mtime`, -/// `hash`, `mime`, `type`, `content_state` and `failure_msg` — and only -/// those; `name`, `parent`, `inode` and `device_id` are not refreshed here. +/// `hash`, `mime`, `type` and `content_state` — and only those; `name` and +/// `parent` are not refreshed here. pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result, String> { let id: Option = tx .prepare_cached( "UPDATE files SET size = ?1, mtime = ?2, hash = ?3, mime = ?4, type = ?5, - content_state = ?6, failure_msg = NULL + content_state = ?6 WHERE path = ?7 RETURNING id", ) @@ -159,13 +153,11 @@ pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result