Compare commits
5 commits
1f19f684f2
...
dc9580a659
| Author | SHA1 | Date | |
|---|---|---|---|
|
dc9580a659 | ||
|
|
89fb0317e8 | ||
|
|
8981584291 | ||
|
|
e8057e596c | ||
|
|
ea224eed40 |
70 changed files with 11610 additions and 2134 deletions
|
|
@ -99,9 +99,15 @@ jobs:
|
|||
# for before it will run.
|
||||
# libcap2-bin provides capsh, which the Test step uses to drop the two
|
||||
# DAC capabilities so root obeys permission bits.
|
||||
# zsync and appstream are for build-appimage.sh: appimagetool shells
|
||||
# out to zsyncmake rather than bundling it, and reports success while
|
||||
# writing nothing when it is absent, so the script checks for it up
|
||||
# front. appstream provides the appstreamcli that validates the
|
||||
# metainfo both packages ship.
|
||||
apt-get install -y --no-install-recommends \
|
||||
build-essential perl pkg-config \
|
||||
binutils dpkg-dev desktop-file-utils gzip libcap2-bin
|
||||
binutils dpkg-dev desktop-file-utils gzip libcap2-bin \
|
||||
zsync appstream
|
||||
|
||||
- name: Trust the workspace
|
||||
# checkout writes as root into a directory git then considers dubiously
|
||||
|
|
@ -187,6 +193,19 @@ jobs:
|
|||
fi
|
||||
echo "OK: glibc floor $floor <= $MAX_GLIBC"
|
||||
|
||||
- name: Build the AppImage
|
||||
# After the glibc gate, so the cheaper check still fails first. --no-build
|
||||
# reuses the binaries from the Build step, as the .deb step does, and
|
||||
# SOURCE_DATE_EPOCH pins the date substituted into the AppStream release
|
||||
# entry the same way it pins the .deb changelog.
|
||||
#
|
||||
# The script downloads appimagetool and the AppImage runtime, both pinned
|
||||
# by sha256, and needs no FUSE: it runs appimagetool with
|
||||
# APPIMAGE_EXTRACT_AND_RUN so the container needs no /dev/fuse.
|
||||
run: |
|
||||
SOURCE_DATE_EPOCH="$(git log -1 --pretty=%ct)" \
|
||||
./packaging/build-appimage.sh --no-build
|
||||
|
||||
- name: Package the binaries
|
||||
# A tarball for anyone not installing the .deb, stripped to match what
|
||||
# build-deb.sh ships.
|
||||
|
|
@ -214,9 +233,15 @@ jobs:
|
|||
- uses: actions/upload-artifact@v3-node20
|
||||
with:
|
||||
name: linux-x86_64
|
||||
# The .zsync is not optional: the update URL baked into every AppImage
|
||||
# points at it, so leaving it unpublished breaks AppImageUpdate for
|
||||
# everyone who already installed one. It is the one asset named without
|
||||
# a version, because that URL has to keep resolving across releases.
|
||||
path: |
|
||||
dist/*.deb
|
||||
dist/*.tar.gz
|
||||
dist/*.AppImage
|
||||
dist/*.zsync
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
|
|
|
|||
153
Cargo.lock
generated
153
Cargo.lock
generated
|
|
@ -783,25 +783,6 @@ dependencies = [
|
|||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
|
||||
dependencies = [
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.20"
|
||||
|
|
@ -869,15 +850,6 @@ dependencies = [
|
|||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deranged"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d630bccd429a5bb5a64b5e94f693bfc48c9f8566418fda4c494cc94f911f87cc"
|
||||
dependencies = [
|
||||
"powerfmt",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
|
|
@ -1099,12 +1071,6 @@ dependencies = [
|
|||
"winit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
|
||||
|
||||
[[package]]
|
||||
name = "emath"
|
||||
version = "0.32.3"
|
||||
|
|
@ -2137,12 +2103,6 @@ dependencies = [
|
|||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linked-hash-map"
|
||||
version = "0.5.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.4.14"
|
||||
|
|
@ -2209,25 +2169,6 @@ version = "0.4.33"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
|
||||
|
||||
[[package]]
|
||||
name = "lopdf"
|
||||
version = "0.32.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e775e4ee264e8a87d50a9efef7b67b4aa988cf94e75630859875fc347e6c872b"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"encoding_rs",
|
||||
"flate2",
|
||||
"itoa",
|
||||
"linked-hash-map",
|
||||
"log",
|
||||
"md5",
|
||||
"nom 7.1.3",
|
||||
"rayon",
|
||||
"time",
|
||||
"weezl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lopdf"
|
||||
version = "0.42.0"
|
||||
|
|
@ -2245,7 +2186,7 @@ dependencies = [
|
|||
"itoa",
|
||||
"log",
|
||||
"md-5",
|
||||
"nom 8.0.0",
|
||||
"nom",
|
||||
"rand 0.10.2",
|
||||
"rangemap",
|
||||
"sha2",
|
||||
|
|
@ -2265,12 +2206,6 @@ dependencies = [
|
|||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "md5"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.4"
|
||||
|
|
@ -2323,12 +2258,6 @@ dependencies = [
|
|||
"unicase",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "minimal-lexical"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.9"
|
||||
|
|
@ -2452,16 +2381,6 @@ version = "0.2.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "7.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"minimal-lexical",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "8.0.0"
|
||||
|
|
@ -2490,12 +2409,6 @@ dependencies = [
|
|||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
|
|
@ -2959,7 +2872,7 @@ dependencies = [
|
|||
"encoding_rs",
|
||||
"euclid",
|
||||
"log",
|
||||
"lopdf 0.42.0",
|
||||
"lopdf",
|
||||
"postscript",
|
||||
"type1-encoding-parser",
|
||||
"unicode-normalization",
|
||||
|
|
@ -3122,12 +3035,6 @@ version = "0.14.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306"
|
||||
|
||||
[[package]]
|
||||
name = "powerfmt"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.17"
|
||||
|
|
@ -3190,9 +3097,10 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "quicksearch-core"
|
||||
version = "0.9.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"argon2",
|
||||
"cfb",
|
||||
"chardetng",
|
||||
"ctrlc",
|
||||
"encoding_rs",
|
||||
|
|
@ -3202,7 +3110,6 @@ dependencies = [
|
|||
"kamadak-exif",
|
||||
"libc",
|
||||
"lofty",
|
||||
"lopdf 0.32.0",
|
||||
"mime_guess",
|
||||
"notify",
|
||||
"pdf-extract",
|
||||
|
|
@ -3222,7 +3129,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "quicksearch-gui"
|
||||
version = "0.9.2"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"eframe",
|
||||
|
|
@ -3331,26 +3238,6 @@ version = "0.6.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539"
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
|
||||
dependencies = [
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
|
||||
dependencies = [
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.4.1"
|
||||
|
|
@ -3956,36 +3843,6 @@ dependencies = [
|
|||
"zune-jpeg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.43"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83bde6f1ec10e72d583d91623c939f623002284ef622b87de38cfd546cbf2031"
|
||||
dependencies = [
|
||||
"deranged",
|
||||
"num-conv",
|
||||
"powerfmt",
|
||||
"serde",
|
||||
"time-core",
|
||||
"time-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "time-core"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b"
|
||||
|
||||
[[package]]
|
||||
name = "time-macros"
|
||||
version = "0.2.24"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3"
|
||||
dependencies = [
|
||||
"num-conv",
|
||||
"time-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinystr"
|
||||
version = "0.7.6"
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ members = [
|
|||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.9.3"
|
||||
version = "1.0.5"
|
||||
edition = "2021"
|
||||
license = "GPL-3.0-or-later"
|
||||
authors = ["Jeremy <jeremy@karsttech.com>"]
|
||||
|
|
|
|||
171
README.md
171
README.md
|
|
@ -6,6 +6,16 @@ sidecar), keeps it fresh automatically with filesystem watchers and
|
|||
periodic reindexing, and serves ranked search-as-you-type results in a
|
||||
compact egui desktop app, or straight to your terminal.
|
||||
|
||||
## AI Disclaimer
|
||||
QuickSearch has a core designed by it's developer and built by hand, however the majority of it's codebase including it's GUI was designed by a human and built using AI agents with human review, improvements, and testing.
|
||||
|
||||
## GitHub Mirror
|
||||
The primary home of this software is:
|
||||
https://code.karsttech.com/jeremy/quick_search
|
||||
|
||||
The code is also mirrored to GitHub for easier bug reporting and issue tracking:
|
||||
https://github.com/DataScienceDIY/quick_search
|
||||
|
||||
## Build & run
|
||||
|
||||
```sh
|
||||
|
|
@ -66,7 +76,7 @@ and exit codes behave normally. On Unix `quicksearch` also does both, and
|
|||
|
||||
```sh
|
||||
./packaging/build-deb.sh
|
||||
sudo apt install ./dist/quicksearch_0.1.0_amd64.deb
|
||||
sudo apt install ./dist/quicksearch_1.0.2_amd64.deb
|
||||
```
|
||||
|
||||
The script builds the release binary, strips it, and assembles a `.deb` with
|
||||
|
|
@ -85,6 +95,7 @@ The package installs:
|
|||
| `/usr/share/applications/quicksearch.desktop` | menu entry, so QuickSearch appears in the app launcher |
|
||||
| `/usr/share/icons/hicolor/{16,22,24,32,48,64,128,256}x*/apps/` | icons at each size |
|
||||
| `/usr/share/icons/hicolor/scalable/apps/quicksearch.svg` | the source icon |
|
||||
| `/usr/share/metainfo/com.karsttech.quicksearch.metainfo.xml` | AppStream data, so software centres show a real listing |
|
||||
| `/usr/share/man/man1/quicksearch{,-cli}.1.gz` | `man quicksearch`; the `-cli` page is a `.so` stub pointing at it |
|
||||
| `/usr/share/doc/quicksearch/` | copyright, changelog, README, `config_example.toml` |
|
||||
|
||||
|
|
@ -118,6 +129,32 @@ largest first, and `save(..., format="ICO", sizes=[...], append_images=rest)`
|
|||
— passing the images rather than one image and a size list is what keeps the
|
||||
committed pixels instead of resampling them.
|
||||
|
||||
## Install (AppImage)
|
||||
|
||||
For anything that is not Debian or Ubuntu. Download
|
||||
`quicksearch-<version>-x86_64.AppImage` from the release page, make it
|
||||
executable and run it:
|
||||
|
||||
```sh
|
||||
chmod +x quicksearch-1.0.4-x86_64.AppImage
|
||||
./quicksearch-1.0.4-x86_64.AppImage
|
||||
```
|
||||
|
||||
If it fails to start with a FUSE error — some distributions no longer install
|
||||
FUSE by default — either install the distribution's FUSE package or run it
|
||||
unpacked:
|
||||
|
||||
```sh
|
||||
APPIMAGE_EXTRACT_AND_RUN=1 ./quicksearch-1.0.4-x86_64.AppImage
|
||||
```
|
||||
|
||||
To build one, `./packaging/build-appimage.sh` takes the same flags as
|
||||
`build-deb.sh` (`--no-build`, `--no-strip`, `-o DIR`). It downloads
|
||||
`appimagetool` and the AppImage runtime, both pinned by sha256 and cached under
|
||||
`~/.cache/quicksearch`, and needs `zsync` and `appstream` installed for
|
||||
`zsyncmake` and `appstreamcli`. `APPIMAGETOOL` points it at a copy you already
|
||||
have. It needs no FUSE itself, which is what lets CI build one in a container.
|
||||
|
||||
## Install (Windows)
|
||||
|
||||
Download `quicksearch-<version>-windows-x86_64-setup.exe` from the release
|
||||
|
|
@ -172,7 +209,7 @@ installer, so it takes `/S` for a silent install and `/D=` for the directory
|
|||
(last argument, unquoted):
|
||||
|
||||
```bat
|
||||
quicksearch-0.9.1-windows-x86_64-setup.exe /S /D=C:\Tools\QuickSearch
|
||||
quicksearch-1.0.2-windows-x86_64-setup.exe /S /D=C:\Tools\QuickSearch
|
||||
```
|
||||
|
||||
The `.zip` on the release page is the alternative to all of this: the same two
|
||||
|
|
@ -211,7 +248,17 @@ inside that folder.
|
|||
what each tab does — pointing here for everything technical.
|
||||
|
||||
The bottom status bar always shows what the indexer is doing (phase,
|
||||
percent, files/sec) or the total indexed file count when idle.
|
||||
percent, files/sec) or the total indexed file count when idle. Applying a
|
||||
settings change to the index counts as something the indexer is doing: it
|
||||
reports its progress there and in the Manage Index tab, and says what it
|
||||
removed for a few seconds after it finishes, so a change that takes a
|
||||
millisecond is as visible as one that takes minutes.
|
||||
|
||||
Quitting while a settings change is still being applied asks first. Leaving
|
||||
is never refused — the work stops promptly and the index stays consistent —
|
||||
but it stops part-way, so entries you excluded can still turn up in search
|
||||
results until indexing runs again. The next launch says so, with a button to
|
||||
start that run; in automatic mode the periodic reindex does it for you.
|
||||
|
||||
### Terminal
|
||||
|
||||
|
|
@ -253,12 +300,9 @@ rebuilds the index — there is no in-place conversion.
|
|||
- Forgot the password? The unlock screen can delete the index and disable
|
||||
protection; your files are untouched and re-indexing rebuilds it.
|
||||
|
||||
What this protects: the index file at rest — disk theft, backups, other
|
||||
accounts reading the file. What it does not protect: a compromised running
|
||||
session (the derived key is in process memory while the app runs), and the
|
||||
files themselves, which are exactly as readable as before. A wrong
|
||||
password can never wipe the index; it is refused without touching the
|
||||
file.
|
||||
This protects the index itself and for attacks like data theft.
|
||||
Anything malicious running with user permissions could bypass this protection,
|
||||
but anything with user permissions can also access all of the same files.
|
||||
|
||||
### Query syntax
|
||||
|
||||
|
|
@ -317,10 +361,12 @@ default index goes to `~/.local/share/quicksearch/index.sqlite`
|
|||
|
||||
Defaults follow the platform. The first indexing root is your home
|
||||
directory or `%USERPROFILE%`; `include_hidden = false` skips dot-files
|
||||
everywhere and additionally anything marked Hidden or System on Windows,
|
||||
which is what keeps `AppData`, `$RECYCLE.BIN` and `System Volume
|
||||
Information` out of the index; and ignore patterns are matched
|
||||
case-insensitively on Windows and macOS, matching the filesystem.
|
||||
everywhere and additionally anything marked Hidden on Windows, which is
|
||||
what keeps `AppData`, `$RECYCLE.BIN` and `System Volume Information` out
|
||||
of the index — the System attribute alone is not enough, because cloud
|
||||
sync roots carry it purely to get a branded folder icon; and ignore
|
||||
patterns are matched case-insensitively on Windows and macOS, matching
|
||||
the filesystem.
|
||||
|
||||
**Portable mode**: a `config.toml` sitting next to the `quicksearch`
|
||||
binary overrides the user config entirely, and relative paths inside any
|
||||
|
|
@ -377,12 +423,25 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
|
|||
open applies the Argon2id-derived raw key (`security.rs`, process-global
|
||||
in `db/key.rs`) before anything reads the file; a wrong key is a tagged
|
||||
`KEY_MISMATCH` error, structurally distinct from the schema drift that
|
||||
may wipe, so it can never destroy an intact index.
|
||||
may wipe, so it can never destroy an intact index. Each kind of connection
|
||||
takes a page cache sized for what it does and how long it lives, rather than
|
||||
one figure applied everywhere (`db/schema.rs` sets six profiles and argues
|
||||
each): the caches are `malloc`ed, so a connection that scans a table and is
|
||||
then held — the coordinator's writer, before it learned to let go when idle —
|
||||
keeps that memory for the life of the process. Search is the one deliberately
|
||||
large one, because it is the only cache reused often enough to pay for
|
||||
itself, and it is released once searching stops.
|
||||
- **Indexing** (`indexing.rs`, `file_handling.rs`): full runs walk each
|
||||
root (`filtered_walk` prunes hidden/ignored subtrees before descending),
|
||||
classify files by mtime into insert/update/skip, batch-write metadata,
|
||||
sweep stale rows, then extract content (plaintext, RTF, Office, PDF,
|
||||
audio tags, EXIF; see `extract/`) for FTS. Files whose extension no MIME
|
||||
sweep stale rows, then extract content (plaintext, RTF, Office — both the
|
||||
OOXML/ODF zip formats and the pre-2007 binary `.doc`/`.xls`/`.ppt`, whose
|
||||
OLE2 streams are read in `extract/ole.rs` — PDF, audio tags, EXIF; see
|
||||
`extract/`) for FTS. PDFs are parsed once, with the text and the `Info`
|
||||
dictionary taken off the same document: the two-parse version that preceded
|
||||
it was a run's largest single memory consumer, and it was what pulled a
|
||||
second copy of `lopdf` — and with it rayon's never-torn-down thread pool —
|
||||
into the build. Files whose extension no MIME
|
||||
table knows — including extensionless ones like `README` or `Makefile` —
|
||||
are sniffed from their head bytes and indexed as text only when that head
|
||||
is provably text: valid UTF-8, or BOM-marked (`mime.rs`, `textenc.rs`).
|
||||
|
|
@ -397,7 +456,12 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
|
|||
it completed or was stopped — with an optimize pass on its own connection:
|
||||
checkpoint, VACUUM if the file has at least 10% slack to reclaim, `PRAGMA
|
||||
optimize`, checkpoint again. Progress streams through a polled
|
||||
`IndexingStatus`, which reads `Optimizing` for the duration of that pass.
|
||||
`IndexingStatus`, which reads `Optimizing` for the duration of that pass —
|
||||
and `Preparing` for everything a run does before its first file is walked:
|
||||
waiting on the previous run's thread, opening the index (a WAL recovery
|
||||
lands here), and reconciling a changed configuration. Each carries the run's
|
||||
start time, so a prologue that outlasts the walk on a large index reads as
|
||||
slow work rather than a hang.
|
||||
- **Scope reconciliation** (`scope.rs`): the index is a cache of what a walk
|
||||
under the configured roots would produce, so a configuration change is a
|
||||
difference between the two rather than a reason to start over.
|
||||
|
|
@ -412,6 +476,20 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
|
|||
closed behave like one edited live. The scan is per-root, by `[lo, hi)`
|
||||
range: a symlink target stored outside every root has no owning root and
|
||||
therefore no rules that could be applied to it, so it is never visited.
|
||||
Whichever of the two applies it, a pass that *finishes* records what it
|
||||
reconciled against — everything but the three rebuild-only keys, which no
|
||||
scan can satisfy. That record is the whole convergence condition: an
|
||||
abandoned pass leaves it alone and the next run picks the work back up,
|
||||
while a completed one stops every later run from re-deriving the same plan
|
||||
and rescanning every row to redo work already done. Both report a live
|
||||
`ReconcileProgress` while they scan, since on a large index this is minutes
|
||||
of work with no files moving to show for it. Both can also be abandoned:
|
||||
`advance` reads a cancel flag before every statement, and the statement
|
||||
already running — one `DELETE` can cover a whole root — is ended by
|
||||
`sqlite3_interrupt`, since a flag alone cannot reach inside SQLite. That is
|
||||
what makes closing the window during a prune immediate instead of a wait
|
||||
the desktop offers to kill. `scope::outstanding_work` asks the record what
|
||||
is still owed, which is how the GUI knows to remind you at the next launch.
|
||||
- **Coordinator** (`coordinator.rs`): the object binaries construct.
|
||||
Owns the `IndexingService`, the debouncing filesystem watcher
|
||||
(`watcher.rs`), and the mode state machine (Auto / Manual, persisted as
|
||||
|
|
@ -433,7 +511,11 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
|
|||
- **Search** (`search/`): `SearchService` runs one worker thread; each
|
||||
query is a *generation*. New queries interrupt the in-flight SQLite
|
||||
statement (`InterruptHandle`) and stale generations stop cooperatively,
|
||||
so typing never waits. The cascade streams rank-ordered batches: one
|
||||
so typing never waits. The worker keeps its connection across requests and
|
||||
drops it once searching stops, so a typing session runs against a page cache
|
||||
that is already warm instead of rebuilding one per keystroke; because a
|
||||
rebuild or clear puts a *new* file at the *same* path, an index generation
|
||||
counter (`db::index_epoch`) is what tells the held connection to reopen. The cascade streams rank-ordered batches: one
|
||||
`files` scan classifies exact/case/substring filename matches (ranks
|
||||
1–4) and, since a path contains its own name, sets aside full-path
|
||||
matches from the same rows (ranks 9–10); one FTS phrase probe verified
|
||||
|
|
@ -444,10 +526,12 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
|
|||
flush last, so weaker matches only ever append. All SQL is
|
||||
parameterized; structured filters from the query language (`query/`)
|
||||
are ANDed onto every pass.
|
||||
- **Baloo compatibility** (`cli.rs`, `mime.rs`): read-only endpoints
|
||||
(`status_for_path`, `list_failed`, `index_size_breakdown`, …) and a
|
||||
Baloo-shaped type model, groundwork for a future `balooctl`-compatible
|
||||
layer.
|
||||
- **Baloo compatibility** (`cli.rs`, `mime.rs`): the read API this repo's
|
||||
parent consumes — `status_for_path`, `list_failed`,
|
||||
`index_size_breakdown`, `pending_content_count`, `clear_path` — plus a
|
||||
Baloo-shaped type model. Only `index_counts` has a caller inside this
|
||||
repository; the rest are a compatibility surface for the parent's
|
||||
`balooctl` layer and are not dead code.
|
||||
- **Logging** (`log.rs`): background reporting goes through `log_info!` /
|
||||
`log_warn!` rather than `println!`/`eprintln!`. Each writes its line to
|
||||
stderr *and* appends it to a bounded in-memory ring (newest 5000 lines,
|
||||
|
|
@ -456,7 +540,8 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
|
|||
search hits, usage, the error a command exits with — stays on stdio.
|
||||
- **Platform differences** (`platform.rs`): the single home for `#[cfg]`.
|
||||
Home directory lookup, what counts as a hidden entry (dot-prefix, plus
|
||||
the Hidden/System attributes on Windows), network-filesystem detection
|
||||
the Hidden attribute on Windows — System deliberately excluded, since
|
||||
cloud sync roots set it to get a folder icon), network-filesystem detection
|
||||
(`/proc/mounts` against `GetDriveTypeW`), path collation, and the
|
||||
watch-registration strategy all live here, so the rest of the crate can
|
||||
ask a question rather than test a target. Anything decidable from a
|
||||
|
|
@ -490,9 +575,31 @@ pagination: the table is virtualized, so a single scroll list capped at
|
|||
- `cargo test -p quicksearch-core`: unit + integration suites (cascade
|
||||
ranking, cancellation, incremental indexing, coordinator modes, config
|
||||
resolution, fuzzy matcher vs. brute-force oracle).
|
||||
- `cargo test -p quicksearch-gui`: formatter/tracker/CLI-parsing units.
|
||||
- `cargo test -p quicksearch-gui`: formatter/tracker/CLI-parsing units plus
|
||||
headless egui tests that drive the real widgets — building an input frame,
|
||||
synthesizing clicks and reading back the painted text (`test_ui.rs`) — over
|
||||
the search and manage tabs, the options editor, the unlock gate, the logs
|
||||
and duplicates tabs, and query highlighting.
|
||||
- `QSB_SNIPPET_PERF=1 cargo test --release -p quicksearch-core --test
|
||||
snippet_perf -- --nocapture`: snippet pipeline benchmark.
|
||||
- `QSB_SEARCH_PERF=1 cargo test --release -p quicksearch-core --test
|
||||
search_perf -- --nocapture`: what a warm page cache is worth to search, swept
|
||||
across cache ceilings and run both encrypted and not. It exists because the
|
||||
right size for `PRAGMAS_SEARCH` is not something to reason about: encrypted,
|
||||
the curve has a cliff at the working set, because SQLCipher caches pages
|
||||
decrypted and a miss below that costs an AES-CBC plus an HMAC-SHA512 per
|
||||
4 KiB page. Unencrypted it is flat. Read it before changing that number.
|
||||
- Memory probes, all under `crates/quicksearch-core/examples/`:
|
||||
`memprobe <cold|warm> <root> <db>` reports an indexing run's peak *and* what
|
||||
it settles at once idle — the gap between those is the memory a process
|
||||
keeps for nothing, since glibc's `free` returns chunks to its arena rather
|
||||
than to the kernel. `rssprobe <pid> [duration_s]` attributes a *running*
|
||||
process's footprint instead, splitting anonymous heap (ours) from
|
||||
file-backed pages (the binary, libc, the GL stack), reading `Private_Dirty`
|
||||
rather than `VmRSS`, and counting glibc's arenas so retention is
|
||||
distinguishable from live data. It reads another process's `/proc`, so it
|
||||
measures a build made without knowing it would be measured.
|
||||
`indexprobe` and `walkprobe` answer "how fast" rather than "how much".
|
||||
- `.forgejo/workflows/ci.yml`: builds both platforms on every push to `master`
|
||||
and every pull request. To cut a release, bump `[workspace.package] version`
|
||||
in `Cargo.toml` and push the commit on a branch named `Release...`; CI runs
|
||||
|
|
@ -501,8 +608,15 @@ pagination: the table is virtualized, so a single scroll list capped at
|
|||
crates, so the `--locked` build after it still fails on a dependency added or
|
||||
bumped without committing `Cargo.lock`. Once both
|
||||
build jobs are green, CI tags that commit `v<version>` and publishes a release
|
||||
with the `.deb`, a Linux tarball, the Windows installer and a Windows zip
|
||||
attached; pushing a `v*` tag by hand does the same thing. The version is never taken from the branch
|
||||
with the `.deb`, an AppImage and its `.zsync` sidecar, a Linux tarball, the
|
||||
Windows installer and a Windows zip attached; pushing a `v*` tag by hand does
|
||||
the same thing. The sidecar is the one asset named without a version, because
|
||||
every released AppImage embeds its URL and that URL has to keep resolving as
|
||||
releases come and go — Forgejo resolves the literal tag `latest` to the newest
|
||||
release and looks an asset up by name, so it is always at
|
||||
`.../releases/download/latest/quicksearch-x86_64.AppImage.zsync`. Note that is
|
||||
`/releases/download/latest/`, not the GitHub-style `/releases/latest/download/`,
|
||||
which Forgejo does not implement. The version is never taken from the branch
|
||||
name, and a tag that already exists at a different commit aborts the release
|
||||
rather than shipping two builds under one version. Every build carries its
|
||||
identity: `crates/quicksearch-gui/build.rs` bakes in the commit CI passes as
|
||||
|
|
@ -512,7 +626,10 @@ pagination: the table is virtualized, so a single scroll list capped at
|
|||
`unknown` there rather than failing. The Linux job runs in an Ubuntu
|
||||
22.04 container on purpose — `packaging/build-deb.sh` reads the package's
|
||||
`libc6` floor from the binary it just built, so the builder's glibc becomes
|
||||
the package's minimum, and 22.04 pins it at 2.35. The Windows job
|
||||
the package's minimum, and 22.04 pins it at 2.35. The AppImage is cut from
|
||||
that same binary and bundles no libraries, so 2.35 is its floor too — it is
|
||||
the one number that decides how far either Linux artifact reaches.
|
||||
The Windows job
|
||||
cross-compiles with mingw-w64 and fails if either `.exe` picks up a
|
||||
dependency on a non-system DLL, then builds both Windows assets from those
|
||||
binaries — `packaging/build-installer.sh` runs `makensis`, which is a Linux
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ database_path = "~/.local/share/quicksearch/index.sqlite"
|
|||
# buttons on the Manage Index tab write this value, so the mode you left the
|
||||
# app in is the mode it starts in.
|
||||
auto_index = true
|
||||
reindex_interval_minutes = 1440
|
||||
reindex_interval_minutes = 60
|
||||
# Follow symbolic links during directory walks. Applies to links pointing at
|
||||
# files as well as at directories: with this off a symlink is not resolved at
|
||||
# all, so its target is never indexed — which matters because a target can
|
||||
|
|
@ -43,8 +43,12 @@ reindex_interval_minutes = 1440
|
|||
# that are no longer in scope; turning it on reindexes to find them.
|
||||
follow_symlinks = false
|
||||
# Index hidden files and directories. That means dot-files everywhere, and
|
||||
# additionally anything carrying the Hidden or System attribute on Windows
|
||||
# (AppData, $RECYCLE.BIN, System Volume Information, pagefile.sys ...).
|
||||
# additionally anything carrying the Hidden attribute on Windows (AppData,
|
||||
# $RECYCLE.BIN, System Volume Information, pagefile.sys ...). The System
|
||||
# attribute on its own does not count: Windows honours the desktop.ini inside
|
||||
# a folder only if the folder carries System or Read-only, so cloud sync roots
|
||||
# (ownCloud, Nextcloud, OneDrive, Google Drive) and any folder given a custom
|
||||
# icon carry it purely to get that icon, and are indexed normally.
|
||||
# Turning this off removes the hidden entries already indexed.
|
||||
include_hidden = false
|
||||
# Empty = extract text from every supported format. Non-empty = content
|
||||
|
|
@ -145,6 +149,13 @@ use_keychain = false
|
|||
# together (0.5 – 2.5). Ctrl +/- and Ctrl 0 adjust it temporarily at
|
||||
# runtime; this value is the persistent baseline.
|
||||
scale = 1.1
|
||||
# Written by QuickSearch, not by you: the folders that have already shown
|
||||
# the "more subfolders than the watcher can follow" warning, so restarting
|
||||
# does not repeat it. Keyed by folder rather than a single flag so that
|
||||
# adding a folder warns again — the trade-off changed — and pruned to the
|
||||
# current folder list whenever it is applied. Deleting it just means the
|
||||
# warnings come back once each.
|
||||
watch_cap_warned_roots = []
|
||||
|
||||
[search]
|
||||
# Start with the fuzzy passes enabled.
|
||||
|
|
|
|||
|
|
@ -29,6 +29,11 @@ walkdir = "2.5.0"
|
|||
# stored, so zstd/bzip2/aes-crypto are all dead weight here.
|
||||
zip = { version = "0.6", default-features = false, features = ["deflate"] }
|
||||
quick-xml = "0.31"
|
||||
# OLE2 compound-file reader, for the pre-2007 binary Office formats
|
||||
# (.doc/.xls/.ppt) whose text lives in named streams rather than a zip. Already
|
||||
# in the lockfile transitively via infer, so naming it directly compiles
|
||||
# nothing new.
|
||||
cfb = "0.7"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
toml = "0.8"
|
||||
mime_guess = "2.0"
|
||||
|
|
@ -44,8 +49,12 @@ chardetng = "1.0"
|
|||
# RTF text extraction. Pure Rust; with the default `jsbindings` feature off
|
||||
# (it exists for the crate's WASM build) it depends only on serde.
|
||||
rtf-parser = { version = "0.4", default-features = false }
|
||||
# `lopdf` is deliberately NOT declared here. `pdf-extract` re-exports it
|
||||
# (`pub use lopdf::*`), and `extract/pdf.rs` reaches it that way. Naming it
|
||||
# directly resolves a *second*, older copy alongside pdf-extract's — which is
|
||||
# what used to make every PDF parse twice, and what pulled in rayon (whose
|
||||
# global thread pool is never torn down), chrono, time, md5 and a second nom.
|
||||
pdf-extract = "0.12"
|
||||
lopdf = "0.32"
|
||||
lofty = "0.19"
|
||||
kamadak-exif = "0.5"
|
||||
notify = "6.1"
|
||||
|
|
@ -62,8 +71,10 @@ regex = "1"
|
|||
libc = "0.2"
|
||||
|
||||
# `GetDriveTypeW` (a mapped drive letter is the only way to spot an SMB share
|
||||
# that isn't written as UNC) plus the FILE_ATTRIBUTE_* constants for hidden
|
||||
# detection. Pinned to 0.52 deliberately: walkdir → winapi-util already
|
||||
# that isn't written as UNC) plus the FILE_ATTRIBUTE_* constants, which
|
||||
# `platform.rs` spells out for itself so its tests run on Linux and then
|
||||
# const-asserts against this crate on Windows builds.
|
||||
# Pinned to 0.52 deliberately: walkdir → winapi-util already
|
||||
# resolves exactly that version, so this adds no new crate compilations.
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
# GetDriveTypeW and the FILE_ATTRIBUTE_* constants live in
|
||||
|
|
@ -73,6 +84,9 @@ libc = "0.2"
|
|||
# GetCurrentThread need Win32_Foundation for HANDLE/BOOL.
|
||||
windows-sys = { version = "0.52", features = [
|
||||
"Win32_Foundation",
|
||||
# SECURITY_ATTRIBUTES, which `CreateFileW`'s signature names even though
|
||||
# the directory-count path passes null for it.
|
||||
"Win32_Security",
|
||||
"Win32_Storage_FileSystem",
|
||||
"Win32_System_Threading",
|
||||
"Win32_System_WindowsProgramming",
|
||||
|
|
|
|||
|
|
@ -198,7 +198,7 @@ fn prose(rng: &mut Rng, target: usize) -> String {
|
|||
let mut s = String::with_capacity(target + 16);
|
||||
while s.len() < target {
|
||||
s.push_str(WORDS[rng.next() as usize % WORDS.len()]);
|
||||
s.push(if rng.next() % 12 == 0 { '\n' } else { ' ' });
|
||||
s.push(if rng.next().is_multiple_of(12) { '\n' } else { ' ' });
|
||||
}
|
||||
s.truncate(target);
|
||||
s
|
||||
|
|
|
|||
|
|
@ -52,13 +52,29 @@
|
|||
//! peak to the walk or to extraction; a sampled peak far under VmHWM means
|
||||
//! the real spike was shorter than the sampling interval.
|
||||
//!
|
||||
//! Nothing here is evictable page cache. The connection sets no `mmap_size`,
|
||||
//! so SQLite reads the index through its own `malloc`'d page cache (`PRAGMA
|
||||
//! cache_size`, 10000 pages ≈ 40 MiB) rather than mapping the file, and the
|
||||
//! `by mapping` breakdown confirms it: the index never appears as a
|
||||
//! file-backed mapping. Every megabyte reported is memory the process
|
||||
//! actually holds. `/usr/bin/time -v` on this binary reports the same VmHWM,
|
||||
//! as a cross-check that nothing here is fooling itself.
|
||||
//! Nothing here is evictable page cache. The connections set no `mmap_size`,
|
||||
//! so SQLite reads the index through its own `malloc`'d page cache (see
|
||||
//! [`quicksearch_core::db::schema`], where every profile's ceiling is set and
|
||||
//! argued) rather than mapping the file, and the `by mapping` breakdown
|
||||
//! confirms it: the index never appears as a file-backed mapping. Every
|
||||
//! megabyte reported is memory the process actually holds. `/usr/bin/time -v`
|
||||
//! on this binary reports the same VmHWM, as a cross-check that nothing here
|
||||
//! is fooling itself.
|
||||
//!
|
||||
//! # Peak, settled, and the difference between them
|
||||
//!
|
||||
//! `settled RSS` is read once the run is over and the service is idle, and it
|
||||
//! is the number that matters for a process that stays open. It is not the
|
||||
//! peak minus the run's buffers: glibc's `free` returns a chunk to its arena
|
||||
//! rather than to the kernel, so without an explicit
|
||||
//! [`quicksearch_core::platform::release_free_heap`] a run's high-water
|
||||
//! becomes the process's floor for as long as it lives. On a 65k-file tree
|
||||
//! that was the difference between settling at 89 MiB and settling at 34 MiB.
|
||||
//!
|
||||
//! For the *idle* footprint of a running GUI — which includes the window, the
|
||||
//! GL stack and everything this binary deliberately excludes — use
|
||||
//! [`rssprobe`](rssprobe.rs), which reads another process's `/proc` and so can
|
||||
//! measure a build that was made without knowing it would be measured.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, Instant};
|
||||
|
|
@ -80,6 +96,18 @@ const MARKER_INTERVAL_MS: u64 = 500;
|
|||
/// be worse than failing.
|
||||
const TIMEOUT: Duration = Duration::from_secs(3 * 3600);
|
||||
|
||||
/// How long to wait for the run to reach `Idle` before measuring what it left
|
||||
/// behind. Generous because the wait is for `VACUUM` on a multi-gigabyte index
|
||||
/// ([`quicksearch_core::db::open`]'s maintenance connection), not for anything
|
||||
/// that scales with the sampling interval.
|
||||
const SETTLE_TIMEOUT: Duration = Duration::from_secs(600);
|
||||
|
||||
/// The pause after `Idle` is published. The writer thread releases its
|
||||
/// connection and returns free pages to the kernel *after* setting the status,
|
||||
/// so sampling the instant it flips would miss exactly the thing being
|
||||
/// measured.
|
||||
const SETTLE_QUIET: Duration = Duration::from_secs(2);
|
||||
|
||||
/// Resident bytes at the peak, grouped by what the mapping is.
|
||||
///
|
||||
/// A peak figure alone cannot be acted on: 60 MiB of heap is a buffer to
|
||||
|
|
@ -207,7 +235,7 @@ fn run(mode: &str, root: &str, db: &Path, interval: Duration, config_path: Optio
|
|||
file,
|
||||
});
|
||||
|
||||
if ticks % marker_every == 0 && db.exists() {
|
||||
if ticks.is_multiple_of(marker_every) && db.exists() {
|
||||
if let Ok(conn) = rusqlite::Connection::open(db) {
|
||||
if quicksearch_core::db::repo::get_last_full_index(&conn).is_some() {
|
||||
done = true;
|
||||
|
|
@ -222,13 +250,40 @@ fn run(mode: &str, root: &str, db: &Path, interval: Duration, config_path: Optio
|
|||
// nothing that VmHWM would forget anyway.
|
||||
let hwm = vm_hwm();
|
||||
assert!(done, "indexing did not finish within {:?}", TIMEOUT);
|
||||
|
||||
let settled = settle(&service);
|
||||
service.stop_indexing().expect("stop");
|
||||
|
||||
report(
|
||||
mode, elapsed, baseline, hwm, &samples, db, interval, &at_peak,
|
||||
mode, elapsed, baseline, hwm, settled, &samples, db, interval, &at_peak,
|
||||
);
|
||||
}
|
||||
|
||||
/// RSS once the run has fully finished and stopped allocating.
|
||||
///
|
||||
/// The peak says what indexing needs; this says what it *keeps*, and the gap
|
||||
/// between them is the number that decides whether an idle QuickSearch is
|
||||
/// holding memory it has no use for. They differ by more than the run's own
|
||||
/// buffers: `free` under glibc returns a chunk to its arena rather than to the
|
||||
/// kernel, so what is measured here is the floor the process will sit at for
|
||||
/// as long as it stays open.
|
||||
///
|
||||
/// The marker this loop's caller watches for is written *during* the run, so
|
||||
/// maintenance and the final release are still ahead of it — hence waiting for
|
||||
/// `Idle` rather than sampling immediately, and then a little longer, because
|
||||
/// the last of it happens after the status is published.
|
||||
fn settle(service: &IndexingService) -> u64 {
|
||||
let deadline = Instant::now() + SETTLE_TIMEOUT;
|
||||
while Instant::now() < deadline {
|
||||
if matches!(service.get_status(), IndexingStatus::Idle) {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
std::thread::sleep(SETTLE_QUIET);
|
||||
rss().unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Flatten per-root progress into one line's worth of numbers. Roots are
|
||||
/// summed: the process has one address space, so a per-root split would not
|
||||
/// explain a peak that several roots contribute to at once.
|
||||
|
|
@ -254,11 +309,15 @@ fn progress(status: &IndexingStatus) -> (usize, usize, &'static str, String) {
|
|||
(walked, extracted, phase, file)
|
||||
}
|
||||
|
||||
/// Eight columns of one probe run, printed as a line. Grouping them into a
|
||||
/// struct would only move the same eight names one level out.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn report(
|
||||
mode: &str,
|
||||
elapsed: Duration,
|
||||
baseline: u64,
|
||||
hwm: Option<u64>,
|
||||
settled: u64,
|
||||
samples: &[Sample],
|
||||
db: &Path,
|
||||
interval: Duration,
|
||||
|
|
@ -267,8 +326,8 @@ fn report(
|
|||
// One line per 5% of the run, so the shape is visible at any duration.
|
||||
let step = (samples.len() / 20).max(1);
|
||||
eprintln!(
|
||||
"\n {:>8} {:>10} {:>9} {:>10} {}",
|
||||
"t", "RSS", "walked", "extracted", "phase"
|
||||
"\n {:>8} {:>10} {:>9} {:>10} phase",
|
||||
"t", "RSS", "walked", "extracted"
|
||||
);
|
||||
for s in samples.iter().step_by(step) {
|
||||
eprintln!(
|
||||
|
|
@ -313,6 +372,17 @@ fn report(
|
|||
);
|
||||
}
|
||||
eprintln!(" baseline RSS {}", mib(baseline));
|
||||
eprintln!(
|
||||
" settled RSS {} (once idle: what the run kept, not what it needed)",
|
||||
mib(settled)
|
||||
);
|
||||
if let Some(h) = hwm {
|
||||
eprintln!(
|
||||
" returned {} of the {} the run took above baseline",
|
||||
mib(h.saturating_sub(settled)),
|
||||
mib(h.saturating_sub(baseline))
|
||||
);
|
||||
}
|
||||
if let Some(h) = hwm {
|
||||
if files > 0 {
|
||||
eprintln!(
|
||||
|
|
|
|||
40
crates/quicksearch-core/examples/oleprobe.rs
Normal file
40
crates/quicksearch-core/examples/oleprobe.rs
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
//! Print what the legacy-Office extractor gets out of a real file.
|
||||
//!
|
||||
//! The unit tests build their own fixtures, which proves the parsers agree
|
||||
//! with the format specs as read. This runs them against files a real
|
||||
//! producer wrote, which is the other half of the question.
|
||||
//!
|
||||
//! cargo run --example oleprobe -- some.doc some.xls some.ppt
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use quicksearch_core::extract::{Extractor, Registry};
|
||||
|
||||
fn main() {
|
||||
let mut failures = 0;
|
||||
for arg in std::env::args().skip(1) {
|
||||
let path = Path::new(&arg);
|
||||
println!("=== {} ===", path.display());
|
||||
match quicksearch_core::extract::office::OfficeExtractor.extract(path) {
|
||||
Ok(content) => {
|
||||
let text = content.text;
|
||||
println!("{} chars", text.chars().count());
|
||||
let preview: String = text.chars().take(400).collect();
|
||||
println!("{}", preview);
|
||||
}
|
||||
Err(e) => {
|
||||
failures += 1;
|
||||
println!("FAILED: {}", e);
|
||||
}
|
||||
}
|
||||
// The dispatch a real index would take: MIME, not extension.
|
||||
let registry = Registry::default_set();
|
||||
let mime = mime_guess::from_path(path)
|
||||
.first()
|
||||
.map(|m| m.essence_str().to_string())
|
||||
.unwrap_or_default();
|
||||
println!("(mime {} claimed: {})", mime, registry.supports(&mime));
|
||||
println!();
|
||||
}
|
||||
std::process::exit(if failures > 0 { 1 } else { 0 });
|
||||
}
|
||||
442
crates/quicksearch-core/examples/rssprobe.rs
Normal file
442
crates/quicksearch-core/examples/rssprobe.rs
Normal file
|
|
@ -0,0 +1,442 @@
|
|||
//! Steady-state memory accounting for an already-running process.
|
||||
//!
|
||||
//! [`memprobe`](memprobe.rs) answers "how much RAM does a run peak at"; this
|
||||
//! answers "how much is the process still holding once it has nothing to do",
|
||||
//! which is a different question with a different answer. Under glibc a peak
|
||||
//! is not returned to the OS when it is freed, so the idle floor is set by the
|
||||
//! largest thing that ever happened rather than by anything currently live.
|
||||
//! Telling those apart is the whole point of this probe.
|
||||
//!
|
||||
//! It reads another process's `/proc` rather than its own, so it measures a
|
||||
//! binary that was built without knowing it would be measured. That matters
|
||||
//! for a before/after: an in-process diagnostic would mean the "before" and
|
||||
//! "after" numbers come from different binaries, and the delta would include
|
||||
//! the diagnostic itself.
|
||||
//!
|
||||
//! ```text
|
||||
//! cargo build -p quicksearch-core --example rssprobe --release
|
||||
//! ./target/release/examples/rssprobe $(pgrep -x quicksearch)
|
||||
//! ./target/release/examples/rssprobe $(pgrep -x quicksearch) 60
|
||||
//! ./target/release/examples/rssprobe $(pgrep -x quicksearch) 60 250
|
||||
//! ```
|
||||
//!
|
||||
//! With no duration it takes one snapshot and prints the full breakdown. With
|
||||
//! one it samples for that many seconds first, so a number can be quoted for
|
||||
//! an *idle* process rather than for whatever the process happened to be doing
|
||||
//! the instant the probe ran.
|
||||
//!
|
||||
//! **`VmRSS` is the wrong number to optimise and it is the one every system
|
||||
//! monitor shows.** Most of it here is `Shared_Clean`: the binary's own text,
|
||||
//! libc, and the Mesa/GL stack the window pulls in. Those pages are shared
|
||||
//! with every other process using them and the kernel drops them under
|
||||
//! pressure. The two numbers worth acting on are:
|
||||
//!
|
||||
//! - **`RssAnon`** — heap and thread stacks. Nothing else. This is the share
|
||||
//! this codebase allocates and can therefore give back.
|
||||
//! - **`Private_Dirty`** — what the process actually costs the machine, from
|
||||
//! `smaps_rollup`. Nobody else is sharing it and it cannot be evicted, only
|
||||
//! freed.
|
||||
//!
|
||||
//! The `glibc arenas` line is the one that says whether an anonymous figure is
|
||||
//! live data or retention. glibc allocates each non-main arena as a 64 MiB
|
||||
//! aligned region and commits into it; it never unmaps one, and `free` returns
|
||||
//! chunks to the arena rather than to the kernel. So a stack of 64 MiB regions
|
||||
//! holding far less than 64 MiB each is freed memory the process is still
|
||||
//! charged for — which `malloc_trim(3)` can return and dropping a buffer
|
||||
//! cannot. Live data does not look like that.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Default gap between samples. One `status` plus one `smaps_rollup` read per
|
||||
/// sample, both small; the interval is set by how fine the timeline should be
|
||||
/// rather than by overhead. `smaps` is read once at the end, not per sample —
|
||||
/// it is the expensive one, being proportional to the mapping count.
|
||||
const DEFAULT_SAMPLE_MS: u64 = 500;
|
||||
|
||||
/// The size glibc reserves per non-main arena (`HEAP_MAX_SIZE` on 64-bit).
|
||||
/// Regions of exactly this size, aligned to it, are arenas rather than
|
||||
/// anything the program asked for.
|
||||
const ARENA_SPAN: u64 = 64 * 1024 * 1024;
|
||||
|
||||
/// One sample: the cheap per-interval reads only.
|
||||
struct Sample {
|
||||
at: Duration,
|
||||
rss: u64,
|
||||
anon: u64,
|
||||
file: u64,
|
||||
private_dirty: u64,
|
||||
pss: u64,
|
||||
/// Carried per sample rather than read once at the end, so it survives the
|
||||
/// process exiting mid-run — it comes from the same `status` read anyway.
|
||||
hwm: u64,
|
||||
}
|
||||
|
||||
/// Resident bytes per mapping, plus the two groupings that answer the
|
||||
/// retention question directly.
|
||||
#[derive(Default)]
|
||||
struct Breakdown {
|
||||
/// Mapping name (or anonymous size class) to resident bytes, descending.
|
||||
entries: Vec<(String, u64)>,
|
||||
/// Non-main glibc arenas: how many, and resident across all of them.
|
||||
arenas: (usize, u64),
|
||||
/// The main arena, which grows by `brk` and is the one the kernel labels.
|
||||
main_heap: u64,
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let mut args = std::env::args().skip(1);
|
||||
let Some(pid) = args.next().and_then(|p| p.parse::<u32>().ok()) else {
|
||||
eprintln!("usage: rssprobe <pid> [duration_s] [interval_ms]");
|
||||
std::process::exit(2);
|
||||
};
|
||||
let duration = Duration::from_secs(
|
||||
args.next()
|
||||
.map(|s| s.parse().expect("duration_s must be a number"))
|
||||
.unwrap_or(0),
|
||||
);
|
||||
let interval = Duration::from_millis(
|
||||
args.next()
|
||||
.map(|s| s.parse().expect("interval_ms must be a number"))
|
||||
.unwrap_or(DEFAULT_SAMPLE_MS)
|
||||
.max(1),
|
||||
);
|
||||
|
||||
if !PathBuf::from(format!("/proc/{}", pid)).exists() {
|
||||
eprintln!("rssprobe: no process {}", pid);
|
||||
std::process::exit(1);
|
||||
}
|
||||
eprintln!("rssprobe pid={} {}", pid, comm(pid));
|
||||
|
||||
let (samples, at_end) = sample_for(pid, duration, interval);
|
||||
report(&samples, &at_end, interval);
|
||||
}
|
||||
|
||||
/// Sample until `duration` elapses. A zero duration still takes one sample,
|
||||
/// so the no-argument form is a snapshot rather than an error.
|
||||
///
|
||||
/// The per-mapping breakdown is taken alongside the last *live* sample rather
|
||||
/// than after the loop, because a process that exits during a long run would
|
||||
/// otherwise report an empty one — which reads like "nothing was resident"
|
||||
/// rather than "nobody was there to ask".
|
||||
fn sample_for(pid: u32, duration: Duration, interval: Duration) -> (Vec<Sample>, Breakdown) {
|
||||
let start = Instant::now();
|
||||
let mut samples = Vec::new();
|
||||
let mut at_end = Breakdown::default();
|
||||
loop {
|
||||
let elapsed = start.elapsed();
|
||||
let Some(s) = sample(pid, elapsed) else {
|
||||
if samples.is_empty() {
|
||||
eprintln!("rssprobe: cannot read /proc/{}: process gone?", pid);
|
||||
std::process::exit(1);
|
||||
}
|
||||
eprintln!(
|
||||
" process exited after {:.1}s; figures below are its last live sample",
|
||||
elapsed.as_secs_f64()
|
||||
);
|
||||
break;
|
||||
};
|
||||
samples.push(s);
|
||||
// smaps is proportional to the mapping count, so it is read once per
|
||||
// sample only when the sample might be the last one — which, until the
|
||||
// loop ends, is every one of them.
|
||||
at_end = breakdown(pid);
|
||||
if start.elapsed() >= duration {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(interval);
|
||||
}
|
||||
(samples, at_end)
|
||||
}
|
||||
|
||||
fn sample(pid: u32, at: Duration) -> Option<Sample> {
|
||||
let status = proc_kv(pid, "status")?;
|
||||
// smaps_rollup is the kernel's own sum over smaps, so it costs one read
|
||||
// rather than one per mapping. Missing only on kernels this old code will
|
||||
// never meet; treat it as zero rather than failing the sample.
|
||||
let rollup = proc_kv(pid, "smaps_rollup").unwrap_or_default();
|
||||
Some(Sample {
|
||||
at,
|
||||
rss: *status.get("VmRSS").unwrap_or(&0),
|
||||
anon: *status.get("RssAnon").unwrap_or(&0),
|
||||
file: *status.get("RssFile").unwrap_or(&0),
|
||||
private_dirty: *rollup.get("Private_Dirty").unwrap_or(&0),
|
||||
pss: *rollup.get("Pss").unwrap_or(&0),
|
||||
hwm: *status.get("VmHWM").unwrap_or(&0),
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse a `/proc/<pid>/<file>` of `Key: N kB` lines into bytes.
|
||||
///
|
||||
/// Both `status` and `smaps_rollup` are in this format, and both carry lines
|
||||
/// that are not sizes at all (`Name:`, `State:`, the rollup's address range).
|
||||
/// Anything that does not end in `kB` is skipped rather than guessed at.
|
||||
fn proc_kv(pid: u32, file: &str) -> Option<HashMap<String, u64>> {
|
||||
let text = std::fs::read_to_string(format!("/proc/{}/{}", pid, file)).ok()?;
|
||||
let mut out = HashMap::new();
|
||||
for line in text.lines() {
|
||||
let Some((key, rest)) = line.split_once(':') else {
|
||||
continue;
|
||||
};
|
||||
let mut fields = rest.split_whitespace();
|
||||
let (Some(value), Some("kB")) = (fields.next(), fields.next()) else {
|
||||
continue;
|
||||
};
|
||||
if let Ok(kib) = value.parse::<u64>() {
|
||||
out.insert(key.to_string(), kib * 1024);
|
||||
}
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
fn comm(pid: u32) -> String {
|
||||
std::fs::read_to_string(format!("/proc/{}/comm", pid))
|
||||
.map(|s| s.trim().to_string())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Resident bytes per mapping from `/proc/<pid>/smaps`, summed by name, with
|
||||
/// glibc's arenas identified.
|
||||
///
|
||||
/// The name is the mapping's path, or `[heap]`/`[stack]` for the ones the
|
||||
/// kernel labels. Everything else is anonymous — thread stacks, arena heaps,
|
||||
/// and any single `malloc` large enough to have gone to `mmap` — and is
|
||||
/// bucketed by size class, because individually they are unnamed and there can
|
||||
/// be hundreds of them.
|
||||
fn breakdown(pid: u32) -> Breakdown {
|
||||
let Ok(smaps) = std::fs::read_to_string(format!("/proc/{}/smaps", pid)) else {
|
||||
return Breakdown::default();
|
||||
};
|
||||
|
||||
// Two passes over the same parse: one for the by-name totals the report
|
||||
// prints, one to find arenas — which needs the raw address ranges the
|
||||
// names throw away.
|
||||
let mut by_name: HashMap<String, u64> = HashMap::new();
|
||||
let mut regions: Vec<Region> = Vec::new();
|
||||
let mut current = String::new();
|
||||
|
||||
for line in smaps.lines() {
|
||||
if let Some(rss_kib) = line.strip_prefix("Rss:") {
|
||||
let kib: u64 = rss_kib
|
||||
.split_whitespace()
|
||||
.next()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or(0);
|
||||
*by_name.entry(current.clone()).or_default() += kib * 1024;
|
||||
if let Some(r) = regions.last_mut() {
|
||||
r.rss = kib * 1024;
|
||||
}
|
||||
} else if let Some(region) = parse_map_header(line) {
|
||||
current = region.name.clone();
|
||||
regions.push(region);
|
||||
}
|
||||
}
|
||||
|
||||
let mut entries: Vec<(String, u64)> = by_name.into_iter().filter(|(_, b)| *b > 0).collect();
|
||||
entries.sort_by_key(|(_, bytes)| std::cmp::Reverse(*bytes));
|
||||
|
||||
Breakdown {
|
||||
entries,
|
||||
arenas: find_arenas(®ions),
|
||||
main_heap: regions
|
||||
.iter()
|
||||
.filter(|r| r.name == "[heap]")
|
||||
.map(|r| r.rss)
|
||||
.sum(),
|
||||
}
|
||||
}
|
||||
|
||||
/// One `smaps` mapping, kept with its address range so arenas can be found.
|
||||
struct Region {
|
||||
lo: u64,
|
||||
hi: u64,
|
||||
anonymous: bool,
|
||||
name: String,
|
||||
rss: u64,
|
||||
}
|
||||
|
||||
/// Count glibc's non-main arenas and their resident bytes.
|
||||
///
|
||||
/// An arena is one `mmap` of [`ARENA_SPAN`], aligned to it, which glibc then
|
||||
/// commits into piecewise — so by the time it reaches `smaps` it has usually
|
||||
/// been split into a readable part and a `---p` remainder. Matching a single
|
||||
/// mapping of the full span therefore misses most of them. Contiguous runs of
|
||||
/// anonymous mappings are coalesced first, and a run that spans exactly one
|
||||
/// aligned [`ARENA_SPAN`] is an arena.
|
||||
///
|
||||
/// This can in principle collide with a program that mmaps 64 MiB at 64 MiB
|
||||
/// alignment on purpose. Nothing here does, and the alternative — parsing
|
||||
/// glibc's internal `heap_info` out of the process — is not worth it for a
|
||||
/// figure whose job is to point at a cause rather than to be exact.
|
||||
fn find_arenas(regions: &[Region]) -> (usize, u64) {
|
||||
let mut count = 0;
|
||||
let mut resident = 0;
|
||||
let mut i = 0;
|
||||
while i < regions.len() {
|
||||
if !regions[i].anonymous {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
let lo = regions[i].lo;
|
||||
let mut j = i;
|
||||
let mut rss = 0;
|
||||
while j < regions.len()
|
||||
&& regions[j].anonymous
|
||||
&& regions[j].lo == if j == i { lo } else { regions[j - 1].hi }
|
||||
&& regions[j].hi <= lo + ARENA_SPAN
|
||||
{
|
||||
rss += regions[j].rss;
|
||||
j += 1;
|
||||
}
|
||||
if j > i && regions[j - 1].hi == lo + ARENA_SPAN && lo.is_multiple_of(ARENA_SPAN) {
|
||||
count += 1;
|
||||
resident += rss;
|
||||
i = j;
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
(count, resident)
|
||||
}
|
||||
|
||||
/// Parse one `smaps` header line: `addr-addr perms offset dev inode [path]`.
|
||||
///
|
||||
/// Anonymous mappings have no path; they are named by size class so that a
|
||||
/// hundred 8 MiB regions read as one line rather than a hundred.
|
||||
fn parse_map_header(line: &str) -> Option<Region> {
|
||||
let mut fields = line.split_whitespace();
|
||||
let range = fields.next()?;
|
||||
let (lo, hi) = range.split_once('-')?;
|
||||
let lo = u64::from_str_radix(lo, 16).ok()?;
|
||||
let hi = u64::from_str_radix(hi, 16).ok()?;
|
||||
// Fields 2-5 are perms, offset, dev, inode; anything after is the path.
|
||||
let path = fields.nth(4).unwrap_or("");
|
||||
Some(Region {
|
||||
lo,
|
||||
hi,
|
||||
anonymous: path.is_empty(),
|
||||
name: if path.is_empty() {
|
||||
format!("anon {}", size_class(hi.saturating_sub(lo)))
|
||||
} else {
|
||||
path.to_string()
|
||||
},
|
||||
rss: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Power-of-two bucket, so mappings group by what allocated them rather than
|
||||
/// by their exact size.
|
||||
fn size_class(bytes: u64) -> String {
|
||||
let mib = bytes as f64 / (1024.0 * 1024.0);
|
||||
if mib < 1.0 {
|
||||
"< 1 MiB".to_string()
|
||||
} else {
|
||||
let bucket = 1u64 << (63 - (bytes / (1024 * 1024)).leading_zeros() as u64);
|
||||
format!("~{} MiB", bucket)
|
||||
}
|
||||
}
|
||||
|
||||
fn report(samples: &[Sample], at_end: &Breakdown, interval: Duration) {
|
||||
if samples.len() > 1 {
|
||||
// One line per 5% of the run, so the shape is visible at any duration.
|
||||
let step = (samples.len() / 20).max(1);
|
||||
eprintln!(
|
||||
"\n {:>8} {:>10} {:>10} {:>10} {:>12} {:>10}",
|
||||
"t", "VmRSS", "RssAnon", "RssFile", "PrivDirty", "Pss"
|
||||
);
|
||||
for s in samples.iter().step_by(step) {
|
||||
eprintln!(
|
||||
" {:>7.1}s {:>10} {:>10} {:>10} {:>12} {:>10}",
|
||||
s.at.as_secs_f64(),
|
||||
mib(s.rss),
|
||||
mib(s.anon),
|
||||
mib(s.file),
|
||||
mib(s.private_dirty),
|
||||
mib(s.pss)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let last = samples.last().expect("at least one sample");
|
||||
let elapsed = last.at;
|
||||
eprintln!(
|
||||
"\nsteady state over {:.1}s ({} samples, {:?} apart)",
|
||||
elapsed.as_secs_f64(),
|
||||
samples.len(),
|
||||
interval
|
||||
);
|
||||
|
||||
// Both ends of each range, because a figure that is drifting is a
|
||||
// different finding from one that is flat, and only the flat one can be
|
||||
// called a floor.
|
||||
range_line("VmRSS", samples, |s| s.rss, "");
|
||||
range_line(
|
||||
"RssAnon",
|
||||
samples,
|
||||
|s| s.anon,
|
||||
"heap + thread stacks: the share this code owns",
|
||||
);
|
||||
range_line(
|
||||
"RssFile",
|
||||
samples,
|
||||
|s| s.file,
|
||||
"binary text, libc, Mesa/GL: shared and evictable",
|
||||
);
|
||||
range_line(
|
||||
"Private_Dirty",
|
||||
samples,
|
||||
|s| s.private_dirty,
|
||||
"what this process costs the machine",
|
||||
);
|
||||
range_line("Pss", samples, |s| s.pss, "");
|
||||
|
||||
eprintln!(
|
||||
" {:<16} {:>10} kernel high-water: the peak this floor was left by",
|
||||
"VmHWM",
|
||||
mib(last.hwm)
|
||||
);
|
||||
|
||||
let b = at_end;
|
||||
let (arenas, arena_rss) = b.arenas;
|
||||
eprintln!(
|
||||
"\n glibc heap: {} non-main arena(s) holding {}, plus [heap] {}",
|
||||
arenas,
|
||||
mib(arena_rss),
|
||||
mib(b.main_heap)
|
||||
);
|
||||
if arenas > 0 {
|
||||
eprintln!(
|
||||
" reserved {} across them, so {:.0}% of the arena space is resident",
|
||||
mib(arenas as u64 * ARENA_SPAN),
|
||||
100.0 * arena_rss as f64 / (arenas as u64 * ARENA_SPAN) as f64
|
||||
);
|
||||
}
|
||||
|
||||
if !b.entries.is_empty() {
|
||||
eprintln!("\n resident bytes by mapping:");
|
||||
for (name, bytes) in b.entries.iter().take(12) {
|
||||
eprintln!(" {:>10} {}", mib(*bytes), name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn range_line(label: &str, samples: &[Sample], get: fn(&Sample) -> u64, note: &str) {
|
||||
let last = get(samples.last().expect("at least one sample"));
|
||||
let min = samples.iter().map(get).min().unwrap_or(0);
|
||||
let max = samples.iter().map(get).max().unwrap_or(0);
|
||||
let spread = if samples.len() > 1 && max > min {
|
||||
format!(" (min {}, max {})", mib(min), mib(max))
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
if note.is_empty() {
|
||||
eprintln!(" {:<16} {:>10}{}", label, mib(last), spread);
|
||||
} else {
|
||||
eprintln!(" {:<16} {:>10} {}{}", label, mib(last), note, spread);
|
||||
}
|
||||
}
|
||||
|
||||
fn mib(bytes: u64) -> String {
|
||||
format!("{:.1} MiB", bytes as f64 / (1024.0 * 1024.0))
|
||||
}
|
||||
|
|
@ -1,10 +1,34 @@
|
|||
//! Programmatic read-only query helpers.
|
||||
//! Programmatic query helpers.
|
||||
//!
|
||||
//! Pure functions that open a DB, run a query, and return structured data.
|
||||
//! No stdout, no CLI framing — callers (GUI, future CLI binaries, Set B
|
||||
//! `balooctl`) format the result as they see fit. Mutating operations live
|
||||
//! on [`crate::indexing::IndexingService`] since they require a running
|
||||
//! worker thread.
|
||||
//! No stdout, no CLI framing — the GUI and `quicksearch-cli` format the
|
||||
//! result as they see fit. Indexing operations live on
|
||||
//! [`crate::indexing::IndexingService`] since they require a running worker
|
||||
//! thread.
|
||||
//!
|
||||
//! # These are consumed from outside this repository
|
||||
//!
|
||||
//! QuickSearch is a sub-repo, and **nothing in this module has a caller in
|
||||
//! this tree**. [`status_for_path`], [`list_failed`], [`index_size_breakdown`],
|
||||
//! [`pending_content_count`] and [`clear_path`] are called by the parent
|
||||
//! repository's Baloo compat daemon, which is what reports them to `balooctl`
|
||||
//! and mirrors them into LMDB.
|
||||
//!
|
||||
//! So they are **not dead code**, and their signatures are a compatibility
|
||||
//! surface rather than an internal detail: a search of this repository alone
|
||||
//! will not turn up the callers that break when one changes.
|
||||
//!
|
||||
//! There used to be an `index_counts` here as well, for the GUI's status bar.
|
||||
//! It ran three `COUNT(*)` scans — one of them unindexed — of which the
|
||||
//! frontend displayed one, on a thread and a connection opened afresh every
|
||||
//! five seconds. The surviving figure is published by
|
||||
//! [`crate::coordinator::IndexerState::files`] instead, off the connection the
|
||||
//! coordinator already has.
|
||||
//!
|
||||
//! One exception to the "query helpers" framing: [`clear_path`] mutates. It
|
||||
//! opens its own writer, which sidesteps the single-writer discipline the
|
||||
//! coordinator maintains, so it is safe only against an index no local
|
||||
//! coordinator is running against.
|
||||
|
||||
use rusqlite::{params, OptionalExtension};
|
||||
|
||||
|
|
@ -189,29 +213,6 @@ pub fn pending_content_count(db_path: &str) -> Result<i64, String> {
|
|||
.map_err(|e| format!("pending_content_count: {}", e))
|
||||
}
|
||||
|
||||
/// Cheap aggregate counts for the GUI's idle status bar ("N files
|
||||
/// indexed"). Callers cache the result; it's three COUNT scans, not
|
||||
/// something to run per frame.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct IndexCounts {
|
||||
pub files: i64,
|
||||
pub content_done: i64,
|
||||
pub content_pending: i64,
|
||||
}
|
||||
|
||||
pub fn index_counts(db_path: &str) -> Result<IndexCounts, String> {
|
||||
let conn = open_existing(db_path, false)?;
|
||||
let count = |sql: &str| -> Result<i64, String> {
|
||||
conn.query_row(sql, [], |r| r.get(0))
|
||||
.map_err(|e| format!("index_counts: {}", e))
|
||||
};
|
||||
Ok(IndexCounts {
|
||||
files: count("SELECT COUNT(*) FROM files")?,
|
||||
content_done: count("SELECT COUNT(*) FROM files WHERE content_state = 1")?,
|
||||
content_pending: count("SELECT COUNT(*) FROM files WHERE content_state = 0")?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Remove a single file from the index. Returns whether a row was deleted.
|
||||
/// Keeps FTS/documents/properties in sync via the repo helpers.
|
||||
pub fn clear_path(db_path: &str, path: &str) -> Result<bool, String> {
|
||||
|
|
@ -233,16 +234,7 @@ mod tests {
|
|||
use crate::mime::FileType;
|
||||
|
||||
fn tmp_path() -> std::path::PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"qs-cli-test-{}-{}.sqlite",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
p
|
||||
crate::testutil::scratch_dir("cli").join("index.sqlite")
|
||||
}
|
||||
|
||||
fn seed_fixture(db_path: &str) -> (i64, i64) {
|
||||
|
|
@ -328,6 +320,79 @@ mod tests {
|
|||
std::fs::remove_file(&p).ok();
|
||||
}
|
||||
|
||||
/// The count is "outstanding extraction work", which is a narrower thing
|
||||
/// than "rows without text": a file nothing extracts is settled, not
|
||||
/// waiting, and would otherwise be reported as a backlog that never
|
||||
/// drains.
|
||||
#[test]
|
||||
fn pending_content_count_counts_only_outstanding_work() {
|
||||
let p = tmp_path();
|
||||
let dbp = p.to_str().unwrap();
|
||||
// a is Done, b is Failed — both resolved, neither pending.
|
||||
let _ = seed_fixture(dbp);
|
||||
assert_eq!(pending_content_count(dbp).unwrap(), 0);
|
||||
|
||||
let mut conn = open_or_recreate(dbp, "trigram").unwrap();
|
||||
{
|
||||
let tx = conn.transaction().unwrap();
|
||||
// Claimed by an extractor, text not read yet: this is the backlog.
|
||||
insert_file(
|
||||
&tx,
|
||||
&NewFile {
|
||||
name: "c.txt",
|
||||
path: "/tmp/c.txt",
|
||||
parent: "/tmp",
|
||||
size: 1,
|
||||
mtime: 1,
|
||||
inode: None,
|
||||
device_id: None,
|
||||
mime: Some("text/plain"),
|
||||
ftype: FileType::TEXT,
|
||||
hash: None,
|
||||
needs_content: true,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.expect("unique path");
|
||||
// Nothing extracts this one, so it is NA on arrival and must not
|
||||
// inflate the figure.
|
||||
insert_file(
|
||||
&tx,
|
||||
&NewFile {
|
||||
name: "d.bin",
|
||||
path: "/tmp/d.bin",
|
||||
parent: "/tmp",
|
||||
size: 1,
|
||||
mtime: 1,
|
||||
inode: None,
|
||||
device_id: None,
|
||||
mime: None,
|
||||
ftype: FileType::EMPTY,
|
||||
hash: None,
|
||||
needs_content: false,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.expect("unique path");
|
||||
tx.commit().unwrap();
|
||||
}
|
||||
drop(conn);
|
||||
|
||||
assert_eq!(
|
||||
pending_content_count(dbp).unwrap(),
|
||||
1,
|
||||
"only the file awaiting extraction counts"
|
||||
);
|
||||
|
||||
std::fs::remove_file(&p).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pending_content_count_on_a_missing_db_is_an_error() {
|
||||
let missing = crate::testutil::scratch_dir("cli-missing").join("nope.sqlite");
|
||||
assert!(pending_content_count(missing.to_str().unwrap()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_size_breakdown_counts_rows() {
|
||||
let p = tmp_path();
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ use std::path::{Path, PathBuf};
|
|||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(default)]
|
||||
#[derive(Default)]
|
||||
pub struct Config {
|
||||
pub paths: PathConfig,
|
||||
pub indexing: IndexingConfig,
|
||||
|
|
@ -192,7 +193,7 @@ impl Default for IndexingConfig {
|
|||
fn default() -> Self {
|
||||
IndexingConfig {
|
||||
auto_index: true,
|
||||
reindex_interval_minutes: 24 * 60,
|
||||
reindex_interval_minutes: 60,
|
||||
follow_symlinks: false,
|
||||
include_hidden: false,
|
||||
content_extensions: Vec::new(),
|
||||
|
|
@ -293,19 +294,6 @@ impl Default for UiConfig {
|
|||
}
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Config {
|
||||
paths: PathConfig::default(),
|
||||
indexing: IndexingConfig::default(),
|
||||
processing: ProcessingConfig::default(),
|
||||
search: SearchConfig::default(),
|
||||
ui: UiConfig::default(),
|
||||
security: SecurityConfig::default(),
|
||||
source: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Directories and files excluded from a fresh index.
|
||||
///
|
||||
|
|
@ -451,8 +439,10 @@ impl Config {
|
|||
cfg.source = Some(path.to_path_buf());
|
||||
Ok(cfg)
|
||||
} else {
|
||||
let mut cfg = Config::default();
|
||||
cfg.source = Some(path.to_path_buf());
|
||||
let cfg = Config {
|
||||
source: Some(path.to_path_buf()),
|
||||
..Config::default()
|
||||
};
|
||||
cfg.save()?;
|
||||
Ok(cfg)
|
||||
}
|
||||
|
|
@ -511,7 +501,7 @@ impl Config {
|
|||
}
|
||||
|
||||
/// `indexing_paths` with the same resolution rules as
|
||||
/// [`resolved_database_path`].
|
||||
/// [`Config::resolved_database_path`].
|
||||
pub fn resolved_indexing_paths(&self) -> Vec<PathBuf> {
|
||||
self.paths
|
||||
.indexing_paths
|
||||
|
|
@ -567,11 +557,44 @@ pub fn content_allowed(path: &Path, cfg: &Config) -> bool {
|
|||
}
|
||||
}
|
||||
|
||||
/// Longest name folded without allocating. Comfortably above `NAME_MAX` on the
|
||||
/// filesystems that matter (255 bytes), so the heap path is effectively dead
|
||||
/// code kept for correctness rather than for use.
|
||||
const FOLD_BUF: usize = 256;
|
||||
|
||||
/// Whether `pat` is a plain name with no glob syntax in it.
|
||||
///
|
||||
/// `\` is included even though it is not glob syntax everywhere: a pattern
|
||||
/// containing one is routed to the path set rather than the component set, so
|
||||
/// treating it as literal here would put it in the wrong place.
|
||||
fn is_literal_name(pat: &str) -> bool {
|
||||
!pat.contains(['*', '?', '[', ']', '{', '}', '/', '\\'])
|
||||
}
|
||||
|
||||
/// Compiled ignore patterns, split by matching scope: patterns without a
|
||||
/// path separator match any single path component; the rest match the full
|
||||
/// path. Both use glob syntax.
|
||||
#[derive(Debug)]
|
||||
pub struct IgnoreSet {
|
||||
/// Component patterns that are plain ASCII names — `.git`, `node_modules`,
|
||||
/// `System Volume Information`. Held apart from `component` purely for
|
||||
/// speed, and it is a large difference on Windows.
|
||||
///
|
||||
/// globset compiles a case-insensitive glob by giving up every fast path it
|
||||
/// has: `Glob::literal`, `ext`, `prefix`, `suffix` and `basename_tokens` all
|
||||
/// return `None` the moment `case_insensitive` is set, so every pattern
|
||||
/// falls through to a `RegexSet` scan. Case-sensitively, `.git` and
|
||||
/// `node_modules` compile to a hash lookup and `*.tmp` to an extension
|
||||
/// lookup. That left Windows — which is case-insensitive *and* carries seven
|
||||
/// extra default patterns — running a 12-pattern regex DFA over every single
|
||||
/// directory entry, where Linux did five hash lookups.
|
||||
///
|
||||
/// Folded and compared as **ASCII**, matching [`PATH_COLLATION`]'s reasoning
|
||||
/// exactly: SQLite's `NOCASE` and `LIKE` fold ASCII only, so the glob layer
|
||||
/// agreeing with them is what keeps a path filter from disagreeing with the
|
||||
/// ignore rules. Patterns that are not ASCII stay in `component` and keep
|
||||
/// globset's Unicode folding, so nothing that matched before stops matching.
|
||||
literal_components: std::collections::HashSet<String>,
|
||||
component: globset::GlobSet,
|
||||
path: globset::GlobSet,
|
||||
empty: bool,
|
||||
|
|
@ -579,6 +602,7 @@ pub struct IgnoreSet {
|
|||
|
||||
impl IgnoreSet {
|
||||
pub fn compile(patterns: &[String]) -> Result<IgnoreSet, String> {
|
||||
let mut literal_components = std::collections::HashSet::new();
|
||||
let mut component = globset::GlobSetBuilder::new();
|
||||
let mut path = globset::GlobSetBuilder::new();
|
||||
for pat in patterns {
|
||||
|
|
@ -605,6 +629,17 @@ impl IgnoreSet {
|
|||
if pat.is_empty() {
|
||||
continue;
|
||||
}
|
||||
// A plain ASCII name needs no glob machinery at all — see
|
||||
// `literal_components`. Everything else, including every non-ASCII
|
||||
// pattern, goes on to globset unchanged.
|
||||
if pat.is_ascii() && is_literal_name(pat) {
|
||||
literal_components.insert(if crate::platform::PATHS_ARE_CASE_INSENSITIVE {
|
||||
pat.to_ascii_lowercase()
|
||||
} else {
|
||||
pat.to_string()
|
||||
});
|
||||
continue;
|
||||
}
|
||||
let glob = globset::GlobBuilder::new(pat)
|
||||
.literal_separator(false)
|
||||
// Match the filesystem's own rules, or `node_modules` fails to
|
||||
|
|
@ -612,7 +647,7 @@ impl IgnoreSet {
|
|||
// half of Windows compatibility on its own: `Candidate` folds
|
||||
// `\` to `/` when matching, and backslash-as-escape is off
|
||||
// wherever `\` is a separator.
|
||||
.case_insensitive(cfg!(any(windows, target_os = "macos")))
|
||||
.case_insensitive(crate::platform::PATHS_ARE_CASE_INSENSITIVE)
|
||||
.build()
|
||||
.map_err(|e| format!("invalid ignore pattern {:?}: {}", pat, e))?;
|
||||
if pat.contains('/') || pat.contains('\\') {
|
||||
|
|
@ -627,18 +662,54 @@ impl IgnoreSet {
|
|||
let path = path
|
||||
.build()
|
||||
.map_err(|e| format!("compile ignore patterns: {}", e))?;
|
||||
let empty = component.is_empty() && path.is_empty();
|
||||
let empty = literal_components.is_empty() && component.is_empty() && path.is_empty();
|
||||
Ok(IgnoreSet {
|
||||
literal_components,
|
||||
component,
|
||||
path,
|
||||
empty,
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether `name` is one of the plain-name patterns, folded per platform.
|
||||
///
|
||||
/// Allocation-free for any name that fits [`FOLD_BUF`], which is every name
|
||||
/// a real filesystem can produce. This runs on every directory entry the
|
||||
/// walker sees, so it is the one place in the ignore path worth keeping off
|
||||
/// the heap.
|
||||
fn matches_literal(&self, name: &str) -> bool {
|
||||
if self.literal_components.is_empty() {
|
||||
return false;
|
||||
}
|
||||
if !crate::platform::PATHS_ARE_CASE_INSENSITIVE {
|
||||
return self.literal_components.contains(name);
|
||||
}
|
||||
// Every stored literal is ASCII, and ASCII case folding maps ASCII to
|
||||
// ASCII, so a name containing any non-ASCII byte cannot equal one.
|
||||
if !name.is_ascii() {
|
||||
return false;
|
||||
}
|
||||
if name.len() <= FOLD_BUF {
|
||||
let mut buf = [0u8; FOLD_BUF];
|
||||
let buf = &mut buf[..name.len()];
|
||||
buf.copy_from_slice(name.as_bytes());
|
||||
buf.make_ascii_lowercase();
|
||||
// Lowercasing ASCII yields ASCII, which is always valid UTF-8.
|
||||
let folded = std::str::from_utf8(buf).expect("ascii stays utf-8");
|
||||
return self.literal_components.contains(folded);
|
||||
}
|
||||
self.literal_components.contains(&name.to_ascii_lowercase())
|
||||
}
|
||||
|
||||
/// Match a single file/directory name. Used by the walker to prune
|
||||
/// subtrees before descending.
|
||||
pub fn matches_component(&self, name: &str) -> bool {
|
||||
!self.empty && self.component.is_match(name)
|
||||
if self.empty {
|
||||
return false;
|
||||
}
|
||||
// The literal set answers almost every call — the defaults are all
|
||||
// plain names — and answers it without touching the regex engine.
|
||||
self.matches_literal(name) || self.component.is_match(name)
|
||||
}
|
||||
|
||||
/// Match a path against the full-path patterns only. The path *and its
|
||||
|
|
@ -669,9 +740,12 @@ impl IgnoreSet {
|
|||
if self.matches_path_pattern(path) {
|
||||
return true;
|
||||
}
|
||||
path.components().any(|c| {
|
||||
matches!(c, std::path::Component::Normal(name)
|
||||
if self.component.is_match(Path::new(name)))
|
||||
path.components().any(|c| match c {
|
||||
// Routed through `matches_component` rather than `self.component`
|
||||
// directly, or the literal patterns would be invisible here and the
|
||||
// watcher would index what the walker prunes.
|
||||
std::path::Component::Normal(name) => self.matches_component(&name.to_string_lossy()),
|
||||
_ => false,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -764,6 +838,42 @@ impl IndexWork {
|
|||
pub fn scans_rows(&self) -> bool {
|
||||
self.prune_scope || self.reconcile_content || self.restore_text || self.drop_text
|
||||
}
|
||||
|
||||
/// The plan in one line, for the log entry that announces the scan.
|
||||
///
|
||||
/// Names what changed rather than what will happen to the rows: the
|
||||
/// reader is someone asking why a run has not started walking yet, and
|
||||
/// the answer they need is which edit of theirs caused it.
|
||||
pub fn summary(&self) -> String {
|
||||
let mut parts: Vec<String> = Vec::new();
|
||||
if !self.drop_roots.is_empty() {
|
||||
parts.push(format!(
|
||||
"{} root(s) no longer indexed",
|
||||
self.drop_roots.len()
|
||||
));
|
||||
}
|
||||
if self.prune_scope {
|
||||
parts.push("narrowed ignore or hidden-file rules".into());
|
||||
}
|
||||
if self.drop_aliases {
|
||||
parts.push("symlinks no longer followed".into());
|
||||
}
|
||||
if self.reconcile_content {
|
||||
parts.push("changed content extensions".into());
|
||||
}
|
||||
if self.restore_text {
|
||||
parts.push("snippet text turned on".into());
|
||||
}
|
||||
if self.drop_text {
|
||||
parts.push("snippet text turned off".into());
|
||||
}
|
||||
if parts.is_empty() {
|
||||
// `touches_index` is false here, so no caller logs this; a
|
||||
// placeholder beats an empty pair of parentheses if one ever does.
|
||||
return "no stored rows affected".into();
|
||||
}
|
||||
parts.join("; ")
|
||||
}
|
||||
}
|
||||
|
||||
/// What running services must do after a config edit. Computed by the GUI
|
||||
|
|
@ -788,8 +898,8 @@ pub struct ConfigActions {
|
|||
/// duplicates are reported once). Nested roots are disallowed: with one
|
||||
/// walker per root they would race for the same files and split progress
|
||||
/// attribution. Comparison is on best-effort canonicalized paths (an
|
||||
/// unresolvable root is compared as spelled) and is component-boundary
|
||||
/// aware — `/a/bc` is not under `/a/b`.
|
||||
/// unresolvable root is compared as spelled), component-wise per
|
||||
/// [`crate::file_handling::UnreadableDirs::covers`].
|
||||
pub fn nested_roots(roots: &[String]) -> Vec<(String, String)> {
|
||||
let resolved: Vec<PathBuf> = roots
|
||||
.iter()
|
||||
|
|
@ -932,17 +1042,7 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
fn tmp_dir() -> PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"quicksearch-config-{}-{}",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
fs::create_dir_all(&p).unwrap();
|
||||
p
|
||||
crate::testutil::scratch_dir("config")
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -1298,6 +1398,89 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
/// Which patterns take the fast path, and that taking it changes nothing
|
||||
/// observable. A plain name is matched whole — never as a prefix, a
|
||||
/// substring or a wildcard — and only its case is allowed to vary.
|
||||
#[test]
|
||||
fn the_literal_fast_path_matches_whole_names_only() {
|
||||
let literal = IgnoreSet::compile(&["node_modules".to_string()]).unwrap();
|
||||
assert!(
|
||||
!literal.literal_components.is_empty(),
|
||||
"a plain name belongs on the fast path"
|
||||
);
|
||||
for globby in ["node_module?", "node_*", "*.tmp", "a[bc]d", "x{1,2}"] {
|
||||
assert!(
|
||||
IgnoreSet::compile(&[globby.to_string()])
|
||||
.unwrap()
|
||||
.literal_components
|
||||
.is_empty(),
|
||||
"{} has glob syntax and must stay with globset",
|
||||
globby
|
||||
);
|
||||
}
|
||||
|
||||
assert!(literal.matches_component("node_modules"));
|
||||
for cased in ["Node_Modules", "NODE_MODULES", "node_moduleS"] {
|
||||
assert_eq!(
|
||||
literal.matches_component(cased),
|
||||
cfg!(any(windows, target_os = "macos")),
|
||||
"only case may vary, and only where the filesystem says so: {}",
|
||||
cased
|
||||
);
|
||||
}
|
||||
for name in [
|
||||
"node_modules_",
|
||||
"_node_modules",
|
||||
"nodemodules",
|
||||
"node_module",
|
||||
"src",
|
||||
"",
|
||||
] {
|
||||
assert!(
|
||||
!literal.matches_component(name),
|
||||
"{} is not the ignored name",
|
||||
name
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A non-ASCII pattern keeps globset's Unicode folding rather than being
|
||||
/// silently downgraded to the ASCII fast path.
|
||||
#[test]
|
||||
fn non_ascii_patterns_stay_on_the_glob_path() {
|
||||
let set = IgnoreSet::compile(&["café".to_string()]).unwrap();
|
||||
assert!(
|
||||
set.literal_components.is_empty(),
|
||||
"a non-ASCII name must not join the ASCII-folded set"
|
||||
);
|
||||
assert!(set.matches_component("café"));
|
||||
assert!(!set.matches_component("cafe"));
|
||||
}
|
||||
|
||||
/// Names longer than the stack fold buffer take the heap path, and must
|
||||
/// come back with the same answer.
|
||||
#[test]
|
||||
fn overlong_names_still_fold_correctly() {
|
||||
let long = "a".repeat(FOLD_BUF + 10);
|
||||
let set = IgnoreSet::compile(std::slice::from_ref(&long)).unwrap();
|
||||
assert!(set.matches_component(&long));
|
||||
assert_eq!(
|
||||
set.matches_component(&long.to_uppercase()),
|
||||
cfg!(any(windows, target_os = "macos"))
|
||||
);
|
||||
assert!(!set.matches_component(&"a".repeat(FOLD_BUF + 9)));
|
||||
}
|
||||
|
||||
/// Watcher events are matched by whole path, and the literal patterns have
|
||||
/// to be visible on that route too — otherwise the watcher indexes exactly
|
||||
/// what the walker prunes and the index churns every cycle.
|
||||
#[test]
|
||||
fn full_path_matching_sees_literal_component_patterns() {
|
||||
let set = IgnoreSet::compile(&["node_modules".to_string()]).unwrap();
|
||||
assert!(set.matches_path(Path::new("/home/me/proj/node_modules/pkg/index.js")));
|
||||
assert!(!set.matches_path(Path::new("/home/me/proj/src/index.js")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_ignore_patterns_cover_the_platform() {
|
||||
let d = IndexingConfig::default().ignore_patterns;
|
||||
|
|
@ -1374,8 +1557,10 @@ mod tests {
|
|||
fn root_workers_round_trip() {
|
||||
let dir = tmp_dir();
|
||||
let path = dir.join("config.toml");
|
||||
let mut cfg = Config::default();
|
||||
cfg.source = Some(path.clone());
|
||||
let mut cfg = Config {
|
||||
source: Some(path.clone()),
|
||||
..Config::default()
|
||||
};
|
||||
cfg.paths.indexing_paths = vec!["/data".into(), "/share".into()];
|
||||
cfg.indexing.root_workers.insert("/share".into(), 24);
|
||||
cfg.save().unwrap();
|
||||
|
|
@ -1705,8 +1890,10 @@ mod tests {
|
|||
fn watch_cap_warned_roots_round_trips() {
|
||||
let dir = tmp_dir();
|
||||
let path = dir.join("config.toml");
|
||||
let mut cfg = Config::default();
|
||||
cfg.source = Some(path.clone());
|
||||
let mut cfg = Config {
|
||||
source: Some(path.clone()),
|
||||
..Config::default()
|
||||
};
|
||||
cfg.ui.watch_cap_warned_roots =
|
||||
vec!["/media/ApolloStore".to_string(), "/media/GSSD".to_string()];
|
||||
cfg.save().unwrap();
|
||||
|
|
@ -1740,8 +1927,10 @@ mod tests {
|
|||
fn fuzzy_max_edits_round_trips() {
|
||||
let dir = tmp_dir();
|
||||
let path = dir.join("config.toml");
|
||||
let mut cfg = Config::default();
|
||||
cfg.source = Some(path.clone());
|
||||
let mut cfg = Config {
|
||||
source: Some(path.clone()),
|
||||
..Config::default()
|
||||
};
|
||||
cfg.search.fuzzy_max_edits = 4;
|
||||
cfg.save().unwrap();
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@
|
|||
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::sync::{mpsc, Arc, Condvar, Mutex};
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::extract::Registry;
|
||||
|
|
@ -175,11 +175,9 @@ impl ContentPass {
|
|||
self.stats.clone()
|
||||
}
|
||||
|
||||
/// Join the workers and report whether every one finished cleanly.
|
||||
///
|
||||
/// The caller needs this for the same reason the walk does: a dead worker
|
||||
/// and a finished worker both close the channel, so from the receiving end
|
||||
/// they are indistinguishable.
|
||||
/// Join the workers and report whether every one finished cleanly. Needed
|
||||
/// for the same reason as [`crate::walk::ParallelWalk::finish`], which
|
||||
/// states it.
|
||||
pub fn finish(&mut self) -> bool {
|
||||
// Dropping the receiver first releases any worker parked in `send`.
|
||||
self.rx = None;
|
||||
|
|
@ -313,7 +311,7 @@ pub fn extract_content(
|
|||
let (registry, config) = (registry.clone(), config.clone());
|
||||
let (stop_flag, suspend_flag) = (stop_flag.clone(), suspend_flag.clone());
|
||||
let stats = stats.clone();
|
||||
thread::spawn(move || {
|
||||
crate::platform::spawn_worker("qs-extract", move || {
|
||||
crate::platform::set_background_priority();
|
||||
worker(
|
||||
&shared,
|
||||
|
|
@ -334,7 +332,7 @@ pub fn extract_content(
|
|||
|
||||
let feeder_handle = {
|
||||
let (shared, db_path, cursor) = (shared.clone(), db_path.to_string(), cursor.clone());
|
||||
thread::spawn(move || {
|
||||
crate::platform::spawn_worker("qs-feeder", move || {
|
||||
crate::platform::set_background_priority();
|
||||
feeder(&shared, &db_path, cursor, max_size)
|
||||
})
|
||||
|
|
@ -352,25 +350,18 @@ pub fn extract_content(
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
// Only the tests sleep; the pass itself spawns through
|
||||
// `platform::spawn_worker` and blocks on channels rather than polling.
|
||||
use std::thread;
|
||||
|
||||
use crate::db::open_or_recreate;
|
||||
use crate::db::repo::{self, insert_file, NewFile};
|
||||
use crate::file_handling::{extract_scope_prepare, store_extracted};
|
||||
use crate::mime::FileType;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
||||
/// A path that does not exist yet — the caller builds the tree under it.
|
||||
fn tmp(tag: &str) -> PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"qs-content-{}-{}-{}",
|
||||
tag,
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
p
|
||||
crate::testutil::scratch_dir(tag).join("tree")
|
||||
}
|
||||
|
||||
/// A tree of `n` text files under `root/sub`, plus an index holding a
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -8,6 +8,10 @@
|
|||
//! size, `clear`) instead use [`open::open_existing`], which never creates or
|
||||
//! wipes — a tokenizer difference or stale version is an error, not data loss.
|
||||
|
||||
use std::sync::Mutex;
|
||||
|
||||
use rusqlite::{Connection, InterruptHandle};
|
||||
|
||||
pub mod key;
|
||||
pub mod open;
|
||||
pub mod repo;
|
||||
|
|
@ -18,3 +22,95 @@ pub use open::{
|
|||
index_needs_rebuild, open_existing, open_or_recreate, verify_process_key,
|
||||
CURRENT_SCHEMA_VERSION, KEY_MISMATCH_PREFIX,
|
||||
};
|
||||
|
||||
/// Bumped whenever the index file is replaced rather than modified.
|
||||
///
|
||||
/// Process-wide, like [`key::process_key`], and for the same reason: it is a
|
||||
/// fact about the index this process is working with, not about any one
|
||||
/// connection to it.
|
||||
///
|
||||
/// Anything holding a connection open across operations needs to know that the
|
||||
/// file underneath it has been swapped, and the path cannot tell it — a
|
||||
/// rebuild and a clear both put a *new* file at the *same* path. A connection
|
||||
/// that missed the change keeps serving the deleted inode: stale results, and
|
||||
/// on Linux the old file's blocks stay allocated for as long as the handle
|
||||
/// lives. [`crate::search`] is the only long-lived reader today; it compares
|
||||
/// this against the value it opened with.
|
||||
static INDEX_EPOCH: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||
|
||||
/// The current index generation. See [`INDEX_EPOCH`].
|
||||
pub fn index_epoch() -> u64 {
|
||||
INDEX_EPOCH.load(std::sync::atomic::Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Declare that the index file has been replaced.
|
||||
///
|
||||
/// Called from [`open::open_or_recreate`]'s wipe path — the one place a wipe
|
||||
/// actually happens, so a schema drift nobody asked for cannot slip past —
|
||||
/// and from the coordinator's explicit rebuild and clear commands, which
|
||||
/// delete the file without going through it.
|
||||
pub fn bump_index_epoch() {
|
||||
INDEX_EPOCH.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
|
||||
/// A shared slot holding the interrupt handle of whatever long statement is
|
||||
/// running, so another thread can cut it short.
|
||||
///
|
||||
/// # Why a flag is not enough
|
||||
///
|
||||
/// SQLite's own interrupt is the only way out of a statement already in
|
||||
/// flight: a `DELETE` over a whole root's range, an FTS merge or a VACUUM
|
||||
/// answer to nothing else, and on a large index each can run for minutes. A
|
||||
/// flag can only stop the *next* statement from starting.
|
||||
///
|
||||
/// So cancelling anything long is two halves — the flag, which prevents the
|
||||
/// next statement, and this, which ends the current one — and a caller that
|
||||
/// wants a bounded wait needs both. Every user of this type is one of those
|
||||
/// pairs: [`crate::scope::advance`] and its `cancel` argument,
|
||||
/// `coordinator::ReconcileStop`, and the stop flag alongside
|
||||
/// `IndexingService::cancel_db_work`. This is the reference statement of the
|
||||
/// rule; those sites record only what it means locally.
|
||||
///
|
||||
/// One further consequence, which is why [`InterruptGuard`] exists rather than
|
||||
/// a set/clear pair: an interrupted statement fails like any other and cannot
|
||||
/// be told apart from a real failure by its error message. The flag the
|
||||
/// canceller set is the only reliable answer, so callers re-read it on the
|
||||
/// error path before deciding whether they were cancelled or broke.
|
||||
pub type InterruptSlot = Mutex<Option<InterruptHandle>>;
|
||||
|
||||
/// Publish `conn`'s interrupt handle in `slot` for as long as this lives.
|
||||
///
|
||||
/// The guard, rather than a set/clear pair, because every path out matters:
|
||||
/// a handle left behind after an early `?` would let a later interrupt —
|
||||
/// aimed at the next long statement, or at a VACUUM — land on whatever that
|
||||
/// connection happens to be running by then.
|
||||
pub struct InterruptGuard<'a> {
|
||||
slot: &'a InterruptSlot,
|
||||
}
|
||||
|
||||
impl<'a> InterruptGuard<'a> {
|
||||
pub fn arm(slot: &'a InterruptSlot, conn: &Connection) -> InterruptGuard<'a> {
|
||||
if let Ok(mut held) = slot.lock() {
|
||||
*held = Some(conn.get_interrupt_handle());
|
||||
}
|
||||
InterruptGuard { slot }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for InterruptGuard<'_> {
|
||||
fn drop(&mut self) {
|
||||
if let Ok(mut held) = self.slot.lock() {
|
||||
*held = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Interrupt the statement `slot` holds a handle for, if there is one.
|
||||
/// A no-op otherwise, and on a connection that has since closed.
|
||||
pub fn interrupt(slot: &InterruptSlot) {
|
||||
if let Ok(held) = slot.lock() {
|
||||
if let Some(handle) = held.as_ref() {
|
||||
handle.interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,19 +7,19 @@
|
|||
//! migrations.
|
||||
//!
|
||||
//! The tradeoff: users pay a re-index cost every time the shipped schema
|
||||
//! changes. Our indexing is fast (see `bench/`) and schema changes are
|
||||
//! rare in practice, so the code-complexity cost of maintaining real
|
||||
//! migration paths wasn't worth it. A single `open_or_recreate` replaces
|
||||
//! what used to be version detection + tokenizer-drift FTS rebuild +
|
||||
//! legacy-layout recovery, all of which ultimately wiped anyway.
|
||||
//! changes. Indexing is fast and schema changes are rare in practice, so the
|
||||
//! code-complexity cost of maintaining real migration paths wasn't worth it.
|
||||
//! A single `open_or_recreate` replaces what used to be version detection +
|
||||
//! tokenizer-drift FTS rebuild + legacy-layout recovery, all of which
|
||||
//! ultimately wiped anyway.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use rusqlite::{params, Connection, OpenFlags, OptionalExtension};
|
||||
|
||||
use super::schema::{
|
||||
effective_tokenizer, fts_create_sql, PRAGMAS_FAST, PRAGMAS_MAINTENANCE, PRAGMAS_READONLY,
|
||||
PRAGMAS_WALK_READER, SCHEMA_CURRENT,
|
||||
effective_tokenizer, fts_create_sql, PRAGMAS_FAST, PRAGMAS_INCREMENTAL, PRAGMAS_MAINTENANCE,
|
||||
PRAGMAS_READONLY, PRAGMAS_SEARCH, PRAGMAS_WALK_READER, SCHEMA_CURRENT,
|
||||
};
|
||||
use crate::security::IndexKey;
|
||||
|
||||
|
|
@ -119,6 +119,35 @@ pub fn open_walk_reader(db_path: &str) -> Result<Connection, String> {
|
|||
)
|
||||
}
|
||||
|
||||
/// The search worker's connection, held across requests.
|
||||
///
|
||||
/// Identical to `open_existing(_, false)` except for the pragma profile: see
|
||||
/// [`PRAGMAS_SEARCH`] for why this one is deliberately larger than the
|
||||
/// one-shot readers' and still smaller than what it replaced.
|
||||
pub fn open_search_reader(db_path: &str) -> Result<Connection, String> {
|
||||
open_keyed_with_pragmas(
|
||||
db_path,
|
||||
false,
|
||||
super::key::process_key().as_ref(),
|
||||
PRAGMAS_SEARCH,
|
||||
)
|
||||
}
|
||||
|
||||
/// The coordinator's write connection, for applying watcher events and
|
||||
/// reconciling a config change.
|
||||
///
|
||||
/// Identical to `open_existing(_, true)` except for the pragma profile: see
|
||||
/// [`PRAGMAS_INCREMENTAL`] for why the process's longest-lived writer must not
|
||||
/// take the cache the bulk indexer does.
|
||||
pub fn open_incremental_writer(db_path: &str) -> Result<Connection, String> {
|
||||
open_keyed_with_pragmas(
|
||||
db_path,
|
||||
true,
|
||||
super::key::process_key().as_ref(),
|
||||
PRAGMAS_INCREMENTAL,
|
||||
)
|
||||
}
|
||||
|
||||
/// A writable connection for post-run compaction, and the only one that may
|
||||
/// VACUUM. See [`PRAGMAS_MAINTENANCE`] for why it cannot be the indexer's.
|
||||
pub fn open_maintenance(db_path: &str) -> Result<Connection, String> {
|
||||
|
|
@ -369,6 +398,10 @@ fn wipe_and_reopen(
|
|||
key: Option<&IndexKey>,
|
||||
) -> Result<Connection, String> {
|
||||
drop(conn);
|
||||
// Before the delete, not after: anything holding a connection to the file
|
||||
// about to disappear must be told, and told even if the removal below
|
||||
// fails partway. See [`super::bump_index_epoch`].
|
||||
super::bump_index_epoch();
|
||||
// Primary file may already be absent (fresh open that just needed
|
||||
// the table applied). Ignore NotFound; anything else is an error.
|
||||
//
|
||||
|
|
@ -414,10 +447,7 @@ fn apply_current_schema(conn: &Connection, tokenizer: &str) -> Result<(), String
|
|||
conn.execute_batch(&fts)
|
||||
.map_err(|e| format!("Failed to create searchabletext: {}", e))?;
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0);
|
||||
let now = crate::log::now_unix();
|
||||
let effective = effective_tokenizer(tokenizer);
|
||||
conn.execute(
|
||||
"INSERT INTO schema_info(key, value) VALUES ('version', ?1), ('created_at', ?2), ('tokenize', ?3)",
|
||||
|
|
@ -437,16 +467,7 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
fn tmp_db_path() -> std::path::PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"quicksearch-test-{}-{}.sqlite",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
p
|
||||
crate::testutil::scratch_dir("open").join("index.sqlite")
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -548,7 +569,7 @@ mod tests {
|
|||
assert_eq!(count, 0);
|
||||
// New columns should exist (just prepare the SELECT — an
|
||||
// unknown column name would parse-error here).
|
||||
let _ = conn
|
||||
conn
|
||||
.query_row(
|
||||
"SELECT basic_state, content_state, type, mime FROM files LIMIT 0",
|
||||
[],
|
||||
|
|
@ -667,15 +688,7 @@ mod tests {
|
|||
fn open_or_recreate_creates_missing_parent_dirs() {
|
||||
// Fresh installs point at ~/.local/share/quicksearch/… which
|
||||
// doesn't exist yet; the owner open must create it.
|
||||
let mut dir = std::env::temp_dir();
|
||||
dir.push(format!(
|
||||
"qs-mkdir-{}-{}",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
let dir = crate::testutil::scratch_dir("mkdir");
|
||||
let db = dir.join("nested/deeper/index.sqlite");
|
||||
let conn = open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
|
||||
drop(conn);
|
||||
|
|
|
|||
|
|
@ -23,6 +23,57 @@ pub const STATE_DONE: i64 = 1;
|
|||
pub const STATE_FAILED: i64 = 2;
|
||||
pub const STATE_NA: i64 = 3;
|
||||
|
||||
/// `prepare_cached` + `execute`, returning the affected row count.
|
||||
///
|
||||
/// `what` is a closure rather than a string so the message is built only on
|
||||
/// the error path — several callers run this per property or per file, where
|
||||
/// formatting a description that is then discarded is the dominant cost.
|
||||
///
|
||||
/// Takes `&Connection`; a `Transaction` derefs to one, so the same call works
|
||||
/// inside and outside a transaction.
|
||||
fn exec(
|
||||
conn: &Connection,
|
||||
sql: &str,
|
||||
params: impl rusqlite::Params,
|
||||
what: impl FnOnce() -> String,
|
||||
) -> Result<usize, String> {
|
||||
conn.prepare_cached(sql)
|
||||
.and_then(|mut stmt| stmt.execute(params))
|
||||
.map_err(|e| format!("{}: {}", what(), e))
|
||||
}
|
||||
|
||||
/// Move a file to a content state that means "nothing is currently wrong with
|
||||
/// this row", clearing any failure record along with it.
|
||||
///
|
||||
/// The pair is one operation, not two that happen to be adjacent: a row that
|
||||
/// succeeded, that nothing can extract, or that has been queued for another
|
||||
/// attempt must not keep a `failed_files` entry explaining a failure that no
|
||||
/// longer applies. `list-failed` reads that table directly, so a stale entry
|
||||
/// is a file the user is told is broken after it stopped being broken.
|
||||
///
|
||||
/// [`set_content_failed`] is deliberately not routed through here — it is the
|
||||
/// one transition that *writes* a failure record.
|
||||
fn set_state_clearing_failure(
|
||||
tx: &Transaction<'_>,
|
||||
file_id: i64,
|
||||
state: i64,
|
||||
transition: &'static str,
|
||||
) -> Result<(), String> {
|
||||
exec(
|
||||
tx,
|
||||
"UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2",
|
||||
params![state, file_id],
|
||||
|| format!("{} content_state {}", transition, file_id),
|
||||
)?;
|
||||
exec(
|
||||
tx,
|
||||
"DELETE FROM failed_files WHERE file_id = ?1",
|
||||
params![file_id],
|
||||
|| format!("clear failed_files {}", file_id),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Everything needed to insert a fresh file row.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NewFile<'a> {
|
||||
|
|
@ -162,18 +213,22 @@ pub fn set_content_done(
|
|||
remove_content_for_id(tx, file_id)?;
|
||||
|
||||
for (k, v) in properties {
|
||||
tx.prepare_cached("INSERT INTO properties(file_id, key, value) VALUES (?1, ?2, ?3)")
|
||||
.and_then(|mut stmt| stmt.execute(params![file_id, k, v]))
|
||||
.map_err(|e| format!("insert property {}={}: {}", k, v, e))?;
|
||||
exec(
|
||||
tx,
|
||||
"INSERT INTO properties(file_id, key, value) VALUES (?1, ?2, ?3)",
|
||||
params![file_id, k, v],
|
||||
|| format!("insert property {}={}", k, v),
|
||||
)?;
|
||||
}
|
||||
let props_blob = encode_properties_for_fts(properties);
|
||||
// Contentless FTS5 still accepts values on INSERT — the tokenizer needs
|
||||
// them — it simply doesn't persist the raw column values.
|
||||
tx.prepare_cached(
|
||||
exec(
|
||||
tx,
|
||||
"INSERT INTO searchabletext(rowid, name, text, properties) VALUES (?1, ?2, ?3, ?4)",
|
||||
)
|
||||
.and_then(|mut stmt| stmt.execute(params![file_id, name, text, props_blob]))
|
||||
.map_err(|e| format!("insert FTS row {}: {}", file_id, e))?;
|
||||
params![file_id, name, text, props_blob],
|
||||
|| format!("insert FTS row {}", file_id),
|
||||
)?;
|
||||
|
||||
// Skip the compressed sidecar when: the config disables snippet storage
|
||||
// outright, or there's no body text (e.g. an image whose extractor
|
||||
|
|
@ -182,21 +237,15 @@ pub fn set_content_done(
|
|||
if store_text && !text.is_empty() {
|
||||
let compressed = zstd::encode_all(text.as_bytes(), ZSTD_LEVEL)
|
||||
.map_err(|e| format!("zstd encode for file {}: {}", file_id, e))?;
|
||||
tx.prepare_cached(
|
||||
exec(
|
||||
tx,
|
||||
"INSERT INTO documents_text(file_id, text_zstd, text_len) VALUES (?1, ?2, ?3)",
|
||||
)
|
||||
.and_then(|mut stmt| stmt.execute(params![file_id, compressed, text.len() as i64]))
|
||||
.map_err(|e| format!("insert documents_text {}: {}", file_id, e))?;
|
||||
params![file_id, compressed, text.len() as i64],
|
||||
|| format!("insert documents_text {}", file_id),
|
||||
)?;
|
||||
}
|
||||
|
||||
tx.prepare_cached("UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2")
|
||||
.and_then(|mut stmt| stmt.execute(params![STATE_DONE, file_id]))
|
||||
.map_err(|e| format!("update content_state DONE {}: {}", file_id, e))?;
|
||||
// Clear any prior failed-file record.
|
||||
tx.prepare_cached("DELETE FROM failed_files WHERE file_id = ?1")
|
||||
.and_then(|mut stmt| stmt.execute(params![file_id]))
|
||||
.map_err(|e| format!("clear failed_files {}: {}", file_id, e))?;
|
||||
Ok(())
|
||||
set_state_clearing_failure(tx, file_id, STATE_DONE, "update DONE")
|
||||
}
|
||||
|
||||
/// zstd level tuned for extracted-text prose. Level 3 hits ~3-5× on English
|
||||
|
|
@ -208,31 +257,26 @@ const ZSTD_LEVEL: i32 = 3;
|
|||
|
||||
/// Mark a file's content extraction as failed. Keeps the basic row in place.
|
||||
pub fn set_content_failed(tx: &Transaction<'_>, file_id: i64, reason: &str) -> Result<(), String> {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0) as i64;
|
||||
tx.prepare_cached("UPDATE files SET content_state = ?1, failure_msg = ?2 WHERE id = ?3")
|
||||
.and_then(|mut stmt| stmt.execute(params![STATE_FAILED, reason, file_id]))
|
||||
.map_err(|e| format!("update content_state FAILED {}: {}", file_id, e))?;
|
||||
tx.prepare_cached(
|
||||
let now = crate::log::now_unix() as i64;
|
||||
exec(
|
||||
tx,
|
||||
"UPDATE files SET content_state = ?1, failure_msg = ?2 WHERE id = ?3",
|
||||
params![STATE_FAILED, reason, file_id],
|
||||
|| format!("update content_state FAILED {}", file_id),
|
||||
)?;
|
||||
exec(
|
||||
tx,
|
||||
"INSERT OR REPLACE INTO failed_files(file_id, reason, ts) VALUES (?1, ?2, ?3)",
|
||||
)
|
||||
.and_then(|mut stmt| stmt.execute(params![file_id, reason, now]))
|
||||
.map_err(|e| format!("insert failed_files {}: {}", file_id, e))?;
|
||||
params![file_id, reason, now],
|
||||
|| format!("insert failed_files {}", file_id),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mark content extraction as not applicable (e.g. binary format we don't
|
||||
/// support). The file row still contributes to filename search.
|
||||
pub fn set_content_na(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> {
|
||||
tx.prepare_cached("UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2")
|
||||
.and_then(|mut stmt| stmt.execute(params![STATE_NA, file_id]))
|
||||
.map_err(|e| format!("update content_state NA {}: {}", file_id, e))?;
|
||||
tx.prepare_cached("DELETE FROM failed_files WHERE file_id = ?1")
|
||||
.and_then(|mut stmt| stmt.execute(params![file_id]))
|
||||
.map_err(|e| format!("clear failed_files {}: {}", file_id, e))?;
|
||||
Ok(())
|
||||
set_state_clearing_failure(tx, file_id, STATE_NA, "update NA")
|
||||
}
|
||||
|
||||
/// Delete a file row by path, keeping FTS in sync. Returns whether a row was
|
||||
|
|
@ -276,15 +320,16 @@ pub fn delete_subtree(tx: &Transaction<'_>, lo: &str, hi: &str) -> Result<usize,
|
|||
(SELECT id FROM files WHERE path >= ?1 AND path < ?2)",
|
||||
table, key
|
||||
);
|
||||
tx.prepare_cached(&sql)
|
||||
.and_then(|mut stmt| stmt.execute(params![lo, hi]))
|
||||
.map_err(|e| format!("delete {} under {}: {}", table, lo, e))?;
|
||||
exec(tx, &sql, params![lo, hi], || {
|
||||
format!("delete {} under {}", table, lo)
|
||||
})?;
|
||||
}
|
||||
let removed = tx
|
||||
.prepare_cached("DELETE FROM files WHERE path >= ?1 AND path < ?2")
|
||||
.and_then(|mut stmt| stmt.execute(params![lo, hi]))
|
||||
.map_err(|e| format!("delete files under {}: {}", lo, e))?;
|
||||
Ok(removed)
|
||||
exec(
|
||||
tx,
|
||||
"DELETE FROM files WHERE path >= ?1 AND path < ?2",
|
||||
params![lo, hi],
|
||||
|| format!("delete files under {}", lo),
|
||||
)
|
||||
}
|
||||
|
||||
/// Delete every row whose path falls in *none* of `ranges`, keeping the four
|
||||
|
|
@ -326,14 +371,14 @@ pub fn delete_outside_ranges(
|
|||
"DELETE FROM {} WHERE {} IN (SELECT id FROM files WHERE {})",
|
||||
table, key, predicate
|
||||
);
|
||||
tx.prepare_cached(&sql)
|
||||
.and_then(|mut stmt| stmt.execute(params_from_iter(bounds.iter())))
|
||||
.map_err(|e| format!("delete {} outside the roots: {}", table, e))?;
|
||||
exec(tx, &sql, params_from_iter(bounds.iter()), || {
|
||||
format!("delete {} outside the roots", table)
|
||||
})?;
|
||||
}
|
||||
let sql = format!("DELETE FROM files WHERE {}", predicate);
|
||||
tx.prepare_cached(&sql)
|
||||
.and_then(|mut stmt| stmt.execute(params_from_iter(bounds.iter())))
|
||||
.map_err(|e| format!("delete files outside the roots: {}", e))
|
||||
exec(tx, &sql, params_from_iter(bounds.iter()), || {
|
||||
"delete files outside the roots".to_string()
|
||||
})
|
||||
}
|
||||
|
||||
/// The tables a file id owns, in the order they must be cleared: everything
|
||||
|
|
@ -381,15 +426,14 @@ pub fn delete_ids(tx: &Transaction<'_>, ids: &[i64]) -> Result<usize, String> {
|
|||
}
|
||||
for (table, key) in DEPENDENT_TABLES {
|
||||
let sql = format!("DELETE FROM {} WHERE {} IN ({})", table, key, placeholders);
|
||||
tx.prepare_cached(&sql)
|
||||
.and_then(|mut stmt| stmt.execute(params_from_iter(chunk.iter())))
|
||||
.map_err(|e| format!("delete {} for {} ids: {}", table, chunk.len(), e))?;
|
||||
exec(tx, &sql, params_from_iter(chunk.iter()), || {
|
||||
format!("delete {} for {} ids", table, chunk.len())
|
||||
})?;
|
||||
}
|
||||
let sql = format!("DELETE FROM files WHERE id IN ({})", placeholders);
|
||||
removed += tx
|
||||
.prepare_cached(&sql)
|
||||
.and_then(|mut stmt| stmt.execute(params_from_iter(chunk.iter())))
|
||||
.map_err(|e| format!("delete {} file rows: {}", chunk.len(), e))?;
|
||||
removed += exec(tx, &sql, params_from_iter(chunk.iter()), || {
|
||||
format!("delete {} file rows", chunk.len())
|
||||
})?;
|
||||
}
|
||||
Ok(removed)
|
||||
}
|
||||
|
|
@ -420,8 +464,11 @@ pub fn dir_rows(
|
|||
Ok(out)
|
||||
}
|
||||
|
||||
/// A row the content pass has yet to extract: `(id, name, path, mime)`.
|
||||
pub type PendingContentRow = (i64, String, String, Option<String>);
|
||||
|
||||
/// One page of rows still awaiting content extraction under `cursor`'s range,
|
||||
/// as `(id, name, path, mime)` ordered by id.
|
||||
/// ordered by id.
|
||||
///
|
||||
/// Keyset, not `OFFSET`: `id > cursor.last_id` means each page is an index
|
||||
/// seek rather than a re-scan of everything already handed out, and — because
|
||||
|
|
@ -434,7 +481,7 @@ pub fn pending_content_page(
|
|||
cursor: &crate::file_handling::ExtractCursor,
|
||||
max_size: i64,
|
||||
limit: i64,
|
||||
) -> Result<Vec<(i64, String, String, Option<String>)>, String> {
|
||||
) -> Result<Vec<PendingContentRow>, String> {
|
||||
let mut stmt = conn
|
||||
.prepare_cached(
|
||||
"SELECT id, name, path, mime FROM files
|
||||
|
|
@ -472,6 +519,17 @@ pub struct ScopeRow {
|
|||
pub content_state: i64,
|
||||
}
|
||||
|
||||
/// How many files the index holds.
|
||||
///
|
||||
/// A denominator for a scan that pages over all of them, so the cost is paid
|
||||
/// once against work measured in pages. SQLite answers it from the smallest
|
||||
/// index rather than the table, so it is a key scan and not a row fetch.
|
||||
pub fn row_count(conn: &Connection) -> Result<usize, String> {
|
||||
conn.query_row("SELECT COUNT(*) FROM files", [], |r| r.get::<_, i64>(0))
|
||||
.map(|n| n.max(0) as usize)
|
||||
.map_err(|e| format!("count indexed files: {}", e))
|
||||
}
|
||||
|
||||
/// One page of rows whose path is `> after` and `< hi`, in path order.
|
||||
///
|
||||
/// Keyset on `path` rather than on `id`: the range is already a seek on
|
||||
|
|
@ -529,10 +587,9 @@ pub fn drop_stored_text(tx: &Transaction<'_>, ids: &[i64]) -> Result<usize, Stri
|
|||
"DELETE FROM documents_text WHERE file_id IN ({})",
|
||||
placeholders
|
||||
);
|
||||
removed += tx
|
||||
.prepare_cached(&sql)
|
||||
.and_then(|mut stmt| stmt.execute(params_from_iter(chunk.iter())))
|
||||
.map_err(|e| format!("drop stored text for {} ids: {}", chunk.len(), e))?;
|
||||
removed += exec(tx, &sql, params_from_iter(chunk.iter()), || {
|
||||
format!("drop stored text for {} ids", chunk.len())
|
||||
})?;
|
||||
}
|
||||
Ok(removed)
|
||||
}
|
||||
|
|
@ -545,13 +602,7 @@ pub fn drop_stored_text(tx: &Transaction<'_>, ids: &[i64]) -> Result<usize, Stri
|
|||
/// and hash from a fresh stat), but its content must be produced again.
|
||||
pub fn reset_content_pending(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> {
|
||||
remove_content_for_id(tx, file_id)?;
|
||||
tx.prepare_cached("UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2")
|
||||
.and_then(|mut stmt| stmt.execute(params![STATE_PENDING, file_id]))
|
||||
.map_err(|e| format!("reset content_state pending {}: {}", file_id, e))?;
|
||||
tx.prepare_cached("DELETE FROM failed_files WHERE file_id = ?1")
|
||||
.and_then(|mut stmt| stmt.execute(params![file_id]))
|
||||
.map_err(|e| format!("clear failed_files {}: {}", file_id, e))?;
|
||||
Ok(())
|
||||
set_state_clearing_failure(tx, file_id, STATE_PENDING, "reset pending")
|
||||
}
|
||||
|
||||
/// The stored mtime for one exact path, or `None` if it isn't indexed.
|
||||
|
|
@ -621,9 +672,9 @@ pub fn remove_content_for_id(tx: &Transaction<'_>, file_id: i64) -> Result<(), S
|
|||
("properties", "file_id"),
|
||||
] {
|
||||
let sql = format!("DELETE FROM {} WHERE {} = ?1", table, key);
|
||||
tx.prepare_cached(&sql)
|
||||
.and_then(|mut stmt| stmt.execute(params![file_id]))
|
||||
.map_err(|e| format!("delete {} for {}: {}", table, file_id, e))?;
|
||||
exec(tx, &sql, params![file_id], || {
|
||||
format!("delete {} for {}", table, file_id)
|
||||
})?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -779,6 +830,70 @@ pub fn set_last_full_index(conn: &Connection, ts: u64) -> Result<(), String> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// `schema_info` key holding one root's last known file count.
|
||||
fn walk_count_key(root: &str) -> String {
|
||||
format!("walk_count:{}", root)
|
||||
}
|
||||
|
||||
/// How many files the last clean walk of `root` reported.
|
||||
///
|
||||
/// The progress bar's denominator. Absent means this root has never been walked
|
||||
/// to completion — a new root, a fresh index, or a run that was stopped — and
|
||||
/// the caller falls back to counting the tree as it goes.
|
||||
///
|
||||
/// Deliberately last run's *file* count rather than a tree-entry count: it is
|
||||
/// the same quantity the numerator counts, where an entry count includes
|
||||
/// directories and ignore-pruned subtrees and so reads high (over 1.6x on a home
|
||||
/// directory). See [`crate::indexing::RootProgress::walk_denominator`].
|
||||
pub fn get_root_walk_count(conn: &Connection, root: &str) -> Option<usize> {
|
||||
conn.query_row(
|
||||
"SELECT value FROM schema_info WHERE key = ?1",
|
||||
params![walk_count_key(root)],
|
||||
|r| r.get::<_, String>(0),
|
||||
)
|
||||
.optional()
|
||||
.ok()
|
||||
.flatten()
|
||||
.and_then(|v| v.parse().ok())
|
||||
}
|
||||
|
||||
/// Record `n` as `root`'s file count, for the next run's progress bar.
|
||||
///
|
||||
/// Written only after a walk that finished cleanly: a stopped or partially
|
||||
/// unreadable walk saw only part of the tree, and storing its count would leave
|
||||
/// every later run dividing by a number that is too small.
|
||||
pub fn set_root_walk_count(conn: &Connection, root: &str, n: usize) -> Result<(), String> {
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO schema_info(key, value) VALUES (?1, ?2)",
|
||||
params![walk_count_key(root), n.to_string()],
|
||||
)
|
||||
.map_err(|e| format!("write walk count for {}: {}", root, e))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the stored counts of roots that are no longer configured.
|
||||
///
|
||||
/// Without this, `schema_info` accumulates a row per root the user ever had —
|
||||
/// harmless in size, but it also means a root removed and later re-added would
|
||||
/// start from a count that predates everything that happened in between.
|
||||
pub fn prune_root_walk_counts(conn: &Connection, keep: &[String]) -> Result<(), String> {
|
||||
let keep: std::collections::HashSet<String> = keep.iter().map(|r| walk_count_key(r)).collect();
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT key FROM schema_info WHERE key LIKE 'walk_count:%'")
|
||||
.map_err(|e| format!("read walk counts: {}", e))?;
|
||||
let stored: Vec<String> = stmt
|
||||
.query_map([], |r| r.get::<_, String>(0))
|
||||
.map_err(|e| format!("read walk counts: {}", e))?
|
||||
.filter_map(|r| r.ok())
|
||||
.collect();
|
||||
drop(stmt);
|
||||
for key in stored.iter().filter(|k| !keep.contains(*k)) {
|
||||
conn.execute("DELETE FROM schema_info WHERE key = ?1", params![key])
|
||||
.map_err(|e| format!("drop walk count {}: {}", key, e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
|
@ -788,16 +903,7 @@ mod tests {
|
|||
use crate::db::open_or_recreate;
|
||||
|
||||
fn tmp_path() -> std::path::PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"quicksearch-repo-{}-{}.sqlite",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
p
|
||||
crate::testutil::scratch_dir("repo").join("index.sqlite")
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,65 @@
|
|||
//! SQL strings for the current schema. Versioned; [`migrate`](super::migrate)
|
||||
//! drives the upgrade path.
|
||||
//! SQL strings for the current schema.
|
||||
//!
|
||||
//! Versioned by [`super::open::CURRENT_SCHEMA_VERSION`], but there is no
|
||||
//! upgrade path: a database written under any other version is wiped and
|
||||
//! recreated from [`SCHEMA_CURRENT`]. See [`super::open`] for why.
|
||||
//!
|
||||
//! # The pragma profiles, and why there are six of them
|
||||
//!
|
||||
//! Each profile below belongs to one kind of connection, and the only field
|
||||
//! that differs between most of them is `cache_size`. They are separate
|
||||
//! constants rather than one shared string because the page cache is the
|
||||
//! largest thing a connection holds, and what the right size is depends
|
||||
//! entirely on what the connection does and how long it lives — a question
|
||||
//! with six different answers here.
|
||||
//!
|
||||
//! **A negative `cache_size` is KiB; a positive one would be a page count.**
|
||||
//! Nothing sets `page_size`, so a page count would be an unfalsifiable claim
|
||||
//! about SQLCipher's default, and SQLCipher reserves per-page bytes for the IV
|
||||
//! and HMAC on top of that, so pages do not convert to bytes by a clean
|
||||
//! multiply. Every value here is a ceiling in KiB, which is the thing actually
|
||||
//! being reasoned about.
|
||||
//!
|
||||
//! It *is* a ceiling and not a reservation — a connection that touches ten
|
||||
//! pages holds ten pages. What matters is which connections can reach the
|
||||
//! ceiling, which is any of them that scans a table, and how long they hold it
|
||||
//! afterwards. Page cache is `malloc`ed in 4 KiB units, far below glibc's mmap
|
||||
//! threshold, so a filled cache is arena memory: closing the connection
|
||||
//! returns it to the arena, not to the kernel. That is why these numbers show
|
||||
//! up in an *idle* process's footprint at all, and why
|
||||
//! [`crate::platform::release_free_heap`] exists alongside them.
|
||||
//!
|
||||
//! Note which way each profile is sized. Every one of them is small because
|
||||
//! its connection either scans once or lives a long time — except
|
||||
//! [`PRAGMAS_SEARCH`], which is large because it is the only cache that is
|
||||
//! *reused* often enough to pay for itself, and which is released when
|
||||
//! searching stops so that it never becomes part of the idle floor.
|
||||
//!
|
||||
//! | Profile | Connection | Lifetime | Cache |
|
||||
//! |---|---|---|---|
|
||||
//! | [`PRAGMAS_FAST`] | bulk indexer writer | one run | 8 MiB |
|
||||
//! | [`PRAGMAS_INCREMENTAL`] | coordinator's writer | released when idle | 4 MiB |
|
||||
//! | [`PRAGMAS_SEARCH`] | search worker | held across a typing session | 32 MiB |
|
||||
//! | [`PRAGMAS_READONLY`] | one-shot readers | a single query | 4 MiB |
|
||||
//! | [`PRAGMAS_MAINTENANCE`] | VACUUM | one bulk copy | 8 MiB |
|
||||
//! | [`PRAGMAS_WALK_READER`] | per-root row prefetch | the walk | 1 MiB |
|
||||
//!
|
||||
//! `PRAGMA mmap_size` is deliberately absent from all of them. It is not
|
||||
//! compile-disabled, and SQLCipher's codec only turns it off at runtime when a
|
||||
//! key is set — so an unprotected index could use it. It stays off because
|
||||
//! mapped pages still count in `VmRSS`, so it would not help the number this
|
||||
//! is all about, and because it would make memory behaviour differ between
|
||||
//! protected and unprotected installs, which is exactly the kind of silent
|
||||
//! per-configuration difference [`crate::platform`] argues against.
|
||||
|
||||
/// Pragmas applied on every writable connection open.
|
||||
/// The bulk indexer's write connection: one per run, dies with it.
|
||||
///
|
||||
/// 8 MiB of page cache, not the 40 MiB this profile used to take. The cache
|
||||
/// buys a writer the chance to batch dirty pages before spilling mid
|
||||
/// transaction, and a run commits every `batch_size` rows (200–500), so the
|
||||
/// working set between commits is nowhere near 40 MiB. The old figure was
|
||||
/// never chosen for this connection — it was chosen once and then inherited by
|
||||
/// every other profile that copied this one.
|
||||
///
|
||||
/// WAL, not journal-off: auto-indexing writes continuously while searches
|
||||
/// stream from their own read-only connections, and WAL is what lets those
|
||||
|
|
@ -18,7 +76,7 @@ pub const PRAGMAS_FAST: &str = "
|
|||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA synchronous = NORMAL;
|
||||
PRAGMA busy_timeout = 5000;
|
||||
PRAGMA cache_size = 10000;
|
||||
PRAGMA cache_size = -8192;
|
||||
PRAGMA temp_store = MEMORY;
|
||||
PRAGMA foreign_keys = ON;
|
||||
";
|
||||
|
|
@ -33,24 +91,98 @@ pub const PRAGMAS_FAST: &str = "
|
|||
/// entire rebuilt index in RAM. See [`super::repo::maintain`], which also
|
||||
/// points the temp directory at the index's own volume.
|
||||
///
|
||||
/// The smaller page cache is because this connection does one bulk copy and
|
||||
/// then closes; the 40 MiB the indexer keeps hot buys it nothing.
|
||||
/// The page cache is sized for one bulk copy followed by a close, which is
|
||||
/// all this connection ever does.
|
||||
pub const PRAGMAS_MAINTENANCE: &str = "
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA synchronous = NORMAL;
|
||||
PRAGMA busy_timeout = 5000;
|
||||
PRAGMA cache_size = 2000;
|
||||
PRAGMA cache_size = -8192;
|
||||
PRAGMA temp_store = FILE;
|
||||
PRAGMA foreign_keys = ON;
|
||||
";
|
||||
|
||||
/// The coordinator's long-lived write connection.
|
||||
///
|
||||
/// [`PRAGMAS_FAST`] without the `journal_mode`/`synchronous` lines it does not
|
||||
/// need to set again, and with a smaller cache, which is why it is its own
|
||||
/// profile. This connection exists to apply a handful of single-row upserts
|
||||
/// and deletes per batch of watcher events; it has no bulk phase to batch for.
|
||||
///
|
||||
/// The size matters more here than anywhere else because of the lifetime. This
|
||||
/// is the one connection that lives as long as the process, so whatever it
|
||||
/// reaches, it holds — and [`super::super::scope::advance`] runs a
|
||||
/// forward-only scan of `files` through it after a config change, which is
|
||||
/// exactly the access pattern that fills a cache to its ceiling. Before this
|
||||
/// profile existed that meant an idle QuickSearch carried a full 40 MiB page
|
||||
/// cache from a reconciliation the user did once. It is now also dropped
|
||||
/// outright when the coordinator settles (see `Inner::go_idle`); this profile
|
||||
/// bounds what it can reach *before* then.
|
||||
pub const PRAGMAS_INCREMENTAL: &str = "
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA synchronous = NORMAL;
|
||||
PRAGMA busy_timeout = 5000;
|
||||
PRAGMA cache_size = -4096;
|
||||
PRAGMA temp_store = MEMORY;
|
||||
PRAGMA foreign_keys = ON;
|
||||
";
|
||||
|
||||
/// The search worker's connection, which is held across requests.
|
||||
///
|
||||
/// A search fires on every character typed, and the worker keeps one
|
||||
/// connection for the whole typing session rather than opening one per
|
||||
/// keystroke (see [`crate::search`]). That inverts what the cache is for: it
|
||||
/// is not there to absorb a single cold cascade, it is there to still be warm
|
||||
/// when the next character arrives.
|
||||
///
|
||||
/// **This is the one profile that is deliberately large, and the size is
|
||||
/// measured rather than reasoned.** `tests/search_perf.rs` sweeps it; on an
|
||||
/// encrypted index the curve is not a gradient but a cliff, and the cliff is
|
||||
/// at the working set:
|
||||
///
|
||||
/// | ceiling | warm, unencrypted | warm, encrypted |
|
||||
/// |---|---|---|
|
||||
/// | 32–40 MiB | ~19 ms | **~19 ms** |
|
||||
/// | 1–16 MiB | ~20 ms | **~47 ms** |
|
||||
///
|
||||
/// Unencrypted, the ceiling makes no difference at all — a miss is a `pread`
|
||||
/// from the OS page cache and a `memcpy`. Encrypted, SQLCipher caches pages
|
||||
/// *decrypted*, so a hit skips an AES-CBC decrypt and an HMAC-SHA512 verify
|
||||
/// per 4 KiB; below the working set every warm query pays for all of them
|
||||
/// again, which is the 2.5× above. One number covers both because sizing this
|
||||
/// by whether a key happens to be set would make search latency depend on a
|
||||
/// setting nobody would connect it to.
|
||||
///
|
||||
/// The ceiling only stands while someone is searching: the worker releases the
|
||||
/// connection after [`crate::search`]'s idle window, and
|
||||
/// [`crate::platform::release_free_heap`] returns the pages. So this buys warm
|
||||
/// search latency without adding to what an idle process holds — which is the
|
||||
/// trade the rest of these profiles are making in the other direction.
|
||||
///
|
||||
/// The knee tracks index size, so on a very large encrypted index even this
|
||||
/// will not hold the working set. That degrades to the old behaviour rather
|
||||
/// than to something worse, and the fix if it ever matters is a bigger number
|
||||
/// here, informed by the same test.
|
||||
pub const PRAGMAS_SEARCH: &str = "
|
||||
PRAGMA busy_timeout = 5000;
|
||||
PRAGMA cache_size = -32768;
|
||||
PRAGMA temp_store = MEMORY;
|
||||
PRAGMA foreign_keys = ON;
|
||||
";
|
||||
|
||||
/// Pragmas safe to apply on a read-only connection, where `journal_mode`
|
||||
/// and `synchronous` can't be changed on the file. Used by
|
||||
/// [`super::open::open_existing`] for read-only opens; write paths get the
|
||||
/// full [`PRAGMAS_FAST`] set.
|
||||
///
|
||||
/// This is the *one-shot* reader now that the search worker has
|
||||
/// [`PRAGMAS_SEARCH`]: the CLI query helpers, the duplicates scan, the
|
||||
/// coordinator's own small reads. Each opens, runs a single query, and closes.
|
||||
/// A cache only pays for itself across queries, and these connections have no
|
||||
/// across.
|
||||
pub const PRAGMAS_READONLY: &str = "
|
||||
PRAGMA busy_timeout = 5000;
|
||||
PRAGMA cache_size = 10000;
|
||||
PRAGMA cache_size = -4096;
|
||||
PRAGMA temp_store = MEMORY;
|
||||
PRAGMA foreign_keys = ON;
|
||||
";
|
||||
|
|
@ -59,17 +191,17 @@ pub const PRAGMAS_READONLY: &str = "
|
|||
///
|
||||
/// Identical to [`PRAGMAS_READONLY`] but for `cache_size`, and that one
|
||||
/// difference is the point. One of these connections exists per indexing
|
||||
/// root, so the 10000-page (~40 MiB) cache the other profiles take would
|
||||
/// cost ~200 MiB across five roots — more than the per-directory
|
||||
/// classification this connection exists to serve was meant to save.
|
||||
/// root, so a cache sized for a connection that runs alone would be
|
||||
/// multiplied by the root count — and this was the first profile to be sized
|
||||
/// for its access pattern rather than copied from [`PRAGMAS_FAST`], which is
|
||||
/// why the others now are too.
|
||||
///
|
||||
/// 256 pages (~1 MiB) is enough to hold the upper levels of
|
||||
/// `idx_files_parent` hot, which is all these queries touch: each one is a
|
||||
/// single index range lookup, and the pages under it are read once and not
|
||||
/// revisited.
|
||||
/// 1 MiB is enough to hold the upper levels of `idx_files_parent` hot, which
|
||||
/// is all these queries touch: each one is a single index range lookup, and
|
||||
/// the pages under it are read once and not revisited.
|
||||
pub const PRAGMAS_WALK_READER: &str = "
|
||||
PRAGMA busy_timeout = 5000;
|
||||
PRAGMA cache_size = 256;
|
||||
PRAGMA cache_size = -1024;
|
||||
PRAGMA temp_store = MEMORY;
|
||||
PRAGMA foreign_keys = ON;
|
||||
";
|
||||
|
|
|
|||
|
|
@ -1,373 +0,0 @@
|
|||
use std::ffi::OsString;
|
||||
use std::fs::File;
|
||||
use std::io::{BufReader, Read};
|
||||
|
||||
use quick_xml::events::Event;
|
||||
use quick_xml::Reader;
|
||||
use zip::ZipArchive;
|
||||
|
||||
/// Extract text from DOCX files by parsing the word/document.xml
|
||||
pub fn extract_text_from_docx(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
|
||||
let file = File::open(file_path)?;
|
||||
let mut archive = ZipArchive::new(BufReader::new(file))?;
|
||||
|
||||
let mut document_xml = archive.by_name("word/document.xml")?;
|
||||
let mut content = String::new();
|
||||
document_xml.read_to_string(&mut content)?;
|
||||
|
||||
let mut reader = Reader::from_str(&content);
|
||||
reader.trim_text(true);
|
||||
|
||||
let mut text_content = String::new();
|
||||
let mut buf = Vec::new();
|
||||
let mut in_text = false;
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(ref e)) => {
|
||||
if e.name().as_ref() == b"w:t" {
|
||||
in_text = true;
|
||||
}
|
||||
}
|
||||
Ok(Event::Text(e)) if in_text => {
|
||||
text_content.push_str(&e.unescape()?.into_owned());
|
||||
}
|
||||
Ok(Event::End(ref e)) => {
|
||||
if e.name().as_ref() == b"w:t" {
|
||||
in_text = false;
|
||||
} else if e.name().as_ref() == b"w:p" {
|
||||
text_content.push('\n');
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
Err(e) => return Err(format!("Error parsing XML: {}", e).into()),
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
|
||||
Ok(text_content)
|
||||
}
|
||||
|
||||
/// Extract text from XLSX files by parsing worksheet XML files
|
||||
pub fn extract_text_from_xlsx(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
|
||||
let file = File::open(file_path)?;
|
||||
let mut archive = ZipArchive::new(BufReader::new(file))?;
|
||||
|
||||
let mut text_content = String::new();
|
||||
|
||||
// First, read shared strings if they exist
|
||||
let mut shared_strings = Vec::new();
|
||||
if let Ok(mut shared_strings_xml) = archive.by_name("xl/sharedStrings.xml") {
|
||||
let mut content = String::new();
|
||||
shared_strings_xml.read_to_string(&mut content)?;
|
||||
|
||||
let mut reader = Reader::from_str(&content);
|
||||
reader.trim_text(true);
|
||||
let mut buf = Vec::new();
|
||||
let mut in_text = false;
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(ref e)) => {
|
||||
if e.name().as_ref() == b"t" {
|
||||
in_text = true;
|
||||
}
|
||||
}
|
||||
Ok(Event::Text(e)) if in_text => {
|
||||
shared_strings.push(e.unescape()?.into_owned());
|
||||
}
|
||||
Ok(Event::End(ref e)) => {
|
||||
if e.name().as_ref() == b"t" {
|
||||
in_text = false;
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
}
|
||||
|
||||
// Read worksheets
|
||||
for i in 0..archive.len() {
|
||||
let file_name = archive.by_index(i)?.name().to_string();
|
||||
if file_name.starts_with("xl/worksheets/sheet") && file_name.ends_with(".xml") {
|
||||
let mut sheet_xml = archive.by_index(i)?;
|
||||
let mut content = String::new();
|
||||
sheet_xml.read_to_string(&mut content)?;
|
||||
|
||||
let mut reader = Reader::from_str(&content);
|
||||
reader.trim_text(true);
|
||||
let mut buf = Vec::new();
|
||||
let mut in_cell = false;
|
||||
let mut cell_type = String::new();
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(ref e)) => {
|
||||
if e.name().as_ref() == b"c" {
|
||||
in_cell = true;
|
||||
cell_type.clear();
|
||||
for attr in e.attributes() {
|
||||
let attr = attr?;
|
||||
if attr.key.as_ref() == b"t" {
|
||||
cell_type = String::from_utf8_lossy(&attr.value).to_string();
|
||||
}
|
||||
}
|
||||
} else if e.name().as_ref() == b"v" && in_cell {
|
||||
// Value element
|
||||
}
|
||||
}
|
||||
Ok(Event::Text(e)) if in_cell => {
|
||||
let text = e.unescape()?.into_owned();
|
||||
if cell_type == "s" {
|
||||
// Shared string reference
|
||||
if let Ok(index) = text.parse::<usize>() {
|
||||
if index < shared_strings.len() {
|
||||
text_content.push_str(&shared_strings[index]);
|
||||
text_content.push(' ');
|
||||
}
|
||||
}
|
||||
} else {
|
||||
text_content.push_str(&text);
|
||||
text_content.push(' ');
|
||||
}
|
||||
}
|
||||
Ok(Event::End(ref e)) => {
|
||||
if e.name().as_ref() == b"c" {
|
||||
in_cell = false;
|
||||
} else if e.name().as_ref() == b"row" {
|
||||
text_content.push('\n');
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(text_content)
|
||||
}
|
||||
|
||||
/// Extract text from PPTX files by parsing slide XML files
|
||||
pub fn extract_text_from_pptx(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
|
||||
let file = File::open(file_path)?;
|
||||
let mut archive = ZipArchive::new(BufReader::new(file))?;
|
||||
|
||||
let mut text_content = String::new();
|
||||
|
||||
// Read all slide files
|
||||
for i in 0..archive.len() {
|
||||
let file_name = archive.by_index(i)?.name().to_string();
|
||||
if file_name.starts_with("ppt/slides/slide") && file_name.ends_with(".xml") {
|
||||
let mut slide_xml = archive.by_index(i)?;
|
||||
let mut content = String::new();
|
||||
slide_xml.read_to_string(&mut content)?;
|
||||
|
||||
let mut reader = Reader::from_str(&content);
|
||||
reader.trim_text(true);
|
||||
let mut buf = Vec::new();
|
||||
let mut in_text = false;
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(ref e)) => {
|
||||
if e.name().as_ref() == b"a:t" {
|
||||
in_text = true;
|
||||
}
|
||||
}
|
||||
Ok(Event::Text(e)) if in_text => {
|
||||
text_content.push_str(&e.unescape()?.into_owned());
|
||||
}
|
||||
Ok(Event::End(ref e)) => {
|
||||
if e.name().as_ref() == b"a:t" {
|
||||
in_text = false;
|
||||
} else if e.name().as_ref() == b"a:p" {
|
||||
text_content.push('\n');
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
text_content.push_str("\n--- New Slide ---\n");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(text_content)
|
||||
}
|
||||
|
||||
/// Extract text from ODT files (OpenDocument Text)
|
||||
pub fn extract_text_from_odt(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
|
||||
let file = File::open(file_path)?;
|
||||
let mut archive = ZipArchive::new(BufReader::new(file))?;
|
||||
|
||||
let mut content_xml = archive.by_name("content.xml")?;
|
||||
let mut content = String::new();
|
||||
content_xml.read_to_string(&mut content)?;
|
||||
|
||||
let mut reader = Reader::from_str(&content);
|
||||
reader.trim_text(true);
|
||||
|
||||
let mut text_content = String::new();
|
||||
let mut buf = Vec::new();
|
||||
let mut in_text = false;
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(ref e)) => {
|
||||
let name = e.name();
|
||||
if name.as_ref() == b"text:p"
|
||||
|| name.as_ref() == b"text:h"
|
||||
|| name.as_ref() == b"text:span"
|
||||
{
|
||||
in_text = true;
|
||||
}
|
||||
}
|
||||
Ok(Event::Text(e)) if in_text => {
|
||||
text_content.push_str(&e.unescape()?.into_owned());
|
||||
}
|
||||
Ok(Event::End(ref e)) => {
|
||||
let name = e.name();
|
||||
if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" {
|
||||
text_content.push('\n');
|
||||
in_text = false;
|
||||
} else if name.as_ref() == b"text:span" {
|
||||
in_text = false;
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
|
||||
Ok(text_content)
|
||||
}
|
||||
|
||||
/// Extract text from ODP files (OpenDocument Presentation)
|
||||
pub fn extract_text_from_odp(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
|
||||
let file = File::open(file_path)?;
|
||||
let mut archive = ZipArchive::new(BufReader::new(file))?;
|
||||
|
||||
let mut content_xml = archive.by_name("content.xml")?;
|
||||
let mut content = String::new();
|
||||
content_xml.read_to_string(&mut content)?;
|
||||
|
||||
let mut reader = Reader::from_str(&content);
|
||||
reader.trim_text(true);
|
||||
|
||||
let mut text_content = String::new();
|
||||
let mut buf = Vec::new();
|
||||
let mut in_text = false;
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(ref e)) => {
|
||||
let name = e.name();
|
||||
if name.as_ref() == b"text:p"
|
||||
|| name.as_ref() == b"text:h"
|
||||
|| name.as_ref() == b"text:span"
|
||||
{
|
||||
in_text = true;
|
||||
}
|
||||
}
|
||||
Ok(Event::Text(e)) if in_text => {
|
||||
text_content.push_str(&e.unescape()?.into_owned());
|
||||
}
|
||||
Ok(Event::End(ref e)) => {
|
||||
let name = e.name();
|
||||
if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" {
|
||||
text_content.push('\n');
|
||||
in_text = false;
|
||||
} else if name.as_ref() == b"text:span" {
|
||||
in_text = false;
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
|
||||
Ok(text_content)
|
||||
}
|
||||
|
||||
/// Extract text from ODS files (OpenDocument Spreadsheet)
|
||||
pub fn extract_text_from_ods(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
|
||||
let file = File::open(file_path)?;
|
||||
let mut archive = ZipArchive::new(BufReader::new(file))?;
|
||||
|
||||
let mut content_xml = archive.by_name("content.xml")?;
|
||||
let mut content = String::new();
|
||||
content_xml.read_to_string(&mut content)?;
|
||||
|
||||
let mut reader = Reader::from_str(&content);
|
||||
reader.trim_text(true);
|
||||
|
||||
let mut text_content = String::new();
|
||||
let mut buf = Vec::new();
|
||||
let mut in_text = false;
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(ref e)) => {
|
||||
let name = e.name();
|
||||
if name.as_ref() == b"text:p" || name.as_ref() == b"text:span" {
|
||||
in_text = true;
|
||||
}
|
||||
}
|
||||
Ok(Event::Text(e)) if in_text => {
|
||||
text_content.push_str(&e.unescape()?.into_owned());
|
||||
text_content.push(' ');
|
||||
}
|
||||
Ok(Event::End(ref e)) => {
|
||||
let name = e.name();
|
||||
if name.as_ref() == b"text:p" {
|
||||
text_content.push('\n');
|
||||
in_text = false;
|
||||
} else if name.as_ref() == b"text:span" {
|
||||
in_text = false;
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
|
||||
Ok(text_content)
|
||||
}
|
||||
|
||||
/// Extract text from various document formats
|
||||
pub fn extract_document_text(
|
||||
file_path: &OsString,
|
||||
extension: &str,
|
||||
) -> Result<String, Box<dyn std::error::Error>> {
|
||||
match extension {
|
||||
"docx" => extract_text_from_docx(file_path),
|
||||
"doc" => {
|
||||
// DOC format is binary and complex to parse without external tools
|
||||
// For now, return an empty result
|
||||
Ok(String::new())
|
||||
}
|
||||
"xlsx" => extract_text_from_xlsx(file_path),
|
||||
"xls" => {
|
||||
// XLS format is binary and complex to parse without external tools
|
||||
Ok(String::new())
|
||||
}
|
||||
"pptx" => extract_text_from_pptx(file_path),
|
||||
"ppt" => {
|
||||
// PPT format is binary and complex to parse without external tools
|
||||
Ok(String::new())
|
||||
}
|
||||
"odt" => extract_text_from_odt(file_path),
|
||||
"odp" => extract_text_from_odp(file_path),
|
||||
"ods" => extract_text_from_ods(file_path),
|
||||
_ => Ok(String::new()),
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
//! Image metadata extraction via [`kamadak_exif`]. Reads EXIF tags (camera
|
||||
//! Image metadata extraction via `kamadak-exif`. Reads EXIF tags (camera
|
||||
//! make/model, date, GPS, dimensions) into properties. `text` is left empty
|
||||
//! — this extractor does not OCR.
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ use std::path::Path;
|
|||
pub mod audio;
|
||||
pub mod image;
|
||||
pub mod office;
|
||||
pub mod ole;
|
||||
pub mod pdf;
|
||||
pub mod plaintext;
|
||||
pub mod rtf;
|
||||
|
|
|
|||
|
|
@ -1,13 +1,27 @@
|
|||
//! Office document extraction: DOCX, XLSX, PPTX, ODT, ODP, ODS.
|
||||
//!
|
||||
//! Delegates to [`crate::document_extraction`] — a single entry point based
|
||||
//! on file extension rather than MIME. We translate the MIME to the
|
||||
//! extension expected by that module.
|
||||
//! All six are zip containers holding XML, and five of the six want the same
|
||||
//! thing from it: the character data of a few named elements, with a newline
|
||||
//! where a paragraph closes. That shape lives in [`collect_xml_text`], driven
|
||||
//! by a per-format [`TextSpec`], so there is one event loop rather than one
|
||||
//! per format.
|
||||
//!
|
||||
//! XLSX is the exception and keeps its own two loops: its text is not in the
|
||||
//! sheet at all but in a shared-string table the cells index into, which is a
|
||||
//! different machine, not a different table of element names.
|
||||
//!
|
||||
//! Dispatch is by file extension rather than MIME. `.docm` carries the same
|
||||
//! MIME as `.docx` but needs the same reader, and the extension is what
|
||||
//! distinguishes them.
|
||||
|
||||
use std::ffi::OsString;
|
||||
use std::error::Error;
|
||||
use std::fs::File;
|
||||
use std::io::{BufReader, Read, Seek};
|
||||
use std::path::Path;
|
||||
|
||||
use crate::document_extraction::extract_document_text;
|
||||
use quick_xml::events::Event;
|
||||
use quick_xml::Reader;
|
||||
use zip::ZipArchive;
|
||||
|
||||
use super::{ExtractError, ExtractedContent, Extractor};
|
||||
|
||||
|
|
@ -28,22 +42,293 @@ fn mime_to_ext(mime: &str) -> Option<&'static str> {
|
|||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The shared XML text walk
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Which elements of a format's XML carry text, and where paragraphs end.
|
||||
///
|
||||
/// `text` and `breaks` are matched independently on a closing tag: an element
|
||||
/// can be in both (ODF's `text:p` both holds text and ends a paragraph), in
|
||||
/// only one (`w:p` breaks but holds nothing directly), or in `text` alone
|
||||
/// (`text:span`, which ends a run without ending the line).
|
||||
struct TextSpec {
|
||||
/// Elements whose character data is body text.
|
||||
text: &'static [&'static [u8]],
|
||||
/// Elements that close a paragraph, emitting `'\n'`.
|
||||
breaks: &'static [&'static [u8]],
|
||||
/// Emitted after each text run. Spreadsheets separate cells with it;
|
||||
/// prose formats leave it `None` so runs within a paragraph stay joined.
|
||||
separator: Option<char>,
|
||||
}
|
||||
|
||||
const DOCX: TextSpec = TextSpec {
|
||||
text: &[b"w:t"],
|
||||
breaks: &[b"w:p"],
|
||||
separator: None,
|
||||
};
|
||||
|
||||
const PPTX: TextSpec = TextSpec {
|
||||
text: &[b"a:t"],
|
||||
breaks: &[b"a:p"],
|
||||
separator: None,
|
||||
};
|
||||
|
||||
/// ODT and ODP are the same format as far as text extraction is concerned —
|
||||
/// both are ODF prose with headings, paragraphs and spans.
|
||||
const ODF_TEXT: TextSpec = TextSpec {
|
||||
text: &[b"text:p", b"text:h", b"text:span"],
|
||||
breaks: &[b"text:p", b"text:h"],
|
||||
separator: None,
|
||||
};
|
||||
|
||||
const ODF_SHEET: TextSpec = TextSpec {
|
||||
text: &[b"text:p", b"text:span"],
|
||||
breaks: &[b"text:p"],
|
||||
separator: Some(' '),
|
||||
};
|
||||
|
||||
/// Append the text `spec` selects out of `xml` to `out`.
|
||||
///
|
||||
/// `in_text` is a flag rather than a depth count, which means a closing
|
||||
/// `</text:span>` ends the run even though its enclosing `<text:p>` is still
|
||||
/// open. That is how every one of the six extractors this replaces behaved.
|
||||
fn collect_xml_text(xml: &str, spec: &TextSpec, out: &mut String) -> Result<(), Box<dyn Error>> {
|
||||
let mut reader = Reader::from_str(xml);
|
||||
reader.trim_text(true);
|
||||
let mut buf = Vec::new();
|
||||
let mut in_text = false;
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(ref e)) => {
|
||||
if spec.text.contains(&e.name().as_ref()) {
|
||||
in_text = true;
|
||||
}
|
||||
}
|
||||
Ok(Event::Text(e)) if in_text => {
|
||||
out.push_str(&e.unescape()?);
|
||||
if let Some(sep) = spec.separator {
|
||||
out.push(sep);
|
||||
}
|
||||
}
|
||||
Ok(Event::End(ref e)) => {
|
||||
let name = e.name();
|
||||
if spec.text.contains(&name.as_ref()) {
|
||||
in_text = false;
|
||||
}
|
||||
if spec.breaks.contains(&name.as_ref()) {
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
// Propagated rather than ignored. Five of the six loops this
|
||||
// replaces had no error arm at all, so a malformed member sent
|
||||
// them round the loop on an error the reader kept re-reporting
|
||||
// without advancing — a hang on a file the user merely happened
|
||||
// to have on disk.
|
||||
Err(e) => return Err(format!("Error parsing XML: {}", e).into()),
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Container access
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type Archive = ZipArchive<BufReader<File>>;
|
||||
|
||||
fn open_container(path: &Path) -> Result<Archive, Box<dyn Error>> {
|
||||
Ok(ZipArchive::new(BufReader::new(File::open(path)?))?)
|
||||
}
|
||||
|
||||
/// One member's bytes as a string.
|
||||
fn member_text<R: Read + Seek>(
|
||||
archive: &mut ZipArchive<R>,
|
||||
name: &str,
|
||||
) -> Result<String, Box<dyn Error>> {
|
||||
let mut member = archive.by_name(name)?;
|
||||
let mut body = String::new();
|
||||
member.read_to_string(&mut body)?;
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
/// Names of the `.xml` members under `prefix`, in archive order.
|
||||
///
|
||||
/// Indexed rather than taken from `file_names()`, which iterates a hash map:
|
||||
/// slide order is the archive's order, and hashing it would shuffle the
|
||||
/// slides of every presentation.
|
||||
fn xml_members_under<R: Read + Seek>(
|
||||
archive: &mut ZipArchive<R>,
|
||||
prefix: &str,
|
||||
) -> Result<Vec<String>, Box<dyn Error>> {
|
||||
let mut names = Vec::new();
|
||||
for i in 0..archive.len() {
|
||||
let name = archive.by_index(i)?.name().to_string();
|
||||
if name.starts_with(prefix) && name.ends_with(".xml") {
|
||||
names.push(name);
|
||||
}
|
||||
}
|
||||
Ok(names)
|
||||
}
|
||||
|
||||
/// A format whose whole text lives in one member under one spec.
|
||||
fn single_member(
|
||||
path: &Path,
|
||||
member: &str,
|
||||
spec: &TextSpec,
|
||||
) -> Result<String, Box<dyn Error>> {
|
||||
let mut archive = open_container(path)?;
|
||||
let xml = member_text(&mut archive, member)?;
|
||||
let mut out = String::new();
|
||||
collect_xml_text(&xml, spec, &mut out)?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn extract_pptx(path: &Path) -> Result<String, Box<dyn Error>> {
|
||||
let mut archive = open_container(path)?;
|
||||
let mut out = String::new();
|
||||
for name in xml_members_under(&mut archive, "ppt/slides/slide")? {
|
||||
let xml = member_text(&mut archive, &name)?;
|
||||
collect_xml_text(&xml, &PPTX, &mut out)?;
|
||||
out.push_str("\n--- New Slide ---\n");
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// XLSX: shared strings plus cells
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// The workbook's shared-string table, in index order. Absent or unreadable
|
||||
/// is not an error: a sheet of nothing but numbers has no table at all.
|
||||
fn shared_strings<R: Read + Seek>(archive: &mut ZipArchive<R>) -> Vec<String> {
|
||||
let Ok(xml) = member_text(archive, "xl/sharedStrings.xml") else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut reader = Reader::from_str(&xml);
|
||||
reader.trim_text(true);
|
||||
let mut buf = Vec::new();
|
||||
let mut strings = Vec::new();
|
||||
let mut in_text = false;
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(ref e)) if e.name().as_ref() == b"t" => in_text = true,
|
||||
Ok(Event::Text(e)) if in_text => match e.unescape() {
|
||||
Ok(s) => strings.push(s.into_owned()),
|
||||
Err(_) => return strings,
|
||||
},
|
||||
Ok(Event::End(ref e)) if e.name().as_ref() == b"t" => in_text = false,
|
||||
Ok(Event::Eof) | Err(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
strings
|
||||
}
|
||||
|
||||
/// One worksheet's cells. A `t="s"` cell holds an index into `strings`
|
||||
/// rather than text of its own; every other type holds its value inline.
|
||||
fn collect_sheet(xml: &str, strings: &[String], out: &mut String) -> Result<(), Box<dyn Error>> {
|
||||
let mut reader = Reader::from_str(xml);
|
||||
reader.trim_text(true);
|
||||
let mut buf = Vec::new();
|
||||
let mut in_cell = false;
|
||||
let mut cell_type = String::new();
|
||||
|
||||
loop {
|
||||
match reader.read_event_into(&mut buf) {
|
||||
Ok(Event::Start(ref e)) if e.name().as_ref() == b"c" => {
|
||||
in_cell = true;
|
||||
cell_type.clear();
|
||||
for attr in e.attributes() {
|
||||
let attr = attr?;
|
||||
if attr.key.as_ref() == b"t" {
|
||||
cell_type = String::from_utf8_lossy(&attr.value).to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Event::Text(e)) if in_cell => {
|
||||
let text = e.unescape()?;
|
||||
if cell_type == "s" {
|
||||
// A shared-string reference. An index past the end of the
|
||||
// table is a corrupt workbook, not something to guess at.
|
||||
if let Some(s) = text.parse::<usize>().ok().and_then(|i| strings.get(i)) {
|
||||
out.push_str(s);
|
||||
out.push(' ');
|
||||
}
|
||||
} else {
|
||||
out.push_str(&text);
|
||||
out.push(' ');
|
||||
}
|
||||
}
|
||||
Ok(Event::End(ref e)) => {
|
||||
let name = e.name();
|
||||
if name.as_ref() == b"c" {
|
||||
in_cell = false;
|
||||
} else if name.as_ref() == b"row" {
|
||||
out.push('\n');
|
||||
}
|
||||
}
|
||||
Ok(Event::Eof) => break,
|
||||
Err(e) => return Err(format!("Error parsing XML: {}", e).into()),
|
||||
_ => {}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn extract_xlsx(path: &Path) -> Result<String, Box<dyn Error>> {
|
||||
let mut archive = open_container(path)?;
|
||||
let strings = shared_strings(&mut archive);
|
||||
let mut out = String::new();
|
||||
for name in xml_members_under(&mut archive, "xl/worksheets/sheet")? {
|
||||
let xml = member_text(&mut archive, &name)?;
|
||||
collect_sheet(&xml, &strings, &mut out)?;
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Dispatch
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Extract text from an office document, chosen by lowercase extension.
|
||||
///
|
||||
/// An extension nothing here handles yields empty text rather than an error:
|
||||
/// the caller reaches this only for a MIME [`mime_to_ext`] claimed, so an
|
||||
/// unrecognized extension means the file was named unlike its type.
|
||||
fn extract_document_text(path: &Path, extension: &str) -> Result<String, Box<dyn Error>> {
|
||||
match extension {
|
||||
"docx" => single_member(path, "word/document.xml", &DOCX),
|
||||
"xlsx" => extract_xlsx(path),
|
||||
"pptx" => extract_pptx(path),
|
||||
"odt" | "odp" => single_member(path, "content.xml", &ODF_TEXT),
|
||||
"ods" => single_member(path, "content.xml", &ODF_SHEET),
|
||||
// Pre-2007 binary formats: a different container entirely.
|
||||
"doc" | "xls" | "ppt" => super::ole::extract_ole_text(path, extension),
|
||||
_ => Ok(String::new()),
|
||||
}
|
||||
}
|
||||
|
||||
impl Extractor for OfficeExtractor {
|
||||
fn supports(&self, mime: &str) -> bool {
|
||||
mime_to_ext(mime).is_some()
|
||||
}
|
||||
|
||||
fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError> {
|
||||
// Recompute the extension from MIME at call time by asking the path.
|
||||
// Using the filesystem extension directly is more robust than round-
|
||||
// tripping through MIME: `.docm` has the same MIME as `.docx` but
|
||||
// `extract_document_text` looks up by extension.
|
||||
// The extension from the path, not from the MIME: `.docm` and `.docx`
|
||||
// share a MIME but the dispatch above is by extension.
|
||||
let ext = path
|
||||
.extension()
|
||||
.and_then(|s| s.to_str())
|
||||
.map(|s| s.to_ascii_lowercase())
|
||||
.unwrap_or_default();
|
||||
let text = extract_document_text(&OsString::from(path.as_os_str()), &ext)
|
||||
let text = extract_document_text(path, &ext)
|
||||
.map_err(|e| format!("office extractor {}: {}", path.display(), e))?;
|
||||
Ok(ExtractedContent::with_text(text))
|
||||
}
|
||||
|
|
@ -52,6 +337,7 @@ impl Extractor for OfficeExtractor {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
#[test]
|
||||
fn supports_docx_and_friends() {
|
||||
|
|
@ -66,4 +352,203 @@ mod tests {
|
|||
}
|
||||
assert!(!e.supports("image/png"));
|
||||
}
|
||||
|
||||
/// A zip container holding `members`, written to a scratch file.
|
||||
fn container(tag: &str, ext: &str, members: &[(&str, &str)]) -> std::path::PathBuf {
|
||||
let path = crate::testutil::scratch_dir(tag).join(format!("doc.{ext}"));
|
||||
let file = File::create(&path).unwrap();
|
||||
let mut zip = zip::ZipWriter::new(file);
|
||||
for (name, body) in members {
|
||||
zip.start_file(*name, zip::write::FileOptions::default())
|
||||
.unwrap();
|
||||
zip.write_all(body.as_bytes()).unwrap();
|
||||
}
|
||||
zip.finish().unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
const DOCX_BODY: &str = "<w:document><w:body>\
|
||||
<w:p><w:r><w:t>Hello</w:t></w:r><w:r><w:t>world</w:t></w:r></w:p>\
|
||||
<w:p><w:r><w:t>Second</w:t></w:r></w:p>\
|
||||
</w:body></w:document>";
|
||||
|
||||
const PPTX_SLIDE: &str = "<p:sld><p:cSld><p:spTree><p:sp><p:txBody>\
|
||||
<a:p><a:r><a:t>Title</a:t></a:r></a:p>\
|
||||
<a:p><a:r><a:t>Body</a:t></a:r></a:p>\
|
||||
</p:txBody></p:sp></p:spTree></p:cSld></p:sld>";
|
||||
|
||||
const ODT_BODY: &str = "<office:document-content><office:body><office:text>\
|
||||
<text:h>Heading</text:h>\
|
||||
<text:p>Para<text:span>span</text:span></text:p>\
|
||||
</office:text></office:body></office:document-content>";
|
||||
|
||||
const ODS_BODY: &str = "<office:document-content><office:body><office:spreadsheet>\
|
||||
<table:table><table:table-row>\
|
||||
<table:table-cell><text:p>A1</text:p></table:table-cell>\
|
||||
<table:table-cell><text:p>B1</text:p></table:table-cell>\
|
||||
</table:table-row></table:table>\
|
||||
</office:spreadsheet></office:body></office:document-content>";
|
||||
|
||||
const XLSX_SHARED: &str = "<sst><si><t>Shared</t></si><si><t>Second</t></si></sst>";
|
||||
|
||||
const XLSX_SHEET: &str = "<worksheet><sheetData>\
|
||||
<row><c t=\"s\"><v>0</v></c><c t=\"n\"><v>42</v></c></row>\
|
||||
<row><c t=\"s\"><v>1</v></c></row>\
|
||||
</sheetData></worksheet>";
|
||||
|
||||
// The golden set. These strings were recorded from the six hand-written
|
||||
// extractors this module replaced, so they pin the rewrite to exactly
|
||||
// what shipped rather than to what it ought to have produced.
|
||||
|
||||
#[test]
|
||||
fn docx() {
|
||||
let p = container("docx", "docx", &[("word/document.xml", DOCX_BODY)]);
|
||||
assert_eq!(
|
||||
extract_document_text(&p, "docx").unwrap(),
|
||||
"Helloworld\nSecond\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pptx_marks_each_slide_and_keeps_archive_order() {
|
||||
let p = container(
|
||||
"pptx",
|
||||
"pptx",
|
||||
&[
|
||||
("ppt/slides/slide1.xml", PPTX_SLIDE),
|
||||
("ppt/slides/slide2.xml", PPTX_SLIDE),
|
||||
],
|
||||
);
|
||||
assert_eq!(
|
||||
extract_document_text(&p, "pptx").unwrap(),
|
||||
"Title\nBody\n\n--- New Slide ---\nTitle\nBody\n\n--- New Slide ---\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn odt_and_odp_are_the_same_extraction() {
|
||||
let odt = container("odt", "odt", &[("content.xml", ODT_BODY)]);
|
||||
let odp = container("odp", "odp", &[("content.xml", ODT_BODY)]);
|
||||
assert_eq!(
|
||||
extract_document_text(&odt, "odt").unwrap(),
|
||||
"Heading\nParaspan\n"
|
||||
);
|
||||
assert_eq!(
|
||||
extract_document_text(&odt, "odt").unwrap(),
|
||||
extract_document_text(&odp, "odp").unwrap(),
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ods_separates_cells_with_a_space() {
|
||||
let p = container("ods", "ods", &[("content.xml", ODS_BODY)]);
|
||||
assert_eq!(extract_document_text(&p, "ods").unwrap(), "A1 \nB1 \n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xlsx_resolves_shared_strings() {
|
||||
let p = container(
|
||||
"xlsx",
|
||||
"xlsx",
|
||||
&[
|
||||
("xl/sharedStrings.xml", XLSX_SHARED),
|
||||
("xl/worksheets/sheet1.xml", XLSX_SHEET),
|
||||
],
|
||||
);
|
||||
assert_eq!(
|
||||
extract_document_text(&p, "xlsx").unwrap(),
|
||||
"Shared 42 \nSecond \n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_extension_nothing_handles_is_empty() {
|
||||
let p = container("none", "bin", &[("whatever", "x")]);
|
||||
assert_eq!(extract_document_text(&p, "zzz").unwrap(), "");
|
||||
}
|
||||
|
||||
/// A workbook of pure numbers has no shared-string table. Its absence is
|
||||
/// normal, not a failure.
|
||||
#[test]
|
||||
fn xlsx_without_a_shared_string_table_still_reads_its_cells() {
|
||||
let p = container(
|
||||
"xlsx-nosst",
|
||||
"xlsx",
|
||||
&[("xl/worksheets/sheet1.xml", "<worksheet><sheetData>\
|
||||
<row><c t=\"n\"><v>7</v></c></row></sheetData></worksheet>")],
|
||||
);
|
||||
assert_eq!(extract_document_text(&p, "xlsx").unwrap(), "7 \n");
|
||||
}
|
||||
|
||||
/// A shared-string index past the end of the table is dropped rather than
|
||||
/// panicking on the slice.
|
||||
#[test]
|
||||
fn an_out_of_range_shared_string_index_is_dropped() {
|
||||
let p = container(
|
||||
"xlsx-oob",
|
||||
"xlsx",
|
||||
&[
|
||||
("xl/sharedStrings.xml", "<sst><si><t>only</t></si></sst>"),
|
||||
(
|
||||
"xl/worksheets/sheet1.xml",
|
||||
"<worksheet><sheetData><row>\
|
||||
<c t=\"s\"><v>0</v></c><c t=\"s\"><v>99</v></c>\
|
||||
</row></sheetData></worksheet>",
|
||||
),
|
||||
],
|
||||
);
|
||||
assert_eq!(extract_document_text(&p, "xlsx").unwrap(), "only \n");
|
||||
}
|
||||
|
||||
/// Malformed XML is an error, not a hang. Five of the six extractors this
|
||||
/// replaced had no error arm, so the reader re-reported the same failure
|
||||
/// forever without advancing.
|
||||
///
|
||||
/// An undefined entity inside a text run is the cheapest way to reach that
|
||||
/// arm, and it is a real shape: a document written by a tool that emitted
|
||||
/// HTML entities into OOXML.
|
||||
#[test]
|
||||
fn malformed_xml_returns_an_error_rather_than_looping() {
|
||||
for (ext, member, body) in [
|
||||
("docx", "word/document.xml", "<w:t>bad &nonsuch; entity</w:t>"),
|
||||
("odt", "content.xml", "<text:p>bad &nonsuch; entity</text:p>"),
|
||||
("ods", "content.xml", "<text:p>bad &nonsuch; entity</text:p>"),
|
||||
("pptx", "ppt/slides/slide1.xml", "<a:t>bad &nonsuch; entity</a:t>"),
|
||||
] {
|
||||
let p = container(&format!("bad-{ext}"), ext, &[(member, body)]);
|
||||
assert!(
|
||||
extract_document_text(&p, ext).is_err(),
|
||||
"{ext} should report malformed XML"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Mismatched tags are caught too — quick_xml checks closing names, and
|
||||
/// that error now reaches the caller instead of being swallowed.
|
||||
#[test]
|
||||
fn mismatched_tags_are_an_error() {
|
||||
let p = container(
|
||||
"mismatch",
|
||||
"docx",
|
||||
&[("word/document.xml", "<w:body><w:t>x</w:body>")],
|
||||
);
|
||||
assert!(extract_document_text(&p, "docx").is_err());
|
||||
}
|
||||
|
||||
/// A container missing the member the format is defined by.
|
||||
#[test]
|
||||
fn a_missing_member_is_an_error() {
|
||||
let p = container("empty", "docx", &[("unrelated.xml", "<x/>")]);
|
||||
assert!(extract_document_text(&p, "docx").is_err());
|
||||
}
|
||||
|
||||
/// Not a zip file at all — the shape a truncated download or a
|
||||
/// misidentified file arrives in.
|
||||
#[test]
|
||||
fn a_non_container_is_an_error() {
|
||||
let dir = crate::testutil::scratch_dir("notzip");
|
||||
let p = dir.join("doc.docx");
|
||||
crate::testutil::touch(&p, b"this is not a zip archive");
|
||||
assert!(extract_document_text(&p, "docx").is_err());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
1097
crates/quicksearch-core/src/extract/ole.rs
Normal file
1097
crates/quicksearch-core/src/extract/ole.rs
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -1,16 +1,30 @@
|
|||
//! PDF text extraction.
|
||||
//!
|
||||
//! Primary path: [`pdf_extract::extract_text`], which handles most modern PDFs
|
||||
//! and is simple to call. It can panic or hard-error on malformed files; any
|
||||
//! failure is surfaced to the caller and marks the file's content state as
|
||||
//! failed. Properties (title, author, etc.) from the PDF `Info` dictionary
|
||||
//! are pulled via `lopdf` where available.
|
||||
//! One `Document::load` per file, then both the text and the `Info` dictionary
|
||||
//! are taken off it. `pdf_extract` can panic or hard-error on malformed files;
|
||||
//! any failure is surfaced to the caller and marks the file's content state as
|
||||
//! failed.
|
||||
//!
|
||||
//! This used to be two loads of the same file — `pdf_extract::extract_text`,
|
||||
//! which parses the document internally, followed by a second parse through a
|
||||
//! directly-declared `lopdf` purely to read six `Info` strings. Both object
|
||||
//! graphs and the full extracted text were live at once, on the extractor that
|
||||
//! `examples/memprobe.rs` already named as a run's largest single consumer.
|
||||
//! Naming `lopdf` as a direct dependency also resolved a *second, older* copy
|
||||
//! of it, which is what dragged `rayon` — and a global thread pool that is
|
||||
//! never torn down — plus `chrono`, `time`, `md5` and a second `nom` into the
|
||||
//! build.
|
||||
//!
|
||||
//! So `lopdf` is reached through `pdf_extract`'s own `pub use lopdf::*` and
|
||||
//! must not be declared in `Cargo.toml` again. The re-export is not a
|
||||
//! semver-guaranteed surface, but a break in it is a compile error rather than
|
||||
//! a silent behaviour change, and the fixtures below cover the behaviour.
|
||||
|
||||
use std::cell::Cell;
|
||||
use std::path::Path;
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use lopdf::{Document as LopdfDocument, Object};
|
||||
use pdf_extract::{Document, Object, PlainTextOutput};
|
||||
|
||||
use super::{ExtractError, ExtractedContent, Extractor};
|
||||
|
||||
|
|
@ -56,45 +70,69 @@ impl Extractor for PdfExtractor {
|
|||
}
|
||||
|
||||
fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError> {
|
||||
// Text. Catch panics from pdf_extract (some PDFs crash its parser)
|
||||
// and keep the default hook from spamming stderr about them.
|
||||
// Catch panics from pdf_extract (some PDFs crash its parser) and keep
|
||||
// the default hook from spamming stderr about them. The whole
|
||||
// operation is inside the guard, document loading included — that used
|
||||
// to sit outside both it and the suppression window, so a panic in the
|
||||
// parser reached the process hook and took the thread with it.
|
||||
install_quiet_panic_hook();
|
||||
let path_buf = path.to_path_buf();
|
||||
SUPPRESS_PANIC_PRINT.with(|flag| flag.set(true));
|
||||
let result = std::panic::catch_unwind(move || pdf_extract::extract_text(&path_buf));
|
||||
let result = std::panic::catch_unwind(move || extract_one_pass(&path_buf));
|
||||
SUPPRESS_PANIC_PRINT.with(|flag| flag.set(false));
|
||||
let text = result
|
||||
.map_err(|panic| format!("pdf_extract panicked: {}", panic_message(&*panic)))?
|
||||
.map_err(|e| format!("pdf_extract: {}", e))?;
|
||||
result.map_err(|panic| format!("pdf_extract panicked: {}", panic_message(&*panic)))?
|
||||
}
|
||||
}
|
||||
|
||||
/// The six `Info` keys worth keeping, in the order they are written.
|
||||
const INFO_KEYS: [&str; 6] = [
|
||||
"Title", "Author", "Subject", "Keywords", "Creator", "Producer",
|
||||
];
|
||||
|
||||
/// Load the document once; take the text and the `Info` dictionary off it.
|
||||
///
|
||||
/// This is `pdf_extract::extract_text` — whose body is load, decrypt,
|
||||
/// `output_doc` — with the `Info` read folded in where the document is still
|
||||
/// in scope, which is the entire reason it is spelled out here rather than
|
||||
/// called.
|
||||
fn extract_one_pass(path: &Path) -> Result<ExtractedContent, ExtractError> {
|
||||
let mut doc = Document::load(path).map_err(|e| format!("pdf_extract: {}", e))?;
|
||||
// What `pdf_extract`'s own (private) `maybe_decrypt` does, and it has to
|
||||
// happen before either the content streams or the `Info` strings mean
|
||||
// anything — the previous two-load version never decrypted for the `Info`
|
||||
// half, so those properties were garbage on any encrypted file. Empty
|
||||
// password only: a real one is the user's to supply and nothing on this
|
||||
// path can ask for it.
|
||||
if doc.is_encrypted() {
|
||||
doc.decrypt("").map_err(|e| format!("pdf_extract: {}", e))?;
|
||||
}
|
||||
|
||||
let mut text = String::new();
|
||||
{
|
||||
let mut sink = PlainTextOutput::new(&mut text);
|
||||
pdf_extract::output_doc(&doc, &mut sink).map_err(|e| format!("pdf_extract: {}", e))?;
|
||||
}
|
||||
let mut out = ExtractedContent::with_text(text);
|
||||
|
||||
// Info dictionary via lopdf. Soft-fail: if lopdf can't open the file
|
||||
// we still return the text.
|
||||
if let Ok(doc) = LopdfDocument::load(path) {
|
||||
if let Ok(info_ref) = doc.trailer.get(b"Info") {
|
||||
if let Ok(info_id) = info_ref.as_reference() {
|
||||
if let Ok(info) = doc.get_object(info_id) {
|
||||
if let Ok(dict) = info.as_dict() {
|
||||
for key in [
|
||||
"Title", "Author", "Subject", "Keywords", "Creator", "Producer",
|
||||
] {
|
||||
if let Ok(val) = dict.get(key.as_bytes()) {
|
||||
if let Some(s) = object_to_string(val) {
|
||||
// Soft-fail, unchanged: a document with no readable `Info` dictionary
|
||||
// still has its text, and the text is the half that matters.
|
||||
let info = doc
|
||||
.trailer
|
||||
.get(b"Info")
|
||||
.ok()
|
||||
.and_then(|o| o.as_reference().ok())
|
||||
.and_then(|id| doc.get_object(id).ok())
|
||||
.and_then(|o| o.as_dict().ok());
|
||||
if let Some(dict) = info {
|
||||
for key in INFO_KEYS {
|
||||
if let Some(s) = dict.get(key.as_bytes()).ok().and_then(object_to_string) {
|
||||
if !s.is_empty() {
|
||||
out.properties.insert(key.to_ascii_lowercase(), s);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
fn object_to_string(obj: &Object) -> Option<String> {
|
||||
|
|
@ -131,4 +169,173 @@ mod tests {
|
|||
assert!(PdfExtractor.supports("application/pdf"));
|
||||
assert!(!PdfExtractor.supports("application/zip"));
|
||||
}
|
||||
|
||||
use pdf_extract::{dictionary, Dictionary, Stream, StringFormat};
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// Write a one-page PDF drawing `body`, with `info` as its `Info`
|
||||
/// dictionary, and return the path.
|
||||
///
|
||||
/// Built here rather than checked in as a blob, following the convention
|
||||
/// [`super::super::ole`] states: a fixture asserted to have a particular
|
||||
/// structure is not reviewable when it is opaque bytes. Everything needed
|
||||
/// is public through `pdf_extract`'s `lopdf` re-export, which is the same
|
||||
/// surface the extractor itself uses — so if that re-export ever moves,
|
||||
/// these fail to compile alongside it rather than silently stopping
|
||||
/// covering anything.
|
||||
///
|
||||
/// The page is deliberately minimal but complete: `output_doc` walks
|
||||
/// Catalog → Pages → Page and needs `MediaBox`, a `Resources` font it can
|
||||
/// resolve, and a content stream. Helvetica is one of the base-14 fonts,
|
||||
/// for which `pdf_extract` carries built-in encoding tables, so no font
|
||||
/// file is involved.
|
||||
fn write_pdf(tag: &str, body: &str, info: Option<Dictionary>) -> PathBuf {
|
||||
let mut doc = Document::with_version("1.5");
|
||||
let font = doc.add_object(dictionary! {
|
||||
"Type" => "Font",
|
||||
"Subtype" => "Type1",
|
||||
"BaseFont" => "Helvetica",
|
||||
});
|
||||
let resources = doc.add_object(dictionary! {
|
||||
"Font" => dictionary! { "F1" => font },
|
||||
});
|
||||
let content = format!("BT /F1 24 Tf 72 720 Td ({}) Tj ET", body);
|
||||
let contents = doc.add_object(Stream::new(dictionary! {}, content.into_bytes()));
|
||||
let pages_id = doc.new_object_id();
|
||||
let page = doc.add_object(dictionary! {
|
||||
"Type" => "Page",
|
||||
"Parent" => pages_id,
|
||||
"Contents" => contents,
|
||||
"MediaBox" => vec![0.into(), 0.into(), 612.into(), 792.into()],
|
||||
});
|
||||
doc.objects.insert(
|
||||
pages_id,
|
||||
Object::Dictionary(dictionary! {
|
||||
"Type" => "Pages",
|
||||
"Kids" => vec![page.into()],
|
||||
"Count" => 1,
|
||||
"Resources" => resources,
|
||||
}),
|
||||
);
|
||||
let catalog = doc.add_object(dictionary! {
|
||||
"Type" => "Catalog",
|
||||
"Pages" => pages_id,
|
||||
});
|
||||
doc.trailer.set("Root", catalog);
|
||||
if let Some(info) = info {
|
||||
let info_id = doc.add_object(Object::Dictionary(info));
|
||||
doc.trailer.set("Info", info_id);
|
||||
}
|
||||
|
||||
let path = crate::testutil::scratch_dir(tag).join("fixture.pdf");
|
||||
doc.save(&path).expect("write fixture pdf");
|
||||
path
|
||||
}
|
||||
|
||||
fn text_string(s: &str) -> Object {
|
||||
Object::String(s.as_bytes().to_vec(), StringFormat::Literal)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_text_and_info_properties() {
|
||||
let path = write_pdf(
|
||||
"pdf-full",
|
||||
"Hello QuickSearch",
|
||||
Some(dictionary! {
|
||||
"Title" => text_string("The Title"),
|
||||
"Author" => text_string("An Author"),
|
||||
"Subject" => text_string("A Subject"),
|
||||
"Keywords" => text_string("alpha beta"),
|
||||
"Creator" => text_string("A Creator"),
|
||||
"Producer" => text_string("A Producer"),
|
||||
}),
|
||||
);
|
||||
|
||||
let out = PdfExtractor.extract(&path).expect("extract");
|
||||
assert!(
|
||||
out.text.contains("Hello QuickSearch"),
|
||||
"drawn text missing from {:?}",
|
||||
out.text
|
||||
);
|
||||
// Lowercased keys, which is the contract the rest of the pipeline
|
||||
// stores under.
|
||||
assert_eq!(out.properties.get("title").map(String::as_str), Some("The Title"));
|
||||
assert_eq!(out.properties.get("author").map(String::as_str), Some("An Author"));
|
||||
assert_eq!(out.properties.get("subject").map(String::as_str), Some("A Subject"));
|
||||
assert_eq!(out.properties.get("keywords").map(String::as_str), Some("alpha beta"));
|
||||
assert_eq!(out.properties.get("creator").map(String::as_str), Some("A Creator"));
|
||||
assert_eq!(out.properties.get("producer").map(String::as_str), Some("A Producer"));
|
||||
}
|
||||
|
||||
/// The soft-fail path: no `Info` dictionary is not an extraction failure,
|
||||
/// because the text is the half that matters.
|
||||
#[test]
|
||||
fn missing_info_dictionary_still_yields_text() {
|
||||
let path = write_pdf("pdf-noinfo", "Body Only", None);
|
||||
let out = PdfExtractor.extract(&path).expect("extract");
|
||||
assert!(out.text.contains("Body Only"));
|
||||
assert!(
|
||||
out.properties.is_empty(),
|
||||
"unexpected properties: {:?}",
|
||||
out.properties
|
||||
);
|
||||
}
|
||||
|
||||
/// An empty `Info` value is absence, not an empty property.
|
||||
#[test]
|
||||
fn empty_info_values_are_not_stored() {
|
||||
let path = write_pdf(
|
||||
"pdf-emptyinfo",
|
||||
"Body",
|
||||
Some(dictionary! {
|
||||
"Title" => text_string(""),
|
||||
"Author" => text_string("Real Author"),
|
||||
}),
|
||||
);
|
||||
let out = PdfExtractor.extract(&path).expect("extract");
|
||||
assert!(!out.properties.contains_key("title"), "empty title stored");
|
||||
assert_eq!(
|
||||
out.properties.get("author").map(String::as_str),
|
||||
Some("Real Author")
|
||||
);
|
||||
}
|
||||
|
||||
/// `Info` values that are not strings or names are skipped rather than
|
||||
/// rendered — pins `object_to_string`'s catch-all arm.
|
||||
#[test]
|
||||
fn non_string_info_values_are_skipped() {
|
||||
let path = write_pdf(
|
||||
"pdf-badinfo",
|
||||
"Body",
|
||||
Some(dictionary! {
|
||||
"Producer" => 42,
|
||||
"Title" => text_string("Kept"),
|
||||
}),
|
||||
);
|
||||
let out = PdfExtractor.extract(&path).expect("extract");
|
||||
assert!(
|
||||
!out.properties.contains_key("producer"),
|
||||
"integer Info value was rendered: {:?}",
|
||||
out.properties
|
||||
);
|
||||
assert_eq!(out.properties.get("title").map(String::as_str), Some("Kept"));
|
||||
}
|
||||
|
||||
/// Malformed input must come back as an error, not take the process down.
|
||||
/// This is the case the widened `catch_unwind` exists for: the document
|
||||
/// load now runs inside it, where it used to run outside.
|
||||
#[test]
|
||||
fn malformed_pdf_fails_without_panicking_the_process() {
|
||||
let path = crate::testutil::scratch_dir("pdf-malformed").join("broken.pdf");
|
||||
std::fs::write(&path, b"%PDF-1.4\n\x00\x01\x02 not a pdf at all \xff\xfe").unwrap();
|
||||
|
||||
let err = PdfExtractor
|
||||
.extract(&path)
|
||||
.expect_err("malformed pdf must fail");
|
||||
assert!(
|
||||
err.starts_with("pdf_extract"),
|
||||
"unexpected failure reason: {}",
|
||||
err
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -124,17 +124,8 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
fn tmp(tag: &str, body: &[u8]) -> std::path::PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"qs-plaintext-{}-{}-{}.txt",
|
||||
tag,
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
std::fs::write(&p, body).unwrap();
|
||||
let p = crate::testutil::scratch_dir(tag).join("sample.txt");
|
||||
crate::testutil::touch(&p, body);
|
||||
p
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -58,17 +58,8 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
fn tmp(tag: &str, body: &[u8]) -> std::path::PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"qs-rtf-{}-{}-{}.rtf",
|
||||
tag,
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
std::fs::write(&p, body).unwrap();
|
||||
let p = crate::testutil::scratch_dir(tag).join("sample.rtf");
|
||||
crate::testutil::touch(&p, body);
|
||||
p
|
||||
}
|
||||
|
||||
|
|
@ -98,7 +89,13 @@ mod tests {
|
|||
fn malformed_input_errors_and_names_the_file() {
|
||||
let p = tmp("broken", br"{\rtf1 truncated");
|
||||
let err = RtfExtractor.extract(&p).unwrap_err();
|
||||
assert!(err.contains("qs-rtf-broken"), "must name the file: {}", err);
|
||||
// The path itself, not a fixed prefix: this is the message a user sees
|
||||
// in `list-failed`, and it is useless without naming the file.
|
||||
assert!(
|
||||
err.contains(&p.display().to_string()),
|
||||
"must name the file: {}",
|
||||
err
|
||||
);
|
||||
std::fs::remove_file(&p).ok();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
use std::collections::HashMap;
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
// Only the Unix entry-count path shells out; Windows walks the tree directly.
|
||||
use std::collections::HashMap;
|
||||
#[cfg(unix)]
|
||||
use std::process::{Command, Stdio};
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
|
@ -67,15 +67,12 @@ pub(crate) fn path_to_db_string(path: &Path) -> String {
|
|||
}
|
||||
}
|
||||
|
||||
/// Canonicalize a root string for storage/comparison, stripping the Windows
|
||||
/// UNC prefix. Multi-root strings (newline-joined) fail canonicalize and
|
||||
/// pass through verbatim, which still compares consistently.
|
||||
///
|
||||
/// The UNC strip is [`path_to_db_string`]'s, not a hand-rolled one: chopping
|
||||
/// four characters would turn `\\?\UNC\server\share` into
|
||||
/// `UNC\server\share`, which is not a path — and no longer looks like a
|
||||
/// share, so the root would walk with the local thread count instead of the
|
||||
/// network one.
|
||||
/// Canonicalize a root string for storage/comparison, via
|
||||
/// [`path_to_db_string`] rather than a hand-rolled prefix strip — the extra
|
||||
/// consequence here being that a mis-stripped share stops looking like one, so
|
||||
/// the root would walk with the local thread count instead of the network one.
|
||||
/// Multi-root strings (newline-joined) fail canonicalize and pass through
|
||||
/// verbatim, which still compares consistently.
|
||||
///
|
||||
/// This is the spelling `files.path` rows are prefixed with, so it is also the
|
||||
/// form roots must be compared in: `~/docs` and `/home/me/docs` name one root
|
||||
|
|
@ -196,8 +193,19 @@ impl UnreadableDirs {
|
|||
/// Whether `path` lies under a directory the walk failed to read, and so
|
||||
/// must not be treated as deleted.
|
||||
///
|
||||
/// Compares by path component, not by string prefix: `/a/bc` does not
|
||||
/// live under `/a/b`.
|
||||
/// # Containment is by path component, never by string prefix
|
||||
///
|
||||
/// `Path::starts_with` and `Path::ancestors` both compare whole
|
||||
/// components, so `/a/bc` does not live under `/a/b` and `/a/b.txt` is not
|
||||
/// swallowed by `/a/b`. A `str::starts_with` would say otherwise, and
|
||||
/// every use of this rule in the codebase decides whether rows get
|
||||
/// deleted, whether a watch is dropped, or whether a subtree is in scope —
|
||||
/// so a sibling matched by accident is a neighbouring folder's index
|
||||
/// disappearing. This is the reference statement of the rule;
|
||||
/// [`crate::incremental::collapse_removal_roots`],
|
||||
/// `coordinator::collapse_pending_removals`, `watcher::WatchRegistry::remove_tree`,
|
||||
/// [`crate::scope::Scope::owning_root`] and
|
||||
/// [`crate::config::nested_roots`] apply the same one.
|
||||
pub fn covers(&self, path: &str) -> bool {
|
||||
let dirs = self.dirs.lock().unwrap();
|
||||
if dirs.is_empty() {
|
||||
|
|
@ -217,7 +225,12 @@ impl UnreadableDirs {
|
|||
///
|
||||
/// Used as a `walkdir` `filter_entry` predicate, so returning `false` for a
|
||||
/// directory prunes the whole subtree instead of merely skipping the entry.
|
||||
fn walk_filter(e: &DirEntry, include_hidden: bool, ignore: &IgnoreSet) -> bool {
|
||||
fn walk_filter(
|
||||
e: &DirEntry,
|
||||
follow_symlinks: bool,
|
||||
include_hidden: bool,
|
||||
ignore: &IgnoreSet,
|
||||
) -> bool {
|
||||
// Depth 0 is the root itself — a `false` here would silence the
|
||||
// entire walk, and users explicitly chose their roots.
|
||||
if e.depth() == 0 {
|
||||
|
|
@ -227,7 +240,21 @@ fn walk_filter(e: &DirEntry, include_hidden: bool, ignore: &IgnoreSet) -> bool {
|
|||
// Free on Windows (walkdir hands back the attributes `FindNextFileW`
|
||||
// already returned) and never called on Unix, so the "no extra lstat"
|
||||
// property below still holds.
|
||||
if !include_hidden && crate::platform::entry_is_hidden(&name, || e.metadata().ok()) {
|
||||
//
|
||||
// The exception is a followed symlink: `DirEntry::metadata` switches to
|
||||
// `fs::metadata` there and would report the *target's* attributes, where
|
||||
// the parallel walker reports the link's own. `entry_hidden_reason`
|
||||
// requires the entry itself, so ask for it explicitly — walkdir was
|
||||
// already making a real call in that case, so this costs nothing extra.
|
||||
if !include_hidden
|
||||
&& crate::platform::entry_is_hidden(&name, || {
|
||||
if follow_symlinks && e.path_is_symlink() {
|
||||
std::fs::symlink_metadata(e.path()).ok()
|
||||
} else {
|
||||
e.metadata().ok()
|
||||
}
|
||||
})
|
||||
{
|
||||
return false;
|
||||
}
|
||||
!ignore.matches_component(&name) && !ignore.matches_path_pattern(e.path())
|
||||
|
|
@ -256,7 +283,7 @@ fn walk_entries<'a>(
|
|||
WalkDir::new(root)
|
||||
.follow_links(follow_symlinks)
|
||||
.into_iter()
|
||||
.filter_entry(move |e| walk_filter(e, include_hidden, ignore))
|
||||
.filter_entry(move |e| walk_filter(e, follow_symlinks, include_hidden, ignore))
|
||||
.filter_map(move |res| match res {
|
||||
Ok(e) => Some(e),
|
||||
Err(err) => {
|
||||
|
|
@ -310,7 +337,6 @@ pub fn filtered_dirs<'a>(
|
|||
fn parse_wc_l_stdout(bytes: &[u8]) -> Result<usize, String> {
|
||||
let s = String::from_utf8_lossy(bytes);
|
||||
let token = s
|
||||
.trim()
|
||||
.split_whitespace()
|
||||
.next()
|
||||
.ok_or_else(|| "wc: empty output".to_string())?;
|
||||
|
|
@ -405,15 +431,11 @@ fn count_find_pipe_wc(
|
|||
|
||||
/// Count tree entries with a plain directory walk.
|
||||
///
|
||||
/// Windows has no `find`, and the obvious substitute — `powershell.exe -Command
|
||||
/// "(Get-ChildItem -Recurse | Measure-Object).Count"` — is a poor trade: 300+
|
||||
/// ms of interpreter startup before any work, `Get-ChildItem -Recurse` is far
|
||||
/// slower than a `FindNextFileW` loop, it pops a console window on a windowed
|
||||
/// process, and the path has to be escaped into a script string. Walking
|
||||
/// directly is faster, quieter, and cancels immediately instead of at the
|
||||
/// 50 ms subprocess-poll granularity.
|
||||
/// Kept as the oracle [`count_tree_entries_win32`] is tested against: the fast
|
||||
/// path replaces it precisely because it must produce the same number, and an
|
||||
/// independent implementation is the only way to assert that.
|
||||
#[cfg(windows)]
|
||||
fn count_tree_entries_native(
|
||||
fn count_tree_entries_walkdir(
|
||||
path: &str,
|
||||
cancel: &std::sync::atomic::AtomicBool,
|
||||
) -> Result<usize, String> {
|
||||
|
|
@ -436,10 +458,151 @@ fn count_tree_entries_native(
|
|||
Ok(n)
|
||||
}
|
||||
|
||||
/// Count tree entries by reading directories in bulk through
|
||||
/// `GetFileInformationByHandleEx`.
|
||||
///
|
||||
/// Windows has no `find`, so the Unix pipeline has nothing to call. The
|
||||
/// obvious substitute — `powershell.exe -Command "(Get-ChildItem -Recurse |
|
||||
/// Measure-Object).Count"` — is a poor trade: 300+ ms of interpreter startup
|
||||
/// before any work, `Get-ChildItem -Recurse` is far slower than a directory
|
||||
/// read, it pops a console window on a windowed process, and the path has to
|
||||
/// be escaped into a script string.
|
||||
///
|
||||
/// The reason this exists rather than [`count_tree_entries_walkdir`] is the
|
||||
/// per-entry cost. `std::fs::read_dir` — which walkdir sits on — issues one
|
||||
/// `FindNextFileW` per entry and allocates a `PathBuf` for each, so counting a
|
||||
/// million-entry tree means a million syscalls and a million allocations to
|
||||
/// produce a single integer. `FileIdBothDirectoryInfo` fills a caller-supplied
|
||||
/// buffer with as many chained records as fit, which turns the syscall count
|
||||
/// into roughly one per 64 KiB of directory data and the allocations into one
|
||||
/// per directory. This is the documented Win32 form of what fast indexers use;
|
||||
/// it needs no new crate, no elevated rights, and no ntdll.
|
||||
///
|
||||
/// Recursion is explicit and iterative — a stack of directories still to read
|
||||
/// — because a deep tree must not be able to exhaust the thread's stack.
|
||||
#[cfg(windows)]
|
||||
fn count_tree_entries_win32(
|
||||
path: &str,
|
||||
cancel: &std::sync::atomic::AtomicBool,
|
||||
) -> Result<usize, String> {
|
||||
use std::ffi::OsString;
|
||||
use std::os::windows::ffi::{OsStrExt, OsStringExt};
|
||||
use std::sync::atomic::Ordering;
|
||||
use windows_sys::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE};
|
||||
use windows_sys::Win32::Storage::FileSystem::{
|
||||
CreateFileW, GetFileInformationByHandleEx, FileIdBothDirectoryInfo, FILE_ID_BOTH_DIR_INFO,
|
||||
FILE_FLAG_BACKUP_SEMANTICS, FILE_LIST_DIRECTORY, FILE_SHARE_DELETE, FILE_SHARE_READ,
|
||||
FILE_SHARE_WRITE, OPEN_EXISTING,
|
||||
};
|
||||
|
||||
/// One call returns as many entries as fit here. 64 KiB holds several
|
||||
/// hundred typical names, so a directory of any ordinary size is one or
|
||||
/// two syscalls rather than one per file.
|
||||
const BUFFER_BYTES: usize = 64 * 1024;
|
||||
|
||||
/// `FILE_ATTRIBUTE_DIRECTORY`, spelled out for the reason
|
||||
/// [`crate::platform`] spells out its own attribute constants, and
|
||||
/// const-asserted against the real header below.
|
||||
const ATTR_DIRECTORY: u32 = 0x10;
|
||||
const _: () = assert!(
|
||||
ATTR_DIRECTORY == windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_DIRECTORY
|
||||
);
|
||||
|
||||
fn wide(path: &std::path::Path) -> Vec<u16> {
|
||||
path.as_os_str().encode_wide().chain(Some(0)).collect()
|
||||
}
|
||||
|
||||
let mut pending = vec![std::path::PathBuf::from(path)];
|
||||
let mut count = 0usize;
|
||||
// One allocation for the whole walk, reused for every directory.
|
||||
let mut buffer = vec![0u8; BUFFER_BYTES];
|
||||
|
||||
while let Some(dir) = pending.pop() {
|
||||
// Checked per directory *and* per entry below: cancellation has to be
|
||||
// immediate on a tree large enough to be worth counting.
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
return Err("count cancelled".to_string());
|
||||
}
|
||||
|
||||
// FILE_FLAG_BACKUP_SEMANTICS is what makes CreateFileW open a
|
||||
// directory rather than fail; the share flags let the tree keep being
|
||||
// used while we count it.
|
||||
let handle = unsafe {
|
||||
CreateFileW(
|
||||
wide(&dir).as_ptr(),
|
||||
FILE_LIST_DIRECTORY,
|
||||
FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
|
||||
std::ptr::null(),
|
||||
OPEN_EXISTING,
|
||||
FILE_FLAG_BACKUP_SEMANTICS,
|
||||
// hTemplateFile: a HANDLE, which windows-sys spells `isize`.
|
||||
0,
|
||||
)
|
||||
};
|
||||
if handle == INVALID_HANDLE_VALUE {
|
||||
// An unreadable directory costs its own entries, not the tree's.
|
||||
continue;
|
||||
}
|
||||
|
||||
loop {
|
||||
// Returns zero both on error and at the end of the listing; either
|
||||
// way this directory is done.
|
||||
let ok = unsafe {
|
||||
GetFileInformationByHandleEx(
|
||||
handle,
|
||||
FileIdBothDirectoryInfo,
|
||||
buffer.as_mut_ptr().cast(),
|
||||
buffer.len() as u32,
|
||||
)
|
||||
};
|
||||
if ok == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
let mut offset = 0usize;
|
||||
loop {
|
||||
if cancel.load(Ordering::Relaxed) {
|
||||
unsafe { CloseHandle(handle) };
|
||||
return Err("count cancelled".to_string());
|
||||
}
|
||||
// SAFETY: the API filled `buffer` with a chain of these
|
||||
// records, each at the offset the previous one gave.
|
||||
let info =
|
||||
unsafe { &*(buffer.as_ptr().add(offset) as *const FILE_ID_BOTH_DIR_INFO) };
|
||||
|
||||
// FileNameLength is in bytes; FileName is UTF-16 and is *not*
|
||||
// NUL-terminated, so the length is the only thing that says
|
||||
// where it ends.
|
||||
let name_units = (info.FileNameLength as usize) / 2;
|
||||
let name = unsafe {
|
||||
std::slice::from_raw_parts(info.FileName.as_ptr(), name_units)
|
||||
};
|
||||
let name = OsString::from_wide(name);
|
||||
|
||||
// "." and ".." are entries of the listing, not of the tree.
|
||||
let is_dot = name == "." || name == "..";
|
||||
if !is_dot {
|
||||
count += 1;
|
||||
if info.FileAttributes & ATTR_DIRECTORY != 0 {
|
||||
pending.push(dir.join(&name));
|
||||
}
|
||||
}
|
||||
|
||||
match info.NextEntryOffset {
|
||||
0 => break,
|
||||
next => offset += next as usize,
|
||||
}
|
||||
}
|
||||
}
|
||||
unsafe { CloseHandle(handle) };
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
/// Rough tree entry count for progress totals. Setting `cancel` stops it
|
||||
/// rather than letting it scan an entire root after the run stopped: on Unix
|
||||
/// that kills the `find`/`wc` subprocesses at ~50 ms granularity, on Windows
|
||||
/// the native walk notices almost immediately. Runs concurrently with
|
||||
/// the bulk directory read notices almost immediately. Runs concurrently with
|
||||
/// indexing — its scope is not identical to the walker's classified file
|
||||
/// count.
|
||||
pub fn count_tree_entries_fast(
|
||||
|
|
@ -448,18 +611,18 @@ pub fn count_tree_entries_fast(
|
|||
) -> Result<usize, String> {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
return count_tree_entries_native(path, cancel);
|
||||
return count_tree_entries_win32(path, cancel);
|
||||
}
|
||||
#[cfg(all(unix, target_os = "linux"))]
|
||||
{
|
||||
return count_find_pipe_wc(path, cancel, true).or_else(|e| {
|
||||
count_find_pipe_wc(path, cancel, true).or_else(|e| {
|
||||
if e.contains("cancelled") {
|
||||
Err(e)
|
||||
} else {
|
||||
// Non-GNU find without -printf: plain listing.
|
||||
count_find_pipe_wc(path, cancel, false)
|
||||
}
|
||||
});
|
||||
})
|
||||
}
|
||||
#[cfg(all(unix, not(target_os = "linux")))]
|
||||
{
|
||||
|
|
@ -509,13 +672,13 @@ pub fn classify_by_mtime(stored: Option<u64>, mtime: u64) -> FileIndexAction {
|
|||
}
|
||||
}
|
||||
|
||||
/// Safely truncate a string to at most max_bytes bytes while respecting UTF-8 character boundaries
|
||||
/// Truncate to at most `max_bytes` bytes, backing up to a UTF-8 character
|
||||
/// boundary. Cuts short of the budget rather than over it.
|
||||
fn safe_truncate_string(s: &str, max_bytes: usize) -> String {
|
||||
if s.len() <= max_bytes {
|
||||
return s.to_string();
|
||||
}
|
||||
|
||||
// Find the last valid UTF-8 character boundary at or before max_bytes
|
||||
let mut end = max_bytes;
|
||||
while end > 0 && !s.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
|
|
@ -553,7 +716,7 @@ fn get_file_hash(
|
|||
f.read_exact(&mut head)?;
|
||||
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&size.to_le_bytes());
|
||||
hasher.update(size.to_le_bytes());
|
||||
hasher.update(&head);
|
||||
Ok((hasher.finalize().to_vec(), head))
|
||||
}
|
||||
|
|
@ -583,7 +746,11 @@ pub struct OwnedNewFile {
|
|||
pub device_id: Option<u64>,
|
||||
pub mime: Option<String>,
|
||||
pub ftype: FileType,
|
||||
pub hash: Vec<u8>,
|
||||
/// `None` only for a file whose contents could not be read without paying
|
||||
/// for them — a dehydrated cloud placeholder. Stored as SQL NULL, which is
|
||||
/// what keeps such files out of duplicate detection: an empty or zero hash
|
||||
/// would make every one of them look identical to every other.
|
||||
pub hash: Option<Vec<u8>>,
|
||||
/// Text extracted from the head bytes during the walk, for files small
|
||||
/// enough that the head *was* the whole file. `Some` means the content
|
||||
/// pass never has to open this file; `None` leaves it pending as before.
|
||||
|
|
@ -611,12 +778,28 @@ impl OwnedNewFile {
|
|||
device_id: self.device_id,
|
||||
mime: self.mime.as_deref(),
|
||||
ftype: self.ftype,
|
||||
hash: Some(&self.hash),
|
||||
hash: self.hash.as_deref(),
|
||||
needs_content: self.needs_content,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Individual "cannot hash" warnings allowed per run before only the count is
|
||||
/// kept. See [`crate::log::Throttle`]; [`reset_run_warnings`] arms it.
|
||||
static HASH_FAILURES: crate::log::Throttle = crate::log::Throttle::new(20);
|
||||
|
||||
/// Arm the per-run warning throttles. Called once at the start of an indexing
|
||||
/// run, so the counts a run reports describe that run and not the process.
|
||||
pub fn reset_run_warnings() {
|
||||
HASH_FAILURES.reset();
|
||||
}
|
||||
|
||||
/// How many files could not be hashed this run, and how many of those went
|
||||
/// unlogged. `(0, 0)` when nothing failed.
|
||||
pub fn hash_failure_counts() -> (u64, u64) {
|
||||
(HASH_FAILURES.seen(), HASH_FAILURES.suppressed())
|
||||
}
|
||||
|
||||
/// Build the `files` row for one on-disk file from a `stat` the caller
|
||||
/// already holds. The single implementation behind both full-run batches
|
||||
/// and incremental watcher updates.
|
||||
|
|
@ -651,12 +834,30 @@ pub fn prepare_file_record(
|
|||
.and_then(|t| t.duration_since(UNIX_EPOCH).ok())
|
||||
.map(|d| d.as_secs())?;
|
||||
|
||||
let (hash, head) = match get_file_hash(size, Path::new(path), config.processing.hash_length) {
|
||||
Ok(v) => v,
|
||||
// A dehydrated cloud file is indexed from its metadata alone. Reading even
|
||||
// the first byte would block on downloading the whole thing, so the head is
|
||||
// never fetched: no hash, no MIME sniff, no inline extraction, and the row
|
||||
// is born with nothing pending. The file is still fully searchable by name,
|
||||
// size and date — only its *contents* wait, and they wait until the user
|
||||
// hydrates the file themselves. Hydration does not change the mtime, so what
|
||||
// picks it up is this same test on a later run, once the attribute clears.
|
||||
let dehydrated = crate::platform::is_cloud_placeholder(meta);
|
||||
|
||||
let (hash, head) = if dehydrated {
|
||||
(None, Vec::new())
|
||||
} else {
|
||||
match get_file_hash(size, Path::new(path), config.processing.hash_length) {
|
||||
Ok((hash, head)) => (Some(hash), head),
|
||||
Err(e) => {
|
||||
// Throttled: on Windows a file another process holds open fails
|
||||
// here as a matter of course, and a large tree can produce
|
||||
// thousands. `run_indexing` reports the total.
|
||||
if HASH_FAILURES.allow() {
|
||||
crate::log_warn!("Skipping file (cannot hash) {}: {}", path, e);
|
||||
}
|
||||
return None;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let name = Path::new(path)
|
||||
|
|
@ -664,7 +865,9 @@ pub fn prepare_file_record(
|
|||
.map(|n| n.to_string_lossy().into_owned())?;
|
||||
let parent = parent_str(path);
|
||||
let (inode, device_id) = inode_and_device(meta);
|
||||
// Sniff from the bytes hashing already read rather than reopening.
|
||||
// Sniff from the bytes hashing already read rather than reopening. With no
|
||||
// head to sniff, the extension is all there is — which is what
|
||||
// `guess_mime_from_head` already falls back to for an empty slice.
|
||||
let mime = guess_mime_from_head(Path::new(path), &head);
|
||||
let ftype = mime.as_deref().map(mime_to_type).unwrap_or(FileType::EMPTY);
|
||||
|
||||
|
|
@ -672,7 +875,8 @@ pub fn prepare_file_record(
|
|||
// registry are all in hand — and stored as the row's `content_state`, so
|
||||
// "pending" downstream means a file that really needs reading rather than
|
||||
// one the content pass would only mark not-applicable.
|
||||
let needs_content = size <= config.processing.maximum_text_file_size
|
||||
let needs_content = !dehydrated
|
||||
&& size <= config.processing.maximum_text_file_size
|
||||
&& content_extractable(Path::new(path), mime.as_deref(), config, registry);
|
||||
|
||||
// When the head is the whole file, an extractor that works from bytes can
|
||||
|
|
@ -1209,17 +1413,7 @@ mod tests {
|
|||
use std::path::MAIN_SEPARATOR;
|
||||
|
||||
fn tmp_tree() -> std::path::PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"quicksearch-walk-{}-{}",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
std::fs::create_dir_all(&p).unwrap();
|
||||
p
|
||||
crate::testutil::scratch_dir("walk")
|
||||
}
|
||||
|
||||
fn touch(p: &Path) {
|
||||
|
|
@ -1619,18 +1813,9 @@ mod count_and_extract_tests {
|
|||
use super::*;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
/// A path that does not exist yet — these tests build the tree themselves.
|
||||
fn tmp(tag: &str) -> std::path::PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"qs-ce-{}-{}-{}",
|
||||
tag,
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
p
|
||||
crate::testutil::scratch_dir(tag).join("tree")
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -1642,8 +1827,59 @@ mod count_and_extract_tests {
|
|||
}
|
||||
let cancel = AtomicBool::new(false);
|
||||
let n = count_tree_entries_fast(root.to_str().unwrap(), &cancel).unwrap();
|
||||
// find lists the root, the subdir, and the three files.
|
||||
assert_eq!(n, 5);
|
||||
// The subdir plus the three files. Unix counts through `find`, which
|
||||
// also lists the root it was given; the Windows directory read only
|
||||
// ever sees entries *inside* a directory and so never counts the root.
|
||||
// Both are correct for a progress denominator — this is an estimate
|
||||
// running concurrently with the walk, not a total to reconcile against.
|
||||
assert_eq!(n, if cfg!(windows) { 4 } else { 5 });
|
||||
std::fs::remove_dir_all(&root).ok();
|
||||
}
|
||||
|
||||
/// The load-bearing property of the Windows fast path: it exists only to
|
||||
/// be quicker than a plain walk, so it has to agree with one exactly.
|
||||
///
|
||||
/// The tree is deliberately wider than one buffer's worth of directory
|
||||
/// data, so `GetFileInformationByHandleEx` is called repeatedly for the
|
||||
/// same directory and the resumption between calls is exercised — the part
|
||||
/// a single-buffer test would never reach.
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn the_bulk_directory_read_agrees_with_a_plain_walk() {
|
||||
let root = tmp("count-oracle");
|
||||
std::fs::create_dir_all(root.join("empty")).unwrap();
|
||||
std::fs::create_dir_all(root.join("a/b/c")).unwrap();
|
||||
// Long names so the chained records fill more than one 64 KiB buffer.
|
||||
for i in 0..600 {
|
||||
let name = format!("{}-{:04}.txt", "padding".repeat(12), i);
|
||||
std::fs::write(root.join(&name), b"x").unwrap();
|
||||
std::fs::write(root.join("a/b/c").join(&name), b"x").unwrap();
|
||||
}
|
||||
|
||||
let cancel = AtomicBool::new(false);
|
||||
let path = root.to_str().unwrap();
|
||||
let fast = count_tree_entries_win32(path, &cancel).unwrap();
|
||||
let plain = count_tree_entries_walkdir(path, &cancel).unwrap();
|
||||
assert_eq!(fast, plain, "the fast count must match a plain walk");
|
||||
// 1200 files + `empty` + `a` + `a/b` + `a/b/c`.
|
||||
assert_eq!(fast, 1204);
|
||||
|
||||
std::fs::remove_dir_all(&root).ok();
|
||||
}
|
||||
|
||||
/// Cancellation is checked per entry, not per directory, so a huge single
|
||||
/// directory stops as promptly as a deep tree.
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn the_bulk_directory_read_stops_when_cancelled() {
|
||||
let root = tmp("count-cancel");
|
||||
std::fs::create_dir_all(&root).unwrap();
|
||||
for i in 0..200 {
|
||||
std::fs::write(root.join(format!("f{:03}.txt", i)), b"x").unwrap();
|
||||
}
|
||||
let cancel = AtomicBool::new(true);
|
||||
let err = count_tree_entries_win32(root.to_str().unwrap(), &cancel).unwrap_err();
|
||||
assert!(err.contains("cancelled"), "{err}");
|
||||
std::fs::remove_dir_all(&root).ok();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -73,8 +73,14 @@ fn upsert_path(
|
|||
if meta.is_dir() {
|
||||
// A moved-in tree surfaces as one directory event; walk it with
|
||||
// the same filters as a full run.
|
||||
//
|
||||
// `filtered_walk` wants a `&str`, so a path that is not valid UTF-8
|
||||
// cannot be walked here. That is a whole subtree missing from the
|
||||
// index, not one file, so it is an error rather than a quiet `Ok`:
|
||||
// the caller answers it by scheduling a full run, which walks from
|
||||
// the configured root and never needs the path as a `str`.
|
||||
let Some(root) = path.to_str() else {
|
||||
return Ok(());
|
||||
return Err(format!("directory path is not valid UTF-8: {:?}", path));
|
||||
};
|
||||
let entries: Vec<_> = filtered_walk(
|
||||
root,
|
||||
|
|
@ -168,11 +174,17 @@ fn remove_path(conn: &mut Connection, path: &Path) -> Result<(), String> {
|
|||
/// `dir` sweeps its whole path range, each descendant event is duplicate work —
|
||||
/// 10 000 of them for a 10 000-file tree.
|
||||
///
|
||||
/// This is for a caller holding a raw removal set. The coordinator is not one:
|
||||
/// it collapses as events *arrive*, in `collapse_pending_removals`, because it
|
||||
/// has to measure its queue after collapsing rather than before (see
|
||||
/// `Inner::drain_events`). Same rule, two entry points, deliberately — a
|
||||
/// mass deletion must be one path by the time either of them is done with it.
|
||||
///
|
||||
/// Ancestor membership is tested against a set rather than by sorting, which
|
||||
/// keeps this obviously correct: no ordering argument to get wrong, and
|
||||
/// `Path::ancestors` walks whole components, so `/a/bc` is never treated as
|
||||
/// living under `/a/b` — the same rule `remove_tree` and
|
||||
/// `UnreadableDirs::covers` use. Paths are shallow, so the cost is linear.
|
||||
/// keeps this obviously correct: there is no ordering argument to get wrong.
|
||||
/// Containment is component-wise, per
|
||||
/// [`crate::file_handling::UnreadableDirs::covers`]. Paths are shallow, so the
|
||||
/// cost is linear.
|
||||
pub fn collapse_removal_roots(paths: Vec<std::path::PathBuf>) -> Vec<std::path::PathBuf> {
|
||||
if paths.len() < 2 {
|
||||
return paths;
|
||||
|
|
@ -188,11 +200,13 @@ pub fn collapse_removal_roots(paths: Vec<std::path::PathBuf>) -> Vec<std::path::
|
|||
/// Delete `paths` and everything indexed beneath them, in transactions of at
|
||||
/// most `chunk` paths.
|
||||
///
|
||||
/// A mass deletion is the case this exists for. Callers hand over the *roots*
|
||||
/// of the removal set (see [`collapse_removal_roots`]), so `rm -rf dir/` is one
|
||||
/// path here rather than one per file, and each one costs a fixed handful of
|
||||
/// range-driven statements ([`repo::delete_subtree`]) instead of a full table
|
||||
/// scan plus five statements per file.
|
||||
/// Correct for any path set, but a mass deletion is the case it exists for,
|
||||
/// and it pays off only if the caller has already reduced that set to its
|
||||
/// *roots* — then `rm -rf dir/` is one path here rather than one per file, and
|
||||
/// costs a fixed handful of range-driven statements
|
||||
/// ([`repo::delete_subtree`]) instead of a full table scan plus five
|
||||
/// statements per file. Both callers do: the coordinator collapses on arrival,
|
||||
/// and anyone else has [`collapse_removal_roots`].
|
||||
///
|
||||
/// Chunking bounds how long any single transaction holds the connection, the
|
||||
/// same way `process_batch_inserts` bounds the indexer's writes.
|
||||
|
|
@ -239,17 +253,10 @@ mod tests {
|
|||
|
||||
impl Fixture {
|
||||
fn new() -> Fixture {
|
||||
let stamp = format!(
|
||||
"{}-{}",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
);
|
||||
let dir = std::env::temp_dir().join(format!("qs-incr-{}", stamp));
|
||||
let scratch = crate::testutil::scratch_dir("incr");
|
||||
let dir = scratch.join("tree");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let db = std::env::temp_dir().join(format!("qs-incr-{}.sqlite", stamp));
|
||||
let db = scratch.join("index.sqlite");
|
||||
let conn = open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
|
||||
let config = Config::default();
|
||||
let ignore = IgnoreSet::compile(&config.indexing.ignore_patterns).unwrap();
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -3,7 +3,6 @@ pub mod config;
|
|||
pub mod content;
|
||||
pub mod coordinator;
|
||||
pub mod db;
|
||||
pub mod document_extraction;
|
||||
pub mod extract;
|
||||
pub mod file_handling;
|
||||
pub mod incremental;
|
||||
|
|
@ -17,6 +16,8 @@ pub mod search;
|
|||
pub mod security;
|
||||
pub mod shutdown;
|
||||
pub mod snippet;
|
||||
#[doc(hidden)]
|
||||
pub mod testutil;
|
||||
pub mod textenc;
|
||||
pub mod walk;
|
||||
pub mod watcher;
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@
|
|||
//! exactly the ones a user needs when something looks wrong, and they were
|
||||
//! going nowhere.
|
||||
//!
|
||||
//! So background reporting goes through [`log_info!`] and [`log_warn!`]
|
||||
//! So background reporting goes through [`crate::log_info!`] and
|
||||
//! [`crate::log_warn!`]
|
||||
//! instead of `println!`/`eprintln!`: each writes the same line to stderr
|
||||
//! *and* appends it to a bounded ring the GUI's Logs tab reads. A terminal
|
||||
//! run looks exactly as it did; a windowed run gains the tab.
|
||||
|
|
@ -66,7 +67,8 @@ macro_rules! log_warn {
|
|||
|
||||
/// Write `message` to stderr and to the ring.
|
||||
///
|
||||
/// Prefer the [`log_info!`] / [`log_warn!`] macros; this is what they call.
|
||||
/// Prefer the [`crate::log_info!`] / [`crate::log_warn!`] macros; this is what
|
||||
/// they call.
|
||||
///
|
||||
/// A failed stderr write is ignored rather than propagated: `eprintln!`
|
||||
/// *panics* when the handle is unwritable, which on a process launched
|
||||
|
|
@ -82,6 +84,56 @@ pub fn record(level: Level, message: String) {
|
|||
lock().push(level, text);
|
||||
}
|
||||
|
||||
/// A cap on how many times one *kind* of warning is allowed to speak.
|
||||
///
|
||||
/// Some failures are per-file and arrive in the thousands: a directory tree the
|
||||
/// user cannot read, or — the case that motivated this — a Windows machine where
|
||||
/// `ERROR_SHARING_VIOLATION` from a file some other process holds open is
|
||||
/// routine and has no Unix equivalent. Logging each one costs a global mutex and
|
||||
/// an unbuffered stderr write on the walk's hottest path, and it evicts the
|
||||
/// [`CAPACITY`]-line ring so thoroughly that the warnings worth reading are gone
|
||||
/// before the run ends.
|
||||
///
|
||||
/// So the first few speak and the rest are counted. The count is the part that
|
||||
/// actually informs: "3 files could not be hashed" and "31,402 files could not
|
||||
/// be hashed" call for very different responses, and neither is legible as a
|
||||
/// wall of identical lines.
|
||||
///
|
||||
/// Declared as a `static` next to the call site it guards, and reset by whoever
|
||||
/// owns the run, so the numbers describe one run rather than the process.
|
||||
pub struct Throttle {
|
||||
limit: u64,
|
||||
seen: std::sync::atomic::AtomicU64,
|
||||
}
|
||||
|
||||
impl Throttle {
|
||||
pub const fn new(limit: u64) -> Throttle {
|
||||
Throttle {
|
||||
limit,
|
||||
seen: std::sync::atomic::AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Count one occurrence, and answer whether it may be logged individually.
|
||||
pub fn allow(&self) -> bool {
|
||||
self.seen.fetch_add(1, std::sync::atomic::Ordering::Relaxed) < self.limit
|
||||
}
|
||||
|
||||
/// Occurrences counted since the last [`Throttle::reset`].
|
||||
pub fn seen(&self) -> u64 {
|
||||
self.seen.load(std::sync::atomic::Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// How many went unlogged — what a summary line should report.
|
||||
pub fn suppressed(&self) -> u64 {
|
||||
self.seen().saturating_sub(self.limit)
|
||||
}
|
||||
|
||||
pub fn reset(&self) {
|
||||
self.seen.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// Every retained line, oldest first.
|
||||
pub fn snapshot() -> Vec<LogLine> {
|
||||
lock().lines.iter().cloned().collect()
|
||||
|
|
@ -152,7 +204,17 @@ impl Ring {
|
|||
}
|
||||
}
|
||||
|
||||
fn now_unix() -> u64 {
|
||||
/// Seconds since the Unix epoch.
|
||||
///
|
||||
/// Lives here because this is the lowest-level module in the crate and every
|
||||
/// layer above it wanted the same four lines — log lines, the `last_full_index`
|
||||
/// stamp, `schema_info.created_at`, failure timestamps, and the GUI's
|
||||
/// "5 min ago".
|
||||
///
|
||||
/// A clock set before 1970 yields `0` rather than an error. Every caller is
|
||||
/// stamping a record or measuring an age, and none has anything better to do
|
||||
/// with a failure than what `0` already does: read as "very long ago".
|
||||
pub fn now_unix() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
|
|
@ -215,6 +277,33 @@ mod tests {
|
|||
assert_eq!(ring.recorded, 4, "the running total survives a clear");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_throttle_lets_the_first_few_through_and_counts_the_rest() {
|
||||
let t = Throttle::new(3);
|
||||
assert_eq!((t.seen(), t.suppressed()), (0, 0));
|
||||
for _ in 0..3 {
|
||||
assert!(t.allow(), "the first `limit` occurrences speak");
|
||||
}
|
||||
for _ in 0..7 {
|
||||
assert!(!t.allow(), "the rest are counted only");
|
||||
}
|
||||
assert_eq!(t.seen(), 10);
|
||||
assert_eq!(t.suppressed(), 7);
|
||||
|
||||
t.reset();
|
||||
assert_eq!((t.seen(), t.suppressed()), (0, 0));
|
||||
assert!(t.allow(), "a reset throttle speaks again");
|
||||
}
|
||||
|
||||
/// A zero limit must silence rather than divide by anything.
|
||||
#[test]
|
||||
fn a_zero_limit_throttle_logs_nothing() {
|
||||
let t = Throttle::new(0);
|
||||
assert!(!t.allow());
|
||||
assert_eq!(t.seen(), 1);
|
||||
assert_eq!(t.suppressed(), 1);
|
||||
}
|
||||
|
||||
/// Through the global: the macros must land in the snapshot, and a
|
||||
/// warning must carry the prefix its terminal line has. Written to
|
||||
/// tolerate lines from tests running in parallel in this process.
|
||||
|
|
|
|||
|
|
@ -9,6 +9,12 @@
|
|||
//! - Anything that can be decided from a string rather than a syscall is
|
||||
//! split out and made testable everywhere ([`is_unc_string`],
|
||||
//! [`PATH_COLLATION`]), because the test suite runs on Linux.
|
||||
//!
|
||||
//! Beyond filesystem and path semantics, this is also where the process's
|
||||
//! dealings with its own allocator and threads live — [`release_free_heap`]
|
||||
//! and [`heap_stats`], which are glibc-only, and [`spawn_worker`], which puts
|
||||
//! the run's worker stack size in one place. They are here for the same reason
|
||||
//! everything else is: they are `#[cfg]`, and `#[cfg]` lives here.
|
||||
|
||||
use std::ffi::OsString;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
|
|
@ -29,12 +35,72 @@ pub fn home_dir() -> Option<OsString> {
|
|||
std::env::var_os("HOME").or_else(|| std::env::var_os("USERPROFILE"))
|
||||
}
|
||||
|
||||
/// Whether a directory entry counts as hidden.
|
||||
/// `FILE_ATTRIBUTE_HIDDEN`.
|
||||
///
|
||||
/// Unix: a leading dot. Windows: a leading dot **or** `FILE_ATTRIBUTE_HIDDEN`
|
||||
/// / `FILE_ATTRIBUTE_SYSTEM` — without which `include_hidden = false` hides
|
||||
/// nothing on Windows, and `$RECYCLE.BIN`, `System Volume Information`,
|
||||
/// `pagefile.sys` and `AppData` all get indexed.
|
||||
/// Spelled out rather than imported, for the reason in the module header:
|
||||
/// `windows-sys` is a `cfg(windows)`-only dependency, and
|
||||
/// [`attributes_are_hidden`] has to compile — and be tested — on Linux, where
|
||||
/// the suite runs.
|
||||
const FILE_ATTRIBUTE_HIDDEN: u32 = 0x2;
|
||||
|
||||
/// The cross-compiled Windows build is where the spelling above is checked
|
||||
/// against the real header value, so a typo fails that job rather than quietly
|
||||
/// indexing `$RECYCLE.BIN`.
|
||||
#[cfg(windows)]
|
||||
const _: () = assert!(
|
||||
FILE_ATTRIBUTE_HIDDEN == windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_HIDDEN
|
||||
);
|
||||
|
||||
/// Whether a Windows attribute word marks an entry hidden.
|
||||
///
|
||||
/// `FILE_ATTRIBUTE_HIDDEN` and nothing else. `FILE_ATTRIBUTE_SYSTEM` was part
|
||||
/// of this test and was removed, because it pruned users' cloud folders:
|
||||
/// Windows honours the `desktop.ini` inside a folder only if the folder itself
|
||||
/// carries Read-only or System, so the ownCloud, Nextcloud, OneDrive and Google
|
||||
/// Drive clients set System on their sync root purely to get a branded icon.
|
||||
/// Such a folder has no Hidden bit and is plainly visible in Explorer, yet the
|
||||
/// whole subtree vanished from the index with nothing to explain it — only
|
||||
/// "Index hidden files" brought it back, which is the opposite of what that
|
||||
/// setting is for. Any folder given a custom icon is the same case.
|
||||
///
|
||||
/// Nothing the System term existed for is lost. `$RECYCLE.BIN`, `System Volume
|
||||
/// Information`, `pagefile.sys` and the legacy per-user junctions (`My
|
||||
/// Documents`, `Local Settings`, `Application Data`) are Hidden **and** System —
|
||||
/// that pairing is Windows' own definition of a protected operating system file
|
||||
/// — and `AppData` is Hidden alone. Hidden catches every one. The drive-root
|
||||
/// names are excluded by [`crate::config`]'s default ignore patterns as well, so
|
||||
/// they have two independent reasons to stay out.
|
||||
#[cfg_attr(not(windows), allow(dead_code))]
|
||||
pub(crate) fn attributes_are_hidden(attributes: u32) -> bool {
|
||||
attributes & FILE_ATTRIBUTE_HIDDEN != 0
|
||||
}
|
||||
|
||||
/// Why an entry counted as hidden.
|
||||
///
|
||||
/// The distinction exists for the walk's log line: a dot prefix explains itself
|
||||
/// and an ignore pattern is something the user typed, but "this visible folder
|
||||
/// was skipped over an attribute you cannot see" has no discoverability at all,
|
||||
/// so only that case is worth reporting.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum HiddenReason {
|
||||
DotPrefix,
|
||||
Attribute,
|
||||
}
|
||||
|
||||
/// Whether a directory entry counts as hidden, and why.
|
||||
///
|
||||
/// Unix: a leading dot. Windows: a leading dot **or** `FILE_ATTRIBUTE_HIDDEN` —
|
||||
/// without which `include_hidden = false` hides nothing on Windows, and
|
||||
/// `$RECYCLE.BIN`, `System Volume Information`, `pagefile.sys` and `AppData` all
|
||||
/// get indexed. `FILE_ATTRIBUTE_SYSTEM` is deliberately not part of it; see
|
||||
/// [`attributes_are_hidden`].
|
||||
///
|
||||
/// `meta` must report the attributes of the entry **itself**, never of a link
|
||||
/// target — callers pass `symlink_metadata` or an already-cached directory
|
||||
/// entry. A hidden symlink is a hidden alias; a visible symlink to a hidden
|
||||
/// target is a visible alias. All four call sites have to agree on that, or a
|
||||
/// full run indexes a file the watcher then refuses to update and the index
|
||||
/// churns on every cycle.
|
||||
///
|
||||
/// `meta` is a closure because on Unix it is never called: the walkers
|
||||
/// deliberately avoid `metadata()`, which would cost an extra `lstat` per
|
||||
|
|
@ -42,28 +108,207 @@ pub fn home_dir() -> Option<OsString> {
|
|||
/// zero anyway — both `std::fs::DirEntry::metadata` and
|
||||
/// `walkdir::DirEntry::metadata` hand back data already cached from
|
||||
/// `FindNextFileW`.
|
||||
pub fn entry_is_hidden<F>(name: &str, meta: F) -> bool
|
||||
pub fn entry_hidden_reason<F>(name: &str, meta: F) -> Option<HiddenReason>
|
||||
where
|
||||
F: FnOnce() -> Option<std::fs::Metadata>,
|
||||
{
|
||||
if name.starts_with('.') {
|
||||
return true;
|
||||
return Some(HiddenReason::DotPrefix);
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::fs::MetadataExt;
|
||||
use windows_sys::Win32::Storage::FileSystem::{
|
||||
FILE_ATTRIBUTE_HIDDEN, FILE_ATTRIBUTE_SYSTEM,
|
||||
};
|
||||
if let Some(m) = meta() {
|
||||
return m.file_attributes() & (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM) != 0;
|
||||
if attributes_are_hidden(m.file_attributes()) {
|
||||
return Some(HiddenReason::Attribute);
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
let _ = meta;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// [`entry_hidden_reason`] for the callers that only need the verdict.
|
||||
pub fn entry_is_hidden<F>(name: &str, meta: F) -> bool
|
||||
where
|
||||
F: FnOnce() -> Option<std::fs::Metadata>,
|
||||
{
|
||||
entry_hidden_reason(name, meta).is_some()
|
||||
}
|
||||
|
||||
/// `FILE_ATTRIBUTE_REPARSE_POINT`. Spelled out for [`FILE_ATTRIBUTE_HIDDEN`]'s
|
||||
/// reason, and checked against the real header value the same way.
|
||||
const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400;
|
||||
|
||||
#[cfg(windows)]
|
||||
const _: () = assert!(
|
||||
FILE_ATTRIBUTE_REPARSE_POINT
|
||||
== windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_REPARSE_POINT
|
||||
);
|
||||
|
||||
/// The attributes a dehydrated cloud file carries.
|
||||
///
|
||||
/// `FILE_ATTRIBUTE_OFFLINE` (0x1000) is the old tape-archive bit that OneDrive
|
||||
/// reused; `RECALL_ON_OPEN` (0x40000) marks a file whose *metadata* is local but
|
||||
/// whose data is not; `RECALL_ON_DATA_ACCESS` (0x400000) is the modern
|
||||
/// Files-On-Demand placeholder. Any one of them means opening the file for read
|
||||
/// pulls it over the network.
|
||||
const FILE_ATTRIBUTE_OFFLINE: u32 = 0x1000;
|
||||
const FILE_ATTRIBUTE_RECALL_ON_OPEN: u32 = 0x4_0000;
|
||||
const FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS: u32 = 0x40_0000;
|
||||
|
||||
#[cfg(windows)]
|
||||
const _: () = {
|
||||
use windows_sys::Win32::Storage::FileSystem as fs_attrs;
|
||||
assert!(FILE_ATTRIBUTE_OFFLINE == fs_attrs::FILE_ATTRIBUTE_OFFLINE);
|
||||
assert!(FILE_ATTRIBUTE_RECALL_ON_OPEN == fs_attrs::FILE_ATTRIBUTE_RECALL_ON_OPEN);
|
||||
assert!(FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS == fs_attrs::FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS);
|
||||
};
|
||||
|
||||
/// Whether reading this file's contents would pull it down from the cloud.
|
||||
///
|
||||
/// OneDrive, and every other Files-On-Demand provider, leaves a placeholder on
|
||||
/// disk with real metadata and no data. Opening one for read is not a local
|
||||
/// operation: it blocks on a network download of the entire file. The default
|
||||
/// indexing root is `%USERPROFILE%`, the OneDrive folder beneath it is not
|
||||
/// hidden, and the walk hashes the first 8 KiB of every new or changed file — so
|
||||
/// without this test a first index quietly downloads the user's whole cloud
|
||||
/// drive, filling their disk with the files they had deliberately offloaded.
|
||||
///
|
||||
/// Named for the question the caller is actually asking ("will reading this
|
||||
/// cost a download?") rather than for the bits, because the answer is what the
|
||||
/// indexing path branches on. Always `false` off Windows: no other platform this
|
||||
/// runs on has an equivalent, and a `stat` there tells the truth about the data.
|
||||
pub fn is_cloud_placeholder(meta: &std::fs::Metadata) -> bool {
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::fs::MetadataExt;
|
||||
return attributes_are_dehydrated(meta.file_attributes());
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
let _ = meta;
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// The bit test behind [`is_cloud_placeholder`], split out so the Linux suite
|
||||
/// exercises it. See this module's second rule.
|
||||
#[cfg_attr(not(windows), allow(dead_code))]
|
||||
pub(crate) fn attributes_are_dehydrated(attributes: u32) -> bool {
|
||||
attributes
|
||||
& (FILE_ATTRIBUTE_OFFLINE
|
||||
| FILE_ATTRIBUTE_RECALL_ON_OPEN
|
||||
| FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS)
|
||||
!= 0
|
||||
}
|
||||
|
||||
/// Whether a Windows attribute word marks an entry as a reparse point.
|
||||
///
|
||||
/// Split out from [`entry_cached_metadata`] so the bit test is exercised by the
|
||||
/// Linux suite, per this module's second rule. Deliberately *not* the same
|
||||
/// question as `FileType::is_symlink`, which additionally requires the
|
||||
/// name-surrogate bit: see [`entry_cached_metadata`] for why that distinction is
|
||||
/// the whole point.
|
||||
#[cfg_attr(not(windows), allow(dead_code))]
|
||||
pub(crate) fn attributes_are_reparse_point(attributes: u32) -> bool {
|
||||
attributes & FILE_ATTRIBUTE_REPARSE_POINT != 0
|
||||
}
|
||||
|
||||
/// Metadata a directory read already handed back, on the platforms that hand
|
||||
/// any back at all.
|
||||
///
|
||||
/// `std::fs::Metadata` on Windows: `FindFirstFileW`/`FindNextFileW` return size,
|
||||
/// timestamps and attributes with every name, and `std::fs::DirEntry::metadata`
|
||||
/// is a copy of that buffer rather than a syscall. A `fs::metadata(path)` on the
|
||||
/// same entry is therefore a whole extra `CreateFileW` +
|
||||
/// `GetFileInformationByHandle` + `CloseHandle` — an `IRP_MJ_CREATE` through
|
||||
/// every antivirus and EDR minifilter on the machine — for data already in hand.
|
||||
///
|
||||
/// Uninhabited everywhere else, because `getdents64` returns only `d_type`: size
|
||||
/// and mtime genuinely need the `statx`, and `DirEntry::metadata` there is a
|
||||
/// *second* syscall rather than a saved one. Uninhabited rather than a unit
|
||||
/// struct so `Option<CachedMetadata>` is zero-sized off Windows — the walker
|
||||
/// queues one per pending file and nothing throttles file chunks (see
|
||||
/// `walk::Job::Files`), so carrying `Option<std::fs::Metadata>` would cost Linux
|
||||
/// 176 bytes per queued file to hold nothing at all.
|
||||
#[cfg(windows)]
|
||||
pub(crate) type CachedMetadata = std::fs::Metadata;
|
||||
#[cfg(not(windows))]
|
||||
pub(crate) type CachedMetadata = std::convert::Infallible;
|
||||
|
||||
/// The zero-cost half of the claim above, enforced rather than asserted in
|
||||
/// prose: a layout change in a future toolchain becomes a compile error instead
|
||||
/// of a silent regression in the walker's memory profile.
|
||||
#[cfg(not(windows))]
|
||||
const _: () = assert!(std::mem::size_of::<Option<CachedMetadata>>() == 0);
|
||||
|
||||
/// The metadata a directory read already produced for one entry — but only
|
||||
/// where trusting it is both free *and* indistinguishable from a fresh `stat`.
|
||||
///
|
||||
/// `meta` is a closure for exactly [`entry_hidden_reason`]'s reason: on Unix it
|
||||
/// is never called, because `std::fs::DirEntry::metadata` there is a real
|
||||
/// `lstat` and the walk spends exactly one `statx` per file by design.
|
||||
///
|
||||
/// `None` for a reparse point even on Windows, and that is the whole subtlety.
|
||||
/// `fs::metadata` *follows* a reparse point; the cached buffer describes the
|
||||
/// link itself. The tags std does not classify as symlinks reach the walk's
|
||||
/// ordinary file arm — `IO_REPARSE_TAG_APPEXECLINK`, the zero-byte Store app
|
||||
/// stubs under `WindowsApps`, and the OneDrive `IO_REPARSE_TAG_CLOUD_*` family —
|
||||
/// and for those the two answers differ: today an AppExecLink fails
|
||||
/// `CreateFileW` with `ERROR_CANT_ACCESS_FILE` and is skipped, while its
|
||||
/// directory entry looks like an ordinary empty file and would get indexed.
|
||||
/// Sending every reparse point back to the path-based `fs::metadata` keeps the
|
||||
/// semantics identical and leaves only the common case on the fast path.
|
||||
///
|
||||
/// Tested against the raw `FILE_ATTRIBUTE_REPARSE_POINT` bit, deliberately not
|
||||
/// against `file_type().is_symlink()`: std's `is_symlink` additionally requires
|
||||
/// the name-surrogate bit `0x20000000`, which is precisely the test that lets
|
||||
/// AppExecLink and the cloud tags through.
|
||||
pub(crate) fn entry_cached_metadata<F>(meta: F) -> Option<CachedMetadata>
|
||||
where
|
||||
F: FnOnce() -> Option<std::fs::Metadata>,
|
||||
{
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::fs::MetadataExt;
|
||||
if let Some(m) = meta().filter(|m| !attributes_are_reparse_point(m.file_attributes())) {
|
||||
return Some(m);
|
||||
}
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
let _ = meta;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// A file's `std::fs::Metadata`, from the directory read where that read
|
||||
/// supplied it and from a `stat` where it did not.
|
||||
///
|
||||
/// The `#[cfg]` lives here rather than at the call site, per this module's first
|
||||
/// rule. Off Windows `cached` is uninhabited and therefore provably `None`,
|
||||
/// which is why the walk still costs exactly one `statx` per file there.
|
||||
#[cfg(windows)]
|
||||
pub(crate) fn metadata_or_stat(
|
||||
path: &Path,
|
||||
cached: Option<CachedMetadata>,
|
||||
) -> std::io::Result<std::fs::Metadata> {
|
||||
match cached {
|
||||
Some(m) => Ok(m),
|
||||
None => std::fs::metadata(path),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
pub(crate) fn metadata_or_stat(
|
||||
path: &Path,
|
||||
_cached: Option<CachedMetadata>,
|
||||
) -> std::io::Result<std::fs::Metadata> {
|
||||
std::fs::metadata(path)
|
||||
}
|
||||
|
||||
/// Whether `path` has a hidden component *below* the root that contains it.
|
||||
|
|
@ -101,7 +346,11 @@ pub fn path_has_hidden_component_under(path: &Path, roots: &[PathBuf]) -> bool {
|
|||
current.push(component);
|
||||
if let Component::Normal(name) = component {
|
||||
let name = name.to_string_lossy();
|
||||
if entry_is_hidden(&name, || std::fs::metadata(¤t).ok()) {
|
||||
// `symlink_metadata`, not `metadata`: each component is judged as
|
||||
// itself, which is what the walkers do. Only the final component
|
||||
// stops being followed, so intermediate ones still resolve
|
||||
// normally. See `entry_hidden_reason`.
|
||||
if entry_is_hidden(&name, || std::fs::symlink_metadata(¤t).ok()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
@ -234,6 +483,15 @@ pub const WATCH_ROOTS_RECURSIVELY: bool = cfg!(windows);
|
|||
/// on both sides, consistently.
|
||||
pub const PATH_COLLATION: &str = if cfg!(windows) { "NOCASE" } else { "BINARY" };
|
||||
|
||||
/// Whether this platform's filesystem matches names without regard to case.
|
||||
///
|
||||
/// What ignore patterns compile against: on Windows and macOS `node_modules`
|
||||
/// has to exclude `Node_Modules`, and on Linux it must not. Named here rather
|
||||
/// than spelled `cfg!(any(windows, target_os = "macos"))` at each use, so the
|
||||
/// two places that must agree — [`crate::config::IgnoreSet`]'s literal set and
|
||||
/// its glob set — cannot drift apart.
|
||||
pub const PATHS_ARE_CASE_INSENSITIVE: bool = cfg!(any(windows, target_os = "macos"));
|
||||
|
||||
/// Drop the **calling thread** to background scheduling priority.
|
||||
///
|
||||
/// Per-thread, not per-process. The GUI shares this process, so lowering the
|
||||
|
|
@ -245,6 +503,19 @@ pub const PATH_COLLATION: &str = if cfg!(windows) { "NOCASE" } else { "BINARY" }
|
|||
///
|
||||
/// Best-effort and idempotent: a refusal is not worth reporting, since the
|
||||
/// only consequence is that indexing competes on equal terms.
|
||||
///
|
||||
/// **CPU only, on every platform.** This used to be
|
||||
/// `THREAD_MODE_BACKGROUND_BEGIN` on Windows, which is background *mode*: it
|
||||
/// lowers the thread to base priority 4 and drops it to `IoPriorityVeryLow`, a
|
||||
/// tier the kernel does not merely deprioritise but actively rate-limits — the
|
||||
/// same one SuperFetch and defrag run in. Linux's `nice` has no I/O half at all
|
||||
/// under the usual schedulers, so the two platforms were not doing remotely the
|
||||
/// same thing: every walker, prefetcher, extractor *and* the single SQLite
|
||||
/// writer thread were throttled on Windows and full-speed on Linux, which is
|
||||
/// most of why Windows indexing was so much slower. Matching `nice`'s CPU-only
|
||||
/// semantics is the deliberate choice; a machine that genuinely needs I/O
|
||||
/// throttling needs it as a user-visible setting, not as a silent per-platform
|
||||
/// difference.
|
||||
pub fn set_background_priority() {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
|
|
@ -256,12 +527,12 @@ pub fn set_background_priority() {
|
|||
#[cfg(windows)]
|
||||
{
|
||||
use windows_sys::Win32::System::Threading::{
|
||||
GetCurrentThread, SetThreadPriority, THREAD_MODE_BACKGROUND_BEGIN,
|
||||
GetCurrentThread, SetThreadPriority, THREAD_PRIORITY_BELOW_NORMAL,
|
||||
};
|
||||
// Background *mode*, not merely a lower priority number: it drops I/O
|
||||
// priority as well, which is what actually keeps a walk from starving
|
||||
// the foreground on a spinning disk.
|
||||
unsafe { SetThreadPriority(GetCurrentThread(), THREAD_MODE_BACKGROUND_BEGIN) };
|
||||
// Yields the CPU to the foreground without touching I/O priority — the
|
||||
// closest Windows equivalent of `nice(10)`. See the note above for why
|
||||
// this is not `THREAD_MODE_BACKGROUND_BEGIN`.
|
||||
unsafe { SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_BELOW_NORMAL) };
|
||||
}
|
||||
// Elsewhere (macOS, BSD): deliberately nothing. `nice` there applies to the
|
||||
// whole process, so it would hit the GUI. The right call is
|
||||
|
|
@ -269,6 +540,106 @@ pub fn set_background_priority() {
|
|||
// adding on its own terms rather than approximating here.
|
||||
}
|
||||
|
||||
/// Stack size for the per-run worker threads spawned by [`spawn_worker`].
|
||||
///
|
||||
/// Rust's default is 8 MiB of *reserved* address space, of which only touched
|
||||
/// pages become resident — so on its own the default costs nothing much. What
|
||||
/// costs is that glibc **caches freed thread stacks** rather than unmapping
|
||||
/// them (`stack_cache_maxsize`, 40 MiB by default) and a cached stack keeps
|
||||
/// its dirty pages. A run spawns walk and extraction workers per root, so the
|
||||
/// pages those threads touched outlive them and sit in that cache for the life
|
||||
/// of the process.
|
||||
///
|
||||
/// 512 KiB is ample for what these threads actually do: bounded loops over a
|
||||
/// directory's entries and over a batch of rows. The one thing here that
|
||||
/// recurses on untrusted input is document parsing — OLE2 compound files and
|
||||
/// zip/XML containers — and that runs on extraction workers, so if a malformed
|
||||
/// document ever overflows this it should be raised rather than reverted, and
|
||||
/// the parser given a depth limit.
|
||||
const WORKER_STACK_SIZE: usize = 512 * 1024;
|
||||
|
||||
/// Spawn one of a run's short-lived worker threads.
|
||||
///
|
||||
/// Exists to put [`WORKER_STACK_SIZE`] in one place rather than at each of the
|
||||
/// four spawn sites, and to name the threads while it is at it — `qs-walk`,
|
||||
/// `qs-extract` and friends show up in `top -H` and in a debugger, which the
|
||||
/// anonymous `thread::spawn` versions did not.
|
||||
///
|
||||
/// Panics if the thread cannot be spawned, exactly as `thread::spawn` does: a
|
||||
/// run that cannot start its workers has nothing to fall back to.
|
||||
pub fn spawn_worker<F, T>(name: &str, f: F) -> std::thread::JoinHandle<T>
|
||||
where
|
||||
F: FnOnce() -> T + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
std::thread::Builder::new()
|
||||
.name(name.to_string())
|
||||
.stack_size(WORKER_STACK_SIZE)
|
||||
.spawn(f)
|
||||
.expect("spawn worker thread")
|
||||
}
|
||||
|
||||
/// Return free heap pages to the kernel.
|
||||
///
|
||||
/// glibc's `free` returns a chunk to its arena's free list, not to the OS —
|
||||
/// only the top of an arena is ever trimmed, and only past `M_TRIM_THRESHOLD`.
|
||||
/// So a transient peak stays in RSS for the life of the process even though
|
||||
/// nothing is using it. Indexing peaks around 200 MiB above baseline on a
|
||||
/// large root (see `examples/memprobe.rs`), and a full-table scan fills a
|
||||
/// connection's SQLite page cache with 4 KiB allocations that are individually
|
||||
/// far below the mmap threshold — both land in an arena and stay there. This
|
||||
/// is what gives them back.
|
||||
///
|
||||
/// Process-wide despite being one call: `malloc_trim(0)` walks *every* arena,
|
||||
/// so calling it on the coordinator's thread also reclaims what the search
|
||||
/// worker and the finished indexing threads left behind. That is why there are
|
||||
/// only two call sites rather than one per subsystem — and why it must not go
|
||||
/// anywhere hot, since the walk plus the `madvise` per free page costs
|
||||
/// milliseconds on a large heap.
|
||||
///
|
||||
/// Best-effort and idempotent, like [`set_background_priority`]: a refusal
|
||||
/// means only that the memory stays where it was.
|
||||
pub fn release_free_heap() {
|
||||
#[cfg(all(target_os = "linux", target_env = "gnu"))]
|
||||
{
|
||||
// SAFETY: no arguments, no pointers, and safe to call from any thread
|
||||
// at any time — glibc takes the arena locks itself.
|
||||
unsafe { libc::malloc_trim(0) };
|
||||
}
|
||||
// Elsewhere: deliberately nothing. `malloc_trim` is a glibc extension —
|
||||
// musl has no equivalent and does not need one (its allocator returns
|
||||
// spans to the kernel on free), and the Windows CRT heap has
|
||||
// `_heapmin`, which is worth adding on its own terms if Windows RSS ever
|
||||
// proves to be a problem rather than assuming it behaves like glibc.
|
||||
}
|
||||
|
||||
/// Live and free-but-retained heap bytes, as `(in_use, free)`.
|
||||
///
|
||||
/// The gap between the two *is* the retention this module's
|
||||
/// [`release_free_heap`] exists to close: `in_use` is memory something still
|
||||
/// holds, `free` is memory the program has already given back to the allocator
|
||||
/// and which glibc is nonetheless still charging the process for. A large
|
||||
/// `free` immediately after a trim is the signal that `malloc_trim` cannot
|
||||
/// reach the fragmentation and a different allocator is the answer.
|
||||
///
|
||||
/// `None` where the platform has no way to answer.
|
||||
pub fn heap_stats() -> Option<(u64, u64)> {
|
||||
#[cfg(all(target_os = "linux", target_env = "gnu"))]
|
||||
{
|
||||
// `mallinfo2`, not `mallinfo`: the older struct is `int`-typed and
|
||||
// silently wraps past 2 GiB, which is exactly the size where the
|
||||
// answer starts to matter.
|
||||
//
|
||||
// SAFETY: no arguments, returns a plain struct by value.
|
||||
let info = unsafe { libc::mallinfo2() };
|
||||
Some((info.uordblks as u64, info.fordblks as u64))
|
||||
}
|
||||
#[cfg(not(all(target_os = "linux", target_env = "gnu")))]
|
||||
{
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// How long to keep retrying a delete that fails because something else holds
|
||||
/// the file open.
|
||||
#[cfg(windows)]
|
||||
|
|
@ -426,12 +797,165 @@ mod tests {
|
|||
assert!(!called, "a dot prefix must short-circuit before any stat");
|
||||
}
|
||||
|
||||
/// Each of the three attributes on its own must be enough: providers do not
|
||||
/// agree on which they set, and getting this wrong means silently
|
||||
/// downloading someone's entire cloud drive.
|
||||
#[test]
|
||||
fn any_recall_attribute_marks_a_file_dehydrated() {
|
||||
for bit in [OFFLINE, RECALL_ON_OPEN, RECALL_ON_DATA_ACCESS] {
|
||||
assert!(attributes_are_dehydrated(bit));
|
||||
assert!(attributes_are_dehydrated(ARCHIVE | REPARSE_POINT | bit));
|
||||
}
|
||||
// A synced-down file keeps the reparse point but drops the recall bits.
|
||||
assert!(!attributes_are_dehydrated(ARCHIVE | REPARSE_POINT));
|
||||
assert!(!attributes_are_dehydrated(ARCHIVE));
|
||||
assert!(!attributes_are_dehydrated(NORMAL));
|
||||
assert!(!attributes_are_dehydrated(0));
|
||||
}
|
||||
|
||||
/// Off Windows there is no such thing, and a local `stat` tells the truth.
|
||||
#[test]
|
||||
#[cfg(not(windows))]
|
||||
fn nothing_is_a_cloud_placeholder_off_windows() {
|
||||
let meta = std::fs::metadata(env!("CARGO_MANIFEST_DIR")).unwrap();
|
||||
assert!(!is_cloud_placeholder(&meta));
|
||||
}
|
||||
|
||||
/// The bit test behind the walk's fast path, exercised where the suite
|
||||
/// actually runs. A junction, an AppExecLink stub and a OneDrive
|
||||
/// placeholder all carry this bit; an ordinary file does not.
|
||||
#[test]
|
||||
fn reparse_points_are_recognised_by_attribute() {
|
||||
assert!(attributes_are_reparse_point(REPARSE_POINT));
|
||||
assert!(attributes_are_reparse_point(DIRECTORY | REPARSE_POINT));
|
||||
// A dehydrated cloud file: reparse point plus the recall attributes.
|
||||
assert!(attributes_are_reparse_point(
|
||||
ARCHIVE | REPARSE_POINT | 0x40_0000
|
||||
));
|
||||
assert!(!attributes_are_reparse_point(ARCHIVE));
|
||||
assert!(!attributes_are_reparse_point(NORMAL));
|
||||
assert!(!attributes_are_reparse_point(DIRECTORY));
|
||||
assert!(!attributes_are_reparse_point(0));
|
||||
}
|
||||
|
||||
/// Off Windows the directory read supplies nothing, and asking it for
|
||||
/// anything would be the `lstat` per entry the walker exists to avoid.
|
||||
#[test]
|
||||
#[cfg(not(windows))]
|
||||
fn nothing_is_served_from_a_directory_read_off_windows() {
|
||||
let mut called = false;
|
||||
let got = entry_cached_metadata(|| {
|
||||
called = true;
|
||||
None
|
||||
});
|
||||
assert!(got.is_none());
|
||||
assert!(
|
||||
!called,
|
||||
"DirEntry::metadata here is an lstat, which is the syscall the walk exists to avoid"
|
||||
);
|
||||
}
|
||||
|
||||
/// `fs::metadata` follows a reparse point and the cached buffer does not,
|
||||
/// so the fast path must decline every one of them — including the tags std
|
||||
/// does not call symlinks, which are precisely the ones that reach the
|
||||
/// walk's ordinary file arm.
|
||||
#[test]
|
||||
#[cfg(windows)]
|
||||
fn a_reparse_point_is_never_served_from_the_directory_read() {
|
||||
let dir = std::env::temp_dir().join(format!("qs-reparse-{}", std::process::id()));
|
||||
let target = dir.join("target");
|
||||
let link = dir.join("link");
|
||||
let plain = dir.join("plain.txt");
|
||||
std::fs::create_dir_all(&target).unwrap();
|
||||
std::fs::write(&plain, b"x").unwrap();
|
||||
|
||||
let made = std::process::Command::new("cmd")
|
||||
.args(["/C", "mklink", "/J"])
|
||||
.arg(&link)
|
||||
.arg(&target)
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false);
|
||||
if made {
|
||||
let m = std::fs::symlink_metadata(&link).unwrap();
|
||||
assert!(
|
||||
entry_cached_metadata(|| Some(m)).is_none(),
|
||||
"a junction must fall back to the path-based stat"
|
||||
);
|
||||
}
|
||||
|
||||
let m = std::fs::metadata(&plain).unwrap();
|
||||
assert!(
|
||||
entry_cached_metadata(|| Some(m)).is_some(),
|
||||
"an ordinary file must take the fast path"
|
||||
);
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_names_are_not_hidden() {
|
||||
assert!(!entry_is_hidden("Documents", || None));
|
||||
assert!(!entry_is_hidden("report.txt", || None));
|
||||
}
|
||||
|
||||
/// The real `FILE_ATTRIBUTE_*` bits, so the cases below read as the files
|
||||
/// they stand for.
|
||||
const READONLY: u32 = 0x1;
|
||||
const HIDDEN: u32 = 0x2;
|
||||
const SYSTEM: u32 = 0x4;
|
||||
const DIRECTORY: u32 = 0x10;
|
||||
const ARCHIVE: u32 = 0x20;
|
||||
const NORMAL: u32 = 0x80;
|
||||
const REPARSE_POINT: u32 = 0x400;
|
||||
const OFFLINE: u32 = 0x1000;
|
||||
const RECALL_ON_OPEN: u32 = 0x4_0000;
|
||||
const RECALL_ON_DATA_ACCESS: u32 = 0x40_0000;
|
||||
|
||||
/// The attribute half of `entry_is_hidden`, which the Windows arm cannot
|
||||
/// be asked about from Linux. Split out precisely so this test runs
|
||||
/// everywhere.
|
||||
#[test]
|
||||
fn only_the_hidden_bit_hides_an_entry() {
|
||||
// AppData: Hidden alone, and `std::env::temp_dir()` lives under it.
|
||||
assert!(attributes_are_hidden(HIDDEN | DIRECTORY));
|
||||
// $RECYCLE.BIN, System Volume Information, and the legacy per-user
|
||||
// junctions: Hidden+System, Windows' own definition of a protected
|
||||
// operating system file.
|
||||
assert!(attributes_are_hidden(HIDDEN | SYSTEM | DIRECTORY));
|
||||
// pagefile.sys.
|
||||
assert!(attributes_are_hidden(HIDDEN | SYSTEM | ARCHIVE));
|
||||
|
||||
assert!(!attributes_are_hidden(0));
|
||||
assert!(!attributes_are_hidden(NORMAL));
|
||||
assert!(!attributes_are_hidden(DIRECTORY));
|
||||
assert!(!attributes_are_hidden(READONLY | DIRECTORY));
|
||||
}
|
||||
|
||||
/// Regression: a cloud sync root carries System and *not* Hidden — Windows
|
||||
/// will not honour the `desktop.ini` supplying its branded icon otherwise —
|
||||
/// and is fully visible in Explorer. Keying on System pruned the folder and
|
||||
/// every file beneath it, silently, and only "include hidden files" brought
|
||||
/// it back.
|
||||
#[test]
|
||||
fn a_sync_root_marked_system_but_not_hidden_is_not_hidden() {
|
||||
assert!(!attributes_are_hidden(SYSTEM | DIRECTORY));
|
||||
// Read-only is the other attribute that enables desktop.ini, and
|
||||
// Explorer sets it when a user picks a custom folder icon.
|
||||
assert!(!attributes_are_hidden(READONLY | SYSTEM | DIRECTORY));
|
||||
assert!(!attributes_are_hidden(SYSTEM));
|
||||
}
|
||||
|
||||
/// The walk announces an attribute prune and stays quiet about a dot
|
||||
/// prefix, so the two must stay distinguishable.
|
||||
#[test]
|
||||
fn a_dot_prefix_reports_itself_as_the_reason() {
|
||||
assert_eq!(
|
||||
entry_hidden_reason(".git", || None),
|
||||
Some(HiddenReason::DotPrefix)
|
||||
);
|
||||
assert_eq!(entry_hidden_reason("Documents", || None), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hidden_components_are_measured_from_the_innermost_root() {
|
||||
let root = PathBuf::from(format!("{}.config", sep_prefix()));
|
||||
|
|
|
|||
|
|
@ -236,7 +236,8 @@ impl TermPattern {
|
|||
}
|
||||
}
|
||||
|
||||
/// Case-insensitive [`find_first`] against an already-folded haystack.
|
||||
/// Case-insensitive [`TermPattern::find_first`] against an already-folded
|
||||
/// haystack.
|
||||
///
|
||||
/// The literal path would otherwise fold the haystack itself, and the
|
||||
/// cascade's full-text passes need the same fold for counting, searching
|
||||
|
|
@ -256,8 +257,8 @@ impl TermPattern {
|
|||
}
|
||||
}
|
||||
|
||||
/// Case-insensitive [`count`] against an already-folded haystack. See
|
||||
/// [`TermPattern::find_first_folded`].
|
||||
/// Case-insensitive [`TermPattern::count`] against an already-folded
|
||||
/// haystack. See [`TermPattern::find_first_folded`].
|
||||
pub fn count_folded(&self, folded: &str) -> usize {
|
||||
match self {
|
||||
TermPattern::Empty => 0,
|
||||
|
|
|
|||
|
|
@ -28,8 +28,25 @@
|
|||
//! indexed, has no owning root, and so has no filtering rules that can be
|
||||
//! applied to it — scanning by range means it is simply never visited, the
|
||||
//! same exemption `aliased_paths` gives it during a full run's stale sweep.
|
||||
//!
|
||||
//! ## Reporting and giving up
|
||||
//!
|
||||
//! On a multi-million-row index this pass is minutes of work with no files
|
||||
//! moving to show for it, so the cursor counts what it has examined against
|
||||
//! the rows in the index and its driver publishes that
|
||||
//! ([`crate::indexing::ReconcileProgress`]).
|
||||
//!
|
||||
//! It can also be abandoned, through the flag-plus-interrupt pair
|
||||
//! [`crate::db::InterruptSlot`] describes — before this, closing the window
|
||||
//! during a prune waited the prune out. What makes giving up safe is that
|
||||
//! nothing here records anything: the stored configuration is stamped by the
|
||||
//! *caller*, only once the cursor reports itself finished, so an abandoned
|
||||
//! pass leaves the index describing the settings it was last reconciled to and
|
||||
//! the next run derives the same plan again. See [`outstanding_work`], which
|
||||
//! is that question asked directly.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use rusqlite::Connection;
|
||||
|
|
@ -38,12 +55,17 @@ use crate::config::{Config, IgnoreSet, IndexWork};
|
|||
use crate::db::repo;
|
||||
use crate::extract::Registry;
|
||||
use crate::file_handling::{content_extractable, fts_finalize_after_text_indexing, ExtractCursor};
|
||||
use crate::indexing::ReconcileProgress;
|
||||
|
||||
/// How long [`advance`] may work before handing control back.
|
||||
///
|
||||
/// Not a throughput knob — the caller decides when to come back — but a bound
|
||||
/// on how long a Stop, a search or a further config edit waits behind a scan
|
||||
/// in progress. The same budget the coordinator gives its watcher queue.
|
||||
///
|
||||
/// It bounds the wait *between* statements only; one statement can outlast the
|
||||
/// whole budget, which is why the pass also publishes its connection through
|
||||
/// [`crate::db::InterruptGuard`]. See [`crate::db::InterruptSlot`].
|
||||
pub const SLICE: Duration = Duration::from_millis(250);
|
||||
|
||||
/// One configured root, with the `files.path` range it owns precomputed.
|
||||
|
|
@ -89,9 +111,8 @@ impl Scope {
|
|||
})
|
||||
}
|
||||
|
||||
/// The configured root `path` lives under, if any. `Path::starts_with`
|
||||
/// compares whole components, so `/a/bc` is never read as living under
|
||||
/// `/a/b`.
|
||||
/// The configured root `path` lives under, if any. Containment is
|
||||
/// component-wise, per [`crate::file_handling::UnreadableDirs::covers`].
|
||||
pub fn owning_root(&self, path: &Path) -> Option<&Path> {
|
||||
self.roots
|
||||
.iter()
|
||||
|
|
@ -107,8 +128,23 @@ impl Scope {
|
|||
/// which is exactly the union of the per-level tests the walker performs
|
||||
/// on its way down. The hidden and component-pattern rules are tested per
|
||||
/// component *below* the root: a root is never filtered, because the user
|
||||
/// chose it (see `walk_parallel`).
|
||||
/// chose it (see [`crate::walk::walk_indexable_files`]).
|
||||
pub fn covers(&self, root: &Path, path: &Path) -> bool {
|
||||
self.covers_cached(root, path, &mut CoverCache::default())
|
||||
}
|
||||
|
||||
/// [`Scope::covers`], reusing the verdicts already reached for the
|
||||
/// directories on the way down.
|
||||
///
|
||||
/// A scan calls this once per stored row, and rows under one root share
|
||||
/// their ancestors densely — every file in `~/src/project/src` asks the same
|
||||
/// four questions before it asks its own. On Unix that repetition is free,
|
||||
/// because `entry_is_hidden` never calls the closure and the whole loop is
|
||||
/// string comparisons. On Windows each component is a `symlink_metadata`,
|
||||
/// which is a `CreateFileW` through the full filter-driver stack: a 3M-row
|
||||
/// index at depth 8 was ~24M file opens where Linux did none. Remembering
|
||||
/// each directory's verdict collapses that to roughly one per directory.
|
||||
pub fn covers_cached(&self, root: &Path, path: &Path, cache: &mut CoverCache) -> bool {
|
||||
if self.ignore.matches_path_pattern(path) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -116,28 +152,85 @@ impl Scope {
|
|||
return false;
|
||||
};
|
||||
let mut current = root.to_path_buf();
|
||||
for component in relative.components() {
|
||||
let depth = relative.components().count();
|
||||
for (i, component) in relative.components().enumerate() {
|
||||
let std::path::Component::Normal(name) = component else {
|
||||
// Stored paths are canonical, so stripping a canonical root
|
||||
// leaves plain names. Anything else did not come from a walk.
|
||||
return false;
|
||||
};
|
||||
current.push(name);
|
||||
let name = name.to_string_lossy();
|
||||
// The metadata closure is only consulted on Windows, where hidden
|
||||
// is an attribute rather than a leading dot; on Unix this stays at
|
||||
// zero syscalls, exactly as it does in the walker.
|
||||
if !self.include_hidden
|
||||
&& crate::platform::entry_is_hidden(&name, || std::fs::metadata(¤t).ok())
|
||||
{
|
||||
// Only the ancestors are worth remembering. The last component is
|
||||
// this row's own file name, asked once and never again, so caching
|
||||
// it would grow the map by one entry per row for no hits at all.
|
||||
let is_leaf = i + 1 == depth;
|
||||
if !is_leaf {
|
||||
if let Some(allowed) = cache.get(¤t) {
|
||||
if !allowed {
|
||||
return false;
|
||||
}
|
||||
if self.ignore.matches_component(&name) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let allowed = self.component_allowed(¤t, &name.to_string_lossy());
|
||||
if !is_leaf {
|
||||
cache.insert(current.clone(), allowed);
|
||||
}
|
||||
if !allowed {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Whether one path component passes the hidden and component-pattern
|
||||
/// rules. `current` is its full path, which the attribute test needs.
|
||||
fn component_allowed(&self, current: &Path, name: &str) -> bool {
|
||||
// The metadata closure is only consulted on Windows, where hidden
|
||||
// is an attribute rather than a leading dot; on Unix this stays at
|
||||
// zero syscalls, exactly as it does in the walker.
|
||||
//
|
||||
// `symlink_metadata` for the same reason the walker uses
|
||||
// `DirEntry::metadata`: the component is judged as itself, never as
|
||||
// what it points at. Following here and not there is exactly the
|
||||
// disagreement this whole type exists to avoid.
|
||||
if !self.include_hidden
|
||||
&& crate::platform::entry_is_hidden(name, || std::fs::symlink_metadata(current).ok())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
!self.ignore.matches_component(name)
|
||||
}
|
||||
}
|
||||
|
||||
/// Directory verdicts already reached by [`Scope::covers_cached`].
|
||||
///
|
||||
/// Bounded rather than unbounded: a scan of a multi-million-row index would
|
||||
/// otherwise hold every directory under every root at once, and the pass is
|
||||
/// already paged precisely so it does not have to. Past the cap the map is
|
||||
/// cleared outright instead of evicting one entry — rows arrive in roughly
|
||||
/// insertion order, so the entries that matter are the ones just added, and a
|
||||
/// clear costs one rebuild of a working set that is small by construction.
|
||||
#[derive(Default)]
|
||||
pub struct CoverCache {
|
||||
dirs: std::collections::HashMap<PathBuf, bool>,
|
||||
}
|
||||
|
||||
impl CoverCache {
|
||||
/// Directories remembered before the map is cleared. Roughly 100 bytes per
|
||||
/// entry, so this is a few megabytes at most.
|
||||
const CAP: usize = 20_000;
|
||||
|
||||
fn get(&self, dir: &Path) -> Option<bool> {
|
||||
self.dirs.get(dir).copied()
|
||||
}
|
||||
|
||||
fn insert(&mut self, dir: PathBuf, allowed: bool) {
|
||||
if self.dirs.len() >= Self::CAP {
|
||||
self.dirs.clear();
|
||||
}
|
||||
self.dirs.insert(dir, allowed);
|
||||
}
|
||||
}
|
||||
|
||||
/// How far an in-progress [`advance`] has got.
|
||||
|
|
@ -146,6 +239,13 @@ impl Scope {
|
|||
/// coordinator's command loop for the length of a full scan; the caller hands
|
||||
/// back the same cursor each tick with a fresh deadline, the way
|
||||
/// `apply_pending` drains the watcher queue.
|
||||
///
|
||||
/// Resumable within one pass only. A cursor abandoned — cancelled, or dropped
|
||||
/// on an error — takes its position with it, and the next attempt starts from
|
||||
/// the beginning of a freshly derived plan; every part of the pass is
|
||||
/// idempotent precisely so that this costs time and nothing else. A config edit
|
||||
/// arriving mid-pass has the same effect (see `Inner::start_work`), which is
|
||||
/// why the counters can go backwards between two published snapshots.
|
||||
pub struct WorkCursor {
|
||||
work: IndexWork,
|
||||
scope: Scope,
|
||||
|
|
@ -165,6 +265,15 @@ pub struct WorkCursor {
|
|||
pub deleted: usize,
|
||||
/// Rows whose content state or stored text was re-decided.
|
||||
pub recontented: usize,
|
||||
/// Rows the scan has re-tested against the current configuration.
|
||||
examined: usize,
|
||||
/// Rows in the index, counted once when the scan first needs a page and
|
||||
/// then left alone — a denominator that moved would make the display walk
|
||||
/// backwards. `None` until then, and for a plan that reads no rows at all:
|
||||
/// removing a root is one whole-range delete with no intermediate state to
|
||||
/// report, so it gets the indeterminate bar rather than an invented
|
||||
/// percentage.
|
||||
total: Option<usize>,
|
||||
}
|
||||
|
||||
impl WorkCursor {
|
||||
|
|
@ -179,6 +288,8 @@ impl WorkCursor {
|
|||
finalized: false,
|
||||
deleted: 0,
|
||||
recontented: 0,
|
||||
examined: 0,
|
||||
total: None,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -186,6 +297,20 @@ impl WorkCursor {
|
|||
self.finalized
|
||||
}
|
||||
|
||||
/// A snapshot for the status the caller publishes.
|
||||
///
|
||||
/// Both places this work runs — a run's prologue and the coordinator's
|
||||
/// between-runs pass — report the same figures from the same counters,
|
||||
/// so the user sees one thing however the reconcile was reached.
|
||||
pub fn progress(&self) -> ReconcileProgress {
|
||||
ReconcileProgress {
|
||||
examined: self.examined,
|
||||
total: self.total,
|
||||
deleted: self.deleted,
|
||||
recontented: self.recontented,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a full walk must follow this reconciliation.
|
||||
pub fn reindex(&self) -> bool {
|
||||
self.work.reindex
|
||||
|
|
@ -207,16 +332,31 @@ impl WorkCursor {
|
|||
/// Apply as much of `cursor` as fits before `deadline`, one page of rows per
|
||||
/// transaction. Returns with the cursor advanced; call again until
|
||||
/// [`WorkCursor::done`].
|
||||
///
|
||||
/// `cancel` means "do not start another statement": it is read before every
|
||||
/// one, and a set flag returns immediately with the cursor un-finished. It is
|
||||
/// only half of cutting the pass short — the statement already running answers
|
||||
/// to [`crate::db::interrupt`] and nothing else — and it is deliberately not
|
||||
/// the same thing as `deadline`, which hands control back to a caller that
|
||||
/// intends to come straight back.
|
||||
///
|
||||
/// A cancelled pass leaves the work owed. Nothing here records what it did:
|
||||
/// the stored configuration is stamped only by a caller that saw the cursor
|
||||
/// finish, so the next run re-derives the same plan and picks it up.
|
||||
pub fn advance(
|
||||
conn: &mut Connection,
|
||||
config: &Config,
|
||||
registry: &Registry,
|
||||
cursor: &mut WorkCursor,
|
||||
deadline: Instant,
|
||||
cancel: &AtomicBool,
|
||||
) -> Result<(), String> {
|
||||
// Whole ranges first: a removed root's rows can never satisfy the scan's
|
||||
// filters anyway, and deleting them by range spares the scan the work.
|
||||
while cursor.drop_idx < cursor.work.drop_roots.len() {
|
||||
if cancelled(cancel) {
|
||||
return Ok(());
|
||||
}
|
||||
let range = ExtractCursor::for_root(&cursor.work.drop_roots[cursor.drop_idx]);
|
||||
let tx = conn
|
||||
.transaction()
|
||||
|
|
@ -231,6 +371,10 @@ pub fn advance(
|
|||
}
|
||||
}
|
||||
|
||||
if cancelled(cancel) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Before the per-root scan and after the root deletions: the ranges it
|
||||
// spares must already be the final set of roots.
|
||||
if !cursor.dropped_aliases && cursor.work.drop_aliases {
|
||||
|
|
@ -254,8 +398,24 @@ pub fn advance(
|
|||
}
|
||||
|
||||
if cursor.work.scans_rows() {
|
||||
// One count, the first time a page is actually needed. The scan it
|
||||
// measures is a keyset walk of every row under every root, so on the
|
||||
// indexes where this matters it is minutes of work against a count of
|
||||
// seconds — and without it the display has no denominator at all.
|
||||
if cursor.total.is_none() {
|
||||
if cancelled(cancel) {
|
||||
return Ok(());
|
||||
}
|
||||
cursor.total = Some(repo::row_count(conn)?);
|
||||
}
|
||||
let page = config.processing.batch_size.max(1) as i64;
|
||||
// Lives across pages, not per page: consecutive pages walk the same
|
||||
// directories, which is exactly the repetition worth remembering.
|
||||
let mut covered = CoverCache::default();
|
||||
while cursor.root_idx < cursor.scope.roots.len() {
|
||||
if cancelled(cancel) {
|
||||
return Ok(());
|
||||
}
|
||||
let root = &cursor.scope.roots[cursor.root_idx];
|
||||
if cursor.after.is_empty() {
|
||||
cursor.after = root.lo.clone();
|
||||
|
|
@ -267,6 +427,7 @@ pub fn advance(
|
|||
continue;
|
||||
};
|
||||
cursor.after = last.path.clone();
|
||||
cursor.examined += rows.len();
|
||||
let root = cursor.scope.roots[cursor.root_idx].path.clone();
|
||||
let (deleted, recontented) = apply_page(
|
||||
conn,
|
||||
|
|
@ -276,6 +437,7 @@ pub fn advance(
|
|||
&cursor.work,
|
||||
&root,
|
||||
&rows,
|
||||
&mut covered,
|
||||
)?;
|
||||
cursor.deleted += deleted;
|
||||
cursor.recontented += recontented;
|
||||
|
|
@ -287,6 +449,12 @@ pub fn advance(
|
|||
|
||||
// Deletions leave the FTS index with tombstones and a long segment list;
|
||||
// the same automerge that follows a run's stale cleanup collapses them.
|
||||
// It is one statement that can run for a while, so it is the last thing
|
||||
// the flag can spare the caller — and skipping it costs only tidiness,
|
||||
// since the next run's own automerge collapses the same segments.
|
||||
if cancelled(cancel) {
|
||||
return Ok(());
|
||||
}
|
||||
if cursor.deleted > 0 || cursor.recontented > 0 {
|
||||
fts_finalize_after_text_indexing(conn);
|
||||
}
|
||||
|
|
@ -294,7 +462,12 @@ pub fn advance(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn cancelled(cancel: &AtomicBool) -> bool {
|
||||
cancel.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Decide and write one page of rows. Returns `(deleted, recontented)`.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn apply_page(
|
||||
conn: &mut Connection,
|
||||
config: &Config,
|
||||
|
|
@ -303,6 +476,7 @@ fn apply_page(
|
|||
work: &IndexWork,
|
||||
root: &Path,
|
||||
rows: &[repo::ScopeRow],
|
||||
covered: &mut CoverCache,
|
||||
) -> Result<(usize, usize), String> {
|
||||
let mut doomed: Vec<i64> = Vec::new();
|
||||
let mut stale_text: Vec<i64> = Vec::new();
|
||||
|
|
@ -311,7 +485,7 @@ fn apply_page(
|
|||
|
||||
for row in rows {
|
||||
let path = Path::new(&row.path);
|
||||
if work.prune_scope && !scope.covers(root, path) {
|
||||
if work.prune_scope && !scope.covers_cached(root, path, covered) {
|
||||
doomed.push(row.id);
|
||||
continue;
|
||||
}
|
||||
|
|
@ -388,18 +562,44 @@ pub fn stored_config(conn: &Connection, config: &Config) -> Result<Config, Strin
|
|||
"store_text_for_snippets" => {
|
||||
stored.processing.store_text_for_snippets = value == "true"
|
||||
}
|
||||
"hash_length" => {
|
||||
if let Ok(n) = value.parse() {
|
||||
stored.processing.hash_length = n;
|
||||
}
|
||||
}
|
||||
"hash_length" => match value.parse() {
|
||||
Ok(n) => stored.processing.hash_length = n,
|
||||
// Keeping the caller's value is the safest of bad options,
|
||||
// but it makes the reconciliation diff describe a config the
|
||||
// index was not built under, so it must not pass in silence.
|
||||
Err(e) => crate::log_warn!("stored hash_length {:?} unreadable: {}", value, e),
|
||||
},
|
||||
"tokenize" => stored.processing.tokenize = value,
|
||||
// An unrecognized key is a record written by a newer build.
|
||||
// Ignoring it leaves that setting at the caller's value, which is
|
||||
// what a build that does not know the key would have used anyway.
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(stored)
|
||||
}
|
||||
|
||||
/// The reconciliation the index still owes `config`, derived from its own
|
||||
/// record of what it was last brought into line with.
|
||||
///
|
||||
/// The one question three callers ask in the same words: a run, to decide what
|
||||
/// its prologue must do; a test, to check a pass recorded itself; and the GUI
|
||||
/// at startup, to tell the user their settings have not reached the index yet.
|
||||
/// Empty is the normal answer — every pass that finishes stamps the record —
|
||||
/// so a non-empty one means a pass was abandoned, or the config was edited
|
||||
/// while the app was closed. Either way the remedy is the same: run indexing.
|
||||
///
|
||||
/// The roots are canonicalized first, because that is the spelling the record
|
||||
/// holds; comparing raw ones would report a `~` or a trailing slash as a
|
||||
/// changed root.
|
||||
pub fn outstanding_work(db_path: &str, config: &Config) -> Result<IndexWork, String> {
|
||||
let conn = crate::db::open_existing(db_path, false)?;
|
||||
let mut current = config.clone();
|
||||
current.paths.indexing_paths = config.normalized_indexing_paths().into_iter().collect();
|
||||
let stored = stored_config(&conn, ¤t)?;
|
||||
Ok(crate::config::diff_actions(&stored, ¤t).work)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
@ -409,18 +609,7 @@ mod tests {
|
|||
use std::sync::Arc;
|
||||
|
||||
fn tmp_tree(tag: &str) -> PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"quicksearch-scope-{}-{}-{}",
|
||||
tag,
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
std::fs::create_dir_all(&p).unwrap();
|
||||
std::fs::canonicalize(&p).unwrap()
|
||||
crate::testutil::scratch_dir_canonical(tag)
|
||||
}
|
||||
|
||||
fn touch(p: &Path) {
|
||||
|
|
@ -456,6 +645,34 @@ mod tests {
|
|||
.collect()
|
||||
}
|
||||
|
||||
/// One `files` row per path, which is all the scan reads.
|
||||
fn seed(conn: &mut Connection, paths: &[PathBuf]) {
|
||||
let tx = conn.transaction().unwrap();
|
||||
for path in paths {
|
||||
let path = path.to_string_lossy();
|
||||
let (parent, name) = path.rsplit_once('/').unwrap();
|
||||
repo::insert_file(
|
||||
&tx,
|
||||
&repo::NewFile {
|
||||
name,
|
||||
path: &path,
|
||||
parent,
|
||||
size: 1,
|
||||
mtime: 1,
|
||||
inode: None,
|
||||
device_id: None,
|
||||
mime: Some("text/plain"),
|
||||
ftype: crate::mime::FileType::TEXT,
|
||||
hash: None,
|
||||
needs_content: false,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.expect("unique path");
|
||||
}
|
||||
tx.commit().unwrap();
|
||||
}
|
||||
|
||||
/// Every file that physically exists under `root`, walker or no walker.
|
||||
fn on_disk(root: &Path) -> Vec<PathBuf> {
|
||||
walkdir::WalkDir::new(root)
|
||||
|
|
@ -566,6 +783,156 @@ mod tests {
|
|||
std::fs::remove_dir_all(&base).ok();
|
||||
}
|
||||
|
||||
/// The counters the status display reads. A scan that reports nothing is
|
||||
/// indistinguishable from a hang, and on the indexes where this pass takes
|
||||
/// minutes that is exactly what it looked like.
|
||||
#[test]
|
||||
fn the_scan_reports_its_way_through_every_row() {
|
||||
let root = tmp_tree("progress");
|
||||
for i in 0..7 {
|
||||
touch(&root.join(format!("f{}.log", i)));
|
||||
}
|
||||
touch(&root.join("keep.txt"));
|
||||
|
||||
let db_dir = tmp_tree("progress-db");
|
||||
let db = empty_db(&db_dir);
|
||||
let mut conn = crate::db::open_existing(db.to_str().unwrap(), true).unwrap();
|
||||
let mut config = Config::default();
|
||||
config.paths.indexing_paths = vec![root.to_string_lossy().into_owned()];
|
||||
// One row per page, so the scan takes as many steps as there are rows
|
||||
// and a counter that only moved at the end would be visible.
|
||||
config.processing.batch_size = 1;
|
||||
seed(&mut conn, &on_disk(&root));
|
||||
|
||||
let mut narrowed = config.clone();
|
||||
narrowed.indexing.ignore_patterns = vec!["*.log".into()];
|
||||
let work = crate::config::diff_actions(&config, &narrowed).work;
|
||||
let mut cursor = WorkCursor::new(work, &narrowed).unwrap();
|
||||
assert_eq!(
|
||||
cursor.progress(),
|
||||
ReconcileProgress::default(),
|
||||
"nothing counted before the first slice"
|
||||
);
|
||||
|
||||
let registry = Registry::default_set();
|
||||
let run = AtomicBool::new(false);
|
||||
let mut seen: Vec<usize> = Vec::new();
|
||||
while !cursor.done() {
|
||||
// A deadline already past, so each call does the least it can and
|
||||
// the counters are sampled at their finest granularity.
|
||||
advance(
|
||||
&mut conn,
|
||||
&narrowed,
|
||||
®istry,
|
||||
&mut cursor,
|
||||
Instant::now(),
|
||||
&run,
|
||||
)
|
||||
.unwrap();
|
||||
seen.push(cursor.progress().examined);
|
||||
}
|
||||
|
||||
let end = cursor.progress();
|
||||
assert_eq!(end.total, Some(8), "counted once, before the first page");
|
||||
assert_eq!(end.examined, 8, "every row was re-tested");
|
||||
assert_eq!(end.deleted, 7, "the logs, and only the logs");
|
||||
assert!(
|
||||
seen.windows(2).all(|w| w[0] <= w[1]),
|
||||
"the count never goes backwards: {:?}",
|
||||
seen
|
||||
);
|
||||
assert!(
|
||||
seen.len() > 2 && seen[0] < end.examined,
|
||||
"progress was reported during the scan, not only at its end: {:?}",
|
||||
seen
|
||||
);
|
||||
|
||||
std::fs::remove_dir_all(&root).ok();
|
||||
std::fs::remove_dir_all(&db_dir).ok();
|
||||
}
|
||||
|
||||
/// Cancelling stops the pass at the next statement boundary and leaves the
|
||||
/// cursor un-finished, so nothing downstream can mistake it for done and
|
||||
/// record the configuration as reconciled. The rows it had already reached
|
||||
/// stay gone — every part of the pass is idempotent, and the next run
|
||||
/// re-derives the same plan and finishes it.
|
||||
#[test]
|
||||
fn cancelling_stops_the_scan_without_finishing_it() {
|
||||
let root = tmp_tree("cancel");
|
||||
for i in 0..6 {
|
||||
touch(&root.join(format!("f{}.log", i)));
|
||||
}
|
||||
touch(&root.join("keep.txt"));
|
||||
|
||||
let db_dir = tmp_tree("cancel-db");
|
||||
let db = empty_db(&db_dir);
|
||||
let mut conn = crate::db::open_existing(db.to_str().unwrap(), true).unwrap();
|
||||
let mut config = Config::default();
|
||||
config.paths.indexing_paths = vec![root.to_string_lossy().into_owned()];
|
||||
config.processing.batch_size = 1;
|
||||
seed(&mut conn, &on_disk(&root));
|
||||
|
||||
let mut narrowed = config.clone();
|
||||
narrowed.indexing.ignore_patterns = vec!["*.log".into()];
|
||||
let work = crate::config::diff_actions(&config, &narrowed).work;
|
||||
let registry = Registry::default_set();
|
||||
|
||||
// Cancelled from the outset: not one statement runs.
|
||||
let stop = AtomicBool::new(true);
|
||||
let mut cursor = WorkCursor::new(work.clone(), &narrowed).unwrap();
|
||||
advance(
|
||||
&mut conn,
|
||||
&narrowed,
|
||||
®istry,
|
||||
&mut cursor,
|
||||
Instant::now() + SLICE,
|
||||
&stop,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!cursor.done(), "a cancelled pass is never finished");
|
||||
assert_eq!(
|
||||
cursor.progress(),
|
||||
ReconcileProgress::default(),
|
||||
"a cancelled pass touched the index"
|
||||
);
|
||||
|
||||
// And part-way through: one slice with the flag clear, the rest with
|
||||
// it set. The counters keep what the first slice earned.
|
||||
let stop = AtomicBool::new(false);
|
||||
let mut cursor = WorkCursor::new(work, &narrowed).unwrap();
|
||||
advance(
|
||||
&mut conn,
|
||||
&narrowed,
|
||||
®istry,
|
||||
&mut cursor,
|
||||
Instant::now(),
|
||||
&stop,
|
||||
)
|
||||
.unwrap();
|
||||
let part_way = cursor.progress();
|
||||
assert!(part_way.examined > 0 && !cursor.done(), "nothing to cancel");
|
||||
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
advance(
|
||||
&mut conn,
|
||||
&narrowed,
|
||||
®istry,
|
||||
&mut cursor,
|
||||
Instant::now() + SLICE,
|
||||
&stop,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!cursor.done(), "the pass finished despite the cancellation");
|
||||
assert_eq!(
|
||||
cursor.progress(),
|
||||
part_way,
|
||||
"the cancelled slice did more work"
|
||||
);
|
||||
|
||||
std::fs::remove_dir_all(&root).ok();
|
||||
std::fs::remove_dir_all(&db_dir).ok();
|
||||
}
|
||||
|
||||
/// A path under no configured root has no rules that could be applied to
|
||||
/// it — a followed symlink's target is the real case. The scan reaches it
|
||||
/// by never visiting it, so `owning_root` returning `None` is what keeps
|
||||
|
|
|
|||
|
|
@ -213,6 +213,29 @@ fn count_frac(count: usize) -> f64 {
|
|||
(1000usize.saturating_sub(count.min(1000))) as f64 / 1000.0
|
||||
}
|
||||
|
||||
/// The `files` columns every pass selects, in the order the passes index
|
||||
/// them: `0` id, `1` name, `2` path, `3` size, `4` mtime. Passes that also
|
||||
/// want the stored document text append `dt.text_zstd` as column `5`.
|
||||
///
|
||||
/// One string rather than seven copies, because the column *order* is what
|
||||
/// every `row.get(n)` in this file is written against — a pass that spelled
|
||||
/// its own list in a different order would compile and then quietly serve
|
||||
/// paths as names.
|
||||
const HIT_COLUMNS: &str = "f.id, f.name, f.path, f.size, f.mtime";
|
||||
|
||||
/// Columns 3 and 4: the two every pass reads identically and stores without
|
||||
/// inspecting.
|
||||
///
|
||||
/// The clamp is the point of having this in one place. `size` is `INTEGER` in
|
||||
/// SQLite and so signed; a corrupt or hand-edited row holding `-1` would
|
||||
/// otherwise become 18 exabytes on the way to `u64` and sort to the top of
|
||||
/// every size-ordered result.
|
||||
fn size_and_mtime(row: &rusqlite::Row<'_>) -> Result<(u64, i64), String> {
|
||||
let size = row.get::<_, i64>(3).map_err(|e| e.to_string())?.max(0) as u64;
|
||||
let mtime = row.get(4).map_err(|e| e.to_string())?;
|
||||
Ok((size, mtime))
|
||||
}
|
||||
|
||||
/// The path tiers only make sense with enough term to be specific — same
|
||||
/// floor the trigram full-text pass uses. Wildcards count only their
|
||||
/// literal content (`a*b` is two characters of specificity, not three).
|
||||
|
|
@ -439,8 +462,9 @@ impl<'a> Cx<'a> {
|
|||
// A path always ends in its own name, so `path LIKE` is the
|
||||
// superset that feeds both the name and the path tiers.
|
||||
let sql = format!(
|
||||
"SELECT f.id, f.name, f.path, f.size, f.mtime FROM files f \
|
||||
"SELECT {} FROM files f \
|
||||
WHERE {} LIKE ? ESCAPE '\\'{}",
|
||||
HIT_COLUMNS,
|
||||
if with_paths { "f.path" } else { "f.name" },
|
||||
query.filter_sql
|
||||
);
|
||||
|
|
@ -469,7 +493,7 @@ impl<'a> Cx<'a> {
|
|||
let mut clock = FlushClock::new();
|
||||
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
|
||||
scanned += 1;
|
||||
if scanned % CANCEL_CHECK_ROWS == 0 && self.cancelled() {
|
||||
if scanned.is_multiple_of(CANCEL_CHECK_ROWS) && self.cancelled() {
|
||||
return Ok(false);
|
||||
}
|
||||
let file_id: i64 = row.get(0).map_err(|e| e.to_string())?;
|
||||
|
|
@ -519,12 +543,13 @@ impl<'a> Cx<'a> {
|
|||
truncated_start: false,
|
||||
truncated_end: false,
|
||||
};
|
||||
let (size, mtime) = size_and_mtime(row)?;
|
||||
let hit = SearchHit {
|
||||
file_id,
|
||||
name,
|
||||
path,
|
||||
size: row.get::<_, i64>(3).map_err(|e| e.to_string())?.max(0) as u64,
|
||||
mtime: row.get(4).map_err(|e| e.to_string())?,
|
||||
size,
|
||||
mtime,
|
||||
rank,
|
||||
stage: rank as u8,
|
||||
snippet: Some(snip),
|
||||
|
|
@ -586,20 +611,22 @@ impl<'a> Cx<'a> {
|
|||
let (sql, params) = match match_expr {
|
||||
Some(expr) => (
|
||||
format!(
|
||||
"SELECT f.id, f.name, f.path, f.size, f.mtime, dt.text_zstd \
|
||||
"SELECT {}, dt.text_zstd \
|
||||
FROM searchabletext \
|
||||
JOIN files f ON f.id = searchabletext.rowid \
|
||||
LEFT JOIN documents_text dt ON dt.file_id = f.id \
|
||||
WHERE searchabletext MATCH ?{}",
|
||||
HIT_COLUMNS,
|
||||
query.filter_sql
|
||||
),
|
||||
self.params_with_filters(vec![rusqlite::types::Value::Text(expr)]),
|
||||
),
|
||||
None => (
|
||||
format!(
|
||||
"SELECT f.id, f.name, f.path, f.size, f.mtime, dt.text_zstd \
|
||||
"SELECT {}, dt.text_zstd \
|
||||
FROM documents_text dt \
|
||||
JOIN files f ON f.id = dt.file_id WHERE 1=1{}",
|
||||
HIT_COLUMNS,
|
||||
query.filter_sql
|
||||
),
|
||||
self.params_with_filters(Vec::new()),
|
||||
|
|
@ -692,12 +719,13 @@ impl<'a> Cx<'a> {
|
|||
continue;
|
||||
}
|
||||
|
||||
let (size, mtime) = size_and_mtime(row)?;
|
||||
buf.push(SearchHit {
|
||||
file_id,
|
||||
name: row.get(1).map_err(|e| e.to_string())?,
|
||||
path,
|
||||
size: row.get::<_, i64>(3).map_err(|e| e.to_string())?.max(0) as u64,
|
||||
mtime: row.get(4).map_err(|e| e.to_string())?,
|
||||
size,
|
||||
mtime,
|
||||
rank,
|
||||
stage,
|
||||
snippet: snip,
|
||||
|
|
@ -733,7 +761,8 @@ impl<'a> Cx<'a> {
|
|||
let with_paths = path_tiers_enabled(&self.query.pattern);
|
||||
|
||||
let sql = format!(
|
||||
"SELECT f.id, f.name, f.path, f.size, f.mtime FROM files f WHERE 1=1{}",
|
||||
"SELECT {} FROM files f WHERE 1=1{}",
|
||||
HIT_COLUMNS,
|
||||
self.query.filter_sql
|
||||
);
|
||||
let params = self.params_with_filters(Vec::new());
|
||||
|
|
@ -750,7 +779,7 @@ impl<'a> Cx<'a> {
|
|||
let mut clock = FlushClock::new();
|
||||
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
|
||||
scanned += 1;
|
||||
if scanned % 1024 == 0 && self.cancelled() {
|
||||
if scanned.is_multiple_of(1024) && self.cancelled() {
|
||||
return Ok(false);
|
||||
}
|
||||
let file_id: i64 = row.get(0).map_err(|e| e.to_string())?;
|
||||
|
|
@ -789,12 +818,13 @@ impl<'a> Cx<'a> {
|
|||
},
|
||||
));
|
||||
let is_path_tier = rank >= 11.0;
|
||||
let (size, mtime) = size_and_mtime(row)?;
|
||||
let hit = SearchHit {
|
||||
file_id,
|
||||
name,
|
||||
path,
|
||||
size: row.get::<_, i64>(3).map_err(|e| e.to_string())?.max(0) as u64,
|
||||
mtime: row.get(4).map_err(|e| e.to_string())?,
|
||||
size,
|
||||
mtime,
|
||||
rank,
|
||||
stage: rank as u8,
|
||||
snippet: snip,
|
||||
|
|
@ -838,8 +868,9 @@ impl<'a> Cx<'a> {
|
|||
};
|
||||
|
||||
let sql = format!(
|
||||
"SELECT f.id, f.name, f.path, f.size, f.mtime, dt.text_zstd \
|
||||
"SELECT {}, dt.text_zstd \
|
||||
FROM documents_text dt JOIN files f ON f.id = dt.file_id WHERE 1=1{}",
|
||||
HIT_COLUMNS,
|
||||
self.query.filter_sql
|
||||
);
|
||||
let params = self.params_with_filters(Vec::new());
|
||||
|
|
@ -882,12 +913,13 @@ impl<'a> Cx<'a> {
|
|||
continue;
|
||||
}
|
||||
let snip = first.map(|range| snippet::window_around(&text, range, &snippet_opts));
|
||||
let (size, mtime) = size_and_mtime(row)?;
|
||||
buf.push(SearchHit {
|
||||
file_id,
|
||||
name: row.get(1).map_err(|e| e.to_string())?,
|
||||
path,
|
||||
size: row.get::<_, i64>(3).map_err(|e| e.to_string())?.max(0) as u64,
|
||||
mtime: row.get(4).map_err(|e| e.to_string())?,
|
||||
size,
|
||||
mtime,
|
||||
rank: 8.0 + count_frac(count),
|
||||
stage: 8,
|
||||
snippet: snip,
|
||||
|
|
@ -911,7 +943,8 @@ impl<'a> Cx<'a> {
|
|||
let query = self.query;
|
||||
let re = query.regex.as_ref().expect("regex-only pass list");
|
||||
let sql = format!(
|
||||
"SELECT f.id, f.name, f.path, f.size, f.mtime FROM files f WHERE 1=1{}",
|
||||
"SELECT {} FROM files f WHERE 1=1{}",
|
||||
HIT_COLUMNS,
|
||||
query.filter_sql
|
||||
);
|
||||
let params = self.params_with_filters(Vec::new());
|
||||
|
|
@ -928,7 +961,7 @@ impl<'a> Cx<'a> {
|
|||
let mut clock = FlushClock::new();
|
||||
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
|
||||
scanned += 1;
|
||||
if scanned % 1024 == 0 && self.cancelled() {
|
||||
if scanned.is_multiple_of(1024) && self.cancelled() {
|
||||
return Ok(false);
|
||||
}
|
||||
let file_id: i64 = row.get(0).map_err(|e| e.to_string())?;
|
||||
|
|
@ -956,12 +989,13 @@ impl<'a> Cx<'a> {
|
|||
truncated_start: false,
|
||||
truncated_end: false,
|
||||
};
|
||||
let (size, mtime) = size_and_mtime(row)?;
|
||||
let hit = SearchHit {
|
||||
file_id,
|
||||
name,
|
||||
path,
|
||||
size: row.get::<_, i64>(3).map_err(|e| e.to_string())?.max(0) as u64,
|
||||
mtime: row.get(4).map_err(|e| e.to_string())?,
|
||||
size,
|
||||
mtime,
|
||||
rank,
|
||||
stage: rank as u8,
|
||||
snippet: Some(snip),
|
||||
|
|
@ -992,8 +1026,9 @@ impl<'a> Cx<'a> {
|
|||
let query = self.query;
|
||||
let re = query.regex.as_ref().expect("regex-only pass list");
|
||||
let sql = format!(
|
||||
"SELECT f.id, f.name, f.path, f.size, f.mtime, dt.text_zstd \
|
||||
"SELECT {}, dt.text_zstd \
|
||||
FROM documents_text dt JOIN files f ON f.id = dt.file_id WHERE 1=1{}",
|
||||
HIT_COLUMNS,
|
||||
query.filter_sql
|
||||
);
|
||||
let params = self.params_with_filters(Vec::new());
|
||||
|
|
@ -1033,12 +1068,13 @@ impl<'a> Cx<'a> {
|
|||
let r = clamp_match_range(&text, r, SNIPPET_WINDOW_CHARS);
|
||||
snippet::window_around(&text, (r.start, r.end), &snippet_opts)
|
||||
});
|
||||
let (size, mtime) = size_and_mtime(row)?;
|
||||
buf.push(SearchHit {
|
||||
file_id,
|
||||
name: row.get(1).map_err(|e| e.to_string())?,
|
||||
path,
|
||||
size: row.get::<_, i64>(3).map_err(|e| e.to_string())?.max(0) as u64,
|
||||
mtime: row.get(4).map_err(|e| e.to_string())?,
|
||||
size,
|
||||
mtime,
|
||||
rank: 6.0 + count_frac(count),
|
||||
stage: 6,
|
||||
snippet: snip,
|
||||
|
|
|
|||
|
|
@ -88,15 +88,7 @@ mod tests {
|
|||
use crate::mime::FileType;
|
||||
|
||||
fn seed_db() -> std::path::PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"qs-dups-{}-{}.sqlite",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
let p = crate::testutil::scratch_dir("dups").join("index.sqlite");
|
||||
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
|
||||
let tx = conn.transaction().unwrap();
|
||||
let add = |name: &str, path: &str, size: u64, hash: Option<&[u8]>| {
|
||||
|
|
|
|||
|
|
@ -320,9 +320,9 @@ mod tests {
|
|||
}
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
for j in 0..=hay.len() {
|
||||
best = best.min(prev[j]);
|
||||
}
|
||||
// The best occurrence may end at any text position, so the answer is
|
||||
// the smallest value in the final row.
|
||||
best = prev.iter().copied().min().unwrap_or(best);
|
||||
if best <= k {
|
||||
Some(best)
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -24,15 +24,35 @@
|
|||
//!
|
||||
//! Consumers that want a plain blocking search (the CLI mode) skip the
|
||||
//! service entirely and call [`cascade::run`] with a collecting sink.
|
||||
//!
|
||||
//! # The worker's connection
|
||||
//!
|
||||
//! The worker holds one connection across requests and releases it after
|
||||
//! [`IDLE_RELEASE`] of quiet. This is not an optimisation of open cost — an
|
||||
//! open is microseconds — but of the page cache behind it: a search runs on
|
||||
//! every character typed, so reopening per request meant paying to warm a
|
||||
//! cache and then discarding it, once per keystroke, forever.
|
||||
//!
|
||||
//! What the old per-request open bought, and what now has to be arranged
|
||||
//! deliberately, is never operating on an index that has been replaced
|
||||
//! underneath it. Two things can do that, and only one is visible in the path:
|
||||
//! the config can point at a different file, and a rebuild, clear or
|
||||
//! schema-drift wipe can put a *new* file at the *same* path. The second is
|
||||
//! why [`crate::db::index_epoch`] exists — see [`Worker::take_connection`].
|
||||
//! Holding a handle on a replaced index would mean stale results and, on
|
||||
//! Linux, its blocks staying allocated until the handle closed.
|
||||
|
||||
pub mod cascade;
|
||||
pub mod duplicates;
|
||||
pub mod fuzzy;
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{mpsc, Arc, Mutex};
|
||||
use std::thread::JoinHandle;
|
||||
use std::time::Duration;
|
||||
|
||||
use rusqlite::Connection;
|
||||
|
||||
use crate::db;
|
||||
use crate::query::split::split_for_cascade;
|
||||
|
|
@ -41,6 +61,19 @@ use crate::snippet::Snippet;
|
|||
pub use cascade::Outcome;
|
||||
pub use duplicates::{find_duplicate_groups, DuplicateGroup};
|
||||
|
||||
/// How long the worker keeps its connection after the last request.
|
||||
///
|
||||
/// The connection is held across requests so a typing session runs against a
|
||||
/// warm page cache (see [`Worker::take_connection`]). It is not held *forever*, for
|
||||
/// two reasons that have nothing to do with the cache: an open handle on a
|
||||
/// deleted index keeps its blocks allocated on disk, and an open reader stops
|
||||
/// SQLite from resetting the WAL, which would then grow toward
|
||||
/// `maximum_wal_size` and never come back down.
|
||||
///
|
||||
/// So: long enough to span the pauses inside a search session, short enough
|
||||
/// that an abandoned one is not still holding the index minutes later.
|
||||
const IDLE_RELEASE: Duration = Duration::from_secs(30);
|
||||
|
||||
/// One search result. `rank` is the sort key (lower = better): integer
|
||||
/// part = cascade stage (1–11), fraction = occurrence-count or
|
||||
/// edit-distance tiebreak. Batches arrive already rank-ordered and later
|
||||
|
|
@ -147,6 +180,18 @@ impl SearchService {
|
|||
pub fn new(
|
||||
db_path: PathBuf,
|
||||
notify: Arc<dyn Fn() + Send + Sync>,
|
||||
) -> (SearchService, mpsc::Receiver<SearchUpdate>) {
|
||||
Self::new_with_idle_release(db_path, notify, IDLE_RELEASE)
|
||||
}
|
||||
|
||||
/// [`Self::new`] with an explicit connection-release window.
|
||||
///
|
||||
/// Tests use short windows; the default [`IDLE_RELEASE`] is right for real
|
||||
/// use, where the window has to span the pauses inside a typing session.
|
||||
pub fn new_with_idle_release(
|
||||
db_path: PathBuf,
|
||||
notify: Arc<dyn Fn() + Send + Sync>,
|
||||
idle_release: Duration,
|
||||
) -> (SearchService, mpsc::Receiver<SearchUpdate>) {
|
||||
let (req_tx, req_rx) = mpsc::channel::<SearchRequest>();
|
||||
let (update_tx, update_rx) = mpsc::channel::<SearchUpdate>();
|
||||
|
|
@ -161,6 +206,8 @@ impl SearchService {
|
|||
latest_gen: latest_gen.clone(),
|
||||
in_flight: in_flight.clone(),
|
||||
db_path: db_path.clone(),
|
||||
open: None,
|
||||
idle_release,
|
||||
};
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("qs-search".into())
|
||||
|
|
@ -268,11 +315,37 @@ struct Worker {
|
|||
latest_gen: Arc<AtomicU64>,
|
||||
in_flight: InFlight,
|
||||
db_path: Arc<Mutex<PathBuf>>,
|
||||
/// The connection, and the index generation and path it was opened
|
||||
/// against. See [`Worker::take_connection`].
|
||||
open: Option<OpenIndex>,
|
||||
/// How long `open` survives with no requests; [`IDLE_RELEASE`] outside
|
||||
/// tests.
|
||||
idle_release: Duration,
|
||||
}
|
||||
|
||||
/// A connection held across requests, tagged with what it was opened on.
|
||||
struct OpenIndex {
|
||||
conn: Connection,
|
||||
epoch: u64,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl Worker {
|
||||
fn run(self) {
|
||||
while let Ok(first) = self.req_rx.recv() {
|
||||
fn run(mut self) {
|
||||
loop {
|
||||
// `recv_timeout`, not `recv`: a search session is a burst of
|
||||
// requests one keystroke apart, and the connection is worth
|
||||
// keeping for the length of one. Past that it is worth strictly
|
||||
// less than nothing — see [`IDLE_RELEASE`].
|
||||
let first = match self.req_rx.recv_timeout(self.idle_release) {
|
||||
Ok(req) => req,
|
||||
Err(mpsc::RecvTimeoutError::Timeout) => {
|
||||
self.open = None;
|
||||
continue;
|
||||
}
|
||||
// The service was dropped; nothing more is coming.
|
||||
Err(mpsc::RecvTimeoutError::Disconnected) => return,
|
||||
};
|
||||
// A fast typist queues several requests; only the newest one
|
||||
// matters.
|
||||
let mut req = first;
|
||||
|
|
@ -286,12 +359,50 @@ impl Worker {
|
|||
}
|
||||
}
|
||||
|
||||
/// Take the connection to run this request on, reopening if the one held
|
||||
/// cannot be reused.
|
||||
///
|
||||
/// Taken out and handed back by the caller, rather than borrowed from
|
||||
/// `self`, for the same reason [`crate::coordinator`]'s writer is: the
|
||||
/// cascade needs the connection for its whole run, and everything else on
|
||||
/// `self` — the update channel, the generation counter, the interrupt slot
|
||||
/// — has to stay reachable while it does.
|
||||
///
|
||||
/// Reuse is what makes [`crate::db::schema::PRAGMAS_SEARCH`]'s page cache
|
||||
/// worth having: searches run on every character typed, so the second
|
||||
/// query of a session and every one after it finds the b-tree interior
|
||||
/// pages and FTS5 segment tips already resident. Opening per request threw
|
||||
/// that away each time.
|
||||
///
|
||||
/// It is reopened when either half of what it was opened against has
|
||||
/// changed. The path moves when the user points the config at a different
|
||||
/// index. The epoch moves when the file at the *same* path is replaced —
|
||||
/// a rebuild, a clear, or a schema-drift wipe — which no comparison of
|
||||
/// paths could ever catch, and which would otherwise leave this worker
|
||||
/// querying a deleted inode and pinning its blocks on disk.
|
||||
fn take_connection(&mut self, db_path: &Path) -> Result<OpenIndex, String> {
|
||||
let epoch = db::index_epoch();
|
||||
if let Some(open) = self.open.take() {
|
||||
if open.epoch == epoch && open.path == db_path {
|
||||
return Ok(open);
|
||||
}
|
||||
// Dropped here, before the open below, so the handle on the old
|
||||
// index is gone before a handle on the new one exists.
|
||||
drop(open);
|
||||
}
|
||||
Ok(OpenIndex {
|
||||
conn: db::open::open_search_reader(&db_path.to_string_lossy())?,
|
||||
epoch,
|
||||
path: db_path.to_path_buf(),
|
||||
})
|
||||
}
|
||||
|
||||
fn send(&self, update: SearchUpdate) {
|
||||
let _ = self.update_tx.send(update);
|
||||
(self.notify)();
|
||||
}
|
||||
|
||||
fn handle(&self, req: SearchRequest) {
|
||||
fn handle(&mut self, req: SearchRequest) {
|
||||
let generation = req.generation;
|
||||
self.send(SearchUpdate::Started { generation });
|
||||
|
||||
|
|
@ -307,9 +418,7 @@ impl Worker {
|
|||
};
|
||||
|
||||
let db_path = self.db_path.lock().unwrap().clone();
|
||||
// Per-request open: microseconds, and always sees a freshly
|
||||
// rebuilt index file rather than pinning a deleted inode.
|
||||
let conn = match db::open_existing(&db_path.to_string_lossy(), false) {
|
||||
let open = match self.take_connection(&db_path) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
self.send(SearchUpdate::Error {
|
||||
|
|
@ -322,13 +431,13 @@ impl Worker {
|
|||
// Publish the handle tagged with the generation it kills, before the
|
||||
// first statement runs. Anyone cancelling from here on can tell this
|
||||
// search apart from the one that supersedes it.
|
||||
*self.in_flight.lock().unwrap() = Some((generation, conn.get_interrupt_handle()));
|
||||
*self.in_flight.lock().unwrap() = Some((generation, open.conn.get_interrupt_handle()));
|
||||
|
||||
let mut sink = |hits: Vec<SearchHit>| {
|
||||
self.send(SearchUpdate::Hits { generation, hits });
|
||||
};
|
||||
let outcome = cascade::run(
|
||||
&conn,
|
||||
&open.conn,
|
||||
&split,
|
||||
&req.options,
|
||||
generation,
|
||||
|
|
@ -338,6 +447,15 @@ impl Worker {
|
|||
|
||||
*self.in_flight.lock().unwrap() = None;
|
||||
|
||||
// Kept only if it still works. A failed cascade may have failed
|
||||
// *because* of this connection — a torn file, or an interrupt that
|
||||
// left it unusable — and putting it back would wedge every search
|
||||
// after this one behind the same bad handle. Dropping it costs one
|
||||
// reopen.
|
||||
if outcome.is_ok() {
|
||||
self.open = Some(open);
|
||||
}
|
||||
|
||||
match outcome {
|
||||
Ok(Some(Outcome { total, limited })) => self.send(SearchUpdate::Completed {
|
||||
generation,
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ impl IndexKey {
|
|||
hex_encode(&self.0)
|
||||
}
|
||||
|
||||
/// Strict inverse of [`to_hex`]: exactly 64 hex digits, any case.
|
||||
/// Strict inverse of [`IndexKey::to_hex`]: exactly 64 hex digits, any case.
|
||||
pub fn from_hex(hex: &str) -> Result<IndexKey, String> {
|
||||
let bytes = hex_decode(hex)?;
|
||||
let arr: [u8; KEY_LEN] = bytes
|
||||
|
|
@ -102,7 +102,7 @@ fn hex_encode(bytes: &[u8]) -> String {
|
|||
}
|
||||
|
||||
fn hex_decode(hex: &str) -> Result<Vec<u8>, String> {
|
||||
if hex.len() % 2 != 0 {
|
||||
if !hex.len().is_multiple_of(2) {
|
||||
return Err("hex string has odd length".to_string());
|
||||
}
|
||||
if !hex.bytes().all(|b| b.is_ascii_hexdigit()) {
|
||||
|
|
|
|||
76
crates/quicksearch-core/src/testutil.rs
Normal file
76
crates/quicksearch-core/src/testutil.rs
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
//! Scratch directories for tests.
|
||||
//!
|
||||
//! Public and `#[doc(hidden)]` rather than `#[cfg(test)]`: the `tests/`
|
||||
//! integration binaries and the GUI crate are separate compilation units, so a
|
||||
//! test-gated item here would be invisible to them. This is the only reason it
|
||||
//! is not private.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
/// Distinguishes directories requested within one process. Two tests running
|
||||
/// on different threads in the same millisecond would otherwise collide — the
|
||||
/// hand-rolled helpers this replaces all keyed off a timestamp, which made
|
||||
/// that rare rather than impossible.
|
||||
static NEXT: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
/// A fresh, empty directory under the system temp dir, named for `tag`.
|
||||
///
|
||||
/// Not cleaned up on drop, deliberately: when a test fails, the tree it built
|
||||
/// is most of the evidence. The OS clears the temp dir eventually.
|
||||
///
|
||||
/// Panics rather than returning a `Result` — a test that cannot create a
|
||||
/// directory has nothing left to assert.
|
||||
#[doc(hidden)]
|
||||
pub fn scratch_dir(tag: &str) -> PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"quicksearch-{}-{}-{}",
|
||||
tag,
|
||||
std::process::id(),
|
||||
NEXT.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
std::fs::create_dir_all(&p).expect("create scratch dir");
|
||||
p
|
||||
}
|
||||
|
||||
/// [`scratch_dir`] canonicalized, for the tests that compare walked paths
|
||||
/// against the root they were given. On macOS `/tmp` is a symlink to
|
||||
/// `/private/tmp`, so an uncanonicalized root and a walked path disagree.
|
||||
#[doc(hidden)]
|
||||
pub fn scratch_dir_canonical(tag: &str) -> PathBuf {
|
||||
std::fs::canonicalize(scratch_dir(tag)).expect("canonicalize scratch dir")
|
||||
}
|
||||
|
||||
/// Write `body` to `path`, creating parent directories as needed.
|
||||
#[doc(hidden)]
|
||||
pub fn touch(path: &std::path::Path, body: &[u8]) {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).expect("create parent dir");
|
||||
}
|
||||
std::fs::write(path, body).expect("write file");
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn every_call_gets_its_own_empty_directory() {
|
||||
let a = scratch_dir("selftest");
|
||||
let b = scratch_dir("selftest");
|
||||
assert_ne!(a, b, "two calls must not collide");
|
||||
for d in [&a, &b] {
|
||||
assert!(d.is_dir());
|
||||
assert_eq!(std::fs::read_dir(d).unwrap().count(), 0, "starts empty");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn touch_creates_missing_parents() {
|
||||
let dir = scratch_dir("selftest-touch");
|
||||
let deep = dir.join("a/b/c.txt");
|
||||
touch(&deep, b"hi");
|
||||
assert_eq!(std::fs::read(&deep).unwrap(), b"hi");
|
||||
}
|
||||
}
|
||||
|
|
@ -27,9 +27,9 @@
|
|||
use std::collections::HashSet;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{mpsc, Arc, Condvar, Mutex};
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::thread::JoinHandle;
|
||||
use std::time::UNIX_EPOCH;
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
|
@ -100,6 +100,11 @@ impl WalkedFile {
|
|||
|
||||
/// What the walk emits. Files as they are classified, plus the per-directory
|
||||
/// verdict on which index rows no longer have a file behind them.
|
||||
// `File` is far larger than the deletion variant, and deliberately so: this
|
||||
// is the walk's hot path, one event per file on a tree of millions. Boxing to
|
||||
// even out the variants would add exactly the per-file allocation the
|
||||
// owned-record design exists to avoid.
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
#[derive(Debug)]
|
||||
pub enum WalkEvent {
|
||||
File(WalkedFile),
|
||||
|
|
@ -109,6 +114,27 @@ pub enum WalkEvent {
|
|||
Stale(Vec<String>),
|
||||
}
|
||||
|
||||
/// One file a directory read produced: its path, plus whatever that read
|
||||
/// already told us about it.
|
||||
///
|
||||
/// The metadata half is `Some` only on Windows, where `FindNextFileW` returned
|
||||
/// size, mtime and attributes alongside the name. On Unix `getdents64` returns
|
||||
/// only `d_type`, so it is always `None`, [`prepare`] does the single `statx`
|
||||
/// the walk has always done, and `Option<CachedMetadata>` is zero-sized — a
|
||||
/// queued file costs exactly the `PathBuf` it cost before.
|
||||
struct PendingFile {
|
||||
path: PathBuf,
|
||||
cached: Option<crate::platform::CachedMetadata>,
|
||||
}
|
||||
|
||||
impl PendingFile {
|
||||
/// A path that did not come from a directory entry — a resolved symlink
|
||||
/// target — so there is nothing cached to carry.
|
||||
fn uncached(path: PathBuf) -> Self {
|
||||
PendingFile { path, cached: None }
|
||||
}
|
||||
}
|
||||
|
||||
/// Work waiting for a thread.
|
||||
enum Job {
|
||||
/// Read this directory and process its files. Carries the directory's
|
||||
|
|
@ -116,8 +142,10 @@ enum Job {
|
|||
Dir(PathBuf, Arc<DirRows>),
|
||||
/// Process this slice of one directory's files, split off because the
|
||||
/// directory was too wide for one worker to be worth serialising on.
|
||||
/// Shares its directory's rows.
|
||||
Files(Vec<PathBuf>, Arc<DirRows>),
|
||||
/// Shares its directory's rows. On Windows each entry also carries the
|
||||
/// metadata its directory read returned, so classifying these files costs
|
||||
/// no syscall at all.
|
||||
Files(Vec<PendingFile>, Arc<DirRows>),
|
||||
/// A resolved symlink target, with the stored mtime for its own path.
|
||||
/// Classified against that rather than against any directory's rows.
|
||||
Alias(PathBuf, Option<u64>),
|
||||
|
|
@ -151,6 +179,12 @@ struct Queue {
|
|||
/// directory pushes many of them. Bounding on the total would let file
|
||||
/// chunks starve directory prefetching and leave the pool waiting on
|
||||
/// rows that were never fetched.
|
||||
///
|
||||
/// The consequence is that file chunks, not this counter, are what bounds
|
||||
/// the walker's memory: nothing throttles them, so one very wide directory
|
||||
/// can hold its whole listing in `jobs` at once — and on Windows each of
|
||||
/// those entries also carries its directory read's metadata
|
||||
/// ([`PendingFile`]), so that is the larger cost of the two.
|
||||
dirs_ready: usize,
|
||||
/// Workers currently holding a job — that is, workers that may still push
|
||||
/// more. The walk is over when this is zero and `jobs` is empty.
|
||||
|
|
@ -342,7 +376,7 @@ enum Found {
|
|||
/// A resolved symlink target. Needs an exact-path mtime lookup.
|
||||
Alias(PathBuf),
|
||||
/// Overflow files from the directory just read, which already has rows.
|
||||
Files(Vec<PathBuf>, Arc<DirRows>),
|
||||
Files(Vec<PendingFile>, Arc<DirRows>),
|
||||
}
|
||||
|
||||
impl Shared {
|
||||
|
|
@ -398,10 +432,57 @@ impl Drop for ActiveJob<'_> {
|
|||
}
|
||||
}
|
||||
|
||||
/// How many entries each filter rejected, for the one-line summary a run logs
|
||||
/// when it finishes.
|
||||
///
|
||||
/// Counted rather than logged per entry, deliberately. The process log is a
|
||||
/// 5,000-line ring ([`crate::log::CAPACITY`]) that evicts oldest-first with no
|
||||
/// level protection, so a line per ignored entry would push every warning —
|
||||
/// including the unreadable-directory ones that distinguish a network blip from
|
||||
/// a deletion — out of the buffer before the run ended. `log::record` also takes
|
||||
/// a global mutex and writes to stderr, which would serialize the worker pool on
|
||||
/// the one path the walker is built to keep syscall-free.
|
||||
///
|
||||
/// A pruned *directory* is one increment, not one per file beneath it: the
|
||||
/// subtree is never enumerated.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct PruneCounts {
|
||||
/// Names beginning with a dot.
|
||||
pub dot_named: AtomicU64,
|
||||
/// Windows entries carrying `FILE_ATTRIBUTE_HIDDEN`.
|
||||
pub attribute: AtomicU64,
|
||||
/// Rejected by a configured ignore pattern, of either kind.
|
||||
pub ignored: AtomicU64,
|
||||
}
|
||||
|
||||
impl PruneCounts {
|
||||
pub fn total(&self) -> u64 {
|
||||
self.dot_named.load(Ordering::Relaxed)
|
||||
+ self.attribute.load(Ordering::Relaxed)
|
||||
+ self.ignored.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// The summary line, or `None` when nothing was pruned — a clean tree
|
||||
/// should not add noise to the log.
|
||||
pub fn summary(&self) -> Option<String> {
|
||||
if self.total() == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(format!(
|
||||
"pruned {} entries: {} hidden by attribute, {} dot-named, {} by ignore pattern",
|
||||
self.total(),
|
||||
self.attribute.load(Ordering::Relaxed),
|
||||
self.dot_named.load(Ordering::Relaxed),
|
||||
self.ignored.load(Ordering::Relaxed),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
struct Ctx {
|
||||
follow_symlinks: bool,
|
||||
include_hidden: bool,
|
||||
ignore: IgnoreSet,
|
||||
pruned: PruneCounts,
|
||||
config: Config,
|
||||
/// Lets a worker finish small text files outright: the head it reads to
|
||||
/// hash them is already their entire contents, so an extractor that works
|
||||
|
|
@ -412,6 +493,16 @@ struct Ctx {
|
|||
suspend_flag: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
/// Individual unreadable-directory warnings allowed per run before only the
|
||||
/// count is kept. Reset by [`reset_run_warnings`].
|
||||
static UNREADABLE_WARNINGS: crate::log::Throttle = crate::log::Throttle::new(20);
|
||||
|
||||
/// Arm this module's per-run warning throttle. See
|
||||
/// [`crate::file_handling::reset_run_warnings`], which the same caller invokes.
|
||||
pub fn reset_run_warnings() {
|
||||
UNREADABLE_WARNINGS.reset();
|
||||
}
|
||||
|
||||
/// Read one directory, apply the hidden/ignore rules, and split the result:
|
||||
/// subdirectories and overflow file chunks go to `found` for the pool, the
|
||||
/// remaining files come back for this worker to handle immediately.
|
||||
|
|
@ -425,18 +516,27 @@ struct Ctx {
|
|||
/// A directory that cannot be read returns before reconciling, so nothing
|
||||
/// under it is ever deleted — an unreadable directory must not read as an
|
||||
/// empty one.
|
||||
///
|
||||
/// Each returned file carries whatever this read already told us about it, so
|
||||
/// that [`prepare`] does not have to ask the filesystem twice. See
|
||||
/// [`PendingFile`].
|
||||
fn read_directory(
|
||||
dir: &Path,
|
||||
rows: &Arc<DirRows>,
|
||||
ctx: &Ctx,
|
||||
found: &mut Vec<Found>,
|
||||
stale: &mut Vec<String>,
|
||||
) -> Vec<PathBuf> {
|
||||
) -> Vec<PendingFile> {
|
||||
let entries = match fs::read_dir(dir) {
|
||||
Ok(entries) => entries,
|
||||
Err(e) => {
|
||||
// Not the same as "this directory is empty": see UnreadableDirs.
|
||||
// Throttled because a permission-denied subtree produces one of
|
||||
// these per directory; `UnreadableDirs` records every one of them
|
||||
// regardless, and the run reports the total.
|
||||
if UNREADABLE_WARNINGS.allow() {
|
||||
crate::log_warn!("cannot read {}: {}", dir.display(), e);
|
||||
}
|
||||
ctx.unreadable.record(dir.to_path_buf());
|
||||
return Vec::new();
|
||||
}
|
||||
|
|
@ -452,7 +552,9 @@ fn read_directory(
|
|||
let entry = match entry {
|
||||
Ok(entry) => entry,
|
||||
Err(e) => {
|
||||
if UNREADABLE_WARNINGS.allow() {
|
||||
crate::log_warn!("cannot read an entry of {}: {}", dir.display(), e);
|
||||
}
|
||||
ctx.unreadable.record(dir.to_path_buf());
|
||||
// The listing is now incomplete, so it cannot be used to
|
||||
// decide what is missing: an entry we failed to read would
|
||||
|
|
@ -465,17 +567,45 @@ fn read_directory(
|
|||
let name = entry.file_name();
|
||||
let name = name.to_string_lossy();
|
||||
// `entry.metadata()` is only consulted on Windows, where it is free —
|
||||
// the attributes came back with the directory read. On Unix the
|
||||
// closure is never called, so this stays at zero extra syscalls.
|
||||
if !ctx.include_hidden && crate::platform::entry_is_hidden(&name, || entry.metadata().ok())
|
||||
// the attributes came back with the directory read, and it reports the
|
||||
// entry itself rather than a link target, which is what
|
||||
// `entry_hidden_reason` requires. On Unix the closure is never called,
|
||||
// so this stays at zero extra syscalls.
|
||||
if !ctx.include_hidden {
|
||||
if let Some(reason) =
|
||||
crate::platform::entry_hidden_reason(&name, || entry.metadata().ok())
|
||||
{
|
||||
match reason {
|
||||
crate::platform::HiddenReason::DotPrefix => {
|
||||
ctx.pruned.dot_named.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
crate::platform::HiddenReason::Attribute => {
|
||||
ctx.pruned.attribute.fetch_add(1, Ordering::Relaxed);
|
||||
// Only the attribute case is announced, and only for a
|
||||
// directory. A dot prefix explains itself and an ignore
|
||||
// pattern is something the user typed, but a plainly
|
||||
// visible folder skipped over an attribute Explorer does
|
||||
// not show has no other way of being discovered — which
|
||||
// is how a whole cloud-sync tree went missing in silence.
|
||||
if entry.file_type().is_ok_and(|ft| ft.is_dir()) {
|
||||
crate::log_info!(
|
||||
"skipping {}: hidden attribute set (enable \"include hidden \
|
||||
files\" to index it)",
|
||||
entry.path().display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if ctx.ignore.matches_component(&name) {
|
||||
ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed);
|
||||
continue;
|
||||
}
|
||||
let path = entry.path();
|
||||
if ctx.ignore.matches_path_pattern(&path) {
|
||||
ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -507,12 +637,12 @@ fn read_directory(
|
|||
// path is what the index stores, and pushing only canonical
|
||||
// directories is what keeps `seen_dirs` able to break cycles.
|
||||
//
|
||||
// Normalized like the roots (walk_parallel), or on Windows the
|
||||
// target keeps `canonicalize`'s `\\?\` prefix: every path below
|
||||
// it would be spelled differently from the plainly-spelled
|
||||
// roots, so full-path ignore patterns would never match under a
|
||||
// followed junction and `seen_dirs` could not dedup against an
|
||||
// overlapping root.
|
||||
// Normalized like the roots (`walk_indexable_files`), or on
|
||||
// Windows the target keeps `canonicalize`'s `\\?\` prefix:
|
||||
// every path below it would be spelled differently from the
|
||||
// plainly-spelled roots, so full-path ignore patterns would
|
||||
// never match under a followed junction and `seen_dirs` could
|
||||
// not dedup against an overlapping root.
|
||||
if let Ok(target) = path.canonicalize() {
|
||||
let target = PathBuf::from(path_to_db_string(&target));
|
||||
match fs::metadata(&target) {
|
||||
|
|
@ -528,7 +658,13 @@ fn read_directory(
|
|||
}
|
||||
Ok(_) => {
|
||||
present.insert(name.into_owned());
|
||||
files.push(path);
|
||||
// Windows already sent this file's size and mtime back with its
|
||||
// name, and `DirEntry::metadata` there is a copy of that buffer
|
||||
// rather than a syscall — so carrying it means `prepare` never
|
||||
// opens the file just to ask the same question again. `None` on
|
||||
// Unix, and on any reparse point: see `entry_cached_metadata`.
|
||||
let cached = crate::platform::entry_cached_metadata(|| entry.metadata().ok());
|
||||
files.push(PendingFile { path, cached });
|
||||
}
|
||||
// Type unknown: the entry exists but we could not classify it.
|
||||
// Mark it present so an existing row survives — seen, not deleted.
|
||||
|
|
@ -583,9 +719,14 @@ pub fn path_digest(path: &str) -> u128 {
|
|||
u128::from_be_bytes(bytes)
|
||||
}
|
||||
|
||||
/// One `stat`, then classify; only files that are actually going to be
|
||||
/// At most one `stat`, then classify; only files that are actually going to be
|
||||
/// written get opened, and small text files are finished outright.
|
||||
fn prepare(path: PathBuf, known: Known<'_>, ctx: &Ctx) -> WalkedFile {
|
||||
///
|
||||
/// "At most", because on Windows the directory read already answered the
|
||||
/// question and [`PendingFile::cached`] carries the answer — see
|
||||
/// [`crate::platform::metadata_or_stat`].
|
||||
fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile {
|
||||
let PendingFile { path, cached } = file;
|
||||
let db_path = path_to_db_string(&path);
|
||||
let digest = path_digest(&db_path);
|
||||
let aliased = matches!(known, Known::Exact(_));
|
||||
|
|
@ -598,7 +739,11 @@ fn prepare(path: PathBuf, known: Known<'_>, ctx: &Ctx) -> WalkedFile {
|
|||
return WalkedFile::skipped(db_path, digest, aliased);
|
||||
}
|
||||
|
||||
let Ok(meta) = fs::metadata(&path) else {
|
||||
// On Windows this is the directory read's own copy of the entry: an
|
||||
// unchanged file now costs no syscall at all, and a changed one costs only
|
||||
// the open the hasher was going to do anyway. On Unix, and for every
|
||||
// reparse point, it is the same single `stat` as before.
|
||||
let Ok(meta) = crate::platform::metadata_or_stat(&path, cached) else {
|
||||
// Seen but unreadable. Emitting it anyway keeps its index row alive:
|
||||
// a transient stat failure must not read as "deleted".
|
||||
return WalkedFile::skipped(db_path, digest, aliased);
|
||||
|
|
@ -663,8 +808,11 @@ fn worker(shared: &Shared, ctx: &Ctx, tx: &mpsc::SyncSender<WalkEvent>) {
|
|||
Job::Files(files, rows) => (files, rows),
|
||||
Job::Alias(path, stored) => {
|
||||
slot.finish(found);
|
||||
// Reached through `canonicalize`, not through a directory
|
||||
// entry, so there is nothing cached to carry.
|
||||
let file = PendingFile::uncached(path);
|
||||
if tx
|
||||
.send(WalkEvent::File(prepare(path, Known::Exact(stored), ctx)))
|
||||
.send(WalkEvent::File(prepare(file, Known::Exact(stored), ctx)))
|
||||
.is_err()
|
||||
{
|
||||
shared.shutdown();
|
||||
|
|
@ -684,13 +832,13 @@ fn worker(shared: &Shared, ctx: &Ctx, tx: &mpsc::SyncSender<WalkEvent>) {
|
|||
return;
|
||||
}
|
||||
|
||||
for path in files {
|
||||
for file in files {
|
||||
if should_abort(&ctx.stop_flag, &ctx.suspend_flag) {
|
||||
shared.shutdown();
|
||||
return;
|
||||
}
|
||||
if tx
|
||||
.send(WalkEvent::File(prepare(path, Known::InDir(&rows), ctx)))
|
||||
.send(WalkEvent::File(prepare(file, Known::InDir(&rows), ctx)))
|
||||
.is_err()
|
||||
{
|
||||
// Receiver gone: the run was stopped or failed. Not an error.
|
||||
|
|
@ -705,7 +853,8 @@ fn worker(shared: &Shared, ctx: &Ctx, tx: &mpsc::SyncSender<WalkEvent>) {
|
|||
/// read-only connection.
|
||||
///
|
||||
/// One per walk. The alternative — a connection per worker — would multiply
|
||||
/// SQLite's page cache by the pool size; see [`PRAGMAS_WALK_READER`]. Every
|
||||
/// SQLite's page cache by the pool size; see
|
||||
/// [`crate::db::schema::PRAGMAS_WALK_READER`]. Every
|
||||
/// query here is a single index lookup, so one thread stays far ahead of a
|
||||
/// pool bound by `stat` latency.
|
||||
///
|
||||
|
|
@ -753,6 +902,9 @@ fn prefetcher(shared: &Shared, db_path: &str) {
|
|||
/// stops the workers and joins them.
|
||||
pub struct ParallelWalk {
|
||||
rx: Option<mpsc::Receiver<WalkEvent>>,
|
||||
/// One event pulled off the channel by [`ParallelWalk::wait_ready`] and not
|
||||
/// yet handed to [`ParallelWalk::try_next`]. See `wait_ready`.
|
||||
pending: Option<WalkEvent>,
|
||||
handles: Vec<JoinHandle<()>>,
|
||||
/// Joined by [`ParallelWalk::finish`] alongside the workers. Held
|
||||
/// separately only so a failure to open its connection is attributable.
|
||||
|
|
@ -768,6 +920,12 @@ impl ParallelWalk {
|
|||
&self.ctx.unreadable
|
||||
}
|
||||
|
||||
/// How many entries each filter rejected. Final on the same terms as
|
||||
/// [`ParallelWalk::unreadable`].
|
||||
pub fn pruned(&self) -> &PruneCounts {
|
||||
&self.ctx.pruned
|
||||
}
|
||||
|
||||
/// Every canonical directory the walk queued, in `files.parent` spelling.
|
||||
///
|
||||
/// The caller's vanished-directory sweep needs this: a directory deleted
|
||||
|
|
@ -802,7 +960,10 @@ impl ParallelWalk {
|
|||
/// identical from the receiving end: both close the channel, so iteration
|
||||
/// simply ends. Treating a panicked walk as a completed one would hand
|
||||
/// stale cleanup a partial file set and delete everything the dead workers
|
||||
/// never reached.
|
||||
/// never reached. So: join before deciding anything about what the walk
|
||||
/// saw. This is the reference statement of that rule; the content pass
|
||||
/// ([`crate::content::ContentPass::finish`]) and the writer loop's
|
||||
/// `TryNext::Finished` arms follow it.
|
||||
pub fn finish(&mut self) -> bool {
|
||||
// Dropping the receiver first releases any worker parked in `send`.
|
||||
self.rx = None;
|
||||
|
|
@ -852,18 +1013,73 @@ pub(crate) fn try_recv_next<T>(rx: Option<&mpsc::Receiver<T>>) -> TryNext<T> {
|
|||
}
|
||||
}
|
||||
|
||||
/// [`try_recv_next`], but willing to wait up to `timeout` for something to
|
||||
/// arrive.
|
||||
///
|
||||
/// What the writer loop backs off with instead of `thread::sleep`. A sleep is
|
||||
/// the wrong instrument twice over: it ignores work that lands a microsecond
|
||||
/// later, and on Windows the default timer resolution is 15.6 ms, so a 2 ms
|
||||
/// backoff actually stalls for 15.6 — nearly eight times the intended pause,
|
||||
/// every time the channels run momentarily dry. `recv_timeout` parks on the
|
||||
/// channel's own condition variable, so a sender wakes it immediately and the
|
||||
/// timeout is only the ceiling.
|
||||
pub(crate) fn recv_next_timeout<T>(
|
||||
rx: Option<&mpsc::Receiver<T>>,
|
||||
timeout: std::time::Duration,
|
||||
) -> TryNext<T> {
|
||||
match rx {
|
||||
None => TryNext::Finished,
|
||||
Some(rx) => match rx.recv_timeout(timeout) {
|
||||
Ok(item) => TryNext::Item(item),
|
||||
Err(mpsc::RecvTimeoutError::Timeout) => TryNext::Empty,
|
||||
Err(mpsc::RecvTimeoutError::Disconnected) => TryNext::Finished,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
impl ParallelWalk {
|
||||
/// Non-blocking variant of `next`, for callers multiplexing several
|
||||
/// walks (the per-root writer loop).
|
||||
pub fn try_next(&mut self) -> TryNext<WalkEvent> {
|
||||
if let Some(event) = self.pending.take() {
|
||||
return TryNext::Item(event);
|
||||
}
|
||||
try_recv_next(self.rx.as_ref())
|
||||
}
|
||||
|
||||
/// Wait up to `timeout` for this walk to produce something, holding
|
||||
/// whatever arrives for the next [`ParallelWalk::try_next`].
|
||||
///
|
||||
/// The writer loop's idle backoff. It multiplexes several walks, so it
|
||||
/// cannot simply block on one of them and consume the result — hence the
|
||||
/// one-slot pushback: the event is taken off the channel, but the loop still
|
||||
/// sees it in its normal round-robin order.
|
||||
///
|
||||
/// Returns whether anything is now ready.
|
||||
pub fn wait_ready(&mut self, timeout: std::time::Duration) -> bool {
|
||||
if self.pending.is_some() {
|
||||
return true;
|
||||
}
|
||||
match recv_next_timeout(self.rx.as_ref(), timeout) {
|
||||
TryNext::Item(event) => {
|
||||
self.pending = Some(event);
|
||||
true
|
||||
}
|
||||
// A finished walk is "ready" in the sense the caller cares about:
|
||||
// there is something to do (notice it ended), so do not keep waiting.
|
||||
TryNext::Finished => true,
|
||||
TryNext::Empty => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Iterator for ParallelWalk {
|
||||
type Item = WalkEvent;
|
||||
|
||||
fn next(&mut self) -> Option<WalkEvent> {
|
||||
if let Some(event) = self.pending.take() {
|
||||
return Some(event);
|
||||
}
|
||||
self.rx.as_ref()?.recv().ok()
|
||||
}
|
||||
}
|
||||
|
|
@ -938,6 +1154,7 @@ pub fn walk_indexable_files(
|
|||
follow_symlinks,
|
||||
include_hidden,
|
||||
ignore,
|
||||
pruned: PruneCounts::default(),
|
||||
config,
|
||||
registry,
|
||||
unreadable: UnreadableDirs::default(),
|
||||
|
|
@ -953,7 +1170,7 @@ pub fn walk_indexable_files(
|
|||
let handles = (0..threads)
|
||||
.map(|_| {
|
||||
let (shared, ctx, tx) = (shared.clone(), ctx.clone(), tx.clone());
|
||||
thread::spawn(move || {
|
||||
crate::platform::spawn_worker("qs-walk", move || {
|
||||
// Walker threads are the bulk of a run's CPU and I/O; the
|
||||
// foreground must stay ahead of them.
|
||||
crate::platform::set_background_priority();
|
||||
|
|
@ -968,7 +1185,7 @@ pub fn walk_indexable_files(
|
|||
|
||||
let prefetch = {
|
||||
let (shared, db_path) = (shared.clone(), db_path.to_string());
|
||||
thread::spawn(move || {
|
||||
crate::platform::spawn_worker("qs-prefetch", move || {
|
||||
crate::platform::set_background_priority();
|
||||
prefetcher(&shared, &db_path)
|
||||
})
|
||||
|
|
@ -976,6 +1193,7 @@ pub fn walk_indexable_files(
|
|||
|
||||
ParallelWalk {
|
||||
rx: Some(rx),
|
||||
pending: None,
|
||||
handles,
|
||||
prefetch: Some(prefetch),
|
||||
shared,
|
||||
|
|
@ -1007,18 +1225,7 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
fn tmp_tree(tag: &str) -> PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"quicksearch-pwalk-{}-{}-{}",
|
||||
tag,
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
fs::create_dir_all(&p).unwrap();
|
||||
p
|
||||
crate::testutil::scratch_dir(tag)
|
||||
}
|
||||
|
||||
fn touch(p: &Path) {
|
||||
|
|
@ -1032,16 +1239,7 @@ mod tests {
|
|||
/// caller passes in, so these tests build the state they are testing
|
||||
/// against the same way the indexer does.
|
||||
fn db_with(tag: &str, rows: &[(String, u64)]) -> PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"quicksearch-pwalk-db-{}-{}-{}.sqlite",
|
||||
tag,
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
let p = crate::testutil::scratch_dir(tag).join("index.sqlite");
|
||||
let conn = crate::db::open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
|
||||
for (path, mtime) in rows {
|
||||
let as_path = Path::new(path);
|
||||
|
|
@ -1234,6 +1432,74 @@ mod tests {
|
|||
fs::remove_dir_all(&root).ok();
|
||||
}
|
||||
|
||||
/// The walk's mtime and a `stat`'s mtime must be the same number.
|
||||
///
|
||||
/// On Windows the walk now reads mtime out of the directory entry while the
|
||||
/// *watcher* writes its rows from `fs::metadata`; if the two ever disagreed,
|
||||
/// every run would reclassify files nothing had touched and the index would
|
||||
/// churn forever. Seeding the index the watcher's way and demanding the walk
|
||||
/// call every file `Skip` is what pins them together.
|
||||
///
|
||||
/// `unchanged_files_are_never_opened` cannot catch this on its own: both of
|
||||
/// its walks read from the same source, so they agree with each other even
|
||||
/// when both disagree with a stat. Vacuous on Unix, where there is only ever
|
||||
/// one source; on Windows it is the whole guarantee.
|
||||
#[test]
|
||||
fn a_walk_agrees_with_a_stat_seeded_index() {
|
||||
let root = tmp_tree("stat-seeded");
|
||||
touch(&root.join("a.txt"));
|
||||
touch(&root.join("sub/b.txt"));
|
||||
|
||||
let seeded: Vec<(String, u64)> = [root.join("a.txt"), root.join("sub/b.txt")]
|
||||
.iter()
|
||||
.map(|p| {
|
||||
let mtime = fs::metadata(p)
|
||||
.unwrap()
|
||||
.modified()
|
||||
.unwrap()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs();
|
||||
(path_to_db_string(p), mtime)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let files = walk(&root, &db_with("stat-seeded", &seeded));
|
||||
assert_eq!(files.len(), 2);
|
||||
for f in &files {
|
||||
assert_eq!(
|
||||
f.action,
|
||||
FileIndexAction::Skip,
|
||||
"the directory read disagreed with a stat about {}",
|
||||
f.path
|
||||
);
|
||||
assert!(f.record.is_none());
|
||||
}
|
||||
fs::remove_dir_all(&root).ok();
|
||||
}
|
||||
|
||||
/// Windows: the three fields `prepare` and `prepare_file_record` read out of
|
||||
/// the cached buffer, pinned against what a `stat` would have said — for the
|
||||
/// case that now skips the `stat` entirely.
|
||||
#[test]
|
||||
#[cfg(windows)]
|
||||
fn cached_directory_metadata_matches_a_stat_field_for_field() {
|
||||
let root = tmp_tree("cached-meta");
|
||||
touch(&root.join("a.txt"));
|
||||
fs::write(root.join("b.bin"), vec![0u8; 5000]).unwrap();
|
||||
|
||||
for entry in fs::read_dir(&root).unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
let cached = crate::platform::entry_cached_metadata(|| entry.metadata().ok())
|
||||
.expect("a plain file is served from the directory read");
|
||||
let fresh = fs::metadata(entry.path()).unwrap();
|
||||
assert_eq!(cached.is_file(), fresh.is_file());
|
||||
assert_eq!(cached.len(), fresh.len());
|
||||
assert_eq!(cached.modified().unwrap(), fresh.modified().unwrap());
|
||||
}
|
||||
fs::remove_dir_all(&root).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_seen_file_is_reported_even_when_it_cannot_be_read() {
|
||||
// A path missing from the stream gets its index row deleted, so
|
||||
|
|
@ -1472,6 +1738,103 @@ mod tests {
|
|||
fs::remove_dir_all(&root).ok();
|
||||
}
|
||||
|
||||
/// The counters behind the one-line summary a run logs.
|
||||
///
|
||||
/// The property worth pinning is that a pruned *directory* costs one
|
||||
/// increment rather than one per file beneath it — the subtree is never
|
||||
/// enumerated, which is exactly why logging per entry was rejected and
|
||||
/// counting was not.
|
||||
#[test]
|
||||
fn pruned_entries_are_counted_by_reason() {
|
||||
let root = tmp_tree("prune-counts");
|
||||
touch(&root.join("keep.txt"));
|
||||
touch(&root.join("sub/keep2.txt"));
|
||||
touch(&root.join("sub/skip.tmp"));
|
||||
// Two files below, one prune.
|
||||
touch(&root.join(".hidden/inside.txt"));
|
||||
touch(&root.join(".hidden/also-inside.txt"));
|
||||
touch(&root.join(".dotfile"));
|
||||
// Three levels below, still one prune.
|
||||
touch(&root.join("node_modules/dep/lib/index.js"));
|
||||
|
||||
let ignore =
|
||||
IgnoreSet::compile(&["*.tmp".to_string(), "node_modules".to_string()]).unwrap();
|
||||
let mut walk = walk_indexable_files(
|
||||
&[root.to_string_lossy().into_owned()],
|
||||
false,
|
||||
false,
|
||||
ignore,
|
||||
empty_db("prune-counts").to_str().unwrap(),
|
||||
Config::default(),
|
||||
Arc::new(Registry::default_set()),
|
||||
Arc::new(AtomicBool::new(false)),
|
||||
Arc::new(AtomicBool::new(false)),
|
||||
4,
|
||||
);
|
||||
let files: Vec<WalkedFile> = (&mut walk)
|
||||
.filter_map(|e| match e {
|
||||
WalkEvent::File(f) => Some(f),
|
||||
WalkEvent::Stale(_) => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(names(&files), vec!["keep.txt", "keep2.txt"]);
|
||||
|
||||
let pruned = walk.pruned();
|
||||
assert_eq!(
|
||||
pruned.dot_named.load(Ordering::Relaxed),
|
||||
2,
|
||||
"`.hidden` and `.dotfile` — not the two files inside `.hidden`"
|
||||
);
|
||||
assert_eq!(
|
||||
pruned.ignored.load(Ordering::Relaxed),
|
||||
2,
|
||||
"`skip.tmp` and `node_modules` — not `index.js` three levels down"
|
||||
);
|
||||
// Attributes are a Windows concept; on Linux nothing can reach this
|
||||
// counter, and on Windows a temp tree carries no Hidden bit.
|
||||
assert_eq!(pruned.attribute.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(pruned.total(), 4);
|
||||
|
||||
let summary = pruned.summary().expect("something was pruned");
|
||||
assert!(summary.contains("4 entries"), "{}", summary);
|
||||
|
||||
fs::remove_dir_all(&root).ok();
|
||||
}
|
||||
|
||||
/// A clean tree must add no line to a log whose whole budget is 5,000
|
||||
/// entries.
|
||||
#[test]
|
||||
fn a_tree_with_nothing_pruned_reports_no_summary() {
|
||||
let root = tmp_tree("prune-none");
|
||||
touch(&root.join("keep.txt"));
|
||||
touch(&root.join("sub/keep2.txt"));
|
||||
|
||||
let mut walk = walk_indexable_files(
|
||||
&[root.to_string_lossy().into_owned()],
|
||||
false,
|
||||
false,
|
||||
IgnoreSet::compile(&[]).unwrap(),
|
||||
empty_db("prune-none").to_str().unwrap(),
|
||||
Config::default(),
|
||||
Arc::new(Registry::default_set()),
|
||||
Arc::new(AtomicBool::new(false)),
|
||||
Arc::new(AtomicBool::new(false)),
|
||||
4,
|
||||
);
|
||||
let files: Vec<WalkedFile> = (&mut walk)
|
||||
.filter_map(|e| match e {
|
||||
WalkEvent::File(f) => Some(f),
|
||||
WalkEvent::Stale(_) => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(names(&files), vec!["keep.txt", "keep2.txt"]);
|
||||
|
||||
assert_eq!(walk.pruned().total(), 0);
|
||||
assert!(walk.pruned().summary().is_none());
|
||||
|
||||
fs::remove_dir_all(&root).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_directory_reports_rows_with_no_file_behind_them() {
|
||||
// The per-directory diff, at the level it is computed: one listing
|
||||
|
|
|
|||
|
|
@ -160,7 +160,7 @@ impl std::error::Error for WatchError {}
|
|||
|
||||
/// Render the directory cap compactly: the default 128_000 reads "128k".
|
||||
fn fmt_cap(cap: usize) -> String {
|
||||
if cap >= 1000 && cap % 1000 == 0 {
|
||||
if cap >= 1000 && cap.is_multiple_of(1000) {
|
||||
format!("{}k", cap / 1000)
|
||||
} else {
|
||||
cap.to_string()
|
||||
|
|
@ -266,12 +266,12 @@ impl WatchRegistry {
|
|||
}
|
||||
|
||||
/// Forget `dir` and every watched directory beneath it, returning how
|
||||
/// many were dropped.
|
||||
/// many were dropped. Containment is component-wise, per
|
||||
/// [`crate::file_handling::UnreadableDirs::covers`].
|
||||
///
|
||||
/// `Path::starts_with` compares whole components, so `/a/bc` is not
|
||||
/// treated as living under `/a/b`. The kernel drops watches for deleted
|
||||
/// directories on its own; unwatching anyway keeps notify's internal
|
||||
/// descriptor map from growing across a long session of directory churn.
|
||||
/// The kernel drops watches for deleted directories on its own; unwatching
|
||||
/// anyway keeps notify's internal descriptor map from growing across a long
|
||||
/// session of directory churn.
|
||||
fn remove_tree(&mut self, dir: &Path) -> usize {
|
||||
// The scan below is O(watched dirs), and the event loop calls this for
|
||||
// every Remove — files included. Deleting a directory of 10k files
|
||||
|
|
@ -564,7 +564,7 @@ fn run_loop(rx: mpsc::Receiver<NotifyEvent>, ctx: LoopCtx) {
|
|||
|
||||
// Periodic GC of abandoned throttle entries.
|
||||
tick_counter = tick_counter.wrapping_add(1);
|
||||
if tick_counter % prune_interval_ticks == 0 {
|
||||
if tick_counter.is_multiple_of(prune_interval_ticks) {
|
||||
let max_age = ctx
|
||||
.config
|
||||
.throttle_window
|
||||
|
|
@ -775,6 +775,13 @@ fn prune_stale(throttle: &mut HashMap<PathBuf, DirThrottleEntry>, max_age: Durat
|
|||
|| entry.immediate
|
||||
|| now.saturating_duration_since(entry.record_time) < max_age
|
||||
});
|
||||
// `retain` frees the entries but keeps the table sized for the busiest
|
||||
// moment this map has ever seen — and a build or an unpack can put tens of
|
||||
// thousands of directories through it in a burst, once, on a process that
|
||||
// then sits idle for days.
|
||||
if throttle.is_empty() {
|
||||
throttle.shrink_to_fit();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -809,19 +816,8 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
/// Unique temp directory; the repo has no `tempfile` dev-dependency.
|
||||
fn tmp_dir(tag: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!(
|
||||
"qs-watch-{}-{}-{}",
|
||||
tag,
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir
|
||||
crate::testutil::scratch_dir(tag)
|
||||
}
|
||||
|
||||
/// A registry with `dirs` seeded directly, so the pure set logic can be
|
||||
|
|
@ -903,8 +899,10 @@ mod tests {
|
|||
);
|
||||
}
|
||||
let (sink, got) = sink_to_vec();
|
||||
let mut config = WatcherConfig::default();
|
||||
config.max_dirs_per_tick = 3;
|
||||
let config = WatcherConfig {
|
||||
max_dirs_per_tick: 3,
|
||||
..WatcherConfig::default()
|
||||
};
|
||||
flush_ready(&mut map, &sink, &config);
|
||||
// Each dir contributes one event because each entry has one path.
|
||||
assert_eq!(got.lock().unwrap().len(), 3);
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
//! Integration tests for the ranked search cascade and the streaming
|
||||
//! search service, against real temp databases.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
|
|
@ -11,19 +11,8 @@ use quicksearch_core::mime::FileType;
|
|||
use quicksearch_core::query::split::split_for_cascade;
|
||||
use quicksearch_core::search::{cascade, SearchHit, SearchOptions, SearchService, SearchUpdate};
|
||||
|
||||
fn tmp_db(tag: &str) -> PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"qs-cascade-{}-{}-{}.sqlite",
|
||||
tag,
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
p
|
||||
}
|
||||
mod common;
|
||||
use common::scratch_db as tmp_db;
|
||||
|
||||
struct Seeder {
|
||||
conn: rusqlite::Connection,
|
||||
|
|
@ -31,7 +20,7 @@ struct Seeder {
|
|||
}
|
||||
|
||||
impl Seeder {
|
||||
fn new(path: &PathBuf, store_text: bool) -> Seeder {
|
||||
fn new(path: &Path, store_text: bool) -> Seeder {
|
||||
Seeder {
|
||||
conn: open_or_recreate(path.to_str().unwrap(), "trigram").unwrap(),
|
||||
store_text,
|
||||
|
|
@ -1083,6 +1072,213 @@ fn service_reports_missing_db_as_error() {
|
|||
service.shutdown();
|
||||
}
|
||||
|
||||
/// Collect the names one search returns, or the error it produced.
|
||||
///
|
||||
/// The worker now keeps its connection between requests, so every test below
|
||||
/// runs *several* searches against one service — which is the situation the
|
||||
/// reuse has to survive, and the reason these are written against the service
|
||||
/// rather than against `cascade::run`.
|
||||
fn search_names(
|
||||
service: &SearchService,
|
||||
updates: &std::sync::mpsc::Receiver<SearchUpdate>,
|
||||
query: &str,
|
||||
) -> Result<Vec<String>, String> {
|
||||
let generation = service.search(query, SearchOptions::default());
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
|
||||
let mut names = Vec::new();
|
||||
while std::time::Instant::now() < deadline {
|
||||
match updates.recv_timeout(std::time::Duration::from_millis(200)) {
|
||||
Ok(SearchUpdate::Hits { generation: g, hits }) if g == generation => {
|
||||
names.extend(hits.into_iter().map(|h| h.name));
|
||||
}
|
||||
Ok(SearchUpdate::Completed { generation: g, .. }) if g == generation => {
|
||||
names.sort();
|
||||
return Ok(names);
|
||||
}
|
||||
Ok(SearchUpdate::Error {
|
||||
generation: g,
|
||||
message,
|
||||
}) if g == generation => return Err(message),
|
||||
Ok(_) => {}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
panic!("search {:?} never completed", query);
|
||||
}
|
||||
|
||||
/// The connection is reused across requests, so a service that has already
|
||||
/// searched must keep answering correctly rather than serving whatever its
|
||||
/// first query happened to warm.
|
||||
#[test]
|
||||
fn repeated_searches_on_one_service_stay_correct() {
|
||||
let p = tmp_db("warmrepeat");
|
||||
let mut s = Seeder::new(&p, true);
|
||||
s.add("alpha.txt", "/d", 1, Some("alpha body"));
|
||||
s.add("beta.txt", "/d", 1, Some("beta body"));
|
||||
drop(s.done());
|
||||
|
||||
let (service, updates) = SearchService::new(p.clone(), Arc::new(|| {}));
|
||||
for _ in 0..3 {
|
||||
assert_eq!(
|
||||
search_names(&service, &updates, "alpha").unwrap(),
|
||||
vec!["alpha.txt"]
|
||||
);
|
||||
assert_eq!(
|
||||
search_names(&service, &updates, "beta").unwrap(),
|
||||
vec!["beta.txt"]
|
||||
);
|
||||
}
|
||||
service.shutdown();
|
||||
|
||||
std::fs::remove_file(&p).ok();
|
||||
}
|
||||
|
||||
/// The hazard the old per-request open made impossible, and the reason
|
||||
/// `db::index_epoch` exists: a rebuild replaces the file at the *same path*,
|
||||
/// so nothing about the path tells a held connection that it is now looking at
|
||||
/// a deleted inode.
|
||||
#[test]
|
||||
fn a_rebuilt_index_at_the_same_path_is_picked_up() {
|
||||
let p = tmp_db("warmrebuild");
|
||||
let mut s = Seeder::new(&p, true);
|
||||
s.add("before.txt", "/d", 1, Some("shared body"));
|
||||
drop(s.done());
|
||||
|
||||
let (service, updates) = SearchService::new(p.clone(), Arc::new(|| {}));
|
||||
assert_eq!(
|
||||
search_names(&service, &updates, "shared").unwrap(),
|
||||
vec!["before.txt"],
|
||||
"the pre-rebuild index answers first"
|
||||
);
|
||||
|
||||
// A rebuild, spelled the way the coordinator spells one: delete the file
|
||||
// and recreate it. `open_or_recreate` on the now-missing path takes its
|
||||
// wipe branch, which is the real call site that bumps the epoch — so this
|
||||
// exercises the actual invalidation rather than poking the counter.
|
||||
for suffix in ["", "-wal", "-shm"] {
|
||||
std::fs::remove_file(format!("{}{}", p.display(), suffix)).ok();
|
||||
}
|
||||
let mut s = Seeder::new(&p, true);
|
||||
s.add("after.txt", "/d", 1, Some("shared body"));
|
||||
drop(s.done());
|
||||
|
||||
assert_eq!(
|
||||
search_names(&service, &updates, "shared").unwrap(),
|
||||
vec!["after.txt"],
|
||||
"a held connection served the replaced index"
|
||||
);
|
||||
service.shutdown();
|
||||
|
||||
std::fs::remove_file(&p).ok();
|
||||
}
|
||||
|
||||
/// The other half of invalidation: the config points somewhere else entirely.
|
||||
#[test]
|
||||
fn set_db_path_repoints_a_held_connection() {
|
||||
let first = tmp_db("warmpath1");
|
||||
let mut s = Seeder::new(&first, true);
|
||||
s.add("infirst.txt", "/d", 1, Some("shared body"));
|
||||
drop(s.done());
|
||||
|
||||
let second = tmp_db("warmpath2");
|
||||
let mut s = Seeder::new(&second, true);
|
||||
s.add("insecond.txt", "/d", 1, Some("shared body"));
|
||||
drop(s.done());
|
||||
|
||||
let (service, updates) = SearchService::new(first.clone(), Arc::new(|| {}));
|
||||
assert_eq!(
|
||||
search_names(&service, &updates, "shared").unwrap(),
|
||||
vec!["infirst.txt"]
|
||||
);
|
||||
|
||||
service.set_db_path(second.clone());
|
||||
assert_eq!(
|
||||
search_names(&service, &updates, "shared").unwrap(),
|
||||
vec!["insecond.txt"],
|
||||
"the connection stayed on the old index after the path moved"
|
||||
);
|
||||
service.shutdown();
|
||||
|
||||
std::fs::remove_file(&first).ok();
|
||||
std::fs::remove_file(&second).ok();
|
||||
}
|
||||
|
||||
/// A failed search drops the connection rather than putting it back, so one
|
||||
/// bad handle cannot wedge every search that follows it.
|
||||
#[test]
|
||||
fn a_failed_search_does_not_wedge_the_next_one() {
|
||||
let p = tmp_db("warmrecover");
|
||||
let (service, updates) = SearchService::new(p.clone(), Arc::new(|| {}));
|
||||
|
||||
search_names(&service, &updates, "anything").expect_err("no index yet");
|
||||
|
||||
let mut s = Seeder::new(&p, true);
|
||||
s.add("arrived.txt", "/d", 1, Some("shared body"));
|
||||
drop(s.done());
|
||||
|
||||
assert_eq!(
|
||||
search_names(&service, &updates, "shared").unwrap(),
|
||||
vec!["arrived.txt"],
|
||||
"the failure left the worker unable to open the index that appeared"
|
||||
);
|
||||
service.shutdown();
|
||||
|
||||
std::fs::remove_file(&p).ok();
|
||||
}
|
||||
|
||||
/// The connection is released once a search session goes quiet.
|
||||
///
|
||||
/// Not about memory here — it is about the two things an open handle costs
|
||||
/// that have nothing to do with the page cache: a deleted index keeps its
|
||||
/// blocks until the last handle closes, and an open reader stops SQLite from
|
||||
/// resetting the WAL. Checked through `/proc/self/fd` because "the file is no
|
||||
/// longer open" is the actual claim, and a reopen-still-works assertion would
|
||||
/// pass whether or not anything was ever released.
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn the_connection_is_released_once_searching_stops() {
|
||||
let p = tmp_db("warmrelease");
|
||||
let mut s = Seeder::new(&p, true);
|
||||
s.add("held.txt", "/d", 1, Some("shared body"));
|
||||
drop(s.done());
|
||||
|
||||
let holds_index = || {
|
||||
let Ok(entries) = std::fs::read_dir("/proc/self/fd") else {
|
||||
return false;
|
||||
};
|
||||
entries
|
||||
.filter_map(|e| e.ok())
|
||||
.filter_map(|e| std::fs::read_link(e.path()).ok())
|
||||
.any(|target| target == p)
|
||||
};
|
||||
|
||||
let (service, updates) = SearchService::new_with_idle_release(
|
||||
p.clone(),
|
||||
Arc::new(|| {}),
|
||||
std::time::Duration::from_millis(150),
|
||||
);
|
||||
assert_eq!(
|
||||
search_names(&service, &updates, "shared").unwrap(),
|
||||
vec!["held.txt"]
|
||||
);
|
||||
assert!(holds_index(), "the connection should be held across requests");
|
||||
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
|
||||
while holds_index() && std::time::Instant::now() < deadline {
|
||||
std::thread::sleep(std::time::Duration::from_millis(50));
|
||||
}
|
||||
assert!(!holds_index(), "the idle connection was never released");
|
||||
|
||||
// And the release must not be terminal: the next search reopens.
|
||||
assert_eq!(
|
||||
search_names(&service, &updates, "shared").unwrap(),
|
||||
vec!["held.txt"]
|
||||
);
|
||||
service.shutdown();
|
||||
|
||||
std::fs::remove_file(&p).ok();
|
||||
}
|
||||
|
||||
/// The point of the whole change: a pass hands hits over *while* it scans, so
|
||||
/// the UI has something to show long before the scan ends.
|
||||
///
|
||||
|
|
|
|||
92
crates/quicksearch-core/tests/common/mod.rs
Normal file
92
crates/quicksearch-core/tests/common/mod.rs
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
//! Shared scaffolding for the integration-test binaries.
|
||||
//!
|
||||
//! Cargo compiles this into each `tests/*.rs` that declares `mod common;`, so
|
||||
//! every binary gets its own copy and each one uses a different subset.
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use quicksearch_core::config::Config;
|
||||
use quicksearch_core::db;
|
||||
use quicksearch_core::indexing::{IndexingService, IndexingStatus};
|
||||
|
||||
#[allow(unused_imports)]
|
||||
pub use quicksearch_core::testutil::{scratch_dir, scratch_dir_canonical, touch};
|
||||
|
||||
/// A scratch database path under a fresh directory. The sidecars SQLite
|
||||
/// creates alongside it (`-wal`, `-shm`) land in the same directory.
|
||||
pub fn scratch_db(tag: &str) -> std::path::PathBuf {
|
||||
scratch_dir(tag).join("index.sqlite")
|
||||
}
|
||||
|
||||
/// How long a single indexing run may take before the test gives up. Generous:
|
||||
/// CI runs these in a container against a cold page cache.
|
||||
const INDEX_TIMEOUT: Duration = Duration::from_secs(120);
|
||||
|
||||
/// One full indexing run, awaited to completion.
|
||||
///
|
||||
/// Completion is read from the `last_full_index` marker rather than the status
|
||||
/// enum, because `run_indexing` writes that marker only on a successful finish.
|
||||
/// Polling for `IndexingStatus::Idle` instead would race: a small tree finishes
|
||||
/// between two polls, leaving `Idle` ambiguous between "not started yet" and
|
||||
/// "already done".
|
||||
pub struct IndexOnce<'a> {
|
||||
pub db: &'a Path,
|
||||
pub roots: Vec<String>,
|
||||
pub config: &'a Config,
|
||||
/// Delete any existing completion marker first, so a second run over the
|
||||
/// same index is distinguishable from the first. Off for suites that index
|
||||
/// into a database whose lifecycle they are themselves testing.
|
||||
pub fresh_marker: bool,
|
||||
/// Poll the marker through the keyed open. An encrypted index cannot be
|
||||
/// read by a plain `rusqlite::Connection::open`, so a run against one would
|
||||
/// otherwise never observe its own completion and time out.
|
||||
pub encrypted: bool,
|
||||
}
|
||||
|
||||
impl IndexOnce<'_> {
|
||||
pub fn run(mut self) {
|
||||
if self.fresh_marker && self.db.exists() {
|
||||
let conn = rusqlite::Connection::open(self.db).unwrap();
|
||||
conn.execute("DELETE FROM schema_info WHERE key = 'last_full_index'", [])
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let service = IndexingService::new();
|
||||
service
|
||||
.start_indexing(
|
||||
std::mem::take(&mut self.roots),
|
||||
self.db.to_string_lossy().into_owned(),
|
||||
self.config.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let deadline = Instant::now() + INDEX_TIMEOUT;
|
||||
let mut done = false;
|
||||
while Instant::now() < deadline {
|
||||
if let IndexingStatus::Error(e) = service.get_status() {
|
||||
panic!("indexing failed: {}", e);
|
||||
}
|
||||
if self.db.exists() && self.completed() {
|
||||
done = true;
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
assert!(done, "indexing did not finish within {:?}", INDEX_TIMEOUT);
|
||||
service.stop_indexing().unwrap();
|
||||
}
|
||||
|
||||
/// Whether the completion marker is present. A database mid-creation is
|
||||
/// simply "not yet", not a failure — the poll comes round again.
|
||||
fn completed(&self) -> bool {
|
||||
let conn = if self.encrypted {
|
||||
db::open_existing(&self.db.to_string_lossy(), false).ok()
|
||||
} else {
|
||||
rusqlite::Connection::open(self.db).ok()
|
||||
};
|
||||
conn.is_some_and(|c| db::repo::get_last_full_index(&c).is_some())
|
||||
}
|
||||
}
|
||||
|
|
@ -7,59 +7,31 @@
|
|||
//! do. Everything runs inside a single #[test] so the key transitions are
|
||||
//! strictly ordered.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use std::path::Path;
|
||||
|
||||
use quicksearch_core::config::Config;
|
||||
use quicksearch_core::db;
|
||||
use quicksearch_core::indexing::{IndexingService, IndexingStatus};
|
||||
use quicksearch_core::indexing::IndexingService;
|
||||
use quicksearch_core::security::{derive_key, salt_from_hex};
|
||||
|
||||
fn tmp_dir(tag: &str) -> PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"quicksearch-enc-{}-{}-{}",
|
||||
tag,
|
||||
std::process::id(),
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
std::fs::create_dir_all(&p).unwrap();
|
||||
p
|
||||
}
|
||||
mod common;
|
||||
use common::scratch_dir as tmp_dir;
|
||||
|
||||
/// Run one full index over `root` and wait for the completion marker,
|
||||
/// reading it through the keyed open so the poll works on encrypted DBs.
|
||||
/// reading it through the keyed open so the poll works on encrypted indexes.
|
||||
///
|
||||
/// The marker is deliberately *not* cleared first: this suite indexes into a
|
||||
/// database whose enable/disable rebuild cycle it is itself testing, and each
|
||||
/// rebuild already starts from a fresh file.
|
||||
fn index_once(root: &Path, db_path: &Path, config: &Config) {
|
||||
let service = IndexingService::new();
|
||||
service
|
||||
.start_indexing(
|
||||
vec![root.to_string_lossy().into_owned()],
|
||||
db_path.to_string_lossy().into_owned(),
|
||||
config.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let deadline = Instant::now() + Duration::from_secs(120);
|
||||
let mut done = false;
|
||||
while Instant::now() < deadline {
|
||||
if let IndexingStatus::Error(e) = service.get_status() {
|
||||
panic!("indexing failed: {}", e);
|
||||
common::IndexOnce {
|
||||
db: db_path,
|
||||
roots: vec![root.to_string_lossy().into_owned()],
|
||||
config,
|
||||
fresh_marker: false,
|
||||
encrypted: true,
|
||||
}
|
||||
if db_path.exists() {
|
||||
if let Ok(conn) = db::open_existing(&db_path.to_string_lossy(), false) {
|
||||
if quicksearch_core::db::repo::get_last_full_index(&conn).is_some() {
|
||||
done = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
assert!(done, "indexing did not finish within the timeout");
|
||||
service.stop_indexing().unwrap();
|
||||
.run()
|
||||
}
|
||||
|
||||
fn header(db_path: &Path) -> [u8; 16] {
|
||||
|
|
|
|||
|
|
@ -6,74 +6,27 @@
|
|||
//! invisible on a first index — `existing_files` is empty, so nothing is
|
||||
//! stale — and only appears on the second run.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use quicksearch_core::config::Config;
|
||||
use quicksearch_core::file_handling::{extract_scope_prepare, ExtractCursor};
|
||||
use quicksearch_core::indexing::{IndexingService, IndexingStatus, RootPhase};
|
||||
|
||||
fn tmp_dir(tag: &str) -> PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"quicksearch-e2e-{}-{}-{}",
|
||||
tag,
|
||||
std::process::id(),
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
std::fs::create_dir_all(&p).unwrap();
|
||||
p
|
||||
}
|
||||
mod common;
|
||||
use common::{scratch_dir as tmp_dir, touch};
|
||||
|
||||
fn touch(p: &Path, body: &[u8]) {
|
||||
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
|
||||
std::fs::write(p, body).unwrap();
|
||||
}
|
||||
|
||||
/// Run one full index and wait for it to finish.
|
||||
///
|
||||
/// Completion is detected via the `last_full_index` marker, which
|
||||
/// `run_indexing` writes only on a successful finish. Polling the status
|
||||
/// enum instead would race: a small tree finishes between two polls, so
|
||||
/// `Idle` is ambiguous between "not started yet" and "already done".
|
||||
/// Run one full index over `root` and wait for it to finish.
|
||||
fn index_once(root: &Path, db: &Path, config: &Config) {
|
||||
if db.exists() {
|
||||
let conn = rusqlite::Connection::open(db).unwrap();
|
||||
conn.execute("DELETE FROM schema_info WHERE key = 'last_full_index'", [])
|
||||
.unwrap();
|
||||
common::IndexOnce {
|
||||
db,
|
||||
roots: vec![root.to_string_lossy().into_owned()],
|
||||
config,
|
||||
fresh_marker: true,
|
||||
encrypted: false,
|
||||
}
|
||||
|
||||
let service = IndexingService::new();
|
||||
service
|
||||
.start_indexing(
|
||||
vec![root.to_string_lossy().into_owned()],
|
||||
db.to_string_lossy().into_owned(),
|
||||
config.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let deadline = Instant::now() + Duration::from_secs(120);
|
||||
let mut done = false;
|
||||
while Instant::now() < deadline {
|
||||
if let IndexingStatus::Error(e) = service.get_status() {
|
||||
panic!("indexing failed: {}", e);
|
||||
}
|
||||
if db.exists() {
|
||||
if let Ok(conn) = rusqlite::Connection::open(db) {
|
||||
if quicksearch_core::db::repo::get_last_full_index(&conn).is_some() {
|
||||
done = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
assert!(done, "indexing did not finish within the timeout");
|
||||
service.stop_indexing().unwrap();
|
||||
.run()
|
||||
}
|
||||
|
||||
/// (path, mtime, content_state) for every indexed row, ordered by path.
|
||||
|
|
@ -91,10 +44,10 @@ fn rows(db: &Path) -> Vec<(String, i64, i64)> {
|
|||
}
|
||||
|
||||
fn test_config() -> Config {
|
||||
let config = Config::default();
|
||||
|
||||
// Keep the run to phase 1 semantics we're asserting on; extraction is
|
||||
// covered elsewhere.
|
||||
config
|
||||
Config::default()
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -393,9 +346,12 @@ fn starting_a_run_claims_the_status_before_it_returns() {
|
|||
)
|
||||
.unwrap();
|
||||
|
||||
// No sleep, no poll: the very next observation must already be Running.
|
||||
// No sleep, no poll: the very next observation must already show the run.
|
||||
// `Preparing` is what a claim looks like before the command thread has
|
||||
// even picked the start up — it is still joining the previous run — and
|
||||
// it holds the index exactly as `Running` does.
|
||||
assert!(
|
||||
matches!(service.get_status(), IndexingStatus::Running { .. }),
|
||||
matches!(service.get_status(), IndexingStatus::Preparing { .. }),
|
||||
"status must be claimed synchronously, got {:?}",
|
||||
service.get_status()
|
||||
);
|
||||
|
|
@ -781,7 +737,9 @@ fn a_directory_that_becomes_unreadable_deletes_nothing() {
|
|||
|
||||
/// Everything about a file's indexed content that a user can observe: its
|
||||
/// state, the stored snippet body, and its property rows.
|
||||
fn content_rows(db: &Path) -> Vec<(String, i64, Option<String>, Option<i64>, String)> {
|
||||
type ContentRow = (String, i64, Option<String>, Option<i64>, String);
|
||||
|
||||
fn content_rows(db: &Path) -> Vec<ContentRow> {
|
||||
let conn = rusqlite::Connection::open(db).unwrap();
|
||||
let mut stmt = conn
|
||||
.prepare(
|
||||
|
|
@ -948,6 +906,41 @@ fn undecodable_small_files_are_reported_as_failures_not_silently_skipped() {
|
|||
std::fs::remove_dir_all(&db_dir).ok();
|
||||
}
|
||||
|
||||
/// A `.doc` that is not a readable OLE2 compound file — a truncated download,
|
||||
/// or something misnamed — records a failure with a reason.
|
||||
///
|
||||
/// This is the end-to-end shape of the legacy-Office support: the walk types
|
||||
/// the file from its extension, the office extractor claims `application/
|
||||
/// msword`, and the OLE2 reader either produces text or says why it could not.
|
||||
/// Until that reader existed, every `.doc` took the third path instead —
|
||||
/// `DONE` with empty text — which reads as "indexed, contains nothing" and is
|
||||
/// indistinguishable from a genuinely empty document.
|
||||
#[test]
|
||||
fn an_unreadable_legacy_office_file_fails_with_a_reason() {
|
||||
let root = tmp_dir("legacy-doc");
|
||||
let db_dir = tmp_dir("legacy-doc-db");
|
||||
let db = db_dir.join("index.sqlite");
|
||||
|
||||
touch(&root.join("broken.doc"), b"D0CF11E0 this is not really a compound file");
|
||||
index_once(&root, &db, &Config::default());
|
||||
|
||||
let conn = rusqlite::Connection::open(&db).unwrap();
|
||||
let (state, msg): (i64, Option<String>) = conn
|
||||
.query_row(
|
||||
"SELECT content_state, failure_msg FROM files WHERE path LIKE '%broken.doc'",
|
||||
[],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(state, 2, "an unreadable .doc is FAILED, not DONE-with-no-text");
|
||||
let msg = msg.unwrap_or_default();
|
||||
assert!(msg.contains("broken.doc"), "names the file: {msg}");
|
||||
assert!(msg.contains("compound file"), "says what went wrong: {msg}");
|
||||
|
||||
std::fs::remove_dir_all(&root).ok();
|
||||
std::fs::remove_dir_all(&db_dir).ok();
|
||||
}
|
||||
|
||||
/// The text sniff end-to-end: extensionless text files (README, Makefile,
|
||||
/// go.sum) are content-indexed off their head bytes, while an extensionless
|
||||
/// binary blob stays NA.
|
||||
|
|
@ -1306,6 +1299,10 @@ fn a_heavy_root_does_not_stall_a_light_one() {
|
|||
}
|
||||
}
|
||||
}
|
||||
// The run is claimed but has not reached its walk yet; there is
|
||||
// nothing to sample, and breaking here would end the watch before
|
||||
// the run it is watching had started.
|
||||
IndexingStatus::Preparing { .. } => {}
|
||||
IndexingStatus::Error(e) => panic!("indexing failed: {}", e),
|
||||
_ => break,
|
||||
}
|
||||
|
|
@ -1410,7 +1407,10 @@ fn the_wal_stays_bounded_during_a_run() {
|
|||
}
|
||||
last = len;
|
||||
match service.get_status() {
|
||||
IndexingStatus::Running { .. } => {}
|
||||
// Preparing included: the run is claimed but has not opened the
|
||||
// database yet, so there is no log to watch and nothing to stop
|
||||
// watching for either.
|
||||
IndexingStatus::Running { .. } | IndexingStatus::Preparing { .. } => {}
|
||||
IndexingStatus::Error(e) => panic!("indexing failed: {}", e),
|
||||
_ => break,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,69 +8,28 @@
|
|||
//! assertion a regression that quietly reintroduces the rebuild would pass
|
||||
//! every other check in this file.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::time::Instant;
|
||||
|
||||
use quicksearch_core::config::{diff_actions, Config};
|
||||
use quicksearch_core::db;
|
||||
use quicksearch_core::extract::Registry;
|
||||
use quicksearch_core::indexing::{IndexingService, IndexingStatus};
|
||||
use quicksearch_core::scope::{advance, WorkCursor, SLICE};
|
||||
|
||||
fn tmp_dir(tag: &str) -> PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"quicksearch-reconcile-{}-{}-{}",
|
||||
tag,
|
||||
std::process::id(),
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
std::fs::create_dir_all(&p).unwrap();
|
||||
std::fs::canonicalize(&p).unwrap()
|
||||
}
|
||||
|
||||
fn touch(p: &Path, body: &[u8]) {
|
||||
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
|
||||
std::fs::write(p, body).unwrap();
|
||||
}
|
||||
mod common;
|
||||
use common::{scratch_dir_canonical as tmp_dir, touch};
|
||||
|
||||
/// Run one full index over `config`'s roots and wait for it to finish.
|
||||
fn index_once(db: &Path, config: &Config) {
|
||||
if db.exists() {
|
||||
let conn = rusqlite::Connection::open(db).unwrap();
|
||||
conn.execute("DELETE FROM schema_info WHERE key = 'last_full_index'", [])
|
||||
.unwrap();
|
||||
common::IndexOnce {
|
||||
db,
|
||||
roots: config.paths.indexing_paths.clone(),
|
||||
config,
|
||||
fresh_marker: true,
|
||||
encrypted: false,
|
||||
}
|
||||
let service = IndexingService::new();
|
||||
service
|
||||
.start_indexing(
|
||||
config.paths.indexing_paths.clone(),
|
||||
db.to_string_lossy().into_owned(),
|
||||
config.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let deadline = Instant::now() + Duration::from_secs(120);
|
||||
let mut done = false;
|
||||
while Instant::now() < deadline {
|
||||
if let IndexingStatus::Error(e) = service.get_status() {
|
||||
panic!("indexing failed: {}", e);
|
||||
}
|
||||
if db.exists() {
|
||||
if let Ok(conn) = rusqlite::Connection::open(db) {
|
||||
if db::repo::get_last_full_index(&conn).is_some() {
|
||||
done = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
assert!(done, "indexing did not finish within the timeout");
|
||||
service.stop_indexing().unwrap();
|
||||
.run()
|
||||
}
|
||||
|
||||
/// Apply the reconciliation `old -> new` implies, exactly as the coordinator
|
||||
|
|
@ -84,6 +43,7 @@ fn reconcile(db: &Path, old: &Config, new: &Config) -> (usize, usize) {
|
|||
let mut conn = db::open_existing(&db.to_string_lossy(), true).unwrap();
|
||||
let registry = Registry::default_set();
|
||||
let mut cursor = WorkCursor::new(actions.work, new).unwrap();
|
||||
let run = AtomicBool::new(false);
|
||||
while !cursor.done() {
|
||||
advance(
|
||||
&mut conn,
|
||||
|
|
@ -91,6 +51,7 @@ fn reconcile(db: &Path, old: &Config, new: &Config) -> (usize, usize) {
|
|||
®istry,
|
||||
&mut cursor,
|
||||
Instant::now() + SLICE,
|
||||
&run,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
|
|
|||
249
crates/quicksearch-core/tests/search_perf.rs
Normal file
249
crates/quicksearch-core/tests/search_perf.rs
Normal file
|
|
@ -0,0 +1,249 @@
|
|||
//! What a warm page cache is worth to search, and how big it has to be.
|
||||
//!
|
||||
//! The search worker holds one connection across requests (see
|
||||
//! [`quicksearch_core::search`]) precisely so that a typing session runs
|
||||
//! against a cache that is already warm. This measures the two things that
|
||||
//! claim rests on:
|
||||
//!
|
||||
//! 1. **Warm beats cold**, by enough to justify holding the connection at all.
|
||||
//! 2. **8 MiB is enough** — the size [`PRAGMAS_SEARCH`] settles on. The hot set
|
||||
//! across queries is the `files` b-tree interior nodes and the tips of the
|
||||
//! FTS5 segments, not the table, so past some point a larger ceiling buys
|
||||
//! nothing and only raises what an idle process is holding.
|
||||
//!
|
||||
//! Queries are run as a *sequence* — `q`, `qu`, `qui`, `quic` — because that is
|
||||
//! what a search-per-keystroke frontend actually does. The first is the
|
||||
//! outlier; the second and later ones are the number that matters.
|
||||
//!
|
||||
//! The encrypted column is the one to watch. Under SQLCipher a page cache miss
|
||||
//! costs an AES-CBC decrypt plus an HMAC-SHA512 verify per 4 KiB page rather
|
||||
//! than a `memcpy`, so if a smaller cache is going to hurt anywhere it is here.
|
||||
//!
|
||||
//! Gated by `QSB_SEARCH_PERF` so the harness doesn't pay the seed cost on every
|
||||
//! `cargo test`. To run it:
|
||||
//!
|
||||
//! ```text
|
||||
//! QSB_SEARCH_PERF=1 cargo test --release -p quicksearch-core \
|
||||
//! --test search_perf -- --nocapture
|
||||
//! ```
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use quicksearch_core::db::repo::{insert_file, set_content_done, NewFile};
|
||||
use quicksearch_core::db::{open_or_recreate, set_process_key};
|
||||
use quicksearch_core::mime::FileType;
|
||||
use quicksearch_core::query::split::split_for_cascade;
|
||||
use quicksearch_core::search::{cascade, SearchHit, SearchOptions};
|
||||
use quicksearch_core::security::IndexKey;
|
||||
use rusqlite::Connection;
|
||||
|
||||
mod common;
|
||||
use common::scratch_db;
|
||||
|
||||
/// Rows to seed. Large enough that the `files` b-tree has real interior levels
|
||||
/// and the FTS index has more than one segment — below that everything fits in
|
||||
/// any cache and the comparison says nothing.
|
||||
const NUM_FILES: usize = 200_000;
|
||||
|
||||
/// Cache ceilings to compare, as `PRAGMA cache_size` values in KiB.
|
||||
///
|
||||
/// `-40960` is what every read connection used to take, `-8192` is
|
||||
/// `PRAGMAS_SEARCH`, and `-1024` is deliberately too small — it is there to
|
||||
/// show the curve has a floor worth being above, so that "8 MiB is enough" is
|
||||
/// a measurement rather than an assumption.
|
||||
const CACHE_SIZES: [i64; 6] = [-40960, -32768, -16384, -8192, -4096, -1024];
|
||||
|
||||
/// The prefixes of one word, typed one character at a time.
|
||||
const SEQUENCE: [&str; 4] = ["quar", "quart", "quartz", "quartzi"];
|
||||
|
||||
fn enabled() -> bool {
|
||||
std::env::var("QSB_SEARCH_PERF").is_ok()
|
||||
}
|
||||
|
||||
/// Deterministic pseudo-random word picker. Same LCG as `indexprobe`, for the
|
||||
/// same reason: a fixed seed makes two runs comparable.
|
||||
struct Lcg(u64);
|
||||
|
||||
impl Lcg {
|
||||
fn next(&mut self) -> u64 {
|
||||
self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1);
|
||||
self.0 >> 33
|
||||
}
|
||||
}
|
||||
|
||||
const WORDS: &[&str] = &[
|
||||
"alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", "iota", "kappa",
|
||||
"lambda", "quartz", "quartzite", "quarry", "quarter", "quantum", "brown", "fox", "jumps",
|
||||
"lazy", "index", "search", "cascade", "snippet", "document", "content", "extract",
|
||||
];
|
||||
|
||||
/// Seed an index with `NUM_FILES` rows, a tenth of them content-indexed.
|
||||
///
|
||||
/// Only a tenth so the FTS index stays smaller than the table, which is the
|
||||
/// real shape — most files in a tree are not text.
|
||||
fn seed(path: &std::path::Path) {
|
||||
let mut conn = open_or_recreate(path.to_str().unwrap(), "trigram").unwrap();
|
||||
let mut rng = Lcg(0x5eed);
|
||||
let tx = conn.transaction().unwrap();
|
||||
for i in 0..NUM_FILES {
|
||||
let w1 = WORDS[(rng.next() as usize) % WORDS.len()];
|
||||
let w2 = WORDS[(rng.next() as usize) % WORDS.len()];
|
||||
let name = format!("{}-{}-{:07}.txt", w1, w2, i);
|
||||
let dir = format!("/seed/{:03}", i % 500);
|
||||
let full = format!("{}/{}", dir, name);
|
||||
let id = insert_file(
|
||||
&tx,
|
||||
&NewFile {
|
||||
name: &name,
|
||||
path: &full,
|
||||
parent: &dir,
|
||||
size: 4096,
|
||||
mtime: 1_700_000_000 + i as u64,
|
||||
inode: None,
|
||||
device_id: None,
|
||||
mime: Some("text/plain"),
|
||||
ftype: FileType::TEXT,
|
||||
hash: None,
|
||||
needs_content: i % 10 == 0,
|
||||
},
|
||||
)
|
||||
.unwrap()
|
||||
.expect("unique path");
|
||||
if i % 10 == 0 {
|
||||
let body: Vec<&str> = (0..60)
|
||||
.map(|_| WORDS[(rng.next() as usize) % WORDS.len()])
|
||||
.collect();
|
||||
set_content_done(&tx, id, &name, &body.join(" "), &[], true).unwrap();
|
||||
}
|
||||
}
|
||||
tx.commit().unwrap();
|
||||
conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);").ok();
|
||||
}
|
||||
|
||||
/// Run one query to completion, returning how long it took and how many hits
|
||||
/// it produced. Hits are counted, not kept — the cost being measured is the
|
||||
/// scan, and holding 200k `SearchHit`s would measure the allocator instead.
|
||||
fn time_query(conn: &Connection, query: &str) -> (Duration, usize) {
|
||||
let split = split_for_cascade(query).unwrap();
|
||||
let latest = std::sync::atomic::AtomicU64::new(1);
|
||||
let mut count = 0usize;
|
||||
let mut sink = |hits: Vec<SearchHit>| count += hits.len();
|
||||
let options = SearchOptions {
|
||||
limit: 1000,
|
||||
..SearchOptions::default()
|
||||
};
|
||||
let start = Instant::now();
|
||||
cascade::run(conn, &split, &options, 1, &latest, &mut sink).unwrap();
|
||||
(start.elapsed(), count)
|
||||
}
|
||||
|
||||
/// Open a connection at an explicit cache ceiling.
|
||||
///
|
||||
/// Spelled out rather than going through `db::open::open_search_reader`
|
||||
/// because the whole point is to compare ceilings, which that function
|
||||
/// deliberately does not expose.
|
||||
fn open_at(path: &std::path::Path, cache_size: i64) -> Connection {
|
||||
let conn = Connection::open(path).unwrap();
|
||||
conn.execute_batch(&format!(
|
||||
"PRAGMA busy_timeout = 5000;
|
||||
PRAGMA cache_size = {};
|
||||
PRAGMA temp_store = MEMORY;
|
||||
PRAGMA foreign_keys = ON;",
|
||||
cache_size
|
||||
))
|
||||
.unwrap();
|
||||
conn
|
||||
}
|
||||
|
||||
fn run_matrix(label: &str, path: &std::path::Path) {
|
||||
println!("\n=== {} ===", label);
|
||||
println!(
|
||||
"{:>12} {:>10} {:>10} {:>10} {:>8}",
|
||||
"cache_size", "cold", "warm avg", "warm best", "hits"
|
||||
);
|
||||
for cache_size in CACHE_SIZES {
|
||||
// A connection per ceiling, held for the whole sequence — the same
|
||||
// lifetime the search worker gives it.
|
||||
let conn = open_at(path, cache_size);
|
||||
let (cold, hits) = time_query(&conn, SEQUENCE[0]);
|
||||
let mut warm = Vec::new();
|
||||
for query in &SEQUENCE[1..] {
|
||||
warm.push(time_query(&conn, query).0);
|
||||
}
|
||||
let avg = warm.iter().sum::<Duration>() / warm.len() as u32;
|
||||
let best = warm.iter().min().copied().unwrap_or_default();
|
||||
println!(
|
||||
"{:>12} {:>9.1?} {:>9.1?} {:>9.1?} {:>8}",
|
||||
cache_size, cold, avg, best, hits
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The headline comparison, printed rather than asserted.
|
||||
///
|
||||
/// Deliberately not a pass/fail threshold: timings on a shared CI box are not
|
||||
/// stable enough for one, and a flaky perf gate gets muted rather than fixed.
|
||||
/// This exists to be *read* when the number in [`PRAGMAS_SEARCH`] is being
|
||||
/// chosen or questioned.
|
||||
#[test]
|
||||
fn cache_size_against_search_latency() {
|
||||
if !enabled() {
|
||||
eprintln!("skipping: set QSB_SEARCH_PERF=1 to run");
|
||||
return;
|
||||
}
|
||||
|
||||
let plain = scratch_db("searchperf-plain");
|
||||
let seeded = Instant::now();
|
||||
seed(&plain);
|
||||
println!(
|
||||
"seeded {} rows in {:.1?} ({} MiB on disk)",
|
||||
NUM_FILES,
|
||||
seeded.elapsed(),
|
||||
std::fs::metadata(&plain).map(|m| m.len()).unwrap_or(0) / (1024 * 1024)
|
||||
);
|
||||
run_matrix("unencrypted", &plain);
|
||||
}
|
||||
|
||||
/// The same matrix against an encrypted index.
|
||||
///
|
||||
/// Separate test, and separate process-wide key, because
|
||||
/// [`set_process_key`] is global: running both in one test would have the
|
||||
/// plain index opened with a key set.
|
||||
#[test]
|
||||
fn cache_size_against_search_latency_encrypted() {
|
||||
if !enabled() {
|
||||
eprintln!("skipping: set QSB_SEARCH_PERF=1 to run");
|
||||
return;
|
||||
}
|
||||
|
||||
set_process_key(Some(
|
||||
IndexKey::from_hex(&"42".repeat(32)).expect("valid 32-byte key"),
|
||||
));
|
||||
let enc = scratch_db("searchperf-enc");
|
||||
seed(&enc);
|
||||
|
||||
println!("\n(encrypted: every cache miss costs an AES-CBC + HMAC-SHA512 per page)");
|
||||
// Both orders. The ceilings are tried largest-first and then
|
||||
// smallest-first because the OS page cache warms as the run proceeds, and
|
||||
// a difference that survives reversing the order is a property of the
|
||||
// ceiling rather than of when it was measured.
|
||||
let mut order: Vec<i64> = CACHE_SIZES.to_vec();
|
||||
order.extend(CACHE_SIZES.iter().rev());
|
||||
// Opened through the keyed path — a raw `Connection::open` cannot read it.
|
||||
for cache_size in order {
|
||||
let conn = quicksearch_core::db::open_existing(&enc.to_string_lossy(), false).unwrap();
|
||||
conn.execute_batch(&format!("PRAGMA cache_size = {};", cache_size))
|
||||
.unwrap();
|
||||
let (cold, hits) = time_query(&conn, SEQUENCE[0]);
|
||||
let mut warm = Vec::new();
|
||||
for query in &SEQUENCE[1..] {
|
||||
warm.push(time_query(&conn, query).0);
|
||||
}
|
||||
let avg = warm.iter().sum::<Duration>() / warm.len() as u32;
|
||||
println!(
|
||||
"{:>12} cold {:>9.1?} warm avg {:>9.1?} hits {}",
|
||||
cache_size, cold, avg, hits
|
||||
);
|
||||
}
|
||||
set_process_key(None);
|
||||
}
|
||||
|
|
@ -16,7 +16,6 @@
|
|||
//! --test snippet_perf -- --nocapture
|
||||
//! ```
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::Instant;
|
||||
|
||||
use quicksearch_core::snippet;
|
||||
|
|
@ -123,19 +122,8 @@ fn seed_text(rng: &mut u64, target_words: usize) -> String {
|
|||
out
|
||||
}
|
||||
|
||||
fn tmp_path(tag: &str) -> PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
p.push(format!(
|
||||
"qs-snippet-perf-{}-{}-{}.sqlite",
|
||||
tag,
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
p
|
||||
}
|
||||
mod common;
|
||||
use common::scratch_db as tmp_path;
|
||||
|
||||
#[test]
|
||||
fn snippet_paths_perf_comparison() {
|
||||
|
|
|
|||
|
|
@ -2,13 +2,14 @@
|
|||
//! status bar, and config-change routing.
|
||||
|
||||
use std::sync::mpsc;
|
||||
use std::time::{Duration, Instant};
|
||||
use std::time::Duration;
|
||||
|
||||
use quicksearch_core::cli::IndexCounts;
|
||||
use quicksearch_core::config::{diff_actions, nested_roots, Config, SecurityConfig};
|
||||
use quicksearch_core::coordinator::{IndexMode, IndexerState, WatcherStatus};
|
||||
use quicksearch_core::coordinator::{IndexMode, IndexerState, ReconcileState, WatcherStatus};
|
||||
use quicksearch_core::db;
|
||||
use quicksearch_core::indexing::{overall_progress, ConfigChange, IndexingStatus, RootPhase};
|
||||
use quicksearch_core::indexing::{
|
||||
overall_progress, ConfigChange, IndexingStatus, PrepStep, RootPhase,
|
||||
};
|
||||
use quicksearch_core::search::SearchOptions;
|
||||
use quicksearch_core::security::{derive_key, generate_salt, salt_to_hex, IndexKey};
|
||||
use quicksearch_core::watcher::WatchError;
|
||||
|
|
@ -79,6 +80,18 @@ fn guard_source(
|
|||
}
|
||||
}
|
||||
|
||||
/// Whether quitting now needs the "settings are still being applied" warning.
|
||||
///
|
||||
/// Only a Quit, and only while a reconciliation is actually running: leaving
|
||||
/// mid-pass leaves entries the user excluded still in the index until an
|
||||
/// indexing run redoes the work, which in manual mode means until they ask for
|
||||
/// one. Its own function because it is a rule, not a rendering decision, and
|
||||
/// because the guard it belongs to has two entrances — the close request, and
|
||||
/// the unsaved-changes prompt resolving to Quit.
|
||||
fn quit_needs_reconcile_warning(intent: NavIntent, reconciling: bool) -> bool {
|
||||
intent == NavIntent::Quit && reconciling
|
||||
}
|
||||
|
||||
pub struct QuickSearchApp {
|
||||
cfg: Config,
|
||||
backend: Backend,
|
||||
|
|
@ -88,8 +101,6 @@ pub struct QuickSearchApp {
|
|||
dups: DuplicatesTab,
|
||||
logs: LogsTab,
|
||||
options: OptionsWindow,
|
||||
/// Cached idle counts for the status bar, refreshed at most every 5 s.
|
||||
counts: Option<(Instant, IndexCounts)>,
|
||||
/// Set when applying a config that invalidates the stored index.
|
||||
rebuild_prompt: Option<Vec<ConfigChange>>,
|
||||
/// Set while the "delete the index?" confirmation is open.
|
||||
|
|
@ -103,6 +114,13 @@ pub struct QuickSearchApp {
|
|||
/// and the next run will replace it; see
|
||||
/// [`QuickSearchApp::stale_index_prompt_ui`].
|
||||
stale_index_prompt: bool,
|
||||
/// Set at startup when the index has not caught up with the settings —
|
||||
/// a reconciliation cut short by a previous quit, or a config edited
|
||||
/// while the app was closed. See [`QuickSearchApp::reconcile_owed_ui`].
|
||||
reconcile_owed: bool,
|
||||
/// `last_full_index` as it read at startup; the run that moves it past
|
||||
/// this is the run that clears `reconcile_owed`.
|
||||
reconcile_owed_since: Option<u64>,
|
||||
/// Set when the watcher gave up on the directory budget and live
|
||||
/// updates are off; see [`QuickSearchApp::check_watch_cap_warning`].
|
||||
watch_cap_prompt: Option<WatchError>,
|
||||
|
|
@ -177,7 +195,20 @@ impl QuickSearchApp {
|
|||
let stale_index_prompt =
|
||||
db::index_needs_rebuild(&cfg.resolved_database_path().to_string_lossy());
|
||||
|
||||
// Also before the backend, and for the same reason: in automatic mode
|
||||
// the coordinator's first run can reconcile — and clear the answer —
|
||||
// before the first frame is drawn. A missing or unreadable index owes
|
||||
// nothing; it has never been reconciled against anything.
|
||||
let db_path = cfg.resolved_database_path().to_string_lossy().into_owned();
|
||||
let reconcile_owed = quicksearch_core::scope::outstanding_work(&db_path, &cfg)
|
||||
.map(|work| work.touches_index())
|
||||
.unwrap_or(false);
|
||||
|
||||
let backend = Backend::start(&cfg, ctx.clone())?;
|
||||
// Read from the coordinator rather than the file: it stamps this at
|
||||
// startup, before its thread can run anything, so it is the same
|
||||
// number the frames below compare against.
|
||||
let reconcile_owed_since = backend.coordinator.state().last_full_index;
|
||||
let fuzzy = cfg.search.fuzzy_default;
|
||||
// Startup validation: a hand-edited config can nest roots, which
|
||||
// per-root pipelines can't accept. Redirect straight to the folder
|
||||
|
|
@ -202,12 +233,13 @@ impl QuickSearchApp {
|
|||
dups: DuplicatesTab::new(),
|
||||
logs: LogsTab::new(),
|
||||
options: OptionsWindow::new(),
|
||||
counts: None,
|
||||
rebuild_prompt: None,
|
||||
clear_prompt: false,
|
||||
nested_prompt,
|
||||
key_source,
|
||||
stale_index_prompt,
|
||||
reconcile_owed,
|
||||
reconcile_owed_since,
|
||||
watch_cap_prompt: None,
|
||||
security_prompt: None,
|
||||
pending_nav: None,
|
||||
|
|
@ -261,8 +293,13 @@ impl QuickSearchApp {
|
|||
.watch_cap_warned_roots
|
||||
.retain(|root| new.paths.indexing_paths.contains(root));
|
||||
let actions = diff_actions(&self.cfg, &new);
|
||||
// A config that could not be written must not take effect either: a
|
||||
// read-only config directory would otherwise apply the settings to
|
||||
// this process, revert them on restart, and — because `is_dirty`
|
||||
// compares against `self.cfg` — show nothing unsaved in between.
|
||||
if let Err(e) = new.save() {
|
||||
self.config_error = Some(e);
|
||||
return false;
|
||||
}
|
||||
if (new.ui.scale - self.cfg.ui.scale).abs() > f32::EPSILON {
|
||||
ctx.set_zoom_factor(clamp_scale(new.ui.scale));
|
||||
|
|
@ -271,7 +308,6 @@ impl QuickSearchApp {
|
|||
self.backend
|
||||
.search()
|
||||
.set_db_path(new.resolved_database_path());
|
||||
self.counts = None;
|
||||
}
|
||||
// Everything the index can reconcile in place — pruning rows a
|
||||
// narrowed filter put out of scope, re-deciding extracted text,
|
||||
|
|
@ -321,35 +357,9 @@ impl QuickSearchApp {
|
|||
|
||||
fn drain_events(&mut self) {
|
||||
// Streamed search results.
|
||||
loop {
|
||||
match self.backend.search_rx.try_recv() {
|
||||
Ok(update) => self
|
||||
.search
|
||||
.apply_update(update, self.cfg.search.display_limit),
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
// Status-bar counts worker.
|
||||
if let Some(rx) = &self.backend.counts_job {
|
||||
match rx.try_recv() {
|
||||
Ok(counts) => {
|
||||
self.counts = Some((Instant::now(), counts));
|
||||
self.backend.counts_job = None;
|
||||
}
|
||||
Err(mpsc::TryRecvError::Empty) => {}
|
||||
// The worker gave up (missing or unreadable index). Keep the
|
||||
// last known figures but restamp them, so the next attempt
|
||||
// waits its turn instead of respawning a thread every frame.
|
||||
Err(mpsc::TryRecvError::Disconnected) => {
|
||||
let last = self.counts.map(|(_, c)| c).unwrap_or(IndexCounts {
|
||||
files: 0,
|
||||
content_done: 0,
|
||||
content_pending: 0,
|
||||
});
|
||||
self.counts = Some((Instant::now(), last));
|
||||
self.backend.counts_job = None;
|
||||
}
|
||||
}
|
||||
while let Ok(update) = self.backend.search_rx.try_recv() {
|
||||
self.search
|
||||
.apply_update(update, self.cfg.search.display_limit);
|
||||
}
|
||||
// Duplicates worker.
|
||||
if let Some(rx) = &self.backend.dup_job {
|
||||
|
|
@ -430,23 +440,72 @@ impl QuickSearchApp {
|
|||
egui::TopBottomPanel::bottom("status-bar").show(ctx, |ui| {
|
||||
ui.horizontal(|ui| {
|
||||
match &state.activity {
|
||||
// A reconcile the coordinator applies between runs: no run
|
||||
// holds the index, but it is scanning every row of it —
|
||||
// and for a moment after, so a pass shorter than a frame
|
||||
// still leaves a trace.
|
||||
IndexingStatus::Idle if state.reconcile.is_some() => {
|
||||
match state.reconcile.expect("matched Some") {
|
||||
ReconcileState::Running(r) => {
|
||||
ui.label(
|
||||
egui::RichText::new(match (r.total, r.fraction()) {
|
||||
(Some(total), Some(frac)) => format!(
|
||||
"Applying configuration change · {} / {} ({:.0}%)",
|
||||
group_thousands(r.examined as u64),
|
||||
group_thousands(total as u64),
|
||||
frac * 100.0
|
||||
),
|
||||
_ => format!(
|
||||
"Applying configuration change · {} entries",
|
||||
group_thousands(r.examined as u64)
|
||||
),
|
||||
})
|
||||
.small(),
|
||||
);
|
||||
progress_widget(ui, r.fraction());
|
||||
}
|
||||
ReconcileState::Finished(r) => {
|
||||
ui.label(
|
||||
egui::RichText::new(crate::format::fmt_reconcile_summary(
|
||||
r.deleted,
|
||||
r.recontented,
|
||||
))
|
||||
.small(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
IndexingStatus::Preparing { start_time, step } => {
|
||||
let (label, frac) = match step {
|
||||
PrepStep::PreviousRun => {
|
||||
("Finishing the previous run…".to_string(), None)
|
||||
}
|
||||
PrepStep::OpeningIndex => ("Opening the index…".to_string(), None),
|
||||
PrepStep::Reconciling(r) => (
|
||||
format!(
|
||||
"Applying configuration change · {} entries",
|
||||
group_thousands(r.examined as u64)
|
||||
),
|
||||
r.fraction(),
|
||||
),
|
||||
};
|
||||
ui.label(
|
||||
egui::RichText::new(format!(
|
||||
"{} · {}",
|
||||
label,
|
||||
crate::format::fmt_duration_clock(start_time.elapsed())
|
||||
))
|
||||
.small(),
|
||||
);
|
||||
progress_widget(ui, frac);
|
||||
}
|
||||
IndexingStatus::Idle => {
|
||||
let mode = match state.mode {
|
||||
IndexMode::Auto => "Auto",
|
||||
IndexMode::ManualStopped => "Manual",
|
||||
IndexMode::ManualRunning => "Manual",
|
||||
};
|
||||
let stale = self
|
||||
.counts
|
||||
.map(|(at, _)| at.elapsed() > Duration::from_secs(5))
|
||||
.unwrap_or(true);
|
||||
if stale {
|
||||
// Kicked off, not awaited: the result lands through
|
||||
// `drain_events` on a later frame.
|
||||
let cfg = self.cfg.clone();
|
||||
self.backend.start_index_counts(&cfg, ctx.clone());
|
||||
}
|
||||
let files = self.counts.map(|(_, c)| c.files).unwrap_or(0);
|
||||
let files = state.files.unwrap_or(0);
|
||||
ui.label(
|
||||
egui::RichText::new(format!(
|
||||
"Idle · {} · {} files indexed",
|
||||
|
|
@ -497,14 +556,7 @@ impl QuickSearchApp {
|
|||
text.push_str(&format!(" · {}/{} workers", active, total_workers));
|
||||
}
|
||||
ui.label(egui::RichText::new(text).small());
|
||||
match frac {
|
||||
Some(frac) => {
|
||||
ui.add(egui::ProgressBar::new(frac as f32).desired_width(120.0));
|
||||
}
|
||||
None => {
|
||||
ui.add(egui::Spinner::new().size(12.0));
|
||||
}
|
||||
}
|
||||
progress_widget(ui, frac);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -525,11 +577,15 @@ impl QuickSearchApp {
|
|||
});
|
||||
});
|
||||
|
||||
// Keep painting while anything is moving.
|
||||
// Keep painting while anything is moving. The reconcile clause is not
|
||||
// redundant: the coordinator's own pass runs with the activity `Idle`,
|
||||
// so without it the counters would freeze mid-scan until the pointer
|
||||
// moved — and the summary that follows would never age off screen.
|
||||
if !matches!(
|
||||
state.activity,
|
||||
IndexingStatus::Idle | IndexingStatus::Error(_)
|
||||
) {
|
||||
) || state.reconcile.is_some()
|
||||
{
|
||||
ctx.request_repaint_after(Duration::from_millis(250));
|
||||
}
|
||||
// Watcher registration walks every root, so its verdict can land
|
||||
|
|
@ -631,8 +687,12 @@ impl QuickSearchApp {
|
|||
return;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
keychain::delete_key(&db_path.to_string_lossy());
|
||||
} else if let Err(e) = keychain::delete_key(&db_path.to_string_lossy()) {
|
||||
// Mirrors the store half: the preference describes what
|
||||
// is on the keychain, so it must not claim the key is
|
||||
// gone while it is still there.
|
||||
self.config_error = Some(e);
|
||||
return;
|
||||
}
|
||||
self.cfg.security.use_keychain = remember;
|
||||
if let Err(e) = self.cfg.save() {
|
||||
|
|
@ -823,16 +883,34 @@ impl QuickSearchApp {
|
|||
}
|
||||
}
|
||||
// Disabling protection, or "remember" off: no stored key may
|
||||
// survive pointing at the previous encryption state.
|
||||
_ => keychain::delete_key(&db_path),
|
||||
// survive pointing at the previous encryption state. A failure
|
||||
// here leaves one that does, which is worth saying out loud.
|
||||
_ => {
|
||||
if let Err(e) = keychain::delete_key(&db_path) {
|
||||
self.config_error = Some(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
db::set_process_key(new_key);
|
||||
self.backend.coordinator.rebuild_index();
|
||||
self.counts = None;
|
||||
self.dups.state = DupState::NotLoaded;
|
||||
}
|
||||
}
|
||||
|
||||
/// The status bar's trailing progress indicator: a bar when the work has a
|
||||
/// denominator, a spinner when it does not. One helper so every kind of
|
||||
/// activity the bar reports ends the same way.
|
||||
fn progress_widget(ui: &mut egui::Ui, fraction: Option<f64>) {
|
||||
match fraction {
|
||||
Some(frac) => {
|
||||
ui.add(egui::ProgressBar::new(frac as f32).desired_width(120.0));
|
||||
}
|
||||
None => {
|
||||
ui.add(egui::Spinner::new().size(12.0));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop egui's retained text-field state (buffer + undo history) for the
|
||||
/// password dialog fields.
|
||||
fn purge_security_field_state(ctx: &egui::Context) {
|
||||
|
|
@ -894,11 +972,53 @@ impl QuickSearchApp {
|
|||
if stale_index_window(ctx, self.key_source) {
|
||||
self.stale_index_prompt = false;
|
||||
self.backend.coordinator.rebuild_index();
|
||||
self.counts = None;
|
||||
self.dups.state = DupState::NotLoaded;
|
||||
}
|
||||
}
|
||||
|
||||
/// Tell the user their settings have not reached the index, and offer the
|
||||
/// one thing that fixes it.
|
||||
///
|
||||
/// The condition is the index's own record: a reconciliation that finishes
|
||||
/// stamps it, so work still owed means a pass was abandoned — quitting
|
||||
/// during one is the ordinary way — or the config was edited while the app
|
||||
/// was closed. In automatic mode the periodic run clears it without the
|
||||
/// user doing anything, which is why the banner is a line and a button
|
||||
/// rather than a modal; in manual mode nothing happens until they ask.
|
||||
///
|
||||
/// Held out of the way while a run or a reconcile is in progress: that is
|
||||
/// the work itself, and it can only be answered by waiting.
|
||||
fn reconcile_owed_ui(&mut self, ctx: &egui::Context) {
|
||||
if !self.reconcile_owed {
|
||||
return;
|
||||
}
|
||||
let state = self.backend.coordinator.state();
|
||||
// A completed run is the proof: it reconciles from the same record
|
||||
// and stamps it. A run the user stops does not move this, and the
|
||||
// banner correctly comes back.
|
||||
if state.last_full_index > self.reconcile_owed_since {
|
||||
self.reconcile_owed = false;
|
||||
return;
|
||||
}
|
||||
if !matches!(
|
||||
state.activity,
|
||||
IndexingStatus::Idle | IndexingStatus::Error(_)
|
||||
) || state.reconcile.is_some()
|
||||
{
|
||||
return;
|
||||
}
|
||||
match reconcile_owed_banner(ctx) {
|
||||
None => {}
|
||||
Some(ReconcileOwedChoice::StartIndexing) => {
|
||||
self.backend.coordinator.reindex_now();
|
||||
// Not cleared here: the run that finishes clears it, and one
|
||||
// that is stopped half-way leaves the reminder standing.
|
||||
ctx.request_repaint_after(Duration::from_millis(100));
|
||||
}
|
||||
Some(ReconcileOwedChoice::Dismiss) => self.reconcile_owed = false,
|
||||
}
|
||||
}
|
||||
|
||||
fn watch_cap_prompt_ui(&mut self, ctx: &egui::Context) {
|
||||
let Some(reason) = &self.watch_cap_prompt else {
|
||||
return;
|
||||
|
|
@ -994,7 +1114,6 @@ impl QuickSearchApp {
|
|||
// launch must not undo that either.
|
||||
self.set_index_mode(false);
|
||||
self.backend.coordinator.clear_index();
|
||||
self.counts = None;
|
||||
self.dups.state = DupState::NotLoaded;
|
||||
close = true;
|
||||
}
|
||||
|
|
@ -1020,6 +1139,23 @@ impl QuickSearchApp {
|
|||
};
|
||||
let dirty = (self.manage.is_dirty(), self.options.is_dirty(&self.cfg));
|
||||
let Some(source) = guard_source(intent, dirty.0, dirty.1) else {
|
||||
// Second in line, and inside the guard rather than beside it: the
|
||||
// Discard-then-quit path sets `quit_confirmed` and never returns
|
||||
// to the close-request check, so a warning that lived only there
|
||||
// would be skipped by exactly the user who dirtied an editor.
|
||||
if quit_needs_reconcile_warning(intent, self.backend.coordinator.reconciling()) {
|
||||
// Repaint on its own: a reconcile that ends while the modal is
|
||||
// up should take the modal with it.
|
||||
ctx.request_repaint_after(Duration::from_millis(250));
|
||||
match reconcile_quit_modal(ctx) {
|
||||
None => return,
|
||||
Some(false) => {
|
||||
self.pending_nav = None;
|
||||
return;
|
||||
}
|
||||
Some(true) => {}
|
||||
}
|
||||
}
|
||||
return self.complete_nav(ctx, intent);
|
||||
};
|
||||
match unsaved_changes_modal(ctx, source) {
|
||||
|
|
@ -1101,6 +1237,11 @@ impl QuickSearchApp {
|
|||
self.backend.coordinator.state().activity
|
||||
}
|
||||
|
||||
/// Let the close request a scripted quit sends through both guards.
|
||||
pub(crate) fn capture_confirm_quit(&mut self) {
|
||||
self.quit_confirmed = true;
|
||||
}
|
||||
|
||||
pub(crate) fn capture_search_settled(&self) -> bool {
|
||||
self.search.capture_settled()
|
||||
}
|
||||
|
|
@ -1113,7 +1254,10 @@ impl QuickSearchApp {
|
|||
/// search runs and the results table clears.
|
||||
pub(crate) fn capture_clear_query(&mut self) {
|
||||
self.search.query.clear();
|
||||
self.search.pending_edit = Some(Instant::now());
|
||||
// Qualified rather than imported: this is the only `Instant` left in
|
||||
// this module, and it is behind `feature = "capture"`, so a plain
|
||||
// `use` would warn in every default build.
|
||||
self.search.pending_edit = Some(std::time::Instant::now());
|
||||
}
|
||||
|
||||
pub(crate) fn capture_focus_search(&mut self) {
|
||||
|
|
@ -1186,6 +1330,85 @@ fn unsaved_changes_modal(ctx: &egui::Context, source: UnsavedSource) -> Option<U
|
|||
choice
|
||||
}
|
||||
|
||||
/// What the user chose in the "settings not applied yet" banner.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
enum ReconcileOwedChoice {
|
||||
StartIndexing,
|
||||
Dismiss,
|
||||
}
|
||||
|
||||
/// The banner's body, as a top panel under the config-error one.
|
||||
///
|
||||
/// A free function for the same reason as [`stale_index_window`]: a test can
|
||||
/// render it and click its buttons without a coordinator behind it.
|
||||
fn reconcile_owed_banner(ctx: &egui::Context) -> Option<ReconcileOwedChoice> {
|
||||
let mut choice = None;
|
||||
egui::TopBottomPanel::top("reconcile-owed").show(ctx, |ui| {
|
||||
ui.horizontal(|ui| {
|
||||
ui.colored_label(
|
||||
ui.visuals().warn_fg_color,
|
||||
"⚠ Your indexing settings have not been applied to the index yet.",
|
||||
);
|
||||
if ui.small_button("Start indexing now").clicked() {
|
||||
choice = Some(ReconcileOwedChoice::StartIndexing);
|
||||
}
|
||||
if ui.small_button("Dismiss").clicked() {
|
||||
choice = Some(ReconcileOwedChoice::Dismiss);
|
||||
}
|
||||
});
|
||||
});
|
||||
choice
|
||||
}
|
||||
|
||||
/// Body of the quit-during-a-reconcile guard; `Some(true)` to quit anyway,
|
||||
/// `Some(false)` to stay. Esc and a backdrop click count as staying.
|
||||
///
|
||||
/// The same blocking `egui::Modal` the unsaved guard uses, and for the same
|
||||
/// reason: this is a decision, not a notice. Quitting is not refused — the
|
||||
/// pass is cancellable and the index stays consistent either way — but the
|
||||
/// consequence is invisible otherwise, since the entries the user excluded go
|
||||
/// on appearing in search results until an indexing run finishes the job.
|
||||
fn reconcile_quit_modal(ctx: &egui::Context) -> Option<bool> {
|
||||
let mut choice = None;
|
||||
let modal = egui::Modal::new(egui::Id::new("reconcile-quit-guard")).show(ctx, |ui| {
|
||||
ui.set_max_width(460.0);
|
||||
ui.heading("Settings are still being applied");
|
||||
ui.label(
|
||||
"QuickSearch is still applying your indexing settings to the index. If you \
|
||||
quit now it stops part-way, and entries you excluded can still turn up in \
|
||||
search results.",
|
||||
);
|
||||
ui.add_space(4.0);
|
||||
ui.label(
|
||||
"Nothing is lost: the next indexing run picks the work up again. In manual \
|
||||
mode, choose \"Start indexing now\" on the Manage Index tab after the next \
|
||||
launch.",
|
||||
);
|
||||
ui.add_space(6.0);
|
||||
ui.horizontal(|ui| {
|
||||
if ui
|
||||
.button(egui::RichText::new("Quit anyway").color(ui.visuals().error_fg_color))
|
||||
.clicked()
|
||||
{
|
||||
choice = Some(true);
|
||||
}
|
||||
if ui
|
||||
.add(crate::ui_util::bordered_button(
|
||||
"Cancel",
|
||||
crate::ui_util::BLUE,
|
||||
))
|
||||
.clicked()
|
||||
{
|
||||
choice = Some(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
if choice.is_none() && modal.should_close() {
|
||||
choice = Some(false);
|
||||
}
|
||||
choice
|
||||
}
|
||||
|
||||
/// The stale-index window's body. Returns whether the user asked for the
|
||||
/// rebuild.
|
||||
///
|
||||
|
|
@ -1274,7 +1497,12 @@ impl eframe::App for QuickSearchApp {
|
|||
// window is gone there is nothing left to ask — and re-sent from
|
||||
// `complete_nav` if the user chooses to leave.
|
||||
if ctx.input(|i| i.viewport().close_requested()) && !self.quit_confirmed {
|
||||
if self.manage.is_dirty() || self.options.is_dirty(&self.cfg) {
|
||||
// A reconciliation in flight gets the same treatment as an unsaved
|
||||
// editor: hold the close and say what leaving now costs.
|
||||
if self.manage.is_dirty()
|
||||
|| self.options.is_dirty(&self.cfg)
|
||||
|| self.backend.coordinator.reconciling()
|
||||
{
|
||||
ctx.send_viewport_cmd(egui::ViewportCommand::CancelClose);
|
||||
// Quitting subsumes any narrower pending navigation.
|
||||
self.pending_nav = Some(NavIntent::Quit);
|
||||
|
|
@ -1335,6 +1563,7 @@ impl eframe::App for QuickSearchApp {
|
|||
});
|
||||
});
|
||||
}
|
||||
self.reconcile_owed_ui(ctx);
|
||||
|
||||
egui::CentralPanel::default().show(ctx, |ui| match self.tab {
|
||||
Tab::Search => {
|
||||
|
|
@ -1426,30 +1655,16 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
fn frame(ctx: &egui::Context, source: KeySource, events: Vec<egui::Event>) -> bool {
|
||||
let input = egui::RawInput {
|
||||
screen_rect: Some(egui::Rect::from_min_size(
|
||||
egui::Pos2::ZERO,
|
||||
egui::vec2(1000.0, 700.0),
|
||||
)),
|
||||
events,
|
||||
..Default::default()
|
||||
};
|
||||
let input = crate::test_ui::raw_input(SCREEN, events);
|
||||
let mut clicked = false;
|
||||
let _ = ctx.run(input, |ctx| clicked = stale_index_window(ctx, source));
|
||||
clicked
|
||||
}
|
||||
|
||||
fn click_at(pos: egui::Pos2) -> Vec<egui::Event> {
|
||||
[true, false]
|
||||
.into_iter()
|
||||
.map(|pressed| egui::Event::PointerButton {
|
||||
pos,
|
||||
button: egui::PointerButton::Primary,
|
||||
pressed,
|
||||
modifiers: egui::Modifiers::default(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
use crate::test_ui::click_at;
|
||||
|
||||
/// The viewport every modal in this module is centred in.
|
||||
const SCREEN: egui::Vec2 = egui::vec2(1000.0, 700.0);
|
||||
|
||||
/// The modal the user sees after unlocking onto an index from an older
|
||||
/// version. It must render under every key source — each produces a
|
||||
|
|
@ -1566,14 +1781,7 @@ mod tests {
|
|||
source: UnsavedSource,
|
||||
events: Vec<egui::Event>,
|
||||
) -> Option<UnsavedChoice> {
|
||||
let input = egui::RawInput {
|
||||
screen_rect: Some(egui::Rect::from_min_size(
|
||||
egui::Pos2::ZERO,
|
||||
egui::vec2(1000.0, 700.0),
|
||||
)),
|
||||
events,
|
||||
..Default::default()
|
||||
};
|
||||
let input = crate::test_ui::raw_input(SCREEN, events);
|
||||
let mut choice = None;
|
||||
let _ = ctx.run(input, |ctx| choice = unsaved_changes_modal(ctx, source));
|
||||
choice
|
||||
|
|
@ -1627,4 +1835,98 @@ mod tests {
|
|||
assert_eq!(esc, Some(UnsavedChoice::Cancel), "Esc must cancel");
|
||||
}
|
||||
}
|
||||
|
||||
/// Quitting mid-reconcile is the one case that needs saying out loud: the
|
||||
/// index is left describing settings the user has already changed, and in
|
||||
/// manual mode nothing fixes that until they ask for a run. Nothing else
|
||||
/// warrants the prompt — a tab switch does not end the pass, and a quit
|
||||
/// with no pass running has nothing to warn about.
|
||||
#[test]
|
||||
fn only_quitting_during_a_reconcile_warns() {
|
||||
use NavIntent::*;
|
||||
assert!(quit_needs_reconcile_warning(Quit, true));
|
||||
assert!(!quit_needs_reconcile_warning(Quit, false));
|
||||
assert!(!quit_needs_reconcile_warning(SwitchTab(Tab::Search), true));
|
||||
assert!(!quit_needs_reconcile_warning(CloseOptions, true));
|
||||
}
|
||||
|
||||
fn reconcile_modal_frame(ctx: &egui::Context, events: Vec<egui::Event>) -> Option<bool> {
|
||||
let input = crate::test_ui::raw_input(SCREEN, events);
|
||||
let mut choice = None;
|
||||
let _ = ctx.run(input, |ctx| choice = reconcile_quit_modal(ctx));
|
||||
choice
|
||||
}
|
||||
|
||||
/// Both ways out of the quit warning work, and neither is the default: a
|
||||
/// modal whose "Quit anyway" did nothing would trap the user in an app
|
||||
/// they asked to close, and one that quit on Esc would make the warning
|
||||
/// pointless.
|
||||
#[test]
|
||||
fn the_quit_warning_reports_both_answers() {
|
||||
let ctx = egui::Context::default();
|
||||
assert_eq!(
|
||||
reconcile_modal_frame(&ctx, Vec::new()),
|
||||
None,
|
||||
"an untouched frame must not decide"
|
||||
);
|
||||
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for y in (250..450).step_by(3) {
|
||||
for x in (250..760).step_by(6) {
|
||||
if let Some(choice) =
|
||||
reconcile_modal_frame(&ctx, click_at(egui::pos2(x as f32, y as f32)))
|
||||
{
|
||||
seen.insert(choice);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(seen.contains(&true), "\"Quit anyway\" never fired");
|
||||
assert!(seen.contains(&false), "Cancel never fired");
|
||||
|
||||
let esc = reconcile_modal_frame(
|
||||
&ctx,
|
||||
vec![egui::Event::Key {
|
||||
key: egui::Key::Escape,
|
||||
physical_key: None,
|
||||
pressed: true,
|
||||
repeat: false,
|
||||
modifiers: egui::Modifiers::NONE,
|
||||
}],
|
||||
);
|
||||
assert_eq!(esc, Some(false), "Esc must keep the app open");
|
||||
}
|
||||
|
||||
fn banner_frame(ctx: &egui::Context, events: Vec<egui::Event>) -> Option<ReconcileOwedChoice> {
|
||||
let input = crate::test_ui::raw_input(SCREEN, events);
|
||||
let mut choice = None;
|
||||
let _ = ctx.run(input, |ctx| choice = reconcile_owed_banner(ctx));
|
||||
choice
|
||||
}
|
||||
|
||||
/// The reminder a quit mid-prune leaves behind. Its Start button is the
|
||||
/// whole point — the banner exists because in manual mode nothing else
|
||||
/// will finish the work — and Dismiss must not be the only live control.
|
||||
#[test]
|
||||
fn the_reconcile_banner_reports_both_buttons() {
|
||||
let ctx = egui::Context::default();
|
||||
assert_eq!(banner_frame(&ctx, Vec::new()), None);
|
||||
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
// A top panel, so it sits in the first rows of the window.
|
||||
for y in (0..60).step_by(2) {
|
||||
for x in (0..1000).step_by(4) {
|
||||
if let Some(choice) = banner_frame(&ctx, click_at(egui::pos2(x as f32, y as f32))) {
|
||||
seen.insert(choice);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
seen.contains(&ReconcileOwedChoice::StartIndexing),
|
||||
"\"Start indexing now\" never fired"
|
||||
);
|
||||
assert!(
|
||||
seen.contains(&ReconcileOwedChoice::Dismiss),
|
||||
"Dismiss never fired"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,11 +3,19 @@
|
|||
//! All communication is non-blocking from the UI's point of view:
|
||||
//! searches stream over an mpsc receiver drained each frame, indexing
|
||||
//! state is polled, and the duplicates query runs on a throwaway worker
|
||||
//! thread. Core threads wake the UI through `ctx.request_repaint()`.
|
||||
//! thread. Core threads wake the UI through `ctx.request_repaint()` — every
|
||||
//! one of them, which is what makes polling enough: the frame that starts the
|
||||
//! UI's own cadence is the one the waking thread asks for.
|
||||
//!
|
||||
//! The duplicates scan is the *only* throwaway thread left here, and it fires
|
||||
//! on a user action rather than a timer. The status bar's file count used to
|
||||
//! spawn one every five seconds, each opening its own connection — a page
|
||||
//! cache and a fresh allocator arena per refresh, neither of which glibc gives
|
||||
//! back. It now rides on `IndexerState`, published by the coordinator off the
|
||||
//! connection it already holds.
|
||||
|
||||
use std::sync::{mpsc, Arc};
|
||||
|
||||
use quicksearch_core::cli::IndexCounts;
|
||||
use quicksearch_core::config::Config;
|
||||
use quicksearch_core::coordinator::IndexCoordinator;
|
||||
use quicksearch_core::search::{DuplicateGroup, SearchService, SearchUpdate};
|
||||
|
|
@ -18,13 +26,20 @@ pub struct Backend {
|
|||
pub search: Option<SearchService>,
|
||||
pub search_rx: mpsc::Receiver<SearchUpdate>,
|
||||
pub dup_job: Option<mpsc::Receiver<Result<Vec<DuplicateGroup>, String>>>,
|
||||
/// In-flight status-bar count; see [`Backend::start_index_counts`].
|
||||
pub counts_job: Option<mpsc::Receiver<IndexCounts>>,
|
||||
}
|
||||
|
||||
impl Backend {
|
||||
pub fn start(config: &Config, ctx: egui::Context) -> Result<Backend, String> {
|
||||
let coordinator = Arc::new(IndexCoordinator::start(config.clone())?);
|
||||
// The coordinator gets the same repaint hook the search worker does,
|
||||
// and needs it for the same reason: eframe is reactive, and a run it
|
||||
// schedules on its own — the periodic reindex a launch is already due —
|
||||
// would otherwise sit unseen behind a settled window until the pointer
|
||||
// moved. It calls this on the edge into work, not on a cadence.
|
||||
let coord_ctx = ctx.clone();
|
||||
let coordinator = Arc::new(IndexCoordinator::start(
|
||||
config.clone(),
|
||||
Arc::new(move || coord_ctx.request_repaint()),
|
||||
)?);
|
||||
if let Err(e) = shutdown::install_signal_handler(coordinator.clone()) {
|
||||
quicksearch_core::log_warn!("failed to install signal handler: {}", e);
|
||||
}
|
||||
|
|
@ -40,35 +55,9 @@ impl Backend {
|
|||
search: Some(search),
|
||||
search_rx,
|
||||
dup_job: None,
|
||||
counts_job: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Refresh the status bar's "N files indexed" on a worker thread.
|
||||
///
|
||||
/// Three `COUNT(*)` scans, and on a multi-million-row index the unfiltered
|
||||
/// one alone reads the whole primary key. Running it inline in `update()`
|
||||
/// froze a frame every refresh for a number that is purely decorative — so
|
||||
/// it goes the same way the duplicates scan does. No-op while one is
|
||||
/// already in flight.
|
||||
pub fn start_index_counts(&mut self, config: &Config, ctx: egui::Context) {
|
||||
if self.counts_job.is_some() {
|
||||
return;
|
||||
}
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let db = config.resolved_database_path();
|
||||
std::thread::spawn(move || {
|
||||
// A missing or unreadable index is not worth reporting here: the
|
||||
// status bar has nothing useful to say about it that the indexing
|
||||
// state does not already say.
|
||||
if let Ok(counts) = quicksearch_core::cli::index_counts(&db.to_string_lossy()) {
|
||||
let _ = tx.send(counts);
|
||||
}
|
||||
ctx.request_repaint();
|
||||
});
|
||||
self.counts_job = Some(rx);
|
||||
}
|
||||
|
||||
pub fn search(&self) -> &SearchService {
|
||||
self.search.as_ref().expect("search service alive")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -193,8 +193,16 @@ fn parse_line(tokens: &[Token], line_no: usize) -> Result<Option<Cmd>, ParseErro
|
|||
"clear_query" => Cmd::ClearQuery,
|
||||
"focus_search" => Cmd::FocusSearch,
|
||||
"window" => {
|
||||
let w = parse_int("width", next_word(rest, line_no, "width in points")?, line_no)?;
|
||||
let h = parse_int("height", next_word(rest, line_no, "height in points")?, line_no)?;
|
||||
let w = parse_int(
|
||||
"width",
|
||||
next_word(rest, line_no, "width in points")?,
|
||||
line_no,
|
||||
)?;
|
||||
let h = parse_int(
|
||||
"height",
|
||||
next_word(rest, line_no, "height in points")?,
|
||||
line_no,
|
||||
)?;
|
||||
if w == 0 || h == 0 {
|
||||
return Err(err("window dimensions must be positive".to_string()));
|
||||
}
|
||||
|
|
@ -203,9 +211,11 @@ fn parse_line(tokens: &[Token], line_no: usize) -> Result<Option<Cmd>, ParseErro
|
|||
h: h as f32,
|
||||
}
|
||||
}
|
||||
"hover_match" => Cmd::HoverMatch(
|
||||
"hover_match" => {
|
||||
Cmd::HoverMatch(
|
||||
parse_int("row", next_word(rest, line_no, "row index")?, line_no)? as usize,
|
||||
),
|
||||
)
|
||||
}
|
||||
"hover_off" => Cmd::HoverOff,
|
||||
"tab" => Cmd::Tab(match next_word(rest, line_no, "tab name")? {
|
||||
"search" => Tab::Search,
|
||||
|
|
@ -452,7 +462,7 @@ impl CaptureDriver {
|
|||
}
|
||||
|
||||
let Some(cmd) = self.cmds.get(self.pc).cloned() else {
|
||||
self.quit(ctx);
|
||||
self.quit(app, ctx);
|
||||
return;
|
||||
};
|
||||
let started = match self.cmd_started {
|
||||
|
|
@ -542,7 +552,7 @@ impl CaptureDriver {
|
|||
ShotTag,
|
||||
)));
|
||||
}
|
||||
Cmd::Quit => self.quit(ctx),
|
||||
Cmd::Quit => self.quit(app, ctx),
|
||||
}
|
||||
if matches!(cmd, Cmd::RecordStop) {
|
||||
self.stop_recorder();
|
||||
|
|
@ -572,7 +582,8 @@ impl CaptureDriver {
|
|||
capped(max_ms)
|
||||
|| matches!(
|
||||
app.capture_indexing_status(),
|
||||
IndexingStatus::Running { .. }
|
||||
IndexingStatus::Preparing { .. }
|
||||
| IndexingStatus::Running { .. }
|
||||
| IndexingStatus::Stopping
|
||||
| IndexingStatus::Optimizing
|
||||
)
|
||||
|
|
@ -590,8 +601,13 @@ impl CaptureDriver {
|
|||
}
|
||||
}
|
||||
|
||||
fn quit(&mut self, ctx: &egui::Context) {
|
||||
fn quit(&mut self, app: &mut QuickSearchApp, ctx: &egui::Context) {
|
||||
self.stop_recorder();
|
||||
// A scripted quit answers the guards up front. Nothing a scenario does
|
||||
// dirties an editor, but a scenario that changed the indexed folders
|
||||
// can leave a reconcile running, and its modal would hold the window
|
||||
// open until the run timed out.
|
||||
app.capture_confirm_quit();
|
||||
ctx.send_viewport_cmd(egui::ViewportCommand::Close);
|
||||
self.finished = true;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -54,6 +54,8 @@ impl DuplicatesTab {
|
|||
.weak(),
|
||||
);
|
||||
}
|
||||
// The spinner and its label are already on the header row above,
|
||||
// so a second progress line here would just repeat them.
|
||||
DupState::Loading => {}
|
||||
DupState::Error(e) => {
|
||||
ui.colored_label(ui.visuals().error_fg_color, e);
|
||||
|
|
|
|||
|
|
@ -29,11 +29,7 @@ pub fn fmt_mtime(unix_secs: i64) -> String {
|
|||
/// "5 min ago", "3 h ago", else `YYYY-MM-DD HH:MM`. Gives instant
|
||||
/// feedback that an action (like a fast index run) actually happened.
|
||||
pub fn fmt_ago(unix_secs: u64) -> String {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0);
|
||||
let age = now.saturating_sub(unix_secs);
|
||||
let age = quicksearch_core::log::now_unix().saturating_sub(unix_secs);
|
||||
if age < 60 {
|
||||
"just now".to_string()
|
||||
} else if age < 3600 {
|
||||
|
|
@ -59,7 +55,8 @@ pub fn fmt_interval(minutes: u64) -> String {
|
|||
if minutes.is_multiple_of(1440) {
|
||||
let days = minutes / 1440;
|
||||
return if days == 1 {
|
||||
// "24 h" reads better than "1 day" for the shipped default.
|
||||
// A staleness window reads better in hours than in days: "1 day"
|
||||
// invites rounding to "about a day", "24 h" does not.
|
||||
"24 h".to_string()
|
||||
} else {
|
||||
format!("{} days", days)
|
||||
|
|
@ -76,7 +73,7 @@ pub fn group_thousands(n: u64) -> String {
|
|||
let digits = n.to_string();
|
||||
let mut out = String::with_capacity(digits.len() + digits.len() / 3);
|
||||
for (i, c) in digits.chars().enumerate() {
|
||||
if i > 0 && (digits.len() - i) % 3 == 0 {
|
||||
if i > 0 && (digits.len() - i).is_multiple_of(3) {
|
||||
out.push(',');
|
||||
}
|
||||
out.push(c);
|
||||
|
|
@ -108,6 +105,50 @@ pub fn fmt_elapsed(d: std::time::Duration) -> String {
|
|||
}
|
||||
}
|
||||
|
||||
/// A running clock: `0:07`, `4:32`, `1:04:12`.
|
||||
///
|
||||
/// For work that is still going, where the question is "how long has this
|
||||
/// been like this?" — [`fmt_elapsed`] answers a different one and would
|
||||
/// render a twenty-minute wait as `1234.5 s`. Seconds are always two digits
|
||||
/// so the text does not change width every tick.
|
||||
pub fn fmt_duration_clock(d: std::time::Duration) -> String {
|
||||
let secs = d.as_secs();
|
||||
let (h, m, s) = (secs / 3600, (secs % 3600) / 60, secs % 60);
|
||||
if h > 0 {
|
||||
format!("{}:{:02}:{:02}", h, m, s)
|
||||
} else {
|
||||
format!("{}:{:02}", m, s)
|
||||
}
|
||||
}
|
||||
|
||||
/// What a finished configuration reconciliation did, in one line.
|
||||
///
|
||||
/// The counts are the point: a prune of a small index is over in a
|
||||
/// millisecond, so this line is the only evidence the user gets that the
|
||||
/// setting they changed reached the index at all. A clause whose count is
|
||||
/// zero is left out rather than printed as "0", and a pass that found nothing
|
||||
/// to change still reports that it ran — that it ran is the answer.
|
||||
pub fn fmt_reconcile_summary(deleted: usize, recontented: usize) -> String {
|
||||
let entries = |n: usize| {
|
||||
format!(
|
||||
"{} {}",
|
||||
group_thousands(n as u64),
|
||||
if n == 1 { "entry" } else { "entries" }
|
||||
)
|
||||
};
|
||||
let mut parts: Vec<String> = Vec::new();
|
||||
if deleted > 0 {
|
||||
parts.push(format!("{} removed", entries(deleted)));
|
||||
}
|
||||
if recontented > 0 {
|
||||
parts.push(format!("{} re-examined", entries(recontented)));
|
||||
}
|
||||
if parts.is_empty() {
|
||||
return "Configuration change applied".to_string();
|
||||
}
|
||||
format!("Configuration change applied · {}", parts.join(" · "))
|
||||
}
|
||||
|
||||
/// Middle-truncate a path to at most `max_chars` characters.
|
||||
pub fn middle_truncate(s: &str, max_chars: usize) -> String {
|
||||
let chars: Vec<char> = s.chars().collect();
|
||||
|
|
@ -141,10 +182,11 @@ mod tests {
|
|||
assert_eq!(fmt_interval(0), "run");
|
||||
assert_eq!(fmt_interval(1), "1 min");
|
||||
assert_eq!(fmt_interval(59), "59 min");
|
||||
assert_eq!(fmt_interval(60), "1 h");
|
||||
assert_eq!(fmt_interval(60), "1 h", "the shipped default");
|
||||
assert_eq!(fmt_interval(90), "1 h 30 min");
|
||||
assert_eq!(fmt_interval(120), "2 h");
|
||||
assert_eq!(fmt_interval(1440), "24 h", "the shipped default");
|
||||
// A whole day is the one multiple-of-1440 case that stays in hours.
|
||||
assert_eq!(fmt_interval(1440), "24 h");
|
||||
assert_eq!(fmt_interval(2880), "2 days");
|
||||
assert_eq!(fmt_interval(10_080), "7 days");
|
||||
}
|
||||
|
|
@ -179,10 +221,7 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn ago_buckets() {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs();
|
||||
let now = quicksearch_core::log::now_unix();
|
||||
assert_eq!(fmt_ago(now), "just now");
|
||||
assert_eq!(fmt_ago(now - 59), "just now");
|
||||
assert_eq!(fmt_ago(now - 120), "2 min ago");
|
||||
|
|
@ -190,6 +229,31 @@ mod tests {
|
|||
assert!(fmt_ago(now - 200_000).contains('-'), "old = absolute date");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reconcile_summaries_omit_what_did_not_happen() {
|
||||
assert_eq!(
|
||||
fmt_reconcile_summary(0, 0),
|
||||
"Configuration change applied",
|
||||
"a pass with nothing to do still reports that it ran"
|
||||
);
|
||||
assert_eq!(
|
||||
fmt_reconcile_summary(1, 0),
|
||||
"Configuration change applied · 1 entry removed"
|
||||
);
|
||||
assert_eq!(
|
||||
fmt_reconcile_summary(1204, 0),
|
||||
"Configuration change applied · 1,204 entries removed"
|
||||
);
|
||||
assert_eq!(
|
||||
fmt_reconcile_summary(0, 7),
|
||||
"Configuration change applied · 7 entries re-examined"
|
||||
);
|
||||
assert_eq!(
|
||||
fmt_reconcile_summary(2, 3),
|
||||
"Configuration change applied · 2 entries removed · 3 entries re-examined"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncation() {
|
||||
assert_eq!(middle_truncate("short", 20), "short");
|
||||
|
|
|
|||
|
|
@ -33,10 +33,16 @@ pub fn load_key(db_path: &str) -> Result<Option<String>, String> {
|
|||
}
|
||||
}
|
||||
|
||||
/// Forget the remembered key. Best-effort: an entry that never existed or
|
||||
/// a dead keychain daemon are both fine outcomes for "forget".
|
||||
pub fn delete_key(db_path: &str) {
|
||||
if let Ok(entry) = entry(db_path) {
|
||||
let _ = entry.delete_credential();
|
||||
/// Forget the remembered key. An entry that never existed is a fine outcome
|
||||
/// for "forget" and reports success.
|
||||
///
|
||||
/// A real failure is not: the derived SQLCipher key is still sitting in the
|
||||
/// OS keychain, so a caller that goes on to record "not remembered" would be
|
||||
/// describing a machine state that isn't true. Callers surface this rather
|
||||
/// than assuming the key is gone.
|
||||
pub fn delete_key(db_path: &str) -> Result<(), String> {
|
||||
match entry(db_path)?.delete_credential() {
|
||||
Ok(()) | Err(keyring::Error::NoEntry) => Ok(()),
|
||||
Err(e) => Err(format!("keychain delete failed: {}", e)),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,6 +24,9 @@ mod options;
|
|||
mod platform;
|
||||
mod query_highlight;
|
||||
mod search_tab;
|
||||
#[cfg(test)]
|
||||
mod test_ui;
|
||||
mod tips;
|
||||
mod tracker;
|
||||
mod ui_util;
|
||||
mod unlock;
|
||||
|
|
|
|||
|
|
@ -5,11 +5,18 @@ use std::path::Path;
|
|||
use std::time::{Duration, Instant};
|
||||
|
||||
use quicksearch_core::config::Config;
|
||||
use quicksearch_core::coordinator::{IndexMode, IndexerState, WatcherStatus};
|
||||
use quicksearch_core::indexing::{IndexingStatus, RootPhase, RootProgress};
|
||||
use quicksearch_core::coordinator::{IndexMode, IndexerState, ReconcileState, WatcherStatus};
|
||||
use quicksearch_core::indexing::{
|
||||
IndexingStatus, PrepStep, ReconcileProgress, RootPhase, RootProgress,
|
||||
};
|
||||
|
||||
use crate::format::{fmt_interval, fmt_rate, group_thousands, human_size, middle_truncate};
|
||||
use crate::format::{
|
||||
fmt_duration_clock, fmt_interval, fmt_rate, fmt_reconcile_summary, group_thousands, human_size,
|
||||
middle_truncate,
|
||||
};
|
||||
use crate::tips::{self, Tipped};
|
||||
use crate::tracker::SpeedTracker;
|
||||
use crate::ui_util::middle_elide;
|
||||
|
||||
/// What the tab asks the app to do after this frame.
|
||||
#[derive(Default)]
|
||||
|
|
@ -65,9 +72,13 @@ impl ManageTab {
|
|||
let total: usize = roots.iter().map(|r| r.walked + r.extracted).sum();
|
||||
self.speed.record(total);
|
||||
}
|
||||
IndexingStatus::Idle | IndexingStatus::Error(_) | IndexingStatus::Optimizing => {
|
||||
self.speed.reset()
|
||||
}
|
||||
// Preparing included: the prologue has its own counters and no
|
||||
// files to rate, and a stale files/sec left over from the last
|
||||
// run would read as progress that is not happening.
|
||||
IndexingStatus::Idle
|
||||
| IndexingStatus::Error(_)
|
||||
| IndexingStatus::Optimizing
|
||||
| IndexingStatus::Preparing { .. } => self.speed.reset(),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
|
@ -202,6 +213,7 @@ impl ManageTab {
|
|||
);
|
||||
if ui
|
||||
.add_enabled(!running, egui::Button::new("Start indexing now"))
|
||||
.tip(&tips::START_NOW)
|
||||
.clicked()
|
||||
{
|
||||
actions.start_now = true;
|
||||
|
|
@ -211,10 +223,7 @@ impl ManageTab {
|
|||
running || state.mode == IndexMode::Auto,
|
||||
egui::Button::new("Stop"),
|
||||
)
|
||||
.on_hover_text(
|
||||
"Stop indexing and switch to manual. Saved right away: it \
|
||||
stays manual on the next launch too.",
|
||||
)
|
||||
.tip(&tips::STOP_INDEXING)
|
||||
.clicked()
|
||||
{
|
||||
actions.stop = true;
|
||||
|
|
@ -224,10 +233,7 @@ impl ManageTab {
|
|||
state.mode != IndexMode::Auto,
|
||||
egui::Button::new("Return to Automatic"),
|
||||
)
|
||||
.on_hover_text(
|
||||
"Watch for changes and reindex periodically again. Also \
|
||||
saved, so this is how the app starts from now on.",
|
||||
)
|
||||
.tip(&tips::RETURN_TO_AUTO)
|
||||
.clicked()
|
||||
{
|
||||
actions.auto = true;
|
||||
|
|
@ -243,7 +249,7 @@ impl ManageTab {
|
|||
.button(
|
||||
egui::RichText::new("Clear index…").color(ui.visuals().error_fg_color),
|
||||
)
|
||||
.on_hover_text("Delete the index database (asks for confirmation)")
|
||||
.tip(&tips::CLEAR_INDEX)
|
||||
.clicked()
|
||||
{
|
||||
actions.clear_index = true;
|
||||
|
|
@ -268,7 +274,7 @@ impl ManageTab {
|
|||
// never push them out of view; the path truncates into
|
||||
// whatever width remains (full path on hover).
|
||||
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
|
||||
if ui.small_button("Remove").clicked() {
|
||||
if ui.small_button("Remove").tip(&tips::REMOVE_ROOT).clicked() {
|
||||
remove = Some(i);
|
||||
}
|
||||
// Per-root walker override; 0 = auto (4 local / 16
|
||||
|
|
@ -295,11 +301,7 @@ impl ManageTab {
|
|||
}
|
||||
}),
|
||||
)
|
||||
.on_hover_text(
|
||||
"Walker threads for this folder. auto = 4 on local \
|
||||
storage, 16 on network mounts. Takes effect on \
|
||||
the next indexing run.",
|
||||
);
|
||||
.tip(&tips::ROOT_WORKERS);
|
||||
#[cfg(test)]
|
||||
tests::record_widget("workers", &response);
|
||||
if response.changed() {
|
||||
|
|
@ -316,12 +318,9 @@ impl ManageTab {
|
|||
egui::Layout::left_to_right(egui::Align::Center),
|
||||
|ui| {
|
||||
let font_id = egui::TextStyle::Monospace.resolve(ui.style());
|
||||
let char_width =
|
||||
ui.fonts(|f| f.glyph_width(&font_id, '0')).max(1.0);
|
||||
let budget =
|
||||
((ui.available_width() / char_width) as usize).max(16);
|
||||
ui.monospace(middle_truncate(root, budget))
|
||||
.on_hover_text(root);
|
||||
let shown =
|
||||
middle_elide(ui, root, ui.available_width(), &font_id);
|
||||
ui.monospace(shown.as_ref()).on_hover_text(root);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
@ -332,7 +331,7 @@ impl ManageTab {
|
|||
draft.indexing.root_workers.remove(&removed);
|
||||
}
|
||||
ui.horizontal(|ui| {
|
||||
if ui.button("Add folder…").clicked() {
|
||||
if ui.button("Add folder…").tip(&tips::ADD_ROOT).clicked() {
|
||||
if let Some(dir) = rfd::FileDialog::new().pick_folder() {
|
||||
let path = dir.to_string_lossy().into_owned();
|
||||
try_add_root(draft, path, &mut self.root_error);
|
||||
|
|
@ -342,8 +341,11 @@ impl ManageTab {
|
|||
egui::TextEdit::singleline(&mut self.new_root)
|
||||
.desired_width(240.0)
|
||||
.hint_text("or type a path"),
|
||||
);
|
||||
if ui.button("Add").clicked() && !self.new_root.trim().is_empty() {
|
||||
)
|
||||
.tip(&tips::ADD_ROOT);
|
||||
if ui.button("Add").tip(&tips::ADD_ROOT).clicked()
|
||||
&& !self.new_root.trim().is_empty()
|
||||
{
|
||||
let path = self.new_root.trim().to_string();
|
||||
if try_add_root(draft, path, &mut self.root_error) {
|
||||
self.new_root.clear();
|
||||
|
|
@ -369,7 +371,9 @@ impl ManageTab {
|
|||
// --- Filters ---------------------------------------------------
|
||||
ui.heading(egui::RichText::new("Content filters").strong());
|
||||
ui.columns(2, |cols| {
|
||||
cols[0].label("Full-text extensions whitelist (empty = all supported):");
|
||||
cols[0]
|
||||
.label("Full-text extensions whitelist (empty = all supported):")
|
||||
.tip(&tips::EXT_WHITELIST);
|
||||
cols[0]
|
||||
.add(
|
||||
egui::TextEdit::multiline(&mut self.ext_filter_text)
|
||||
|
|
@ -377,15 +381,10 @@ impl ManageTab {
|
|||
.desired_width(f32::INFINITY)
|
||||
.hint_text("txt\nmd\npdf # comments allowed\n(none)"),
|
||||
)
|
||||
.on_hover_text(
|
||||
"One extension per line, leading dot optional. A non-empty \
|
||||
list also excludes files that have no extension at all \
|
||||
(Makefile, README, .bashrc) — add the line \"(none)\" to \
|
||||
keep extracting text from those.\n\n\
|
||||
\"#\" starts a comment, either on its own line or after an \
|
||||
entry, so a type can be commented out without losing it.",
|
||||
);
|
||||
cols[1].label("Ignore patterns (excluded entirely):");
|
||||
.tip(&tips::EXT_WHITELIST);
|
||||
cols[1]
|
||||
.label("Ignore patterns (excluded entirely):")
|
||||
.tip(&tips::IGNORE_PATTERNS);
|
||||
let mut remove_pat: Option<usize> = None;
|
||||
// The list grows and shrinks — including from outside
|
||||
// this tab — so it is kept off the id of the editor
|
||||
|
|
@ -425,7 +424,11 @@ impl ManageTab {
|
|||
);
|
||||
let submitted =
|
||||
response.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter));
|
||||
if ui.add_enabled(valid, egui::Button::new("Add")).clicked()
|
||||
response.tip(&tips::IGNORE_PATTERNS);
|
||||
if ui
|
||||
.add_enabled(valid, egui::Button::new("Add"))
|
||||
.tip(&tips::IGNORE_PATTERNS)
|
||||
.clicked()
|
||||
|| (submitted && valid)
|
||||
{
|
||||
let pat = self.new_ignore.trim().to_string();
|
||||
|
|
@ -463,14 +466,16 @@ impl ManageTab {
|
|||
|
||||
let dirty = self.is_dirty();
|
||||
ui.horizontal(|ui| {
|
||||
let apply = ui.add(crate::ui_util::bordered_button(
|
||||
let apply = ui
|
||||
.add(crate::ui_util::bordered_button(
|
||||
"Apply & Save",
|
||||
if dirty {
|
||||
crate::ui_util::ORANGE
|
||||
} else {
|
||||
crate::ui_util::BLUE
|
||||
},
|
||||
));
|
||||
))
|
||||
.tip(&tips::APPLY_SAVE);
|
||||
#[cfg(test)]
|
||||
tests::record_widget("apply", &apply);
|
||||
// The label comes and goes with the dirty state; keep it
|
||||
|
|
@ -572,7 +577,7 @@ fn db_size_tooltip(ui: &mut egui::Ui) {
|
|||
ui.set_max_width(440.0);
|
||||
ui.strong("To reduce the index size");
|
||||
for lever in [
|
||||
"Add ignore filters for files and folders you never search — the ignore \
|
||||
"Add ignore filters for files and folders you never search, in the ignore \
|
||||
pattern list further down this tab.",
|
||||
"Remove indexed folders you do not need, in Indexed folders above.",
|
||||
"Narrow the full-text extension whitelist, so text is only extracted \
|
||||
|
|
@ -588,7 +593,7 @@ fn db_size_tooltip(ui: &mut egui::Ui) {
|
|||
ui.label(
|
||||
egui::RichText::new(
|
||||
"Narrowing any of these removes the entries it excludes straight away, \
|
||||
but the file does not shrink on its own: the freed space is reused by \
|
||||
but the file does not shrink on its own. The freed space is reused by \
|
||||
the index rather than returned to the disk, until an indexing run's \
|
||||
optimize pass compacts it.",
|
||||
)
|
||||
|
|
@ -682,6 +687,21 @@ fn status_panel(ui: &mut egui::Ui, state: &IndexerState, speed: &SpeedTracker) {
|
|||
fn status_contents(ui: &mut egui::Ui, state: &IndexerState, speed: &SpeedTracker) {
|
||||
match &state.activity {
|
||||
IndexingStatus::Idle => {
|
||||
// A reconcile the coordinator is applying between runs. No run
|
||||
// owns the index, so the activity really is Idle — but the thread
|
||||
// is scanning every row, which on a large index is minutes of
|
||||
// work that used to be reported as nothing at all.
|
||||
match &state.reconcile {
|
||||
Some(ReconcileState::Running(r)) => return reconcile_row(ui, r, None),
|
||||
// Kept on screen for a few seconds after the work ends: a
|
||||
// narrowed filter on a small index is applied faster than the
|
||||
// display could show it happening.
|
||||
Some(ReconcileState::Finished(r)) => {
|
||||
ui.label(fmt_reconcile_summary(r.deleted, r.recontented));
|
||||
return;
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
// Relative wording makes even a milliseconds-fast run visibly
|
||||
// register ("just now") instead of looking like a dead button.
|
||||
let last = state
|
||||
|
|
@ -690,6 +710,9 @@ fn status_contents(ui: &mut egui::Ui, state: &IndexerState, speed: &SpeedTracker
|
|||
.unwrap_or_else(|| "never".to_string());
|
||||
ui.label(format!("Idle; last full index: {}", last));
|
||||
}
|
||||
IndexingStatus::Preparing { start_time, step } => {
|
||||
prep_row(ui, step, start_time.elapsed());
|
||||
}
|
||||
IndexingStatus::Error(e) => {
|
||||
ui.colored_label(ui.visuals().error_fg_color, format!("Error: {}", e));
|
||||
}
|
||||
|
|
@ -707,7 +730,11 @@ fn status_contents(ui: &mut egui::Ui, state: &IndexerState, speed: &SpeedTracker
|
|||
}
|
||||
if let Some(rate) = speed.files_per_sec() {
|
||||
ui.label(
|
||||
egui::RichText::new(format!("overall: {}", fmt_rate(rate)))
|
||||
egui::RichText::new(format!(
|
||||
"last {}s: {}",
|
||||
crate::tracker::WINDOW.as_secs(),
|
||||
fmt_rate(rate)
|
||||
))
|
||||
.small()
|
||||
.weak(),
|
||||
);
|
||||
|
|
@ -716,6 +743,99 @@ fn status_contents(ui: &mut egui::Ui, state: &IndexerState, speed: &SpeedTracker
|
|||
}
|
||||
}
|
||||
|
||||
/// What a run is doing before it walks its first file.
|
||||
///
|
||||
/// Every one of these can outlast the walk itself on a large index, and they
|
||||
/// all used to render as one motionless "Starting…". The elapsed clock is the
|
||||
/// point of the row: it is the only thing that distinguishes slow work from a
|
||||
/// hang, and it is why each label carries one even when there is nothing to
|
||||
/// count.
|
||||
fn prep_row(ui: &mut egui::Ui, step: &PrepStep, elapsed: Duration) {
|
||||
match step {
|
||||
PrepStep::PreviousRun => waiting_row(ui, "Finishing the previous run…", elapsed),
|
||||
PrepStep::OpeningIndex => waiting_row(ui, "Opening the index…", elapsed),
|
||||
PrepStep::Reconciling(r) => reconcile_row(ui, r, Some(elapsed)),
|
||||
}
|
||||
}
|
||||
|
||||
/// A prologue step with no counters: label, clock, indeterminate bar.
|
||||
fn waiting_row(ui: &mut egui::Ui, label: &str, elapsed: Duration) {
|
||||
ui.horizontal(|ui| {
|
||||
ui.label(label);
|
||||
ui.label(
|
||||
egui::RichText::new(fmt_duration_clock(elapsed))
|
||||
.small()
|
||||
.weak(),
|
||||
);
|
||||
ui.add(
|
||||
egui::ProgressBar::new(0.0)
|
||||
.animate(true)
|
||||
.desired_width(160.0),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// A configuration reconciliation, from either place one runs.
|
||||
///
|
||||
/// `elapsed` is `Some` for a run's prologue, which has a start time to
|
||||
/// measure from; the coordinator's between-runs pass has none, and its
|
||||
/// examined count moves often enough to serve the same purpose.
|
||||
fn reconcile_row(ui: &mut egui::Ui, r: &ReconcileProgress, elapsed: Option<Duration>) {
|
||||
ui.horizontal(|ui| {
|
||||
ui.label("Applying configuration change");
|
||||
ui.label(egui::RichText::new("|").weak());
|
||||
match (r.total, r.fraction()) {
|
||||
(Some(total), Some(frac)) => {
|
||||
ui.label(format!(
|
||||
"{} / {} ({:.0}%) entries checked",
|
||||
group_thousands(r.examined as u64),
|
||||
group_thousands(total as u64),
|
||||
frac * 100.0
|
||||
));
|
||||
}
|
||||
_ => {
|
||||
ui.label(format!(
|
||||
"{} entries checked",
|
||||
group_thousands(r.examined as u64)
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Some(elapsed) = elapsed {
|
||||
ui.label(
|
||||
egui::RichText::new(fmt_duration_clock(elapsed))
|
||||
.small()
|
||||
.weak(),
|
||||
);
|
||||
}
|
||||
match r.fraction() {
|
||||
Some(frac) => {
|
||||
ui.add(egui::ProgressBar::new(frac as f32).desired_width(160.0));
|
||||
}
|
||||
// Whole-range deletions read no rows, so they reach the bar with
|
||||
// no denominator — the same indeterminate form a walk uses before
|
||||
// its count lands.
|
||||
None => {
|
||||
ui.add(
|
||||
egui::ProgressBar::new(0.0)
|
||||
.animate(true)
|
||||
.desired_width(160.0),
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
if r.deleted > 0 || r.recontented > 0 {
|
||||
ui.label(
|
||||
egui::RichText::new(format!(
|
||||
"{} entries removed, {} re-examined for text extraction",
|
||||
group_thousands(r.deleted as u64),
|
||||
group_thousands(r.recontented as u64)
|
||||
))
|
||||
.small()
|
||||
.weak(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// One root's progress: path, phase, bar, counters, current file.
|
||||
fn root_row(ui: &mut egui::Ui, r: &RootProgress) {
|
||||
// Weak "|" separators split the row into folder | status | numbers.
|
||||
|
|
@ -828,8 +948,10 @@ mod tests {
|
|||
mode: IndexMode::Auto,
|
||||
activity: IndexingStatus::Idle,
|
||||
last_full_index: Some(0),
|
||||
files: Some(0),
|
||||
queued_events: 0,
|
||||
watcher: WatcherStatus::Active { dirs: 10 },
|
||||
reconcile: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -858,28 +980,31 @@ mod tests {
|
|||
/// contents — not just their widget ids — are under test.
|
||||
fn state_with(roots: Vec<RootProgress>) -> IndexerState {
|
||||
IndexerState {
|
||||
mode: IndexMode::Auto,
|
||||
activity: IndexingStatus::Running {
|
||||
start_time: std::time::Instant::now(),
|
||||
roots,
|
||||
},
|
||||
last_full_index: Some(0),
|
||||
queued_events: 0,
|
||||
watcher: WatcherStatus::Active { dirs: 10 },
|
||||
..idle_state()
|
||||
}
|
||||
}
|
||||
|
||||
fn raw_input(events: Vec<egui::Event>) -> egui::RawInput {
|
||||
egui::RawInput {
|
||||
screen_rect: Some(egui::Rect::from_min_size(
|
||||
egui::Pos2::ZERO,
|
||||
egui::vec2(1000.0, 900.0),
|
||||
)),
|
||||
events,
|
||||
..Default::default()
|
||||
/// A run still in its prologue, before the walk has produced anything.
|
||||
fn preparing_state(step: PrepStep) -> IndexerState {
|
||||
IndexerState {
|
||||
activity: IndexingStatus::Preparing {
|
||||
start_time: std::time::Instant::now(),
|
||||
step,
|
||||
},
|
||||
..idle_state()
|
||||
}
|
||||
}
|
||||
|
||||
use crate::test_ui::{click_at, painted_text};
|
||||
|
||||
fn raw_input(events: Vec<egui::Event>) -> egui::RawInput {
|
||||
crate::test_ui::raw_input(egui::vec2(1000.0, 900.0), events)
|
||||
}
|
||||
|
||||
/// One frame of the real tab, with `events` delivered to it.
|
||||
fn frame(
|
||||
ctx: &egui::Context,
|
||||
|
|
@ -917,23 +1042,7 @@ mod tests {
|
|||
tab.ui(ui, state, cfg);
|
||||
});
|
||||
});
|
||||
let mut text = Vec::new();
|
||||
for clipped in &out.shapes {
|
||||
collect_text(&clipped.shape, &mut text);
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
fn collect_text(shape: &egui::epaint::Shape, out: &mut Vec<String>) {
|
||||
match shape {
|
||||
egui::epaint::Shape::Text(t) => out.push(t.galley.text().to_string()),
|
||||
egui::epaint::Shape::Vec(shapes) => {
|
||||
for s in shapes {
|
||||
collect_text(s, out);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
painted_text(&out)
|
||||
}
|
||||
|
||||
fn pointer(pos: egui::Pos2, pressed: bool) -> egui::Event {
|
||||
|
|
@ -945,14 +1054,6 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
fn click_at(pos: egui::Pos2) -> Vec<egui::Event> {
|
||||
vec![
|
||||
egui::Event::PointerMoved(pos),
|
||||
pointer(pos, true),
|
||||
pointer(pos, false),
|
||||
]
|
||||
}
|
||||
|
||||
fn cfg_with_root() -> Config {
|
||||
let mut cfg = Config::default();
|
||||
cfg.paths.indexing_paths = vec!["/data".into()];
|
||||
|
|
@ -1181,14 +1282,154 @@ mod tests {
|
|||
assert!(!text.contains(" / "), "invented a denominator: {}", text);
|
||||
}
|
||||
|
||||
/// Every step of the prologue names itself. These used to be one static
|
||||
/// "Starting…" with no clock, which on a large index is indistinguishable
|
||||
/// from a hang for as long as the step takes.
|
||||
#[test]
|
||||
fn each_prologue_step_says_what_it_is_waiting_on() {
|
||||
let ctx = egui::Context::default();
|
||||
let mut tab = ManageTab::new();
|
||||
|
||||
for (step, expected) in [
|
||||
(PrepStep::PreviousRun, "Finishing the previous run…"),
|
||||
(PrepStep::OpeningIndex, "Opening the index…"),
|
||||
] {
|
||||
let text = frame_text(&ctx, &mut tab, &preparing_state(step)).join(" | ");
|
||||
assert!(text.contains(expected), "{}", text);
|
||||
// The clock is the point of the row.
|
||||
assert!(text.contains("0:00"), "no elapsed time shown: {}", text);
|
||||
}
|
||||
}
|
||||
|
||||
/// The reconcile is the long one, and the only prologue step with
|
||||
/// something to count. It reports its position in the scan.
|
||||
#[test]
|
||||
fn a_reconcile_reports_how_far_through_the_index_it_is() {
|
||||
let ctx = egui::Context::default();
|
||||
let mut tab = ManageTab::new();
|
||||
let text = frame_text(
|
||||
&ctx,
|
||||
&mut tab,
|
||||
&preparing_state(PrepStep::Reconciling(ReconcileProgress {
|
||||
examined: 2_500_000,
|
||||
total: Some(8_000_000),
|
||||
deleted: 1_204,
|
||||
recontented: 0,
|
||||
})),
|
||||
)
|
||||
.join(" | ");
|
||||
|
||||
assert!(text.contains("Applying configuration change"), "{}", text);
|
||||
assert!(
|
||||
text.contains("2,500,000 / 8,000,000 (31%) entries checked"),
|
||||
"{}",
|
||||
text
|
||||
);
|
||||
assert!(text.contains("1,204 entries removed"), "{}", text);
|
||||
}
|
||||
|
||||
/// Whole-range deletions read no rows, so the scan can reach the display
|
||||
/// with nothing to divide by. It must not invent a denominator — the same
|
||||
/// rule the walk follows before its count lands.
|
||||
#[test]
|
||||
fn a_reconcile_without_a_row_count_shows_no_denominator() {
|
||||
let ctx = egui::Context::default();
|
||||
let mut tab = ManageTab::new();
|
||||
let text = frame_text(
|
||||
&ctx,
|
||||
&mut tab,
|
||||
&preparing_state(PrepStep::Reconciling(ReconcileProgress::default())),
|
||||
)
|
||||
.join(" | ");
|
||||
assert!(text.contains("0 entries checked"), "{}", text);
|
||||
assert!(!text.contains(" / "), "invented a denominator: {}", text);
|
||||
}
|
||||
|
||||
/// The coordinator applies a prune between runs, so no run owns the index
|
||||
/// and the activity really is `Idle` — but the thread is scanning every
|
||||
/// row of it. Reporting "Idle" for the minutes that takes is what made a
|
||||
/// working app look like a stopped one.
|
||||
#[test]
|
||||
fn a_prune_between_runs_is_reported_instead_of_idle() {
|
||||
let ctx = egui::Context::default();
|
||||
let mut tab = ManageTab::new();
|
||||
let state = IndexerState {
|
||||
reconcile: Some(ReconcileState::Running(ReconcileProgress {
|
||||
examined: 40_000,
|
||||
total: Some(80_000),
|
||||
deleted: 0,
|
||||
recontented: 0,
|
||||
})),
|
||||
..idle_state()
|
||||
};
|
||||
|
||||
let text = frame_text(&ctx, &mut tab, &state).join(" | ");
|
||||
assert!(
|
||||
text.contains("Applying configuration change"),
|
||||
"the scan is invisible: {}",
|
||||
text
|
||||
);
|
||||
assert!(text.contains("40,000 / 80,000 (50%)"), "{}", text);
|
||||
assert!(
|
||||
!text.contains("Idle; last full index"),
|
||||
"reported idle while scanning: {}",
|
||||
text
|
||||
);
|
||||
}
|
||||
|
||||
/// The pass itself can be over between two frames — narrowing a filter on
|
||||
/// a small index is one transaction. Without a tail the user changes a
|
||||
/// setting, sees nothing move, and has no way to tell whether it took.
|
||||
#[test]
|
||||
fn a_finished_prune_reports_what_it_did() {
|
||||
let ctx = egui::Context::default();
|
||||
let mut tab = ManageTab::new();
|
||||
let state = IndexerState {
|
||||
reconcile: Some(ReconcileState::Finished(ReconcileProgress {
|
||||
examined: 80_000,
|
||||
total: Some(80_000),
|
||||
deleted: 1_204,
|
||||
recontented: 7,
|
||||
})),
|
||||
..idle_state()
|
||||
};
|
||||
|
||||
let text = frame_text(&ctx, &mut tab, &state).join(" | ");
|
||||
assert!(
|
||||
text.contains("Configuration change applied"),
|
||||
"the finished pass left no trace: {}",
|
||||
text
|
||||
);
|
||||
assert!(text.contains("1,204 entries removed"), "{}", text);
|
||||
assert!(text.contains("7 entries re-examined"), "{}", text);
|
||||
assert!(
|
||||
!text.contains("Idle; last full index"),
|
||||
"the summary was replaced by the idle line: {}",
|
||||
text
|
||||
);
|
||||
}
|
||||
|
||||
/// The placeholder this whole prologue display replaced. It carried no
|
||||
/// phase, no counters and no clock, and it was on screen for every one of
|
||||
/// the steps above.
|
||||
#[test]
|
||||
fn the_starting_placeholder_is_gone() {
|
||||
let ctx = egui::Context::default();
|
||||
let mut tab = ManageTab::new();
|
||||
for state in [
|
||||
preparing_state(PrepStep::PreviousRun),
|
||||
preparing_state(PrepStep::OpeningIndex),
|
||||
preparing_state(PrepStep::Reconciling(ReconcileProgress::default())),
|
||||
idle_state(),
|
||||
] {
|
||||
let text = frame_text(&ctx, &mut tab, &state).join(" | ");
|
||||
assert!(!text.contains("Starting"), "{}", text);
|
||||
}
|
||||
}
|
||||
|
||||
/// A scratch directory of its own for each size test, so two of them
|
||||
/// running in parallel cannot see each other's files.
|
||||
fn scratch_dir(tag: &str) -> std::path::PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("qs-dbsize-{}-{}", std::process::id(), tag));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).expect("scratch dir");
|
||||
dir
|
||||
}
|
||||
use quicksearch_core::testutil::scratch_dir;
|
||||
|
||||
fn write_bytes(path: &Path, len: usize) {
|
||||
std::fs::write(path, vec![b'x'; len]).expect("write");
|
||||
|
|
@ -1600,4 +1841,40 @@ mod tests {
|
|||
);
|
||||
assert_eq!(widget("apply").0, clean_id, "the label renamed the button");
|
||||
}
|
||||
|
||||
/// The status area gains a row while a prune runs, another while its
|
||||
/// summary lingers, and loses both afterwards. Every one of those frames
|
||||
/// must leave the widgets below it with the ids they had: an editor whose
|
||||
/// id changes mid-edit loses its buffer, and a prune is exactly when the
|
||||
/// user is looking at the filter list they just changed.
|
||||
#[test]
|
||||
fn the_prune_rows_come_and_go_without_renaming_anything_below() {
|
||||
let ctx = egui::Context::default();
|
||||
let mut tab = ManageTab::new();
|
||||
let cfg = cfg_with_root();
|
||||
let progress = ReconcileProgress {
|
||||
examined: 40_000,
|
||||
total: Some(80_000),
|
||||
deleted: 12,
|
||||
recontented: 0,
|
||||
};
|
||||
|
||||
frame_text_with(&ctx, &mut tab, &cfg, &idle_state());
|
||||
let (apply, _) = widget("apply");
|
||||
let (workers, _) = widget("workers");
|
||||
|
||||
for reconcile in [
|
||||
Some(ReconcileState::Running(progress)),
|
||||
Some(ReconcileState::Finished(progress)),
|
||||
None,
|
||||
] {
|
||||
let state = IndexerState {
|
||||
reconcile,
|
||||
..idle_state()
|
||||
};
|
||||
frame_text_with(&ctx, &mut tab, &cfg, &state);
|
||||
assert_eq!(widget("apply").0, apply, "renamed by {reconcile:?}");
|
||||
assert_eq!(widget("workers").0, workers, "renamed by {reconcile:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
//! validates, saves, and hands the new config to the app.
|
||||
|
||||
use crate::keychain;
|
||||
use crate::tips::{self, tip_row, Tipped};
|
||||
use quicksearch_core::config::Config;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
|
|
@ -144,12 +145,12 @@ impl OptionsWindow {
|
|||
.show(ui, |ui| {
|
||||
ui.heading(egui::RichText::new("Paths").strong());
|
||||
egui::Grid::new("opt-paths").num_columns(2).show(ui, |ui| {
|
||||
ui.label("Database file");
|
||||
tip_row(ui, "Database file", &tips::DATABASE_PATH, |ui| {
|
||||
ui.add(
|
||||
egui::TextEdit::singleline(&mut draft.paths.database_path)
|
||||
.desired_width(260.0),
|
||||
);
|
||||
ui.end_row();
|
||||
)
|
||||
});
|
||||
});
|
||||
ui.label(
|
||||
egui::RichText::new(
|
||||
|
|
@ -182,18 +183,13 @@ impl OptionsWindow {
|
|||
|
||||
ui.heading(egui::RichText::new("Interface").strong());
|
||||
egui::Grid::new("opt-ui").num_columns(2).show(ui, |ui| {
|
||||
ui.label("UI scale");
|
||||
tip_row(ui, "UI scale", &tips::UI_SCALE, |ui| {
|
||||
ui.add(
|
||||
egui::Slider::new(&mut draft.ui.scale, 0.5..=2.5)
|
||||
.step_by(0.05)
|
||||
.fixed_decimals(2),
|
||||
)
|
||||
.on_hover_text(
|
||||
"Zooms the whole interface: fonts, spacing, and \
|
||||
widgets. Ctrl +/- and Ctrl 0 adjust it temporarily \
|
||||
at runtime.",
|
||||
);
|
||||
ui.end_row();
|
||||
});
|
||||
});
|
||||
ui.separator();
|
||||
|
||||
|
|
@ -208,14 +204,16 @@ impl OptionsWindow {
|
|||
|
||||
ui.separator();
|
||||
ui.horizontal(|ui| {
|
||||
let apply = ui.add(crate::ui_util::bordered_button(
|
||||
let apply = ui
|
||||
.add(crate::ui_util::bordered_button(
|
||||
"Apply & Save",
|
||||
if dirty {
|
||||
crate::ui_util::ORANGE
|
||||
} else {
|
||||
crate::ui_util::BLUE
|
||||
},
|
||||
));
|
||||
))
|
||||
.tip(&tips::APPLY_SAVE);
|
||||
if apply.clicked() {
|
||||
out.applied = Some(draft.clone());
|
||||
}
|
||||
|
|
@ -264,27 +262,36 @@ fn security_ui(
|
|||
ui.label("The index is encrypted; a password is required at startup.");
|
||||
}
|
||||
ui.horizontal(|ui| {
|
||||
if ui.button("Change password…").clicked() {
|
||||
if ui
|
||||
.button("Change password…")
|
||||
.tip(&tips::CHANGE_PASSWORD)
|
||||
.clicked()
|
||||
{
|
||||
action = Some(SecurityAction::ChangePassword);
|
||||
}
|
||||
if ui.button("Disable protection…").clicked() {
|
||||
if ui
|
||||
.button("Disable protection…")
|
||||
.tip(&tips::DISABLE_PASSWORD)
|
||||
.clicked()
|
||||
{
|
||||
action = Some(SecurityAction::Disable);
|
||||
}
|
||||
});
|
||||
let mut remember = current.security.use_keychain;
|
||||
if ui
|
||||
.checkbox(&mut remember, "Remember on this device")
|
||||
.on_hover_text(
|
||||
"Stores the derived key (not the password) in the OS keychain \
|
||||
and skips the startup prompt on this machine.",
|
||||
)
|
||||
.tip(&tips::REMEMBER_KEYCHAIN)
|
||||
.changed()
|
||||
{
|
||||
action = Some(SecurityAction::SetKeychain(remember));
|
||||
}
|
||||
} else {
|
||||
ui.label("The index is not encrypted.");
|
||||
if ui.button("Enable password protection…").clicked() {
|
||||
if ui
|
||||
.button("Enable password protection…")
|
||||
.tip(&tips::ENABLE_PASSWORD)
|
||||
.clicked()
|
||||
{
|
||||
action = Some(SecurityAction::Enable);
|
||||
}
|
||||
ui.label(
|
||||
|
|
@ -299,8 +306,11 @@ fn security_ui(
|
|||
action
|
||||
}
|
||||
|
||||
/// One implementation of the per-section config controls, shared by the
|
||||
/// Options window and the Manage tab.
|
||||
/// The per-section config controls of the Options window.
|
||||
///
|
||||
/// Every row goes through [`crate::tips::tip_row`], which takes the tooltip
|
||||
/// that explains it: a setting cannot arrive here without one, and hovering
|
||||
/// the name works as well as hovering the control.
|
||||
pub fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section) {
|
||||
match section {
|
||||
Section::Indexing => {
|
||||
|
|
@ -311,30 +321,31 @@ pub fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section
|
|||
// state, switched (and saved) by the Stop / Return to
|
||||
// Automatic buttons on the Manage Index tab. A staged copy
|
||||
// of it here would fight those buttons.
|
||||
ui.label("Full reindex every");
|
||||
tip_row(ui, "Full reindex every", &tips::REINDEX_INTERVAL, |ui| {
|
||||
ui.horizontal(|ui| {
|
||||
ui.add(
|
||||
egui::DragValue::new(&mut config.indexing.reindex_interval_minutes)
|
||||
.range(5..=60 * 24 * 30),
|
||||
);
|
||||
ui.label("minutes");
|
||||
})
|
||||
.response
|
||||
});
|
||||
ui.end_row();
|
||||
|
||||
ui.label("Follow symlinks");
|
||||
ui.checkbox(&mut config.indexing.follow_symlinks, "");
|
||||
ui.end_row();
|
||||
tip_row(ui, "Follow symlinks", &tips::FOLLOW_SYMLINKS, |ui| {
|
||||
ui.checkbox(&mut config.indexing.follow_symlinks, "")
|
||||
});
|
||||
|
||||
ui.label("Include hidden files");
|
||||
ui.checkbox(&mut config.indexing.include_hidden, "");
|
||||
ui.end_row();
|
||||
tip_row(ui, "Include hidden files", &tips::INCLUDE_HIDDEN, |ui| {
|
||||
ui.checkbox(&mut config.indexing.include_hidden, "")
|
||||
});
|
||||
});
|
||||
}
|
||||
Section::Processing => {
|
||||
egui::Grid::new("cfg-processing")
|
||||
.num_columns(2)
|
||||
.show(ui, |ui| {
|
||||
ui.label("Tokenizer");
|
||||
tip_row(ui, "Tokenizer", &tips::TOKENIZER, |ui| {
|
||||
egui::ComboBox::from_id_salt("cfg-tokenize")
|
||||
.selected_text(&config.processing.tokenize)
|
||||
.show_ui(ui, |ui| {
|
||||
|
|
@ -345,8 +356,9 @@ pub fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section
|
|||
opt,
|
||||
);
|
||||
}
|
||||
})
|
||||
.response
|
||||
});
|
||||
ui.end_row();
|
||||
|
||||
ui.label("");
|
||||
ui.hyperlink_to(
|
||||
|
|
@ -355,101 +367,85 @@ pub fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section
|
|||
);
|
||||
ui.end_row();
|
||||
|
||||
ui.label("Hash sample size (bytes)");
|
||||
tip_row(ui, "Hash sample size (bytes)", &tips::HASH_LENGTH, |ui| {
|
||||
ui.add(
|
||||
egui::DragValue::new(&mut config.processing.hash_length)
|
||||
.range(512..=1_048_576),
|
||||
);
|
||||
ui.end_row();
|
||||
)
|
||||
});
|
||||
|
||||
ui.label("Max stored text (bytes)");
|
||||
tip_row(
|
||||
ui,
|
||||
"Max stored text (bytes)",
|
||||
&tips::MAX_STORED_TEXT,
|
||||
|ui| {
|
||||
ui.add(
|
||||
egui::DragValue::new(&mut config.processing.maximum_text_size)
|
||||
.range(1024..=16_777_216),
|
||||
)
|
||||
},
|
||||
);
|
||||
ui.end_row();
|
||||
|
||||
ui.label("Max text file size (bytes)");
|
||||
tip_row(
|
||||
ui,
|
||||
"Max text file size (bytes)",
|
||||
&tips::MAX_TEXT_FILE_SIZE,
|
||||
|ui| {
|
||||
ui.add(
|
||||
egui::DragValue::new(&mut config.processing.maximum_text_file_size)
|
||||
.range(1024..=1_073_741_824),
|
||||
)
|
||||
},
|
||||
);
|
||||
ui.end_row();
|
||||
|
||||
ui.label("Batch size");
|
||||
tip_row(ui, "Batch size", &tips::BATCH_SIZE, |ui| {
|
||||
ui.add(
|
||||
egui::DragValue::new(&mut config.processing.batch_size).range(10..=100_000),
|
||||
);
|
||||
ui.end_row();
|
||||
egui::DragValue::new(&mut config.processing.batch_size)
|
||||
.range(10..=100_000),
|
||||
)
|
||||
});
|
||||
|
||||
ui.label("Max WAL size (bytes)");
|
||||
tip_row(ui, "Max WAL size (bytes)", &tips::MAX_WAL_SIZE, |ui| {
|
||||
ui.add(
|
||||
egui::DragValue::new(&mut config.processing.maximum_wal_size)
|
||||
.range(0u64..=8_589_934_592u64),
|
||||
)
|
||||
.on_hover_text(
|
||||
"How large index.sqlite-wal may grow during a run before the \
|
||||
indexer forces a checkpoint. 0 disables forced checkpoints; \
|
||||
anything below 16 MiB is raised to it.",
|
||||
);
|
||||
ui.end_row();
|
||||
});
|
||||
|
||||
ui.label("Store text for snippets");
|
||||
tip_row(ui, "Store text for snippets", &tips::STORE_TEXT, |ui| {
|
||||
ui.checkbox(&mut config.processing.store_text_for_snippets, "")
|
||||
.on_hover_text(
|
||||
"Off: smaller index, but no previews, occurrence ranking, \
|
||||
case verification, or fuzzy full-text search",
|
||||
);
|
||||
ui.end_row();
|
||||
});
|
||||
});
|
||||
}
|
||||
Section::Search => {
|
||||
egui::Grid::new("cfg-search").num_columns(2).show(ui, |ui| {
|
||||
ui.label("Fuzzy search ON by default");
|
||||
ui.checkbox(&mut config.search.fuzzy_default, "");
|
||||
ui.end_row();
|
||||
tip_row(
|
||||
ui,
|
||||
"Fuzzy search ON by default",
|
||||
&tips::FUZZY_DEFAULT,
|
||||
|ui| ui.checkbox(&mut config.search.fuzzy_default, ""),
|
||||
);
|
||||
|
||||
ui.label("Fuzzy edit distance");
|
||||
ui.vertical(|ui| {
|
||||
tip_row(ui, "Fuzzy edit distance", &tips::FUZZY_EDITS, |ui| {
|
||||
ui.add(egui::DragValue::new(&mut config.search.fuzzy_max_edits).range(0..=8))
|
||||
.on_hover_text(
|
||||
"Ceiling on the typo budget. The allowance grows with the \
|
||||
search term, one edit per three characters, up to this \
|
||||
value, so 2 means \"1 edit for short terms, 2 for longer \
|
||||
ones\". 0 turns the fuzzy stages off.",
|
||||
);
|
||||
if let Some(warning) = config.search.fuzzy_edits_warning() {
|
||||
ui.label(
|
||||
egui::RichText::new(warning)
|
||||
.small()
|
||||
.color(crate::ui_util::ORANGE),
|
||||
);
|
||||
}
|
||||
});
|
||||
ui.end_row();
|
||||
|
||||
ui.label("Display limit");
|
||||
ui.add(egui::DragValue::new(&mut config.search.display_limit).range(50..=100_000));
|
||||
ui.end_row();
|
||||
|
||||
ui.label("Stream batch size");
|
||||
tip_row(ui, "Display limit", &tips::DISPLAY_LIMIT, |ui| {
|
||||
ui.add(
|
||||
egui::DragValue::new(&mut config.search.results_per_page).range(10..=10_000),
|
||||
);
|
||||
ui.end_row();
|
||||
egui::DragValue::new(&mut config.search.display_limit).range(50..=100_000),
|
||||
)
|
||||
});
|
||||
|
||||
ui.label("Debounce (ms)");
|
||||
ui.add(egui::DragValue::new(&mut config.search.debounce_ms).range(0..=2000));
|
||||
ui.end_row();
|
||||
tip_row(ui, "Stream batch size", &tips::RESULTS_PER_PAGE, |ui| {
|
||||
ui.add(
|
||||
egui::DragValue::new(&mut config.search.results_per_page)
|
||||
.range(10..=10_000),
|
||||
)
|
||||
});
|
||||
|
||||
ui.label("Fuzzy max edits");
|
||||
ui.add(egui::DragValue::new(&mut config.search.fuzzy_max_edits).range(0..=8))
|
||||
.on_hover_text(
|
||||
"Ceiling on fuzzy edit distance (the budget grows one \
|
||||
edit per three characters of the term). 0 disables \
|
||||
the fuzzy passes.",
|
||||
);
|
||||
ui.end_row();
|
||||
tip_row(ui, "Debounce (ms)", &tips::DEBOUNCE, |ui| {
|
||||
ui.add(egui::DragValue::new(&mut config.search.debounce_ms).range(0..=2000))
|
||||
});
|
||||
});
|
||||
// A warning that comes and goes as the value is edited would
|
||||
// otherwise move every widget below it in the window; see
|
||||
|
|
@ -525,29 +521,151 @@ mod tests {
|
|||
assert!(w.draft.is_none());
|
||||
}
|
||||
|
||||
/// Where `needle` was painted this frame, as the center of its galley —
|
||||
/// a click target that follows the layout instead of pinning it.
|
||||
fn painted_text_center(out: &egui::FullOutput, needle: &str) -> Option<egui::Pos2> {
|
||||
fn walk(shape: &egui::epaint::Shape, needle: &str, found: &mut Option<egui::Pos2>) {
|
||||
match shape {
|
||||
egui::epaint::Shape::Text(t) => {
|
||||
if t.galley.text() == needle {
|
||||
*found = Some(t.pos + t.galley.size() / 2.0);
|
||||
use crate::test_ui::{painted_text, painted_text_center};
|
||||
|
||||
/// Every row of every section, with the tip it must show. `tip_row`
|
||||
/// makes a row without *a* tooltip impossible; this table is what makes
|
||||
/// a row with the *wrong* one impossible.
|
||||
const ROWS: &[(Section, &str, &tips::Tip)] = &[
|
||||
(
|
||||
Section::Indexing,
|
||||
"Full reindex every",
|
||||
&tips::REINDEX_INTERVAL,
|
||||
),
|
||||
(Section::Indexing, "Follow symlinks", &tips::FOLLOW_SYMLINKS),
|
||||
(
|
||||
Section::Indexing,
|
||||
"Include hidden files",
|
||||
&tips::INCLUDE_HIDDEN,
|
||||
),
|
||||
(Section::Processing, "Tokenizer", &tips::TOKENIZER),
|
||||
(
|
||||
Section::Processing,
|
||||
"Hash sample size (bytes)",
|
||||
&tips::HASH_LENGTH,
|
||||
),
|
||||
(
|
||||
Section::Processing,
|
||||
"Max stored text (bytes)",
|
||||
&tips::MAX_STORED_TEXT,
|
||||
),
|
||||
(
|
||||
Section::Processing,
|
||||
"Max text file size (bytes)",
|
||||
&tips::MAX_TEXT_FILE_SIZE,
|
||||
),
|
||||
(Section::Processing, "Batch size", &tips::BATCH_SIZE),
|
||||
(
|
||||
Section::Processing,
|
||||
"Max WAL size (bytes)",
|
||||
&tips::MAX_WAL_SIZE,
|
||||
),
|
||||
(
|
||||
Section::Processing,
|
||||
"Store text for snippets",
|
||||
&tips::STORE_TEXT,
|
||||
),
|
||||
(
|
||||
Section::Search,
|
||||
"Fuzzy search ON by default",
|
||||
&tips::FUZZY_DEFAULT,
|
||||
),
|
||||
(Section::Search, "Fuzzy edit distance", &tips::FUZZY_EDITS),
|
||||
(Section::Search, "Display limit", &tips::DISPLAY_LIMIT),
|
||||
(
|
||||
Section::Search,
|
||||
"Stream batch size",
|
||||
&tips::RESULTS_PER_PAGE,
|
||||
),
|
||||
(Section::Search, "Debounce (ms)", &tips::DEBOUNCE),
|
||||
];
|
||||
|
||||
/// Hovering a row's name paints that row's own explanation. Rendered
|
||||
/// without the window's scroll area so nothing sits below the fold.
|
||||
#[test]
|
||||
fn every_row_shows_its_own_tip() {
|
||||
for (section, label, tip) in ROWS {
|
||||
let ctx = egui::Context::default();
|
||||
ctx.style_mut(|s| {
|
||||
s.interaction.tooltip_delay = 0.0;
|
||||
s.interaction.show_tooltips_only_when_still = false;
|
||||
});
|
||||
let mut cfg = Config::default();
|
||||
let mut run = |events: Vec<egui::Event>| {
|
||||
let input = crate::test_ui::raw_input(egui::vec2(600.0, 800.0), events);
|
||||
ctx.run(input, |ctx| {
|
||||
egui::CentralPanel::default()
|
||||
.show(ctx, |ui| config_editor_ui(ui, &mut cfg, *section));
|
||||
})
|
||||
};
|
||||
|
||||
let first = run(vec![]);
|
||||
let pos = painted_text_center(&first, label)
|
||||
.unwrap_or_else(|| panic!("{label} was not painted"));
|
||||
|
||||
// Enough of the body to be unique, and short enough to survive
|
||||
// an edit to the sentence it starts.
|
||||
let opening: String = tip.body.chars().take(40).collect();
|
||||
let mut out = run(vec![egui::Event::PointerMoved(pos)]);
|
||||
let mut found = false;
|
||||
for _ in 0..3 {
|
||||
// The tooltip is an area of its own, so it can land a frame
|
||||
// late.
|
||||
if painted_text(&out).join("\n").contains(&opening) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
out = run(vec![]);
|
||||
}
|
||||
assert!(found, "hovering {label:?} did not show {:?}", tip.title);
|
||||
}
|
||||
}
|
||||
egui::epaint::Shape::Vec(v) => {
|
||||
for s in v {
|
||||
walk(s, needle, found);
|
||||
|
||||
/// Hovering a setting's *name* explains it, not just its control: the
|
||||
/// label is the larger target and the one a reader's eye is already on.
|
||||
/// Checks the wiring, so the tooltip timing is turned off.
|
||||
#[test]
|
||||
fn hovering_a_setting_label_explains_it() {
|
||||
let ctx = egui::Context::default();
|
||||
ctx.style_mut(|s| {
|
||||
s.interaction.tooltip_delay = 0.0;
|
||||
s.interaction.show_tooltips_only_when_still = false;
|
||||
});
|
||||
let cfg = Config::default();
|
||||
let mut w = OptionsWindow::new();
|
||||
w.open_with(&cfg);
|
||||
|
||||
let run = |w: &mut OptionsWindow, events: Vec<egui::Event>| {
|
||||
let input = crate::test_ui::raw_input(egui::vec2(1000.0, 900.0), events);
|
||||
ctx.run(input, |ctx| {
|
||||
w.ui(ctx, &cfg);
|
||||
})
|
||||
};
|
||||
|
||||
// The window spends its first frames sizing itself and painting
|
||||
// nothing; run until the label is on screen.
|
||||
let mut target = None;
|
||||
for _ in 0..5 {
|
||||
let full = run(&mut w, vec![]);
|
||||
target = painted_text_center(&full, "Tokenizer");
|
||||
if target.is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
let target = target.expect("the Tokenizer label was not painted");
|
||||
|
||||
// The tooltip is an area of its own, so it can land a frame late.
|
||||
let mut out = run(&mut w, vec![egui::Event::PointerMoved(target)]);
|
||||
for _ in 0..3 {
|
||||
let painted = painted_text(&out).join("\n");
|
||||
if painted.contains(crate::tips::TOKENIZER.title)
|
||||
&& painted.contains("cut up so that it can be")
|
||||
{
|
||||
return;
|
||||
}
|
||||
out = run(&mut w, vec![]);
|
||||
}
|
||||
let mut found = None;
|
||||
for clipped in &out.shapes {
|
||||
walk(&clipped.shape, needle, &mut found);
|
||||
}
|
||||
found
|
||||
panic!("no tooltip appeared over the Tokenizer label");
|
||||
}
|
||||
|
||||
/// One real frame of the window in a headless context: it renders, and
|
||||
|
|
@ -561,14 +679,7 @@ mod tests {
|
|||
w.draft.as_mut().unwrap().search.debounce_ms += 100;
|
||||
|
||||
let run = |w: &mut OptionsWindow, events: Vec<egui::Event>| {
|
||||
let input = egui::RawInput {
|
||||
screen_rect: Some(egui::Rect::from_min_size(
|
||||
egui::Pos2::ZERO,
|
||||
egui::vec2(1000.0, 900.0),
|
||||
)),
|
||||
events,
|
||||
..Default::default()
|
||||
};
|
||||
let input = crate::test_ui::raw_input(egui::vec2(1000.0, 900.0), events);
|
||||
let mut out = OptionsOutput::default();
|
||||
let full = ctx.run(input, |ctx| out = w.ui(ctx, &cfg));
|
||||
(out, full)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
152
crates/quicksearch-gui/src/test_ui.rs
Normal file
152
crates/quicksearch-gui/src/test_ui.rs
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
//! Driving egui headlessly from tests: build an input frame, synthesize a
|
||||
//! click, read back what was painted.
|
||||
//!
|
||||
//! Every tab's test module wants the same three things, and each had grown its
|
||||
//! own copy. The per-tab `frame(…)` wrappers stay where they are — each drives
|
||||
//! a different widget with a different return type — but they are built on
|
||||
//! these.
|
||||
|
||||
/// A frame of input at `size`, carrying `events`.
|
||||
///
|
||||
/// The viewport size is explicit rather than defaulted because it is load
|
||||
/// bearing: a modal is centred in it, so tests that locate a button by where
|
||||
/// it was painted get different coordinates from a different size, and a panel
|
||||
/// that does not fit is simply not painted at all.
|
||||
pub fn raw_input(size: egui::Vec2, events: Vec<egui::Event>) -> egui::RawInput {
|
||||
egui::RawInput {
|
||||
screen_rect: Some(egui::Rect::from_min_size(egui::Pos2::ZERO, size)),
|
||||
events,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// A primary-button press and release at `pos`, preceded by the pointer moving
|
||||
/// there.
|
||||
///
|
||||
/// The move is not decoration: egui hit-tests against the pointer's *current*
|
||||
/// position, so a press delivered without one lands wherever the pointer was
|
||||
/// last frame — which for the first frame of a fresh context is nowhere.
|
||||
pub fn click_at(pos: egui::Pos2) -> Vec<egui::Event> {
|
||||
let button = |pressed| egui::Event::PointerButton {
|
||||
pos,
|
||||
button: egui::PointerButton::Primary,
|
||||
pressed,
|
||||
modifiers: egui::Modifiers::NONE,
|
||||
};
|
||||
vec![
|
||||
egui::Event::PointerMoved(pos),
|
||||
button(true),
|
||||
button(false),
|
||||
]
|
||||
}
|
||||
|
||||
/// A `Ui` from a real (headless) egui pass, so measuring helpers see the same
|
||||
/// fonts the app paints with — the whole point of `middle_elide` and of the
|
||||
/// snippet row arithmetic is that they agree with egui's own layout.
|
||||
pub fn with_ui<R>(f: impl FnOnce(&mut egui::Ui) -> R) -> R {
|
||||
let ctx = egui::Context::default();
|
||||
let mut f = Some(f);
|
||||
let mut out = None;
|
||||
let _ = ctx.run(egui::RawInput::default(), |ctx| {
|
||||
egui::CentralPanel::default().show(ctx, |ui| {
|
||||
if let Some(f) = f.take() {
|
||||
out = Some(f(ui));
|
||||
}
|
||||
});
|
||||
});
|
||||
out.expect("the central panel ran")
|
||||
}
|
||||
|
||||
/// Every text galley painted this frame, in paint order, each with the
|
||||
/// rectangle it occupies.
|
||||
fn painted_galleys(out: &egui::FullOutput) -> Vec<(&std::sync::Arc<egui::Galley>, egui::Rect)> {
|
||||
fn walk<'a>(
|
||||
shape: &'a egui::epaint::Shape,
|
||||
into: &mut Vec<(&'a std::sync::Arc<egui::Galley>, egui::Rect)>,
|
||||
) {
|
||||
match shape {
|
||||
egui::epaint::Shape::Text(t) => {
|
||||
into.push((&t.galley, egui::Rect::from_min_size(t.pos, t.galley.size())))
|
||||
}
|
||||
egui::epaint::Shape::Vec(shapes) => {
|
||||
for s in shapes {
|
||||
walk(s, into);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let mut galleys = Vec::new();
|
||||
for clipped in &out.shapes {
|
||||
walk(&clipped.shape, &mut galleys);
|
||||
}
|
||||
galleys
|
||||
}
|
||||
|
||||
/// Every text galley painted this frame, each with the rectangle it occupies.
|
||||
///
|
||||
/// Labels carry no widget id worth recording, so reading the shapes back is
|
||||
/// the only way to check the text a user actually sees — and the only way to
|
||||
/// find a click target that follows the layout instead of pinning it.
|
||||
pub fn painted(out: &egui::FullOutput) -> Vec<(String, egui::Rect)> {
|
||||
painted_galleys(out)
|
||||
.into_iter()
|
||||
.map(|(g, rect)| (g.text().to_string(), rect))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Every string painted this frame, in paint order.
|
||||
pub fn painted_text(out: &egui::FullOutput) -> Vec<String> {
|
||||
painted(out).into_iter().map(|(text, _)| text).collect()
|
||||
}
|
||||
|
||||
/// Every *visible* row of every galley painted this frame, in paint order.
|
||||
///
|
||||
/// Not the same thing as [`painted_text`]: a galley's `text()` is the job it
|
||||
/// was laid out from, including the rows epaint dropped at `wrap.max_rows`. A
|
||||
/// label that silently truncated away the very thing it was meant to show
|
||||
/// still reads as complete there; the laid-out rows are the only place the
|
||||
/// loss is visible.
|
||||
pub fn painted_rows(out: &egui::FullOutput) -> Vec<String> {
|
||||
painted_galleys(out)
|
||||
.into_iter()
|
||||
.flat_map(|(g, _)| g.rows.iter().map(|r| r.text()).collect::<Vec<_>>())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Every mesh painted this frame, in paint order.
|
||||
///
|
||||
/// The app paints text and rectangles; a mesh means a shape assembled
|
||||
/// vertex by vertex, which is the only way to get a gradient out of egui.
|
||||
/// Reading the vertices back is the only way to check one, since the colour
|
||||
/// that matters varies across the shape rather than being a property of it.
|
||||
pub fn painted_meshes(out: &egui::FullOutput) -> Vec<&egui::Mesh> {
|
||||
fn walk<'a>(shape: &'a egui::epaint::Shape, into: &mut Vec<&'a egui::Mesh>) {
|
||||
match shape {
|
||||
egui::epaint::Shape::Mesh(mesh) => into.push(mesh),
|
||||
egui::epaint::Shape::Vec(shapes) => {
|
||||
for s in shapes {
|
||||
walk(s, into);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let mut meshes = Vec::new();
|
||||
for clipped in &out.shapes {
|
||||
walk(&clipped.shape, &mut meshes);
|
||||
}
|
||||
meshes
|
||||
}
|
||||
|
||||
/// The centre of `needle`'s galley, as a click target.
|
||||
///
|
||||
/// The *last* match wins, so a string painted both behind a modal and on it
|
||||
/// resolves to the one on top — which is the one a click would reach.
|
||||
pub fn painted_text_center(out: &egui::FullOutput, needle: &str) -> Option<egui::Pos2> {
|
||||
painted(out)
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|(text, _)| text == needle)
|
||||
.map(|(_, rect)| rect.center())
|
||||
}
|
||||
758
crates/quicksearch-gui/src/tips.rs
Normal file
758
crates/quicksearch-gui/src/tips.rs
Normal file
|
|
@ -0,0 +1,758 @@
|
|||
//! Plain-language tooltips for the configuration controls.
|
||||
//!
|
||||
//! Every setting in the Options window, and every configuration control on
|
||||
//! the Manage Index tab, explains itself on hover: what it does in ordinary
|
||||
//! words first, then what changing it costs, then concrete values and when
|
||||
//! someone would pick them. The text lives here rather than inline so the
|
||||
//! layout code stays readable and the wording can be checked on its own.
|
||||
//!
|
||||
//! Written for someone who does not know what a tokenizer or a write-ahead
|
||||
//! log is, and should not have to.
|
||||
|
||||
use crate::ui_util::ORANGE;
|
||||
|
||||
/// How wide a tooltip may get. Matches `manage_tab::db_size_tooltip`: wide
|
||||
/// enough that a sentence is not shredded into three lines, narrow enough
|
||||
/// that the eye finds the next line.
|
||||
const TIP_WIDTH: f32 = 420.0;
|
||||
|
||||
/// One control's tooltip.
|
||||
pub struct Tip {
|
||||
/// The setting's name, in bold at the top.
|
||||
pub title: &'static str,
|
||||
/// The explanation. One or more paragraphs separated by `"\n\n"`.
|
||||
pub body: &'static str,
|
||||
/// Concrete values and when to choose them, rendered small underneath.
|
||||
/// Empty where a setting has nothing to weigh up.
|
||||
pub examples: &'static [&'static str],
|
||||
/// A consequence worth seeing before the click, rendered small and
|
||||
/// orange. Reserved for rebuilds and deletions, so that it keeps
|
||||
/// meaning something.
|
||||
pub caution: Option<&'static str>,
|
||||
}
|
||||
|
||||
impl Tip {
|
||||
/// Render into a hover popup.
|
||||
pub fn show(&self, ui: &mut egui::Ui) {
|
||||
ui.set_max_width(TIP_WIDTH);
|
||||
ui.strong(self.title);
|
||||
ui.label(self.body);
|
||||
if !self.examples.is_empty() {
|
||||
ui.add_space(4.0);
|
||||
// One example reads as a sentence; several read as a list.
|
||||
let single = self.examples.len() == 1;
|
||||
for example in self.examples {
|
||||
let line = if single {
|
||||
format!("Example: {}", example)
|
||||
} else {
|
||||
format!("• {}", example)
|
||||
};
|
||||
ui.label(egui::RichText::new(line).small());
|
||||
}
|
||||
}
|
||||
if let Some(caution) = self.caution {
|
||||
ui.add_space(4.0);
|
||||
ui.label(egui::RichText::new(caution).small().color(ORANGE));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Attach a [`Tip`] to any widget.
|
||||
pub trait Tipped {
|
||||
/// Show `tip` on hover, whether or not the widget is enabled: a greyed
|
||||
/// out button is exactly when someone wants to know what it would do.
|
||||
fn tip(self, tip: &'static Tip) -> Self;
|
||||
}
|
||||
|
||||
impl Tipped for egui::Response {
|
||||
fn tip(self, tip: &'static Tip) -> egui::Response {
|
||||
self.on_hover_ui(|ui| tip.show(ui))
|
||||
.on_disabled_hover_ui(|ui| tip.show(ui))
|
||||
}
|
||||
}
|
||||
|
||||
/// One row of a two-column settings grid: the label and the control share a
|
||||
/// tooltip, so hovering the name works as well as hovering the widget.
|
||||
///
|
||||
/// Taking the tip by value rather than as an option is deliberate: a row
|
||||
/// added later cannot forget to explain itself.
|
||||
pub fn tip_row(
|
||||
ui: &mut egui::Ui,
|
||||
label: &str,
|
||||
tip: &'static Tip,
|
||||
widget: impl FnOnce(&mut egui::Ui) -> egui::Response,
|
||||
) {
|
||||
ui.label(label).tip(tip);
|
||||
widget(ui).tip(tip);
|
||||
ui.end_row();
|
||||
}
|
||||
|
||||
// --- Options: Paths ------------------------------------------------------
|
||||
|
||||
pub static DATABASE_PATH: Tip = Tip {
|
||||
title: "Database file",
|
||||
body: "Where QuickSearch keeps its index: a single file holding the \
|
||||
names, locations and text of everything it has indexed. It grows \
|
||||
with the number of files indexed, so it wants a drive with room \
|
||||
to spare.\n\n\
|
||||
Pointing this somewhere else switches to the index at the new \
|
||||
location, building a fresh one there if nothing exists yet. The \
|
||||
old file stays on disk until you delete it. Keep it off a folder \
|
||||
that syncs to the cloud, such as OneDrive or Dropbox: it is far \
|
||||
too large and too busy to synchronise.",
|
||||
examples: &["a path on a second, larger drive when the system drive is tight on space."],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
// --- Options: Indexing ---------------------------------------------------
|
||||
|
||||
pub static REINDEX_INTERVAL: Tip = Tip {
|
||||
title: "Full reindex every",
|
||||
body: "Indexed folders are watched, so ordinary changes appear within \
|
||||
seconds. This is the safety net behind that: a full sweep that \
|
||||
picks up anything the watching missed, such as changes made while \
|
||||
QuickSearch was closed, or on a drive that does not report them.\n\n\
|
||||
A sweep costs some disk activity for a few minutes, and searching \
|
||||
keeps working throughout.",
|
||||
examples: &[
|
||||
"1440, once a day, suits an ordinary laptop.",
|
||||
"60 when you work on a network share, or a drive other machines write to.",
|
||||
"10080, once a week, for an archive that barely changes.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static FOLLOW_SYMLINKS: Tip = Tip {
|
||||
title: "Follow symlinks",
|
||||
body: "Symlinks are entries that stand in for a file or folder living \
|
||||
somewhere else. Off, QuickSearch steps over them, so nothing is \
|
||||
indexed twice under two names. On, it looks through them and \
|
||||
indexes what they point at, stored under the real location rather \
|
||||
than the link's.\n\n\
|
||||
Turning this off removes the entries that are no longer in scope; \
|
||||
turning it on indexes whatever it now reaches. Neither rebuilds \
|
||||
the index.",
|
||||
examples: &[
|
||||
"on when a folder you search lives outside your indexed folders and is reached \
|
||||
through a link.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static INCLUDE_HIDDEN: Tip = Tip {
|
||||
title: "Include hidden files",
|
||||
body: "Files and folders the system keeps out of sight: names beginning \
|
||||
with a dot on Linux and macOS, plus anything carrying the Hidden \
|
||||
attribute on Windows, such as AppData and $RECYCLE.BIN. They are \
|
||||
mostly program settings and caches, so leaving them out keeps the \
|
||||
index smaller and the results cleaner.\n\n\
|
||||
Folders marked System but not Hidden are indexed either way. Cloud \
|
||||
sync folders and folders given a custom icon carry that mark only \
|
||||
to get the icon, and are ordinary folders otherwise.",
|
||||
examples: &["on when you want to find configuration files such as .bashrc or .gitconfig."],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
// --- Options: Processing -------------------------------------------------
|
||||
|
||||
pub static TOKENIZER: Tip = Tip {
|
||||
title: "Tokenizer",
|
||||
body: "How the text inside your files is cut up so that it can be \
|
||||
searched.\n\n\
|
||||
trigram indexes every run of three characters, so a search for \
|
||||
\"port\" also finds \"airport\", and it works for languages that \
|
||||
do not put spaces between words. The other two index whole words \
|
||||
instead: the index is smaller and faster, but a search matches \
|
||||
only from the start of a word.",
|
||||
examples: &[
|
||||
"trigram, the default, for finding a fragment anywhere inside a word.",
|
||||
"unicode61 when you only ever search whole words and want the smallest, \
|
||||
fastest index.",
|
||||
"porter to also match English word endings, so \"running\" finds \"run\".",
|
||||
],
|
||||
caution: Some("Changing this builds the whole index again from scratch."),
|
||||
};
|
||||
|
||||
pub static HASH_LENGTH: Tip = Tip {
|
||||
title: "Hash sample size",
|
||||
body: "How much of the start of each file QuickSearch reads in order to \
|
||||
recognise it. Those first bytes give the file its fingerprint, \
|
||||
which is how the Duplicates tab knows two files are identical; \
|
||||
they also say what kind of file it is, and for a small text file \
|
||||
they are the whole text.\n\n\
|
||||
Reading more is more reliable and slower, particularly over a \
|
||||
network.",
|
||||
examples: &[
|
||||
"8192, the default, is right for almost everyone.",
|
||||
"higher when the Duplicates tab groups files that are not really identical, \
|
||||
which happens with disk images and other formats that begin with a lot of \
|
||||
empty space.",
|
||||
],
|
||||
caution: Some("Changing this builds the whole index again from scratch."),
|
||||
};
|
||||
|
||||
pub static MAX_STORED_TEXT: Tip = Tip {
|
||||
title: "Max stored text",
|
||||
body: "How much text QuickSearch keeps out of any one file. Text past \
|
||||
this point is not stored, so a search will not find a word that \
|
||||
appears only deep inside a very long document. File names, sizes \
|
||||
and dates are unaffected.\n\n\
|
||||
Along with the two settings below it, this is one of the largest \
|
||||
influences on how big the index becomes.",
|
||||
examples: &[
|
||||
"262144, 256 KB, covers the whole of most documents.",
|
||||
"65536, 64 KB, to shrink the index when you mostly search the opening pages.",
|
||||
"higher when you search long books, transcripts or logs and expect to find \
|
||||
words near the end.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static MAX_TEXT_FILE_SIZE: Tip = Tip {
|
||||
title: "Max text file size",
|
||||
body: "Files larger than this are indexed by name only: QuickSearch does \
|
||||
not open them to read the text inside. It keeps one stray huge \
|
||||
file from holding up an indexing run.\n\n\
|
||||
Those files still appear in results, found by their name, size or \
|
||||
date.",
|
||||
examples: &[
|
||||
"2097152, 2 MB, skips very few ordinary documents.",
|
||||
"52428800, 50 MB, when you search inside large log files or scanned PDFs.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static BATCH_SIZE: Tip = Tip {
|
||||
title: "Batch size",
|
||||
body: "How many files QuickSearch handles per write to the index while \
|
||||
indexing. Larger batches mean fewer, bigger writes, which is a \
|
||||
little faster and uses more memory.\n\n\
|
||||
A speed setting only: it changes nothing about what you can find, \
|
||||
and most people never need to touch it.",
|
||||
examples: &[
|
||||
"500, the default, balances speed against memory.",
|
||||
"lower, around 50, on a machine with very little memory to spare.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static MAX_WAL_SIZE: Tip = Tip {
|
||||
title: "Max WAL size",
|
||||
body: "While indexing, changes are written to a companion file beside \
|
||||
the index and folded in afterwards. That normally happens by \
|
||||
itself, but during a long run with searches going on at the same \
|
||||
time the companion file keeps growing, sometimes past the size of \
|
||||
the index. This is the point at which QuickSearch pauses and folds \
|
||||
it in regardless.\n\n\
|
||||
Another speed setting; the default suits most machines.",
|
||||
examples: &[
|
||||
"536870912, 512 MB, is the default.",
|
||||
"67108864, 64 MB, when disk space is tight.",
|
||||
"0 to never force it and let the database decide. Any other value below 16 MB \
|
||||
is treated as 16 MB.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static STORE_TEXT: Tip = Tip {
|
||||
title: "Store text for snippets",
|
||||
body: "Keeps the text QuickSearch reads out of your files, rather than \
|
||||
only the index needed to search it. It is what makes the preview \
|
||||
line underneath a result possible.\n\n\
|
||||
Off, searching inside files still works, but there are no \
|
||||
previews, no ranking by how often a word appears, no telling \
|
||||
Report from report, and no allowance for typos inside file \
|
||||
contents. The index shrinks considerably.\n\n\
|
||||
Turning it off discards the stored text at once; turning it back \
|
||||
on reads your files again.",
|
||||
examples: &[
|
||||
"off when the index has grown larger than you want and you can do without previews.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
// --- Options: Search -----------------------------------------------------
|
||||
|
||||
pub static FUZZY_DEFAULT: Tip = Tip {
|
||||
title: "Fuzzy search ON by default",
|
||||
body: "Whether the Fuzzy box on the Search tab starts ticked each time \
|
||||
QuickSearch opens. Fuzzy search also finds matches with typos in \
|
||||
them, at some cost in speed. Either way you can tick and untick \
|
||||
it whenever you like.",
|
||||
examples: &["on when you often look for names you are not sure how to spell."],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static FUZZY_EDITS: Tip = Tip {
|
||||
title: "Fuzzy edit distance",
|
||||
body: "How far a word may sit from what you typed and still count as a \
|
||||
match while Fuzzy is on. One edit is one letter added, removed or \
|
||||
changed, so \"reciept\" is one edit away from \"receipt\".\n\n\
|
||||
The allowance grows with the length of what you type, one edit per \
|
||||
three characters, up to the value set here. Short searches stay \
|
||||
strict, so that three letters do not match half the index.",
|
||||
examples: &[
|
||||
"2, the default, allows one edit for short words and two for longer ones.",
|
||||
"0 turns typo matching off altogether, even with the Fuzzy box ticked.",
|
||||
"3 or more is allowed, but searches get slower and pull in a lot of \
|
||||
unrelated files.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static DISPLAY_LIMIT: Tip = Tip {
|
||||
title: "Display limit",
|
||||
body: "The most results one search will gather and show. A search \
|
||||
matching thousands of files stops here, which keeps the list quick \
|
||||
to scroll and cheap to hold in memory.\n\n\
|
||||
The best matches come first, so a lower limit rarely hides what \
|
||||
you were looking for.",
|
||||
examples: &[
|
||||
"1000, the default, is more than anyone scrolls through.",
|
||||
"higher when you use QuickSearch to list every file of a kind, such as \
|
||||
type:Image, and want them all at once.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static RESULTS_PER_PAGE: Tip = Tip {
|
||||
title: "Stream batch size",
|
||||
body: "Results arrive in batches while a search runs, and this is how \
|
||||
many are in each one. Smaller batches put the first results on \
|
||||
screen sooner and update the list more often; larger ones do less \
|
||||
work in total.\n\n\
|
||||
This is not a page size: scrolling the results does not go back \
|
||||
for more.",
|
||||
examples: &[
|
||||
"100, the default, feels immediate on most machines.",
|
||||
"25 when the first results are slow to appear on a large index.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static DEBOUNCE: Tip = Tip {
|
||||
title: "Debounce",
|
||||
body: "How long QuickSearch waits after your last keystroke before it \
|
||||
searches, so that typing a word does not fire off a search for \
|
||||
every letter in it. 1000 milliseconds is one second.",
|
||||
examples: &[
|
||||
"150, the default, keeps up with ordinary typing.",
|
||||
"0 to chase every keystroke on a fast machine with a modest index.",
|
||||
"300 or more when the results flicker or stutter as you type.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
// --- Options: Interface --------------------------------------------------
|
||||
|
||||
pub static UI_SCALE: Tip = Tip {
|
||||
title: "UI scale",
|
||||
body: "Zooms the whole window: text, spacing and controls together. 1.00 \
|
||||
is the ordinary size for your screen.\n\n\
|
||||
Ctrl with + or - changes it for the moment without saving, and \
|
||||
Ctrl 0 puts it back. This slider is the size QuickSearch starts \
|
||||
at.",
|
||||
examples: &[
|
||||
"1.40 or more on a high resolution screen where the text looks small.",
|
||||
"0.80 to fit more results on screen at once.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
// --- Options: Security ---------------------------------------------------
|
||||
|
||||
pub static ENABLE_PASSWORD: Tip = Tip {
|
||||
title: "Enable password protection",
|
||||
body: "Encrypts the index with a password of your choosing. The index \
|
||||
holds the names and the text of your files, so anyone who can \
|
||||
read that file can read those; encrypting it means they cannot.\n\n\
|
||||
QuickSearch then asks for the password each time it starts, unless \
|
||||
you let it remember.",
|
||||
examples: &[],
|
||||
caution: Some(
|
||||
"Turning protection on deletes the index and builds it again. Your files are \
|
||||
not touched.",
|
||||
),
|
||||
};
|
||||
|
||||
pub static CHANGE_PASSWORD: Tip = Tip {
|
||||
title: "Change password",
|
||||
body: "Replaces the password the index is encrypted with. You are asked \
|
||||
for a new one, and the index is encrypted again under it.",
|
||||
examples: &[],
|
||||
caution: Some(
|
||||
"Changing the password deletes the index and builds it again. Your files are \
|
||||
not touched.",
|
||||
),
|
||||
};
|
||||
|
||||
pub static DISABLE_PASSWORD: Tip = Tip {
|
||||
title: "Disable protection",
|
||||
body: "Removes the password and leaves the index unencrypted on disk. \
|
||||
Anyone able to read that file can then see the names of your files \
|
||||
and the text inside them.",
|
||||
examples: &[],
|
||||
caution: Some(
|
||||
"Turning protection off deletes the index and builds it again. Your files are \
|
||||
not touched.",
|
||||
),
|
||||
};
|
||||
|
||||
pub static REMEMBER_KEYCHAIN: Tip = Tip {
|
||||
title: "Remember on this device",
|
||||
body: "Hands the key to the password store your system already has, such \
|
||||
as GNOME Keyring, KWallet, or Windows Credential Manager, so that \
|
||||
QuickSearch can unlock the index without asking at startup.\n\n\
|
||||
The password itself is never stored, only the key worked out from \
|
||||
it, and only on this machine. Off, you type the password each time \
|
||||
QuickSearch starts.",
|
||||
examples: &[],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
// --- Manage Index tab: indexing controls ---------------------------------
|
||||
|
||||
pub static START_NOW: Tip = Tip {
|
||||
title: "Start indexing now",
|
||||
body: "Runs a full pass over your indexed folders straight away instead \
|
||||
of waiting for the next scheduled one. Worth doing after adding a \
|
||||
folder, after changing a filter, or when the computer has been off \
|
||||
for a while.\n\n\
|
||||
Searching carries on working while it runs. Unavailable while a \
|
||||
run is already under way.",
|
||||
examples: &[],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static STOP_INDEXING: Tip = Tip {
|
||||
title: "Stop",
|
||||
body: "Stops the run in progress and switches to manual, so QuickSearch \
|
||||
no longer watches for changes or reindexes on a schedule. The \
|
||||
index stays as it is and searching still works, but it drifts out \
|
||||
of date as your files change.\n\n\
|
||||
Saved immediately: QuickSearch is still in manual the next time it \
|
||||
starts.",
|
||||
examples: &[],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static RETURN_TO_AUTO: Tip = Tip {
|
||||
title: "Return to Automatic",
|
||||
body: "Goes back to watching your folders and reindexing on a schedule, \
|
||||
catching up on everything that changed while indexing was manual.\n\n\
|
||||
Also saved, so this is how QuickSearch starts from now on.",
|
||||
examples: &[],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static CLEAR_INDEX: Tip = Tip {
|
||||
title: "Clear index",
|
||||
body: "Deletes the index database. Searching finds nothing until it is \
|
||||
built again, which for a large folder takes a while. Your own \
|
||||
files are never touched.\n\n\
|
||||
QuickSearch asks for confirmation first, then drops to manual so \
|
||||
that it does not immediately rebuild what you just deleted.",
|
||||
examples: &[],
|
||||
caution: Some("This cannot be undone: the index has to be built from scratch again."),
|
||||
};
|
||||
|
||||
// --- Manage Index tab: indexed folders -----------------------------------
|
||||
|
||||
pub static ADD_ROOT: Tip = Tip {
|
||||
title: "Add an indexed folder",
|
||||
body: "Adds a folder for QuickSearch to index, along with everything \
|
||||
inside it. Choose it with the browser, or type the path and press \
|
||||
Add.\n\n\
|
||||
Indexed folders may not overlap, so a folder already inside \
|
||||
another one is refused. Adding a folder starts an indexing pass to \
|
||||
pick it up and leaves the rest of the index alone.",
|
||||
examples: &["a second drive, or a network share you search often."],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static REMOVE_ROOT: Tip = Tip {
|
||||
title: "Remove this folder",
|
||||
body: "Stops indexing this folder and removes its entries from the \
|
||||
index. The rest of the index is left alone, and the files \
|
||||
themselves are not touched.\n\n\
|
||||
Takes effect when you click Apply & Save.",
|
||||
examples: &[],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static ROOT_WORKERS: Tip = Tip {
|
||||
title: "Workers",
|
||||
body: "How many folders QuickSearch explores at once inside this indexed \
|
||||
folder. More of them finish sooner on storage that answers many \
|
||||
requests at a time, which network drives do especially well, but \
|
||||
they compete for the same disk.\n\n\
|
||||
auto reads 4 on local storage and 16 on a network mount. Takes \
|
||||
effect on the next indexing run.",
|
||||
examples: &[
|
||||
"auto unless indexing is slower than you would expect.",
|
||||
"16 or more for a network share that is slow to answer each request.",
|
||||
"2 to keep indexing out of the way on an older machine.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
// --- Manage Index tab: content filters -----------------------------------
|
||||
|
||||
pub static EXT_WHITELIST: Tip = Tip {
|
||||
title: "Full-text extensions whitelist",
|
||||
body: "Which kinds of file QuickSearch reads the text out of, one \
|
||||
extension per line, the leading dot optional. Empty means every \
|
||||
kind it understands.\n\n\
|
||||
Every file is still indexed by name whatever you put here. A list \
|
||||
also leaves out files with no extension at all, such as Makefile \
|
||||
or README, unless you add the line (none). Anything after a # is a \
|
||||
comment, so a file type can be switched off without losing the \
|
||||
line.\n\n\
|
||||
Narrowing the list discards the text it now excludes; widening it \
|
||||
reads those files again.",
|
||||
examples: &[
|
||||
"txt, md and pdf to keep the index small and focused on documents.",
|
||||
"empty to search inside everything QuickSearch can read.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
pub static IGNORE_PATTERNS: Tip = Tip {
|
||||
title: "Ignore patterns",
|
||||
body: "Files and folders left out of the index entirely, by name and by \
|
||||
content alike. Type one pattern and click Add.\n\n\
|
||||
A pattern without a slash matches a file or folder name anywhere, \
|
||||
and must match the whole name: .jpg matches only something called \
|
||||
exactly that, while *.jpg matches every JPEG. A pattern with a \
|
||||
slash in it is matched against the whole path, and skips \
|
||||
everything underneath. * stands for any run of characters and ? \
|
||||
for a single one.\n\n\
|
||||
Adding a pattern removes the entries it matches; removing one \
|
||||
indexes them again.",
|
||||
examples: &[
|
||||
"node_modules to skip that folder wherever it turns up.",
|
||||
"*.tmp to skip temporary files by extension.",
|
||||
"a full path such as the Videos folder to skip it and everything inside it.",
|
||||
],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
// --- Shared --------------------------------------------------------------
|
||||
|
||||
pub static APPLY_SAVE: Tip = Tip {
|
||||
title: "Apply & Save",
|
||||
body: "Writes these settings to the configuration file and puts them to \
|
||||
work straight away. Until you click here, your edits are only \
|
||||
staged.\n\n\
|
||||
Narrowing a setting removes the entries it now excludes; widening \
|
||||
one indexes whatever it now allows. Only the tokenizer, the hash \
|
||||
sample size and password protection need the index built again \
|
||||
from scratch, and those ask first.",
|
||||
examples: &[],
|
||||
caution: None,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Every tip in the file. A tip missing from here is only missing from
|
||||
/// the checks below, so keep it in step when adding one.
|
||||
const ALL: &[&Tip] = &[
|
||||
&DATABASE_PATH,
|
||||
&REINDEX_INTERVAL,
|
||||
&FOLLOW_SYMLINKS,
|
||||
&INCLUDE_HIDDEN,
|
||||
&TOKENIZER,
|
||||
&HASH_LENGTH,
|
||||
&MAX_STORED_TEXT,
|
||||
&MAX_TEXT_FILE_SIZE,
|
||||
&BATCH_SIZE,
|
||||
&MAX_WAL_SIZE,
|
||||
&STORE_TEXT,
|
||||
&FUZZY_DEFAULT,
|
||||
&FUZZY_EDITS,
|
||||
&DISPLAY_LIMIT,
|
||||
&RESULTS_PER_PAGE,
|
||||
&DEBOUNCE,
|
||||
&UI_SCALE,
|
||||
&ENABLE_PASSWORD,
|
||||
&CHANGE_PASSWORD,
|
||||
&DISABLE_PASSWORD,
|
||||
&REMEMBER_KEYCHAIN,
|
||||
&START_NOW,
|
||||
&STOP_INDEXING,
|
||||
&RETURN_TO_AUTO,
|
||||
&CLEAR_INDEX,
|
||||
&ADD_ROOT,
|
||||
&REMOVE_ROOT,
|
||||
&ROOT_WORKERS,
|
||||
&EXT_WHITELIST,
|
||||
&IGNORE_PATTERNS,
|
||||
&APPLY_SAVE,
|
||||
];
|
||||
|
||||
/// Everything a tip can put on screen, as one string.
|
||||
fn all_text(tip: &Tip) -> String {
|
||||
let mut text = format!("{}\n{}", tip.title, tip.body);
|
||||
for example in tip.examples {
|
||||
text.push('\n');
|
||||
text.push_str(example);
|
||||
}
|
||||
if let Some(caution) = tip.caution {
|
||||
text.push('\n');
|
||||
text.push_str(caution);
|
||||
}
|
||||
text
|
||||
}
|
||||
|
||||
/// House style, and the one rule that is easy to break by pasting from
|
||||
/// the source comments: these tooltips use no em-dashes.
|
||||
#[test]
|
||||
fn no_tip_uses_an_em_dash() {
|
||||
for tip in ALL {
|
||||
assert!(
|
||||
!all_text(tip).contains('—'),
|
||||
"{} uses an em-dash",
|
||||
tip.title
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_tip_is_filled_in() {
|
||||
for tip in ALL {
|
||||
assert!(!tip.title.trim().is_empty(), "a tip has no title");
|
||||
assert!(
|
||||
!tip.title.ends_with('.'),
|
||||
"{}: title is not a sentence",
|
||||
tip.title
|
||||
);
|
||||
assert!(
|
||||
tip.body.trim().len() > 40,
|
||||
"{}: body says too little",
|
||||
tip.title
|
||||
);
|
||||
assert!(
|
||||
tip.body.trim_end().ends_with('.'),
|
||||
"{}: body is not a finished sentence",
|
||||
tip.title
|
||||
);
|
||||
// "Stops the run in progress" under the title "Stop" is fine;
|
||||
// "Stop. Stops the run" is the restatement worth catching, so
|
||||
// the title only counts as repeated when a word ends there.
|
||||
let restates = tip
|
||||
.body
|
||||
.strip_prefix(tip.title)
|
||||
.is_some_and(|rest| !rest.starts_with(|c: char| c.is_alphanumeric()));
|
||||
assert!(!restates, "{}: body repeats the title", tip.title);
|
||||
for example in tip.examples {
|
||||
assert!(
|
||||
!example.trim().is_empty() && example.trim_end().ends_with('.'),
|
||||
"{}: bad example {:?}",
|
||||
tip.title,
|
||||
example
|
||||
);
|
||||
}
|
||||
if let Some(caution) = tip.caution {
|
||||
assert!(
|
||||
caution.trim_end().ends_with('.'),
|
||||
"{}: caution is not a finished sentence",
|
||||
tip.title
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Descriptive, but a tooltip nobody reads to the end helps nobody.
|
||||
#[test]
|
||||
fn no_tip_is_a_wall_of_text() {
|
||||
for tip in ALL {
|
||||
let len = all_text(tip).chars().count();
|
||||
assert!(len <= 900, "{} is {} characters long", tip.title, len);
|
||||
}
|
||||
}
|
||||
|
||||
/// Two controls sharing a title means one of them was pasted from the
|
||||
/// other and never renamed.
|
||||
#[test]
|
||||
fn titles_are_distinct() {
|
||||
let mut seen: Vec<&str> = ALL.iter().map(|t| t.title).collect();
|
||||
seen.sort_unstable();
|
||||
let count = seen.len();
|
||||
seen.dedup();
|
||||
assert_eq!(count, seen.len(), "two tips share a title: {:?}", seen);
|
||||
}
|
||||
|
||||
/// The renderer puts every part on screen: title, body, examples and
|
||||
/// caution. Written against the tip with all four.
|
||||
#[test]
|
||||
fn show_paints_every_part() {
|
||||
let ctx = egui::Context::default();
|
||||
let input = crate::test_ui::raw_input(egui::vec2(800.0, 600.0), vec![]);
|
||||
let full = ctx.run(input, |ctx| {
|
||||
egui::CentralPanel::default().show(ctx, |ui| TOKENIZER.show(ui));
|
||||
});
|
||||
let painted = crate::test_ui::painted_text(&full).join("\n");
|
||||
assert!(painted.contains(TOKENIZER.title), "no title: {painted}");
|
||||
assert!(painted.contains("trigram indexes every run"), "no body");
|
||||
assert!(painted.contains("• trigram, the default"), "no examples");
|
||||
assert!(
|
||||
painted.contains(TOKENIZER.caution.unwrap()),
|
||||
"no caution line"
|
||||
);
|
||||
}
|
||||
|
||||
/// A lone example reads as a sentence rather than a one-item list.
|
||||
#[test]
|
||||
fn a_single_example_is_prefixed_with_example() {
|
||||
let ctx = egui::Context::default();
|
||||
let input = crate::test_ui::raw_input(egui::vec2(800.0, 600.0), vec![]);
|
||||
let full = ctx.run(input, |ctx| {
|
||||
egui::CentralPanel::default().show(ctx, |ui| INCLUDE_HIDDEN.show(ui));
|
||||
});
|
||||
let painted = crate::test_ui::painted_text(&full).join("\n");
|
||||
assert!(
|
||||
painted.contains(&format!("Example: {}", INCLUDE_HIDDEN.examples[0])),
|
||||
"{painted}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A greyed-out control still explains itself. egui shows nothing on a
|
||||
/// disabled widget unless the *disabled* tooltip is set too, and "why
|
||||
/// can I not click this" is exactly when the answer is wanted: Stop,
|
||||
/// Start indexing now and Return to Automatic are greyed out by turns.
|
||||
#[test]
|
||||
fn a_disabled_control_still_explains_itself() {
|
||||
let ctx = egui::Context::default();
|
||||
ctx.style_mut(|s| {
|
||||
s.interaction.tooltip_delay = 0.0;
|
||||
s.interaction.show_tooltips_only_when_still = false;
|
||||
});
|
||||
let run = |events: Vec<egui::Event>| {
|
||||
let input = crate::test_ui::raw_input(egui::vec2(600.0, 400.0), events);
|
||||
ctx.run(input, |ctx| {
|
||||
egui::CentralPanel::default().show(ctx, |ui| {
|
||||
ui.add_enabled(false, egui::Button::new("Stop"))
|
||||
.tip(&STOP_INDEXING);
|
||||
});
|
||||
})
|
||||
};
|
||||
|
||||
run(vec![]);
|
||||
let settled = run(vec![]);
|
||||
let pos = crate::test_ui::painted_text_center(&settled, "Stop").expect("button painted");
|
||||
let opening: String = STOP_INDEXING.body.chars().take(40).collect();
|
||||
let mut out = run(vec![egui::Event::PointerMoved(pos)]);
|
||||
for _ in 0..3 {
|
||||
if crate::test_ui::painted_text(&out)
|
||||
.join("\n")
|
||||
.contains(&opening)
|
||||
{
|
||||
return;
|
||||
}
|
||||
out = run(vec![]);
|
||||
}
|
||||
panic!("a disabled control said nothing on hover");
|
||||
}
|
||||
}
|
||||
|
|
@ -1,17 +1,22 @@
|
|||
//! Indexing-rate estimation for the status displays.
|
||||
//!
|
||||
//! The old tracker sampled the counter every poll tick but pruned to a
|
||||
//! 1-second window, so anything slower than ~1 file/sec measured a
|
||||
//! genuine zero and displayed "0.0 files/sec" despite progress. This one
|
||||
//! records a point only when the counter *changes*, keeps up to 60 s of
|
||||
//! history but never fewer than two points (so slow rates stay
|
||||
//! computable), and measures against `now` so the estimate decays during
|
||||
//! stalls instead of freezing at the last burst.
|
||||
//! The rate shown is a rolling [`WINDOW`] average, not a run average: what
|
||||
//! matters while watching a run is what it is doing *now*, and a run that
|
||||
//! spent its first minute on a fast SSD root drags a whole-run mean far
|
||||
//! above the rate the slow root it is on is actually achieving.
|
||||
//!
|
||||
//! The tracker records a point only when the counter *changes*, prunes
|
||||
//! points older than the window but never below two (so a rate slower than
|
||||
//! one file per window stays computable rather than measuring a genuine
|
||||
//! zero), and measures against `now` so the estimate decays during stalls
|
||||
//! instead of freezing at the last burst.
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const HISTORY: Duration = Duration::from_secs(60);
|
||||
/// The averaging window. Public so the display can name it and stay in
|
||||
/// sync with it.
|
||||
pub const WINDOW: Duration = Duration::from_secs(30);
|
||||
|
||||
pub struct SpeedTracker {
|
||||
/// (when, counter value) — appended only on counter change. A deque
|
||||
|
|
@ -44,15 +49,20 @@ impl SpeedTracker {
|
|||
_ => {}
|
||||
}
|
||||
self.points.push_back((now, files_processed));
|
||||
// Prune old points, but always keep at least two so a slow but
|
||||
// steady rate never becomes unmeasurable.
|
||||
while self.points.len() > 2 && now.duration_since(self.points[0].0) > HISTORY {
|
||||
// Prune points that have fallen out of the window, but always keep
|
||||
// at least two so a slow but steady rate never becomes unmeasurable.
|
||||
while self.points.len() > 2 && now.duration_since(self.points[0].0) > WINDOW {
|
||||
self.points.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
/// Estimated files/sec, measured from the oldest retained progress
|
||||
/// point to *now*. `None` until two data points exist.
|
||||
/// Estimated files/sec over the last [`WINDOW`], measured from the
|
||||
/// oldest retained progress point to *now*. `None` until two data
|
||||
/// points exist.
|
||||
///
|
||||
/// The span is shorter than the window early in a run, and longer than
|
||||
/// it during a stall (nothing is recorded then, so nothing prunes and
|
||||
/// the growing span decays the estimate toward zero).
|
||||
pub fn files_per_sec(&self) -> Option<f64> {
|
||||
self.files_per_sec_at(Instant::now())
|
||||
}
|
||||
|
|
@ -139,6 +149,29 @@ mod tests {
|
|||
.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn window_forgets_an_older_burst() {
|
||||
// 10,000 files in the first second, then a steady 10/s. A run
|
||||
// average would still read in the hundreds; the window must report
|
||||
// what the run is doing now.
|
||||
let mut t = SpeedTracker::new();
|
||||
let base = Instant::now();
|
||||
t.record_at(base, 0);
|
||||
t.record_at(base + Duration::from_secs(1), 10_000);
|
||||
for i in 2..=90 {
|
||||
t.record_at(base + Duration::from_secs(i), 10_000 + 10 * i as usize);
|
||||
}
|
||||
let now = base + Duration::from_secs(90);
|
||||
let rate = t.files_per_sec_at(now).unwrap();
|
||||
assert!((rate - 10.0).abs() < 0.5, "expected ~10/s, got {}", rate);
|
||||
assert!(
|
||||
t.points
|
||||
.iter()
|
||||
.all(|&(at, _)| now.duration_since(at) <= WINDOW),
|
||||
"no point older than the window survives while others remain"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counter_regression_resets() {
|
||||
let mut t = SpeedTracker::new();
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
//! Shared UI helpers: emphasis colors, bordered widgets, ignore-pattern
|
||||
//! validation, and the "more content below" scroll hint.
|
||||
//! validation, text eliding, and the "more content below" scroll hint.
|
||||
|
||||
use quicksearch_core::config::IgnoreSet;
|
||||
use std::borrow::Cow;
|
||||
|
||||
/// Warning/emphasis orange, also used for the fuzzy-edit-distance warning.
|
||||
pub const ORANGE: egui::Color32 = egui::Color32::from_rgb(220, 150, 40);
|
||||
|
|
@ -136,6 +137,81 @@ pub fn pattern_edit(
|
|||
(response, valid)
|
||||
}
|
||||
|
||||
/// Middle-elide `text` so it fits `max_width` pixels when laid out in
|
||||
/// `font_id`, returning it borrowed and untouched when it already fits.
|
||||
///
|
||||
/// A path's two ends are the informative ones — the head says which volume
|
||||
/// or home it lives under, the tail names the deepest directories — so a
|
||||
/// column too narrow for the whole thing should drop out of the middle
|
||||
/// rather than tail-truncate the way egui does by default.
|
||||
///
|
||||
/// The budget is in pixels, summed from the font's own glyph advances (the
|
||||
/// same numbers egui's layout adds up), not a character count scaled by the
|
||||
/// width of one sample glyph. The proportional body font makes that estimate
|
||||
/// wrong in both directions: overshoot and egui elides the result a *second*
|
||||
/// time, painting two ellipses; undershoot and the column sits visibly short
|
||||
/// of full.
|
||||
///
|
||||
/// The borrowed/owned distinction is also the caller's signal that something
|
||||
/// was dropped, which is what a "full text on hover" tooltip keys off.
|
||||
pub fn middle_elide<'a>(
|
||||
ui: &egui::Ui,
|
||||
text: &'a str,
|
||||
max_width: f32,
|
||||
font_id: &egui::FontId,
|
||||
) -> Cow<'a, str> {
|
||||
ui.fonts(|f| {
|
||||
let width_of = |c: char| f.glyph_width(font_id, c);
|
||||
if text.chars().map(width_of).sum::<f32>() <= max_width {
|
||||
return Cow::Borrowed(text);
|
||||
}
|
||||
let budget = max_width - width_of('…');
|
||||
|
||||
// Grow a head and a tail toward each other through the middle,
|
||||
// each step feeding whichever side is currently narrower so the cut
|
||||
// lands near the middle. Indices advance by whole characters, so
|
||||
// they always land on UTF-8 boundaries.
|
||||
let (mut head, mut tail) = (0usize, text.len());
|
||||
let (mut head_w, mut tail_w) = (0.0f32, 0.0f32);
|
||||
while head < tail {
|
||||
let rest = &text[head..tail];
|
||||
let front = rest.chars().next().expect("head < tail");
|
||||
let back = rest.chars().next_back().expect("head < tail");
|
||||
let (front_w, back_w) = (width_of(front), width_of(back));
|
||||
let used = head_w + tail_w;
|
||||
let (front_fits, back_fits) = (used + front_w <= budget, used + back_w <= budget);
|
||||
if !front_fits && !back_fits {
|
||||
break;
|
||||
}
|
||||
// The preferred side wins when it fits; otherwise the other one
|
||||
// does, since at least one of them just did.
|
||||
let take_front = if head_w <= tail_w {
|
||||
front_fits
|
||||
} else {
|
||||
!back_fits
|
||||
};
|
||||
if take_front {
|
||||
head += front.len_utf8();
|
||||
head_w += front_w;
|
||||
} else {
|
||||
tail -= back.len_utf8();
|
||||
tail_w += back_w;
|
||||
}
|
||||
}
|
||||
if head >= tail {
|
||||
// The two halves met without dropping anything — splicing an
|
||||
// ellipsis in now would only lengthen a string that fits.
|
||||
return Cow::Borrowed(text);
|
||||
}
|
||||
|
||||
let mut out = String::with_capacity(head + '…'.len_utf8() + (text.len() - tail));
|
||||
out.push_str(&text[..head]);
|
||||
out.push('…');
|
||||
out.push_str(&text[tail..]);
|
||||
Cow::Owned(out)
|
||||
})
|
||||
}
|
||||
|
||||
/// Paint a semitransparent down-arrow near the bottom edge of a scroll
|
||||
/// area while more content lies below the fold. Painter-only, so it can
|
||||
/// never swallow clicks. (The bundled fonts have no ▼ glyph — this is a
|
||||
|
|
@ -173,9 +249,187 @@ pub fn more_below_hint<R>(ui: &egui::Ui, out: &egui::scroll_area::ScrollAreaOutp
|
|||
));
|
||||
}
|
||||
|
||||
/// Height of the wipe's soft edge, as a fraction of the section it travels
|
||||
/// over — a proportional band so the transition reads the same on a tall
|
||||
/// window as on a short one.
|
||||
const WIPE_BAND: f32 = 0.45;
|
||||
/// …but never thinner than this, so a two-row table still gets a gradient
|
||||
/// rather than a hard cut.
|
||||
const WIPE_BAND_MIN: f32 = 24.0;
|
||||
|
||||
/// The two y-coordinates a wipe's scrim ramps between: fully clear at and
|
||||
/// above the first, fully opaque at and below the second.
|
||||
///
|
||||
/// `wipe` is 1 when the section is entirely covered and 0 when it is
|
||||
/// entirely on screen, and the edge travels monotonically with it — so
|
||||
/// walking `wipe` down from 1 slides the covered region off the bottom of
|
||||
/// the section, uncovering the top first.
|
||||
fn wipe_edges(rect: egui::Rect, wipe: f32) -> (f32, f32) {
|
||||
let band = (rect.height() * WIPE_BAND).max(WIPE_BAND_MIN);
|
||||
let covered = rect.bottom() + band - wipe * (rect.height() + band);
|
||||
(covered - band, covered)
|
||||
}
|
||||
|
||||
/// The scrim hiding the `wipe` of `rect` not yet revealed: a vertical
|
||||
/// gradient from transparent to solid `fill`, or `None` once nothing is
|
||||
/// covered.
|
||||
///
|
||||
/// Painting the background colour over the results is equivalent to fading
|
||||
/// them into it, and unlike a per-row opacity it reaches the parts of an
|
||||
/// `egui_extras` table that the caller never gets a `Ui` for — the stripes,
|
||||
/// the selection fill, the scroll bar.
|
||||
pub fn wipe_mesh(rect: egui::Rect, wipe: f32, fill: egui::Color32) -> Option<egui::Mesh> {
|
||||
if wipe <= 0.0 || rect.height() <= 0.0 || rect.width() <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
let (clear, covered) = wipe_edges(rect, wipe);
|
||||
let alpha_at = |y: f32| ((y - clear) / (covered - clear)).clamp(0.0, 1.0);
|
||||
|
||||
// The gradient is linear between the two edges and flat outside them, so
|
||||
// the quads only have to break where an edge falls inside the rect.
|
||||
let mut stops = vec![rect.top(), rect.bottom()];
|
||||
stops.extend(
|
||||
[clear, covered]
|
||||
.into_iter()
|
||||
.filter(|&y| rect.y_range().contains(y)),
|
||||
);
|
||||
stops.sort_by(f32::total_cmp);
|
||||
|
||||
let mut mesh = egui::Mesh::default();
|
||||
for pair in stops.windows(2) {
|
||||
let (top, bottom) = (pair[0], pair[1]);
|
||||
let (a_top, a_bottom) = (alpha_at(top), alpha_at(bottom));
|
||||
// Sub-point slivers and the still-clear stretch above the edge would
|
||||
// contribute nothing but vertices.
|
||||
if bottom - top < 0.5 || (a_top <= 0.0 && a_bottom <= 0.0) {
|
||||
continue;
|
||||
}
|
||||
let base = mesh.vertices.len() as u32;
|
||||
for (y, alpha) in [(top, a_top), (bottom, a_bottom)] {
|
||||
// Mesh vertices carry premultiplied colors, which is exactly
|
||||
// what scaling an opaque one by `gamma_multiply` produces.
|
||||
let color = fill.gamma_multiply(alpha);
|
||||
for x in [rect.left(), rect.right()] {
|
||||
mesh.colored_vertex(egui::pos2(x, y), color);
|
||||
}
|
||||
}
|
||||
mesh.add_triangle(base, base + 1, base + 2);
|
||||
mesh.add_triangle(base + 1, base + 2, base + 3);
|
||||
}
|
||||
(!mesh.is_empty()).then_some(mesh)
|
||||
}
|
||||
|
||||
/// Paint [`wipe_mesh`] over `rect` in the panel's own background color.
|
||||
///
|
||||
/// Drawn through the layer painter, like [`more_below_hint`]: last in the
|
||||
/// caller's layer so it covers the content painted before it, and — the
|
||||
/// reason it cannot use `ui.painter()` — outside the section-wide opacity
|
||||
/// the caller has already set, which would otherwise scale the scrim along
|
||||
/// with what it is meant to hide.
|
||||
pub fn wipe_scrim(ui: &egui::Ui, rect: egui::Rect, wipe: f32) {
|
||||
let Some(mesh) = wipe_mesh(rect, wipe, ui.visuals().panel_fill) else {
|
||||
return;
|
||||
};
|
||||
ui.ctx()
|
||||
.layer_painter(ui.layer_id())
|
||||
.add(egui::Shape::mesh(mesh));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{ignore_pattern_valid, pattern_border, pattern_hint, INVALID_RED, VALID_GREEN};
|
||||
use super::{
|
||||
ignore_pattern_valid, middle_elide, pattern_border, pattern_hint, wipe_mesh, Cow,
|
||||
INVALID_RED, VALID_GREEN, WIPE_BAND_MIN,
|
||||
};
|
||||
use crate::test_ui::with_ui;
|
||||
|
||||
fn body_font(ui: &egui::Ui) -> egui::FontId {
|
||||
egui::TextStyle::Body.resolve(ui.style())
|
||||
}
|
||||
|
||||
const DEEP: &str = "/media/shared/QuickSearch/crates/quicksearch-gui/src/search_tab.rs";
|
||||
|
||||
#[test]
|
||||
fn text_that_fits_comes_back_untouched() {
|
||||
with_ui(|ui| {
|
||||
let font = body_font(ui);
|
||||
let out = middle_elide(ui, DEEP, 10_000.0, &font);
|
||||
assert!(matches!(out, Cow::Borrowed(_)), "borrowed when it fits");
|
||||
assert_eq!(out, DEEP);
|
||||
// Nothing to elide, however little room there is.
|
||||
assert!(matches!(middle_elide(ui, "", 0.0, &font), Cow::Borrowed(_)));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eliding_keeps_both_ends_and_cuts_once() {
|
||||
with_ui(|ui| {
|
||||
let out = middle_elide(ui, DEEP, 200.0, &body_font(ui));
|
||||
assert!(matches!(out, Cow::Owned(_)), "{out}");
|
||||
assert_eq!(out.matches('…').count(), 1, "{out}");
|
||||
let (head, tail) = out.split_once('…').expect("one ellipsis");
|
||||
assert!(
|
||||
!head.is_empty() && !tail.is_empty(),
|
||||
"both ends survive: {out}"
|
||||
);
|
||||
assert!(DEEP.starts_with(head), "{out}");
|
||||
assert!(DEEP.ends_with(tail), "{out}");
|
||||
assert!(
|
||||
tail.ends_with("search_tab.rs"),
|
||||
"the filename survives: {out}"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// The load-bearing property. If the result laid out in the same font
|
||||
/// were wider than the budget, egui would elide it a *second* time and
|
||||
/// paint two ellipses.
|
||||
#[test]
|
||||
fn the_result_fits_the_budget_it_was_given() {
|
||||
with_ui(|ui| {
|
||||
let font = body_font(ui);
|
||||
for width in [40.0f32, 60.0, 121.5, 200.0, 337.5, 480.0] {
|
||||
let out = middle_elide(ui, DEEP, width, &font);
|
||||
let painted = ui.fonts(|f| {
|
||||
f.layout_no_wrap(out.to_string(), font.clone(), egui::Color32::WHITE)
|
||||
.size()
|
||||
.x
|
||||
});
|
||||
assert!(
|
||||
painted <= width,
|
||||
"at {width}: {out:?} lays out at {painted}"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// An ellipsis is the least that can stand for the text; a column too
|
||||
/// narrow even for that gets it anyway rather than a panic.
|
||||
#[test]
|
||||
fn degenerate_widths_never_panic() {
|
||||
with_ui(|ui| {
|
||||
let font = body_font(ui);
|
||||
for width in [f32::NEG_INFINITY, -50.0, 0.0] {
|
||||
assert_eq!(middle_elide(ui, DEEP, width, &font), "…", "at {width}");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Indices walk by whole characters, so a path of multi-byte glyphs
|
||||
/// slices cleanly instead of panicking mid-codepoint.
|
||||
#[test]
|
||||
fn multi_byte_paths_split_on_character_boundaries() {
|
||||
with_ui(|ui| {
|
||||
let font = body_font(ui);
|
||||
let path = "/srv/données/日本語/архив/файл-très-long.txt";
|
||||
for width in [30.0f32, 55.0, 90.0, 140.0, 210.0, 400.0] {
|
||||
let out = middle_elide(ui, path, width, &font);
|
||||
let (head, tail) = out.split_once('…').unwrap_or((out.as_ref(), ""));
|
||||
assert!(path.starts_with(head), "at {width}: {out}");
|
||||
assert!(path.ends_with(tail), "at {width}: {out}");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// The trap behind "my ignore filters don't work" reports: ".jpg" is an
|
||||
/// exact-name pattern, and the hint must say so and offer "*.jpg".
|
||||
|
|
@ -266,4 +520,134 @@ mod tests {
|
|||
// worth flagging, unlike a box the user simply has not filled in.
|
||||
assert_eq!(pattern_border("/"), Some(INVALID_RED));
|
||||
}
|
||||
|
||||
// --- The results wipe ---------------------------------------------------
|
||||
|
||||
const SECTION: egui::Rect = egui::Rect {
|
||||
min: egui::pos2(10.0, 100.0),
|
||||
max: egui::pos2(410.0, 500.0),
|
||||
};
|
||||
const FILL: egui::Color32 = egui::Color32::from_rgb(27, 27, 27);
|
||||
|
||||
/// Every vertex of `mesh` as (y, alpha), in paint order.
|
||||
fn ramp(mesh: &egui::Mesh) -> Vec<(f32, u8)> {
|
||||
mesh.vertices
|
||||
.iter()
|
||||
.map(|v| (v.pos.y, v.color.a()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The y ranges the mesh's quads cover, merged where they touch.
|
||||
fn covered_spans(mesh: &egui::Mesh) -> Vec<(f32, f32)> {
|
||||
let mut spans: Vec<(f32, f32)> = Vec::new();
|
||||
for quad in mesh.vertices.chunks(4) {
|
||||
let (top, bottom) = (quad[0].pos.y, quad[3].pos.y);
|
||||
match spans.last_mut() {
|
||||
Some(last) if (last.1 - top).abs() < 1e-3 => last.1 = bottom,
|
||||
_ => spans.push((top, bottom)),
|
||||
}
|
||||
}
|
||||
spans
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_revealed_section_paints_no_scrim() {
|
||||
// The steady state is the common one: no shape, no vertices, no cost.
|
||||
assert!(wipe_mesh(SECTION, 0.0, FILL).is_none());
|
||||
assert!(wipe_mesh(SECTION, -0.5, FILL).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_degenerate_section_paints_no_scrim() {
|
||||
let flat = egui::Rect::from_min_max(egui::pos2(10.0, 100.0), egui::pos2(410.0, 100.0));
|
||||
assert!(wipe_mesh(flat, 0.5, FILL).is_none());
|
||||
let sliver = egui::Rect::from_min_max(egui::pos2(10.0, 100.0), egui::pos2(10.0, 500.0));
|
||||
assert!(wipe_mesh(sliver, 0.5, FILL).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unstarted_wipe_covers_the_whole_section() {
|
||||
let mesh = wipe_mesh(SECTION, 1.0, FILL).expect("fully hidden");
|
||||
assert!(
|
||||
ramp(&mesh).iter().all(|&(_, a)| a == 255),
|
||||
"nothing may show through before the reveal starts: {:?}",
|
||||
ramp(&mesh)
|
||||
);
|
||||
assert_eq!(
|
||||
covered_spans(&mesh),
|
||||
vec![(SECTION.top(), SECTION.bottom())],
|
||||
"the quads must tile the section with no gap"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_scrim_ramps_clear_at_the_top_to_solid_at_the_bottom() {
|
||||
let mesh = wipe_mesh(SECTION, 0.5, FILL).expect("mid-travel");
|
||||
let ramp = ramp(&mesh);
|
||||
assert_eq!(ramp.first().expect("vertices").1, 0, "the top is untouched");
|
||||
assert_eq!(ramp.last().expect("vertices").1, 255, "the bottom is gone");
|
||||
// Alpha only ever increases downward, and the quads stay contiguous:
|
||||
// a gradient, not a stack of steps.
|
||||
for pair in ramp.windows(2) {
|
||||
assert!(
|
||||
pair[1].0 >= pair[0].0 && pair[1].1 >= pair[0].1,
|
||||
"vertices run down the section, clear to opaque: {ramp:?}"
|
||||
);
|
||||
}
|
||||
assert_eq!(covered_spans(&mesh).len(), 1, "one contiguous scrim");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn less_of_the_section_shows_the_further_the_wipe_is_from_done() {
|
||||
// How much of the section's height the scrim swallows: the integral
|
||||
// of alpha down it, so a widening gradient counts as well as a
|
||||
// growing solid block.
|
||||
let hidden_height = |wipe: f32| {
|
||||
let mesh = wipe_mesh(SECTION, wipe, FILL).expect("travelling");
|
||||
mesh.vertices
|
||||
.chunks(4)
|
||||
.map(|quad| {
|
||||
let alpha = |v: &egui::epaint::Vertex| v.color.a() as f32 / 255.0;
|
||||
(quad[3].pos.y - quad[0].pos.y) * (alpha(&quad[0]) + alpha(&quad[3])) / 2.0
|
||||
})
|
||||
.sum::<f32>()
|
||||
};
|
||||
let mut previous = 0.0;
|
||||
for step in 1..=10 {
|
||||
let hidden = hidden_height(step as f32 / 10.0);
|
||||
assert!(
|
||||
hidden > previous,
|
||||
"each step hides more than the last: {hidden} after {previous}"
|
||||
);
|
||||
previous = hidden;
|
||||
}
|
||||
assert!(
|
||||
(previous - SECTION.height()).abs() < 0.5,
|
||||
"and the section is entirely gone by the end: {previous} of {}",
|
||||
SECTION.height()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_short_section_still_gets_a_gradient() {
|
||||
// Two rows tall: the proportional band would be a few points, small
|
||||
// enough to read as a hard cut, so the floor takes over.
|
||||
let short = egui::Rect::from_min_max(egui::pos2(10.0, 100.0), egui::pos2(410.0, 130.0));
|
||||
let mesh = wipe_mesh(short, 0.5, FILL).expect("mid-travel");
|
||||
let gradient = mesh
|
||||
.vertices
|
||||
.chunks(4)
|
||||
.any(|quad| quad[0].color.a() < quad[3].color.a());
|
||||
assert!(
|
||||
gradient,
|
||||
"the edge ramps rather than cutting: {:?}",
|
||||
ramp(&mesh)
|
||||
);
|
||||
let (clear, covered) = super::wipe_edges(short, 0.5);
|
||||
assert!(
|
||||
(covered - clear - WIPE_BAND_MIN).abs() < 1e-3,
|
||||
"the band is held at its floor: {}",
|
||||
covered - clear
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,6 +38,9 @@ pub enum KeySource {
|
|||
|
||||
/// The application shell handed to eframe: locked (unlock screen) or
|
||||
/// running (the real app).
|
||||
// Exactly one of these exists for the lifetime of the process, and it is
|
||||
// already boxed on the side that would matter.
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
pub enum Gate {
|
||||
Locked(UnlockScreen),
|
||||
Running(Box<QuickSearchApp>),
|
||||
|
|
@ -339,12 +342,11 @@ impl UnlockScreen {
|
|||
/// UI-side buffers.
|
||||
fn submit(&mut self, ctx: &egui::Context) {
|
||||
self.error = None;
|
||||
if matches!(self.mode, Mode::Create) {
|
||||
if self.password.is_empty() {
|
||||
if matches!(self.mode, Mode::Create)
|
||||
&& self.password.is_empty() {
|
||||
self.error = Some("The password may not be empty.".to_string());
|
||||
return;
|
||||
}
|
||||
}
|
||||
let Ok(salt) = self.cfg.security.salt_bytes() else {
|
||||
return; // BrokenSalt mode never reaches submit
|
||||
};
|
||||
|
|
@ -381,8 +383,10 @@ impl UnlockScreen {
|
|||
// stick. Surface it in the running app's banner.
|
||||
self.config_error = Some(e);
|
||||
}
|
||||
} else {
|
||||
keychain::delete_key(&db_path.to_string_lossy());
|
||||
} else if let Err(e) = keychain::delete_key(&db_path.to_string_lossy()) {
|
||||
// Same treatment as the store half above: unlock proceeds, but
|
||||
// the banner says the old key is still on the keychain.
|
||||
self.config_error = Some(e);
|
||||
}
|
||||
if self.cfg.security.use_keychain != self.remember {
|
||||
self.cfg.security.use_keychain = self.remember;
|
||||
|
|
@ -447,7 +451,12 @@ impl UnlockScreen {
|
|||
if let Err(e) = delete_index_files(&db_path) {
|
||||
self.error = Some(e);
|
||||
} else {
|
||||
keychain::delete_key(&db_path.to_string_lossy());
|
||||
// The index files are already gone; a surviving
|
||||
// entry would point at a database that no longer
|
||||
// exists. Non-fatal, but not silent.
|
||||
if let Err(e) = keychain::delete_key(&db_path.to_string_lossy()) {
|
||||
self.config_error = Some(e);
|
||||
}
|
||||
db::set_process_key(None);
|
||||
self.cfg.security = SecurityConfig::default();
|
||||
if let Err(e) = self.cfg.save() {
|
||||
|
|
@ -508,6 +517,9 @@ fn delete_index_files(db_path: &std::path::Path) -> Result<(), String> {
|
|||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The unlock screen is the whole window, so its viewport is the window's.
|
||||
const SCREEN: egui::Vec2 = egui::vec2(900.0, 600.0);
|
||||
|
||||
/// A protected config whose salt parses, so the screen lands in a real
|
||||
/// password mode rather than `BrokenSalt`.
|
||||
fn locked_config() -> Config {
|
||||
|
|
@ -522,13 +534,7 @@ mod tests {
|
|||
}
|
||||
|
||||
fn frame(ctx: &egui::Context, screen: &mut UnlockScreen) {
|
||||
let input = egui::RawInput {
|
||||
screen_rect: Some(egui::Rect::from_min_size(
|
||||
egui::Pos2::ZERO,
|
||||
egui::vec2(900.0, 600.0),
|
||||
)),
|
||||
..Default::default()
|
||||
};
|
||||
let input = crate::test_ui::raw_input(SCREEN, Vec::new());
|
||||
let _ = ctx.run(input, |ctx| {
|
||||
// `update` only builds the app on a successful unlock, which needs
|
||||
// a derived key — so with nothing submitted this stays on screen.
|
||||
|
|
@ -591,13 +597,7 @@ mod tests {
|
|||
fn the_lock_screen_shows_the_build_id() {
|
||||
let ctx = egui::Context::default();
|
||||
let mut screen = UnlockScreen::new(locked_config(), None, None);
|
||||
let input = egui::RawInput {
|
||||
screen_rect: Some(egui::Rect::from_min_size(
|
||||
egui::Pos2::ZERO,
|
||||
egui::vec2(900.0, 600.0),
|
||||
)),
|
||||
..Default::default()
|
||||
};
|
||||
let input = crate::test_ui::raw_input(SCREEN, Vec::new());
|
||||
|
||||
let out = ctx.run(input, |ctx| {
|
||||
assert!(screen.update(ctx).is_none());
|
||||
|
|
|
|||
299
packaging/build-appimage.sh
Normal file
299
packaging/build-appimage.sh
Normal file
|
|
@ -0,0 +1,299 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Build an AppImage for QuickSearch.
|
||||
#
|
||||
# ./packaging/build-appimage.sh build and package
|
||||
# ./packaging/build-appimage.sh --no-build package an existing release binary
|
||||
# ./packaging/build-appimage.sh --no-strip keep debug symbols
|
||||
# ./packaging/build-appimage.sh -o /tmp/out write the AppImage somewhere else
|
||||
#
|
||||
# Environment: SOURCE_DATE_EPOCH, APPIMAGETOOL.
|
||||
#
|
||||
# The AppDir bundles no libraries at all. That is not an oversight: the binary's
|
||||
# dynamic section names only libgcc_s, libm and libc, because SQLCipher, OpenSSL
|
||||
# and libdbus are linked statically, and winit and glutin dlopen the whole
|
||||
# display stack at runtime. Those dlopened libraries - libGL, libEGL, libX11,
|
||||
# libxcb, libXcursor, libXi, libXrender, libxkbcommon{,-x11} and
|
||||
# libwayland-{client,egl} - are exactly the ones an AppImage must take from the
|
||||
# host, since they have to match the user's graphics driver and compositor.
|
||||
# Bundling them is how AppImages break on other people's machines.
|
||||
#
|
||||
# So the glibc floor is the only portability limit, and it is the same one the
|
||||
# .deb carries: whatever the machine that built the binary provides. Release
|
||||
# builds happen in a 22.04 container to keep that at 2.35.
|
||||
|
||||
set -euo pipefail
|
||||
# Directories created along the way must be 0755, not whatever the caller's
|
||||
# umask happens to be, matching build-deb.sh.
|
||||
umask 022
|
||||
|
||||
readonly PKG=quicksearch
|
||||
readonly REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
readonly ICON_SRC="$REPO_ROOT/crates/quicksearch-gui/assets/icons"
|
||||
readonly ICON_SVG="$ICON_SRC/quicksearch_icon.svg"
|
||||
readonly METAINFO=com.karsttech.quicksearch.metainfo.xml
|
||||
|
||||
# Pinned rather than tracking the `continuous` tag, so a rebuild of an old
|
||||
# commit uses the tool that commit was tested with. Bump the version and the
|
||||
# checksum together; the checksum is the release asset's own digest.
|
||||
readonly APPIMAGETOOL_VERSION=1.9.1
|
||||
readonly APPIMAGETOOL_SHA256=ed4ce84f0d9caff66f50bcca6ff6f35aae54ce8135408b3fa33abfc3cb384eb0
|
||||
readonly APPIMAGETOOL_URL="https://github.com/AppImage/appimagetool/releases/download/$APPIMAGETOOL_VERSION/appimagetool-x86_64.AppImage"
|
||||
|
||||
# The runtime is the ~1 MB ELF stub prepended to the squashfs: it is the first
|
||||
# thing that executes on a user's machine, so it matters more than the tool that
|
||||
# assembles it. Left alone, appimagetool downloads it from the type2-runtime
|
||||
# `continuous` tag mid-build, which would put an unpinned binary inside every
|
||||
# release and make the checksum above mostly decorative. Pinned to a dated tag
|
||||
# and passed in with --runtime-file instead.
|
||||
readonly RUNTIME_VERSION=20251108
|
||||
readonly RUNTIME_SHA256=2fca8b443c92510f1483a883f60061ad09b46b978b2631c807cd873a47ec260d
|
||||
readonly RUNTIME_URL="https://github.com/AppImage/type2-runtime/releases/download/$RUNTIME_VERSION/runtime-x86_64"
|
||||
|
||||
# Baked into every AppImage this produces, so AppImageUpdate can find newer
|
||||
# builds. It cannot be corrected after the fact - a released binary keeps
|
||||
# pointing wherever this said at the time - so moving the forge means the old
|
||||
# URL has to keep resolving.
|
||||
#
|
||||
# Forgejo resolves the literal tag `latest` to the newest release and looks the
|
||||
# asset up by name, so this path stays valid across releases. The GitHub-style
|
||||
# /releases/latest/download/<asset> form is not implemented and 404s. The .zsync
|
||||
# name therefore carries no version, while the AppImage it points at does; zsync
|
||||
# resolves that from the sidecar's own headers, relative to this URL.
|
||||
readonly UPDATE_URL="https://code.karsttech.com/jeremy/quick_search/releases/download/latest/$PKG-x86_64.AppImage.zsync"
|
||||
|
||||
do_build=1
|
||||
do_strip=1
|
||||
out_dir="$REPO_ROOT/dist"
|
||||
|
||||
die() { printf 'build-appimage: %s\n' "$*" >&2; exit 1; }
|
||||
say() { printf '\033[1m==>\033[0m %s\n' "$*"; }
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--no-build) do_build=0 ;;
|
||||
--no-strip) do_strip=0 ;;
|
||||
-o|--output-dir) shift; [ $# -gt 0 ] || die "--output-dir needs a path"; out_dir="$1" ;;
|
||||
# Print the header comment block, however long it grows.
|
||||
-h|--help) awk 'NR > 1 { if ($0 !~ /^#/) exit; sub(/^# ?/, ""); print }' "${BASH_SOURCE[0]}"; exit 0 ;;
|
||||
*) die "unknown option: $1 (try --help)" ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# zsyncmake comes from the zsync package. appimagetool looks it up in PATH
|
||||
# rather than bundling it, and - worse - reports success and produces nothing
|
||||
# when it is missing, so it is checked for here instead. appstreamcli comes from
|
||||
# the appstream package. Unlike build-deb.sh, which is deliberately buildable
|
||||
# with nothing but a stock Debian install, this script already downloads
|
||||
# appimagetool, so a couple more packages cost nothing in reach.
|
||||
for tool in curl sha256sum desktop-file-validate zsyncmake appstreamcli strings; do
|
||||
command -v "$tool" >/dev/null 2>&1 || die "missing required tool: $tool"
|
||||
done
|
||||
[ "$do_strip" -eq 0 ] || command -v strip >/dev/null 2>&1 || die "missing strip (install binutils, or pass --no-strip)"
|
||||
|
||||
# Version comes from [workspace.package] so the AppImage can never drift from
|
||||
# the crate version.
|
||||
version="$(sed -n '/^\[workspace\.package\]/,/^\[/{ s/^version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p }' "$REPO_ROOT/Cargo.toml")"
|
||||
[ -n "$version" ] || die "could not read version from Cargo.toml"
|
||||
|
||||
# Absolute from here on: appimagetool is run from a scratch directory below, and
|
||||
# -o could well have been given a relative path.
|
||||
mkdir -p "$out_dir"
|
||||
out_dir="$(cd -- "$out_dir" && pwd)"
|
||||
|
||||
appdir="$out_dir/$PKG-$version-x86_64.AppDir"
|
||||
appimage="$out_dir/$PKG-$version-x86_64.AppImage"
|
||||
# Version-less on purpose: see UPDATE_URL above.
|
||||
zsync="$out_dir/$PKG-x86_64.AppImage.zsync"
|
||||
|
||||
# Scratch space for appimagetool's stray output and the verification unpack.
|
||||
scratch="$(mktemp -d)"
|
||||
trap 'rm -rf "$scratch"' EXIT
|
||||
|
||||
# ------------------------------------------------------- appimagetool ------
|
||||
|
||||
cache_dir="${XDG_CACHE_HOME:-$HOME/.cache}/quicksearch"
|
||||
appimagetool="${APPIMAGETOOL:-$cache_dir/appimagetool-$APPIMAGETOOL_VERSION-x86_64.AppImage}"
|
||||
runtime="$cache_dir/runtime-$RUNTIME_VERSION-x86_64"
|
||||
|
||||
# Fetch to $1 from $2 if it is not cached, then check it against $3 on every
|
||||
# run, not only after a download: a cached file is as much a supply-chain input
|
||||
# as a freshly fetched one, and this is all that stands between a poisoned cache
|
||||
# and a published release.
|
||||
fetch_pinned() {
|
||||
local dest="$1" url="$2" sum="$3"
|
||||
if [ ! -f "$dest" ]; then
|
||||
say "Fetching $(basename -- "$dest")"
|
||||
mkdir -p "$(dirname -- "$dest")"
|
||||
curl --proto '=https' --tlsv1.2 -fsSL -o "$dest.part" "$url" \
|
||||
|| die "could not download $url"
|
||||
mv -- "$dest.part" "$dest"
|
||||
fi
|
||||
printf '%s %s\n' "$sum" "$dest" | sha256sum -c - >/dev/null 2>&1 \
|
||||
|| die "checksum mismatch for $dest (delete it and retry)"
|
||||
}
|
||||
|
||||
if [ -z "${APPIMAGETOOL:-}" ]; then
|
||||
fetch_pinned "$appimagetool" "$APPIMAGETOOL_URL" "$APPIMAGETOOL_SHA256"
|
||||
# curl leaves it 0644, and the cached copy is ours to chmod. A caller-supplied
|
||||
# one might be root-owned in /usr/bin, so that branch only checks.
|
||||
chmod +x "$appimagetool"
|
||||
else
|
||||
[ -x "$appimagetool" ] || die "APPIMAGETOOL is not executable: $appimagetool"
|
||||
fi
|
||||
|
||||
fetch_pinned "$runtime" "$RUNTIME_URL" "$RUNTIME_SHA256"
|
||||
|
||||
# ---------------------------------------------------------------- build ----
|
||||
|
||||
if [ "$do_build" -eq 1 ]; then
|
||||
say "Building quicksearch $version (release)"
|
||||
( cd "$REPO_ROOT" && cargo build --release -p quicksearch-gui )
|
||||
fi
|
||||
|
||||
[ -x "$REPO_ROOT/target/release/$PKG" ] \
|
||||
|| die "no release binary at target/release/$PKG (drop --no-build?)"
|
||||
[ -f "$ICON_SVG" ] || die "no icon at $ICON_SVG"
|
||||
[ -f "$REPO_ROOT/packaging/$METAINFO" ] || die "no metainfo at packaging/$METAINFO"
|
||||
|
||||
say "Validating desktop entry"
|
||||
desktop-file-validate "$REPO_ROOT/packaging/$PKG.desktop"
|
||||
|
||||
# --------------------------------------------------------------- stage -----
|
||||
|
||||
say "Staging $appdir"
|
||||
rm -rf "$appdir"
|
||||
mkdir -p "$appdir"
|
||||
|
||||
# Only the GUI binary ships, where build-deb.sh installs both: an AppImage has a
|
||||
# single entry point, and on Unix `quicksearch` already does the terminal job
|
||||
# too - quicksearch-cli is the same tool under a clearer name, which matters for
|
||||
# something on PATH and not for a self-contained file the user runs directly.
|
||||
install -Dm755 "$REPO_ROOT/target/release/$PKG" "$appdir/usr/bin/$PKG"
|
||||
install -Dm644 "$REPO_ROOT/packaging/$PKG.desktop" "$appdir/usr/share/applications/$PKG.desktop"
|
||||
install -Dm644 "$ICON_SVG" "$appdir/usr/share/icons/hicolor/scalable/apps/$PKG.svg"
|
||||
install -Dm644 "$REPO_ROOT/packaging/copyright" "$appdir/usr/share/doc/$PKG/copyright"
|
||||
install -Dm644 "$REPO_ROOT/config_example.toml" "$appdir/usr/share/doc/$PKG/config_example.toml"
|
||||
install -Dm644 "$REPO_ROOT/README.md" "$appdir/usr/share/doc/$PKG/README.md"
|
||||
|
||||
# Same glob-driven discovery as build-deb.sh, including the guard: the glob
|
||||
# would also catch a non-size name like quicksearch-cli.png.
|
||||
shopt -s nullglob
|
||||
icons=("$ICON_SRC"/$PKG-*.png)
|
||||
shopt -u nullglob
|
||||
[ "${#icons[@]}" -gt 0 ] || die "no icons in $ICON_SRC"
|
||||
for png in "${icons[@]}"; do
|
||||
n="$(basename "$png" .png)"; n="${n#$PKG-}"
|
||||
case "$n" in ''|*[!0-9]*) die "unexpected icon name: $(basename "$png")" ;; esac
|
||||
install -Dm644 "$png" "$appdir/usr/share/icons/hicolor/${n}x${n}/apps/$PKG.png"
|
||||
done
|
||||
say "Installed ${#icons[@]} icon sizes plus the scalable SVG"
|
||||
|
||||
# @VERSION@/@DATE@ substitution, as build-deb.sh does for the man page .TH line,
|
||||
# so [workspace.package] version stays the only place a release is bumped.
|
||||
if [ -n "${SOURCE_DATE_EPOCH:-}" ]; then
|
||||
metainfo_date="$(date -u -d "@$SOURCE_DATE_EPOCH" +%Y-%m-%d)"
|
||||
else
|
||||
metainfo_date="$(date -u +%Y-%m-%d)"
|
||||
fi
|
||||
install -dm755 "$appdir/usr/share/metainfo"
|
||||
sed -e "s/@VERSION@/$version/" -e "s/@DATE@/$metainfo_date/" \
|
||||
"$REPO_ROOT/packaging/$METAINFO" > "$appdir/usr/share/metainfo/$METAINFO"
|
||||
chmod 644 "$appdir/usr/share/metainfo/$METAINFO"
|
||||
|
||||
# Validated here rather than in CI so a local build fails the same way, and on
|
||||
# the substituted file rather than the @VERSION@ template. build-deb.sh ships
|
||||
# the identical file, so this covers both packages. appimagetool would run its
|
||||
# own check, but against whatever appstreamcli happens to be in PATH at the
|
||||
# time; --no-appstream below turns that off in favour of this one.
|
||||
say "Validating AppStream metainfo"
|
||||
appstreamcli validate --no-net --explain "$appdir/usr/share/metainfo/$METAINFO"
|
||||
|
||||
# appimagetool looks for these three at the AppDir root. AppRun is a relative
|
||||
# symlink rather than a wrapper script because nothing is bundled: there is no
|
||||
# LD_LIBRARY_PATH or XDG_DATA_DIRS to set up, so a wrapper would only put a
|
||||
# shell between the runtime and the app.
|
||||
ln -s usr/bin/$PKG "$appdir/AppRun"
|
||||
install -Dm644 "$ICON_SRC/$PKG-256.png" "$appdir/$PKG.png"
|
||||
ln -s $PKG.png "$appdir/.DirIcon"
|
||||
install -Dm644 "$REPO_ROOT/packaging/$PKG.desktop" "$appdir/$PKG.desktop"
|
||||
|
||||
if [ "$do_strip" -eq 1 ]; then
|
||||
before="$(du -h "$appdir/usr/bin/$PKG" | cut -f1)"
|
||||
strip --strip-unneeded "$appdir/usr/bin/$PKG"
|
||||
say "Stripped $PKG: $before -> $(du -h "$appdir/usr/bin/$PKG" | cut -f1)"
|
||||
fi
|
||||
|
||||
# -------------------------------------------------------------- package ----
|
||||
|
||||
say "Building $appimage"
|
||||
rm -f "$appimage" "$zsync"
|
||||
# Run from the scratch directory: -u makes appimagetool shell out to zsyncmake,
|
||||
# which writes its sidecar into the working directory rather than beside the
|
||||
# output, and that stray would otherwise land in the repo root. The one this
|
||||
# script actually ships is generated below.
|
||||
#
|
||||
# APPIMAGE_EXTRACT_AND_RUN makes appimagetool unpack itself instead of mounting
|
||||
# itself, which is what lets this run in a container with no /dev/fuse. ARCH is
|
||||
# set explicitly rather than left to autodetection.
|
||||
( cd "$scratch" && ARCH=x86_64 APPIMAGE_EXTRACT_AND_RUN=1 \
|
||||
"$appimagetool" --no-appstream --runtime-file "$runtime" \
|
||||
-u "zsync|$UPDATE_URL" "$appdir" "$appimage" )
|
||||
|
||||
[ -f "$appimage" ] || die "appimagetool produced no $appimage"
|
||||
chmod 755 "$appimage"
|
||||
|
||||
# The shipped sidecar is generated here rather than left to appimagetool, which
|
||||
# reports Success even when zsyncmake is absent and wrote nothing. Doing it
|
||||
# directly also pins both headers: a bare relative name in URL: is what makes
|
||||
# the stable, version-less sidecar name work, because zsync resolves it against
|
||||
# wherever the sidecar was fetched from and so lands on the current release's
|
||||
# versioned AppImage.
|
||||
say "Generating $zsync"
|
||||
zsyncmake -u "$(basename -- "$appimage")" -f "$(basename -- "$appimage")" \
|
||||
-o "$zsync" "$appimage"
|
||||
[ -f "$zsync" ] || die "zsyncmake produced no $zsync"
|
||||
|
||||
rm -rf "$appdir"
|
||||
|
||||
# --------------------------------------------------------------- verify ----
|
||||
|
||||
say "Verifying $appimage"
|
||||
( cd "$scratch" && "$appimage" --appimage-extract >/dev/null )
|
||||
|
||||
root="$scratch/squashfs-root"
|
||||
for path in AppRun "$PKG.desktop" "$PKG.png" .DirIcon \
|
||||
"usr/bin/$PKG" "usr/share/applications/$PKG.desktop" \
|
||||
"usr/share/metainfo/$METAINFO"; do
|
||||
[ -e "$root/$path" ] || die "$path missing from the AppImage"
|
||||
done
|
||||
# The whole point of the layout: no bundled libraries.
|
||||
[ ! -d "$root/usr/lib" ] || die "the AppDir grew a usr/lib - see the header comment"
|
||||
|
||||
# --version is handled before any window is created, so this works headless.
|
||||
reported="$("$root/usr/bin/$PKG" --version)"
|
||||
case "$reported" in
|
||||
*"v$version"*) ;;
|
||||
*) die "the packaged binary reports '$reported', expected v$version" ;;
|
||||
esac
|
||||
|
||||
# A sidecar describing a different file is worse than no sidecar: AppImageUpdate
|
||||
# would fetch and then reject every delta. Length is the cheap way to catch it.
|
||||
zsync_len="$(sed -n 's/^Length: //p' "$zsync" | head -1)"
|
||||
appimage_len="$(stat -c %s "$appimage")"
|
||||
[ "$zsync_len" = "$appimage_len" ] \
|
||||
|| die "$zsync describes $zsync_len bytes, but the AppImage is $appimage_len"
|
||||
|
||||
# The update information the runtime itself carries, which is what
|
||||
# AppImageUpdate reads before it ever looks for a sidecar.
|
||||
embedded="$(strings -a "$appimage" | grep -m1 '^zsync|' || true)"
|
||||
[ "$embedded" = "zsync|$UPDATE_URL" ] \
|
||||
|| die "embedded update info is '$embedded', expected 'zsync|$UPDATE_URL'"
|
||||
|
||||
echo
|
||||
say "Done: $appimage"
|
||||
echo " reports: $reported"
|
||||
echo " update info: $UPDATE_URL"
|
||||
echo " sidecar: $zsync"
|
||||
echo " run with: chmod +x $appimage && $appimage"
|
||||
|
|
@ -26,6 +26,7 @@ readonly BINARIES=(quicksearch quicksearch-cli)
|
|||
readonly REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
readonly ICON_SRC="$REPO_ROOT/crates/quicksearch-gui/assets/icons"
|
||||
readonly ICON_SVG="$ICON_SRC/quicksearch_icon.svg"
|
||||
readonly METAINFO=com.karsttech.quicksearch.metainfo.xml
|
||||
|
||||
do_build=1
|
||||
do_strip=1
|
||||
|
|
@ -78,6 +79,7 @@ done
|
|||
# Both binaries link the same crates, so either gives the same glibc floor.
|
||||
readonly primary_binary="$REPO_ROOT/target/release/$PKG"
|
||||
[ -f "$ICON_SVG" ] || die "no icon at $ICON_SVG"
|
||||
[ -f "$REPO_ROOT/packaging/$METAINFO" ] || die "no metainfo at packaging/$METAINFO"
|
||||
|
||||
say "Validating desktop entry"
|
||||
desktop-file-validate "$REPO_ROOT/packaging/$PKG.desktop"
|
||||
|
|
@ -119,12 +121,35 @@ for png in "${icons[@]}"; do
|
|||
done
|
||||
say "Installed ${#icons[@]} icon sizes plus the scalable SVG"
|
||||
|
||||
# AppStream metadata, so software centres show a real listing rather than a bare
|
||||
# package name. @VERSION@/@DATE@ are substituted the same way the man page .TH
|
||||
# line is below, keeping [workspace.package] version the only thing a release
|
||||
# bumps. build-appimage.sh ships the identical file and is where it gets
|
||||
# validated - appstreamcli is not something a stock Debian install has, and this
|
||||
# script deliberately needs nothing beyond dpkg-deb and desktop-file-utils.
|
||||
if [ -n "${SOURCE_DATE_EPOCH:-}" ]; then
|
||||
metainfo_date="$(date -u -d "@$SOURCE_DATE_EPOCH" +%Y-%m-%d)"
|
||||
else
|
||||
metainfo_date="$(date -u +%Y-%m-%d)"
|
||||
fi
|
||||
install -dm755 "$stage/usr/share/metainfo"
|
||||
sed -e "s/@VERSION@/$version/" -e "s/@DATE@/$metainfo_date/" \
|
||||
"$REPO_ROOT/packaging/$METAINFO" > "$stage/usr/share/metainfo/$METAINFO"
|
||||
chmod 644 "$stage/usr/share/metainfo/$METAINFO"
|
||||
|
||||
# Debian wants man pages and the changelog compressed, with no gzip timestamp so
|
||||
# repeat builds are byte-identical. quicksearch-cli.1 is a one-line .so stub
|
||||
# pointing at quicksearch.1, which documents both binaries.
|
||||
#
|
||||
# The .TH version field is rewritten from $version on the way through, so the
|
||||
# footer of an installed page tracks [workspace.package] version instead of
|
||||
# needing a hand-edit every release. The stub has no .TH, so the sed is a
|
||||
# no-op there.
|
||||
install -dm755 "$stage/usr/share/man/man1"
|
||||
for page in "${BINARIES[@]}"; do
|
||||
gzip -9nc "$REPO_ROOT/packaging/$page.1" > "$stage/usr/share/man/man1/$page.1.gz"
|
||||
sed "/^\.TH /s/\"quicksearch [^\"]*\"/\"quicksearch $version\"/" \
|
||||
"$REPO_ROOT/packaging/$page.1" \
|
||||
| gzip -9nc > "$stage/usr/share/man/man1/$page.1.gz"
|
||||
chmod 644 "$stage/usr/share/man/man1/$page.1.gz"
|
||||
done
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ wait_index_idle # hard wait: the full index must exist
|
|||
tab search
|
||||
focus_search # tab switches drop egui focus; re-arm it
|
||||
clear_query
|
||||
window 1120 750 # compact clip: the smallest layout at
|
||||
window 1200 600 # compact clip: the smallest layout at
|
||||
# which every results column still fits
|
||||
# (any narrower clips the Match column
|
||||
# away, defeating the demo), at 1.5x
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ ignore_patterns = []
|
|||
# 1.5x zoom + proportionally larger windows (set in the scenario) render the
|
||||
# same layout at ~1.5x the pixel density, for crisper website assets.
|
||||
[ui]
|
||||
scale = 1.5
|
||||
scale = 1.25
|
||||
EOF
|
||||
|
||||
# --- run the scripted app ---------------------------------------------------
|
||||
|
|
|
|||
70
packaging/com.karsttech.quicksearch.metainfo.xml
Normal file
70
packaging/com.karsttech.quicksearch.metainfo.xml
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
AppStream metadata for QuickSearch.
|
||||
|
||||
@VERSION@ and @DATE@ are substituted at package time by build-deb.sh and
|
||||
build-appimage.sh, the same way build-deb.sh rewrites the man page .TH version.
|
||||
[workspace.package] version in Cargo.toml stays the single source of truth, so
|
||||
nothing here needs a hand-edit at release time.
|
||||
|
||||
The file name has to match the component id, which is why it is not simply
|
||||
quicksearch.metainfo.xml.
|
||||
-->
|
||||
<component type="desktop-application">
|
||||
<id>com.karsttech.quicksearch</id>
|
||||
<name>QuickSearch</name>
|
||||
<summary>Fast full-text search across your files</summary>
|
||||
|
||||
<metadata_license>CC0-1.0</metadata_license>
|
||||
<project_license>GPL-3.0-or-later</project_license>
|
||||
|
||||
<!-- A reverse-DNS id cannot also be the desktop file's basename, so this is
|
||||
what ties the component to packaging/quicksearch.desktop. -->
|
||||
<launchable type="desktop-id">quicksearch.desktop</launchable>
|
||||
|
||||
<description>
|
||||
<p>
|
||||
QuickSearch keeps a SQLite/FTS5 index of the directories you choose and
|
||||
searches them by both filename and file content. It extracts text from
|
||||
documents, PDFs, archives and office files, watches the indexed paths for
|
||||
changes and reindexes in the background while the application is open.
|
||||
</p>
|
||||
<p>
|
||||
The same binary doubles as a terminal search tool: "quicksearch" followed
|
||||
by search terms prints ranked results and exits without starting the
|
||||
indexer, the file watcher or any background thread.
|
||||
</p>
|
||||
</description>
|
||||
|
||||
<url type="homepage">https://quicksearch.karsttech.com</url>
|
||||
<url type="bugtracker">https://github.com/DataScienceDIY/quick_search/issues</url>
|
||||
<url type="vcs-browser">https://code.karsttech.com/jeremy/quick_search</url>
|
||||
|
||||
<provides>
|
||||
<binary>quicksearch</binary>
|
||||
<binary>quicksearch-cli</binary>
|
||||
</provides>
|
||||
|
||||
<!-- Kept in step with packaging/quicksearch.desktop. A distribution's metadata
|
||||
generator merges these from the desktop file, but an AppImage has no such
|
||||
step, so they are spelled out here too. -->
|
||||
<categories>
|
||||
<category>Utility</category>
|
||||
<category>Filesystem</category>
|
||||
</categories>
|
||||
|
||||
<keywords>
|
||||
<keyword>search</keyword>
|
||||
<keyword>find</keyword>
|
||||
<keyword>index</keyword>
|
||||
<keyword>full-text</keyword>
|
||||
<keyword>files</keyword>
|
||||
<keyword>content</keyword>
|
||||
</keywords>
|
||||
|
||||
<content_rating type="oars-1.1"/>
|
||||
|
||||
<releases>
|
||||
<release version="@VERSION@" date="@DATE@"/>
|
||||
</releases>
|
||||
</component>
|
||||
|
|
@ -1,4 +1,7 @@
|
|||
.TH QUICKSEARCH 1 "2026-08-02" "quicksearch 0.1.0" "User Commands"
|
||||
.\" The version in the .TH line above is rewritten from [workspace.package]
|
||||
.\" version by packaging/build-deb.sh at package time; the literal here is
|
||||
.\" only what an uninstalled read of this file shows.
|
||||
.TH QUICKSEARCH 1 "2026-08-05" "quicksearch 1.0.2" "User Commands"
|
||||
.SH NAME
|
||||
quicksearch, quicksearch\-cli \- fast full\-text search across your files
|
||||
.SH SYNOPSIS
|
||||
|
|
@ -60,29 +63,78 @@ of bare paths. Highlights the match in bold when stdout is a terminal.
|
|||
.TP
|
||||
.BR \-h ", " \-\-help
|
||||
Print usage and exit.
|
||||
.TP
|
||||
.BR \-V ", " \-\-version
|
||||
Print the version and the commit it was built from, then exit. Quote this in
|
||||
bug reports.
|
||||
.PP
|
||||
An unrecognised option given without any query terms is passed through and the
|
||||
application is opened, since it may be an option for the windowing backend.
|
||||
.SH QUERY SYNTAX
|
||||
Plain words form a single phrase. Filters may be combined with it:
|
||||
Plain words form a single phrase, matched against names, contents and paths.
|
||||
Filters may be combined with it:
|
||||
.TP
|
||||
.B \(dq\fIexact phrase\fB\(dq
|
||||
Quoting keeps spaces, stars and filter\-like words literal.
|
||||
.B \(dq\(dq
|
||||
escapes a quote.
|
||||
.TP
|
||||
.BI bud * port
|
||||
.B *
|
||||
matches any run of characters, staying within one line of content.
|
||||
.B %
|
||||
and
|
||||
.B _
|
||||
are always literal.
|
||||
.TP
|
||||
.B regex:\fIpattern\fR
|
||||
Match a regular expression against names, contents and paths, for example
|
||||
.BR regex:\(dq(foo|bar)\ed+\(dq .
|
||||
Case\-insensitive by default;
|
||||
.B (?\-i:\(dq\(dq)
|
||||
overrides. Quote patterns containing spaces or
|
||||
.BR "( ) : = < > \(dq" .
|
||||
.TP
|
||||
.B type:\fIName\fR
|
||||
Match a file class, for example
|
||||
.IR type:Document ", " type:Image ", " type:Audio .
|
||||
Match a file class: one of
|
||||
.IR Audio ", " Image ", " Video ", " Document ", " Text ", " Archive ", "
|
||||
.IR Spreadsheet ", " Presentation ", " Folder .
|
||||
.TP
|
||||
.B modified:\fIexpr\fR
|
||||
Compare against the modification date, for example
|
||||
.IR modified:>=2024-01-01 .
|
||||
Also
|
||||
.BR < ", " <= ", " > " and " = ;
|
||||
dates are
|
||||
.IR yyyy-mm-dd .
|
||||
.B mtime:
|
||||
is an alias.
|
||||
.TP
|
||||
.B path:\fI/dir\fR
|
||||
Restrict results to a directory.
|
||||
Restrict results to a directory and its subdirectories.
|
||||
.B folder:
|
||||
and
|
||||
.B includefolder:
|
||||
are aliases.
|
||||
.B *
|
||||
is literal here.
|
||||
.TP
|
||||
.B mime:\fItype\fR
|
||||
Match a MIME type, for example
|
||||
Match a MIME type exactly, for example
|
||||
.IR mime:application/pdf .
|
||||
.TP
|
||||
.B name:\fIfragment\fR
|
||||
Match a fragment of the filename.
|
||||
Match a fragment of the filename. A filter, so it does not affect ranking.
|
||||
.B filename:
|
||||
is an alias; an unquoted
|
||||
.B *
|
||||
globs.
|
||||
.PP
|
||||
Unrecognised
|
||||
.I key:value
|
||||
text stays part of the search phrase.
|
||||
.BR AND ", " OR
|
||||
and parentheses are treated as plain words.
|
||||
.SH PASSWORD PROTECTION
|
||||
The index can be encrypted with a password (application Options, Security).
|
||||
A protected index must be unlocked every time either binary starts. The
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue