diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 7ad3524..8743d60 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -29,6 +29,12 @@ env: # cached target/ tree for nothing. CARGO_INCREMENTAL: '0' RUST_BACKTRACE: '1' + # Baked into the binaries by crates/quicksearch-gui/build.rs and shown in the + # GUI status bar, so a screenshot or a bug report identifies the exact build. + # Handed over rather than shelled out to git: checkout leaves a shallow clone + # this would otherwise have to trust, and the runner already knows the SHA. + # The version itself still comes from [workspace.package], as everywhere else. + QS_COMMIT: ${{ github.sha }} jobs: # ---------------------------------------------------------------- linux ---- @@ -238,9 +244,12 @@ jobs: - name: Install build dependencies run: | apt-get update -qq + # nsis is what compiles the Windows installer, and it compiles it here + # rather than on Windows: makensis is a Linux binary that emits a PE, + # so both Windows assets come out of this one job. apt-get install -y --no-install-recommends \ build-essential perl make pkg-config \ - gcc-mingw-w64-x86-64 zip + gcc-mingw-w64-x86-64 zip nsis - name: Trust the workspace run: git config --global --add safe.directory "$GITHUB_WORKSPACE" @@ -288,6 +297,15 @@ jobs: fi done + - name: Build the installer + # --no-build reuses the binaries from the Build step rather than + # cross-compiling them a second time. The installer and the .zip below + # are alternatives, not a two-step download: the installer puts the app + # in Program Files with a Start menu entry and an uninstall entry, the + # .zip is the same binaries for anyone who wants them unpacked by hand + # or run portably. + run: ./packaging/build-installer.sh --no-build + - name: Package the binaries run: | version=$(sed -n '/^\[workspace\.package\]/,/^\[/{ s/^version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p }' Cargo.toml) @@ -313,7 +331,9 @@ jobs: - uses: actions/upload-artifact@v3-node20 with: name: windows-x86_64 - path: dist/*.zip + path: | + dist/*-setup.exe + dist/*.zip if-no-files-found: error retention-days: 14 diff --git a/.gitignore b/.gitignore index 202c536..d931c2f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /dist *.db config.toml +/packaging/captures/ diff --git a/Cargo.lock b/Cargo.lock index b602e65..1aea478 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3190,7 +3190,7 @@ dependencies = [ [[package]] name = "quicksearch-core" -version = "0.9.1" +version = "0.9.2" dependencies = [ "argon2", "chardetng", @@ -3222,7 +3222,7 @@ dependencies = [ [[package]] name = "quicksearch-gui" -version = "0.9.1" +version = "0.9.2" dependencies = [ "chrono", "eframe", diff --git a/Cargo.toml b/Cargo.toml index 87f125a..652f7bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ ] [workspace.package] -version = "0.9.1" +version = "0.9.2" edition = "2021" license = "GPL-3.0-or-later" authors = ["Jeremy "] diff --git a/README.md b/README.md index 41984f7..82f98bd 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,9 @@ the distribution's package manager via `sudo` (apt/dnf/pacman/zypper) and the Rust toolchain with rustup; `build.bat` uses winget and rustup. Both take `--check` to report dependency status without installing or building, `--no-run` to stop after the build, and `--` to pass the rest to the binary. +`build.sh` also takes `--installer`, which adds NSIS and the mingw-w64 cross +toolchain to what it installs and builds the Windows installer instead of +launching anything — see [Install (Windows)](#install-windows). Building by hand needs a Rust toolchain plus, on every platform, a C toolchain and Perl: SQLCipher, zstd and OpenSSL are compiled from bundled C sources, and @@ -63,15 +66,15 @@ and exit codes behave normally. On Unix `quicksearch` also does both, and ```sh ./packaging/build-deb.sh -sudo apt install ./dist/quicksearch_0.1.0-1_amd64.deb +sudo apt install ./dist/quicksearch_0.1.0_amd64.deb ``` The script builds the release binary, strips it, and assembles a `.deb` with `dpkg-deb`. It needs no `cargo-deb`, no `debhelper` and no SVG rasteriser — only `dpkg-deb` and `desktop-file-utils`, both standard on Debian and Ubuntu. Useful flags: `--no-build` to package a binary you already built, `--no-strip` -to keep debug symbols, `-o DIR` to write elsewhere. `DEB_REVISION` and -`DEB_MAINTAINER` override the packaging revision and maintainer. +to keep debug symbols, `-o DIR` to write elsewhere. `DEB_MAINTAINER` overrides +the packaging maintainer. The package installs: @@ -107,6 +110,76 @@ matches the app id (`quicksearch`) against the installed `quicksearch.desktop`, so under Wayland the titlebar icon appears only once the package is installed. +`quicksearch.ico` in the same directory bundles the 16–256px PNGs unchanged +(one PNG-compressed entry per size) for the Windows installer, which uses it +for the installer window, the shortcuts and the Add/Remove Programs entry. +Regenerate it from the PNGs with Pillow: open each `quicksearch-N.png`, +largest first, and `save(..., format="ICO", sizes=[...], append_images=rest)` +— passing the images rather than one image and a size list is what keeps the +committed pixels instead of resampling them. + +## Install (Windows) + +Download `quicksearch--windows-x86_64-setup.exe` from the release +page and run it, or build it on a Linux machine: + +```sh +./build.sh --installer # installs the two extra packages first +./packaging/build-installer.sh # or straight to the build +``` + +That cross-compiles for `x86_64-pc-windows-gnu` and compiles the installer +with NSIS, which runs on Linux — no Windows machine is involved, and CI +produces the installer in the same job as the `.zip`. It needs `nsis` and +`gcc-mingw-w64-x86-64` (`mingw32-nsis` and `mingw64-gcc` on Fedora, `nsis` and +`mingw-w64-gcc` on Arch; on openSUSE both come from the `windows:mingw` OBS +project, so `build.sh` names them and leaves the repository to you). The same +flags as `build-deb.sh` apply: `--no-build` to package binaries you already +built, `--no-strip`, `-o DIR`; after `--`, `build.sh --installer` passes them +straight through. + +The install is per-machine and asks for elevation. Into +`C:\Program Files\QuickSearch` go: + +| File | Contents | +| --- | --- | +| `quicksearch.exe` | the desktop app | +| `quicksearch-cli.exe` | terminal search | +| `quicksearch.ico` | icon for the shortcuts and Add/Remove Programs | +| `README.md`, `LICENSE.txt`, `config_example.toml` | documentation | +| `uninstall.exe` | written by the installer; Add/Remove Programs runs it | + +The components page offers a Start menu shortcut (on) and a desktop shortcut +(off); both are created for all users. No `config.toml` is installed, for the +same reason the `.deb` ships none — one next to the binaries is portable mode +(see [Configuration](#configuration)) and would override the personal config +of every account. The app writes `%APPDATA%\quicksearch\config.toml` on first +run instead. + +Installing over an older version reuses wherever that one went, taken from its +registry entry rather than guessed. Both the installer and the uninstaller +stop with a message if QuickSearch is still running, since Windows will not +replace a running executable and the alternative is a half-replaced install. + +Uninstalling removes what was installed and nothing else. The index in +`%LOCALAPPDATA%\quicksearch` and the config in `%APPDATA%\quicksearch` stay, +so reinstalling picks up the existing index; the program directory is removed +only if empty, which leaves a portable-mode `config.toml` and its index alone. + +`PATH` is deliberately untouched — add `C:\Program Files\QuickSearch` to it +yourself if you want `quicksearch-cli` on every prompt. It is an NSIS +installer, so it takes `/S` for a silent install and `/D=` for the directory +(last argument, unquoted): + +```bat +quicksearch-0.9.1-windows-x86_64-setup.exe /S /D=C:\Tools\QuickSearch +``` + +The `.zip` on the release page is the alternative to all of this: the same two +binaries, no registry entries and nothing to uninstall. Unpack it anywhere, +and drop a `config.toml` next to the binaries to keep the config and index +inside that folder. + ## Usage ### GUI @@ -256,6 +329,23 @@ containing the binary, its config, and its index can be moved wholesale. The GUI edits the config live; external edits apply on next start. +**Changing what is indexed** does not throw the index away. Narrowing the +scope — removing a folder, adding an ignore pattern, turning off hidden +files or symlink following, shortening `content_extensions` — deletes +exactly the entries that fell out of scope, in place. Widening it — adding +a folder, deleting a pattern, lengthening the extension list — schedules a +reindex to find what is newly in scope. Both happen automatically, in +automatic and manual mode alike, and neither asks first: it is the edit you +just made. Order and spelling are not changes at all, so reordering the +folder list or writing `~/docs` where you wrote `/home/you/docs` costs +nothing. + +Only three settings still delete and rebuild the index, because nothing +stored survives them: `processing.tokenize` (part of the FTS table's +definition), `processing.hash_length` (existing hashes become +incomparable), and turning password protection on or off or changing the +password. In manual mode those ask for confirmation first. + ## Engineering overview Two crates: @@ -308,6 +398,20 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. checkpoint, VACUUM if the file has at least 10% slack to reclaim, `PRAGMA optimize`, checkpoint again. Progress streams through a polled `IndexingStatus`, which reads `Optimizing` for the duration of that pass. +- **Scope reconciliation** (`scope.rs`): the index is a cache of what a walk + under the configured roots would produce, so a configuration change is a + difference between the two rather than a reason to start over. + `config::diff_actions` turns old-versus-new into an `IndexWork` plan — + roots to delete by path range, rows to re-test against the walker's own + filtering rules (`Scope::covers` mirrors `read_directory` exactly, or the + next run would re-add what the last prune removed), stored text to + re-decide, and whether a walk must follow. The coordinator applies it in + 250 ms slices so a multi-million-row scan never blocks its command loop, + and every run applies it once more against the `config_validation` + fingerprint, which is what makes a config hand-edited while the app was + closed behave like one edited live. The scan is per-root, by `[lo, hi)` + range: a symlink target stored outside every root has no owning root and + therefore no rules that could be applied to it, so it is never visited. - **Coordinator** (`coordinator.rs`): the object binaries construct. Owns the `IndexingService`, the debouncing filesystem watcher (`watcher.rs`), and the mode state machine (Auto / Manual, persisted as @@ -316,7 +420,8 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime. events become single-file transactions (`incremental.rs`) that keep `files`, FTS, and the text sidecar consistent per commit; a full reindex runs on a configurable interval. Incremental writes defer while - a full run is active, so there is exactly one writer at a time. + a full run is active, so there is exactly one writer at a time — scope + reconciliation defers with them, for the same reason. Registration follows what the platform's notification API can do: inotify covers one directory per watch, so the roots are walked and each surviving directory registered individually (skipping `.git`, @@ -396,16 +501,35 @@ pagination: the table is virtualized, so a single scroll list capped at crates, so the `--locked` build after it still fails on a dependency added or bumped without committing `Cargo.lock`. Once both build jobs are green, CI tags that commit `v` and publishes a release - with the `.deb`, a Linux tarball and a Windows zip attached; pushing a `v*` - tag by hand does the same thing. The version is never taken from the branch + with the `.deb`, a Linux tarball, the Windows installer and a Windows zip + attached; pushing a `v*` tag by hand does the same thing. The version is never taken from the branch name, and a tag that already exists at a different commit aborts the release - rather than shipping two builds under one version. The Linux job runs in an Ubuntu + rather than shipping two builds under one version. Every build carries its + identity: `crates/quicksearch-gui/build.rs` bakes in the commit CI passes as + `QS_COMMIT`, and the pair shows up as `v ()` in the + bottom-right of the status bar, from `quicksearch-cli --version`, and in the + Windows `.exe` properties. A build made outside a git checkout reads + `unknown` there rather than failing. The Linux job runs in an Ubuntu 22.04 container on purpose — `packaging/build-deb.sh` reads the package's `libc6` floor from the binary it just built, so the builder's glibc becomes the package's minimum, and 22.04 pins it at 2.35. The Windows job cross-compiles with mingw-w64 and fails if either `.exe` picks up a - dependency on a non-system DLL. + dependency on a non-system DLL, then builds both Windows assets from those + binaries — `packaging/build-installer.sh` runs `makensis`, which is a Linux + program, so the installer needs no Windows runner either. - New extractors: implement `extract::Extractor` and register it in `Registry::default_set()` — order matters, the first extractor whose `supports` accepts a MIME wins. New cascade behavior: `search/cascade.rs` documents the rank invariants that keep streamed results append-only. +- `packaging/capture.sh`: regenerates the website assets — `search.webm`, + `manage-indexing.webm`, `duplicates.png`, `query-highlight.png` — into + `packaging/captures/` (gitignored). It builds the GUI with the `capture` + feature, whose scripted driver types, switches tabs, waits on indexer + state, and captures both screenshots and video frames from the app's own + framebuffer (piped to ffmpeg), so the display server never matters — X11 + and Wayland record identically, and overlapping windows can't leak into + the footage; `packaging/capture-scenario.txt` is the choreography and is + meant to be edited. Runs against a throwaway index of this repository plus + `~/.cargo/registry/src` under scratch XDG dirs, so your real config and + index are untouched. Needs a graphical session and ffmpeg with + `libx264rgb` and `libvpx-vp9`. diff --git a/build.sh b/build.sh index 1036c2b..566538a 100755 --- a/build.sh +++ b/build.sh @@ -8,6 +8,7 @@ # ./build.sh install what is missing, build release, launch # ./build.sh --no-run stop after the build # ./build.sh --check report dependency status; install and build nothing +# ./build.sh --installer build the Windows installer instead of running # ./build.sh -- pass everything after -- to the launched binary # # Anything the script does not recognise ends its own option parsing, so a bare @@ -18,6 +19,13 @@ # with rustup. Every stage is skipped when what it provides is already present, # so the everyday run costs one `cargo build`. # Other Unixes only get the Rust stage — see the README for their toolchains. +# +# --installer adds NSIS and the mingw-w64 cross toolchain to that list and hands +# off to packaging/build-installer.sh, which cross-compiles and produces +# dist/quicksearch--windows-x86_64-setup.exe. Arguments after -- go to +# that script, so `./build.sh --installer -- --no-strip` works. It is opt-in +# because neither tool has anything to do with building or running QuickSearch +# here: an ordinary ./build.sh should not install a Windows installer compiler. set -e # Not `dirname`: a script whose job is to install missing tools should lean on @@ -34,11 +42,14 @@ need_cmd() { command -v "$1" >/dev/null 2>&1; } do_run=1 mode=run +want_installer=0 while [ $# -gt 0 ]; do case "$1" in --no-run) do_run=0 ;; --check) mode=check ;; + # There is no Linux binary to launch at the end of an installer build. + --installer) want_installer=1; do_run=0 ;; # Print the header comment block, however long it grows. -h|--help) awk 'NR > 1 { if ($0 !~ /^#/) exit; sub(/^# ?/, ""); print }' "$0"; exit 0 ;; --) shift; break ;; @@ -94,24 +105,51 @@ check_deps() { need_cmd pkg-config || missing="$missing pkg-config" # curl is a build dependency only while rustup still has to be downloaded. if ! have_cargo && ! need_cmd curl; then missing="$missing curl"; fi + # Only for --installer: makensis compiles the .nsi into the installer, and + # the mingw-w64 gcc is the linker and the C compiler for the bundled + # SQLCipher, OpenSSL and zstd sources on the Windows target. It also brings + # in the cross binutils, which is where the windres the GUI's build script + # uses to compile the .exe version resource comes from. The Rust side + # needs nothing extra — rust-toolchain.toml already lists the target, so + # rustup installs it with the pinned toolchain. + if [ "$want_installer" = 1 ]; then + need_cmd makensis || missing="$missing makensis" + need_cmd x86_64-w64-mingw32-gcc || missing="$missing x86_64-w64-mingw32-gcc" + fi } # The tool -> package mapping, per package manager. Packages named twice (a # single build-essential covers both cc and make) are deduplicated by the caller. +# Printing nothing means "this distribution has no package for it in its default +# repositories", which the caller reports rather than guessing a name; the two +# --installer tools are the only ones where that happens. packages_for() { case "$1:$2" in apt-get:cc|apt-get:make) echo build-essential ;; + apt-get:makensis) echo nsis ;; + apt-get:x86_64-w64-mingw32-gcc) echo gcc-mingw-w64-x86-64 ;; apt-get:*) echo "$2" ;; dnf:cc) echo gcc gcc-c++ ;; dnf:pkg-config) echo pkgconf-pkg-config ;; + # Fedora ships NSIS as part of its mingw stack, so the name looks + # cross-ish; makensis in it is a native Linux binary all the same. + dnf:makensis) echo mingw32-nsis ;; + dnf:x86_64-w64-mingw32-gcc) echo mingw64-gcc ;; dnf:*) echo "$2" ;; pacman:cc|pacman:make) echo base-devel ;; pacman:pkg-config) echo pkgconf ;; + pacman:makensis) echo nsis ;; + pacman:x86_64-w64-mingw32-gcc) echo mingw-w64-gcc ;; pacman:*) echo "$2" ;; zypper:cc) echo gcc gcc-c++ ;; + # Neither NSIS nor the mingw toolchain is in the openSUSE distribution + # repositories — both live in the windows:mingw OBS project, which is a + # repository the user has to add and not something to do behind their + # back. Hence no mapping. + zypper:makensis|zypper:x86_64-w64-mingw32-gcc) ;; zypper:*) echo "$2" ;; esac } @@ -161,11 +199,26 @@ ensure_system_deps() { [ -n "$pm" ] || die "missing build dependencies:$missing (no apt-get, dnf, pacman or zypper here — install them with your distribution's tools)" pkgs='' + unmapped='' for tool in $missing; do - pkgs="$pkgs $(packages_for "$pm" "$tool")" + mapped="$(packages_for "$pm" "$tool")" + if [ -n "$mapped" ]; then + pkgs="$pkgs $mapped" + else + unmapped="$unmapped $tool" + fi done pkgs="$(printf '%s\n' $pkgs | sort -u | tr '\n' ' ')" + if [ -n "$unmapped" ]; then + say "no package for$unmapped in this distribution's repositories — install it yourself, see the README" + # Nothing left to install means there is nothing to say beyond that. + if [ -z "$pkgs" ]; then + if [ "$mode" = run ]; then die "cannot continue without$unmapped"; fi + return 0 + fi + fi + can_install=1 resolve_sudo || can_install=0 @@ -215,6 +268,14 @@ fi ensure_rust cd "$REPO_ROOT" + +# The installer build has its own cargo invocation — a different target, its own +# staging and makensis - so there is nothing for the native build below to +# contribute, and no Linux binary to launch afterwards. +if [ "$want_installer" = 1 ]; then + exec "$REPO_ROOT/packaging/build-installer.sh" "$@" +fi + cargo build --release -p quicksearch-gui if [ "$do_run" = 0 ]; then diff --git a/config_example.toml b/config_example.toml index 819e8c7..68faf69 100644 --- a/config_example.toml +++ b/config_example.toml @@ -11,6 +11,9 @@ [paths] # One or more directory roots to index. Walked in order; duplicate and # nested roots are de-duplicated automatically. `~` expands to home. +# Adding a folder reindexes to pick it up; removing one deletes its entries +# and leaves the rest of the index alone. Order and spelling do not matter: +# "~/docs", "/home/you/docs" and "/home/you/docs/" are one folder. indexing_paths = ["~"] # SQLite index location. Default: ~/.local/share/quicksearch/index.sqlite # On Windows the default is %LOCALAPPDATA%\quicksearch\index.sqlite. Write @@ -36,12 +39,13 @@ reindex_interval_minutes = 1440 # files as well as at directories: with this off a symlink is not resolved at # all, so its target is never indexed — which matters because a target can # live outside every folder listed above. A resolved target is stored under -# its own real path, not the link's. Changing this changes what is in the -# index, so it prompts for a rebuild. +# its own real path, not the link's. Turning this off removes the entries +# that are no longer in scope; turning it on reindexes to find them. follow_symlinks = false # Index hidden files and directories. That means dot-files everywhere, and # additionally anything carrying the Hidden or System attribute on Windows # (AppData, $RECYCLE.BIN, System Volume Information, pagefile.sys ...). +# Turning this off removes the hidden entries already indexed. include_hidden = false # Empty = extract text from every supported format. Non-empty = content # indexing only for these extensions; other files are still listed for @@ -51,6 +55,9 @@ include_hidden = false # Inside an entry, "#" starts a comment that runs to its end, so entries may # be annotated or commented out: # content_extensions = ["txt", "md # docs", "# pdf — too slow", "(none)"] +# Narrowing this drops the stored text of the files it now excludes, leaving +# them findable by name; widening it reindexes to extract the ones it now +# allows. Order, case, a leading dot and comments make no difference. content_extensions = [] # Excluded from the index entirely. A pattern without a separator matches # any single path component (so ".git" prunes whole subtrees); patterns @@ -115,6 +122,8 @@ tokenize = "trigram" # Store extracted text (zstd-compressed) alongside the FTS index. Off: # the index shrinks to roughly stock-Baloo size, but search loses snippet # previews, occurrence ranking, case verification, and fuzzy full-text. +# Turning it off discards the stored text immediately; turning it on +# re-extracts, because the text of files already indexed was never kept. store_text_for_snippets = true [security] diff --git a/crates/quicksearch-core/src/config.rs b/crates/quicksearch-core/src/config.rs index 818c06e..659d04e 100644 --- a/crates/quicksearch-core/src/config.rs +++ b/crates/quicksearch-core/src/config.rs @@ -13,6 +13,7 @@ //! watcher, repoint search). use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; use std::fs; use std::path::{Path, PathBuf}; @@ -494,6 +495,21 @@ impl Config { } } + /// `resolved_indexing_paths` canonicalized and spelled the way + /// `files.path` prefixes them. + /// + /// The form roots must be compared in: `~/docs`, `docs` in a portable + /// config and `/home/me/docs` are one root under three spellings, and a + /// re-spelling is not a configuration change. Duplicates collapse, order + /// is not preserved — a caller that needs one uses + /// [`Config::resolved_indexing_paths`]. + pub fn normalized_indexing_paths(&self) -> BTreeSet { + self.resolved_indexing_paths() + .iter() + .map(|p| crate::file_handling::normalize_root_string(&p.to_string_lossy())) + .collect() + } + /// `indexing_paths` with the same resolution rules as /// [`resolved_database_path`]. pub fn resolved_indexing_paths(&self) -> Vec { @@ -664,15 +680,106 @@ impl IgnoreSet { } } +/// What must happen to the *stored index* to bring it back in line with the +/// configuration, short of deleting and rebuilding it. +/// +/// Every field is independently satisfiable and the whole thing is +/// idempotent: applying it twice does nothing the second time, which is what +/// lets the same plan be produced from a live config edit and from the +/// `config_validation` fingerprint of a config that was hand-edited while the +/// app was closed. See [`crate::scope`] for the pass that applies it. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct IndexWork { + /// Roots that are no longer configured, in `files.path` spelling. Every + /// row beneath one is deleted; no filesystem access is involved, so a + /// root whose folder is gone is handled the same as one that still + /// exists. + pub drop_roots: Vec, + /// The ignore/hidden rules narrowed. Stored rows under the surviving + /// roots are re-tested against them and the ones the walker would no + /// longer emit are deleted. + pub prune_scope: bool, + /// Symlink following was turned off. A followed target is stored under + /// its own canonical path, which can be outside every root; with links + /// off no walk can produce such a row, and no root's range would ever + /// visit it again, so every row outside the roots goes. + pub drop_aliases: bool, + /// The `content_extensions` filter changed. Kept rows are re-tested + /// against it in both directions: newly-included files go back to + /// pending, newly-excluded ones give up their text, properties and FTS + /// row but keep the name/path row that filename search needs. + pub reconcile_content: bool, + /// `store_text_for_snippets` turned on. Rows that finished extraction + /// under the old setting kept no text, so they must run again. + pub restore_text: bool, + /// `store_text_for_snippets` turned off. The stored text is dead weight + /// now; dropping it leaves full-text search working and only costs + /// snippets. + pub drop_text: bool, + /// Files that are newly in scope exist only on disk — nothing in the + /// index points at them, so a full walk has to go and find them. + pub reindex: bool, +} + +impl IndexWork { + /// Whether there is nothing to do at all. + pub fn is_empty(&self) -> bool { + *self == IndexWork::default() + } + + /// Fold `other` in, so one pass satisfies both. + /// + /// For a second config edit arriving while the first is still being + /// applied: the plans are computed against different configurations and + /// neither knows what the other left undone, so the union of the two is + /// the only thing that is certainly enough. Every part is idempotent, so + /// re-doing the finished half of the first costs time and nothing else. + pub fn merge_from(&mut self, other: &IndexWork) { + for root in &other.drop_roots { + if !self.drop_roots.contains(root) { + self.drop_roots.push(root.clone()); + } + } + self.drop_aliases |= other.drop_aliases; + self.prune_scope |= other.prune_scope; + self.reconcile_content |= other.reconcile_content; + self.restore_text |= other.restore_text; + self.drop_text |= other.drop_text; + self.reindex |= other.reindex; + } + + /// Whether any part of this touches stored rows, as opposed to only + /// asking for another walk. + pub fn touches_index(&self) -> bool { + !self.drop_roots.is_empty() + || self.drop_aliases + || self.prune_scope + || self.reconcile_content + || self.restore_text + || self.drop_text + } + + /// Whether applying this means scanning the rows under each surviving + /// root, rather than just deleting whole ranges. + pub fn scans_rows(&self) -> bool { + self.prune_scope || self.reconcile_content || self.restore_text || self.drop_text + } +} + /// What running services must do after a config edit. Computed by the GUI -/// (the only runtime editor) after saving. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// (the only runtime editor) after saving, and by the coordinator from the +/// config it was already holding. +#[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct ConfigActions { - /// The stored index no longer matches how it would be built — offer the - /// user a rebuild (mirrors the `config_validation` mechanism). + /// The stored file cannot be read or compared under the new + /// configuration and must be deleted and rebuilt from scratch. Reserved + /// for the three settings that leave no other option: the FTS tokenizer + /// (baked into the table definition), the hash length (stored hashes + /// become incomparable) and the encryption key. pub requires_rebuild: bool, - /// Watched roots or link semantics changed — restart the watcher. - pub restart_watcher: bool, + /// Reconciliation the index can do in place. Empty when + /// `requires_rebuild` is set — a wipe subsumes all of it. + pub work: IndexWork, /// Searches must reopen against a different database file. pub search_db_changed: bool, } @@ -709,32 +816,113 @@ pub fn nested_roots(roots: &[String]) -> Vec<(String, String)> { out } +/// The `content_extensions` entries that decide what a file is matched +/// against, normalized the way [`content_allowed`] compares them: comments +/// stripped, a leading dot optional, case-insensitive. Two lists with the +/// same set here filter identically, however they are spelled or ordered. +fn content_filter_set(list: &[String]) -> BTreeSet { + content_filter_entries(list) + .map(|e| e.trim_start_matches('.').to_ascii_lowercase()) + .collect() +} + +/// Whether `new` accepts everything `old` did and more. +/// +/// An empty list means "no filter, everything allowed", so it is a superset +/// of every other list rather than the empty set — the one case plain set +/// arithmetic gets backwards. +fn filter_widened(old: &BTreeSet, new: &BTreeSet) -> bool { + match (old.is_empty(), new.is_empty()) { + (_, true) => !old.is_empty(), + (true, false) => false, + (false, false) => new.difference(old).next().is_some(), + } +} + +/// What an edit means for the running services and the stored index. +/// +/// The guiding rule: a wipe is only for data that cannot be read or compared +/// any more. Everything else is a difference between what the index holds and +/// what the configuration would produce, and a difference can be reconciled — +/// rows that fell out of scope are deleted ([`IndexWork::prune_scope`], +/// [`IndexWork::drop_roots`]), rows whose content scope moved are re-tested +/// ([`IndexWork::reconcile_content`]), and files that came *into* scope are +/// found by another walk ([`IndexWork::reindex`]). +/// +/// Roots, ignore patterns and content extensions are compared as **sets** +/// after normalization, so reordering a list or re-spelling a root is not a +/// change at all. pub fn diff_actions(old: &Config, new: &Config) -> ConfigActions { - let roots_changed = old.paths.indexing_paths != new.paths.indexing_paths; + let old_roots = old.normalized_indexing_paths(); + let new_roots = new.normalized_indexing_paths(); + // Encryption on↔off or a different salt (⇒ a different key) makes the - // on-disk file unreadable to the new configuration: rebuild. The GUI's - // security flows drive their own explicit rebuild dialog; this covers - // hand-edited configs applied through the generic path. `use_keychain` - // only changes where the key is remembered, not the file. + // on-disk file unreadable to the new configuration. The tokenizer is part + // of the FTS table's definition, and `hash_length` decides what bytes a + // stored hash covers, so old and new hashes cannot be compared. Those + // three are the whole of it; the GUI's security flows drive their own + // explicit dialog, and this covers hand-edited configs applied through + // the generic path. `use_keychain` only changes where the key is + // remembered, not the file. let requires_rebuild = old.processing.hash_length != new.processing.hash_length || old.processing.tokenize != new.processing.tokenize - || old.indexing.include_hidden != new.indexing.include_hidden - // Not merely a walk-behaviour knob: with links off, a symlink's target - // is not indexed at all, so turning it off leaves rows for files that - // are no longer in scope — including ones whose real parent lies - // outside every root, which no sweep will ever reach. - || old.indexing.follow_symlinks != new.indexing.follow_symlinks - || old.indexing.ignore_patterns != new.indexing.ignore_patterns - // Comments are not part of the filter, so annotating the list is not - // a reason to rebuild — only a change to what it actually matches is. - || !content_filter_entries(&old.indexing.content_extensions) - .eq(content_filter_entries(&new.indexing.content_extensions)) || old.security.password_protected != new.security.password_protected - || old.security.salt != new.security.salt - || roots_changed; + || old.security.salt != new.security.salt; + + let mut work = IndexWork::default(); + if !requires_rebuild { + work.drop_roots = old_roots.difference(&new_roots).cloned().collect(); + + let old_ignores: BTreeSet<&str> = old + .indexing + .ignore_patterns + .iter() + .map(|s| s.trim()) + .collect(); + let new_ignores: BTreeSet<&str> = new + .indexing + .ignore_patterns + .iter() + .map(|s| s.trim()) + .collect(); + + // Hidden files narrow the walk exactly the way an added ignore pattern + // does, so they take the same route. + work.prune_scope = new_ignores.difference(&old_ignores).next().is_some() + || (old.indexing.include_hidden && !new.indexing.include_hidden); + + // Symlinks do not: a followed target is stored under its own canonical + // path, which is either inside a root — where a direct walk produces + // exactly the same row, so nothing changes — or outside every root, + // where turning links off strands it somewhere no walk and no + // per-root scan will ever look again. + work.drop_aliases = old.indexing.follow_symlinks && !new.indexing.follow_symlinks; + + let old_content = content_filter_set(&old.indexing.content_extensions); + let new_content = content_filter_set(&new.indexing.content_extensions); + let content_widened = filter_widened(&old_content, &new_content); + work.reconcile_content = old_content != new_content; + + let old_store = old.processing.store_text_for_snippets; + let new_store = new.processing.store_text_for_snippets; + work.restore_text = !old_store && new_store; + work.drop_text = old_store && !new_store; + + // Widening only ever *adds* files, and a file that is not in the index + // is not findable from it: only a walk can produce those rows. Text + // that has to be extracted again needs a run for the same reason — + // the content pass runs as part of one. + work.reindex = new_roots.difference(&old_roots).next().is_some() + || old_ignores.difference(&new_ignores).next().is_some() + || (!old.indexing.include_hidden && new.indexing.include_hidden) + || (!old.indexing.follow_symlinks && new.indexing.follow_symlinks) + || content_widened + || work.restore_text; + } + ConfigActions { requires_rebuild, - restart_watcher: requires_rebuild || roots_changed, + work, search_db_changed: old.paths.database_path != new.paths.database_path, } } @@ -947,24 +1135,56 @@ mod tests { assert!(content_allowed(Path::new("/a/Makefile"), &all_comments)); } + /// Comments, spelling and order are not part of the content filter, so + /// editing them is no work at all — and a real change to it is never a + /// rebuild, only a re-decision of the text already stored. #[test] - fn comment_only_edit_does_not_force_rebuild() { + fn comment_only_edit_is_no_work_at_all() { let mut old = Config::default(); old.indexing.content_extensions = vec!["txt".into(), "md".into()]; - let mut new = old.clone(); - new.indexing.content_extensions = - vec!["# my notes".into(), "txt".into(), "md # markdown".into()]; - assert!(!diff_actions(&old, &new).requires_rebuild); - // Changing what the list matches still does. - let mut changed = old.clone(); - changed.indexing.content_extensions = vec!["txt".into(), "md".into(), "(none)".into()]; - assert!(diff_actions(&old, &changed).requires_rebuild); + for cosmetic in [ + vec!["# my notes".into(), "txt".into(), "md # markdown".into()], + vec!["md".into(), "txt".into()], + vec![".TXT".into(), ".Md".into()], + ] { + let mut new = old.clone(); + new.indexing.content_extensions = cosmetic; + let a = diff_actions(&old, &new); + assert_eq!(a, ConfigActions::default(), "cosmetic edit is not a change"); + } - // ... including commenting an entry out. - let mut disabled = old.clone(); - disabled.indexing.content_extensions = vec!["txt".into(), "# md".into()]; - assert!(diff_actions(&old, &disabled).requires_rebuild); + // Adding an extension widens the filter: files already indexed by + // name need their text extracted, which takes a run. + let mut widened = old.clone(); + widened.indexing.content_extensions = vec!["txt".into(), "md".into(), "(none)".into()]; + let a = diff_actions(&old, &widened); + assert!(!a.requires_rebuild); + assert!(a.work.reconcile_content && a.work.reindex); + + // Commenting one out narrows it: the stored text goes, and nothing + // needs walking to make that true. + let mut narrowed = old.clone(); + narrowed.indexing.content_extensions = vec!["txt".into(), "# md".into()]; + let a = diff_actions(&old, &narrowed); + assert!(!a.requires_rebuild); + assert!(a.work.reconcile_content && !a.work.reindex); + } + + /// An empty list means "everything allowed", so it is a superset of every + /// other list — the case plain set arithmetic reads backwards. + #[test] + fn an_empty_content_filter_is_the_widest_one() { + let mut listed = Config::default(); + listed.indexing.content_extensions = vec!["txt".into()]; + let mut unfiltered = listed.clone(); + unfiltered.indexing.content_extensions = vec![]; + + let widening = diff_actions(&listed, &unfiltered).work; + assert!(widening.reconcile_content && widening.reindex); + + let narrowing = diff_actions(&unfiltered, &listed).work; + assert!(narrowing.reconcile_content && !narrowing.reindex); } #[test] @@ -1169,40 +1389,148 @@ mod tests { fs::remove_dir_all(&dir).ok(); } + /// Only three settings may wipe the index: the FTS tokenizer, the hash + /// length and the encryption key. Anything else that reaches + /// `requires_rebuild` is a bug — it costs the user everything the index + /// took hours to learn. #[test] - fn diff_actions_matrix() { + fn only_unreadable_data_forces_a_rebuild() { let base = Config::default(); - let same = diff_actions(&base, &base.clone()); - assert_eq!( - same, - ConfigActions { - requires_rebuild: false, - restart_watcher: false, - search_db_changed: false - } + let mut tokenizer = base.clone(); + tokenizer.processing.tokenize = "unicode61".into(); + let mut hash = base.clone(); + hash.processing.hash_length = base.processing.hash_length + 1; + let mut protect = base.clone(); + protect.security.password_protected = true; + let mut salt = base.clone(); + salt.security.salt = Some("00".repeat(16)); + + for c in [&tokenizer, &hash, &protect, &salt] { + let a = diff_actions(&base, c); + assert!(a.requires_rebuild, "must wipe"); + assert!( + a.work.is_empty(), + "a wipe subsumes reconciliation; leaving work behind would run it \ + against a file that is about to be deleted" + ); + } + + // The keychain only decides where the key is remembered, not what the + // file was written with. + let mut keychain = base.clone(); + keychain.security.use_keychain = true; + assert_eq!(diff_actions(&base, &keychain), ConfigActions::default()); + } + + /// Narrowing deletes; widening walks. Nothing here may wipe. + #[test] + fn diff_actions_matrix() { + let dir = tmp_dir(); + let kept = dir.join("kept"); + let dropped = dir.join("dropped"); + fs::create_dir_all(&kept).unwrap(); + fs::create_dir_all(&dropped).unwrap(); + let (kept, dropped) = ( + kept.to_string_lossy().into_owned(), + dropped.to_string_lossy().into_owned(), ); - let mut c = base.clone(); - c.processing.tokenize = "unicode61".into(); - assert!(diff_actions(&base, &c).requires_rebuild); + let mut base = Config::default(); + base.paths.indexing_paths = vec![kept.clone(), dropped.clone()]; + base.indexing.ignore_patterns = vec!["node_modules".into()]; + base.indexing.include_hidden = true; + base.indexing.follow_symlinks = true; - let mut c = base.clone(); - c.indexing.ignore_patterns.push("*.log".into()); - let a = diff_actions(&base, &c); - assert!(a.requires_rebuild && a.restart_watcher); + assert_eq!(diff_actions(&base, &base.clone()), ConfigActions::default()); - // `follow_symlinks` decides what is in the index, not just how the walk - // moves, so it rebuilds as well as restarting the watcher. + // Removing a root: its rows are deleted by range, and no walk is + // needed to establish that they should go. let mut c = base.clone(); - c.indexing.follow_symlinks = true; + c.paths.indexing_paths = vec![kept.clone()]; let a = diff_actions(&base, &c); - assert!(a.requires_rebuild && a.restart_watcher); + assert!(!a.requires_rebuild); + assert_eq!(a.work.drop_roots, vec![dropped.clone()]); + assert!(!a.work.reindex && !a.work.prune_scope); + + // Adding one: nothing stored is wrong, there is just more to find. + let a = diff_actions(&c, &base); + assert!(!a.requires_rebuild); + assert!(a.work.drop_roots.is_empty() && a.work.reindex && !a.work.prune_scope); + + for (narrow, widen, what) in [ + ( + { + let mut c = base.clone(); + c.indexing.ignore_patterns.push("*.log".into()); + c + }, + { + let mut c = base.clone(); + c.indexing.ignore_patterns.clear(); + c + }, + "ignore patterns", + ), + ( + { + let mut c = base.clone(); + c.indexing.include_hidden = false; + c + }, + base.clone(), + "hidden files", + ), + ] { + let a = diff_actions(&base, &narrow); + assert!(!a.requires_rebuild, "{} must not wipe", what); + assert!( + a.work.prune_scope && !a.work.reindex, + "narrowing {} prunes and needs no walk", + what + ); + let a = diff_actions(&narrow, &widen); + assert!(!a.requires_rebuild, "{} must not wipe", what); + assert!( + a.work.reindex && !a.work.prune_scope, + "widening {} walks and deletes nothing", + what + ); + } + + // Symlinks take their own route: with links on, a target inside a root + // is stored under exactly the path a direct walk would produce, so + // nothing in scope changes. What turning them off strands is the rows + // *outside* every root, which no per-root scan would ever revisit. + let mut no_links = base.clone(); + no_links.indexing.follow_symlinks = false; + let a = diff_actions(&base, &no_links); + assert!(!a.requires_rebuild); + assert!( + a.work.drop_aliases && !a.work.prune_scope && !a.work.reindex, + "turning links off sweeps outside the roots and nothing else" + ); + let a = diff_actions(&no_links, &base); + assert!( + a.work.reindex && !a.work.drop_aliases && !a.work.prune_scope, + "turning links on only adds" + ); + + // Stored text: turning it on means re-extracting, turning it off means + // throwing the blobs away — never a rebuild either way. + let mut off = base.clone(); + off.processing.store_text_for_snippets = false; + let mut on = base.clone(); + on.processing.store_text_for_snippets = true; + let a = diff_actions(&on, &off); + assert!(a.work.drop_text && !a.work.restore_text && !a.work.reindex); + let a = diff_actions(&off, &on); + assert!(a.work.restore_text && !a.work.drop_text && a.work.reindex); let mut c = base.clone(); c.paths.database_path = "/elsewhere.sqlite".into(); let a = diff_actions(&base, &c); - assert!(a.search_db_changed && !a.requires_rebuild); + assert!(a.search_db_changed && !a.requires_rebuild && a.work.is_empty()); let mut c = base.clone(); c.search.display_limit = 5000; @@ -1210,28 +1538,92 @@ mod tests { c.processing.maximum_wal_size = 0; c.indexing.auto_index = false; c.indexing.reindex_interval_minutes = 5; - let a = diff_actions(&base, &c); assert_eq!( - a, - ConfigActions { - requires_rebuild: false, - restart_watcher: false, - search_db_changed: false - }, - "soft knobs never force restarts" + diff_actions(&base, &c), + ConfigActions::default(), + "soft knobs are not index work" ); - // Security: protection on↔off and salt changes rebuild; the - // keychain preference is a soft knob. - let mut c = base.clone(); - c.security.password_protected = true; - assert!(diff_actions(&base, &c).requires_rebuild); - let mut c = base.clone(); - c.security.salt = Some("00".repeat(16)); - assert!(diff_actions(&base, &c).requires_rebuild); - let mut c = base.clone(); - c.security.use_keychain = true; - assert!(!diff_actions(&base, &c).requires_rebuild); + fs::remove_dir_all(&dir).ok(); + } + + /// A second edit landing while the first is still being applied must not + /// lose the first's work: the two plans are computed against different + /// configurations, so neither knows what the other left undone. + #[test] + fn merging_two_plans_loses_nothing() { + let first = IndexWork { + drop_roots: vec!["/gone".into(), "/shared".into()], + prune_scope: true, + drop_text: true, + ..IndexWork::default() + }; + let second = IndexWork { + drop_roots: vec!["/shared".into(), "/also-gone".into()], + reconcile_content: true, + reindex: true, + ..IndexWork::default() + }; + + let mut merged = second.clone(); + merged.merge_from(&first); + assert_eq!( + merged.drop_roots, + vec!["/shared", "/also-gone", "/gone"], + "every root from both, each once" + ); + assert!(merged.prune_scope && merged.drop_text); + assert!(merged.reconcile_content && merged.reindex); + + // Merging an empty plan changes nothing, and merging a plan into + // itself is the identity — both are what make a restart safe. + let mut untouched = first.clone(); + untouched.merge_from(&IndexWork::default()); + assert_eq!(untouched, first); + untouched.merge_from(&first); + assert_eq!(untouched, first); + } + + /// Order and spelling are not configuration. Reordering the folder list, + /// reordering the ignore patterns, or writing a root a different way used + /// to wipe a multi-million-file index for nothing. + #[test] + fn respelling_a_list_is_not_a_change() { + let dir = tmp_dir(); + let a_dir = dir.join("alpha"); + let b_dir = dir.join("beta"); + fs::create_dir_all(&a_dir).unwrap(); + fs::create_dir_all(&b_dir).unwrap(); + + let mut base = Config::default(); + base.paths.indexing_paths = vec![ + a_dir.to_string_lossy().into_owned(), + b_dir.to_string_lossy().into_owned(), + ]; + base.indexing.ignore_patterns = vec!["node_modules".into(), "*.tmp".into()]; + + let mut reordered = base.clone(); + reordered.paths.indexing_paths.reverse(); + reordered.indexing.ignore_patterns.reverse(); + assert_eq!(diff_actions(&base, &reordered), ConfigActions::default()); + + // A trailing separator, a `.` hop and a duplicate entry all name the + // same two roots. + let mut respelled = base.clone(); + respelled.paths.indexing_paths = vec![ + format!("{}{}", a_dir.to_string_lossy(), std::path::MAIN_SEPARATOR), + b_dir.join(".").to_string_lossy().into_owned(), + a_dir.to_string_lossy().into_owned(), + ]; + assert_eq!(diff_actions(&base, &respelled), ConfigActions::default()); + + // Whitespace around an ignore pattern is trimmed before it compiles, + // so it cannot be a change either. + let mut padded = base.clone(); + padded.indexing.ignore_patterns = vec![" node_modules ".into(), "*.tmp".into()]; + assert_eq!(diff_actions(&base, &padded), ConfigActions::default()); + + fs::remove_dir_all(&dir).ok(); } #[test] @@ -1306,15 +1698,7 @@ mod tests { let base = Config::default(); let mut c = base.clone(); c.ui.watch_cap_warned_roots = vec!["/media/ApolloStore".to_string()]; - let a = diff_actions(&base, &c); - assert_eq!( - a, - ConfigActions { - requires_rebuild: false, - restart_watcher: false, - search_db_changed: false - } - ); + assert_eq!(diff_actions(&base, &c), ConfigActions::default()); } #[test] diff --git a/crates/quicksearch-core/src/coordinator.rs b/crates/quicksearch-core/src/coordinator.rs index decfef8..1b17dbf 100644 --- a/crates/quicksearch-core/src/coordinator.rs +++ b/crates/quicksearch-core/src/coordinator.rs @@ -31,11 +31,12 @@ use std::time::{Duration, Instant}; use rusqlite::Connection; -use crate::config::{Config, IgnoreSet}; +use crate::config::{diff_actions, Config, IgnoreSet, IndexWork}; use crate::db; use crate::extract::Registry; use crate::incremental::apply_fs_event; use crate::indexing::{ConfigChange, IndexingService, IndexingStatus}; +use crate::scope::WorkCursor; use crate::watcher::{FsEvent, WatchError, WatchFilters, Watcher, WatcherConfig}; /// Pending-event ceiling; beyond this a full run is cheaper than replay. @@ -145,6 +146,7 @@ impl IndexCoordinator { last_event_at: None, pending_since: None, needs_full_run: false, + pending_work: None, saw_running: false, write_conn: None, ignore: Arc::new(IgnoreSet::compile(&[]).expect("empty ignore set")), @@ -214,7 +216,7 @@ impl IndexCoordinator { config: &Config, ) -> Result>, String> { let db = config.resolved_database_path(); - let roots = joined_roots(config); + let roots: Vec = config.normalized_indexing_paths().into_iter().collect(); self.indexing .check_config_validation(&db.to_string_lossy(), config, &roots) } @@ -238,17 +240,6 @@ impl Drop for IndexCoordinator { } } -/// Newline-joined resolved roots — the shape `start_indexing` / -/// `config_validation` store. -fn joined_roots(config: &Config) -> String { - config - .resolved_indexing_paths() - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect::>() - .join("\n") -} - /// Fold `event` into the last-event-wins pending map. Renames split into /// their halves so downstream application never needs pair handling. fn enqueue(pending: &mut HashMap, event: FsEvent) { @@ -327,6 +318,13 @@ struct Inner { /// defer application past `pending_max_defer`. pending_since: Option, needs_full_run: bool, + /// Reconciliation owed to a config change, part-applied across ticks. + /// + /// Deliberately not folded into `needs_full_run`: that flag also carries + /// watcher overflow and incremental failure, which must stay dormant in + /// manual mode, whereas a config change the user just made is acted on in + /// either mode. + pending_work: Option, /// A start was requested; set false once the service reports running, /// so idle-after-running transitions are detectable. saw_running: bool, @@ -343,7 +341,17 @@ impl Inner { self.enter_auto(); } loop { - match cmd_rx.recv_timeout(Duration::from_secs(1)) { + // Reconciliation is applied one slice per tick, so while any is + // owed the idle wait has to shrink or the slices are a second + // apart and a large index's prune stretches over minutes of wall + // clock. Coming straight back keeps the duty cycle high while + // still servicing every queued command between slices. + let idle = if self.pending_work.is_some() { + Duration::from_millis(1) + } else { + Duration::from_secs(1) + }; + match cmd_rx.recv_timeout(idle) { Ok(CoordCmd::Shutdown) => break, Ok(cmd) => self.handle_cmd(cmd), Err(mpsc::RecvTimeoutError::Timeout) => self.tick(), @@ -368,12 +376,22 @@ impl Inner { } CoordCmd::ConfigChanged(new) => { let want_auto = new.indexing.auto_index; + let actions = diff_actions(&self.config, &new); self.config = new; if let Err(e) = self.reload_filters() { crate::log_warn!("coordinator: {}", e); } // The write connection may point at an old database_path. self.write_conn = None; + // A wipe stays the caller's decision — it is destructive and + // the GUI may have to ask first (see `rebuild_index`). + // Everything short of one this thread reconciles itself, in + // both modes and without asking: deleting rows the user just + // put out of scope is not a change to confirm, it is the + // change they made. + if !actions.requires_rebuild && !actions.work.is_empty() { + self.start_work(actions.work); + } if want_auto && self.mode != IndexMode::Auto { // The mode lives in `auto_index`, so a config that // disagrees with the running mode *is* a mode change. @@ -389,6 +407,8 @@ impl Inner { CoordCmd::RebuildIndex => { let db = self.db_path(); self.write_conn = None; + // Nothing to reconcile against once the file is gone. + self.pending_work = None; if let Err(e) = self.indexing.delete_index_for_rebuild(&db) { crate::log_warn!("coordinator: rebuild: {}", e); } @@ -402,6 +422,7 @@ impl Inner { // missing index and rebuild what was just deleted. self.enter_manual_stopped(); self.write_conn = None; + self.pending_work = None; let db = self.db_path(); if let Err(e) = self.indexing.delete_index_for_rebuild(&db) { crate::log_warn!("coordinator: clear index: {}", e); @@ -441,6 +462,14 @@ impl Inner { } } + // Ahead of the mode gate: a config edit is reconciled in manual mode + // too. It may end by starting a run, which is why this cannot wait for + // the Auto-only scheduling below. + if self.pending_work.is_some() { + self.apply_work(); + return; + } + if self.mode != IndexMode::Auto { if self.mode == IndexMode::ManualStopped { self.clear_pending(); @@ -481,6 +510,84 @@ impl Inner { } } + /// Queue reconciliation for a config change. + /// + /// A plan still in flight is folded in and restarted rather than dropped: + /// it was computed against the previous configuration, so the new one — + /// which diffs against that same previous config — cannot know what it + /// had left undone. Restarting re-does the finished half, which every + /// part of the pass is idempotent precisely so that it can. + fn start_work(&mut self, mut work: IndexWork) { + if let Some(outstanding) = self.pending_work.take() { + work.merge_from(outstanding.work()); + } + match WorkCursor::new(work, &self.config) { + Ok(cursor) => self.pending_work = Some(cursor), + // Only an uncompilable ignore pattern gets here, and the GUI + // validates those before saving. Refusing to reconcile is the + // safe half: nothing is deleted on a filter nobody could build. + Err(e) => crate::log_warn!("coordinator: cannot reconcile config change: {}", e), + } + } + + /// Advance the queued reconciliation by one slice, and start the full run + /// it asked for once it is finished. + fn apply_work(&mut self) { + let mut conn = match self.ensure_write_conn() { + Ok(conn) => conn, + Err(e) => { + // No index to reconcile: a run builds it under the new config + // anyway, which reaches the same place by a longer road. + crate::log_warn!("coordinator: reconcile unavailable ({}); scheduling run", e); + self.pending_work = None; + self.needs_full_run = true; + return; + } + }; + let mut cursor = self.pending_work.take().expect("caller checked"); + let outcome = crate::scope::advance( + &mut conn, + &self.config, + &self.registry, + &mut cursor, + Instant::now() + crate::scope::SLICE, + ); + self.write_conn = Some(conn); + if let Err(e) = outcome { + // Abandoned rather than retried: the cursor is already dropped, + // and a database error that persists would otherwise spin this + // loop for the life of the process. The next full run reconciles + // from the stored fingerprint, which is the backstop for exactly + // this. + crate::log_warn!( + "coordinator: reconcile: {}; leaving it to the next indexing run", + e + ); + return; + } + if !cursor.done() { + self.pending_work = Some(cursor); + return; + } + if cursor.deleted > 0 || cursor.recontented > 0 { + crate::log_info!( + "configuration change: {} index entries removed, {} re-examined \ + for text extraction", + cursor.deleted, + cursor.recontented + ); + } + // Widening the configuration adds files that exist only on disk, so + // only a walk can produce their rows. `ReindexNow`'s exact behaviour, + // including the manual-mode round trip back to stopped. + if cursor.reindex() { + self.start_full_run(); + if self.mode != IndexMode::Auto { + self.mode = IndexMode::ManualRunning; + } + } + } + /// Drop the queue and the timers that describe it, so a stale /// `pending_since` cannot force an immediate apply of the next event. fn clear_pending(&mut self) { @@ -666,6 +773,12 @@ impl Inner { self.config.indexing.auto_index = false; self.stop_watcher(); self.clear_pending(); + // Stopping means "no runs now", so a config change that also widened + // the scope loses its walk — but keeps its pruning. Deleting rows the + // user put out of scope is the edit they made, not indexing work. + if let Some(cursor) = self.pending_work.as_mut() { + cursor.cancel_reindex(); + } let status = self.indexing.get_status(); if !matches!(status, IndexingStatus::Idle | IndexingStatus::Error(_)) { // Signal only — waiting up to 5 s here would stall every @@ -929,6 +1042,122 @@ mod tests { coord.shutdown(); } + /// A narrowed filter is applied to the stored index without a prompt and + /// without a run — including in manual mode, where the user has said not + /// to index anything. Deleting entries they just excluded is not indexing + /// work; it is the edit they made. + #[test] + fn manual_mode_prunes_a_narrowed_filter_without_running() { + let f = Fixture::new(false); + std::fs::write(f.dir.join("keep.txt"), "kept content").unwrap(); + std::fs::write(f.dir.join("drop.log"), "dropped content").unwrap(); + + let coord = IndexCoordinator::start(f.config.clone()).unwrap(); + coord.reindex_now(); + wait_for("initial run", Duration::from_secs(20), || { + let s = coord.state(); + s.last_full_index.is_some() && s.mode == IndexMode::ManualStopped && f.file_count() == 2 + }); + let stamped = coord.state().last_full_index; + + // Appended, not replaced: dropping the default patterns at the same + // time would be a widening too, and this is about narrowing alone. + let mut narrowed = f.config.clone(); + narrowed.indexing.ignore_patterns.push("*.log".into()); + coord.apply_config(narrowed); + + wait_for( + "the log entry to be pruned", + Duration::from_secs(20), + || f.file_count() == 1, + ); + std::thread::sleep(Duration::from_millis(500)); + assert_eq!( + coord.state().mode, + IndexMode::ManualStopped, + "still stopped" + ); + assert_eq!( + coord.state().last_full_index, + stamped, + "no run happened — narrowing needs no walk" + ); + coord.shutdown(); + } + + /// Widening it does the opposite: nothing is deleted, and the walk that + /// finds the newly-eligible files starts on its own, returning manual mode + /// to stopped afterwards the way `reindex_now` does. + #[test] + fn manual_mode_reindexes_a_widened_filter_and_returns_to_stopped() { + let f = Fixture::new(false); + std::fs::write(f.dir.join("keep.txt"), "kept content").unwrap(); + std::fs::write(f.dir.join("later.log"), "arrives later").unwrap(); + + let mut narrowed = f.config.clone(); + narrowed.indexing.ignore_patterns.push("*.log".into()); + let coord = IndexCoordinator::start(narrowed.clone()).unwrap(); + coord.reindex_now(); + wait_for("initial run", Duration::from_secs(20), || { + let s = coord.state(); + s.last_full_index.is_some() && s.mode == IndexMode::ManualStopped && f.file_count() == 1 + }); + + coord.apply_config(f.config.clone()); + wait_for( + "the widened walk to find it", + Duration::from_secs(20), + || f.file_count() == 2 && coord.state().mode == IndexMode::ManualStopped, + ); + coord.shutdown(); + } + + /// Stopping is the user saying "no runs now", so a widening edit already + /// in flight loses its walk — but keeps the pruning half, which is not + /// indexing work. + #[test] + fn stopping_cancels_a_queued_walk_but_not_a_queued_prune() { + let f = Fixture::new(false); + std::fs::write(f.dir.join("keep.txt"), "kept content").unwrap(); + std::fs::write(f.dir.join("drop.log"), "dropped content").unwrap(); + + let coord = IndexCoordinator::start(f.config.clone()).unwrap(); + coord.reindex_now(); + wait_for("initial run", Duration::from_secs(20), || { + let s = coord.state(); + s.last_full_index.is_some() && s.mode == IndexMode::ManualStopped && f.file_count() == 2 + }); + let stamped = coord.state().last_full_index; + + // Narrow and widen at once: one new pattern to prune by, one root + // added to walk for. Turning auto off in the same edit is what makes + // the coordinator enter manual-stopped with the plan already queued. + let extra = f.dir.join("extra"); + std::fs::create_dir_all(&extra).unwrap(); + std::fs::write(extra.join("new.txt"), "in the new root").unwrap(); + let mut edited = f.config.clone(); + edited.indexing.ignore_patterns.push("*.log".into()); + edited + .paths + .indexing_paths + .push(extra.to_string_lossy().into_owned()); + edited.indexing.auto_index = false; + coord.set_mode(IndexMode::ManualStopped); + coord.apply_config(edited); + + wait_for("the prune to land", Duration::from_secs(20), || { + f.file_count() == 1 + }); + std::thread::sleep(Duration::from_millis(500)); + assert_eq!( + coord.state().last_full_index, + stamped, + "the walk the widening asked for was cancelled by the stop" + ); + assert_eq!(f.file_count(), 1, "and the new root is still unindexed"); + coord.shutdown(); + } + /// Short debounce windows so trailing-edge events flush within test /// timeouts (production defaults are 30 s / 2 s). fn fast_watcher() -> WatcherConfig { diff --git a/crates/quicksearch-core/src/db/repo.rs b/crates/quicksearch-core/src/db/repo.rs index 5d0a6c0..d81e96c 100644 --- a/crates/quicksearch-core/src/db/repo.rs +++ b/crates/quicksearch-core/src/db/repo.rs @@ -14,7 +14,7 @@ //! | 2 | failed | //! | 3 | not applicable (content only) | -use rusqlite::{params, Connection, OptionalExtension, Transaction}; +use rusqlite::{params, params_from_iter, Connection, OptionalExtension, Transaction}; use crate::mime::FileType; @@ -270,12 +270,7 @@ pub fn delete_file_by_path(tx: &Transaction<'_>, path: &str) -> Result, lo: &str, hi: &str) -> Result { // Every dependent table is keyed by the file id, so they share one // sub-select; `files` itself goes last, once nothing references it. - for (table, key) in [ - ("searchabletext", "rowid"), - ("documents_text", "file_id"), - ("properties", "file_id"), - ("failed_files", "file_id"), - ] { + for (table, key) in DEPENDENT_TABLES { let sql = format!( "DELETE FROM {} WHERE {} IN \ (SELECT id FROM files WHERE path >= ?1 AND path < ?2)", @@ -292,6 +287,113 @@ pub fn delete_subtree(tx: &Transaction<'_>, lo: &str, hi: &str) -> Result, + ranges: &[(String, String)], +) -> Result { + if ranges.is_empty() { + return Ok(0); + } + let mut predicate = String::new(); + for i in 0..ranges.len() { + if i > 0 { + predicate.push_str(" AND "); + } + predicate.push_str(&format!( + "NOT (path >= ?{} AND path < ?{})", + i * 2 + 1, + i * 2 + 2 + )); + } + let bounds: Vec<&String> = ranges.iter().flat_map(|(lo, hi)| [lo, hi]).collect(); + for (table, key) in DEPENDENT_TABLES { + let sql = format!( + "DELETE FROM {} WHERE {} IN (SELECT id FROM files WHERE {})", + table, key, predicate + ); + tx.prepare_cached(&sql) + .and_then(|mut stmt| stmt.execute(params_from_iter(bounds.iter()))) + .map_err(|e| format!("delete {} outside the roots: {}", table, e))?; + } + let sql = format!("DELETE FROM files WHERE {}", predicate); + tx.prepare_cached(&sql) + .and_then(|mut stmt| stmt.execute(params_from_iter(bounds.iter()))) + .map_err(|e| format!("delete files outside the roots: {}", e)) +} + +/// The tables a file id owns, in the order they must be cleared: everything +/// keyed to `files.id` first, then `files` itself once nothing references it. +/// +/// `searchabletext` is an FTS5 virtual table with no foreign key at all, so +/// none of this can be left to `ON DELETE CASCADE` without splitting one rule +/// across two mechanisms — half declarative, half manual, free to drift the +/// moment someone opens a connection without `PRAGMA foreign_keys`. Shared by +/// [`delete_subtree`] and [`delete_ids`] so the two ways of removing a file +/// cannot disagree about what a file owns. +const DEPENDENT_TABLES: [(&str, &str); 4] = [ + ("searchabletext", "rowid"), + ("documents_text", "file_id"), + ("properties", "file_id"), + ("failed_files", "file_id"), +]; + +/// How many ids [`delete_ids`] binds into one statement. +/// +/// Fixed so `prepare_cached` sees a bounded set of distinct SQL texts: every +/// full chunk of a batch shares one statement and only the short final chunk +/// varies, where a per-call length would mint a new prepared statement each +/// time. +const DELETE_IDS_CHUNK: usize = 512; + +/// Delete the given file ids and everything keyed to them, keeping FTS, +/// `documents_text`, `properties` and `failed_files` in step. Returns how many +/// `files` rows went. +/// +/// The id counterpart to [`delete_subtree`], for rows chosen by a predicate no +/// SQL range can express — a glob ignore pattern, say. Five statements per +/// [`DELETE_IDS_CHUNK`] ids rather than five per file, which is the difference +/// that lets a newly-added ignore pattern prune a large index instead of +/// forcing a rebuild. +pub fn delete_ids(tx: &Transaction<'_>, ids: &[i64]) -> Result { + let mut removed = 0; + for chunk in ids.chunks(DELETE_IDS_CHUNK) { + let mut placeholders = String::with_capacity(chunk.len() * 2); + for i in 0..chunk.len() { + if i > 0 { + placeholders.push(','); + } + placeholders.push('?'); + } + for (table, key) in DEPENDENT_TABLES { + let sql = format!("DELETE FROM {} WHERE {} IN ({})", table, key, placeholders); + tx.prepare_cached(&sql) + .and_then(|mut stmt| stmt.execute(params_from_iter(chunk.iter()))) + .map_err(|e| format!("delete {} for {} ids: {}", table, chunk.len(), e))?; + } + let sql = format!("DELETE FROM files WHERE id IN ({})", placeholders); + removed += tx + .prepare_cached(&sql) + .and_then(|mut stmt| stmt.execute(params_from_iter(chunk.iter()))) + .map_err(|e| format!("delete {} file rows: {}", chunk.len(), e))?; + } + Ok(removed) +} + /// Every indexed file directly inside `parent`, as `name -> mtime`. /// /// The walk's unit of classification. Keyed by name rather than full path @@ -359,6 +461,99 @@ pub fn pending_content_page( .map_err(|e| format!("read pending content row: {}", e)) } +/// A stored row as the scope reconciler sees it: enough to decide both +/// whether the path is still in scope and whether its content still is. +#[derive(Debug, Clone)] +pub struct ScopeRow { + pub id: i64, + pub path: String, + pub size: u64, + pub mime: Option, + pub content_state: i64, +} + +/// One page of rows whose path is `> after` and `< hi`, in path order. +/// +/// Keyset on `path` rather than on `id`: the range is already a seek on +/// `UNIQUE(files.path)`, so paging by the same column keeps every page an +/// index walk with no sort step, and — because the cursor only moves forward +/// — a row is served at most once even though the caller is deleting behind +/// the reader. Seed `after` with the range's `lo` bound, which is +/// `root + separator` and so can never equal a stored path. +pub fn rows_in_range_page( + conn: &Connection, + after: &str, + hi: &str, + limit: i64, +) -> Result, String> { + let mut stmt = conn + .prepare_cached( + "SELECT id, path, size, mime, content_state FROM files + WHERE path > ?1 AND path < ?2 + ORDER BY path + LIMIT ?3", + ) + .map_err(|e| format!("prepare range page: {}", e))?; + let rows = stmt + .query_map(params![after, hi, limit], |row| { + Ok(ScopeRow { + id: row.get(0)?, + path: row.get(1)?, + size: row.get::<_, i64>(2)?.max(0) as u64, + mime: row.get(3)?, + content_state: row.get(4)?, + }) + }) + .map_err(|e| format!("query range page after {}: {}", after, e))?; + rows.collect::, _>>() + .map_err(|e| format!("read range page row: {}", e)) +} + +/// Drop the stored text of the given file ids, leaving their FTS row and +/// `files` row intact. +/// +/// Turning `store_text_for_snippets` off means exactly this: full-text search +/// keeps working from the FTS index, and only the snippet/occurrence source +/// goes away. Re-extracting to achieve it would re-read every file for nothing. +pub fn drop_stored_text(tx: &Transaction<'_>, ids: &[i64]) -> Result { + let mut removed = 0; + for chunk in ids.chunks(DELETE_IDS_CHUNK) { + let mut placeholders = String::with_capacity(chunk.len() * 2); + for i in 0..chunk.len() { + if i > 0 { + placeholders.push(','); + } + placeholders.push('?'); + } + let sql = format!( + "DELETE FROM documents_text WHERE file_id IN ({})", + placeholders + ); + removed += tx + .prepare_cached(&sql) + .and_then(|mut stmt| stmt.execute(params_from_iter(chunk.iter()))) + .map_err(|e| format!("drop stored text for {} ids: {}", chunk.len(), e))?; + } + Ok(removed) +} + +/// Put a file's content back in the pending queue without touching its row's +/// metadata, clearing whatever the last extraction left behind. +/// +/// For a config change that widens what gets extracted: the file itself has +/// not changed, so `update_file_basic` would be wrong (it rewrites size, mtime +/// and hash from a fresh stat), but its content must be produced again. +pub fn reset_content_pending(tx: &Transaction<'_>, file_id: i64) -> Result<(), String> { + remove_content_for_id(tx, file_id)?; + tx.prepare_cached("UPDATE files SET content_state = ?1, failure_msg = NULL WHERE id = ?2") + .and_then(|mut stmt| stmt.execute(params![STATE_PENDING, file_id])) + .map_err(|e| format!("reset content_state pending {}: {}", file_id, e))?; + tx.prepare_cached("DELETE FROM failed_files WHERE file_id = ?1") + .and_then(|mut stmt| stmt.execute(params![file_id])) + .map_err(|e| format!("clear failed_files {}: {}", file_id, e))?; + Ok(()) +} + /// The stored mtime for one exact path, or `None` if it isn't indexed. /// /// For files the walk reaches by a spelling whose parent isn't the directory @@ -921,6 +1116,329 @@ mod tests { std::fs::remove_file(&p).ok(); } + /// Seed a database with `paths` as fully-indexed rows, each carrying an + /// FTS entry, stored text and a property. Returns `path -> id`. + fn seeded(conn: &mut Connection, paths: &[&str]) -> std::collections::HashMap { + let tx = conn.transaction().unwrap(); + let mut ids = std::collections::HashMap::new(); + for path in paths { + let name = path.rsplit('/').next().unwrap(); + let parent = &path[..path.rfind('/').unwrap()]; + let id = insert_file( + &tx, + &NewFile { + name, + path, + parent, + size: 1, + mtime: 1, + inode: None, + device_id: None, + mime: Some("text/plain"), + ftype: FileType::TEXT, + hash: None, + needs_content: true, + }, + ) + .unwrap() + .expect("unique path"); + set_content_done( + &tx, + id, + name, + "body text", + &[("k".into(), "v".into())], + true, + ) + .unwrap(); + ids.insert((*path).to_string(), id); + } + tx.commit().unwrap(); + ids + } + + /// The out-of-root sweep must keep every configured root's rows and take + /// everything else — including a path that merely *starts* with a root's + /// name, which is a different folder. + #[test] + fn delete_outside_ranges_keeps_exactly_the_configured_roots() { + let p = tmp_path(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + seeded( + &mut conn, + &[ + "/roots/one/a.txt", + "/roots/one/deep/b.txt", + "/roots/two/c.txt", + "/roots/onefold/d.txt", + "/elsewhere/target.txt", + ], + ); + + let ranges: Vec<(String, String)> = ["/roots/one", "/roots/two"] + .iter() + .map(|r| { + let range = crate::file_handling::ExtractCursor::for_root(r); + (range.lo, range.hi) + }) + .collect(); + let removed = { + let tx = conn.transaction().unwrap(); + let n = delete_outside_ranges(&tx, &ranges).unwrap(); + tx.commit().unwrap(); + n + }; + assert_eq!(removed, 2, "the name-prefix sibling and the outsider"); + + let survivors: Vec = { + let mut stmt = conn + .prepare("SELECT path FROM files ORDER BY path") + .unwrap(); + let v = stmt + .query_map([], |r| r.get::<_, String>(0)) + .unwrap() + .map(|r| r.unwrap()) + .collect(); + v + }; + assert_eq!( + survivors, + vec![ + "/roots/one/a.txt", + "/roots/one/deep/b.txt", + "/roots/two/c.txt" + ] + ); + assert_eq!( + conn.query_row("SELECT COUNT(*) FROM searchabletext", [], |r| r + .get::<_, i64>(0)) + .unwrap(), + 3 + ); + assert_eq!( + conn.query_row("SELECT COUNT(*) FROM documents_text", [], |r| r + .get::<_, i64>(0)) + .unwrap(), + 3 + ); + + // No roots configured is a half-written config, not an instruction to + // delete the entire index. + let tx = conn.transaction().unwrap(); + assert_eq!(delete_outside_ranges(&tx, &[]).unwrap(), 0); + tx.commit().unwrap(); + assert_eq!( + conn.query_row("SELECT COUNT(*) FROM files", [], |r| r.get::<_, i64>(0)) + .unwrap(), + 3 + ); + + drop(conn); + std::fs::remove_file(&p).ok(); + } + + /// `delete_ids` is what a newly-added ignore pattern prunes with, and a + /// file is only really gone when its name row, its FTS postings, its + /// stored text, its properties and any failure record all go. A survivor + /// in any one of them keeps the file findable, which is the whole thing + /// the user asked to stop. + #[test] + fn delete_ids_clears_every_dependent_table() { + let p = tmp_path(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + let ids = seeded( + &mut conn, + &["/t/a.txt", "/t/b.log", "/t/deep/c.log", "/t/keep.txt"], + ); + { + let tx = conn.transaction().unwrap(); + set_content_failed(&tx, ids["/t/deep/c.log"], "bad parse").unwrap(); + tx.commit().unwrap(); + } + + let doomed = vec![ids["/t/b.log"], ids["/t/deep/c.log"]]; + let removed = { + let tx = conn.transaction().unwrap(); + let n = delete_ids(&tx, &doomed).unwrap(); + tx.commit().unwrap(); + n + }; + assert_eq!(removed, 2); + + let count = |sql: &str| -> i64 { conn.query_row(sql, [], |r| r.get(0)).unwrap() }; + assert_eq!(count("SELECT COUNT(*) FROM files"), 2); + assert_eq!(count("SELECT COUNT(*) FROM searchabletext"), 2); + assert_eq!(count("SELECT COUNT(*) FROM documents_text"), 2); + assert_eq!(count("SELECT COUNT(*) FROM properties"), 2); + assert_eq!(count("SELECT COUNT(*) FROM failed_files"), 0); + + // The FTS index really lost them, not just the `files` row: a + // contentless table keeps serving deleted rowids without the tombstone. + let hits: i64 = conn + .query_row( + "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'body'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(hits, 2); + + // Empty input is a no-op, not a statement with an empty `IN ()`. + let tx = conn.transaction().unwrap(); + assert_eq!(delete_ids(&tx, &[]).unwrap(), 0); + tx.commit().unwrap(); + + drop(conn); + std::fs::remove_file(&p).ok(); + } + + /// More ids than `DELETE_IDS_CHUNK`, so the short final chunk and the + /// full ones both run — the boundary a fixed-size placeholder list makes + /// easy to get wrong. + #[test] + fn delete_ids_spans_chunk_boundaries() { + let p = tmp_path(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + let paths: Vec = (0..DELETE_IDS_CHUNK + 7) + .map(|i| format!("/t/f{:05}.txt", i)) + .collect(); + let refs: Vec<&str> = paths.iter().map(String::as_str).collect(); + let ids = seeded(&mut conn, &refs); + + let mut all: Vec = ids.values().copied().collect(); + all.sort_unstable(); + let keep = all.pop().unwrap(); + let removed = { + let tx = conn.transaction().unwrap(); + let n = delete_ids(&tx, &all).unwrap(); + tx.commit().unwrap(); + n + }; + assert_eq!(removed, DELETE_IDS_CHUNK + 6); + let left: i64 = conn + .query_row("SELECT id FROM files", [], |r| r.get(0)) + .unwrap(); + assert_eq!(left, keep); + + drop(conn); + std::fs::remove_file(&p).ok(); + } + + /// Dropping stored text must cost the file its snippets and nothing else: + /// the FTS postings are what full-text search runs on, and re-extracting + /// every file to turn a storage setting off would be absurd. + #[test] + fn drop_stored_text_keeps_the_file_searchable() { + let p = tmp_path(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + let ids = seeded(&mut conn, &["/t/a.txt", "/t/b.txt"]); + + { + let tx = conn.transaction().unwrap(); + drop_stored_text(&tx, &[ids["/t/a.txt"]]).unwrap(); + tx.commit().unwrap(); + } + + let count = |sql: &str| -> i64 { conn.query_row(sql, [], |r| r.get(0)).unwrap() }; + assert_eq!(count("SELECT COUNT(*) FROM documents_text"), 1); + assert_eq!(count("SELECT COUNT(*) FROM files"), 2); + assert_eq!(count("SELECT COUNT(*) FROM searchabletext"), 2); + assert_eq!( + count("SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'body'"), + 2, + "both files still match on content" + ); + assert_eq!(count("SELECT COUNT(*) FROM properties"), 2); + + drop(conn); + std::fs::remove_file(&p).ok(); + } + + /// Re-queuing content leaves the row's metadata alone — the file has not + /// changed, the configuration has — but clears what the last extraction + /// produced, so a second pass cannot double-insert into the FTS table. + #[test] + fn reset_content_pending_clears_the_last_extraction() { + let p = tmp_path(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + let ids = seeded(&mut conn, &["/t/a.txt", "/t/b.txt"]); + let id = ids["/t/a.txt"]; + { + let tx = conn.transaction().unwrap(); + set_content_failed(&tx, ids["/t/b.txt"], "bad parse").unwrap(); + tx.commit().unwrap(); + } + + { + let tx = conn.transaction().unwrap(); + reset_content_pending(&tx, id).unwrap(); + reset_content_pending(&tx, ids["/t/b.txt"]).unwrap(); + tx.commit().unwrap(); + } + + let row: (i64, i64, Option) = conn + .query_row( + "SELECT content_state, mtime, failure_msg FROM files WHERE id = ?1", + params![id], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .unwrap(); + assert_eq!(row.0, STATE_PENDING); + assert_eq!(row.1, 1, "metadata untouched — the file did not change"); + assert_eq!(row.2, None); + + let count = |sql: &str| -> i64 { conn.query_row(sql, [], |r| r.get(0)).unwrap() }; + assert_eq!(count("SELECT COUNT(*) FROM files"), 2, "rows stay"); + assert_eq!(count("SELECT COUNT(*) FROM searchabletext"), 0); + assert_eq!(count("SELECT COUNT(*) FROM documents_text"), 0); + assert_eq!(count("SELECT COUNT(*) FROM properties"), 0); + assert_eq!( + count("SELECT COUNT(*) FROM failed_files"), + 0, + "a stale failure must not outlive the retry it was queued for" + ); + + drop(conn); + std::fs::remove_file(&p).ok(); + } + + /// The reconciliation scan pages by path, so it must serve every row in + /// the range exactly once, in order, and stop at the range bound rather + /// than at a name prefix. + #[test] + fn rows_in_range_page_walks_the_range_once() { + let p = tmp_path(); + let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap(); + seeded( + &mut conn, + &[ + "/t/a.txt", + "/t/deep/b.txt", + "/t/deep/deeper/c.txt", + "/t2/outside.txt", + "/tX/outside.txt", + ], + ); + + let range = crate::file_handling::ExtractCursor::for_root("/t"); + let mut seen = Vec::new(); + let mut after = range.lo.clone(); + loop { + let page = rows_in_range_page(&conn, &after, &range.hi, 2).unwrap(); + let Some(last) = page.last() else { break }; + after = last.path.clone(); + seen.extend(page.into_iter().map(|r| r.path)); + } + assert_eq!( + seen, + vec!["/t/a.txt", "/t/deep/b.txt", "/t/deep/deeper/c.txt"], + "in path order, once each, and the prefix siblings are outside" + ); + + drop(conn); + std::fs::remove_file(&p).ok(); + } + /// The walk's row prefetcher runs `dir_rows` once per directory, against a /// deliberately tiny page cache, so it must not have to touch the table /// heap at all. `idx_files_parent` carries `name` and `mtime` for exactly diff --git a/crates/quicksearch-core/src/file_handling.rs b/crates/quicksearch-core/src/file_handling.rs index 955299a..9a84ff0 100644 --- a/crates/quicksearch-core/src/file_handling.rs +++ b/crates/quicksearch-core/src/file_handling.rs @@ -1,6 +1,6 @@ use std::fs::File; use std::io::Read; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; // Only the Unix entry-count path shells out; Windows walks the tree directly. @@ -67,6 +67,26 @@ pub(crate) fn path_to_db_string(path: &Path) -> String { } } +/// Canonicalize a root string for storage/comparison, stripping the Windows +/// UNC prefix. Multi-root strings (newline-joined) fail canonicalize and +/// pass through verbatim, which still compares consistently. +/// +/// The UNC strip is [`path_to_db_string`]'s, not a hand-rolled one: chopping +/// four characters would turn `\\?\UNC\server\share` into +/// `UNC\server\share`, which is not a path — and no longer looks like a +/// share, so the root would walk with the local thread count instead of the +/// network one. +/// +/// This is the spelling `files.path` rows are prefixed with, so it is also the +/// form roots must be compared in: `~/docs` and `/home/me/docs` name one root +/// and must not read as a change. See [`crate::config::diff_actions`]. +pub(crate) fn normalize_root_string(indexing_path: &str) -> String { + let path = Path::new(indexing_path) + .canonicalize() + .unwrap_or_else(|_| PathBuf::from(indexing_path)); + path_to_db_string(&path) +} + /// Warn and report `true` for a path that cannot round-trip through /// `files.path`. /// diff --git a/crates/quicksearch-core/src/indexing.rs b/crates/quicksearch-core/src/indexing.rs index 8504281..e438822 100644 --- a/crates/quicksearch-core/src/indexing.rs +++ b/crates/quicksearch-core/src/indexing.rs @@ -11,7 +11,7 @@ use crate::db::repo; use crate::extract::Registry; use crate::file_handling::{ cleanup_stale_index_entries, count_tree_entries_fast, extract_scope_prepare, - fts_finalize_after_text_indexing, path_to_db_string, process_batch_inserts, + fts_finalize_after_text_indexing, normalize_root_string, process_batch_inserts, process_batch_updates, store_extracted, ExtractCursor, FileIndexAction, OwnedNewFile, }; use crate::walk::{thread_count_for, walk_indexable_files, ParallelWalk, TryNext, WalkEvent}; @@ -518,10 +518,10 @@ impl IndexingService { &self, db_path: &str, config: &Config, - indexing_path: &str, + roots: &[String], ) -> Result>, String> { match db::open_existing(db_path, false) { - Ok(conn) => Self::validate_config(&conn, config, indexing_path), + Ok(conn) => Self::validate_config(&conn, config, roots), Err(_) => Ok(None), } } @@ -733,13 +733,24 @@ impl IndexingService { let worker_overrides = resolved_root_workers(config); // Open and migrate the database to the current schema version. - let conn = db::open_or_recreate(db_path, &config.processing.tokenize)?; + let mut conn = db::open_or_recreate(db_path, &config.processing.tokenize)?; - // Update configuration (for new installations or when no validation issues). - // `indexing_path` in the validation table stores the joined list so - // adding/removing a root triggers the same rebuild prompt as changing - // the legacy single path did. - Self::update_config(&conn, config, &roots.join("\n"))?; + // Reconcile against the settings the index was last written under, + // *before* stamping the new ones over them — the old record is the + // only thing that knows a root was dropped. This is what makes a + // config hand-edited while the app was closed behave like one edited + // live: the walk below handles narrowed ignore rules on its own (a + // filtered-out entry is simply absent from the directory listing it + // reconciles against), but nothing in it ever visits a root that is no + // longer configured, or revisits the content of a file whose mtime has + // not moved. + // + // Against `roots`, not `config.paths.indexing_paths`: the roots to + // walk are a parameter of this call, and nothing makes the caller + // pass the ones its config names. Reconciling against the config + // would then delete every row of the tree actually being indexed. + Self::reconcile_stored_config(&mut conn, config, &roots, stop_flag)?; + Self::update_config(&conn, config, &roots)?; // No up-front load of the whole `files` table: each walk's prefetcher // fetches one directory's rows at a time, so classification data is @@ -1237,13 +1248,17 @@ impl IndexingService { Ok(()) } - /// The config keys whose change invalidates the stored index, paired - /// with their current values. One list drives both [`validate_config`] - /// and [`update_config`] so the two can never drift apart. - fn config_validation_entries( - config: &Config, - indexing_path: &str, - ) -> Vec<(&'static str, String)> { + /// The settings the index was built under, paired with their current + /// values. One list drives [`validate_config`], [`update_config`] and + /// [`crate::scope::stored_config`], so the record, the comparison and the + /// reconstruction can never drift apart. + /// + /// Every list value is sorted before joining, and the roots arrive already + /// canonicalized: the record describes what the walk *did*, and reordering + /// or re-spelling a list does not change that. `stored_config` parses these + /// back, so a key added here becomes a key a hand-edited config can be + /// reconciled against. + fn config_validation_entries(config: &Config, roots: &[String]) -> Vec<(&'static str, String)> { let sorted_joined = |v: &[String]| { let mut v: Vec = v.to_vec(); v.sort(); @@ -1256,12 +1271,11 @@ impl IndexingService { // whenever the digest input changes so existing indexes are // offered a rebuild instead of silently mixing schemes. ("hash_algorithm", "size+head".to_string()), - ("indexing_path", normalize_root_string(indexing_path)), + ("indexing_path", sorted_joined(roots)), ("tokenize", config.processing.tokenize.clone()), ("include_hidden", config.indexing.include_hidden.to_string()), // Decides whether symlink targets are in the index at all, so a - // change leaves rows that no longer belong — the rebuild prompt has - // to be able to name it. + // change leaves rows that no longer belong. ( "follow_symlinks", config.indexing.follow_symlinks.to_string(), @@ -1274,21 +1288,48 @@ impl IndexingService { "content_extensions", sorted_joined(&config.indexing.content_extensions), ), + ( + "store_text_for_snippets", + config.processing.store_text_for_snippets.to_string(), + ), ] } - /// Compare current config against the values stored in the index. - /// Returns `Some(changes)` when the index was built under settings - /// that no longer match — the caller offers a rebuild. A key absent - /// from the DB (older index) only counts as changed when the DB has - /// stored *any* validation state before. + /// Every recorded `config_validation` key, for + /// [`crate::scope::stored_config`] to rebuild the configuration the index + /// was last written under. + pub(crate) fn stored_validation(conn: &Connection) -> Result, String> { + let mut stmt = conn + .prepare("SELECT key, value FROM config_validation") + .map_err(|e| format!("prepare config_validation read: {}", e))?; + let rows = stmt + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?))) + .map_err(|e| format!("read config_validation: {}", e))?; + rows.collect::, _>>() + .map_err(|e| format!("read config_validation row: {}", e)) + } + + /// The recorded settings a difference in which cannot be reconciled: the + /// FTS tokenizer is part of the table definition, and a hash written under + /// a different length or algorithm cannot be compared with a new one. + /// Everything else `config_validation_entries` records is recoverable — + /// see [`crate::scope`] — and naming it in a rebuild prompt would be + /// asking the user to pay for a wipe that is not needed. + const REBUILD_KEYS: [&'static str; 3] = ["hash_length", "hash_algorithm", "tokenize"]; + + /// The settings the index was built under that no longer match and cannot + /// be reconciled — the list a rebuild prompt shows. `None` when there are + /// none. A key absent from the DB (older index) never counts as changed. fn validate_config( conn: &Connection, config: &Config, - indexing_path: &str, + roots: &[String], ) -> Result>, String> { let mut changes = Vec::new(); - for (key, current) in Self::config_validation_entries(config, indexing_path) { + for (key, current) in Self::config_validation_entries(config, roots) + .into_iter() + .filter(|(key, _)| Self::REBUILD_KEYS.contains(key)) + { let stored: Option = conn .query_row( "SELECT value FROM config_validation WHERE key = ?1", @@ -1314,13 +1355,60 @@ impl IndexingService { }) } - /// Stamp the index with the settings it's being built under. - fn update_config( - conn: &Connection, + /// Bring the index into line with `config` before a run walks anything. + /// + /// A no-op in the normal case: the coordinator already reconciled when the + /// config was edited, so the stored record matches and the plan is empty. + /// It earns its place for configs changed while the app was not running, + /// where nothing else ever compares the two. + /// + /// Runs to completion rather than under a deadline — the run owns the + /// database until it finishes anyway, and a caller waiting on the walk is + /// not a caller that would rather have a half-reconciled index. + fn reconcile_stored_config( + conn: &mut Connection, config: &Config, - indexing_path: &str, + roots: &[String], + stop_flag: &Arc, ) -> Result<(), String> { - for (key, current) in Self::config_validation_entries(config, indexing_path) { + // What this run is about to index, which is `config` everywhere except + // its roots — see the caller. + let mut current = config.clone(); + current.paths.indexing_paths = roots.to_vec(); + + let stored = crate::scope::stored_config(conn, ¤t)?; + let work = crate::config::diff_actions(&stored, ¤t).work; + if !work.touches_index() { + return Ok(()); + } + let registry = Registry::default_set(); + let mut cursor = crate::scope::WorkCursor::new(work, ¤t)?; + while !cursor.done() { + if stop_flag.load(Ordering::Relaxed) { + return Ok(()); + } + crate::scope::advance( + conn, + ¤t, + ®istry, + &mut cursor, + Instant::now() + crate::scope::SLICE, + )?; + } + if cursor.deleted > 0 || cursor.recontented > 0 { + crate::log_info!( + "configuration changed since the last run: {} index entries removed, \ + {} re-examined for text extraction", + cursor.deleted, + cursor.recontented + ); + } + Ok(()) + } + + /// Stamp the index with the settings it's being built under. + fn update_config(conn: &Connection, config: &Config, roots: &[String]) -> Result<(), String> { + for (key, current) in Self::config_validation_entries(config, roots) { conn.execute( "INSERT OR REPLACE INTO config_validation (key, value) VALUES (?1, ?2)", params![key, current], @@ -1331,22 +1419,6 @@ impl IndexingService { } } -/// Canonicalize a root string for storage/comparison, stripping the Windows -/// UNC prefix. Multi-root strings (newline-joined) fail canonicalize and -/// pass through verbatim, which still compares consistently. -/// -/// The UNC strip is [`path_to_db_string`]'s, not a hand-rolled one: chopping -/// four characters would turn `\\?\UNC\server\share` into -/// `UNC\server\share`, which is not a path — and no longer looks like a -/// share, so the root would walk with the local thread count instead of the -/// network one. -fn normalize_root_string(indexing_path: &str) -> String { - let path = std::path::Path::new(indexing_path) - .canonicalize() - .unwrap_or_else(|_| std::path::PathBuf::from(indexing_path)); - path_to_db_string(&path) -} - impl Drop for IndexingService { fn drop(&mut self) { // Ensure graceful shutdown when the service is dropped diff --git a/crates/quicksearch-core/src/lib.rs b/crates/quicksearch-core/src/lib.rs index a022b07..d7ef1d1 100644 --- a/crates/quicksearch-core/src/lib.rs +++ b/crates/quicksearch-core/src/lib.rs @@ -12,6 +12,7 @@ pub mod log; pub mod mime; pub mod platform; pub mod query; +pub mod scope; pub mod search; pub mod security; pub mod shutdown; diff --git a/crates/quicksearch-core/src/scope.rs b/crates/quicksearch-core/src/scope.rs new file mode 100644 index 0000000..7e63331 --- /dev/null +++ b/crates/quicksearch-core/src/scope.rs @@ -0,0 +1,587 @@ +//! Bringing a stored index back in line with a changed configuration, +//! without deleting it. +//! +//! The index is a cache of what a walk under the configured roots would +//! produce. When the configuration changes, the two disagree — and almost +//! always in a way that can be *reconciled* rather than rebuilt: +//! +//! * A root was removed. Its rows are a contiguous `files.path` range, so +//! they go in five statements ([`crate::db::repo::delete_subtree`]). +//! * An ignore pattern was added, hidden files were switched off, symlinks +//! stopped being followed. The rows to drop are picked out by a predicate +//! no SQL range can express, so [`Scope::covers`] re-runs the walker's own +//! filtering rules against each stored path. +//! * The content filter moved. The rows stay; only their extracted text, +//! properties and FTS entry are re-decided. +//! +//! Only settings that make stored data unreadable or incomparable — the FTS +//! tokenizer, the hash length, the encryption key — still force a wipe. See +//! [`crate::config::diff_actions`], which decides which of these applies, and +//! [`crate::config::IndexWork`], the plan it produces. +//! +//! ## Why the scan is per-root +//! +//! [`advance`] walks each configured root's `[lo, hi)` range rather than the +//! whole `files` table. That is not only a matter of using the index: with +//! `follow_symlinks` on, a symlink target is stored under its own canonical +//! path, which may lie outside every root. Such a row is legitimately +//! indexed, has no owning root, and so has no filtering rules that can be +//! applied to it — scanning by range means it is simply never visited, the +//! same exemption `aliased_paths` gives it during a full run's stale sweep. + +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use rusqlite::Connection; + +use crate::config::{Config, IgnoreSet, IndexWork}; +use crate::db::repo; +use crate::extract::Registry; +use crate::file_handling::{content_extractable, fts_finalize_after_text_indexing, ExtractCursor}; + +/// How long [`advance`] may work before handing control back. +/// +/// Not a throughput knob — the caller decides when to come back — but a bound +/// on how long a Stop, a search or a further config edit waits behind a scan +/// in progress. The same budget the coordinator gives its watcher queue. +pub const SLICE: Duration = Duration::from_millis(250); + +/// One configured root, with the `files.path` range it owns precomputed. +struct Root { + path: PathBuf, + lo: String, + hi: String, +} + +/// The set of paths the current configuration would index. +/// +/// The walker applies these rules on the way down, pruning a directory before +/// it descends. `Scope` applies the same rules to a path that is already +/// stored, which is what lets a narrowed filter delete the rows that fell out +/// of scope instead of rebuilding the whole index. The two must agree exactly, +/// or every run would re-add what the last prune removed; the tests below pin +/// that agreement against [`crate::walk`]'s own behaviour. +pub struct Scope { + roots: Vec, + ignore: IgnoreSet, + include_hidden: bool, +} + +impl Scope { + pub fn from_config(config: &Config) -> Result { + let roots = config + .normalized_indexing_paths() + .into_iter() + .map(|root| { + let range = ExtractCursor::for_root(&root); + Root { + path: PathBuf::from(root), + lo: range.lo, + hi: range.hi, + } + }) + .collect(); + Ok(Scope { + roots, + ignore: IgnoreSet::compile(&config.indexing.ignore_patterns) + .map_err(|e| format!("ignore patterns: {}", e))?, + include_hidden: config.indexing.include_hidden, + }) + } + + /// The configured root `path` lives under, if any. `Path::starts_with` + /// compares whole components, so `/a/bc` is never read as living under + /// `/a/b`. + pub fn owning_root(&self, path: &Path) -> Option<&Path> { + self.roots + .iter() + .map(|r| r.path.as_path()) + .find(|root| path.starts_with(root) && path != *root) + } + + /// Whether the walker would still emit `path` while walking `root`. + /// + /// Mirrors `read_directory`'s three `continue`s. The full-path ignore + /// patterns are tested once against the whole path because + /// [`IgnoreSet::matches_path_pattern`] already walks every ancestor, + /// which is exactly the union of the per-level tests the walker performs + /// on its way down. The hidden and component-pattern rules are tested per + /// component *below* the root: a root is never filtered, because the user + /// chose it (see `walk_parallel`). + pub fn covers(&self, root: &Path, path: &Path) -> bool { + if self.ignore.matches_path_pattern(path) { + return false; + } + let Ok(relative) = path.strip_prefix(root) else { + return false; + }; + let mut current = root.to_path_buf(); + for component in relative.components() { + let std::path::Component::Normal(name) = component else { + // Stored paths are canonical, so stripping a canonical root + // leaves plain names. Anything else did not come from a walk. + return false; + }; + current.push(name); + let name = name.to_string_lossy(); + // The metadata closure is only consulted on Windows, where hidden + // is an attribute rather than a leading dot; on Unix this stays at + // zero syscalls, exactly as it does in the walker. + if !self.include_hidden + && crate::platform::entry_is_hidden(&name, || std::fs::metadata(¤t).ok()) + { + return false; + } + if self.ignore.matches_component(&name) { + return false; + } + } + true + } +} + +/// How far an in-progress [`advance`] has got. +/// +/// The work is resumable because a multi-million-row index must not hold the +/// coordinator's command loop for the length of a full scan; the caller hands +/// back the same cursor each tick with a fresh deadline, the way +/// `apply_pending` drains the watcher queue. +pub struct WorkCursor { + work: IndexWork, + scope: Scope, + /// Index into `work.drop_roots` of the next range to delete outright. + drop_idx: usize, + /// Set once the out-of-root sweep has run; it is a single statement set, + /// so it either happened or it did not. + dropped_aliases: bool, + /// Index into `scope.roots` of the range being scanned. + root_idx: usize, + /// Last path served by the scan — the keyset cursor. Empty means "start + /// this root's range from its `lo` bound". + after: String, + /// Set once the FTS automerge that follows a batch of deletions has run. + finalized: bool, + /// Rows deleted so far, for the log line when the work completes. + pub deleted: usize, + /// Rows whose content state or stored text was re-decided. + pub recontented: usize, +} + +impl WorkCursor { + pub fn new(work: IndexWork, config: &Config) -> Result { + Ok(WorkCursor { + work, + scope: Scope::from_config(config)?, + drop_idx: 0, + dropped_aliases: false, + root_idx: 0, + after: String::new(), + finalized: false, + deleted: 0, + recontented: 0, + }) + } + + pub fn done(&self) -> bool { + self.finalized + } + + /// Whether a full walk must follow this reconciliation. + pub fn reindex(&self) -> bool { + self.work.reindex + } + + /// The plan being applied, for a caller that has to restart against a + /// newer configuration and must not lose what this one had left to do. + pub fn work(&self) -> &IndexWork { + &self.work + } + + /// Drop the walk this reconciliation asked for, keeping the rest. For a + /// caller that has since been told not to run anything. + pub fn cancel_reindex(&mut self) { + self.work.reindex = false; + } +} + +/// Apply as much of `cursor` as fits before `deadline`, one page of rows per +/// transaction. Returns with the cursor advanced; call again until +/// [`WorkCursor::done`]. +pub fn advance( + conn: &mut Connection, + config: &Config, + registry: &Registry, + cursor: &mut WorkCursor, + deadline: Instant, +) -> Result<(), String> { + // Whole ranges first: a removed root's rows can never satisfy the scan's + // filters anyway, and deleting them by range spares the scan the work. + while cursor.drop_idx < cursor.work.drop_roots.len() { + let range = ExtractCursor::for_root(&cursor.work.drop_roots[cursor.drop_idx]); + let tx = conn + .transaction() + .map_err(|e| format!("begin drop-root transaction: {}", e))?; + let removed = repo::delete_subtree(&tx, &range.lo, &range.hi)?; + tx.commit() + .map_err(|e| format!("commit drop-root transaction: {}", e))?; + cursor.deleted += removed; + cursor.drop_idx += 1; + if Instant::now() >= deadline { + return Ok(()); + } + } + + // Before the per-root scan and after the root deletions: the ranges it + // spares must already be the final set of roots. + if !cursor.dropped_aliases && cursor.work.drop_aliases { + let ranges: Vec<(String, String)> = cursor + .scope + .roots + .iter() + .map(|r| (r.lo.clone(), r.hi.clone())) + .collect(); + let tx = conn + .transaction() + .map_err(|e| format!("begin drop-alias transaction: {}", e))?; + let removed = repo::delete_outside_ranges(&tx, &ranges)?; + tx.commit() + .map_err(|e| format!("commit drop-alias transaction: {}", e))?; + cursor.deleted += removed; + cursor.dropped_aliases = true; + if Instant::now() >= deadline { + return Ok(()); + } + } + + if cursor.work.scans_rows() { + let page = config.processing.batch_size.max(1) as i64; + while cursor.root_idx < cursor.scope.roots.len() { + let root = &cursor.scope.roots[cursor.root_idx]; + if cursor.after.is_empty() { + cursor.after = root.lo.clone(); + } + let rows = repo::rows_in_range_page(conn, &cursor.after, &root.hi, page)?; + let Some(last) = rows.last() else { + cursor.root_idx += 1; + cursor.after.clear(); + continue; + }; + cursor.after = last.path.clone(); + let root = cursor.scope.roots[cursor.root_idx].path.clone(); + let (deleted, recontented) = apply_page( + conn, + config, + registry, + &cursor.scope, + &cursor.work, + &root, + &rows, + )?; + cursor.deleted += deleted; + cursor.recontented += recontented; + if Instant::now() >= deadline { + return Ok(()); + } + } + } + + // Deletions leave the FTS index with tombstones and a long segment list; + // the same automerge that follows a run's stale cleanup collapses them. + if cursor.deleted > 0 || cursor.recontented > 0 { + fts_finalize_after_text_indexing(conn); + } + cursor.finalized = true; + Ok(()) +} + +/// Decide and write one page of rows. Returns `(deleted, recontented)`. +fn apply_page( + conn: &mut Connection, + config: &Config, + registry: &Registry, + scope: &Scope, + work: &IndexWork, + root: &Path, + rows: &[repo::ScopeRow], +) -> Result<(usize, usize), String> { + let mut doomed: Vec = Vec::new(); + let mut stale_text: Vec = Vec::new(); + let mut to_pending: Vec = Vec::new(); + let mut to_na: Vec = Vec::new(); + + for row in rows { + let path = Path::new(&row.path); + if work.prune_scope && !scope.covers(root, path) { + doomed.push(row.id); + continue; + } + if work.drop_text { + stale_text.push(row.id); + } + if work.reconcile_content || work.restore_text { + // The walker's own decision, recomputed from the columns it wrote + // it into. Both directions run whenever either flag is set: the + // answer comes from the *current* config, so a row that disagrees + // with it is wrong however it got that way. + let wants = row.size <= config.processing.maximum_text_file_size + && content_extractable(path, row.mime.as_deref(), config, registry); + if !wants && row.content_state != repo::STATE_NA { + to_na.push(row.id); + } else if wants + && (row.content_state == repo::STATE_NA + || (work.restore_text && row.content_state == repo::STATE_DONE)) + { + to_pending.push(row.id); + } + } + } + + let tx = conn + .transaction() + .map_err(|e| format!("begin reconcile transaction: {}", e))?; + let deleted = if doomed.is_empty() { + 0 + } else { + repo::delete_ids(&tx, &doomed)? + }; + if !stale_text.is_empty() { + repo::drop_stored_text(&tx, &stale_text)?; + } + for id in &to_pending { + repo::reset_content_pending(&tx, *id)?; + } + for id in &to_na { + repo::remove_content_for_id(&tx, *id)?; + repo::set_content_na(&tx, *id)?; + } + tx.commit() + .map_err(|e| format!("commit reconcile transaction: {}", e))?; + Ok((deleted, to_pending.len() + to_na.len())) +} + +/// The configuration the index was last built with, as far as +/// `config_validation` records it: `config` with the recorded fields +/// substituted back in. +/// +/// Fields the table does not record keep `config`'s own values, so they never +/// read as changed — the table is a record of what the walk used, not a second +/// copy of the config. Feeding this to [`crate::config::diff_actions`] is what +/// lets a config edited while the app was closed produce exactly the same plan +/// as one edited live, from one decision table rather than two. +pub fn stored_config(conn: &Connection, config: &Config) -> Result { + let mut stored = config.clone(); + let recorded = crate::indexing::IndexingService::stored_validation(conn)?; + let lines = |value: &str| -> Vec { + value + .split('\n') + .map(str::to_string) + .filter(|s| !s.is_empty()) + .collect() + }; + for (key, value) in recorded { + match key.as_str() { + "indexing_path" => stored.paths.indexing_paths = lines(&value), + "ignore_patterns" => stored.indexing.ignore_patterns = lines(&value), + "content_extensions" => stored.indexing.content_extensions = lines(&value), + "include_hidden" => stored.indexing.include_hidden = value == "true", + "follow_symlinks" => stored.indexing.follow_symlinks = value == "true", + "store_text_for_snippets" => { + stored.processing.store_text_for_snippets = value == "true" + } + "hash_length" => { + if let Ok(n) = value.parse() { + stored.processing.hash_length = n; + } + } + "tokenize" => stored.processing.tokenize = value, + _ => {} + } + } + Ok(stored) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::walk::{walk_indexable_files, WalkEvent}; + use std::collections::HashSet; + use std::sync::atomic::AtomicBool; + use std::sync::Arc; + + fn tmp_tree(tag: &str) -> PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!( + "quicksearch-scope-{}-{}-{}", + tag, + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&p).unwrap(); + std::fs::canonicalize(&p).unwrap() + } + + fn touch(p: &Path) { + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, b"x").unwrap(); + } + + fn empty_db(dir: &Path) -> PathBuf { + let db = dir.join("index.sqlite"); + crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap(); + db + } + + /// Every file the walker actually emits under `config`'s single root. + fn walked(config: &Config, db: &Path) -> HashSet { + let root = config.paths.indexing_paths[0].clone(); + walk_indexable_files( + &[root], + config.indexing.follow_symlinks, + config.indexing.include_hidden, + IgnoreSet::compile(&config.indexing.ignore_patterns).unwrap(), + db.to_str().unwrap(), + config.clone(), + Arc::new(Registry::default_set()), + Arc::new(AtomicBool::new(false)), + Arc::new(AtomicBool::new(false)), + 2, + ) + .filter_map(|e| match e { + WalkEvent::File(f) => Some(PathBuf::from(f.path)), + WalkEvent::Stale(_) => None, + }) + .collect() + } + + /// Every file that physically exists under `root`, walker or no walker. + fn on_disk(root: &Path) -> Vec { + walkdir::WalkDir::new(root) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + .map(|e| e.into_path()) + .collect() + } + + /// The whole point of `Scope`: it must reach the same verdict the walker + /// does for every file on disk. If it is stricter, every prune deletes + /// rows the next run puts straight back; if it is laxer, the rows the + /// user excluded survive. Either way the index never settles. + #[test] + fn scope_agrees_with_the_walker() { + let root = tmp_tree("agree"); + touch(&root.join("keep.txt")); + touch(&root.join("sub/keep2.txt")); + touch(&root.join("sub/skip.tmp")); + touch(&root.join("sub/node_modules/dep/index.js")); + touch(&root.join(".hidden/inside.txt")); + touch(&root.join(".dotfile")); + touch(&root.join("build/out/artifact.o")); + touch(&root.join("build/keep3.txt")); + touch(&root.join("nested/build/also.o")); + + let mut config = Config::default(); + config.paths.indexing_paths = vec![root.to_string_lossy().into_owned()]; + config.indexing.ignore_patterns = vec![ + "*.tmp".into(), + "node_modules".into(), + // A full-path pattern: prunes this one directory, not every + // directory called `out`. + root.join("build/out").to_string_lossy().into_owned(), + ]; + + for include_hidden in [false, true] { + config.indexing.include_hidden = include_hidden; + let db = empty_db(&tmp_tree("agree-db")); + let emitted = walked(&config, &db); + let scope = Scope::from_config(&config).unwrap(); + + for path in on_disk(&root) { + assert_eq!( + scope.covers(&root, &path), + emitted.contains(&path), + "disagreement on {} (include_hidden = {})", + path.display(), + include_hidden + ); + } + } + std::fs::remove_dir_all(&root).ok(); + } + + /// A root is never filtered — the user chose it. A component pattern + /// naming the root must not empty it out, but a full-path pattern that + /// matches the root still prunes everything below it, because that is + /// what the walker's ancestor check does when it reads the children. + #[test] + fn a_root_is_never_filtered_but_its_children_still_are() { + let base = tmp_tree("root-name"); + let root = base.join("node_modules"); + touch(&root.join("keep.txt")); + touch(&root.join("node_modules/nested.txt")); + + let mut config = Config::default(); + config.paths.indexing_paths = vec![root.to_string_lossy().into_owned()]; + config.indexing.ignore_patterns = vec!["node_modules".into()]; + let scope = Scope::from_config(&config).unwrap(); + assert!(scope.covers(&root, &root.join("keep.txt"))); + assert!(!scope.covers(&root, &root.join("node_modules/nested.txt"))); + + let db = empty_db(&tmp_tree("root-name-db")); + let emitted = walked(&config, &db); + for path in on_disk(&root) { + assert_eq!(scope.covers(&root, &path), emitted.contains(&path)); + } + + // A full-path pattern reaching the root itself takes the whole tree. + config.indexing.ignore_patterns = vec![root.to_string_lossy().into_owned()]; + let scope = Scope::from_config(&config).unwrap(); + assert!(!scope.covers(&root, &root.join("keep.txt"))); + + std::fs::remove_dir_all(&base).ok(); + } + + /// Root ownership compares whole components, so a sibling whose name + /// merely starts with a root's is not inside it — a prune that got this + /// wrong would delete a neighbouring folder's entire index. + #[test] + fn owning_root_does_not_match_name_prefixes() { + let base = tmp_tree("prefix"); + let root = base.join("data"); + let sibling = base.join("database"); + std::fs::create_dir_all(&root).unwrap(); + std::fs::create_dir_all(&sibling).unwrap(); + + let mut config = Config::default(); + config.paths.indexing_paths = vec![root.to_string_lossy().into_owned()]; + let scope = Scope::from_config(&config).unwrap(); + + assert_eq!(scope.owning_root(&root.join("f.txt")), Some(root.as_path())); + assert_eq!(scope.owning_root(&sibling.join("f.txt")), None); + // The root itself is a directory, never a row, and owns nothing. + assert_eq!(scope.owning_root(&root), None); + std::fs::remove_dir_all(&base).ok(); + } + + /// A path under no configured root has no rules that could be applied to + /// it — a followed symlink's target is the real case. The scan reaches it + /// by never visiting it, so `owning_root` returning `None` is what keeps + /// it alive. + #[test] + fn a_path_outside_every_root_has_no_owner() { + let base = tmp_tree("outside"); + let root = base.join("indexed"); + std::fs::create_dir_all(&root).unwrap(); + + let mut config = Config::default(); + config.paths.indexing_paths = vec![root.to_string_lossy().into_owned()]; + config.indexing.ignore_patterns = vec!["*".into()]; + let scope = Scope::from_config(&config).unwrap(); + + assert_eq!(scope.owning_root(Path::new("/elsewhere/target.txt")), None); + std::fs::remove_dir_all(&base).ok(); + } +} diff --git a/crates/quicksearch-core/src/search/cascade.rs b/crates/quicksearch-core/src/search/cascade.rs index fe267b7..3d76ed2 100644 --- a/crates/quicksearch-core/src/search/cascade.rs +++ b/crates/quicksearch-core/src/search/cascade.rs @@ -169,6 +169,15 @@ pub fn run( Ok(true) => {} Ok(false) => return Ok(None), // cancelled mid-pass Err(e) => { + // A kill from `interrupt()` arrives as an ordinary SQL error, + // and SQLite's interrupt flag carries no ordering edge to the + // generation counter the canceller bumped just before it. On a + // weakly-ordered CPU the plain load in `cancelled()` may still + // read the pre-bump value here, which would report routine + // cancellation as `Search failed: interrupted`. Fence first, so + // having seen the kill implies seeing the bump that caused it. + // Error path only — the per-row checks stay relaxed. + std::sync::atomic::fence(Ordering::Acquire); if cx.cancelled() { return Ok(None); // interrupt() killed the statement } diff --git a/crates/quicksearch-core/src/search/mod.rs b/crates/quicksearch-core/src/search/mod.rs index 1523751..17afe9f 100644 --- a/crates/quicksearch-core/src/search/mod.rs +++ b/crates/quicksearch-core/src/search/mod.rs @@ -15,6 +15,13 @@ //! phases like FTS candidate gathering). An interrupted stale search is //! normal cancellation, not an error. //! +//! The interrupt handle is stored tagged with the generation that owns it, +//! and only ever fired at a generation the counter has already moved past. +//! Interrupting the *current* generation would surface to the user as +//! "Search failed: interrupted" instead of results, and the window for it +//! is real: the worker can dequeue and start a request before the caller +//! that queued it gets back onto the CPU. +//! //! Consumers that want a plain blocking search (the CLI mode) skip the //! service entirely and call [`cascade::run`] with a collecting sink. @@ -118,10 +125,16 @@ struct SearchRequest { options: SearchOptions, } +/// The search the worker is executing right now, and the handle that kills +/// its statement. Tagged with the generation so a caller can tell whether +/// the thing it is about to interrupt is the search it means to cancel or +/// one that has since replaced it. +type InFlight = Arc>>; + pub struct SearchService { req_tx: mpsc::Sender, latest_gen: Arc, - interrupt: Arc>>, + in_flight: InFlight, db_path: Arc>, handle: Option>, } @@ -138,7 +151,7 @@ impl SearchService { let (req_tx, req_rx) = mpsc::channel::(); let (update_tx, update_rx) = mpsc::channel::(); let latest_gen = Arc::new(AtomicU64::new(0)); - let interrupt = Arc::new(Mutex::new(None)); + let in_flight: InFlight = Arc::new(Mutex::new(None)); let db_path = Arc::new(Mutex::new(db_path)); let worker = Worker { @@ -146,7 +159,7 @@ impl SearchService { update_tx, notify, latest_gen: latest_gen.clone(), - interrupt: interrupt.clone(), + in_flight: in_flight.clone(), db_path: db_path.clone(), }; let handle = std::thread::Builder::new() @@ -158,7 +171,7 @@ impl SearchService { SearchService { req_tx, latest_gen, - interrupt, + in_flight, db_path, handle: Some(handle), }, @@ -170,21 +183,22 @@ impl SearchService { /// generation whose events to keep. pub fn search(&self, input: &str, options: SearchOptions) -> u64 { let generation = self.latest_gen.fetch_add(1, Ordering::SeqCst) + 1; + // Interrupt before enqueueing: an idle worker can dequeue the new + // request and be mid-statement within microseconds, and there is no + // point handing it a kill the worker has to survive. + self.interrupt_stale(); let _ = self.req_tx.send(SearchRequest { generation, input: input.to_string(), options, }); - // The new request can't be running yet (the worker hasn't dequeued - // it), so this only ever kills a stale generation's statement. - self.interrupt_current(); generation } /// Cancel without starting anything new. pub fn cancel(&self) { self.latest_gen.fetch_add(1, Ordering::SeqCst); - self.interrupt_current(); + self.interrupt_stale(); } /// Point subsequent searches at a different index file. @@ -193,10 +207,24 @@ impl SearchService { self.cancel(); } - fn interrupt_current(&self) { - if let Ok(guard) = self.interrupt.lock() { - if let Some(handle) = guard.as_ref() { - handle.interrupt(); + /// Kill the running statement — but only if the generation counter has + /// already moved past the search that owns it. + /// + /// Callers bump `latest_gen` first, so anything still tagged with an + /// older generation is stale by definition. The tag is what makes this + /// safe rather than merely well-timed: interrupting the *newest* search + /// does not cancel anything, it fails it, and the cascade reports that + /// as `Search failed: interrupted` because its own generation is still + /// current. On a loaded machine the worker really can pick up and start + /// a request before the thread that queued it runs again, so "the new + /// search cannot have started yet" is not an assumption to build on. + fn interrupt_stale(&self) { + let latest = self.latest_gen.load(Ordering::SeqCst); + if let Ok(guard) = self.in_flight.lock() { + if let Some((generation, handle)) = guard.as_ref() { + if *generation != latest { + handle.interrupt(); + } } } } @@ -238,7 +266,7 @@ struct Worker { update_tx: mpsc::Sender, notify: Arc, latest_gen: Arc, - interrupt: Arc>>, + in_flight: InFlight, db_path: Arc>, } @@ -291,7 +319,10 @@ impl Worker { return; } }; - *self.interrupt.lock().unwrap() = Some(conn.get_interrupt_handle()); + // Publish the handle tagged with the generation it kills, before the + // first statement runs. Anyone cancelling from here on can tell this + // search apart from the one that supersedes it. + *self.in_flight.lock().unwrap() = Some((generation, conn.get_interrupt_handle())); let mut sink = |hits: Vec| { self.send(SearchUpdate::Hits { generation, hits }); @@ -305,7 +336,7 @@ impl Worker { &mut sink, ); - *self.interrupt.lock().unwrap() = None; + *self.in_flight.lock().unwrap() = None; match outcome { Ok(Some(Outcome { total, limited })) => self.send(SearchUpdate::Completed { @@ -327,6 +358,91 @@ impl Worker { mod tests { use super::*; + /// Start a query long enough to be killed while it is executing, on its + /// own thread. Returns the handle that kills it and the result channel. + fn spawn_slow_query() -> ( + rusqlite::InterruptHandle, + mpsc::Receiver>, + ) { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + let handle = conn.get_interrupt_handle(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + let counted = conn.query_row( + "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c WHERE x < 1000000) \ + SELECT count(*) FROM c", + [], + |row| row.get::<_, i64>(0), + ); + let _ = tx.send(counted); + }); + (handle, rx) + } + + /// Cancel repeatedly for as long as the query runs, so every window in + /// which an interrupt could land is exercised rather than hoped past. + fn cancel_until_done( + service: &SearchService, + rx: &mpsc::Receiver>, + ) -> rusqlite::Result { + loop { + match rx.recv_timeout(std::time::Duration::from_millis(1)) { + Ok(result) => return result, + Err(mpsc::RecvTimeoutError::Timeout) => service.interrupt_stale(), + Err(mpsc::RecvTimeoutError::Disconnected) => panic!("query thread died"), + } + } + } + + fn idle_service() -> SearchService { + // Nothing is ever enqueued, so the path is never opened. + SearchService::new(PathBuf::from("/nonexistent"), Arc::new(|| {})).0 + } + + /// Typing the next character must not kill the search that character + /// started. The worker can be mid-statement on the newest generation by + /// the time the caller gets around to cancelling — on a slow machine that + /// is common, not exotic — and killing it there surfaces as + /// "Search failed: interrupted" instead of results. + #[test] + fn cancelling_spares_the_newest_generation() { + let service = idle_service(); + let (handle, rx) = spawn_slow_query(); + service.latest_gen.store(7, Ordering::SeqCst); + *service.in_flight.lock().unwrap() = Some((7, handle)); + + let result = cancel_until_done(&service, &rx); + *service.in_flight.lock().unwrap() = None; + assert_eq!( + result.ok(), + Some(1_000_000), + "the newest generation was interrupted" + ); + service.shutdown(); + } + + /// The other half: a generation the counter has moved past still dies + /// promptly, which is what keeps a keystroke from waiting on the previous + /// query. + #[test] + fn cancelling_kills_a_superseded_generation() { + let service = idle_service(); + let (handle, rx) = spawn_slow_query(); + service.latest_gen.store(8, Ordering::SeqCst); + *service.in_flight.lock().unwrap() = Some((7, handle)); + + let err = cancel_until_done(&service, &rx) + .expect_err("a superseded generation must be interrupted"); + *service.in_flight.lock().unwrap() = None; + assert_eq!( + err.sqlite_error_code(), + Some(rusqlite::ErrorCode::OperationInterrupted), + "unexpected error: {}", + err + ); + service.shutdown(); + } + #[test] fn key_mismatch_is_never_classified_as_corruption() { let msg = format!( diff --git a/crates/quicksearch-core/tests/reconcile.rs b/crates/quicksearch-core/tests/reconcile.rs new file mode 100644 index 0000000..55e7831 --- /dev/null +++ b/crates/quicksearch-core/tests/reconcile.rs @@ -0,0 +1,633 @@ +//! End-to-end tests for reconciling a real index against a changed +//! configuration. +//! +//! The thing every test here really asserts is that the index file *survived*. +//! Losing it is silent — the next run rebuilds and everything looks fine, only +//! hours later and with every extracted document read again — so each test +//! pins `schema_info.created_at`, which only a wipe can change. Without that +//! assertion a regression that quietly reintroduces the rebuild would pass +//! every other check in this file. + +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use quicksearch_core::config::{diff_actions, Config}; +use quicksearch_core::db; +use quicksearch_core::extract::Registry; +use quicksearch_core::indexing::{IndexingService, IndexingStatus}; +use quicksearch_core::scope::{advance, WorkCursor, SLICE}; + +fn tmp_dir(tag: &str) -> PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!( + "quicksearch-reconcile-{}-{}-{}", + tag, + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&p).unwrap(); + std::fs::canonicalize(&p).unwrap() +} + +fn touch(p: &Path, body: &[u8]) { + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, body).unwrap(); +} + +/// Run one full index over `config`'s roots and wait for it to finish. +fn index_once(db: &Path, config: &Config) { + if db.exists() { + let conn = rusqlite::Connection::open(db).unwrap(); + conn.execute("DELETE FROM schema_info WHERE key = 'last_full_index'", []) + .unwrap(); + } + let service = IndexingService::new(); + service + .start_indexing( + config.paths.indexing_paths.clone(), + db.to_string_lossy().into_owned(), + config.clone(), + ) + .unwrap(); + + let deadline = Instant::now() + Duration::from_secs(120); + let mut done = false; + while Instant::now() < deadline { + if let IndexingStatus::Error(e) = service.get_status() { + panic!("indexing failed: {}", e); + } + if db.exists() { + if let Ok(conn) = rusqlite::Connection::open(db) { + if db::repo::get_last_full_index(&conn).is_some() { + done = true; + break; + } + } + } + std::thread::sleep(Duration::from_millis(10)); + } + assert!(done, "indexing did not finish within the timeout"); + service.stop_indexing().unwrap(); +} + +/// Apply the reconciliation `old -> new` implies, exactly as the coordinator +/// would: to completion, in slices, against the live index. +fn reconcile(db: &Path, old: &Config, new: &Config) -> (usize, usize) { + let actions = diff_actions(old, new); + assert!( + !actions.requires_rebuild, + "this change must not need a wipe" + ); + let mut conn = db::open_existing(&db.to_string_lossy(), true).unwrap(); + let registry = Registry::default_set(); + let mut cursor = WorkCursor::new(actions.work, new).unwrap(); + while !cursor.done() { + advance( + &mut conn, + new, + ®istry, + &mut cursor, + Instant::now() + SLICE, + ) + .unwrap(); + } + (cursor.deleted, cursor.recontented) +} + +fn conn(db: &Path) -> rusqlite::Connection { + rusqlite::Connection::open(db).unwrap() +} + +fn paths(db: &Path) -> Vec { + let c = conn(db); + let mut stmt = c.prepare("SELECT path FROM files ORDER BY path").unwrap(); + let out = stmt + .query_map([], |r| r.get::<_, String>(0)) + .unwrap() + .map(|r| r.unwrap()) + .collect(); + out +} + +fn count(db: &Path, sql: &str) -> i64 { + conn(db).query_row(sql, [], |r| r.get(0)).unwrap() +} + +/// The index's birth certificate. A rebuild deletes the file, so this value +/// changing (or the row vanishing) is proof the index was thrown away. +fn created_at(db: &Path) -> String { + conn(db) + .query_row( + "SELECT value FROM schema_info WHERE key = 'created_at'", + [], + |r| r.get(0), + ) + .unwrap() +} + +/// FTS postings and stored-text blobs still in the index. +fn residue(db: &Path) -> (i64, i64) { + ( + count(db, "SELECT COUNT(*) FROM searchabletext"), + count(db, "SELECT COUNT(*) FROM documents_text"), + ) +} + +/// Rows in a dependent table whose file is gone. +/// +/// `searchabletext` is the one that matters and the one this exists for: it +/// is an FTS5 virtual table with no foreign key, so a delete that forgets it +/// leaves postings behind — and a contentless table happily keeps serving a +/// rowid nothing can resolve, which surfaces as a search hit for a file that +/// is no longer indexed. The other three cascade, and are checked so that a +/// future connection opened without `PRAGMA foreign_keys` cannot make this +/// quietly untrue. +fn orphans(db: &Path) -> i64 { + [ + ("searchabletext", "rowid"), + ("documents_text", "file_id"), + ("properties", "file_id"), + ("failed_files", "file_id"), + ] + .iter() + .map(|(table, key)| { + count( + db, + &format!( + "SELECT COUNT(*) FROM {0} t \ + WHERE NOT EXISTS (SELECT 1 FROM files f WHERE f.id = t.{1})", + table, key + ), + ) + }) + .sum() +} + +fn tree(root: &Path) { + touch(&root.join("keep.txt"), b"alpha keep"); + touch(&root.join("notes.md"), b"bravo notes"); + touch(&root.join("build/output.log"), b"charlie log"); + touch(&root.join("build/keep2.txt"), b"delta keep"); + touch(&root.join("node_modules/dep/index.js"), b"echo dep"); +} + +fn base_config(root: &Path, db: &Path) -> Config { + let mut config = Config::default(); + config.paths.indexing_paths = vec![root.to_string_lossy().into_owned()]; + config.paths.database_path = db.to_string_lossy().into_owned(); + // Start with nothing excluded, so each test narrows from a full index. + config.indexing.ignore_patterns = vec![]; + config +} + +/// Adding an ignore pattern must remove exactly the entries it matches — +/// their name row, their FTS postings and their extracted text — and leave +/// the index file itself alone. +#[test] +fn adding_an_ignore_pattern_prunes_instead_of_rebuilding() { + let root = tmp_dir("ignore-add"); + let db_dir = tmp_dir("ignore-add-db"); + let db = db_dir.join("index.sqlite"); + tree(&root); + + let old = base_config(&root, &db); + index_once(&db, &old); + let born = created_at(&db); + assert_eq!(paths(&db).len(), 5); + assert_eq!(residue(&db), (5, 5), "every file was extracted and stored"); + + let mut new = old.clone(); + new.indexing.ignore_patterns = vec!["*.log".into(), "node_modules".into()]; + let (deleted, _) = reconcile(&db, &old, &new); + + assert_eq!(deleted, 2, "the log and the dependency"); + assert_eq!( + paths(&db) + .iter() + .map(|p| Path::new(p) + .file_name() + .unwrap() + .to_string_lossy() + .into_owned()) + .collect::>(), + vec!["keep2.txt", "keep.txt", "notes.md"] + ); + assert_eq!( + residue(&db), + (3, 3), + "the FTS postings and the extracted text went with the rows" + ); + assert_eq!(orphans(&db), 0); + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'echo'" + ), + 0, + "the ignored file's content is no longer findable" + ); + assert_eq!(created_at(&db), born, "the index was not rebuilt"); + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +/// Removing a pattern only ever *adds* files, so it must delete nothing and +/// ask for a walk. Running that walk brings the entries back without the +/// index having been thrown away in between. +#[test] +fn removing_an_ignore_pattern_reindexes_and_deletes_nothing() { + let root = tmp_dir("ignore-remove"); + let db_dir = tmp_dir("ignore-remove-db"); + let db = db_dir.join("index.sqlite"); + tree(&root); + + let mut old = base_config(&root, &db); + old.indexing.ignore_patterns = vec!["*.log".into()]; + index_once(&db, &old); + let born = created_at(&db); + assert_eq!(paths(&db).len(), 4, "the log was never indexed"); + + let mut new = old.clone(); + new.indexing.ignore_patterns = vec![]; + let actions = diff_actions(&old, &new); + assert!(!actions.requires_rebuild); + assert!(actions.work.reindex, "a walk must follow"); + let (deleted, recontented) = reconcile(&db, &old, &new); + assert_eq!((deleted, recontented), (0, 0), "nothing was touched"); + assert_eq!(paths(&db).len(), 4, "still four until the walk runs"); + + index_once(&db, &new); + assert_eq!(paths(&db).len(), 5, "the walk found the log"); + assert_eq!(created_at(&db), born, "the index was not rebuilt"); + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +/// Removing a folder takes its entries and only its entries. This used to be +/// a full wipe, so a user with two roots paid for both to be walked again to +/// stop indexing one of them. +#[test] +fn removing_a_root_takes_only_its_own_entries() { + let base = tmp_dir("roots"); + let kept = base.join("kept"); + let dropped = base.join("dropped"); + let db_dir = tmp_dir("roots-db"); + let db = db_dir.join("index.sqlite"); + touch(&kept.join("a.txt"), b"alpha"); + touch(&kept.join("sub/b.txt"), b"bravo"); + touch(&dropped.join("c.txt"), b"charlie"); + touch(&dropped.join("sub/deep/d.txt"), b"delta"); + + let mut old = base_config(&kept, &db); + old.paths.indexing_paths = vec![ + kept.to_string_lossy().into_owned(), + dropped.to_string_lossy().into_owned(), + ]; + index_once(&db, &old); + let born = created_at(&db); + assert_eq!(paths(&db).len(), 4); + + let mut new = old.clone(); + new.paths.indexing_paths = vec![kept.to_string_lossy().into_owned()]; + let actions = diff_actions(&old, &new); + assert!(!actions.work.reindex, "nothing new to find"); + let (deleted, _) = reconcile(&db, &old, &new); + + assert_eq!(deleted, 2); + assert!( + paths(&db) + .iter() + .all(|p| p.starts_with(kept.to_str().unwrap())), + "only the kept root survives: {:?}", + paths(&db) + ); + assert_eq!(residue(&db), (2, 2), "the kept root keeps its text"); + assert_eq!(orphans(&db), 0, "nothing left behind"); + assert_eq!(created_at(&db), born, "the index was not rebuilt"); + + std::fs::remove_dir_all(&base).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +/// Adding a folder is pure widening: nothing stored is wrong, there is just +/// more to find. The existing root's rows must not even be re-examined. +#[test] +fn adding_a_root_keeps_everything_already_indexed() { + let base = tmp_dir("root-add"); + let first = base.join("first"); + let second = base.join("second"); + let db_dir = tmp_dir("root-add-db"); + let db = db_dir.join("index.sqlite"); + touch(&first.join("a.txt"), b"alpha"); + touch(&second.join("b.txt"), b"bravo"); + + let old = base_config(&first, &db); + index_once(&db, &old); + let born = created_at(&db); + let before = paths(&db); + assert_eq!(before.len(), 1); + + let mut new = old.clone(); + new.paths.indexing_paths = vec![ + first.to_string_lossy().into_owned(), + second.to_string_lossy().into_owned(), + ]; + let (deleted, recontented) = reconcile(&db, &old, &new); + assert_eq!((deleted, recontented), (0, 0)); + assert_eq!(paths(&db), before); + + index_once(&db, &new); + assert_eq!(paths(&db).len(), 2); + assert_eq!(created_at(&db), born, "the index was not rebuilt"); + + std::fs::remove_dir_all(&base).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +/// Narrowing the content filter costs the excluded files their text, not +/// their existence: they must stay findable by name. Widening it queues them +/// for extraction again. +#[test] +fn narrowing_content_extensions_keeps_the_file_findable_by_name() { + let root = tmp_dir("content"); + let db_dir = tmp_dir("content-db"); + let db = db_dir.join("index.sqlite"); + touch(&root.join("notes.md"), b"markdown body"); + touch(&root.join("readme.txt"), b"plain body"); + + let old = base_config(&root, &db); + index_once(&db, &old); + let born = created_at(&db); + assert_eq!(residue(&db).0, 2, "both extracted"); + + let mut narrowed = old.clone(); + narrowed.indexing.content_extensions = vec!["txt".into()]; + let (deleted, recontented) = reconcile(&db, &old, &narrowed); + assert_eq!(deleted, 0, "no file left the index"); + assert_eq!(recontented, 1); + assert_eq!(paths(&db).len(), 2, "both rows are still there"); + assert_eq!(residue(&db).0, 1, "only the .txt keeps its postings"); + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'markdown'" + ), + 0 + ); + assert_eq!( + count(&db, "SELECT COUNT(*) FROM files WHERE content_state = 3"), + 1, + "the excluded file is parked, not pending" + ); + + // Widening again queues it for another extraction, which the next run + // performs. + let (deleted, recontented) = reconcile(&db, &narrowed, &old); + assert_eq!(deleted, 0); + assert_eq!(recontented, 1); + assert_eq!( + count(&db, "SELECT COUNT(*) FROM files WHERE content_state = 0"), + 1, + "pending again" + ); + index_once(&db, &old); + assert_eq!(residue(&db).0, 2, "its text came back"); + assert_eq!(created_at(&db), born, "the index was never rebuilt"); + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +/// `store_text_for_snippets` off throws the blobs away and keeps full-text +/// search working; on again re-extracts, because the text of files already +/// indexed was never kept. +#[test] +fn store_text_toggles_without_losing_full_text_search() { + let root = tmp_dir("store-text"); + let db_dir = tmp_dir("store-text-db"); + let db = db_dir.join("index.sqlite"); + touch(&root.join("a.txt"), b"alpha unique-token"); + + let on = base_config(&root, &db); + index_once(&db, &on); + let born = created_at(&db); + assert_eq!(residue(&db).1, 1, "text stored"); + + let mut off = on.clone(); + off.processing.store_text_for_snippets = false; + reconcile(&db, &on, &off); + assert_eq!(residue(&db).1, 0, "blobs dropped"); + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH 'unique'" + ), + 1, + "full-text search is unaffected" + ); + + reconcile(&db, &off, &on); + index_once(&db, &on); + assert_eq!(residue(&db).1, 1, "text re-extracted"); + assert_eq!(created_at(&db), born, "the index was never rebuilt"); + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +/// Turning hidden files off must reach entries several levels inside a hidden +/// directory, not just the dot-name itself. +#[test] +fn turning_hidden_files_off_prunes_whole_hidden_subtrees() { + let root = tmp_dir("hidden"); + let db_dir = tmp_dir("hidden-db"); + let db = db_dir.join("index.sqlite"); + touch(&root.join("visible.txt"), b"alpha"); + touch(&root.join(".config/app/settings.txt"), b"bravo"); + touch(&root.join(".dotfile"), b"charlie"); + + let mut on = base_config(&root, &db); + on.indexing.include_hidden = true; + index_once(&db, &on); + let born = created_at(&db); + assert_eq!(paths(&db).len(), 3); + + let mut off = on.clone(); + off.indexing.include_hidden = false; + let (deleted, _) = reconcile(&db, &on, &off); + assert_eq!(deleted, 2); + assert_eq!( + paths(&db) + .iter() + .map(|p| Path::new(p) + .file_name() + .unwrap() + .to_string_lossy() + .into_owned()) + .collect::>(), + vec!["visible.txt"] + ); + assert_eq!(residue(&db), (1, 1)); + assert_eq!(orphans(&db), 0); + assert_eq!(created_at(&db), born, "the index was not rebuilt"); + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +/// A followed symlink's target is stored under its own canonical path, which +/// can be outside every configured root. Such a row has no owning root and +/// therefore no filtering rules that could be applied to it — a prune that +/// tested it anyway would delete it on every config change and the next run +/// would put it straight back, forever. +/// +/// The patterns here are the hostile half: `**` and `../*` describe the whole +/// filesystem, and one names the neighbour tree outright. None of them may +/// reach a row the scan never visits. +#[cfg(unix)] +#[test] +fn a_symlink_target_outside_every_root_survives_a_prune() { + let base = tmp_dir("alias"); + let indexed = base.join("indexed"); + let neighbour = base.join("neighbour"); + let db_dir = tmp_dir("alias-db"); + let db = db_dir.join("index.sqlite"); + touch(&indexed.join("a.txt"), b"alpha"); + touch(&indexed.join("sub/b.log"), b"bravo"); + touch(&neighbour.join("target.txt"), b"charlie outside"); + std::os::unix::fs::symlink(neighbour.join("target.txt"), indexed.join("link.txt")).unwrap(); + + let mut old = base_config(&indexed, &db); + old.indexing.follow_symlinks = true; + index_once(&db, &old); + let born = created_at(&db); + let target = neighbour.join("target.txt").to_string_lossy().into_owned(); + assert!( + paths(&db).contains(&target), + "the alias was indexed under the target's own path: {:?}", + paths(&db) + ); + + let mut new = old.clone(); + new.indexing.ignore_patterns = vec![ + "*.log".into(), + "**".into(), + "../*".into(), + neighbour.join("*").to_string_lossy().into_owned(), + ]; + reconcile(&db, &old, &new); + + // `**` matches every component, so everything under the configured root + // goes — and only that. The target, which lives outside it, stays. + assert_eq!(paths(&db), vec![target]); + assert_eq!(orphans(&db), 0, "nothing left behind"); + assert_eq!(created_at(&db), born, "the index was not rebuilt"); + + std::fs::remove_dir_all(&base).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +/// Turning symlink following off leaves rows for targets that live outside +/// every root, which no walk and no stale sweep ever reaches — the reason +/// this setting used to force a wipe. +#[cfg(unix)] +#[test] +fn turning_symlinks_off_reaches_targets_outside_the_roots() { + let base = tmp_dir("links-off"); + let indexed = base.join("indexed"); + let neighbour = base.join("neighbour"); + let db_dir = tmp_dir("links-off-db"); + let db = db_dir.join("index.sqlite"); + touch(&indexed.join("a.txt"), b"alpha"); + touch(&neighbour.join("target.txt"), b"bravo outside"); + std::os::unix::fs::symlink(neighbour.join("target.txt"), indexed.join("link.txt")).unwrap(); + + let mut on = base_config(&indexed, &db); + on.indexing.follow_symlinks = true; + index_once(&db, &on); + let born = created_at(&db); + assert_eq!(paths(&db).len(), 2); + + let mut off = on.clone(); + off.indexing.follow_symlinks = false; + let (deleted, _) = reconcile(&db, &on, &off); + + assert_eq!(deleted, 1); + assert_eq!(paths(&db), vec![indexed.join("a.txt").to_string_lossy()]); + assert_eq!(orphans(&db), 0); + assert_eq!(created_at(&db), born, "the index was not rebuilt"); + + std::fs::remove_dir_all(&base).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +/// A config changed while the app was not running is reconciled by the next +/// run, from the `config_validation` record of what the index was built with +/// — the only thing that knows a root was dropped. +#[test] +fn a_run_reconciles_a_config_edited_while_it_was_closed() { + let base = tmp_dir("offline"); + let kept = base.join("kept"); + let dropped = base.join("dropped"); + let db_dir = tmp_dir("offline-db"); + let db = db_dir.join("index.sqlite"); + touch(&kept.join("a.txt"), b"alpha"); + touch(&dropped.join("b.txt"), b"bravo"); + touch(&dropped.join("sub/c.txt"), b"charlie"); + + let mut old = base_config(&kept, &db); + old.paths.indexing_paths = vec![ + kept.to_string_lossy().into_owned(), + dropped.to_string_lossy().into_owned(), + ]; + index_once(&db, &old); + let born = created_at(&db); + assert_eq!(paths(&db).len(), 3); + + // No reconcile() here: the edit happened with nothing running, so the run + // itself has to notice. + let mut new = old.clone(); + new.paths.indexing_paths = vec![kept.to_string_lossy().into_owned()]; + index_once(&db, &new); + + assert_eq!(paths(&db), vec![kept.join("a.txt").to_string_lossy()]); + assert_eq!(residue(&db), (1, 1)); + assert_eq!(orphans(&db), 0); + assert_eq!(created_at(&db), born, "the index was not rebuilt"); + + std::fs::remove_dir_all(&base).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} + +/// Reconciling twice must be a no-op the second time. The run-start pass and +/// the coordinator's pass can both fire for one edit, and a plan that is not +/// idempotent would delete rows the walk had just re-added. +#[test] +fn reconciling_is_idempotent() { + let root = tmp_dir("idempotent"); + let db_dir = tmp_dir("idempotent-db"); + let db = db_dir.join("index.sqlite"); + tree(&root); + + let old = base_config(&root, &db); + index_once(&db, &old); + let mut new = old.clone(); + new.indexing.ignore_patterns = vec!["*.log".into()]; + new.indexing.content_extensions = vec!["txt".into()]; + + let first = reconcile(&db, &old, &new); + let after_first = paths(&db); + assert!(first.0 > 0 && first.1 > 0, "the first pass did work"); + + let second = reconcile(&db, &old, &new); + assert_eq!(second, (0, 0), "the second pass found nothing left to do"); + assert_eq!(paths(&db), after_first); + + std::fs::remove_dir_all(&root).ok(); + std::fs::remove_dir_all(&db_dir).ok(); +} diff --git a/crates/quicksearch-gui/Cargo.toml b/crates/quicksearch-gui/Cargo.toml index 4e1d1bd..a600178 100644 --- a/crates/quicksearch-gui/Cargo.toml +++ b/crates/quicksearch-gui/Cargo.toml @@ -21,6 +21,12 @@ path = "src/main.rs" name = "quicksearch-cli" path = "src/cli_main.rs" +[features] +# Scripted self-capture driver used by packaging/capture.sh to regenerate the +# website screenshots and screencasts. Adds no dependencies and is inert +# unless QS_CAPTURE_SCRIPT is set at runtime. +capture = [] + [dependencies] quicksearch-core = { path = "../quicksearch-core" } diff --git a/crates/quicksearch-gui/assets/icons/quicksearch.ico b/crates/quicksearch-gui/assets/icons/quicksearch.ico new file mode 100644 index 0000000..dd56d1e Binary files /dev/null and b/crates/quicksearch-gui/assets/icons/quicksearch.ico differ diff --git a/crates/quicksearch-gui/build.rs b/crates/quicksearch-gui/build.rs new file mode 100644 index 0000000..38ca2e4 --- /dev/null +++ b/crates/quicksearch-gui/build.rs @@ -0,0 +1,247 @@ +//! Build identity for the two binaries. +//! +//! Bakes the commit the tree was built from into `QS_COMMIT`, and on Windows +//! compiles a VERSIONINFO resource so the `.exe` reports a version in +//! Explorer's Properties rather than nothing at all. +//! +//! The version number itself is not handled here: `env!("CARGO_PKG_VERSION")` +//! already carries `[workspace.package] version`, which is the one source of +//! truth the CI tag check, build-deb.sh and build-installer.sh all read. +//! +//! No dependencies on purpose — a build script that pulled a crate in would +//! land in Cargo.lock, and every build in this repo runs `--locked`. + +use std::path::PathBuf; +use std::process::Command; + +/// What the version reads as when there is no git and no `QS_COMMIT` — an +/// unpacked source tarball, say. Never a build failure. +const UNKNOWN: &str = "unknown"; + +/// Abbreviated-hash length, matching `git rev-parse --short=7` and the hashes +/// the forge shows. +const SHORT_LEN: usize = 7; + +fn main() { + let commit = resolve_commit(); + println!("cargo::rustc-env=QS_COMMIT={commit}"); + + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("windows") { + emit_version_resource(&commit); + } +} + +// ------------------------------------------------------------- commit ---- + +/// The short commit hash: `QS_COMMIT` first, then git, then [`UNKNOWN`]. +/// +/// CI sets `QS_COMMIT` from the event's SHA rather than letting this shell out +/// to git, because `actions/checkout` leaves a shallow clone owned by another +/// user — the SHA the runner already knows is both cheaper and more trustworthy +/// than anything read back out of that tree. +fn resolve_commit() -> String { + println!("cargo::rerun-if-env-changed=QS_COMMIT"); + watch_git_head(); + + let supplied = std::env::var("QS_COMMIT").unwrap_or_default(); + let supplied = supplied.trim(); + if !supplied.is_empty() { + if let Some(hash) = short_hash(supplied) { + return hash; + } + println!( + "cargo::warning=QS_COMMIT is not a commit hash ({supplied:?}); \ + falling back to git" + ); + } + git_commit().unwrap_or_else(|| UNKNOWN.to_string()) +} + +/// The first [`SHORT_LEN`] characters, lowercased, or `None` when `raw` is not +/// a hex hash. Accepts a full 40-character SHA (what CI passes) and an already +/// abbreviated one alike. +fn short_hash(raw: &str) -> Option { + if raw.is_empty() || !raw.chars().all(|c| c.is_ascii_hexdigit()) { + return None; + } + // `raw` is ASCII here, so slicing by byte cannot split a character. + Some(raw[..raw.len().min(SHORT_LEN)].to_ascii_lowercase()) +} + +fn git_commit() -> Option { + let out = git(&["rev-parse", &format!("--short={SHORT_LEN}"), "HEAD"])?; + short_hash(out.trim()) +} + +/// Rebuild when HEAD moves. +/// +/// Without this cargo only reruns the script when a file in the package +/// changes, so committing anything outside this crate would leave the previous +/// hash baked into the binary. +fn watch_git_head() { + let Some(git_dir) = git_dir() else { return }; + + let head = git_dir.join("HEAD"); + watch(&head); + // On a branch, HEAD itself only changes on checkout — the ref it names is + // what moves on commit. A detached HEAD holds the hash directly and needs + // nothing more. The reflog covers the case where the branch ref is packed + // and so has no loose file to watch. + if let Ok(contents) = std::fs::read_to_string(&head) { + if let Some(reference) = contents.trim().strip_prefix("ref:") { + watch(&git_dir.join(reference.trim())); + } + } + watch(&git_dir.join("logs").join("HEAD")); +} + +/// Watch `path`, but only if it exists: cargo treats a `rerun-if-changed` path +/// it cannot stat as permanently dirty, which would recompile this crate on +/// every single build. +fn watch(path: &std::path::Path) { + if path.exists() { + println!("cargo::rerun-if-changed={}", path.display()); + } +} + +fn git_dir() -> Option { + let path = PathBuf::from(git(&["rev-parse", "--absolute-git-dir"])?.trim()); + path.is_dir().then_some(path) +} + +/// Run git in the crate directory, `None` on any failure — a missing git, a +/// tree that is not a repository, and a repository git refuses to trust all +/// mean the same thing here. +fn git(args: &[&str]) -> Option { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").ok()?; + let out = Command::new("git") + .arg("-C") + .arg(manifest_dir) + .args(args) + .output() + .ok()?; + out.status + .success() + .then(|| String::from_utf8(out.stdout).ok()) + .flatten() +} + +// ------------------------------------------------ windows VERSIONINFO ---- + +/// The binaries this crate builds, with the description Explorer shows in the +/// Properties dialog and in the details pane. +const BINARIES: [(&str, &str); 2] = [ + ("quicksearch", "QuickSearch"), + ("quicksearch-cli", "QuickSearch terminal search"), +]; + +/// Compile a VERSIONINFO resource per binary and link it in. +/// +/// The string values mirror `packaging/quicksearch.nsi` so the app and the +/// installer that ships it never disagree about who published what. +fn emit_version_resource(commit: &str) { + // rust-toolchain.toml lists x86_64-pc-windows-gnu and nothing else, and + // windres is what compiles a .rc there. An MSVC target would need rc.exe + // and a different invocation, so it is skipped rather than half-supported. + if std::env::var("CARGO_CFG_TARGET_ENV").as_deref() != Ok("gnu") { + return; + } + + let out_dir = PathBuf::from(env("OUT_DIR")); + let version = env("CARGO_PKG_VERSION"); + // Windows insists on exactly four numeric fields. Cargo splits the version + // for us, so unlike build-installer.sh there is no suffix to strip. + let quad = format!( + "{},{},{},0", + env("CARGO_PKG_VERSION_MAJOR"), + env("CARGO_PKG_VERSION_MINOR"), + env("CARGO_PKG_VERSION_PATCH"), + ); + let quad_text = quad.replace(',', "."); + + for (bin, description) in BINARIES { + let rc = format!( + r#"1 VERSIONINFO +FILEVERSION {quad} +PRODUCTVERSION {quad} +FILEOS 0x4L +FILETYPE 0x1L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904B0" + BEGIN + VALUE "CompanyName", "Jeremy " + VALUE "FileDescription", "{description}" + VALUE "FileVersion", "{quad_text}" + VALUE "InternalName", "{bin}" + VALUE "LegalCopyright", "GPL-3.0-or-later" + VALUE "OriginalFilename", "{bin}.exe" + VALUE "ProductName", "QuickSearch" + VALUE "ProductVersion", "{version} ({commit})" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END +"# + ); + + let rc_path = out_dir.join(format!("{bin}.rc")); + let res_path = out_dir.join(format!("{bin}.res")); + std::fs::write(&rc_path, rc).expect("OUT_DIR is writable"); + compile_resource(&rc_path, &res_path); + + // Per-binary, because OriginalFilename differs between the two. Linked + // as a plain object rather than through a static library: nothing + // references a resource by symbol, so an archive member holding one + // would be dropped as unused. + println!("cargo::rustc-link-arg-bin={bin}={}", res_path.display()); + } +} + +fn compile_resource(rc: &std::path::Path, res: &std::path::Path) { + println!("cargo::rerun-if-env-changed=QS_WINDRES"); + let explicit = std::env::var("QS_WINDRES").ok(); + let candidates: Vec<&str> = match &explicit { + Some(tool) => vec![tool.as_str()], + // The cross compiler's windres first, then the plain name for a native + // mingw shell where the tools are unprefixed. + None => vec!["x86_64-w64-mingw32-windres", "windres"], + }; + + let mut attempts = Vec::new(); + for tool in &candidates { + match Command::new(tool) + .arg("-O") + .arg("coff") + .arg(rc) + .arg("-o") + .arg(res) + .output() + { + Ok(out) if out.status.success() => return, + Ok(out) => attempts.push(format!( + "{tool}: exited {} — {}", + out.status, + String::from_utf8_lossy(&out.stderr).trim() + )), + Err(e) => attempts.push(format!("{tool}: {e}")), + } + } + // A hard error, not a warning. A Windows build already needs mingw for the + // linker, so this only fires on a genuinely broken toolchain — and silently + // shipping an .exe with no version is exactly what this exists to prevent. + panic!( + "could not compile the Windows version resource. Install \ + binutils-mingw-w64-x86-64 (or set QS_WINDRES to a resource compiler). \ + Tried:\n {}", + attempts.join("\n ") + ); +} + +fn env(key: &str) -> String { + std::env::var(key).unwrap_or_else(|_| panic!("cargo sets {key} for build scripts")) +} diff --git a/crates/quicksearch-gui/src/app.rs b/crates/quicksearch-gui/src/app.rs index 0844f2d..6d1c659 100644 --- a/crates/quicksearch-gui/src/app.rs +++ b/crates/quicksearch-gui/src/app.rs @@ -25,7 +25,7 @@ use crate::search_tab::SearchTab; use crate::unlock::KeySource; #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Tab { +pub(crate) enum Tab { Search, Manage, Duplicates, @@ -114,6 +114,10 @@ pub struct QuickSearchApp { /// The guard resolved a Quit: let the next close request through. quit_confirmed: bool, config_error: Option, + /// Scripted self-capture driver; `None` unless a `capture` build has + /// `QS_CAPTURE_SCRIPT` set. See [`crate::capture`]. + #[cfg(feature = "capture")] + pub(crate) capture: Option>, } /// The two-step security flow: collect a password (enable/change), derive @@ -209,6 +213,8 @@ impl QuickSearchApp { pending_nav: None, quit_confirmed: false, config_error, + #[cfg(feature = "capture")] + capture: crate::capture::CaptureDriver::from_env(), }) } @@ -267,24 +273,16 @@ impl QuickSearchApp { .set_db_path(new.resolved_database_path()); self.counts = None; } + // Everything the index can reconcile in place — pruning rows a + // narrowed filter put out of scope, re-deciding extracted text, + // walking for files a widened one brought in — the coordinator does + // on its own, in either mode and without asking. Only the three + // settings that leave the stored file unreadable get this far. self.backend.coordinator.apply_config(new.clone()); if actions.requires_rebuild { if self.backend.coordinator.state().mode == IndexMode::Auto { - // Automatic mode is hands-off: reconcile immediately, no - // prompt. Root-only changes need just a full run — the - // walk indexes new roots and the stale sweep drops removed - // ones. Anything else (tokenizer, hashing, filters, hidden - // files) invalidates stored data and gets the real wipe. - let roots_only = { - let mut probe = new.clone(); - probe.paths.indexing_paths = self.cfg.paths.indexing_paths.clone(); - !diff_actions(&self.cfg, &probe).requires_rebuild - }; - if roots_only { - self.backend.coordinator.reindex_now(); - } else { - self.backend.coordinator.rebuild_index(); - } + // Automatic mode is hands-off: wipe and start over. + self.backend.coordinator.rebuild_index(); } else { let changes = self .backend @@ -510,10 +508,16 @@ impl QuickSearchApp { } } - // Right corner: search result count. + // Right corner: build id, then the search result count. In a + // right-to-left layout the first widget added is the rightmost, + // so the version is the fixed anchor and the count grows away + // from it rather than shoving it around. ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + ui.label(egui::RichText::new(crate::version::BUILD_ID).small().weak()) + .on_hover_text(crate::version::BUILD_ID_HINT); if self.tab == Tab::Search { if let Some(label) = self.search.result_count_label() { + ui.label(egui::RichText::new("·").small().weak()); ui.label(egui::RichText::new(label).small().weak()); } } @@ -551,7 +555,7 @@ impl QuickSearchApp { ui.label("These settings differ from what the index was built with:"); ui.add_space(4.0); if changes.is_empty() { - ui.monospace("indexing settings changed"); + ui.monospace("the index cannot be read with the new settings"); } for change in &changes { ui.strong(format!("{}:", change.key)); @@ -568,8 +572,10 @@ impl QuickSearchApp { } ui.label( egui::RichText::new( - "A full rebuild applies them everywhere. Until then, existing \ - entries keep the old settings.", + "Unlike folders, filters and hidden files — which are applied \ + to the existing index in place — these cannot be, so the \ + index has to be built again. Until it is, existing entries \ + keep the old settings.", ) .small() .weak(), @@ -1077,6 +1083,44 @@ impl QuickSearchApp { } } +/// State the scripted capture driver steers and waits on; see +/// [`crate::capture`]. +#[cfg(feature = "capture")] +impl QuickSearchApp { + /// Route through the same pending-nav path a click takes: `complete_nav` + /// resolves it at the end of this frame, so the Duplicates auto-scan + /// still fires and the unsaved-changes guard keeps its invariants + /// (capture scenarios never dirty an editor, so the guard never prompts). + pub(crate) fn capture_request_tab(&mut self, tab: Tab) { + if self.pending_nav.is_none() { + self.pending_nav = Some(NavIntent::SwitchTab(tab)); + } + } + + pub(crate) fn capture_indexing_status(&self) -> IndexingStatus { + self.backend.coordinator.state().activity + } + + pub(crate) fn capture_search_settled(&self) -> bool { + self.search.capture_settled() + } + + pub(crate) fn capture_dups_done(&self) -> bool { + matches!(self.dups.state, DupState::Loaded(_) | DupState::Error(_)) + } + + /// Empty the query through the same edit path typing uses, so the empty + /// search runs and the results table clears. + pub(crate) fn capture_clear_query(&mut self) { + self.search.query.clear(); + self.search.pending_edit = Some(Instant::now()); + } + + pub(crate) fn capture_focus_search(&mut self) { + self.search.capture_focus(); + } +} + /// Overwrite the fields a config draft must never carry back. /// /// Both are live state the GUI changes through their own controls — the @@ -1213,6 +1257,11 @@ fn clamp_scale(scale: f32) -> f32 { impl eframe::App for QuickSearchApp { fn update(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) { + // First, so a command's effect is fully rendered before the next one + // starts, and `previous_tab` below sees pre-navigation state. + #[cfg(feature = "capture")] + self.capture_tick(ctx); + self.drain_events(); self.tick_debounce(ctx); diff --git a/crates/quicksearch-gui/src/capture.rs b/crates/quicksearch-gui/src/capture.rs new file mode 100644 index 0000000..fccbd34 --- /dev/null +++ b/crates/quicksearch-gui/src/capture.rs @@ -0,0 +1,1002 @@ +//! Scripted self-capture (feature `capture`): the app drives itself through +//! a scenario so `packaging/capture.sh` can regenerate the website +//! screenshots and screencasts as the software changes. +//! +//! With `QS_CAPTURE_SCRIPT` set, a [`CaptureDriver`] runs one command at a +//! time: it injects keystrokes as real `egui::Event::Text` input (so typing +//! goes through the same debounce and streaming-search path a user's would), +//! switches tabs through the same pending-nav route a click takes, waits on +//! live indexer/search/duplicates state, saves pixel-perfect screenshots via +//! `ViewportCommand::Screenshot`, and records video the same way: frames +//! read back from the GL framebuffer, piped to `ffmpeg` as raw video. +//! +//! Everything is captured from inside the app on purpose. Screen-grabbing +//! (`x11grab` and friends) depends on the display server — it records black +//! frames from a rootless XWayland, needs portals on Wayland proper, and +//! picks up whatever overlaps the window — while the framebuffer readback +//! behind `ViewportCommand::Screenshot` works identically on X11, Wayland +//! and Windows, and sees nothing but the app. +//! +//! Driving from the inside is what keeps the captures maintainable: there +//! are no screen coordinates to rot when the layout changes, and no external +//! automation tooling to install. The scenario file is the only thing to +//! edit when re-choreographing. +//! +//! Exit codes, for the orchestrator: 2 script parse error, 3 wait timeout, +//! 4 screenshot/recording I/O failure. + +use std::io::Write as _; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use quicksearch_core::indexing::IndexingStatus; + +use crate::app::{QuickSearchApp, Tab}; + +// --------------------------------------------------------------------------- +// Scenario script +// --------------------------------------------------------------------------- + +/// One scenario command. Line-based script, `#` comments outside strings: +/// +/// ```text +/// wait_ms INT +/// type "STRING" [cps FLOAT] # default 7 chars/sec +/// clear_query | focus_search +/// window INT INT # resize to width x height, in the same +/// # logical points as the startup size +/// tab (search|manage|duplicates|logs|help) +/// wait_index_running [max INT] # caps in ms; a capped wait cannot fail +/// wait_index_idle [max INT] +/// wait_search_done [max INT] +/// wait_dups_done [max INT] +/// record_start NAME | record_stop # NAME: [A-Za-z0-9._-]+, no separators +/// screenshot NAME +/// quit +/// ``` +#[derive(Debug, Clone, PartialEq)] +pub(crate) enum Cmd { + WaitMs(u64), + Type { text: String, cps: f32 }, + ClearQuery, + FocusSearch, + Window { w: f32, h: f32 }, + Tab(Tab), + WaitIndexRunning { max_ms: Option }, + WaitIndexIdle { max_ms: Option }, + WaitSearchDone { max_ms: Option }, + WaitDupsDone { max_ms: Option }, + RecordStart(String), + RecordStop, + Screenshot(String), + Quit, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct ParseError { + /// 1-based line in the scenario file. + pub line: usize, + pub msg: String, +} + +#[derive(Debug, PartialEq, Eq)] +enum Token { + Word(String), + Str(String), +} + +/// Split one line into bare words and quoted strings. `#` starts a comment +/// except inside a string; `\"` and `\\` are the only escapes. +fn tokenize(line: &str, line_no: usize) -> Result, ParseError> { + let err = |msg: String| ParseError { line: line_no, msg }; + let mut tokens = Vec::new(); + let mut chars = line.chars().peekable(); + while let Some(&c) = chars.peek() { + if c.is_whitespace() { + chars.next(); + } else if c == '#' { + break; + } else if c == '"' { + chars.next(); + let mut s = String::new(); + loop { + match chars.next() { + None => return Err(err("unclosed string".to_string())), + Some('"') => break, + Some('\\') => match chars.next() { + Some(e @ ('"' | '\\')) => s.push(e), + Some(e) => return Err(err(format!("unknown escape \\{e}"))), + None => return Err(err("unclosed string".to_string())), + }, + Some(other) => s.push(other), + } + } + tokens.push(Token::Str(s)); + } else { + let mut w = String::new(); + while let Some(&c) = chars.peek() { + if c.is_whitespace() || c == '#' { + break; + } + if c == '"' { + return Err(err("quotes may only start a token".to_string())); + } + w.push(c); + chars.next(); + } + tokens.push(Token::Word(w)); + } + } + Ok(tokens) +} + +pub(crate) fn parse_script(src: &str) -> Result, ParseError> { + let mut cmds = Vec::new(); + for (i, line) in src.lines().enumerate() { + let line_no = i + 1; + let tokens = tokenize(line, line_no)?; + if let Some(cmd) = parse_line(&tokens, line_no)? { + cmds.push(cmd); + } + } + Ok(cmds) +} + +fn parse_line(tokens: &[Token], line_no: usize) -> Result, ParseError> { + let err = |msg: String| ParseError { line: line_no, msg }; + let Some(first) = tokens.first() else { + return Ok(None); + }; + let Token::Word(name) = first else { + return Err(err( + "a line must start with a command, not a string".to_string() + )); + }; + + let rest = &mut tokens[1..].iter(); + let cmd = match name.as_str() { + "wait_ms" => Cmd::WaitMs(parse_int( + "duration", + next_word(rest, line_no, "duration in ms")?, + line_no, + )?), + "type" => { + let text = match rest.next() { + Some(Token::Str(s)) => s.clone(), + Some(Token::Word(_)) => { + return Err(err("the text to type must be quoted".to_string())); + } + None => return Err(err("missing text to type".to_string())), + }; + let cps = match rest.next() { + None => 7.0, + Some(Token::Word(w)) if w == "cps" => { + let v = next_word(rest, line_no, "cps value")?; + let cps: f32 = v + .parse() + .map_err(|_| err(format!("invalid cps {v:?}: expected a number")))?; + if !(cps.is_finite() && cps > 0.0) { + return Err(err("cps must be positive".to_string())); + } + cps + } + Some(other) => return Err(err(format!("expected `cps`, found {other:?}"))), + }; + Cmd::Type { text, cps } + } + "clear_query" => Cmd::ClearQuery, + "focus_search" => Cmd::FocusSearch, + "window" => { + let w = parse_int("width", next_word(rest, line_no, "width in points")?, line_no)?; + let h = parse_int("height", next_word(rest, line_no, "height in points")?, line_no)?; + if w == 0 || h == 0 { + return Err(err("window dimensions must be positive".to_string())); + } + Cmd::Window { + w: w as f32, + h: h as f32, + } + } + "tab" => Cmd::Tab(match next_word(rest, line_no, "tab name")? { + "search" => Tab::Search, + "manage" => Tab::Manage, + "duplicates" => Tab::Duplicates, + "logs" => Tab::Logs, + "help" => Tab::Help, + other => { + return Err(err(format!( + "unknown tab {other:?}: expected search, manage, duplicates, logs or help" + ))); + } + }), + "wait_index_running" | "wait_index_idle" | "wait_search_done" | "wait_dups_done" => { + let max_ms = match rest.next() { + None => None, + Some(Token::Word(w)) if w == "max" => Some(parse_int( + "max", + next_word(rest, line_no, "max value in ms")?, + line_no, + )?), + Some(other) => return Err(err(format!("expected `max`, found {other:?}"))), + }; + match name.as_str() { + "wait_index_running" => Cmd::WaitIndexRunning { max_ms }, + "wait_index_idle" => Cmd::WaitIndexIdle { max_ms }, + "wait_search_done" => Cmd::WaitSearchDone { max_ms }, + _ => Cmd::WaitDupsDone { max_ms }, + } + } + "record_start" => Cmd::RecordStart(parse_name( + next_word(rest, line_no, "output name")?, + line_no, + )?), + "record_stop" => Cmd::RecordStop, + "screenshot" => Cmd::Screenshot(parse_name( + next_word(rest, line_no, "output name")?, + line_no, + )?), + "quit" => Cmd::Quit, + other => return Err(err(format!("unknown command {other:?}"))), + }; + + if let Some(extra) = rest.next() { + return Err(err(format!( + "unexpected {extra:?} after a complete command" + ))); + } + Ok(Some(cmd)) +} + +fn next_word<'a>( + rest: &mut std::slice::Iter<'a, Token>, + line_no: usize, + what: &str, +) -> Result<&'a str, ParseError> { + match rest.next() { + Some(Token::Word(w)) => Ok(w.as_str()), + Some(Token::Str(_)) => Err(ParseError { + line: line_no, + msg: format!("expected {what}, found a string"), + }), + None => Err(ParseError { + line: line_no, + msg: format!("missing {what}"), + }), + } +} + +fn parse_int(what: &str, w: &str, line_no: usize) -> Result { + w.parse::().map_err(|_| ParseError { + line: line_no, + msg: format!("invalid {what} {w:?}: expected an integer"), + }) +} + +/// Output names stay inside `$QS_CAPTURE_OUT`: a plain filename stem, the +/// driver appends the extension. +fn parse_name(w: &str, line_no: usize) -> Result { + let ok = !w.is_empty() + && w.chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-')); + if ok { + Ok(w.to_string()) + } else { + Err(ParseError { + line: line_no, + msg: format!("invalid name {w:?}: use only letters, digits, `.`, `_`, `-`"), + }) + } +} + +// --------------------------------------------------------------------------- +// Driver +// --------------------------------------------------------------------------- + +/// In-flight keystroke injection for one `type` command. +struct Typing { + chars: std::vec::IntoIter, + /// Nominal seconds per keystroke; each interval is jittered ±25%. + interval_s: f32, + due: Instant, + typed: u64, +} + +/// Recording frame rate. Readback requests are paced to this, and +/// [`CaptureDriver::feed_frame`] duplicates or drops frames so the encoded +/// timeline tracks wall time even when frames arrive unevenly. +const RECORD_FPS: u32 = 30; + +/// A recording in progress: paced framebuffer readbacks piped to ffmpeg. +struct Recorder { + path: PathBuf, + /// Nominal time per frame (1 / [`RECORD_FPS`]). + interval: Duration, + /// When to ask for the next framebuffer readback. + next_request: Instant, + /// Spawned when the first frame arrives — only then are the exact pixel + /// dimensions known, and ffmpeg needs them up front for raw video. + encoder: Option, +} + +struct Encoder { + child: Child, + size: [usize; 2], + /// When the first frame arrived; the video's t = 0. + started: Instant, + frames_written: u64, +} + +/// Marker in a screenshot's `UserData` for the `screenshot` command. +struct ShotTag; + +/// Marker in a screenshot's `UserData` for one recording frame. +struct FrameTag; + +pub(crate) struct CaptureDriver { + cmds: Vec, + /// Index of the command currently executing. + pc: usize, + /// When `cmds[pc]`'s one-shot enter action ran; `None` before it has. + cmd_started: Option, + typing: Option, + /// Screenshot in flight: requested, PNG not yet written. + shot: Option, + rec: Option, + out_dir: PathBuf, + /// Set by `quit`; the app drops the driver once it is. + pub(crate) finished: bool, +} + +impl CaptureDriver { + /// `None` unless `QS_CAPTURE_SCRIPT` names a scenario. A script that + /// cannot be read or parsed exits immediately — an automation harness + /// wants a loud parse error, not a window that sits there doing nothing. + pub(crate) fn from_env() -> Option> { + let script = std::env::var_os("QS_CAPTURE_SCRIPT")?; + let src = match std::fs::read_to_string(&script) { + Ok(src) => src, + Err(e) => { + eprintln!( + "capture: cannot read {}: {}", + Path::new(&script).display(), + e + ); + std::process::exit(2); + } + }; + let cmds = match parse_script(&src) { + Ok(cmds) => cmds, + Err(e) => { + eprintln!( + "capture: {}:{}: {}", + Path::new(&script).display(), + e.line, + e.msg + ); + std::process::exit(2); + } + }; + let out_dir = std::env::var_os("QS_CAPTURE_OUT") + .map(PathBuf::from) + .unwrap_or_else(|| PathBuf::from(".")); + if let Err(e) = std::fs::create_dir_all(&out_dir) { + eprintln!("capture: cannot create {}: {}", out_dir.display(), e); + std::process::exit(4); + } + Some(Box::new(CaptureDriver { + cmds, + pc: 0, + cmd_started: None, + typing: None, + shot: None, + rec: None, + out_dir, + finished: false, + })) + } + + /// Advance the script by at most one command per frame. Runs at the top + /// of `update()`, so a command's effect is on screen before the next + /// command starts. + pub(crate) fn tick(&mut self, app: &mut QuickSearchApp, ctx: &egui::Context) { + if self.finished { + return; + } + // The app repaints on demand when idle; the driver needs frames to + // keep its own clock ticking, and a steady cadence is also what + // keeps recorded footage smooth. + ctx.request_repaint_after(Duration::from_millis(15)); + + // Pump the recording: one framebuffer readback per frame interval, + // harvested in `on_raw_input` a frame later. + if let Some(rec) = self.rec.as_mut() { + let now = Instant::now(); + if now >= rec.next_request { + ctx.send_viewport_cmd(egui::ViewportCommand::Screenshot(egui::UserData::new( + FrameTag, + ))); + rec.next_request = now + rec.interval; + } + } + + let Some(cmd) = self.cmds.get(self.pc).cloned() else { + self.quit(ctx); + return; + }; + let started = match self.cmd_started { + Some(t) => t, + None => { + let now = Instant::now(); + self.cmd_started = Some(now); + self.enter(&cmd, app, ctx); + now + } + }; + if self.finished { + return; // `quit` just ran + } + + let elapsed = started.elapsed(); + if self.done(&cmd, app, elapsed) { + self.pc += 1; + self.cmd_started = None; + } else if elapsed >= Duration::from_millis(hard_timeout_ms(&cmd)) { + eprintln!( + "capture: command #{} ({:?}) timed out after {:.1?}", + self.pc + 1, + cmd, + elapsed + ); + self.stop_recorder(); + std::process::exit(3); + } + } + + /// One-shot action when a command starts. + fn enter(&mut self, cmd: &Cmd, app: &mut QuickSearchApp, ctx: &egui::Context) { + match cmd { + Cmd::WaitMs(_) + | Cmd::WaitIndexRunning { .. } + | Cmd::WaitIndexIdle { .. } + | Cmd::WaitSearchDone { .. } + | Cmd::WaitDupsDone { .. } + | Cmd::RecordStop => {} + Cmd::Type { text, cps } => { + let interval_s = 1.0 / cps; + self.typing = Some(Typing { + chars: text.chars().collect::>().into_iter(), + interval_s, + due: Instant::now() + Duration::from_secs_f32(interval_s), + typed: 0, + }); + } + Cmd::ClearQuery => app.capture_clear_query(), + Cmd::FocusSearch => app.capture_focus_search(), + Cmd::Window { w, h } => { + // Scenario sizes use the same logical points as the startup + // size in main.rs, so `window 1000 700` restores it exactly. + // ViewportCommand sizes are in egui points, which fold in the + // UI zoom ([ui] scale) — divide it back out. The app's own + // floor is 640x400; lower it first so compact clip sizes + // actually take effect. The resize lands asynchronously (the + // window manager has the last word), so scenarios follow + // this with a wait_ms before recording. + let size = egui::vec2(*w, *h) / ctx.zoom_factor(); + ctx.send_viewport_cmd(egui::ViewportCommand::MinInnerSize(size)); + ctx.send_viewport_cmd(egui::ViewportCommand::InnerSize(size)); + } + Cmd::Tab(tab) => app.capture_request_tab(*tab), + Cmd::RecordStart(name) => { + self.rec = Some(Recorder { + path: self.out_dir.join(format!("{name}.cap.mkv")), + interval: Duration::from_secs_f64(1.0 / f64::from(RECORD_FPS)), + next_request: Instant::now(), + encoder: None, + }); + } + Cmd::Screenshot(name) => { + self.shot = Some(self.out_dir.join(format!("{name}.png"))); + ctx.send_viewport_cmd(egui::ViewportCommand::Screenshot(egui::UserData::new( + ShotTag, + ))); + } + Cmd::Quit => self.quit(ctx), + } + if matches!(cmd, Cmd::RecordStop) { + self.stop_recorder(); + } + } + + /// Whether the current command has finished. Wait conditions treat a + /// `max` cap as "done anyway": the caps exist to bound clip length and to + /// tolerate a state change that happened before the wait began. + fn done(&self, cmd: &Cmd, app: &QuickSearchApp, elapsed: Duration) -> bool { + let capped = + |max_ms: &Option| max_ms.is_some_and(|ms| elapsed >= Duration::from_millis(ms)); + match cmd { + Cmd::WaitMs(ms) => elapsed >= Duration::from_millis(*ms), + Cmd::Type { .. } => self.typing.is_none(), + Cmd::ClearQuery + | Cmd::FocusSearch + | Cmd::Window { .. } + | Cmd::Tab(_) + | Cmd::RecordStart(_) + | Cmd::RecordStop + | Cmd::Quit => true, + Cmd::WaitIndexRunning { max_ms } => { + capped(max_ms) + || matches!( + app.capture_indexing_status(), + IndexingStatus::Running { .. } + | IndexingStatus::Stopping + | IndexingStatus::Optimizing + ) + } + Cmd::WaitIndexIdle { max_ms } => { + let status = app.capture_indexing_status(); + if let IndexingStatus::Error(e) = &status { + eprintln!("capture: indexing reported an error: {e}"); + } + capped(max_ms) || matches!(status, IndexingStatus::Idle | IndexingStatus::Error(_)) + } + Cmd::WaitSearchDone { max_ms } => capped(max_ms) || app.capture_search_settled(), + Cmd::WaitDupsDone { max_ms } => capped(max_ms) || app.capture_dups_done(), + Cmd::Screenshot(_) => self.shot.is_none(), + } + } + + fn quit(&mut self, ctx: &egui::Context) { + self.stop_recorder(); + ctx.send_viewport_cmd(egui::ViewportCommand::Close); + self.finished = true; + } + + /// Runs in `raw_input_hook`, before egui processes this frame's input: + /// due keystrokes are appended as `Event::Text` (landing in the focused + /// search box exactly as real typing would), and a finished screenshot is + /// harvested from the incoming events and written out. + pub(crate) fn on_raw_input(&mut self, raw: &mut egui::RawInput) { + let mut drained = false; + if let Some(t) = self.typing.as_mut() { + let now = Instant::now(); + while t.due <= now { + let Some(c) = t.chars.next() else { + drained = true; + break; + }; + raw.events.push(egui::Event::Text(c.to_string())); + t.typed += 1; + let interval = t.interval_s * jitter(t.typed); + t.due += Duration::from_secs_f32(interval); + } + } + if drained { + self.typing = None; + } + + // One pass over the incoming events harvests both kinds of + // framebuffer readback: recording frames and still screenshots. + for event in &raw.events { + let egui::Event::Screenshot { + user_data, image, .. + } = event + else { + continue; + }; + let Some(data) = user_data.data.as_ref() else { + continue; + }; + if data.downcast_ref::().is_some() { + if let Err(e) = self.feed_frame(image) { + eprintln!("capture: recording failed: {e}"); + std::process::exit(4); + } + } else if data.downcast_ref::().is_some() { + if let Some(path) = self.shot.take() { + if let Err(e) = write_png(image, &path) { + eprintln!("capture: cannot write {}: {}", path.display(), e); + std::process::exit(4); + } + } + } + } + } + + // -- recording ---------------------------------------------------------- + + /// Append one readback to the recording, spawning the encoder on the + /// first frame (which fixes the dimensions). The frame is written as many + /// times as whole intervals have elapsed since the recording began — + /// duplicated to catch up after a slow frame, dropped when readbacks + /// outpace [`RECORD_FPS`] — so the video's length tracks wall time. + fn feed_frame(&mut self, image: &egui::ColorImage) -> Result<(), String> { + let Some(rec) = self.rec.as_mut() else { + return Ok(()); // stopped while this readback was in flight + }; + if rec.encoder.is_none() { + rec.encoder = Some(Encoder { + child: spawn_encoder(&rec.path, image.size)?, + size: image.size, + started: Instant::now(), + frames_written: 0, + }); + } + let encoder = rec.encoder.as_mut().expect("spawned above"); + if encoder.size != image.size { + return Err(format!( + "window resized mid-recording ({:?} -> {:?})", + encoder.size, image.size + )); + } + let elapsed = encoder.started.elapsed().as_secs_f64(); + let target = (elapsed / rec.interval.as_secs_f64()).floor() as u64 + 1; + let stdin = encoder + .child + .stdin + .as_mut() + .ok_or("the encoder's stdin is gone")?; + while encoder.frames_written < target { + stdin + .write_all(image.as_raw()) + .map_err(|e| format!("writing to ffmpeg: {e}"))?; + encoder.frames_written += 1; + } + Ok(()) + } + + /// Close the encoder's stdin — end-of-input, on which ffmpeg encodes the + /// tail and exits — then wait, with a kill as backstop. Blocking the UI + /// thread here is fine: the recording has already ended, so there is + /// nothing to miss on screen. + fn stop_recorder(&mut self) { + let Some(rec) = self.rec.take() else { + return; + }; + let Some(mut encoder) = rec.encoder else { + eprintln!( + "capture: recording {} captured no frames", + rec.path.display() + ); + std::process::exit(4); + }; + drop(encoder.child.stdin.take()); + let deadline = Instant::now() + Duration::from_secs(10); + loop { + match encoder.child.try_wait() { + Ok(Some(_)) => break, + Ok(None) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(50)); + } + _ => { + let _ = encoder.child.kill(); + let _ = encoder.child.wait(); + break; + } + } + } + let ok = std::fs::metadata(&rec.path) + .map(|m| m.len() > 0) + .unwrap_or(false); + if !ok { + eprintln!( + "capture: recording {} is missing or empty", + rec.path.display() + ); + std::process::exit(4); + } + } +} + +/// ffmpeg encoding raw RGBA frames from stdin into a *lossless* intermediate; +/// `capture.sh` transcodes to VP9 afterwards. Realtime VP9 at capture quality +/// drops frames, while `libx264rgb -qp 0 -preset ultrafast` is cheap, keeps +/// text crisp (no chroma subsampling at capture time), and mkv survives an +/// unclean stop. +fn spawn_encoder(path: &Path, size: [usize; 2]) -> Result { + Command::new("ffmpeg") + .args(["-hide_banner", "-loglevel", "error", "-y"]) + .args(["-f", "rawvideo", "-pixel_format", "rgba"]) + .args(["-video_size", &format!("{}x{}", size[0], size[1])]) + .args(["-framerate", &RECORD_FPS.to_string()]) + .args(["-i", "pipe:0"]) + .args([ + "-c:v", + "libx264rgb", + "-qp", + "0", + "-preset", + "ultrafast", + "-g", + "60", + ]) + .arg(path) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .spawn() + .map_err(|e| format!("failed to spawn ffmpeg: {e}")) +} + +/// Deterministic per-keystroke pacing factor in [0.75, 1.25] — human enough +/// on video, identical on every run, and no rand dependency. +fn jitter(keystroke: u64) -> f32 { + let mut x = keystroke + .wrapping_mul(0x9E37_79B9_7F4A_7C15) + .wrapping_add(1); + x ^= x >> 33; + x = x.wrapping_mul(0xFF51_AFD7_ED55_8CCD); + x ^= x >> 33; + 0.75 + (x % 1000) as f32 / 1000.0 * 0.5 +} + +/// Ceiling after which a wait without `max` aborts the run: generous enough +/// for a full index of the demo tree, small enough that a wedged run fails +/// instead of hanging the orchestrator. +fn hard_timeout_ms(cmd: &Cmd) -> u64 { + match cmd { + // Always finish on their own; the bound is just a backstop. + Cmd::WaitMs(ms) => ms + 60_000, + Cmd::Type { text, cps } => (text.chars().count() as f32 / cps * 1000.0) as u64 + 30_000, + Cmd::ClearQuery + | Cmd::FocusSearch + | Cmd::Window { .. } + | Cmd::Tab(_) + | Cmd::RecordStart(_) + | Cmd::RecordStop + | Cmd::Quit => 10_000, + Cmd::Screenshot(_) => 10_000, + Cmd::WaitIndexRunning { .. } => 120_000, + Cmd::WaitIndexIdle { .. } => 1_800_000, + Cmd::WaitSearchDone { .. } => 60_000, + Cmd::WaitDupsDone { .. } => 300_000, + } +} + +/// The GL framebuffer is opaque, so premultiplied and straight alpha agree +/// and the pixels can be reused as-is. eframe's icon helper brings the PNG +/// encoder — no extra dependency. +fn write_png(image: &egui::ColorImage, path: &Path) -> Result<(), String> { + use eframe::icon_data::IconDataExt as _; + let icon = egui::IconData { + width: image.size[0] as u32, + height: image.size[1] as u32, + rgba: image.as_raw().to_vec(), + }; + std::fs::write(path, icon.to_png_bytes()?).map_err(|e| e.to_string()) +} + +// --------------------------------------------------------------------------- +// App glue +// --------------------------------------------------------------------------- + +/// Take/call/put wrappers: the driver borrows the whole app mutably, so it +/// cannot stay a field of it during the call. +impl QuickSearchApp { + pub(crate) fn capture_tick(&mut self, ctx: &egui::Context) { + let Some(mut driver) = self.capture.take() else { + return; + }; + driver.tick(self, ctx); + if !driver.finished { + self.capture = Some(driver); + } + } + + pub(crate) fn capture_raw_input(&mut self, raw: &mut egui::RawInput) { + let Some(mut driver) = self.capture.take() else { + return; + }; + driver.on_raw_input(raw); + self.capture = Some(driver); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse_one(line: &str) -> Cmd { + let cmds = parse_script(line).expect("line should parse"); + assert_eq!(cmds.len(), 1, "expected exactly one command from {line:?}"); + cmds.into_iter().next().unwrap() + } + + fn parse_err(src: &str) -> ParseError { + parse_script(src).expect_err("script should be rejected") + } + + #[test] + fn every_command_parses() { + let script = r#" + wait_ms 250 + type "hello world" + type "fast" cps 30 + clear_query + focus_search + window 500 350 + tab search + tab manage + tab duplicates + tab logs + tab help + wait_index_running + wait_index_running max 15000 + wait_index_idle max 13000 + wait_search_done max 6000 + wait_dups_done + record_start manage-indexing + record_stop + screenshot query-highlight.v2 + quit + "#; + let cmds = parse_script(script).expect("script should parse"); + assert_eq!( + cmds, + vec![ + Cmd::WaitMs(250), + Cmd::Type { + text: "hello world".to_string(), + cps: 7.0 + }, + Cmd::Type { + text: "fast".to_string(), + cps: 30.0 + }, + Cmd::ClearQuery, + Cmd::FocusSearch, + Cmd::Window { w: 500.0, h: 350.0 }, + Cmd::Tab(Tab::Search), + Cmd::Tab(Tab::Manage), + Cmd::Tab(Tab::Duplicates), + Cmd::Tab(Tab::Logs), + Cmd::Tab(Tab::Help), + Cmd::WaitIndexRunning { max_ms: None }, + Cmd::WaitIndexRunning { + max_ms: Some(15000) + }, + Cmd::WaitIndexIdle { + max_ms: Some(13000) + }, + Cmd::WaitSearchDone { max_ms: Some(6000) }, + Cmd::WaitDupsDone { max_ms: None }, + Cmd::RecordStart("manage-indexing".to_string()), + Cmd::RecordStop, + Cmd::Screenshot("query-highlight.v2".to_string()), + Cmd::Quit, + ] + ); + } + + #[test] + fn string_escapes_and_hash_inside_strings() { + assert_eq!( + parse_one(r#"type "say \"hi\" \\ done""#), + Cmd::Type { + text: r#"say "hi" \ done"#.to_string(), + cps: 7.0 + } + ); + // `#` inside a quoted string is content, not a comment. + assert_eq!( + parse_one(r##"type "a # b""##), + Cmd::Type { + text: "a # b".to_string(), + cps: 7.0 + } + ); + } + + #[test] + fn comments_and_blank_lines_are_skipped() { + let cmds = + parse_script("\n# a full-line comment\n \nquit # trailing comment\n#another\n") + .expect("should parse"); + assert_eq!(cmds, vec![Cmd::Quit]); + } + + #[test] + fn errors_carry_the_right_line_number() { + let e = parse_err("wait_ms 100\nquit\nfrobnicate\n"); + assert_eq!(e.line, 3); + assert!(e.msg.contains("frobnicate"), "got: {}", e.msg); + } + + #[test] + fn unclosed_string_is_rejected() { + let e = parse_err("type \"never closed\n"); + assert_eq!(e.line, 1); + assert!(e.msg.contains("unclosed"), "got: {}", e.msg); + } + + #[test] + fn unknown_escape_is_rejected() { + let e = parse_err(r#"type "a\nb""#); + assert!(e.msg.contains("escape"), "got: {}", e.msg); + } + + #[test] + fn non_numeric_int_is_rejected() { + let e = parse_err("wait_ms soon"); + assert!(e.msg.contains("integer"), "got: {}", e.msg); + let e = parse_err("wait_index_idle max never"); + assert!(e.msg.contains("integer"), "got: {}", e.msg); + } + + #[test] + fn names_with_path_separators_are_rejected() { + for bad in [ + "screenshot ../escape", + "screenshot a/b", + "record_start a\\b", + ] { + let e = parse_err(bad); + assert!(e.msg.contains("invalid name"), "{bad:?} got: {}", e.msg); + } + } + + #[test] + fn missing_arguments_are_rejected() { + assert!(parse_err("wait_ms").msg.contains("missing")); + assert!(parse_err("type").msg.contains("missing")); + assert!(parse_err("tab").msg.contains("missing")); + assert!(parse_err("screenshot").msg.contains("missing")); + assert!(parse_err("window").msg.contains("missing")); + assert!(parse_err("window 500").msg.contains("missing")); + } + + #[test] + fn degenerate_window_sizes_are_rejected() { + assert!(parse_err("window 0 350").msg.contains("positive")); + assert!(parse_err("window 500 0").msg.contains("positive")); + assert!(parse_err("window 500 -1").msg.contains("integer")); + } + + #[test] + fn trailing_garbage_is_rejected() { + let e = parse_err("quit now"); + assert!(e.msg.contains("unexpected"), "got: {}", e.msg); + let e = parse_err("wait_ms 100 200"); + assert!(e.msg.contains("unexpected"), "got: {}", e.msg); + } + + #[test] + fn unknown_tab_and_bad_cps_are_rejected() { + assert!(parse_err("tab settings").msg.contains("unknown tab")); + assert!(parse_err(r#"type "x" cps 0"#).msg.contains("positive")); + assert!(parse_err(r#"type "x" cps -3"#).msg.contains("positive")); + } + + #[test] + fn unquoted_type_text_is_rejected() { + let e = parse_err("type hello"); + assert!(e.msg.contains("quoted"), "got: {}", e.msg); + } + + /// The scenario that ships in packaging/ must always parse — this pins + /// the file to the grammar so neither can drift without failing tests. + #[test] + fn the_shipped_scenario_parses() { + let src = include_str!("../../../packaging/capture-scenario.txt"); + let cmds = parse_script(src).expect("packaging/capture-scenario.txt should parse"); + assert!( + cmds.len() > 10, + "scenario looks truncated: {} commands", + cmds.len() + ); + assert_eq!( + cmds.last(), + Some(&Cmd::Quit), + "scenario should end with quit" + ); + } + + #[test] + fn jitter_is_deterministic_and_bounded() { + for i in 0..10_000 { + let j = jitter(i); + assert!((0.75..=1.25).contains(&j), "jitter({i}) = {j}"); + assert_eq!(j, jitter(i)); + } + } +} diff --git a/crates/quicksearch-gui/src/cli.rs b/crates/quicksearch-gui/src/cli.rs index d94b023..31599db 100644 --- a/crates/quicksearch-gui/src/cli.rs +++ b/crates/quicksearch-gui/src/cli.rs @@ -32,6 +32,7 @@ FLAGS: --limit maximum results (default: [search].display_limit) --long rank, size, mtime, and snippets instead of bare paths -h, --help this help + -V, --version version and the commit it was built from Query syntax matches the GUI: plain words form one phrase; filters like type:Document, modified:>=2024-01-01, path:/dir, mime:application/pdf, @@ -54,8 +55,12 @@ variables are visible to other processes of the same user."; /// /// [`IndexCoordinator`]: quicksearch_core::coordinator::IndexCoordinator pub fn maybe_run_cli() -> Option { - let args: Vec = std::env::args().skip(1).collect(); + run_cli(std::env::args().skip(1).collect()) +} +/// The body of [`maybe_run_cli`], taking argv rather than reading it, so the +/// flag surface can be exercised from tests. +fn run_cli(args: Vec) -> Option { let mut fuzzy = false; let mut long = false; let mut limit: Option = None; @@ -68,6 +73,10 @@ pub fn maybe_run_cli() -> Option { println!("{}", USAGE); return Some(0); } + "-V" | "--version" => { + println!("QuickSearch {}", crate::version::BUILD_ID); + return Some(0); + } "--fuzzy" => fuzzy = true, "--long" => long = true, "--limit" => match it.next().and_then(|v| v.parse().ok()) { @@ -374,6 +383,38 @@ mod tests { Err(format!("{}wrong password", db::KEY_MISMATCH_PREFIX)) } + fn argv(args: &[&str]) -> Vec { + args.iter().map(|a| a.to_string()).collect() + } + + /// The flags that answer and exit without touching the index. Each must + /// report success, because a shell script asking `quicksearch --version` + /// reads the exit code, not the text. + #[test] + fn informational_flags_answer_and_succeed() { + for flag in ["-V", "--version", "-h", "--help"] { + assert_eq!(run_cli(argv(&[flag])), Some(0), "{flag} should exit 0"); + } + } + + /// Nothing to search for means "open the GUI", and that must survive the + /// new flag: `quicksearch` with no arguments is how most people start it. + #[test] + fn nothing_to_search_for_opens_the_gui() { + assert_eq!(run_cli(argv(&[])), None); + // Including flags the GUI stack might want for itself. + assert_eq!(run_cli(argv(&["--some-winit-flag"])), None); + } + + /// A malformed value is a usage error, not a silent default — in both + /// spellings, since only one of them goes through `it.next()`. + #[test] + fn a_non_numeric_limit_is_a_usage_error() { + assert_eq!(run_cli(argv(&["--limit", "x", "term"])), Some(2)); + assert_eq!(run_cli(argv(&["--limit=x", "term"])), Some(2)); + assert_eq!(run_cli(argv(&["--limit"])), Some(2)); + } + #[test] fn unprotected_needs_nothing() { let sec = SecurityConfig::default(); diff --git a/crates/quicksearch-gui/src/cli_main.rs b/crates/quicksearch-gui/src/cli_main.rs index 0edd284..19dccdf 100644 --- a/crates/quicksearch-gui/src/cli_main.rs +++ b/crates/quicksearch-gui/src/cli_main.rs @@ -12,6 +12,10 @@ mod format; // The GUI stores/deletes keychain entries; the CLI only reads them. #[allow(dead_code)] mod keychain; +// The GUI also shows the build id in its status bar; --version is all the CLI +// needs from it. +#[allow(dead_code)] +mod version; fn main() { // `maybe_run_cli` returns `None` for "no query given", which the combined diff --git a/crates/quicksearch-gui/src/help_tab.rs b/crates/quicksearch-gui/src/help_tab.rs index c759fb6..6134608 100644 --- a/crates/quicksearch-gui/src/help_tab.rs +++ b/crates/quicksearch-gui/src/help_tab.rs @@ -109,16 +109,45 @@ pub fn ui(ui: &mut egui::Ui) { ); ui.add_space(12.0); - ui.label( - egui::RichText::new( - "Everything else — building from source, configuration, the \ - complete query reference — is covered in README.md in the \ - QuickSearch folder (installed under /usr/share/doc/quicksearch/ \ - on Debian and Ubuntu).", - ) - .small() - .weak(), - ); + ui.horizontal_wrapped(|ui| { + // The sentence is assembled from several widgets, so the + // spacing between them has to come from the text itself. + ui.spacing_mut().item_spacing.x = 0.0; + let quiet = |text: &str| egui::RichText::new(text).small().weak(); + ui.label(quiet( + "Building from source, configuration, query structuring and more \ + are covered in ", + )); + if let Some(path) = readme_path() { + let path = path.display().to_string(); + if ui + .link(egui::RichText::new("README.md").small()) + .on_hover_text(&path) + .clicked() + { + crate::platform::open_file(&path); + } + } else { + ui.label(quiet("README.md")); + } + ui.label(quiet(".")); + }); }); crate::ui_util::more_below_hint(ui, &scroll); } + +/// Where this build left the README: under the install prefix's `share/doc` +/// (the .deb puts it in `/usr/share/doc/quicksearch/`), beside the executable +/// (the Windows installer and portable copies), or at the top of a build tree +/// a few levels above `target/`. +fn readme_path() -> Option { + let exe = std::env::current_exe().ok()?; + let dir = exe.parent()?; + let installed = dir + .parent() + .map(|prefix| prefix.join("share/doc/quicksearch/README.md")); + installed + .into_iter() + .chain(dir.ancestors().take(4).map(|d| d.join("README.md"))) + .find(|p| p.is_file()) +} diff --git a/crates/quicksearch-gui/src/main.rs b/crates/quicksearch-gui/src/main.rs index 317993f..a2ee0d3 100644 --- a/crates/quicksearch-gui/src/main.rs +++ b/crates/quicksearch-gui/src/main.rs @@ -10,6 +10,8 @@ mod app; mod backend; +#[cfg(feature = "capture")] +mod capture; #[cfg(not(windows))] mod cli; mod duplicates_tab; @@ -25,6 +27,7 @@ mod search_tab; mod tracker; mod ui_util; mod unlock; +mod version; use quicksearch_core::config::Config; diff --git a/crates/quicksearch-gui/src/manage_tab.rs b/crates/quicksearch-gui/src/manage_tab.rs index 30e3215..f641660 100644 --- a/crates/quicksearch-gui/src/manage_tab.rs +++ b/crates/quicksearch-gui/src/manage_tab.rs @@ -355,6 +355,15 @@ impl ManageTab { ui.colored_label(ui.visuals().error_fg_color, err); } }); + ui.label( + egui::RichText::new( + "Removing a folder removes its entries and leaves the rest of \ + the index untouched; adding one reindexes to pick it up. \ + Neither rebuilds.", + ) + .small() + .weak(), + ); ui.separator(); // --- Filters --------------------------------------------------- @@ -429,7 +438,9 @@ impl ManageTab { crate::ui_util::pattern_hint_label(&mut cols[1], &self.new_ignore); cols[1].label( egui::RichText::new( - "Changes apply on Apply & Save (may trigger index rebuild).", + "Changes apply on Apply & Save. A new pattern removes the \ + entries it matches; removing one reindexes to bring them \ + back.", ) .small() .weak(), @@ -576,9 +587,10 @@ fn db_size_tooltip(ui: &mut egui::Ui) { ui.add_space(6.0); ui.label( egui::RichText::new( - "The file does not shrink on its own: freed space is reused by the \ - index rather than returned to the disk. To hand it back after \ - narrowing the filters, use Clear index… and reindex.", + "Narrowing any of these removes the entries it excludes straight away, \ + but the file does not shrink on its own: the freed space is reused by \ + the index rather than returned to the disk, until an indexing run's \ + optimize pass compacts it.", ) .small() .weak(), diff --git a/crates/quicksearch-gui/src/options.rs b/crates/quicksearch-gui/src/options.rs index eb2360a..29c7b26 100644 --- a/crates/quicksearch-gui/src/options.rs +++ b/crates/quicksearch-gui/src/options.rs @@ -233,8 +233,9 @@ impl OptionsWindow { }); ui.label( egui::RichText::new( - "Changes to tokenizer, filters, hidden files, or hashing \ - prompt an index rebuild.", + "Narrowing a filter removes the entries it excludes; widening \ + one reindexes to find what it now allows. Only the tokenizer \ + and hash length require a full rebuild.", ) .small() .weak(), diff --git a/crates/quicksearch-gui/src/search_tab.rs b/crates/quicksearch-gui/src/search_tab.rs index 26d3e36..ebf191a 100644 --- a/crates/quicksearch-gui/src/search_tab.rs +++ b/crates/quicksearch-gui/src/search_tab.rs @@ -150,6 +150,20 @@ impl SearchTab { self.pending_edit = Some(Instant::now()); } + /// The query has executed and the fade/stage swap has landed — what the + /// capture driver's `wait_search_done` means by "done". + #[cfg(feature = "capture")] + pub(crate) fn capture_settled(&self) -> bool { + !self.running && self.pending_edit.is_none() && !self.swap_pending + } + + /// Re-arm the one-shot first-frame focus: tab switches drop egui focus, + /// and injected text needs the caret back in the search box. + #[cfg(feature = "capture")] + pub(crate) fn capture_focus(&mut self) { + self.focus_query = true; + } + /// A new search was submitted under `generation`. The previous /// results stay on screen (fading out); the new ones stage until the /// fade reaches zero. @@ -1023,23 +1037,24 @@ fn centered_match_job( job } -/// Tier-list chip color per cascade stage — lower rank, higher tier: -/// S-red for exact case-sensitive filename matches down through the -/// pastel ramp to purple for fuzzy full-text and on to the grey path -/// tiers. Dark text on these pastels stays readable in both themes. +/// Jet-colormap chip color per cascade stage — the rank reads as a +/// colorbar: cool blue for the strongest matches, warming through cyan, +/// green and yellow to red for the weakest path tiers. Pastel rather than +/// true jet, since every channel stays at or above 127 so the chip's dark +/// text keeps its contrast in both themes. fn rank_tier_color(stage: u8) -> egui::Color32 { match stage { - 1 => egui::Color32::from_rgb(255, 127, 127), // S - 2 => egui::Color32::from_rgb(255, 191, 127), // A - 3 => egui::Color32::from_rgb(255, 223, 127), // B - 4 => egui::Color32::from_rgb(255, 255, 127), // C - 5 => egui::Color32::from_rgb(191, 255, 127), // D - 6 => egui::Color32::from_rgb(127, 255, 127), // E - 7 => egui::Color32::from_rgb(127, 191, 255), // F - 8 => egui::Color32::from_rgb(191, 127, 255), // G - 9 => egui::Color32::from_rgb(223, 159, 255), // H — path, exact case - 10 => egui::Color32::from_rgb(239, 191, 239), // I — path, any case - _ => egui::Color32::from_rgb(199, 199, 199), // J — fuzzy path + 1 => egui::Color32::from_rgb(127, 127, 255), // name exact, exact case + 2 => egui::Color32::from_rgb(127, 178, 255), // name exact, any case + 3 => egui::Color32::from_rgb(127, 229, 255), // name substring, exact case + 4 => egui::Color32::from_rgb(127, 255, 229), // name substring, any case + 5 => egui::Color32::from_rgb(127, 255, 178), // full text, exact case + 6 => egui::Color32::from_rgb(127, 255, 127), // full text, any case + 7 => egui::Color32::from_rgb(178, 255, 127), // fuzzy name + 8 => egui::Color32::from_rgb(229, 255, 127), // fuzzy full text + 9 => egui::Color32::from_rgb(255, 229, 127), // path substring, exact case + 10 => egui::Color32::from_rgb(255, 178, 127), // path substring, any case + _ => egui::Color32::from_rgb(255, 127, 127), // fuzzy path } } @@ -1383,4 +1398,41 @@ mod tests { assert_eq!(dir_ignore_pattern(Path::new(r"C:\")), r"C:\*"); } } + + /// The rank chips read as a jet colorbar: blue at the best ranks + /// warming monotonically to red at the worst, and never so dark that + /// the chip's fixed dark text loses its contrast. Stage 12 stands in + /// for the catch-all arm. + #[test] + fn the_rank_ramp_runs_blue_to_red_and_stays_light() { + let ramp: Vec = (1..=11).map(rank_tier_color).collect(); + let (first, last) = (ramp[0], ramp[10]); + assert!( + first.b() > first.r(), + "the best rank should be blue: {first:?}" + ); + assert!( + last.r() > last.b(), + "the worst rank should be red: {last:?}" + ); + + for pair in ramp.windows(2) { + let (a, b) = (pair[0], pair[1]); + assert!(a.r() <= b.r(), "red must not cool off: {a:?} then {b:?}"); + assert!(a.b() >= b.b(), "blue must not warm up: {a:?} then {b:?}"); + } + + for stage in 1..=12u8 { + let c = rank_tier_color(stage); + assert!( + c.r() >= 127 && c.g() >= 127 && c.b() >= 127, + "stage {stage} is too dark for the chip's dark text: {c:?}" + ); + } + assert_eq!( + rank_tier_color(12), + last, + "out-of-range stages share the fuzzy-path chip" + ); + } } diff --git a/crates/quicksearch-gui/src/unlock.rs b/crates/quicksearch-gui/src/unlock.rs index babde8e..c2c7d61 100644 --- a/crates/quicksearch-gui/src/unlock.rs +++ b/crates/quicksearch-gui/src/unlock.rs @@ -83,6 +83,17 @@ impl eframe::App for Gate { app.on_exit(gl); } } + + /// The scripted capture driver injects keystrokes and harvests + /// screenshots here, before egui sees the frame's input. While locked + /// there is nothing to drive; capture runs use an unprotected config, so + /// the gate is `Running` from the first frame. + #[cfg(feature = "capture")] + fn raw_input_hook(&mut self, _ctx: &egui::Context, raw_input: &mut egui::RawInput) { + if let Gate::Running(app) = self { + app.capture_raw_input(raw_input); + } + } } /// Try to unlock with the keychain before any window exists. `true` means @@ -200,6 +211,18 @@ impl UnlockScreen { } } + // The lock screen owns the whole window and so has no status bar to + // carry the build id. Give it the same corner the unlocked app uses, + // so a screenshot of a machine that never got past the password is + // still identifiable. Declared before the central panel, as egui + // requires. + egui::TopBottomPanel::bottom("version-bar").show(ctx, |ui| { + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + ui.label(egui::RichText::new(crate::version::BUILD_ID).small().weak()) + .on_hover_text(crate::version::BUILD_ID_HINT); + }); + }); + let mut submitted = false; egui::CentralPanel::default().show(ctx, |ui| { ui.vertical_centered(|ui| { @@ -559,4 +582,35 @@ mod tests { frame(&ctx, &mut screen); assert!(ctx.memory(|m| m.has_focus(pw_field_id()))); } + + /// This screen is the whole window on a protected index, so the build id + /// in its corner is the only thing identifying a machine that never got + /// past the password. Assert it is painted rather than merely laid out — + /// a panel declared after the central one would compile and show nothing. + #[test] + fn the_lock_screen_shows_the_build_id() { + let ctx = egui::Context::default(); + let mut screen = UnlockScreen::new(locked_config(), None, None); + let input = egui::RawInput { + screen_rect: Some(egui::Rect::from_min_size( + egui::Pos2::ZERO, + egui::vec2(900.0, 600.0), + )), + ..Default::default() + }; + + let out = ctx.run(input, |ctx| { + assert!(screen.update(ctx).is_none()); + }); + + let painted = out.shapes.iter().any(|clipped| match &clipped.shape { + egui::epaint::Shape::Text(text) => text.galley.text() == crate::version::BUILD_ID, + _ => false, + }); + assert!( + painted, + "{} is not painted on the lock screen", + crate::version::BUILD_ID + ); + } } diff --git a/crates/quicksearch-gui/src/version.rs b/crates/quicksearch-gui/src/version.rs new file mode 100644 index 0000000..d1e75d4 --- /dev/null +++ b/crates/quicksearch-gui/src/version.rs @@ -0,0 +1,66 @@ +//! Which build this is: the release version and the commit it came from. +//! +//! Both halves are compile-time literals — the version from +//! `[workspace.package]`, the commit from `build.rs` — so [`BUILD_ID`] costs +//! nothing to build and can be shown on every frame. + +/// The one string the user sees: `v0.9.1 (a46cbc2)`. +/// +/// The version half is `[workspace.package] version` — the same value the +/// `.deb`, the Windows installer and the release tag carry, because they all +/// read it from there too. The commit half is the short hash `build.rs` +/// resolved, or `unknown` for a build made outside a git checkout. +pub const BUILD_ID: &str = concat!("v", env!("CARGO_PKG_VERSION"), " (", env!("QS_COMMIT"), ")"); + +/// Hover text wherever [`BUILD_ID`] is shown. +pub const BUILD_ID_HINT: &str = "QuickSearch version and the commit it was built from"; + +#[cfg(test)] +mod tests { + use super::*; + + /// Take the build id apart the way someone reading a bug report does. + /// Panics rather than returning an error: a build id that cannot be split + /// is the failure these tests exist to catch. + fn halves() -> (&'static str, &'static str) { + let rest = BUILD_ID + .strip_prefix('v') + .unwrap_or_else(|| panic!("{BUILD_ID:?} should lead with a v, like the release tags")); + let (version, commit) = rest + .split_once(" (") + .unwrap_or_else(|| panic!("{BUILD_ID:?} should be a version then a commit")); + let commit = commit + .strip_suffix(')') + .unwrap_or_else(|| panic!("{BUILD_ID:?} has an unclosed commit")); + (version, commit) + } + + /// The whole point of the constant is that a screenshot of the status bar + /// identifies a build, so the version half has to be the release version + /// exactly — not a prefix of it, and not something reformatted. + #[test] + fn the_build_id_names_the_release_version() { + let (version, _) = halves(); + assert_eq!(version, env!("CARGO_PKG_VERSION")); + } + + /// `unknown` is the documented fallback for a build with no git and no + /// `QS_COMMIT`; anything else has to be an abbreviated hash. A build script + /// that quietly emitted a branch name, a tag, or a full 40-character SHA + /// would widen the status bar and stop matching what the forge shows. + #[test] + fn the_build_id_names_a_short_commit() { + let (_, commit) = halves(); + if commit == "unknown" { + return; + } + assert!( + (1..=7).contains(&commit.len()), + "commit {commit:?} is not an abbreviated hash" + ); + assert!( + commit.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')), + "commit {commit:?} is not lowercase hex" + ); + } +} diff --git a/packaging/build-deb.sh b/packaging/build-deb.sh index d098570..e0b048e 100755 --- a/packaging/build-deb.sh +++ b/packaging/build-deb.sh @@ -7,7 +7,7 @@ # ./packaging/build-deb.sh --no-strip keep debug symbols (25 MB vs 20 MB) # ./packaging/build-deb.sh -o /tmp/out write the .deb somewhere else # -# Environment: DEB_REVISION (default 1), DEB_MAINTAINER, SOURCE_DATE_EPOCH. +# Environment: DEB_MAINTAINER, SOURCE_DATE_EPOCH. # # Deliberately does not use cargo-deb, debhelper, fakeroot or an SVG rasteriser: # dpkg-deb and desktop-file-utils are the only tools required, and both are part @@ -56,12 +56,13 @@ done version="$(sed -n '/^\[workspace\.package\]/,/^\[/{ s/^version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p }' "$REPO_ROOT/Cargo.toml")" [ -n "$version" ] || die "could not read version from Cargo.toml" -revision="${DEB_REVISION:-1}" +# No Debian revision: QuickSearch is only ever packaged from its own source, so +# the version is native (no hyphen) and the .deb is named for the crate version +# alone, matching the Windows installer. maintainer="${DEB_MAINTAINER:-Jeremy }" arch="$(dpkg --print-architecture)" -deb_version="${version}-${revision}" -stage="$out_dir/${PKG}_${deb_version}_${arch}" -deb="$out_dir/${PKG}_${deb_version}_${arch}.deb" +stage="$out_dir/${PKG}_${version}_${arch}" +deb="$out_dir/${PKG}_${version}_${arch}.deb" # ---------------------------------------------------------------- build ---- @@ -132,14 +133,16 @@ if [ -n "${SOURCE_DATE_EPOCH:-}" ]; then else changelog_date="$(date -R)" fi -gzip -9nc < "$stage/usr/share/doc/$PKG/changelog.Debian.gz" -$PKG ($deb_version) unstable; urgency=medium +# changelog.gz, not changelog.Debian.gz: with a native version there is no +# separate Debian revision to log, and policy puts the one changelog here. +gzip -9nc < "$stage/usr/share/doc/$PKG/changelog.gz" +$PKG ($version) unstable; urgency=medium * Package build of $PKG $version. -- $maintainer $changelog_date EOF -chmod 644 "$stage/usr/share/doc/$PKG/changelog.Debian.gz" +chmod 644 "$stage/usr/share/doc/$PKG/changelog.gz" # No config.toml is installed anywhere. Config::config_path() treats a # config.toml sitting next to the executable as portable mode and lets it @@ -191,7 +194,7 @@ recommends="desktop-file-utils, xdg-utils, dbus-bin, xdg-desktop-portal" install -dm755 "$stage/DEBIAN" cat > "$stage/DEBIAN/control" <-windows-x86_64-setup.exe. +# The binaries come from the x86_64-pc-windows-gnu target and makensis compiles +# the installer, so no Windows machine is involved - CI builds this in the same +# job that produces the .zip. +# +# Needs: makensis (the nsis package) and a mingw-w64 toolchain +# (gcc-mingw-w64-x86-64, which brings x86_64-w64-mingw32-strip). +# +# The installer itself is packaging/quicksearch.nsi; this script only decides +# what goes into it. + +set -euo pipefail +umask 022 + +readonly PKG=quicksearch +readonly TARGET=x86_64-pc-windows-gnu +# The GUI app and the console-subsystem terminal binary. Windows needs both as +# separate executables - see the [[bin]] comment in crates/quicksearch-gui. +readonly BINARIES=(quicksearch.exe quicksearch-cli.exe) +readonly REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +readonly ICON="$REPO_ROOT/crates/quicksearch-gui/assets/icons/$PKG.ico" + +do_build=1 +do_strip=1 +out_dir="$REPO_ROOT/dist" + +die() { printf 'build-installer: %s\n' "$*" >&2; exit 1; } +say() { printf '\033[1m==>\033[0m %s\n' "$*"; } + +while [ $# -gt 0 ]; do + case "$1" in + --no-build) do_build=0 ;; + --no-strip) do_strip=0 ;; + -o|--output-dir) shift; [ $# -gt 0 ] || die "--output-dir needs a path"; out_dir="$1" ;; + # Print the header comment block, however long it grows. + -h|--help) awk 'NR > 1 { if ($0 !~ /^#/) exit; sub(/^# ?/, ""); print }' "${BASH_SOURCE[0]}"; exit 0 ;; + *) die "unknown option: $1 (try --help)" ;; + esac + shift +done + +command -v makensis >/dev/null 2>&1 || die "missing makensis (install nsis)" +[ "$do_strip" -eq 0 ] || command -v x86_64-w64-mingw32-strip >/dev/null 2>&1 \ + || die "missing x86_64-w64-mingw32-strip (install gcc-mingw-w64-x86-64, or pass --no-strip)" + +# Same source of truth as build-deb.sh and the CI asset names: the crate version, +# never a tag or a branch name. +version="$(sed -n '/^\[workspace\.package\]/,/^\[/{ s/^version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p }' "$REPO_ROOT/Cargo.toml")" +[ -n "$version" ] || die "could not read version from Cargo.toml" + +# VIProductVersion accepts nothing but four numeric fields, so 0.9.1 has to +# become 0.9.1.0 and a pre-release suffix has to come off. The visible version +# string keeps whatever Cargo.toml says. +version_quad="${version%%[-+]*}" +case "$version_quad" in + *[!0-9.]*) die "cannot build a Windows version number from '$version'" ;; +esac +while [ "$(printf '%s' "$version_quad" | tr -cd . | wc -c)" -lt 3 ]; do + version_quad="$version_quad.0" +done + +installer="$out_dir/$PKG-$version-windows-x86_64-setup.exe" +stage="$out_dir/.installer-stage" + +# ---------------------------------------------------------------- build ---- + +if [ "$do_build" -eq 1 ]; then + say "Building $PKG $version for $TARGET (release)" + # rustc and the cc crate both derive these from the triple on most setups, + # but rusqlite, zstd-sys and openssl-src shell out to a C compiler and a + # host cc would produce ELF objects the mingw linker then rejects with an + # error that names neither. Setting them costs nothing and CI's identical + # values win by ':=' anyway. + : "${CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER:=x86_64-w64-mingw32-gcc}" + : "${CC_x86_64_pc_windows_gnu:=x86_64-w64-mingw32-gcc}" + : "${AR_x86_64_pc_windows_gnu:=x86_64-w64-mingw32-ar}" + export CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER CC_x86_64_pc_windows_gnu AR_x86_64_pc_windows_gnu + ( cd "$REPO_ROOT" && cargo build --release -p quicksearch-gui --target "$TARGET" ) +fi + +for bin in "${BINARIES[@]}"; do + [ -f "$REPO_ROOT/target/$TARGET/release/$bin" ] \ + || die "no binary at target/$TARGET/release/$bin (drop --no-build?)" +done +[ -f "$ICON" ] || die "no icon at $ICON" + +# --------------------------------------------------------------- stage ----- + +say "Staging $stage" +rm -rf "$stage" +mkdir -p "$stage" + +for bin in "${BINARIES[@]}"; do + install -m755 "$REPO_ROOT/target/$TARGET/release/$bin" "$stage/$bin" + if [ "$do_strip" -eq 1 ]; then + before="$(du -h "$stage/$bin" | cut -f1)" + x86_64-w64-mingw32-strip --strip-unneeded "$stage/$bin" + say "Stripped $bin: $before -> $(du -h "$stage/$bin" | cut -f1)" + fi +done + +# The icon is the installer's own icon, the uninstaller's, the Start menu +# shortcut's, and what Add/Remove Programs shows. The .exe files carry no icon +# resource of their own, so it has to be installed as a file. +install -m644 "$ICON" "$stage/$PKG.ico" + +# CRLF for the text files. The license page is a Windows edit control fed the +# file verbatim, and LF-only text arrives there as one long paragraph; the same +# conversion is what makes the installed copies readable in Notepad. LICENSE +# also gains the extension Windows needs to open it on a double-click. +for doc in README.md:README.md LICENSE:LICENSE.txt config_example.toml:config_example.toml; do + sed 's/$/\r/' "$REPO_ROOT/${doc%%:*}" > "$stage/${doc##*:}" + chmod 644 "$stage/${doc##*:}" +done + +# ----------------------------------------------------------- makensis ----- + +say "Building $installer" +rm -f "$installer" +makensis -V3 \ + "-DVERSION=$version" \ + "-DVERSION_QUAD=$version_quad" \ + "-DSTAGE=$stage" \ + "-DOUTFILE=$installer" \ + "$REPO_ROOT/packaging/$PKG.nsi" + +rm -rf "$stage" +[ -f "$installer" ] || die "makensis reported success but wrote no installer" + +echo +say "Done: $installer ($(du -h "$installer" | cut -f1))" diff --git a/packaging/capture-scenario.txt b/packaging/capture-scenario.txt new file mode 100644 index 0000000..d32f40c --- /dev/null +++ b/packaging/capture-scenario.txt @@ -0,0 +1,55 @@ +# QuickSearch website-asset capture scenario, run by packaging/capture.sh. +# Grammar and command list: crates/quicksearch-gui/src/capture.rs (feature +# "capture"). Edit this file to re-choreograph the captures; timings are in +# milliseconds and `max` caps bound clip length without failing the run. + +# The demo config sets [ui] scale = 1.5, so every window size here is 1.5x +# the layout it shows: high-resolution captures of an unchanged layout. + +# --- 1. manage-indexing.webm: the fresh auto-index in progress ------------- +window 1400 980 # full-size captures render at 1400x980 +wait_ms 800 # the resize lands asynchronously +tab manage +wait_index_running max 15000 # the auto run starts moments after launch; + # the cap tolerates "already running" +record_start manage-indexing +wait_index_idle max 13000 # ~13 s of progress bars and files/sec, + # shorter if the run finishes early +wait_ms 1500 # linger on the completed state +record_stop +wait_index_idle # hard wait: the full index must exist + # before the search captures + +# --- 2. search.webm: slow typing, streaming full-text results -------------- +tab search +focus_search # tab switches drop egui focus; re-arm it +clear_query +window 1120 750 # compact clip: the smallest layout at + # which every results column still fits + # (any narrower clips the Match column + # away, defeating the demo), at 1.5x +wait_ms 800 # the resize lands asynchronously +record_start search +type "fn main" cps 4 # a phrase that lives in file bodies, not + # names -- results stream in with content + # snippets in the Match column +wait_search_done max 8000 +wait_ms 2000 +record_stop +window 1400 980 # back to full size for the screenshots +wait_ms 800 + +# --- 3. query-highlight.png: syntax highlighting in the search box --------- +clear_query +wait_ms 300 +type "type:Text modified:>=2024-01-01 index" cps 30 +wait_search_done max 8000 +wait_ms 500 # let the results fade settle +screenshot query-highlight + +# --- 4. duplicates.png ----------------------------------------------------- +tab duplicates # entering the tab starts the scan +wait_dups_done +wait_ms 600 +screenshot duplicates +quit diff --git a/packaging/capture.sh b/packaging/capture.sh new file mode 100644 index 0000000..a9235ed --- /dev/null +++ b/packaging/capture.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash +# Regenerate the website capture assets: search.webm, manage-indexing.webm, +# duplicates.png, query-highlight.png — all landing in packaging/captures/ +# (gitignored). Needs a graphical session (X11 or Wayland — screenshots and +# video frames are read back from the app's own framebuffer, so the display +# server does not matter) and ffmpeg with libx264rgb and libvpx-vp9. +# +# The app is built with the `capture` feature and drives itself through +# packaging/capture-scenario.txt (see crates/quicksearch-gui/src/capture.rs +# for the command grammar). It runs against a throwaway index of this +# repository plus ~/.cargo/registry/src, under scratch XDG dirs — the real +# ~/.config/quicksearch and index are never touched. +# +# The scratch dirs live OUTSIDE both index roots on purpose: the demo config +# has an empty ignore list, so a scratch index database inside an indexed +# tree would be indexed and watched by the very run that writes it. +set -euo pipefail + +here="$(cd "$(dirname "$0")" && pwd)" +repo="$(dirname "$here")" +out="$here/captures" +work="${TMPDIR:-/tmp}/quicksearch-capture" + +# --- preflight -------------------------------------------------------------- +if [ -z "${DISPLAY:-}" ] && [ -z "${WAYLAND_DISPLAY:-}" ]; then + echo "capture needs a graphical session (neither DISPLAY nor" \ + "WAYLAND_DISPLAY is set)" >&2 + exit 1 +fi +for tool in ffmpeg ffprobe cargo; do + command -v "$tool" >/dev/null || { echo "missing tool: $tool" >&2; exit 1; } +done +# The encoder list is captured first: `grep -q` closing the pipe early would +# make ffmpeg exit on SIGPIPE, which pipefail reports as failure. +encoders="$(ffmpeg -hide_banner -encoders 2>/dev/null)" +for enc in libx264rgb libvpx-vp9; do + grep -q "$enc" <<< "$encoders" \ + || { echo "ffmpeg lacks the $enc encoder" >&2; exit 1; } +done +# A config.toml beside the binary would silently override the scratch XDG +# config entirely (portable mode) — refuse to run with one present. +if [ -e "$repo/target/release/config.toml" ]; then + echo "remove $repo/target/release/config.toml first: portable mode would" \ + "override the capture config" >&2 + exit 1 +fi +registry="$HOME/.cargo/registry/src" +[ -d "$registry" ] || { echo "missing demo index root: $registry" >&2; exit 1; } + +# --- build + parser tests --------------------------------------------------- +cargo test --manifest-path "$repo/Cargo.toml" --release --locked \ + -p quicksearch-gui --features capture capture:: +cargo build --manifest-path "$repo/Cargo.toml" --release --locked \ + -p quicksearch-gui --features capture + +# --- scratch environment ---------------------------------------------------- +# Fresh every run: no stale index, no schema-mismatch prompt, and a fresh +# app.ron so the window opens at its default 1000x700 geometry. +rm -rf "$work" +mkdir -p "$work/config/quicksearch" "$work/data" "$work/tmp" "$out" +cat > "$work/config/quicksearch/config.toml" <&2; exit 1; } + +# --- transcode the lossless intermediates to VP9 webm ----------------------- +# The recording itself is lossless, so webm quality is decided entirely here. +# QS_WEBM_CRF is the knob: constant-quality factor, 0 (best) to 63; UI text +# stays crisp around 24-32, and each step of ~6 roughly halves/doubles the +# file size. QS_WEBM_CPU trades encode time for compression efficiency +# (0 slowest/best to 5 fastest). +crf="${QS_WEBM_CRF:-26}" +cpu="${QS_WEBM_CPU:-0}" +for clip in manage-indexing search; do + [ -s "$work/tmp/$clip.cap.mkv" ] \ + || { echo "missing recording: $clip.cap.mkv" >&2; exit 1; } + # The crop drops at most one row/column: yuv420p needs even dimensions, + # and fractional display scaling can make the window an odd size. + ffmpeg -y -hide_banner -loglevel warning -i "$work/tmp/$clip.cap.mkv" \ + -vf "crop=trunc(iw/2)*2:trunc(ih/2)*2" \ + -c:v libvpx-vp9 -b:v 0 -crf "$crf" -deadline good -cpu-used "$cpu" \ + -row-mt 1 -pix_fmt yuv420p -an "$out/$clip.webm" +done +mv "$work/tmp/query-highlight.png" "$work/tmp/duplicates.png" "$out/" + +# --- verify ----------------------------------------------------------------- +fail=0 +for f in "$out/search.webm" "$out/manage-indexing.webm"; do + codec=$(ffprobe -v error -select_streams v:0 \ + -show_entries stream=codec_name -of csv=p=0 "$f") + dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$f") + [ "$codec" = vp9 ] || { echo "FAIL: $f codec=$codec" >&2; fail=1; } + awk -v d="$dur" 'BEGIN{exit !(d>=4 && d<=60)}' \ + || { echo "FAIL: $f duration=${dur}s (expected 4-60s)" >&2; fail=1; } + echo "OK: $f (vp9, ${dur}s)" +done +for f in "$out/duplicates.png" "$out/query-highlight.png"; do + dims=$(ffprobe -v error -select_streams v:0 \ + -show_entries stream=width,height -of csv=p=0 "$f") + echo "OK: $f (${dims})" +done +[ "$fail" -eq 0 ] + +# $work is kept for post-mortems (app stderr is on this terminal; the scratch +# index and raw .cap.mkv files live there) and recreated fresh next run. +echo +echo "Assets:" +ls -l "$out"/*.webm "$out"/*.png diff --git a/packaging/quicksearch.nsi b/packaging/quicksearch.nsi new file mode 100644 index 0000000..dd8de57 --- /dev/null +++ b/packaging/quicksearch.nsi @@ -0,0 +1,222 @@ +; +; NSIS installer for QuickSearch. +; +; Compiled by packaging/build-installer.sh, which supplies every define below +; and stages the files this installs. Running makensis on this file by hand +; will fail on the !error checks rather than build something half-configured. +; +; VERSION workspace version, e.g. 0.9.1 +; VERSION_QUAD the same padded to a.b.c.d, which is all VIProductVersion takes +; STAGE directory holding the exact files to install +; OUTFILE path of the installer to write +; +; makensis runs on Linux, so the installer comes out of the same cross-compile +; job as the .zip and no Windows machine is involved anywhere in the pipeline. +; That also means file paths below use forward slashes: they are read by +; makensis on the build host, unlike $INSTDIR paths, which are Windows strings. + +!ifndef VERSION | VERSION_QUAD | STAGE | OUTFILE + !error "build this with packaging/build-installer.sh, which defines VERSION, VERSION_QUAD, STAGE and OUTFILE" +!endif + +Unicode true +; The payload is two ~20 MB binaries; solid LZMA is worth the compression time. +SetCompressor /SOLID lzma + +!include "MUI2.nsh" +!include "LogicLib.nsh" +!include "FileFunc.nsh" +!include "x64.nsh" + +!define APP "QuickSearch" +!define PUBLISHER "Jeremy " +!define HOMEPAGE "https://code.karsttech.com/jeremy/quick_search" +; Where Add/Remove Programs looks, and where an upgrade finds the directory the +; previous version went into. +!define UNINST_KEY "Software\Microsoft\Windows\CurrentVersion\Uninstall\${APP}" + +Name "${APP} ${VERSION}" +OutFile "${OUTFILE}" +InstallDir "$PROGRAMFILES64\${APP}" +; Program Files and HKLM both need elevation, and it is what puts the shortcuts +; in front of every account on the machine rather than just the installing one. +RequestExecutionLevel admin +ShowInstDetails show +ShowUninstDetails show + +VIProductVersion "${VERSION_QUAD}" +VIAddVersionKey "ProductName" "${APP}" +VIAddVersionKey "ProductVersion" "${VERSION}" +VIAddVersionKey "FileVersion" "${VERSION_QUAD}" +VIAddVersionKey "CompanyName" "${PUBLISHER}" +VIAddVersionKey "LegalCopyright" "GPL-3.0-or-later" +VIAddVersionKey "FileDescription" "${APP} ${VERSION} installer" + +!define MUI_ABORTWARNING +!define MUI_ICON "${STAGE}/quicksearch.ico" +!define MUI_UNICON "${STAGE}/quicksearch.ico" + +!insertmacro MUI_PAGE_WELCOME +!insertmacro MUI_PAGE_LICENSE "${STAGE}/LICENSE.txt" +!insertmacro MUI_PAGE_COMPONENTS +!insertmacro MUI_PAGE_DIRECTORY +!insertmacro MUI_PAGE_INSTFILES +!define MUI_FINISHPAGE_RUN +!define MUI_FINISHPAGE_RUN_TEXT "Run ${APP}" +!define MUI_FINISHPAGE_RUN_FUNCTION LaunchAsUser +!insertmacro MUI_PAGE_FINISH + +!insertmacro MUI_UNPAGE_CONFIRM +!insertmacro MUI_UNPAGE_INSTFILES + +!insertmacro MUI_LANGUAGE "English" + +; ------------------------------------------------------------------ init ---- + +Function .onInit + ${IfNot} ${RunningX64} + MessageBox MB_OK|MB_ICONSTOP \ + "${APP} is 64-bit only and this is a 32-bit Windows installation." /SD IDOK + Abort + ${EndIf} + ; The installer stub is 32-bit (it runs anywhere), so without this every + ; HKLM write below would be redirected into Wow6432Node - out of sight of + ; 64-bit Add/Remove Programs and of the lookup two lines down. + SetRegView 64 + ; Shortcuts and the uninstall entry are machine-wide, matching where the + ; files go. + SetShellVarContext all + + ; Deliberately not InstallDirRegKey: that is read before .onInit runs, which + ; is before SetRegView 64, so it would look in the wrong registry view. + ; $INSTDIR still holds the InstallDir default unless /D= overrode it on the + ; command line, and an explicit /D= must win over the previous location. + ${If} $INSTDIR == "$PROGRAMFILES64\${APP}" + ReadRegStr $0 HKLM "${UNINST_KEY}" "InstallLocation" + ${If} $0 != "" + StrCpy $INSTDIR $0 + ${EndIf} + ${EndIf} +FunctionEnd + +Function un.onInit + SetRegView 64 + SetShellVarContext all +FunctionEnd + +; -------------------------------------------------------------- sections ---- + +Section "!${APP}" SecApp + SectionIn RO + SetOutPath "$INSTDIR" + + ; Windows will not replace a running executable, and the File commands below + ; would stop halfway through with a write error. Delete fails on a mapped + ; image and succeeds quietly when there is nothing there, which makes it a + ; plugin-free "is it still running?" test. + ClearErrors + Delete "$INSTDIR\quicksearch.exe" + Delete "$INSTDIR\quicksearch-cli.exe" + ${If} ${Errors} + MessageBox MB_OK|MB_ICONSTOP \ + "${APP} is still running. Close it and start this installer again." /SD IDOK + Abort + ${EndIf} + + File "${STAGE}/quicksearch.exe" + File "${STAGE}/quicksearch-cli.exe" + File "${STAGE}/quicksearch.ico" + File "${STAGE}/README.md" + File "${STAGE}/LICENSE.txt" + File "${STAGE}/config_example.toml" + ; No config.toml is installed, for the same reason the .deb ships none: one + ; sitting next to the binary is portable mode, and it would override the + ; personal config of every account on the machine. The app writes + ; %APPDATA%\quicksearch\config.toml on first run instead. + + WriteUninstaller "$INSTDIR\uninstall.exe" + + WriteRegStr HKLM "${UNINST_KEY}" "DisplayName" "${APP}" + WriteRegStr HKLM "${UNINST_KEY}" "DisplayVersion" "${VERSION}" + WriteRegStr HKLM "${UNINST_KEY}" "DisplayIcon" "$INSTDIR\quicksearch.ico" + WriteRegStr HKLM "${UNINST_KEY}" "Publisher" "${PUBLISHER}" + WriteRegStr HKLM "${UNINST_KEY}" "URLInfoAbout" "${HOMEPAGE}" + WriteRegStr HKLM "${UNINST_KEY}" "InstallLocation" "$INSTDIR" + WriteRegStr HKLM "${UNINST_KEY}" "UninstallString" '"$INSTDIR\uninstall.exe"' + WriteRegStr HKLM "${UNINST_KEY}" "QuietUninstallString" '"$INSTDIR\uninstall.exe" /S' + WriteRegDWORD HKLM "${UNINST_KEY}" "NoModify" 1 + WriteRegDWORD HKLM "${UNINST_KEY}" "NoRepair" 1 + ; Add/Remove Programs reads this as a DWORD of kilobytes, which is exactly + ; what /S=0K returns. + ${GetSize} "$INSTDIR" "/S=0K" $0 $1 $2 + IntFmt $0 "0x%08X" $0 + WriteRegDWORD HKLM "${UNINST_KEY}" "EstimatedSize" $0 +SectionEnd + +Section "Start Menu shortcut" SecStartMenu + ; One shortcut, no program folder: a single-application folder is noise in + ; the Windows 10/11 Start menu, and the uninstaller lives in Add/Remove + ; Programs rather than next to it. + CreateShortcut "$SMPROGRAMS\${APP}.lnk" "$INSTDIR\quicksearch.exe" "" "$INSTDIR\quicksearch.ico" +SectionEnd + +Section /o "Desktop shortcut" SecDesktop + CreateShortcut "$DESKTOP\${APP}.lnk" "$INSTDIR\quicksearch.exe" "" "$INSTDIR\quicksearch.ico" +SectionEnd + +; There is deliberately no "add to PATH" section, tempting as one is for +; quicksearch-cli. Editing PATH means reading the machine value, appending and +; writing it back, and NSIS strings in the standard build are capped at +; NSIS_MAX_STRLEN (1024 characters, `makensis -HDRINFO`). ReadRegStr truncates +; silently at that cap, so on any machine with a long PATH the write-back would +; destroy the rest of it - a spectacular failure for an optional checkbox. The +; README tells people to add the directory themselves. + +!insertmacro MUI_FUNCTION_DESCRIPTION_BEGIN + !insertmacro MUI_DESCRIPTION_TEXT ${SecApp} \ + "The desktop app and quicksearch-cli, the terminal search tool." + !insertmacro MUI_DESCRIPTION_TEXT ${SecStartMenu} \ + "Add ${APP} to the Start menu for all users." + !insertmacro MUI_DESCRIPTION_TEXT ${SecDesktop} \ + "Add a ${APP} shortcut to the desktop." +!insertmacro MUI_FUNCTION_DESCRIPTION_END + +; The installer is elevated and anything it starts inherits that. Handing the +; path to the already-running Explorer starts the app as the logged-on user +; instead, so a first run creates its config and index in the right profile. +Function LaunchAsUser + Exec '"$WINDIR\explorer.exe" "$INSTDIR\quicksearch.exe"' +FunctionEnd + +; ----------------------------------------------------------- uninstaller ---- + +Section "Uninstall" + ; Same running-process test as the install side. Stopping here leaves the + ; installation intact rather than gutted. + ClearErrors + Delete "$INSTDIR\quicksearch.exe" + Delete "$INSTDIR\quicksearch-cli.exe" + ${If} ${Errors} + MessageBox MB_OK|MB_ICONSTOP \ + "${APP} is still running. Close it and try again." /SD IDOK + Abort + ${EndIf} + + Delete "$INSTDIR\quicksearch.ico" + Delete "$INSTDIR\README.md" + Delete "$INSTDIR\LICENSE.txt" + Delete "$INSTDIR\config_example.toml" + Delete "$INSTDIR\uninstall.exe" + Delete "$SMPROGRAMS\${APP}.lnk" + Delete "$DESKTOP\${APP}.lnk" + ; Plain RMDir, never /r: a portable-mode config.toml and the index beside it + ; may be sitting in this directory, and neither is ours to delete. A + ; directory holding anything the installer did not put there simply stays. + RMDir "$INSTDIR" + + DeleteRegKey HKLM "${UNINST_KEY}" + ; The per-user config in %APPDATA%\quicksearch and the index in + ; %LOCALAPPDATA%\quicksearch are left alone, the way apt leaves ~/.config + ; alone: reinstalling picks up where the last install left off, and nobody + ; loses an index to an upgrade. +SectionEnd