jeremy/quick_search
jeremy/quick_search
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

Added better visual feeback for adding file extensions to ignore patterns and a prompt to save modified settings.
Some checks are pending
CI / linux (push) Waiting to run
Details
CI / windows-cross (push) Waiting to run
Details
CI / release (push) Blocked by required conditions
Details

Browse source 
Added a Forgejo CI pipeline
This commit is contained in:
= 2026-08-04 03:27:05 -04:00
parent c7e11c57d5
commit b8eedf2137
49 changed files with 3077 additions and 1077 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

293
.forgejo/workflows/ci.yml Normal file
View file

@ -0,0 +1,293 @@
name: CI
# Forgejo reads .forgejo/workflows before .github/workflows. Actions referenced
# bare (actions/checkout, actions/cache, ...) resolve through the instance's
# DEFAULT_ACTIONS_URL, which points at data.forgejo.org, so nothing here reaches
# out to github.com.
on:
push:
branches: [master]
tags: ['v*']
pull_request:
branches: [master]
workflow_dispatch:
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
CARGO_TERM_COLOR: always
# Incremental artifacts are never reused between CI runs and would bloat the
# cached target/ tree for nothing.
CARGO_INCREMENTAL: '0'
RUST_BACKTRACE: '1'
jobs:
# ---------------------------------------------------------------- linux ----
#
# The container base is load-bearing: packaging/build-deb.sh derives the
# package's libc6 floor with objdump from the binary it just built, so the
# .deb inherits the *builder's* glibc. Ubuntu 22.04 fixes that floor at 2.35,
# which covers 22.04 LTS and newer plus Debian 12 and newer. Packages built by
# hand on a dev machine declared libc6 (>= 2.43) and installed on almost
# nothing.
#
# catthehacker/ubuntu is the act-compatible image family. A bare ubuntu:22.04
# will not work: JS actions need Node already present in the image, and no
# step can install it before actions/checkout runs.
linux:
runs-on: forgejo-runner
container:
image: catthehacker/ubuntu:act-22.04
env:
# crates/quicksearch-core/src/config.rs has a test that expects a home
# directory and panics without one.
HOME: /root
# The highest libc6 version the .deb is allowed to require.
MAX_GLIBC: '2.35'
steps:
- uses: actions/checkout@v4
- name: Install build dependencies
run: |
apt-get update -qq
# rusqlite's bundled-sqlcipher-vendored-openssl and keyring's vendored
# feature compile SQLCipher, OpenSSL and libdbus from source, so a C
# toolchain plus perl covers them and no -dev packages are needed.
# winit and glutin dlopen the whole display stack, so there are no X11
# or Wayland headers here either. The rest is what build-deb.sh checks
# for before it will run.
apt-get install -y --no-install-recommends \
build-essential perl pkg-config \
binutils dpkg-dev desktop-file-utils gzip
- name: Trust the workspace
# checkout writes as root into a directory git then considers dubiously
# owned; build-deb.sh and the version scrape both shell out to git.
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Install Rust
run: |
# --default-toolchain none defers to rust-toolchain.toml, so the pinned
# channel and its targets are downloaded exactly once.
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
| sh -s -- -y --profile minimal --default-toolchain none
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
# Materialise the pinned toolchain here rather than partway through the
# build, so a toolchain problem shows up as its own failed step.
"$HOME/.cargo/bin/rustup" show
- uses: actions/cache@v4
with:
path: |
~/.cargo/registry/index
~/.cargo/registry/cache
~/.cargo/git/db
target
# Keyed per job so this tree never collides with the cross-compiled one.
key: ${{ github.job }}-cargo-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }}
restore-keys: ${{ github.job }}-cargo-
- name: Build
run: cargo build --release --locked -p quicksearch-gui
- name: Test
# Release mode is not a nicety: tests/encrypted.rs derives an Argon2id key
# at m=64 MiB, t=3, which takes about half a second in release and minutes
# in debug. tests/snippet_perf.rs self-skips without QSB_SNIPPET_PERF=1.
run: cargo test --release --locked --workspace
- name: Build the .deb
# --no-build reuses the binaries from the Build step rather than
# recompiling. SOURCE_DATE_EPOCH pins the generated changelog date so
# repeat builds of the same commit are byte-identical.
run: |
SOURCE_DATE_EPOCH="$(git log -1 --pretty=%ct)" \
./packaging/build-deb.sh --no-build
- name: Check the glibc floor
# The whole point of pinning the container. If someone bumps the image,
# this fails loudly instead of quietly shipping an uninstallable package.
run: |
deb=$(ls dist/*.deb)
depends=$(dpkg-deb -f "$deb" Depends)
echo "$depends"
floor=$(printf '%s' "$depends" | sed -n 's/.*libc6 (>= \([0-9][0-9.]*\)).*/\1/p')
[ -n "$floor" ] || { echo "could not read the libc6 floor from $deb" >&2; exit 1; }
# dpkg's own comparator, so 2.9 does not sort above 2.35.
if ! dpkg --compare-versions "$floor" le "$MAX_GLIBC"; then
echo "ERROR: the .deb requires glibc $floor, above the $MAX_GLIBC target." >&2
echo "The build container base has probably changed." >&2
exit 1
fi
echo "OK: glibc floor $floor <= $MAX_GLIBC"
- name: Package the binaries
# A tarball for anyone not installing the .deb, stripped to match what
# build-deb.sh ships.
run: |
version=$(sed -n '/^\[workspace\.package\]/,/^\[/{ s/^version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p }' Cargo.toml)
[ -n "$version" ] || { echo "could not read the version from Cargo.toml" >&2; exit 1; }
stage="quicksearch-$version-linux-x86_64"
mkdir -p "dist/$stage"
for bin in quicksearch quicksearch-cli; do
install -m755 "target/release/$bin" "dist/$stage/$bin"
strip --strip-unneeded "dist/$stage/$bin"
done
install -m644 README.md config_example.toml LICENSE "dist/$stage/"
tar -czf "dist/$stage.tar.gz" -C dist "$stage"
rm -rf "dist/$stage"
ls -l dist/
- uses: actions/upload-artifact@v4
with:
name: linux-x86_64
path: |
dist/*.deb
dist/*.tar.gz
if-no-files-found: error
retention-days: 14
# -------------------------------------------------------- windows-cross ----
#
# Deliberately a newer base than the linux job. This job emits a PE binary
# linked against msvcrt.dll, so the container's glibc cannot affect what the
# .exe runs on; pinning it to 22.04 would buy nothing while forcing the build
# through mingw-w64 10.3 instead of 13.2.
windows-cross:
runs-on: forgejo-runner
container:
image: catthehacker/ubuntu:act-24.04
env:
HOME: /root
TARGET: x86_64-pc-windows-gnu
# rusqlite, zstd-sys and openssl-src all shell out to a C compiler, which
# has to be the cross one rather than the host's cc.
CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER: x86_64-w64-mingw32-gcc
CC_x86_64_pc_windows_gnu: x86_64-w64-mingw32-gcc
AR_x86_64_pc_windows_gnu: x86_64-w64-mingw32-ar
steps:
- uses: actions/checkout@v4
- name: Install build dependencies
run: |
apt-get update -qq
apt-get install -y --no-install-recommends \
build-essential perl make pkg-config \
gcc-mingw-w64-x86-64 zip
- name: Trust the workspace
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Install Rust
run: |
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
| sh -s -- -y --profile minimal --default-toolchain none
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
# Materialise the pinned toolchain here rather than partway through the
# build, so a toolchain problem shows up as its own failed step.
"$HOME/.cargo/bin/rustup" show
- uses: actions/cache@v4
with:
path: |
~/.cargo/registry/index
~/.cargo/registry/cache
~/.cargo/git/db
target
key: ${{ github.job }}-cargo-${{ hashFiles('**/Cargo.lock', 'rust-toolchain.toml') }}
restore-keys: ${{ github.job }}-cargo-
- name: Build
# rust-toolchain.toml already lists the target, so no `rustup target add`.
run: cargo build --release --locked -p quicksearch-gui --target "$TARGET"
- name: Check for non-system DLL dependencies
# Debian's default mingw alternative uses posix threads, which can pull
# in libwinpthread-1.dll or libgcc_s_seh-1.dll and produce an .exe that
# refuses to start on a clean Windows machine. The binaries are currently
# clean - every import is a system DLL - and this keeps them that way.
run: |
for exe in "target/$TARGET"/release/quicksearch.exe "target/$TARGET"/release/quicksearch-cli.exe; do
dlls=$(x86_64-w64-mingw32-objdump -p "$exe" | sed -n 's/^[[:space:]]*DLL Name: //p' | sort -fu)
printf '%s:\n%s\n\n' "$exe" "$dlls"
if printf '%s\n' "$dlls" | grep -qiE '^(libgcc|libwinpthread|libstdc\+\+|libssp)'; then
echo "ERROR: $exe imports a non-system DLL and will not run on a clean Windows install." >&2
exit 1
fi
done
- name: Package the binaries
run: |
version=$(sed -n '/^\[workspace\.package\]/,/^\[/{ s/^version[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p }' Cargo.toml)
[ -n "$version" ] || { echo "could not read the version from Cargo.toml" >&2; exit 1; }
stage="quicksearch-$version-windows-x86_64"
mkdir -p "dist/$stage"
for exe in quicksearch.exe quicksearch-cli.exe; do
install -m755 "target/$TARGET/release/$exe" "dist/$stage/$exe"
x86_64-w64-mingw32-strip --strip-unneeded "dist/$stage/$exe"
done
install -m644 README.md config_example.toml LICENSE "dist/$stage/"
(cd dist && zip -qr "$stage.zip" "$stage")
rm -rf "dist/$stage"
ls -l dist/
- uses: actions/upload-artifact@v4
with:
name: windows-x86_64
path: dist/*.zip
if-no-files-found: error
retention-days: 14
# -------------------------------------------------------------- release ----
release:
needs: [linux, windows-cross]
if: startsWith(github.ref, 'refs/tags/v')
runs-on: forgejo-runner
container:
image: catthehacker/ubuntu:act-24.04
steps:
- uses: actions/download-artifact@v4
with:
path: artifacts
- name: Publish the release
# Forgejo populates GITHUB_API_URL, GITHUB_REPOSITORY and the token by
# itself, so this needs no configuration. Talking to the API directly
# keeps the release step off any third-party action's release cadence.
env:
RELEASE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
api="${GITHUB_API_URL:-$GITHUB_SERVER_URL/api/v1}"
tag="${GITHUB_REF#refs/tags/}"
auth="Authorization: token $RELEASE_TOKEN"
mkdir -p dist
find artifacts -type f -exec mv -t dist -- {} +
ls -l dist/
# Reuse the release when it already exists, so re-running a tag build
# replaces assets instead of failing.
id=$(curl -sS -H "$auth" "$api/repos/$GITHUB_REPOSITORY/releases/tags/$tag" | jq -r '.id // empty')
if [ -z "$id" ]; then
id=$(curl -fsS -X POST "$api/repos/$GITHUB_REPOSITORY/releases" \
-H "$auth" -H 'Content-Type: application/json' \
-d "$(jq -n --arg t "$tag" '{tag_name: $t, name: $t, draft: false, prerelease: false}')" \
| jq -r '.id // empty')
fi
[ -n "$id" ] || { echo "could not create or find a release for $tag" >&2; exit 1; }
for f in dist/*; do
name=$(basename "$f")
echo "uploading $name"
# Replace an asset of the same name left by an earlier run.
old=$(curl -sS -H "$auth" "$api/repos/$GITHUB_REPOSITORY/releases/$id/assets" \
| jq -r --arg n "$name" '.[] | select(.name == $n) | .id')
[ -z "$old" ] || curl -fsS -X DELETE -H "$auth" \
"$api/repos/$GITHUB_REPOSITORY/releases/$id/assets/$old"
curl -fsS -X POST "$api/repos/$GITHUB_REPOSITORY/releases/$id/assets?name=$name" \
-H "$auth" -F "attachment=@$f" -o /dev/null
done
echo "published $tag"

2
Cargo.toml
View file

@ -6,7 +6,7 @@ members = [
]
[workspace.package]
version = "0.8.8"
version = "0.9.0"
edition = "2021"
license = "GPL-3.0-or-later"
authors = ["Jeremy <jeremy@karsttech.com>"]

24
README.md
View file

@ -22,9 +22,11 @@ Rust toolchain with rustup; `build.bat` uses winget and rustup. Both take
`--check` to report dependency status without installing or building,
`--no-run` to stop after the build, and `--` to pass the rest to the binary.
Building by hand needs a Rust toolchain (edition 2021) plus, on every
platform, a C toolchain and Perl: SQLCipher, zstd and OpenSSL are compiled
from bundled C sources, and OpenSSL's `Configure` is a Perl script. The old
Building by hand needs a Rust toolchain plus, on every platform, a C toolchain
and Perl: SQLCipher, zstd and OpenSSL are compiled from bundled C sources, and
OpenSSL's `Configure` is a Perl script. `rust-toolchain.toml` pins the compiler
version and the cross-compilation targets, so rustup installs the right ones on
the first `cargo` command and no `rustup target add` is needed. The old
WebKit/WebView dependencies (`setup.sh`) are gone; the GUI renders with
OpenGL via egui.
@ -35,9 +37,11 @@ OpenGL via egui.
run time, so only the runtime libraries matter.
- Windows: Visual Studio 2022 Build Tools with the "Desktop development
with C++" workload (MSVC v143 plus a Windows SDK), and Perl (Strawberry
Perl); NASM is optional and only enables OpenSSL's assembly paths. For the
GNU target instead, `rustup target add x86_64-pc-windows-gnu` and a
mingw-w64 toolchain. Note that Windows ships only a software OpenGL 1.1
Perl); NASM is optional and only enables OpenSSL's assembly paths. The GNU
target needs only a mingw-w64 toolchain, and cross-compiles from Linux —
`cargo build --release -p quicksearch-gui --target x86_64-pc-windows-gnu`
with `gcc-mingw-w64-x86-64` installed, which is how CI produces the Windows
binaries. Note that Windows ships only a software OpenGL 1.1
driver, so a bare VM or an RDP session without a vendor GPU driver cannot
create a context and the window will fail to open.
- macOS: Xcode command line tools (`build.sh` does not auto-install these —
@ -382,6 +386,14 @@ pagination: the table is virtualized, so a single scroll list capped at
- `cargo test -p quicksearch-gui`: formatter/tracker/CLI-parsing units.
- `QSB_SNIPPET_PERF=1 cargo test --release -p quicksearch-core --test
snippet_perf -- --nocapture`: snippet pipeline benchmark.
- `.forgejo/workflows/ci.yml`: builds both platforms on every push to `master`
and every pull request, and attaches the `.deb`, a Linux tarball and a
Windows zip to a release on a `v*` tag. The Linux job runs in an Ubuntu
22.04 container on purpose — `packaging/build-deb.sh` reads the package's
`libc6` floor from the binary it just built, so the builder's glibc becomes
the package's minimum, and 22.04 pins it at 2.35. The Windows job
cross-compiles with mingw-w64 and fails if either `.exe` picks up a
dependency on a non-system DLL.
- New extractors: implement `extract::Extractor` and register it in
`Registry::default_set()` — order matters, the first extractor whose
`supports` accepts a MIME wins. New cascade behavior: `search/cascade.rs`

8
config_example.toml
View file

@ -54,9 +54,11 @@ include_hidden = false
content_extensions = []
# Excluded from the index entirely. A pattern without a separator matches
# any single path component (so ".git" prunes whole subtrees); patterns
# containing one match full paths. Glob syntax (*, ?, [..]). Matching is
# case-insensitive on Windows and macOS, case-sensitive elsewhere,
# following the filesystem.
# containing one match full paths, including a bare drive root like 'D:\'.
# Glob syntax (*, ?, [..]). A name pattern must match the whole name:
# ".jpg" only matches something named exactly ".jpg" — ignoring an
# extension needs the wildcard, "*.jpg". Matching is case-insensitive on
# Windows and macOS, case-sensitive elsewhere, following the filesystem.
#
# The Windows defaults add: "$RECYCLE.BIN", "System Volume Information",
# "pagefile.sys", "hiberfil.sys", "swapfile.sys", "Thumbs.db",

58
crates/quicksearch-core/examples/indexprobe.rs
View file

@ -50,11 +50,42 @@ const LARGE_TEXT: usize = 100;
const BINARY: usize = 100;
const WORDS: &[&str] = &[
"alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta",
"quick", "brown", "fox", "jumps", "over", "lazy", "dog", "indexer",
"rust", "cargo", "sqlite", "baloo", "tokenizer", "trigram", "snippet",
"ocean", "forest", "mountain", "river", "valley", "bridge", "tunnel",
"morning", "afternoon", "evening", "midnight", "yesterday", "today",
"alpha",
"beta",
"gamma",
"delta",
"epsilon",
"zeta",
"eta",
"theta",
"quick",
"brown",
"fox",
"jumps",
"over",
"lazy",
"dog",
"indexer",
"rust",
"cargo",
"sqlite",
"baloo",
"tokenizer",
"trigram",
"snippet",
"ocean",
"forest",
"mountain",
"river",
"valley",
"bridge",
"tunnel",
"morning",
"afternoon",
"evening",
"midnight",
"yesterday",
"today",
];
/// Deterministic so two runs index byte-identical trees and their timings are
@ -63,7 +94,10 @@ struct Rng(u64);
impl Rng {
fn next(&mut self) -> u64 {
self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
self.0 = self
.0
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
self.0 >> 33
}
@ -74,12 +108,20 @@ impl Rng {
fn main() {
let mode = std::env::args().nth(1).unwrap_or_default();
let tree = PathBuf::from(std::env::args().nth(2).expect("usage: indexprobe <gen|cold|warm> <tree> [db]"));
let tree = PathBuf::from(
std::env::args()
.nth(2)
.expect("usage: indexprobe <gen|cold|warm> <tree> [db]"),
);
match mode.as_str() {
"gen" => generate(&tree),
"cold" | "warm" => {
let db = PathBuf::from(std::env::args().nth(3).expect("usage: indexprobe <cold|warm> <tree> <db>"));
let db = PathBuf::from(
std::env::args()
.nth(3)
.expect("usage: indexprobe <cold|warm> <tree> <db>"),
);
if mode == "cold" {
for suffix in ["", "-wal", "-shm"] {
let _ = std::fs::remove_file(format!("{}{}", db.display(), suffix));

26
crates/quicksearch-core/examples/memprobe.rs
View file

@ -202,7 +202,9 @@ fn run(mode: &str, root: &str, db: &Path, interval: Duration) {
assert!(done, "indexing did not finish within {:?}", TIMEOUT);
service.stop_indexing().expect("stop");
report(mode, elapsed, baseline, hwm, &samples, db, interval, &at_peak);
report(
mode, elapsed, baseline, hwm, &samples, db, interval, &at_peak,
);
}
/// Flatten per-root progress into one line's worth of numbers. Roots are
@ -242,7 +244,10 @@ fn report(
) {
// One line per 5% of the run, so the shape is visible at any duration.
let step = (samples.len() / 20).max(1);
eprintln!("\n {:>8} {:>10} {:>9} {:>10} {}", "t", "RSS", "walked", "extracted", "phase");
eprintln!(
"\n {:>8} {:>10} {:>9} {:>10} {}",
"t", "RSS", "walked", "extracted", "phase"
);
for s in samples.iter().step_by(step) {
eprintln!(
" {:>7.1}s {:>10} {:>9} {:>10} {}",
@ -260,7 +265,12 @@ fn report(
let files = samples.iter().map(|s| s.walked).max().unwrap_or(0);
let db_bytes = db_size(db);
eprintln!("\n{} run: {:.1}s, {} files walked", mode, elapsed.as_secs_f64(), files);
eprintln!(
"\n{} run: {:.1}s, {} files walked",
mode,
elapsed.as_secs_f64(),
files
);
match hwm {
Some(h) => eprintln!(" peak RSS (VmHWM) {}", mib(h)),
None => eprintln!(" peak RSS (VmHWM) unavailable"),
@ -289,7 +299,10 @@ fn report(
);
}
}
eprintln!(" database on disk {} (counts toward RSS as page cache)", mib(db_bytes));
eprintln!(
" database on disk {} (counts toward RSS as page cache)",
mib(db_bytes)
);
if !at_peak.entries.is_empty() {
eprintln!("\n resident bytes at the peak, by mapping:");
@ -308,7 +321,10 @@ fn report(
.collect();
jumps.sort_by_key(|(delta, _)| std::cmp::Reverse(*delta));
if !jumps.is_empty() {
eprintln!("\n largest RSS rises between samples ({:?} apart):", interval);
eprintln!(
"\n largest RSS rises between samples ({:?} apart):",
interval
);
for (delta, s) in jumps.iter().take(8) {
eprintln!(
" +{:>9} to {:>10} at t={:>6.1}s {} {}",

9
crates/quicksearch-core/examples/walkprobe.rs
View file

@ -49,7 +49,10 @@ fn main() {
// Phase 1 in isolation: an empty index, so every file classifies as new.
// The parallel walker reads its classification data from a database now,
// so it gets a scratch one rather than an empty map.
let db = std::env::temp_dir().join(format!("quicksearch-walkprobe-{}.sqlite", std::process::id()));
let db = std::env::temp_dir().join(format!(
"quicksearch-walkprobe-{}.sqlite",
std::process::id()
));
let _ = std::fs::remove_file(&db);
quicksearch_core::db::open_or_recreate(db.to_str().unwrap(), &config.processing.tokenize)
.expect("scratch index");
@ -122,7 +125,9 @@ fn parallel(root: &str, config: &Config, db_path: &str) -> (usize, usize) {
Arc::new(AtomicBool::new(false)),
4,
) {
let WalkEvent::File(file) = event else { continue };
let WalkEvent::File(file) = event else {
continue;
};
seen += 1;
if file.record.is_some() {
prepared += 1;

12
crates/quicksearch-core/src/cli.rs
View file

@ -144,9 +144,7 @@ pub fn list_failed(db_path: &str, limit: Option<u32>) -> Result<Vec<FailedEntry>
/// Return a rough size breakdown of the database on disk and by table.
pub fn index_size_breakdown(db_path: &str) -> Result<SizeReport, String> {
let file_size_bytes = std::fs::metadata(db_path)
.map(|m| m.len())
.unwrap_or(0);
let file_size_bytes = std::fs::metadata(db_path).map(|m| m.len()).unwrap_or(0);
let conn = open_existing(db_path, false)?;
let count = |table: &str| -> Result<i64, String> {
conn.query_row(&format!("SELECT COUNT(*) FROM {}", table), [], |r| r.get(0))
@ -384,8 +382,12 @@ mod tests {
let ratio = r.documents_text_ratio().expect("has rows");
// Repeating a 44-byte sentence 500x → zstd should hit <20% ratio
// trivially. Loose bound protects the test from zstd version churn.
assert!(ratio < 0.3, "ratio too high: {ratio} raw={} comp={}",
r.documents_text_raw_bytes, r.documents_text_compressed_bytes);
assert!(
ratio < 0.3,
"ratio too high: {ratio} raw={} comp={}",
r.documents_text_raw_bytes,
r.documents_text_compressed_bytes
);
std::fs::remove_file(&p).ok();
}

137
crates/quicksearch-core/src/config.rs
View file

@ -253,9 +253,11 @@ impl SecurityConfig {
/// around.
pub fn salt_bytes(&self) -> Result<[u8; crate::security::SALT_LEN], String> {
match &self.salt {
None => Err("password protection is enabled but the config has no salt; \
None => Err(
"password protection is enabled but the config has no salt; \
disable protection or set the password again"
.to_string()),
.to_string(),
),
Some(hex) => crate::security::salt_from_hex(hex)
.map_err(|e| format!("invalid salt in config: {}", e)),
}
@ -457,10 +459,7 @@ impl Config {
/// location), creating parent directories as needed. Raw values are
/// written verbatim — relative paths in a portable config stay relative.
pub fn save(&self) -> Result<(), String> {
let path = self
.source
.clone()
.unwrap_or_else(Self::config_path);
let path = self.source.clone().unwrap_or_else(Self::config_path);
if let Some(dir) = path.parent() {
fs::create_dir_all(dir)
.map_err(|e| format!("Failed to create config dir {}: {}", dir.display(), e))?;
@ -566,8 +565,25 @@ impl IgnoreSet {
let mut path = globset::GlobSetBuilder::new();
for pat in patterns {
// Trailing separators are how people naturally write directory
// patterns ("/tmp/"); paths compare without them, so strip.
let pat = pat.trim().trim_end_matches(['/', '\\']);
// patterns ("/tmp/"); paths compare without them, so strip —
// except a drive root ("D:\" or "D:/"), where the separator is
// the whole point: trimmed to "D:" it would become a component
// pattern that can never match. Drive roots keep a normalized
// "D:/" spelling, which the ancestor walk in
// `matches_path_pattern` does reach.
let raw = pat.trim();
let trimmed = raw.trim_end_matches(['/', '\\']);
let is_drive_root = raw.len() > trimmed.len()
&& trimmed.len() == 2
&& trimmed.as_bytes()[0].is_ascii_alphabetic()
&& trimmed.as_bytes()[1] == b':';
let drive_root;
let pat: &str = if is_drive_root {
drive_root = format!("{}/", trimmed);
&drive_root
} else {
trimmed
};
if pat.is_empty() {
continue;
}
@ -781,7 +797,11 @@ mod tests {
fn partial_file_gets_section_defaults() {
let dir = tmp_dir();
let path = dir.join("config.toml");
fs::write(&path, "[paths]\nindexing_paths=[\"/x\"]\ndatabase_path=\"db.sqlite\"\n").unwrap();
fs::write(
&path,
"[paths]\nindexing_paths=[\"/x\"]\ndatabase_path=\"db.sqlite\"\n",
)
.unwrap();
let cfg = Config::load_from(&path).unwrap();
assert_eq!(cfg.paths.indexing_paths, vec!["/x".to_string()]);
assert_eq!(cfg.processing.batch_size, 500, "missing sections default");
@ -844,10 +864,16 @@ mod tests {
cfg.indexing.content_extensions = vec!["txt".into(), ".MD".into()];
assert!(content_allowed(Path::new("/a/b.txt"), &cfg));
assert!(content_allowed(Path::new("/a/B.TXT"), &cfg));
assert!(content_allowed(Path::new("/a/readme.md"), &cfg), "leading dot + case in filter");
assert!(
content_allowed(Path::new("/a/readme.md"), &cfg),
"leading dot + case in filter"
);
assert!(!content_allowed(Path::new("/a/b.pdf"), &cfg));
assert!(!content_allowed(Path::new("/a/noext"), &cfg));
assert!(!content_allowed(Path::new("/a/.bashrc"), &cfg), "dot-only name has no ext");
assert!(
!content_allowed(Path::new("/a/.bashrc"), &cfg),
"dot-only name has no ext"
);
}
#[test]
@ -855,9 +881,18 @@ mod tests {
let mut cfg = Config::default();
cfg.indexing.content_extensions = vec!["txt".into(), " (NonE) ".into()];
assert!(content_allowed(Path::new("/a/Makefile"), &cfg));
assert!(content_allowed(Path::new("/a/.bashrc"), &cfg), "dot-only name");
assert!(content_allowed(Path::new("/a/b.txt"), &cfg), "real extensions still work");
assert!(!content_allowed(Path::new("/a/b.pdf"), &cfg), "sentinel is not a wildcard");
assert!(
content_allowed(Path::new("/a/.bashrc"), &cfg),
"dot-only name"
);
assert!(
content_allowed(Path::new("/a/b.txt"), &cfg),
"real extensions still work"
);
assert!(
!content_allowed(Path::new("/a/b.pdf"), &cfg),
"sentinel is not a wildcard"
);
// The sentinel is not itself an extension: a file literally named
// `x.none` is not whitelisted by it.
assert!(!content_allowed(Path::new("/a/x.none"), &cfg));
@ -888,8 +923,14 @@ mod tests {
"(none) # Makefile, LICENSE, ...".into(),
];
assert!(content_allowed(Path::new("/a/b.rs"), &cfg));
assert!(content_allowed(Path::new("/a/b.md"), &cfg), "dot + trailing comment");
assert!(content_allowed(Path::new("/a/Makefile"), &cfg), "sentinel + comment");
assert!(
content_allowed(Path::new("/a/b.md"), &cfg),
"dot + trailing comment"
);
assert!(
content_allowed(Path::new("/a/Makefile"), &cfg),
"sentinel + comment"
);
assert!(!content_allowed(Path::new("/a/b.pdf"), &cfg));
// Comment text is not itself a filter entry.
assert!(!content_allowed(Path::new("/a/b.rust"), &cfg));
@ -950,10 +991,10 @@ mod tests {
#[test]
fn directory_patterns_with_trailing_slash() {
let set = IgnoreSet::compile(&[
"/tmp/".to_string(), // absolute dir, natural spelling
"cache/".to_string(), // becomes a component pattern
"*/target/".to_string(), // dir anywhere by suffix
"/".to_string(), // degenerate: trims to nothing, skipped
"/tmp/".to_string(), // absolute dir, natural spelling
"cache/".to_string(), // becomes a component pattern
"*/target/".to_string(), // dir anywhere by suffix
"/".to_string(), // degenerate: trims to nothing, skipped
])
.unwrap();
// The directory itself and everything beneath it.
@ -968,6 +1009,41 @@ mod tests {
assert!(!set.matches_path(Path::new("/etc/passwd")));
}
/// A drive-root pattern must survive the trailing-separator trim as a
/// path pattern — trimmed to "D:" it would land in the component set,
/// where nothing is ever named "D:".
#[test]
fn drive_root_patterns_are_not_component_patterns() {
let set = IgnoreSet::compile(&[r"D:\".to_string(), "E:/".to_string()]).unwrap();
assert!(!set.matches_component("D:"));
assert!(!set.matches_component(r"D:\"));
assert!(!set.matches_component("E:"));
}
/// The full drive-root behavior needs Windows path semantics:
/// `Path::parent` only walks up to `D:\` there, and globset only folds
/// `\` to `/` where `\` is a separator.
#[cfg(windows)]
#[test]
fn drive_root_pattern_ignores_the_whole_drive() {
let set = IgnoreSet::compile(&[r"D:\".to_string()]).unwrap();
assert!(set.matches_path(Path::new(r"D:\")));
assert!(set.matches_path(Path::new(r"D:\Users\x\file.txt")));
assert!(set.matches_path(Path::new(r"d:\case\folded.txt")));
assert!(!set.matches_path(Path::new(r"E:\file.txt")));
}
/// A bare "D:" (no separator) compiles but can only match a component
/// literally named "D:", which no file ever is. The GUI warns about
/// this shape; the compiler intentionally leaves it alone.
#[test]
fn bare_drive_letter_stays_a_component_pattern() {
let set = IgnoreSet::compile(&["D:".to_string()]).unwrap();
assert!(set.matches_component("D:"));
#[cfg(windows)]
assert!(!set.matches_path(Path::new(r"D:\file.txt")));
}
#[test]
fn ignore_set_invalid_pattern_errors() {
let err = IgnoreSet::compile(&["[".to_string()]).unwrap_err();
@ -987,7 +1063,10 @@ mod tests {
#[test]
fn ignore_matching_follows_platform_case_rules() {
let set = IgnoreSet::compile(&["node_modules".to_string()]).unwrap();
assert!(set.matches_component("node_modules"), "exact always matches");
assert!(
set.matches_component("node_modules"),
"exact always matches"
);
let folded = cfg!(any(windows, target_os = "macos"));
assert_eq!(
@ -1080,7 +1159,11 @@ mod tests {
cfg.save().unwrap();
let loaded = Config::load_from(&path).unwrap();
assert_eq!(loaded.indexing.root_workers.get("/share"), Some(&24));
assert_eq!(loaded.indexing.root_workers.get("/data"), None, "absent = auto");
assert_eq!(
loaded.indexing.root_workers.get("/data"),
None,
"absent = auto"
);
fs::remove_dir_all(&dir).ok();
}
@ -1305,11 +1388,17 @@ mod tests {
let mut cfg = SearchConfig::default();
for quiet in 0..=FUZZY_EDITS_WARN_ABOVE {
cfg.fuzzy_max_edits = quiet;
assert!(cfg.fuzzy_edits_warning().is_none(), "{} should be quiet", quiet);
assert!(
cfg.fuzzy_edits_warning().is_none(),
"{} should be quiet",
quiet
);
}
for loud in [FUZZY_EDITS_WARN_ABOVE + 1, 8, usize::MAX] {
cfg.fuzzy_max_edits = loud;
let msg = cfg.fuzzy_edits_warning().expect("warns above the threshold");
let msg = cfg
.fuzzy_edits_warning()
.expect("warns above the threshold");
assert!(msg.contains(&loud.to_string()));
assert!(msg.contains(&FUZZY_EDITS_WARN_ABOVE.to_string()));
}

33
crates/quicksearch-core/src/content.rs
View file

@ -223,19 +223,16 @@ fn feeder(shared: &Shared, db_path: &str, mut cursor: ExtractCursor, max_size: i
};
while shared.take_feed_slot().is_some() {
let page = match crate::db::repo::pending_content_page(
&conn,
&cursor,
max_size,
FEED_PAGE as i64,
) {
Ok(page) => page,
Err(e) => {
crate::log_warn!("{}", e);
shared.shutdown();
return;
}
};
let page =
match crate::db::repo::pending_content_page(&conn, &cursor, max_size, FEED_PAGE as i64)
{
Ok(page) => page,
Err(e) => {
crate::log_warn!("{}", e);
shared.shutdown();
return;
}
};
let last_page = page.len() < FEED_PAGE;
if let Some((id, _, _, _)) = page.last() {
cursor.last_id = *id;
@ -318,7 +315,15 @@ pub fn extract_content(
let stats = stats.clone();
thread::spawn(move || {
crate::platform::set_background_priority();
worker(&shared, &tx, &registry, &config, &stop_flag, &suspend_flag, &stats)
worker(
&shared,
&tx,
&registry,
&config,
&stop_flag,
&suspend_flag,
&stats,
)
})
})
.collect();

51
crates/quicksearch-core/src/coordinator.rs
View file

@ -209,7 +209,10 @@ impl IndexCoordinator {
}
/// Compare `config` against what the index was built with. Read-only.
pub fn check_config_validation(&self, config: &Config) -> Result<Option<Vec<ConfigChange>>, String> {
pub fn check_config_validation(
&self,
config: &Config,
) -> Result<Option<Vec<ConfigChange>>, String> {
let db = config.resolved_database_path();
let roots = joined_roots(config);
self.indexing
@ -511,7 +514,10 @@ impl Inner {
Ok(conn) => conn,
Err(e) => {
// Missing or stale DB: incremental can't help, rebuild.
crate::log_warn!("coordinator: incremental unavailable ({}); scheduling full run", e);
crate::log_warn!(
"coordinator: incremental unavailable ({}); scheduling full run",
e
);
self.needs_full_run = true;
return;
}
@ -609,7 +615,8 @@ impl Inner {
for (child, parent) in &nested {
crate::log_warn!(
"coordinator: refusing to index: root {} is nested under {}",
child, parent
child,
parent
);
}
return;
@ -1012,9 +1019,11 @@ mod tests {
// Periodic reindex is the fallback and must still run: mode stays
// Auto, only the watcher is off.
assert_eq!(coord.state().mode, IndexMode::Auto);
wait_for("full run despite no watcher", Duration::from_secs(20), || {
coord.state().last_full_index.is_some() && f.file_count() == 1
});
wait_for(
"full run despite no watcher",
Duration::from_secs(20),
|| coord.state().last_full_index.is_some() && f.file_count() == 1,
);
coord.shutdown();
}
@ -1111,10 +1120,11 @@ mod tests {
f.file_count() == 1
});
let extra_root = f.dir.parent().unwrap().join(format!(
"qs-coord-extra-{}",
std::process::id()
));
let extra_root = f
.dir
.parent()
.unwrap()
.join(format!("qs-coord-extra-{}", std::process::id()));
std::fs::create_dir_all(&extra_root).unwrap();
std::fs::write(extra_root.join("second.txt"), "two").unwrap();
@ -1147,7 +1157,10 @@ mod tests {
&mut pending,
FsEvent::Remove(dir.join(format!("sub{}/f{}.txt", i % 5, i))),
);
enqueue(&mut pending, FsEvent::Remove(dir.join(format!("sub{}", i % 5))));
enqueue(
&mut pending,
FsEvent::Remove(dir.join(format!("sub{}", i % 5))),
);
}
// Not under the removed tree, and not a removal: both must survive.
enqueue(&mut pending, FsEvent::Remove(PathBuf::from("/x/treehouse")));
@ -1200,9 +1213,11 @@ mod tests {
});
std::fs::remove_dir_all(&tree).unwrap();
wait_for("subtree removed from the index", Duration::from_secs(30), || {
f.file_count() == 1
});
wait_for(
"subtree removed from the index",
Duration::from_secs(30),
|| f.file_count() == 1,
);
coord.shutdown();
}
@ -1234,9 +1249,11 @@ mod tests {
std::fs::remove_dir_all(f.dir.join("d0")).unwrap();
// 300 files, 50 of them under d0.
wait_for("deletion applied after the run", Duration::from_secs(60), || {
f.file_count() == 250
});
wait_for(
"deletion applied after the run",
Duration::from_secs(60),
|| f.file_count() == 250,
);
coord.shutdown();
}

5
crates/quicksearch-core/src/db/key.rs
View file

@ -22,7 +22,10 @@ pub fn set_process_key(key: Option<IndexKey>) {
/// Snapshot of the current key for a single open.
pub(crate) fn process_key() -> Option<IndexKey> {
PROCESS_KEY.read().expect("process key lock poisoned").clone()
PROCESS_KEY
.read()
.expect("process key lock poisoned")
.clone()
}
/// Hex form of the installed key, if any. Exists for exactly one consumer:

36
crates/quicksearch-core/src/db/open.rs
View file

@ -134,7 +134,11 @@ pub(crate) fn open_existing_keyed(
write: bool,
key: Option<&IndexKey>,
) -> Result<Connection, String> {
let pragmas = if write { PRAGMAS_FAST } else { PRAGMAS_READONLY };
let pragmas = if write {
PRAGMAS_FAST
} else {
PRAGMAS_READONLY
};
open_keyed_with_pragmas(db_path, write, key, pragmas)
}
@ -295,8 +299,10 @@ fn key_mismatch_message(db_path: &str, had_key: bool) -> String {
})
.unwrap_or(false);
let detail = match (had_key, plaintext) {
(true, true) => "password protection is enabled but the index is not encrypted; \
rebuild the index to encrypt it",
(true, true) => {
"password protection is enabled but the index is not encrypted; \
rebuild the index to encrypt it"
}
(true, false) => "wrong password (or the file is not a QuickSearch index)",
(false, _) => "the index is password-protected; a password is required",
};
@ -709,12 +715,16 @@ mod tests {
let p = tmp_db_path();
{
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let indexer: i64 = conn.query_row("PRAGMA temp_store", [], |r| r.get(0)).unwrap();
let indexer: i64 = conn
.query_row("PRAGMA temp_store", [], |r| r.get(0))
.unwrap();
assert_eq!(indexer, 2, "the indexer's own profile is MEMORY");
}
let conn = open_maintenance(p.to_str().unwrap()).unwrap();
let store: i64 = conn.query_row("PRAGMA temp_store", [], |r| r.get(0)).unwrap();
let store: i64 = conn
.query_row("PRAGMA temp_store", [], |r| r.get(0))
.unwrap();
assert_eq!(store, 1, "maintenance must build its temporaries on disk");
// And the directory those temporaries land in is steerable, which is
@ -726,7 +736,8 @@ mod tests {
.query_row("PRAGMA temp_store_directory", [], |r| r.get(0))
.unwrap();
assert_eq!(set, dir);
conn.execute_batch("PRAGMA temp_store_directory = '';").unwrap();
conn.execute_batch("PRAGMA temp_store_directory = '';")
.unwrap();
drop(conn);
std::fs::remove_file(&p).ok();
@ -754,8 +765,11 @@ mod tests {
// Age it, exactly as a version bump does.
{
let conn = open_existing(p.to_str().unwrap(), true).unwrap();
conn.execute("UPDATE schema_info SET value = '1' WHERE key = 'version'", [])
.unwrap();
conn.execute(
"UPDATE schema_info SET value = '1' WHERE key = 'version'",
[],
)
.unwrap();
}
assert!(index_needs_rebuild(p.to_str().unwrap()));
@ -836,7 +850,7 @@ mod tests {
conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \
VALUES ('secret', '/secret.txt', '/', 0, 0)",
[],
[],
)
.unwrap();
}
@ -877,8 +891,8 @@ mod tests {
}
let before = file_bytes(&p);
for write in [false, true] {
let err = open_existing_keyed(p.to_str().unwrap(), write, Some(&test_key(0xb2)))
.unwrap_err();
let err =
open_existing_keyed(p.to_str().unwrap(), write, Some(&test_key(0xb2))).unwrap_err();
assert!(err.starts_with(KEY_MISMATCH_PREFIX), "got: {err}");
}
// The owner path must error too — a wrong key is never a "schema

46
crates/quicksearch-core/src/db/repo.rs
View file

@ -106,10 +106,7 @@ fn initial_content_state(f: &NewFile<'_>) -> i64 {
/// `failure_msg` — and only those. `name`, `parent`, `inode` and `device_id`
/// are not refreshed here despite being present on the `NewFile`: the row is
/// found by path, and the first two are functions of it.
pub fn update_file_basic(
tx: &Transaction<'_>,
f: &NewFile<'_>,
) -> Result<Option<i64>, String> {
pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option<i64>, String> {
// One statement, not a lookup then an update: `RETURNING` hands back the
// id of the row it just wrote, and a miss is simply no row returned.
let id: Option<i64> = tx
@ -210,11 +207,7 @@ pub fn set_content_done(
const ZSTD_LEVEL: i32 = 3;
/// Mark a file's content extraction as failed. Keeps the basic row in place.
pub fn set_content_failed(
tx: &Transaction<'_>,
file_id: i64,
reason: &str,
) -> Result<(), String> {
pub fn set_content_failed(tx: &Transaction<'_>, file_id: i64, reason: &str) -> Result<(), String> {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
@ -794,7 +787,9 @@ mod tests {
row.mtime = 3;
row.mime = Some("video/mp4");
row.needs_content = false;
update_file_basic(&tx, &row).unwrap().expect("row still there");
update_file_basic(&tx, &row)
.unwrap()
.expect("row still there");
tx.commit().unwrap();
}
assert_eq!(content_state(&conn), STATE_NA);
@ -910,7 +905,9 @@ mod tests {
);
let survivors: Vec<String> = {
let mut stmt = conn.prepare("SELECT path FROM files ORDER BY path").unwrap();
let mut stmt = conn
.prepare("SELECT path FROM files ORDER BY path")
.unwrap();
let v = stmt
.query_map([], |r| r.get::<_, String>(0))
.unwrap()
@ -1051,8 +1048,15 @@ mod tests {
)
.unwrap()
.unwrap();
set_content_done(&tx, id, &name, &"lorem ipsum dolor sit amet ".repeat(64), &[], true)
.unwrap();
set_content_done(
&tx,
id,
&name,
&"lorem ipsum dolor sit amet ".repeat(64),
&[],
true,
)
.unwrap();
}
tx.commit().unwrap();
}
@ -1193,7 +1197,10 @@ mod tests {
let forced = run(&bounded, 4);
std::fs::remove_file(&bounded).ok();
eprintln!("peak WAL: autocheckpoint only {}, forced {}", left_alone, forced);
eprintln!(
"peak WAL: autocheckpoint only {}, forced {}",
left_alone, forced
);
assert!(
forced * 2 < left_alone,
"forcing checkpoints did not bound the log: {} vs {}",
@ -1226,8 +1233,15 @@ mod tests {
assert!(freelist > 0, "the deletions should have freed pages");
let dir = p.parent().unwrap().to_string_lossy().into_owned();
assert!(maintain(&conn, &dir).unwrap(), "that much slack is worth a vacuum");
assert_eq!(wal_bytes(&p), 0, "the vacuum's own writes are checkpointed too");
assert!(
maintain(&conn, &dir).unwrap(),
"that much slack is worth a vacuum"
);
assert_eq!(
wal_bytes(&p),
0,
"the vacuum's own writes are checkpointed too"
);
assert!(
std::fs::metadata(&p).unwrap().len() < before,
"the file should have shrunk"

10
crates/quicksearch-core/src/db/schema.rs
View file

@ -203,8 +203,14 @@ mod tests {
#[test]
fn plain_trigram_gets_accent_stripping() {
assert_eq!(effective_tokenizer("trigram"), "trigram remove_diacritics 1");
assert_eq!(effective_tokenizer(" trigram "), "trigram remove_diacritics 1");
assert_eq!(
effective_tokenizer("trigram"),
"trigram remove_diacritics 1"
);
assert_eq!(
effective_tokenizer(" trigram "),
"trigram remove_diacritics 1"
);
}
#[test]

737
crates/quicksearch-core/src/document_extraction.rs
View file

@ -1,364 +1,373 @@
use std::ffi::OsString;
use std::fs::File;
use std::io::{Read, BufReader};
use zip::ZipArchive;
use quick_xml::Reader;
use quick_xml::events::Event;
/// Extract text from DOCX files by parsing the word/document.xml
pub fn extract_text_from_docx(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
let file = File::open(file_path)?;
let mut archive = ZipArchive::new(BufReader::new(file))?;
let mut document_xml = archive.by_name("word/document.xml")?;
let mut content = String::new();
document_xml.read_to_string(&mut content)?;
let mut reader = Reader::from_str(&content);
reader.trim_text(true);
let mut text_content = String::new();
let mut buf = Vec::new();
let mut in_text = false;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
if e.name().as_ref() == b"w:t" {
in_text = true;
}
}
Ok(Event::Text(e)) if in_text => {
text_content.push_str(&e.unescape()?.into_owned());
}
Ok(Event::End(ref e)) => {
if e.name().as_ref() == b"w:t" {
in_text = false;
} else if e.name().as_ref() == b"w:p" {
text_content.push('\n');
}
}
Ok(Event::Eof) => break,
Err(e) => return Err(format!("Error parsing XML: {}", e).into()),
_ => {}
}
buf.clear();
}
Ok(text_content)
}
/// Extract text from XLSX files by parsing worksheet XML files
pub fn extract_text_from_xlsx(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
let file = File::open(file_path)?;
let mut archive = ZipArchive::new(BufReader::new(file))?;
let mut text_content = String::new();
// First, read shared strings if they exist
let mut shared_strings = Vec::new();
if let Ok(mut shared_strings_xml) = archive.by_name("xl/sharedStrings.xml") {
let mut content = String::new();
shared_strings_xml.read_to_string(&mut content)?;
let mut reader = Reader::from_str(&content);
reader.trim_text(true);
let mut buf = Vec::new();
let mut in_text = false;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
if e.name().as_ref() == b"t" {
in_text = true;
}
}
Ok(Event::Text(e)) if in_text => {
shared_strings.push(e.unescape()?.into_owned());
}
Ok(Event::End(ref e)) => {
if e.name().as_ref() == b"t" {
in_text = false;
}
}
Ok(Event::Eof) => break,
_ => {}
}
buf.clear();
}
}
// Read worksheets
for i in 0..archive.len() {
let file_name = archive.by_index(i)?.name().to_string();
if file_name.starts_with("xl/worksheets/sheet") && file_name.ends_with(".xml") {
let mut sheet_xml = archive.by_index(i)?;
let mut content = String::new();
sheet_xml.read_to_string(&mut content)?;
let mut reader = Reader::from_str(&content);
reader.trim_text(true);
let mut buf = Vec::new();
let mut in_cell = false;
let mut cell_type = String::new();
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
if e.name().as_ref() == b"c" {
in_cell = true;
cell_type.clear();
for attr in e.attributes() {
let attr = attr?;
if attr.key.as_ref() == b"t" {
cell_type = String::from_utf8_lossy(&attr.value).to_string();
}
}
} else if e.name().as_ref() == b"v" && in_cell {
// Value element
}
}
Ok(Event::Text(e)) if in_cell => {
let text = e.unescape()?.into_owned();
if cell_type == "s" {
// Shared string reference
if let Ok(index) = text.parse::<usize>() {
if index < shared_strings.len() {
text_content.push_str(&shared_strings[index]);
text_content.push(' ');
}
}
} else {
text_content.push_str(&text);
text_content.push(' ');
}
}
Ok(Event::End(ref e)) => {
if e.name().as_ref() == b"c" {
in_cell = false;
} else if e.name().as_ref() == b"row" {
text_content.push('\n');
}
}
Ok(Event::Eof) => break,
_ => {}
}
buf.clear();
}
}
}
Ok(text_content)
}
/// Extract text from PPTX files by parsing slide XML files
pub fn extract_text_from_pptx(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
let file = File::open(file_path)?;
let mut archive = ZipArchive::new(BufReader::new(file))?;
let mut text_content = String::new();
// Read all slide files
for i in 0..archive.len() {
let file_name = archive.by_index(i)?.name().to_string();
if file_name.starts_with("ppt/slides/slide") && file_name.ends_with(".xml") {
let mut slide_xml = archive.by_index(i)?;
let mut content = String::new();
slide_xml.read_to_string(&mut content)?;
let mut reader = Reader::from_str(&content);
reader.trim_text(true);
let mut buf = Vec::new();
let mut in_text = false;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
if e.name().as_ref() == b"a:t" {
in_text = true;
}
}
Ok(Event::Text(e)) if in_text => {
text_content.push_str(&e.unescape()?.into_owned());
}
Ok(Event::End(ref e)) => {
if e.name().as_ref() == b"a:t" {
in_text = false;
} else if e.name().as_ref() == b"a:p" {
text_content.push('\n');
}
}
Ok(Event::Eof) => break,
_ => {}
}
buf.clear();
}
text_content.push_str("\n--- New Slide ---\n");
}
}
Ok(text_content)
}
/// Extract text from ODT files (OpenDocument Text)
pub fn extract_text_from_odt(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
let file = File::open(file_path)?;
let mut archive = ZipArchive::new(BufReader::new(file))?;
let mut content_xml = archive.by_name("content.xml")?;
let mut content = String::new();
content_xml.read_to_string(&mut content)?;
let mut reader = Reader::from_str(&content);
reader.trim_text(true);
let mut text_content = String::new();
let mut buf = Vec::new();
let mut in_text = false;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
let name = e.name();
if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" || name.as_ref() == b"text:span" {
in_text = true;
}
}
Ok(Event::Text(e)) if in_text => {
text_content.push_str(&e.unescape()?.into_owned());
}
Ok(Event::End(ref e)) => {
let name = e.name();
if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" {
text_content.push('\n');
in_text = false;
} else if name.as_ref() == b"text:span" {
in_text = false;
}
}
Ok(Event::Eof) => break,
_ => {}
}
buf.clear();
}
Ok(text_content)
}
/// Extract text from ODP files (OpenDocument Presentation)
pub fn extract_text_from_odp(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
let file = File::open(file_path)?;
let mut archive = ZipArchive::new(BufReader::new(file))?;
let mut content_xml = archive.by_name("content.xml")?;
let mut content = String::new();
content_xml.read_to_string(&mut content)?;
let mut reader = Reader::from_str(&content);
reader.trim_text(true);
let mut text_content = String::new();
let mut buf = Vec::new();
let mut in_text = false;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
let name = e.name();
if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" || name.as_ref() == b"text:span" {
in_text = true;
}
}
Ok(Event::Text(e)) if in_text => {
text_content.push_str(&e.unescape()?.into_owned());
}
Ok(Event::End(ref e)) => {
let name = e.name();
if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" {
text_content.push('\n');
in_text = false;
} else if name.as_ref() == b"text:span" {
in_text = false;
}
}
Ok(Event::Eof) => break,
_ => {}
}
buf.clear();
}
Ok(text_content)
}
/// Extract text from ODS files (OpenDocument Spreadsheet)
pub fn extract_text_from_ods(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
let file = File::open(file_path)?;
let mut archive = ZipArchive::new(BufReader::new(file))?;
let mut content_xml = archive.by_name("content.xml")?;
let mut content = String::new();
content_xml.read_to_string(&mut content)?;
let mut reader = Reader::from_str(&content);
reader.trim_text(true);
let mut text_content = String::new();
let mut buf = Vec::new();
let mut in_text = false;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
let name = e.name();
if name.as_ref() == b"text:p" || name.as_ref() == b"text:span" {
in_text = true;
}
}
Ok(Event::Text(e)) if in_text => {
text_content.push_str(&e.unescape()?.into_owned());
text_content.push(' ');
}
Ok(Event::End(ref e)) => {
let name = e.name();
if name.as_ref() == b"text:p" {
text_content.push('\n');
in_text = false;
} else if name.as_ref() == b"text:span" {
in_text = false;
}
}
Ok(Event::Eof) => break,
_ => {}
}
buf.clear();
}
Ok(text_content)
}
/// Extract text from various document formats
pub fn extract_document_text(file_path: &OsString, extension: &str) -> Result<String, Box<dyn std::error::Error>> {
match extension {
"docx" => extract_text_from_docx(file_path),
"doc" => {
// DOC format is binary and complex to parse without external tools
// For now, return an empty result
Ok(String::new())
}
"xlsx" => extract_text_from_xlsx(file_path),
"xls" => {
// XLS format is binary and complex to parse without external tools
Ok(String::new())
}
"pptx" => extract_text_from_pptx(file_path),
"ppt" => {
// PPT format is binary and complex to parse without external tools
Ok(String::new())
}
"odt" => extract_text_from_odt(file_path),
"odp" => extract_text_from_odp(file_path),
"ods" => extract_text_from_ods(file_path),
_ => Ok(String::new())
}
}
use std::ffi::OsString;
use std::fs::File;
use std::io::{BufReader, Read};
use quick_xml::events::Event;
use quick_xml::Reader;
use zip::ZipArchive;
/// Extract text from DOCX files by parsing the word/document.xml
pub fn extract_text_from_docx(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
let file = File::open(file_path)?;
let mut archive = ZipArchive::new(BufReader::new(file))?;
let mut document_xml = archive.by_name("word/document.xml")?;
let mut content = String::new();
document_xml.read_to_string(&mut content)?;
let mut reader = Reader::from_str(&content);
reader.trim_text(true);
let mut text_content = String::new();
let mut buf = Vec::new();
let mut in_text = false;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
if e.name().as_ref() == b"w:t" {
in_text = true;
}
}
Ok(Event::Text(e)) if in_text => {
text_content.push_str(&e.unescape()?.into_owned());
}
Ok(Event::End(ref e)) => {
if e.name().as_ref() == b"w:t" {
in_text = false;
} else if e.name().as_ref() == b"w:p" {
text_content.push('\n');
}
}
Ok(Event::Eof) => break,
Err(e) => return Err(format!("Error parsing XML: {}", e).into()),
_ => {}
}
buf.clear();
}
Ok(text_content)
}
/// Extract text from XLSX files by parsing worksheet XML files
pub fn extract_text_from_xlsx(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
let file = File::open(file_path)?;
let mut archive = ZipArchive::new(BufReader::new(file))?;
let mut text_content = String::new();
// First, read shared strings if they exist
let mut shared_strings = Vec::new();
if let Ok(mut shared_strings_xml) = archive.by_name("xl/sharedStrings.xml") {
let mut content = String::new();
shared_strings_xml.read_to_string(&mut content)?;
let mut reader = Reader::from_str(&content);
reader.trim_text(true);
let mut buf = Vec::new();
let mut in_text = false;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
if e.name().as_ref() == b"t" {
in_text = true;
}
}
Ok(Event::Text(e)) if in_text => {
shared_strings.push(e.unescape()?.into_owned());
}
Ok(Event::End(ref e)) => {
if e.name().as_ref() == b"t" {
in_text = false;
}
}
Ok(Event::Eof) => break,
_ => {}
}
buf.clear();
}
}
// Read worksheets
for i in 0..archive.len() {
let file_name = archive.by_index(i)?.name().to_string();
if file_name.starts_with("xl/worksheets/sheet") && file_name.ends_with(".xml") {
let mut sheet_xml = archive.by_index(i)?;
let mut content = String::new();
sheet_xml.read_to_string(&mut content)?;
let mut reader = Reader::from_str(&content);
reader.trim_text(true);
let mut buf = Vec::new();
let mut in_cell = false;
let mut cell_type = String::new();
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
if e.name().as_ref() == b"c" {
in_cell = true;
cell_type.clear();
for attr in e.attributes() {
let attr = attr?;
if attr.key.as_ref() == b"t" {
cell_type = String::from_utf8_lossy(&attr.value).to_string();
}
}
} else if e.name().as_ref() == b"v" && in_cell {
// Value element
}
}
Ok(Event::Text(e)) if in_cell => {
let text = e.unescape()?.into_owned();
if cell_type == "s" {
// Shared string reference
if let Ok(index) = text.parse::<usize>() {
if index < shared_strings.len() {
text_content.push_str(&shared_strings[index]);
text_content.push(' ');
}
}
} else {
text_content.push_str(&text);
text_content.push(' ');
}
}
Ok(Event::End(ref e)) => {
if e.name().as_ref() == b"c" {
in_cell = false;
} else if e.name().as_ref() == b"row" {
text_content.push('\n');
}
}
Ok(Event::Eof) => break,
_ => {}
}
buf.clear();
}
}
}
Ok(text_content)
}
/// Extract text from PPTX files by parsing slide XML files
pub fn extract_text_from_pptx(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
let file = File::open(file_path)?;
let mut archive = ZipArchive::new(BufReader::new(file))?;
let mut text_content = String::new();
// Read all slide files
for i in 0..archive.len() {
let file_name = archive.by_index(i)?.name().to_string();
if file_name.starts_with("ppt/slides/slide") && file_name.ends_with(".xml") {
let mut slide_xml = archive.by_index(i)?;
let mut content = String::new();
slide_xml.read_to_string(&mut content)?;
let mut reader = Reader::from_str(&content);
reader.trim_text(true);
let mut buf = Vec::new();
let mut in_text = false;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
if e.name().as_ref() == b"a:t" {
in_text = true;
}
}
Ok(Event::Text(e)) if in_text => {
text_content.push_str(&e.unescape()?.into_owned());
}
Ok(Event::End(ref e)) => {
if e.name().as_ref() == b"a:t" {
in_text = false;
} else if e.name().as_ref() == b"a:p" {
text_content.push('\n');
}
}
Ok(Event::Eof) => break,
_ => {}
}
buf.clear();
}
text_content.push_str("\n--- New Slide ---\n");
}
}
Ok(text_content)
}
/// Extract text from ODT files (OpenDocument Text)
pub fn extract_text_from_odt(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
let file = File::open(file_path)?;
let mut archive = ZipArchive::new(BufReader::new(file))?;
let mut content_xml = archive.by_name("content.xml")?;
let mut content = String::new();
content_xml.read_to_string(&mut content)?;
let mut reader = Reader::from_str(&content);
reader.trim_text(true);
let mut text_content = String::new();
let mut buf = Vec::new();
let mut in_text = false;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
let name = e.name();
if name.as_ref() == b"text:p"
|| name.as_ref() == b"text:h"
|| name.as_ref() == b"text:span"
{
in_text = true;
}
}
Ok(Event::Text(e)) if in_text => {
text_content.push_str(&e.unescape()?.into_owned());
}
Ok(Event::End(ref e)) => {
let name = e.name();
if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" {
text_content.push('\n');
in_text = false;
} else if name.as_ref() == b"text:span" {
in_text = false;
}
}
Ok(Event::Eof) => break,
_ => {}
}
buf.clear();
}
Ok(text_content)
}
/// Extract text from ODP files (OpenDocument Presentation)
pub fn extract_text_from_odp(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
let file = File::open(file_path)?;
let mut archive = ZipArchive::new(BufReader::new(file))?;
let mut content_xml = archive.by_name("content.xml")?;
let mut content = String::new();
content_xml.read_to_string(&mut content)?;
let mut reader = Reader::from_str(&content);
reader.trim_text(true);
let mut text_content = String::new();
let mut buf = Vec::new();
let mut in_text = false;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
let name = e.name();
if name.as_ref() == b"text:p"
|| name.as_ref() == b"text:h"
|| name.as_ref() == b"text:span"
{
in_text = true;
}
}
Ok(Event::Text(e)) if in_text => {
text_content.push_str(&e.unescape()?.into_owned());
}
Ok(Event::End(ref e)) => {
let name = e.name();
if name.as_ref() == b"text:p" || name.as_ref() == b"text:h" {
text_content.push('\n');
in_text = false;
} else if name.as_ref() == b"text:span" {
in_text = false;
}
}
Ok(Event::Eof) => break,
_ => {}
}
buf.clear();
}
Ok(text_content)
}
/// Extract text from ODS files (OpenDocument Spreadsheet)
pub fn extract_text_from_ods(file_path: &OsString) -> Result<String, Box<dyn std::error::Error>> {
let file = File::open(file_path)?;
let mut archive = ZipArchive::new(BufReader::new(file))?;
let mut content_xml = archive.by_name("content.xml")?;
let mut content = String::new();
content_xml.read_to_string(&mut content)?;
let mut reader = Reader::from_str(&content);
reader.trim_text(true);
let mut text_content = String::new();
let mut buf = Vec::new();
let mut in_text = false;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
let name = e.name();
if name.as_ref() == b"text:p" || name.as_ref() == b"text:span" {
in_text = true;
}
}
Ok(Event::Text(e)) if in_text => {
text_content.push_str(&e.unescape()?.into_owned());
text_content.push(' ');
}
Ok(Event::End(ref e)) => {
let name = e.name();
if name.as_ref() == b"text:p" {
text_content.push('\n');
in_text = false;
} else if name.as_ref() == b"text:span" {
in_text = false;
}
}
Ok(Event::Eof) => break,
_ => {}
}
buf.clear();
}
Ok(text_content)
}
/// Extract text from various document formats
pub fn extract_document_text(
file_path: &OsString,
extension: &str,
) -> Result<String, Box<dyn std::error::Error>> {
match extension {
"docx" => extract_text_from_docx(file_path),
"doc" => {
// DOC format is binary and complex to parse without external tools
// For now, return an empty result
Ok(String::new())
}
"xlsx" => extract_text_from_xlsx(file_path),
"xls" => {
// XLS format is binary and complex to parse without external tools
Ok(String::new())
}
"pptx" => extract_text_from_pptx(file_path),
"ppt" => {
// PPT format is binary and complex to parse without external tools
Ok(String::new())
}
"odt" => extract_text_from_odt(file_path),
"odp" => extract_text_from_odp(file_path),
"ods" => extract_text_from_ods(file_path),
_ => Ok(String::new()),
}
}

3
crates/quicksearch-core/src/extract/image.rs
View file

@ -18,8 +18,7 @@ impl Extractor for ImageExtractor {
}
fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError> {
let file = File::open(path)
.map_err(|e| format!("image open {}: {}", path.display(), e))?;
let file = File::open(path).map_err(|e| format!("image open {}: {}", path.display(), e))?;
let mut bufreader = BufReader::new(&file);
let mut out = ExtractedContent::default();

37
crates/quicksearch-core/src/extract/mod.rs
View file

@ -114,7 +114,9 @@ pub struct Registry {
impl Registry {
pub fn new() -> Self {
Self { extractors: Vec::new() }
Self {
extractors: Vec::new(),
}
}
pub fn with(mut self, e: impl Extractor + 'static) -> Self {
@ -167,7 +169,8 @@ impl Registry {
mime: &str,
head: &[u8],
) -> Option<Result<ExtractedContent, ExtractError>> {
self.find(mime).and_then(|e| e.extract_from_head(path, head))
self.find(mime)
.and_then(|e| e.extract_from_head(path, head))
}
/// The default set: RTF, plaintext, office docs, PDF, audio tags,
@ -220,11 +223,17 @@ mod tests {
// A format that seeks or reads a trailer must not be handed a buffer.
// `None` here is what routes it back to the on-disk extractor.
assert!(r.extract_complete_head(p, "application/pdf", b"%PDF-1.4").is_none());
assert!(r.extract_complete_head(p, "image/png", b"\x89PNG").is_none());
assert!(r
.extract_complete_head(p, "application/pdf", b"%PDF-1.4")
.is_none());
assert!(r
.extract_complete_head(p, "image/png", b"\x89PNG")
.is_none());
// No extractor claims the MIME at all.
assert!(r.extract_complete_head(p, "application/x-nonesuch", b"..").is_none());
assert!(r
.extract_complete_head(p, "application/x-nonesuch", b"..")
.is_none());
}
#[test]
@ -233,16 +242,23 @@ mod tests {
// file's text would depend on which pass happened to handle it.
let r = Registry::default_set();
let p = Path::new("/tmp/whatever");
for mime in ["text/plain", "TEXT/PLAIN", "application/json", "application/x-sql"] {
for mime in [
"text/plain",
"TEXT/PLAIN",
"application/json",
"application/x-sql",
] {
assert!(
r.extract_complete_head(p, mime, b"x").is_some(),
"{} should extract from a head", mime
"{} should extract from a head",
mime
);
}
for mime in ["application/rtf", "text/rtf"] {
assert!(
r.extract_complete_head(p, mime, br"{\rtf1 x}").is_some(),
"{} should extract from a head", mime
"{} should extract from a head",
mime
);
}
}
@ -305,7 +321,10 @@ mod tests {
.with_property("a", "1");
assert_eq!(
c.properties_sorted(),
vec![("a".to_string(), "1".to_string()), ("b".to_string(), "2".to_string())]
vec![
("a".to_string(), "1".to_string()),
("b".to_string(), "2".to_string())
]
);
}
}

4
crates/quicksearch-core/src/extract/pdf.rs
View file

@ -76,7 +76,9 @@ impl Extractor for PdfExtractor {
if let Ok(info_id) = info_ref.as_reference() {
if let Ok(info) = doc.get_object(info_id) {
if let Ok(dict) = info.as_dict() {
for key in ["Title", "Author", "Subject", "Keywords", "Creator", "Producer"] {
for key in [
"Title", "Author", "Subject", "Keywords", "Creator", "Producer",
] {
if let Ok(val) = dict.get(key.as_bytes()) {
if let Some(s) = object_to_string(val) {
if !s.is_empty() {

41
crates/quicksearch-core/src/extract/plaintext.rs
View file

@ -78,8 +78,8 @@ impl Extractor for PlaintextExtractor {
/// Neither case was ever atomic — a concurrent writer can tear a file
/// across any read sequence, including `read_to_string`'s.
fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError> {
let mut f = File::open(path)
.map_err(|e| format!("plaintext read {}: {}", path.display(), e))?;
let mut f =
File::open(path).map_err(|e| format!("plaintext read {}: {}", path.display(), e))?;
let size = f
.metadata()
.map_err(|e| format!("plaintext read {}: {}", path.display(), e))?
@ -148,10 +148,16 @@ mod tests {
#[test]
fn head_extraction_matches_reading_the_file() {
let p = tmp("agree", b"shared body with unicode: caf\xc3\xa9 \xe2\x9c\x93");
let p = tmp(
"agree",
b"shared body with unicode: caf\xc3\xa9 \xe2\x9c\x93",
);
let from_disk = PlaintextExtractor.extract(&p).unwrap();
let bytes = std::fs::read(&p).unwrap();
let from_head = PlaintextExtractor.extract_from_head(&p, &bytes).unwrap().unwrap();
let from_head = PlaintextExtractor
.extract_from_head(&p, &bytes)
.unwrap()
.unwrap();
assert_eq!(from_disk.text, from_head.text);
assert_eq!(from_disk.properties, from_head.properties);
std::fs::remove_file(&p).ok();
@ -168,7 +174,11 @@ mod tests {
.unwrap()
.unwrap_err();
assert_eq!(disk_err, head_err, "one decode path, one message");
assert!(disk_err.contains("binary"), "the failure names the file: {}", disk_err);
assert!(
disk_err.contains("binary"),
"the failure names the file: {}",
disk_err
);
std::fs::remove_file(&p).ok();
}
@ -177,7 +187,10 @@ mod tests {
let body = b"une journ\xe9e agr\xe9able pr\xe8s de la rivi\xe8re";
let p = tmp("latin1", body);
let from_disk = PlaintextExtractor.extract(&p).unwrap();
let from_head = PlaintextExtractor.extract_from_head(&p, body).unwrap().unwrap();
let from_head = PlaintextExtractor
.extract_from_head(&p, body)
.unwrap()
.unwrap();
assert_eq!(from_disk.text, from_head.text);
assert_eq!(from_disk.text, "une journée agréable près de la rivière");
std::fs::remove_file(&p).ok();
@ -190,9 +203,15 @@ mod tests {
body.extend(src.encode_utf16().flat_map(|u| u.to_le_bytes()));
let p = tmp("utf16", &body);
let from_disk = PlaintextExtractor.extract(&p).unwrap();
let from_head = PlaintextExtractor.extract_from_head(&p, &body).unwrap().unwrap();
let from_head = PlaintextExtractor
.extract_from_head(&p, &body)
.unwrap()
.unwrap();
assert_eq!(from_disk.text, from_head.text);
assert_eq!(from_disk.text, src, "stored text is the UTF-8 decode, BOM stripped");
assert_eq!(
from_disk.text, src,
"stored text is the UTF-8 decode, BOM stripped"
);
std::fs::remove_file(&p).ok();
}
@ -213,7 +232,11 @@ mod tests {
let p = tmp("empty", b"");
assert_eq!(PlaintextExtractor.extract(&p).unwrap().text, "");
assert_eq!(
PlaintextExtractor.extract_from_head(&p, &[]).unwrap().unwrap().text,
PlaintextExtractor
.extract_from_head(&p, &[])
.unwrap()
.unwrap()
.text,
""
);
std::fs::remove_file(&p).ok();

4
crates/quicksearch-core/src/extract/rtf.rs
View file

@ -37,8 +37,8 @@ impl Extractor for RtfExtractor {
fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError> {
// Plain read: RTF files are rare and small enough that plaintext's
// sized-read syscall trimming would be tuning without a workload.
let bytes = std::fs::read(path)
.map_err(|e| format!("rtf read {}: {}", path.display(), e))?;
let bytes =
std::fs::read(path).map_err(|e| format!("rtf read {}: {}", path.display(), e))?;
parse(bytes, path)
}

141
crates/quicksearch-core/src/file_handling.rs
View file

@ -1,17 +1,17 @@
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Mutex, Arc};
use std::fs::File;
use std::io::Read;
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
// Only the Unix entry-count path shells out; Windows walks the tree directly.
use std::collections::HashMap;
#[cfg(unix)]
use std::process::{Command, Stdio};
use std::time::UNIX_EPOCH;
use std::collections::HashMap;
use sha2::{Sha256, Digest};
use walkdir::{DirEntry, WalkDir};
use rusqlite::Connection;
use sha2::{Digest, Sha256};
use walkdir::{DirEntry, WalkDir};
use crate::config::{Config, IgnoreSet};
use crate::db::repo::{self, NewFile};
@ -57,7 +57,10 @@ pub(crate) fn path_to_db_string(path: &Path) -> String {
let s = path.to_string_lossy();
if let Some(rest) = s.strip_prefix(r"\\?\UNC\") {
format!(r"\\{}", rest)
} else if let Some(rest) = s.strip_prefix(r"\\?\").filter(|r| starts_with_drive_letter(r)) {
} else if let Some(rest) = s
.strip_prefix(r"\\?\")
.filter(|r| starts_with_drive_letter(r))
{
rest.to_string()
} else {
s.into_owned()
@ -143,7 +146,6 @@ fn parent_str(path: &str) -> String {
.unwrap_or_default()
}
/// Paths a walk could not read, collected as it runs.
///
/// A full run deletes index rows for everything it did not see, so "I could
@ -218,6 +220,12 @@ fn walk_filter(e: &DirEntry, include_hidden: bool, ignore: &IgnoreSet) -> bool {
/// Directories that cannot be read are recorded in `failures` rather than
/// silently skipped, so the caller can tell an unreadable subtree apart
/// from a deleted one.
///
/// Unlike the parallel walker's symlink resolution, entries here never grow
/// a Windows `\\?\` prefix: walkdir does not canonicalize — every yielded
/// path is `root` plus name components — and the callers pass plainly-spelled
/// roots, so `walk_filter`'s full-path ignore matching sees the same spelling
/// the patterns use.
fn walk_entries<'a>(
root: &str,
follow_symlinks: bool,
@ -278,7 +286,6 @@ pub fn filtered_dirs<'a>(
.filter(|entry| entry.file_type().is_dir())
}
#[cfg(unix)]
fn parse_wc_l_stdout(bytes: &[u8]) -> Result<usize, String> {
let s = String::from_utf8_lossy(bytes);
@ -425,15 +432,14 @@ pub fn count_tree_entries_fast(
}
#[cfg(all(unix, target_os = "linux"))]
{
return count_find_pipe_wc(path, cancel, true)
.or_else(|e| {
if e.contains("cancelled") {
Err(e)
} else {
// Non-GNU find without -printf: plain listing.
count_find_pipe_wc(path, cancel, false)
}
});
return count_find_pipe_wc(path, cancel, true).or_else(|e| {
if e.contains("cancelled") {
Err(e)
} else {
// Non-GNU find without -printf: plain listing.
count_find_pipe_wc(path, cancel, false)
}
});
}
#[cfg(all(unix, not(target_os = "linux")))]
{
@ -488,13 +494,13 @@ fn safe_truncate_string(s: &str, max_bytes: usize) -> String {
if s.len() <= max_bytes {
return s.to_string();
}
// Find the last valid UTF-8 character boundary at or before max_bytes
let mut end = max_bytes;
while end > 0 && !s.is_char_boundary(end) {
end -= 1;
}
s[..end].to_string()
}
@ -779,8 +785,7 @@ pub fn content_extractable(
config: &Config,
registry: &Registry,
) -> bool {
crate::config::content_allowed(path, config)
&& mime.is_some_and(|m| registry.supports(m))
crate::config::content_allowed(path, config) && mime.is_some_and(|m| registry.supports(m))
}
/// Read `path` and decide what its content row should say. No database access,
@ -1165,8 +1170,7 @@ pub fn store_extracted(
.unchecked_transaction()
.map_err(|e| format!("Failed to begin transaction: {}", e))?;
for row in batch {
if let Err(e) =
store_content_outcome(&tx, row.file_id, &row.name, &row.outcome, config)
if let Err(e) = store_content_outcome(&tx, row.file_id, &row.name, &row.outcome, config)
{
crate::log_warn!("content indexing for {}: {}", row.name, e);
continue;
@ -1221,18 +1225,33 @@ mod tests {
])
.unwrap();
let mut names: Vec<String> = filtered_walk(root.to_str().unwrap(), false, false, &ignore, &UnreadableDirs::default())
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
let mut names: Vec<String> = filtered_walk(
root.to_str().unwrap(),
false,
false,
&ignore,
&UnreadableDirs::default(),
)
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
names.sort();
assert_eq!(names, vec!["keep.txt", "keep2.txt"]);
// include_hidden brings back dotfiles but ignores still apply.
let mut names: Vec<String> = filtered_walk(root.to_str().unwrap(), false, true, &ignore, &UnreadableDirs::default())
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
let mut names: Vec<String> = filtered_walk(
root.to_str().unwrap(),
false,
true,
&ignore,
&UnreadableDirs::default(),
)
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
names.sort();
assert_eq!(names, vec![".dotfile", "inside.txt", "keep.txt", "keep2.txt"]);
assert_eq!(
names,
vec![".dotfile", "inside.txt", "keep.txt", "keep2.txt"]
);
std::fs::remove_dir_all(&root).ok();
}
@ -1332,9 +1351,15 @@ mod tests {
let root = base.join(".config");
touch(&root.join("app.conf"));
let ignore = IgnoreSet::compile(&[]).unwrap();
let names: Vec<String> = filtered_walk(root.to_str().unwrap(), false, false, &ignore, &UnreadableDirs::default())
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
let names: Vec<String> = filtered_walk(
root.to_str().unwrap(),
false,
false,
&ignore,
&UnreadableDirs::default(),
)
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
assert_eq!(names, vec!["app.conf"]);
std::fs::remove_dir_all(&base).ok();
}
@ -1372,8 +1397,14 @@ mod tests {
#[test]
fn db_path_strips_windows_prefixes() {
assert_eq!(path_to_db_string(Path::new("/plain/unix/path")), "/plain/unix/path");
assert_eq!(path_to_db_string(Path::new(r"\\?\C:\docs\a.txt")), r"C:\docs\a.txt");
assert_eq!(
path_to_db_string(Path::new("/plain/unix/path")),
"/plain/unix/path"
);
assert_eq!(
path_to_db_string(Path::new(r"\\?\C:\docs\a.txt")),
r"C:\docs\a.txt"
);
// A share must come back as \\server\share, not UNC\server\share —
// stripping a fixed four characters produces a path that cannot be
// opened, and every file beneath it would be misfiled.
@ -1448,9 +1479,8 @@ mod tests {
let missing = real.join("gone").join("deeper.txt");
let key = db_key_for_missing_path(&missing);
let expected = path_to_db_string(
&real.canonicalize().unwrap().join("gone").join("deeper.txt"),
);
let expected =
path_to_db_string(&real.canonicalize().unwrap().join("gone").join("deeper.txt"));
assert_eq!(key, expected, "existing prefix resolved, missing tail kept");
// A redundant component in the *existing* part is collapsed, which is
@ -1504,20 +1534,19 @@ mod tests {
let ignore = IgnoreSet::compile(&[]).unwrap();
let failures = UnreadableDirs::default();
let names: Vec<String> = filtered_walk(
root.to_str().unwrap(),
false,
false,
&ignore,
&failures,
)
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
let names: Vec<String> =
filtered_walk(root.to_str().unwrap(), false, false, &ignore, &failures)
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
// Restore before asserting so a failure still cleans up.
std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).ok();
assert_eq!(names, vec!["a.txt"], "the unreadable subtree yields nothing");
assert_eq!(
names,
vec!["a.txt"],
"the unreadable subtree yields nothing"
);
assert!(!failures.is_empty(), "and that failure must be recorded");
assert!(
failures.covers(locked.join("hidden-from-us.txt").to_str().unwrap()),
@ -1542,7 +1571,11 @@ mod tests {
// Differs within the head window.
std::fs::write(&c, [b"DIFF".as_slice(), &[0u8; 64], b"AAAA"].concat()).unwrap();
let h = |p: &Path| get_file_hash(std::fs::metadata(p).unwrap().len(), p, 8).unwrap().0;
let h = |p: &Path| {
get_file_hash(std::fs::metadata(p).unwrap().len(), p, 8)
.unwrap()
.0
};
assert_eq!(h(&a), h(&b), "tail differences are invisible by design");
assert_ne!(h(&a), h(&c), "head differences are caught");
@ -1699,9 +1732,15 @@ mod count_and_extract_tests {
let cfg = Config::default();
assert!(needs(&cfg, "notes.txt"), "plaintext is claimed");
assert!(needs(&cfg, "song.mp3"), "audio tags are content too");
assert!(needs(&cfg, "README"), "an extensionless text head sniffs as text/plain");
assert!(
needs(&cfg, "README"),
"an extensionless text head sniffs as text/plain"
);
assert!(!needs(&cfg, "movie.mp4"), "no extractor claims video");
assert!(!needs(&cfg, "blob.bin"), "binary content: no MIME, no extractor");
assert!(
!needs(&cfg, "blob.bin"),
"binary content: no MIME, no extractor"
);
// Over `maximum_text_file_size`, so the content pass would never read
// it even though plaintext claims the MIME.

17
crates/quicksearch-core/src/incremental.rs
View file

@ -21,8 +21,8 @@ use crate::config::{Config, IgnoreSet};
use crate::db::repo;
use crate::extract::Registry;
use crate::file_handling::{
db_key_for_missing_path, extract_and_store, filtered_walk, ExtractCursor, UnreadableDirs,
prepare_file_record_from_path, store_inline_text,
db_key_for_missing_path, extract_and_store, filtered_walk, prepare_file_record_from_path,
store_inline_text, ExtractCursor, UnreadableDirs,
};
use crate::platform::path_has_hidden_component_under;
use crate::watcher::FsEvent;
@ -154,7 +154,8 @@ fn upsert_file(
)?;
}
tx.commit().map_err(|e| format!("commit incremental tx: {}", e))
tx.commit()
.map_err(|e| format!("commit incremental tx: {}", e))
}
fn remove_path(conn: &mut Connection, path: &Path) -> Result<(), String> {
@ -263,8 +264,14 @@ mod tests {
}
fn apply(&mut self, event: &FsEvent) {
apply_fs_event(&mut self.conn, event, &self.config, &self.ignore, &self.registry)
.unwrap();
apply_fs_event(
&mut self.conn,
event,
&self.config,
&self.ignore,
&self.registry,
)
.unwrap();
}
fn write(&self, name: &str, content: &str) -> std::path::PathBuf {

126
crates/quicksearch-core/src/indexing.rs
View file

@ -1,28 +1,20 @@
use rusqlite::{params, Connection, InterruptHandle, OptionalExtension};
use std::collections::{HashMap, HashSet};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, mpsc};
use std::sync::{mpsc, Arc, Mutex};
use std::thread;
use std::time::{Duration, Instant};
use std::collections::{HashMap, HashSet};
use rusqlite::{params, Connection, InterruptHandle, OptionalExtension};
use crate::extract::Registry;
use crate::file_handling::{
cleanup_stale_index_entries,
count_tree_entries_fast,
extract_scope_prepare,
store_extracted,
fts_finalize_after_text_indexing,
process_batch_inserts,
process_batch_updates,
path_to_db_string,
ExtractCursor,
FileIndexAction,
OwnedNewFile,
};
use crate::config::Config;
use crate::walk::{thread_count_for, walk_indexable_files, ParallelWalk, TryNext, WalkEvent};
use crate::db;
use crate::db::repo;
use crate::extract::Registry;
use crate::file_handling::{
cleanup_stale_index_entries, count_tree_entries_fast, extract_scope_prepare,
fts_finalize_after_text_indexing, path_to_db_string, process_batch_inserts,
process_batch_updates, store_extracted, ExtractCursor, FileIndexAction, OwnedNewFile,
};
use crate::walk::{thread_count_for, walk_indexable_files, ParallelWalk, TryNext, WalkEvent};
/// Where one root's pipeline is in its life cycle.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@ -291,10 +283,7 @@ fn wal_len(path: &str) -> u64 {
/// caller should abort the operation. While the suspend flag is set and stop
/// is not, this parks the thread by sleeping in short increments so a later
/// `resume()` unblocks it. Cheap to call in tight loops.
pub(crate) fn should_abort(
stop: &Arc<AtomicBool>,
suspend: &Arc<AtomicBool>,
) -> bool {
pub(crate) fn should_abort(stop: &Arc<AtomicBool>, suspend: &Arc<AtomicBool>) -> bool {
loop {
if stop.load(Ordering::Relaxed) {
return true;
@ -454,7 +443,11 @@ impl IndexingService {
};
}
self.command_tx
.send(IndexingCommand::Start { paths, db_path, config })
.send(IndexingCommand::Start {
paths,
db_path,
config,
})
.map_err(|e| {
// The service is gone; leave the status honest rather than
// stuck on a run that will never happen.
@ -478,7 +471,8 @@ impl IndexingService {
// Wait for indexing to transition to stopping state
let mut attempts = 0;
while attempts < 50 { // Wait up to 5 seconds
while attempts < 50 {
// Wait up to 5 seconds
match self.get_status() {
IndexingStatus::Stopping => break,
IndexingStatus::Idle => return Ok(()), // Already stopped
@ -520,7 +514,12 @@ impl IndexingService {
/// confirmed the rebuild dialog). A missing or incompatible DB means
/// there is nothing to validate — the indexer will (re)build under its
/// own policy anyway.
pub fn check_config_validation(&self, db_path: &str, config: &Config, indexing_path: &str) -> Result<Option<Vec<ConfigChange>>, String> {
pub fn check_config_validation(
&self,
db_path: &str,
config: &Config,
indexing_path: &str,
) -> Result<Option<Vec<ConfigChange>>, String> {
match db::open_existing(db_path, false) {
Ok(conn) => Self::validate_config(&conn, config, indexing_path),
Err(_) => Ok(None),
@ -539,7 +538,8 @@ impl IndexingService {
// Wait for indexing to actually stop
let mut attempts = 0;
while attempts < 50 { // Wait up to 5 seconds
while attempts < 50 {
// Wait up to 5 seconds
match self.get_status() {
IndexingStatus::Idle => break,
// Optimizing holds the file about to be deleted, so it is
@ -576,10 +576,14 @@ impl IndexingService {
) {
let stop_flag = Arc::new(AtomicBool::new(false));
let mut indexing_handle: Option<thread::JoinHandle<()>> = None;
while let Ok(command) = command_rx.recv() {
match command {
IndexingCommand::Start { paths, db_path, config } => {
IndexingCommand::Start {
paths,
db_path,
config,
} => {
// `start_indexing` already claimed the status and rejected
// a concurrent start, so there is nothing to re-check here.
@ -606,7 +610,15 @@ impl IndexingService {
// The writer thread: every DB write and every text
// extraction a run performs happens here.
crate::platform::set_background_priority();
let result = Self::run_indexing(&status_clone, &paths_owned, &db_path_owned, &stop_flag_clone, &suspend_clone, &config_owned, &db_connection_clone);
let result = Self::run_indexing(
&status_clone,
&paths_owned,
&db_path_owned,
&stop_flag_clone,
&suspend_clone,
&config_owned,
&db_connection_clone,
);
// Released before maintenance, not after: VACUUM needs
// its own connection (see `db::open::open_maintenance`)
@ -641,7 +653,7 @@ impl IndexingService {
}
}
}
// Clean up any remaining indexing thread
if let Some(handle) = indexing_handle {
let _ = handle.join();
@ -735,7 +747,7 @@ impl IndexingService {
// of after a full table scan.
let conn_mutex = Arc::new(Mutex::new(conn));
// Store the database connection for proper cleanup on stop
if let Ok(mut db_opt) = db_connection.lock() {
*db_opt = Some(conn_mutex.clone());
@ -785,20 +797,22 @@ impl IndexingService {
let root = root.clone();
let cancel = count_cancel.clone();
let total = count_total.clone();
let _ = thread::Builder::new().name("qs-count".into()).spawn(move || {
crate::platform::set_background_priority();
match count_tree_entries_fast(&root, &cancel) {
// A genuinely empty root stores 1 so "known" stays
// distinguishable from the 0 = unknown sentinel; an
// empty root's walk finishes instantly anyway.
Ok(n) => total.store(n.max(1), Ordering::Relaxed),
Err(e) => {
if !e.contains("cancelled") {
crate::log_warn!("count for {}: {}", root, e);
let _ = thread::Builder::new()
.name("qs-count".into())
.spawn(move || {
crate::platform::set_background_priority();
match count_tree_entries_fast(&root, &cancel) {
// A genuinely empty root stores 1 so "known" stays
// distinguishable from the 0 = unknown sentinel; an
// empty root's walk finishes instantly anyway.
Ok(n) => total.store(n.max(1), Ordering::Relaxed),
Err(e) => {
if !e.contains("cancelled") {
crate::log_warn!("count for {}: {}", root, e);
}
}
}
}
});
});
}
pipelines.push(RootPipeline {
@ -843,7 +857,10 @@ impl IndexingService {
.collect();
if let Ok(mut g) = status.lock() {
if !matches!(*g, IndexingStatus::Stopping) {
*g = IndexingStatus::Running { start_time: run_start, roots };
*g = IndexingStatus::Running {
start_time: run_start,
roots,
};
}
}
};
@ -1117,8 +1134,7 @@ impl IndexingService {
}
if !stale_paths.is_empty() {
if let Some(first) = pipelines.first_mut() {
first.current_file =
Some("Removing stale index entries…".to_string());
first.current_file = Some("Removing stale index entries…".to_string());
}
stale_deleted = cleanup_stale_index_entries(
&conn_mutex,
@ -1224,7 +1240,10 @@ impl IndexingService {
/// The config keys whose change invalidates the stored index, paired
/// with their current values. One list drives both [`validate_config`]
/// and [`update_config`] so the two can never drift apart.
fn config_validation_entries(config: &Config, indexing_path: &str) -> Vec<(&'static str, String)> {
fn config_validation_entries(
config: &Config,
indexing_path: &str,
) -> Vec<(&'static str, String)> {
let sorted_joined = |v: &[String]| {
let mut v: Vec<String> = v.to_vec();
v.sort();
@ -1288,11 +1307,19 @@ impl IndexingService {
}
}
}
Ok(if changes.is_empty() { None } else { Some(changes) })
Ok(if changes.is_empty() {
None
} else {
Some(changes)
})
}
/// Stamp the index with the settings it's being built under.
fn update_config(conn: &Connection, config: &Config, indexing_path: &str) -> Result<(), String> {
fn update_config(
conn: &Connection,
config: &Config,
indexing_path: &str,
) -> Result<(), String> {
for (key, current) in Self::config_validation_entries(config, indexing_path) {
conn.execute(
"INSERT OR REPLACE INTO config_validation (key, value) VALUES (?1, ?2)",
@ -1333,7 +1360,6 @@ impl Default for IndexingService {
}
}
#[cfg(test)]
mod tests {
use super::*;

61
crates/quicksearch-core/src/mime.rs
View file

@ -263,10 +263,9 @@ pub fn mime_to_type(mime: &str) -> FileType {
// `text/*` (see `extract::plaintext::EXTRA_TEXT_MIMES` and the
// cross-check test below). Keyed on the subtype alone, so playlists
// stay AUDIO|TEXT and SVG stays IMAGE|TEXT.
"xml" | "json" | "json5" | "geo+json" | "javascript" | "mbox" | "rfc822"
| "vnd.dart" | "x-csh" | "x-httpd-php" | "x-perl" | "x-sh" | "x-sql"
| "x-subrip" | "x-tcl" | "x-tex" | "x-texinfo" | "x-troff" | "x-troff-man"
| "x-mpegurl" | "scpls" | "svg+xml" => {
"xml" | "json" | "json5" | "geo+json" | "javascript" | "mbox" | "rfc822" | "vnd.dart"
| "x-csh" | "x-httpd-php" | "x-perl" | "x-sh" | "x-sql" | "x-subrip" | "x-tcl"
| "x-tex" | "x-texinfo" | "x-troff" | "x-troff-man" | "x-mpegurl" | "scpls" | "svg+xml" => {
t |= FileType::TEXT;
}
_ => {}
@ -292,17 +291,14 @@ mod tests {
#[test]
fn docx_is_document_and_office() {
let t = mime_to_type(
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
);
let t =
mime_to_type("application/vnd.openxmlformats-officedocument.wordprocessingml.document");
assert!(t.contains(FileType::DOCUMENT));
}
#[test]
fn xlsx_is_spreadsheet_and_document() {
let t = mime_to_type(
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
);
let t = mime_to_type("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
assert!(t.contains(FileType::DOCUMENT));
assert!(t.contains(FileType::SPREADSHEET));
}
@ -341,8 +337,17 @@ mod tests {
#[test]
fn from_name_round_trip() {
for n in ["Audio", "Image", "Video", "Document", "Text", "Archive",
"Spreadsheet", "Presentation", "Folder"] {
for n in [
"Audio",
"Image",
"Video",
"Document",
"Text",
"Archive",
"Spreadsheet",
"Presentation",
"Folder",
] {
assert_ne!(FileType::from_name(n), FileType::EMPTY, "{}", n);
}
assert_eq!(FileType::from_name("Weird"), FileType::EMPTY);
@ -435,7 +440,11 @@ mod tests {
let head_bytes = crate::config::ProcessingConfig::default().hash_length;
let samples: &[(&str, &[u8], &str)] = &[
("png", &[0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a], "image/png"),
(
"png",
&[0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a],
"image/png",
),
("gif", b"GIF89a", "image/gif"),
("pdf", b"%PDF-1.7", "application/pdf"),
("zip", &[0x50, 0x4b, 0x03, 0x04], "application/zip"),
@ -520,11 +529,16 @@ mod tests {
];
for (name, head) in samples {
let path = PathBuf::from(name);
let mime = guess_mime_from_head(&path, head)
.unwrap_or_else(|| panic!("{} has no MIME", name));
let mime =
guess_mime_from_head(&path, head).unwrap_or_else(|| panic!("{} has no MIME", name));
let extracted = registry
.extract_complete_head(&path, &mime, head)
.unwrap_or_else(|| panic!("{} -> {} not claimed by a head-capable extractor", name, mime))
.unwrap_or_else(|| {
panic!(
"{} -> {} not claimed by a head-capable extractor",
name, mime
)
})
.unwrap_or_else(|e| panic!("{} -> {} failed to extract: {}", name, mime, e));
assert!(
!extracted.text.is_empty(),
@ -551,7 +565,10 @@ mod tests {
Some("text/plain")
);
let blob = PathBuf::from("blob");
assert_eq!(guess_mime_from_head(&blob, &[0x00, 0x01, 0x02, 0xFF]).as_deref(), None);
assert_eq!(
guess_mime_from_head(&blob, &[0x00, 0x01, 0x02, 0xFF]).as_deref(),
None
);
}
/// Ambiguous extensions resolve by content in both directions: source
@ -582,8 +599,11 @@ mod tests {
);
assert_eq!(
guess_mime_from_head(&PathBuf::from("go.mod"), b"module example.com/x\n\ngo 1.22\n")
.as_deref(),
guess_mime_from_head(
&PathBuf::from("go.mod"),
b"module example.com/x\n\ngo 1.22\n"
)
.as_deref(),
Some("text/plain")
);
@ -607,8 +627,7 @@ mod tests {
Some("text/plain")
);
assert_eq!(
guess_mime_from_head(&PathBuf::from("disk.vhd"), &[0x00, 0x01, 0x02, 0x03])
.as_deref(),
guess_mime_from_head(&PathBuf::from("disk.vhd"), &[0x00, 0x01, 0x02, 0x03]).as_deref(),
Some("application/x-virtualbox-vhd")
);
}

24
crates/quicksearch-core/src/platform.rs
View file

@ -126,7 +126,14 @@ pub(crate) fn is_unc_string(s: &str) -> bool {
/// Filesystem types whose operations are network round trips.
#[cfg(target_os = "linux")]
const NETWORK_FS_TYPES: [&str; 8] = [
"cifs", "smb3", "smbfs", "nfs", "nfs4", "afs", "fuse.sshfs", "9p",
"cifs",
"smb3",
"smbfs",
"nfs",
"nfs4",
"afs",
"fuse.sshfs",
"9p",
];
/// Whether `path` lives on a network filesystem.
@ -312,7 +319,10 @@ pub fn deny_read(dir: &Path) -> std::io::Result<()> {
}
#[cfg(windows)]
{
icacls(dir, &["/deny", &format!("{}:(OI)(CI)(RD)", current_user()?)])
icacls(
dir,
&["/deny", &format!("{}:(OI)(CI)(RD)", current_user()?)],
)
}
#[cfg(not(any(unix, windows)))]
{
@ -400,7 +410,10 @@ mod tests {
fn collation_matches_like_case_folding() {
// LIKE folds ASCII case on every platform; the `=` half of a path
// filter has to agree with it, which is what this constant is for.
assert_eq!(PATH_COLLATION, if cfg!(windows) { "NOCASE" } else { "BINARY" });
assert_eq!(
PATH_COLLATION,
if cfg!(windows) { "NOCASE" } else { "BINARY" }
);
}
#[test]
@ -427,7 +440,10 @@ mod tests {
// The root itself is hidden, but it was chosen explicitly — the walk
// keeps it, so the watcher must too.
assert!(!path_has_hidden_component_under(&root, &roots));
assert!(!path_has_hidden_component_under(&root.join("app.conf"), &roots));
assert!(!path_has_hidden_component_under(
&root.join("app.conf"),
&roots
));
// A dot *below* the root still counts.
assert!(path_has_hidden_component_under(

4
crates/quicksearch-core/src/query/pattern.rs
View file

@ -273,9 +273,7 @@ impl TermPattern {
pub fn count(&self, text: &str, case_insensitive: bool) -> usize {
match self {
TermPattern::Empty => 0,
TermPattern::Literal(l) => {
snippet::count_occurrences(text, &l.text, !case_insensitive)
}
TermPattern::Literal(l) => snippet::count_occurrences(text, &l.text, !case_insensitive),
TermPattern::Wildcard(w) => {
let re = if case_insensitive {
&w.search_ci

18
crates/quicksearch-core/src/query/split.rs
View file

@ -113,8 +113,7 @@ pub fn split_for_cascade(input: &str) -> Result<CascadeQuery, TranslateError> {
if let Some(value) = value {
// Quoted values keep `*` literal; only a bare word's
// stars act as wildcards (`name:` honors this too).
let value_is_word =
matches!(tokens.get(value_idx), Some(Token::Word(_)));
let value_is_word = matches!(tokens.get(value_idx), Some(Token::Word(_)));
if word.eq_ignore_ascii_case("regex") {
// Not a SQL filter: compiled here, matched in
// Rust against name, path and content.
@ -150,8 +149,7 @@ pub fn split_for_cascade(input: &str) -> Result<CascadeQuery, TranslateError> {
while let Some(Token::Op(next_op)) = tokens.get(i) {
glued.push_str(op_str(*next_op));
i += 1;
if let Some(Token::Word(v)) | Some(Token::Quoted(v)) = tokens.get(i)
{
if let Some(Token::Word(v)) | Some(Token::Quoted(v)) = tokens.get(i) {
glued.push_str(v);
i += 1;
}
@ -295,10 +293,14 @@ mod tests {
fn a_windows_drive_path_reaches_the_filter_intact() {
let q = split_for_cascade(r"path:C:\Users\me\docs").unwrap();
assert_eq!(q.term, "", "the whole input is a filter");
assert!(matches!(
&q.filter_params[0],
Value::Text(t) if t == r"C:\Users\me\docs"
), "{:?}", q.filter_params);
assert!(
matches!(
&q.filter_params[0],
Value::Text(t) if t == r"C:\Users\me\docs"
),
"{:?}",
q.filter_params
);
}
#[test]

50
crates/quicksearch-core/src/query/translator.rs
View file

@ -14,10 +14,7 @@ pub enum TranslateError {
UnknownProperty(String),
BadDate(String),
BadRegex(String),
UnsupportedOp {
key: String,
op: Op,
},
UnsupportedOp { key: String, op: Op },
}
impl std::fmt::Display for TranslateError {
@ -27,7 +24,11 @@ impl std::fmt::Display for TranslateError {
TranslateError::BadDate(s) => write!(f, "bad date '{}'", s),
TranslateError::BadRegex(s) => write!(f, "regex error: {}", s),
TranslateError::UnsupportedOp { key, op } => {
write!(f, "operator {:?} is not supported for property '{}'", op, key)
write!(
f,
"operator {:?} is not supported for property '{}'",
op, key
)
}
}
}
@ -66,8 +67,15 @@ pub struct FilterFragment {
pub fn is_filter_key(key: &str) -> bool {
matches!(
key.to_ascii_lowercase().as_str(),
"type" | "modified" | "mtime" | "path" | "folder" | "includefolder" | "name"
| "filename" | "mime"
"type"
| "modified"
| "mtime"
| "path"
| "folder"
| "includefolder"
| "name"
| "filename"
| "mime"
)
}
@ -295,7 +303,9 @@ mod tests {
assert_eq!(f.sql, "(f.type & ?) != 0");
assert_eq!(
f.params,
vec![rusqlite::types::Value::Integer(FileType::AUDIO.bits() as i64)]
vec![rusqlite::types::Value::Integer(
FileType::AUDIO.bits() as i64
)]
);
}
@ -372,8 +382,17 @@ mod tests {
#[test]
fn is_filter_key_covers_exactly_the_supported_keys() {
for key in [
"type", "modified", "mtime", "path", "folder", "includefolder", "name", "filename",
"mime", "TYPE", "Path",
"type",
"modified",
"mtime",
"path",
"folder",
"includefolder",
"name",
"filename",
"mime",
"TYPE",
"Path",
] {
assert!(is_filter_key(key), "{} should be a filter key", key);
}
@ -395,11 +414,11 @@ mod tests {
let base = format!("{}a{}b", root_prefix(), SEP);
let rows = [
format!("{}{}sub", base, SEP), // inside
format!("{}{}sub", base, SEP), // inside
format!("{}{}sub{}deep", base, SEP, SEP), // deeper
base.clone(), // the folder itself
format!("{}a{}bc", root_prefix(), SEP), // prefix sibling: outside
format!("{}a", root_prefix()), // parent: outside
base.clone(), // the folder itself
format!("{}a{}bc", root_prefix(), SEP), // prefix sibling: outside
format!("{}a", root_prefix()), // parent: outside
];
for r in &rows {
conn.execute("INSERT INTO files (parent) VALUES (?1)", [r])
@ -426,7 +445,7 @@ mod tests {
let base = format!("{}a_b", root_prefix());
for r in [
format!("{}{}inside", base, SEP), // real child
format!("{}{}inside", base, SEP), // real child
format!("{}axb{}bait", root_prefix(), SEP), // `_` must not glob to `x`
format!("{}100%_done{}x", root_prefix(), SEP),
] {
@ -483,5 +502,4 @@ mod tests {
// 2000-02-29 is valid (leap year)
assert_eq!(civil_to_unix(2000, 2, 29), 951_782_400);
}
}

47
crates/quicksearch-core/src/search/cascade.rs
View file

@ -317,12 +317,7 @@ impl<'a> Cx<'a> {
/// check covers both; content is fetched (and decompressed) only for
/// rows whose path missed — bounded by the pass's hit count, not its
/// scan count. Pass `text` when the pass already has the content.
fn regex_accepts(
&self,
file_id: i64,
path: &str,
text: Option<&str>,
) -> Result<bool, String> {
fn regex_accepts(&self, file_id: i64, path: &str, text: Option<&str>) -> Result<bool, String> {
let Some(re) = &self.query.regex else {
return Ok(true);
};
@ -449,10 +444,8 @@ impl<'a> Cx<'a> {
.map(|s| escape_like(s))
.collect::<Vec<_>>()
.join("%");
let params = self.params_with_filters(vec![rusqlite::types::Value::Text(format!(
"%{}%",
like
))]);
let params =
self.params_with_filters(vec![rusqlite::types::Value::Text(format!("%{}%", like))]);
let mut stmt = self.conn.prepare(&sql).map_err(|e| e.to_string())?;
let mut rows = stmt
@ -509,7 +502,11 @@ impl<'a> Cx<'a> {
// the matched span marked — the GUI renders it as [the field].
let snip = snippet::Snippet {
ranges: vec![match_range],
window: if is_path_tier { path.clone() } else { name.clone() },
window: if is_path_tier {
path.clone()
} else {
name.clone()
},
truncated_start: false,
truncated_end: false,
};
@ -605,7 +602,9 @@ impl<'a> Cx<'a> {
.query(rusqlite::params_from_iter(params))
.map_err(|e| e.to_string())?;
let snippet_opts = snippet::Options { approx_chars: SNIPPET_WINDOW_CHARS };
let snippet_opts = snippet::Options {
approx_chars: SNIPPET_WINDOW_CHARS,
};
let mut buf: Vec<SearchHit> = Vec::new();
let mut overflowed = false;
let mut clock = FlushClock::new();
@ -620,9 +619,9 @@ impl<'a> Cx<'a> {
continue;
}
let blob: Option<Vec<u8>> = row.get(5).map_err(|e| e.to_string())?;
let text = blob.and_then(|b| zstd::decode_all(b.as_slice()).ok()).map(
|raw| String::from_utf8_lossy(&raw).into_owned(),
);
let text = blob
.and_then(|b| zstd::decode_all(b.as_slice()).ok())
.map(|raw| String::from_utf8_lossy(&raw).into_owned());
let (rank, stage, snip) = match &text {
Some(text) => {
@ -761,9 +760,7 @@ impl<'a> Cx<'a> {
None if with_paths => {
let folded_path = path.to_ascii_lowercase();
match bitap.best_distance_and_first(folded_path.as_bytes()) {
Some((distance, range)) => {
(11.0 + 0.1 * distance as f64, &path, range)
}
Some((distance, range)) => (11.0 + 0.1 * distance as f64, &path, range),
None => continue,
}
}
@ -842,7 +839,9 @@ impl<'a> Cx<'a> {
.query(rusqlite::params_from_iter(params))
.map_err(|e| e.to_string())?;
let snippet_opts = snippet::Options { approx_chars: SNIPPET_WINDOW_CHARS };
let snippet_opts = snippet::Options {
approx_chars: SNIPPET_WINDOW_CHARS,
};
let mut buf: Vec<SearchHit> = Vec::new();
let mut overflowed = false;
let mut clock = FlushClock::new();
@ -940,7 +939,11 @@ impl<'a> Cx<'a> {
};
let snip = snippet::Snippet {
ranges: vec![match_range],
window: if is_path_tier { path.clone() } else { name.clone() },
window: if is_path_tier {
path.clone()
} else {
name.clone()
},
truncated_start: false,
truncated_end: false,
};
@ -990,7 +993,9 @@ impl<'a> Cx<'a> {
.query(rusqlite::params_from_iter(params))
.map_err(|e| e.to_string())?;
let snippet_opts = snippet::Options { approx_chars: SNIPPET_WINDOW_CHARS };
let snippet_opts = snippet::Options {
approx_chars: SNIPPET_WINDOW_CHARS,
};
let mut buf: Vec<SearchHit> = Vec::new();
let mut overflowed = false;
let mut clock = FlushClock::new();

4
crates/quicksearch-core/src/search/duplicates.rs
View file

@ -58,9 +58,7 @@ pub fn find_duplicate_groups(
}
let mut member_stmt = conn
.prepare(
"SELECT id, name, path, size, mtime FROM files WHERE hash = ?1 ORDER BY path",
)
.prepare("SELECT id, name, path, size, mtime FROM files WHERE hash = ?1 ORDER BY path")
.map_err(|e| e.to_string())?;
for group in &mut groups {
let rows = member_stmt

8
crates/quicksearch-core/src/search/fuzzy.rs
View file

@ -71,7 +71,7 @@ impl Bitap {
let done = 1u64 << (self.len - 1);
let mut hit = None;
let mut prev_old = r[0]; // R_old[d-1] for the d-th iteration
// d = 0: exact prefix extension only.
// d = 0: exact prefix extension only.
r[0] = ((r[0] << 1) | 1) & mask;
if r[0] & done != 0 {
hit = Some(0);
@ -81,7 +81,7 @@ impl Bitap {
r[d] = (((old << 1) | 1) & mask) // extend a ≤d-error state
| prev_old // insertion in text
| (prev_old << 1) // substitution
| ((r[d - 1] << 1) | 1); // deletion (pattern byte skipped)
| ((r[d - 1] << 1) | 1); // deletion (pattern byte skipped)
prev_old = old;
if hit.is_none() && r[d] & done != 0 {
hit = Some(d);
@ -335,7 +335,9 @@ mod tests {
// Deterministic LCG so the test is reproducible.
let mut seed: u64 = 0x2545F4914F6CDD1D;
let mut rng = move || {
seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
seed = seed
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
(seed >> 33) as usize
};
let alphabet = b"abcx";

25
crates/quicksearch-core/src/search/mod.rs
View file

@ -55,10 +55,22 @@ pub struct SearchHit {
#[derive(Debug, Clone)]
pub enum SearchUpdate {
Started { generation: u64 },
Hits { generation: u64, hits: Vec<SearchHit> },
Completed { generation: u64, total: usize, limited: bool },
Error { generation: u64, message: String },
Started {
generation: u64,
},
Hits {
generation: u64,
hits: Vec<SearchHit>,
},
Completed {
generation: u64,
total: usize,
limited: bool,
},
Error {
generation: u64,
message: String,
},
}
impl SearchUpdate {
@ -335,8 +347,9 @@ mod tests {
#[test]
fn corruption_and_syntax_classification_still_work() {
assert!(classify_sql_err("database disk image is malformed")
.starts_with("DATABASE_CORRUPTED:"));
assert!(
classify_sql_err("database disk image is malformed").starts_with("DATABASE_CORRUPTED:")
);
assert!(classify_sql_err("fts5: syntax error near \"NEAR\"").starts_with("Search syntax"));
assert!(classify_sql_err("no such table: files").starts_with("Search failed:"));
}

7
crates/quicksearch-core/src/snippet.rs
View file

@ -142,7 +142,12 @@ pub fn extract_folded(text: &str, folded: &str, terms: &[&str], opts: &Options)
let ranges = matches
.iter()
.filter(|(s, e)| *e > win_start && *s < win_end)
.map(|(s, e)| ((*s).max(win_start) - win_start, (*e).min(win_end) - win_start))
.map(|(s, e)| {
(
(*s).max(win_start) - win_start,
(*e).min(win_end) - win_start,
)
})
.collect();
Snippet {

21
crates/quicksearch-core/src/textenc.rs
View file

@ -110,8 +110,7 @@ pub fn decode_text(bytes: Vec<u8>, path: &Path) -> Result<String, String> {
TextClass::Legacy => {
// ISO-2022-JP detection is safe here: the browser caveat about
// it concerns script-running web content, not indexed files.
let mut det =
chardetng::EncodingDetector::new(chardetng::Iso2022JpDetection::Allow);
let mut det = chardetng::EncodingDetector::new(chardetng::Iso2022JpDetection::Allow);
det.feed(&bytes, true);
// Deny UTF-8: strict UTF-8 was already ruled out, so a UTF-8
// guess could only mean malformed UTF-8.
@ -119,10 +118,7 @@ pub fn decode_text(bytes: Vec<u8>, path: &Path) -> Result<String, String> {
let (text, _, _) = enc.decode(&bytes);
Ok(text.into_owned())
}
TextClass::Binary => Err(format!(
"plaintext read {}: binary content",
path.display()
)),
TextClass::Binary => Err(format!("plaintext read {}: binary content", path.display())),
}
}
@ -139,7 +135,10 @@ mod tests {
fn utf8_decodes_unchanged() {
let body = "plain ascii and café über 日本語".as_bytes().to_vec();
assert!(looks_like_text(&body));
assert_eq!(decode_text(body, &p()).unwrap(), "plain ascii and café über 日本語");
assert_eq!(
decode_text(body, &p()).unwrap(),
"plain ascii and café über 日本語"
);
}
#[test]
@ -203,7 +202,10 @@ mod tests {
assert!(!looks_like_text(&body));
let err = decode_text(body, &p()).unwrap_err();
assert!(err.contains("binary content"), "{err}");
assert!(err.contains("textenc-test-file"), "error must name the file: {err}");
assert!(
err.contains("textenc-test-file"),
"error must name the file: {err}"
);
}
#[test]
@ -217,7 +219,8 @@ mod tests {
#[test]
fn ansi_log_is_text() {
// ESC-heavy colored log output stays text.
let body = b"\x1b[31mERROR\x1b[0m something failed\n\x1b[33mWARN\x1b[0m retrying\n".to_vec();
let body =
b"\x1b[31mERROR\x1b[0m something failed\n\x1b[33mWARN\x1b[0m retrying\n".to_vec();
assert!(looks_like_text(&body));
assert!(decode_text(body, &p()).is_ok());
}

180
crates/quicksearch-core/src/walk.rs
View file

@ -88,7 +88,13 @@ impl WalkedFile {
/// Seen, but with nothing to write. Distinct from not being emitted at
/// all: the row stays.
fn skipped(path: String, digest: u128, aliased: bool) -> Self {
WalkedFile { path, action: FileIndexAction::Skip, record: None, digest, aliased }
WalkedFile {
path,
action: FileIndexAction::Skip,
record: None,
digest,
aliased,
}
}
}
@ -205,7 +211,10 @@ pub struct WorkerStats {
impl WorkerStats {
pub(crate) fn new(total: usize) -> Self {
WorkerStats { busy: Arc::new(AtomicUsize::new(0)), total }
WorkerStats {
busy: Arc::new(AtomicUsize::new(0)),
total,
}
}
/// Count the calling thread as busy until the returned guard drops.
@ -243,7 +252,13 @@ impl Shared {
}
// The prefetcher may have been parked behind PREFETCH_AHEAD.
self.idle.notify_all();
return Some((job, ActiveJob { shared: self, finished: false }));
return Some((
job,
ActiveJob {
shared: self,
finished: false,
},
));
}
// Nothing runnable. Only "nobody anywhere holds work" proves the
// walk is over — a directory sitting in the prefetch stage still
@ -452,8 +467,7 @@ fn read_directory(
// `entry.metadata()` is only consulted on Windows, where it is free —
// the attributes came back with the directory read. On Unix the
// closure is never called, so this stays at zero extra syscalls.
if !ctx.include_hidden
&& crate::platform::entry_is_hidden(&name, || entry.metadata().ok())
if !ctx.include_hidden && crate::platform::entry_is_hidden(&name, || entry.metadata().ok())
{
continue;
}
@ -492,7 +506,15 @@ fn read_directory(
// Resolve aliases where they are found. The target's canonical
// path is what the index stores, and pushing only canonical
// directories is what keeps `seen_dirs` able to break cycles.
//
// Normalized like the roots (walk_parallel), or on Windows the
// target keeps `canonicalize`'s `\\?\` prefix: every path below
// it would be spelled differently from the plainly-spelled
// roots, so full-path ignore patterns would never match under a
// followed junction and `seen_dirs` could not dedup against an
// overlapping root.
if let Ok(target) = path.canonicalize() {
let target = PathBuf::from(path_to_db_string(&target));
match fs::metadata(&target) {
Ok(m) if m.is_dir() => found.push(Found::Dir(target)),
// The row for a resolved target belongs to the
@ -611,7 +633,13 @@ fn prepare(path: PathBuf, known: Known<'_>, ctx: &Ctx) -> WalkedFile {
_ => prepare_file_record(&db_path, &meta, &ctx.config, &ctx.registry),
};
WalkedFile { path: db_path, action, record, digest, aliased }
WalkedFile {
path: db_path,
action,
record,
digest,
aliased,
}
}
fn worker(shared: &Shared, ctx: &Ctx, tx: &mpsc::SyncSender<WalkEvent>) {
@ -635,7 +663,10 @@ fn worker(shared: &Shared, ctx: &Ctx, tx: &mpsc::SyncSender<WalkEvent>) {
Job::Files(files, rows) => (files, rows),
Job::Alias(path, stored) => {
slot.finish(found);
if tx.send(WalkEvent::File(prepare(path, Known::Exact(stored), ctx))).is_err() {
if tx
.send(WalkEvent::File(prepare(path, Known::Exact(stored), ctx)))
.is_err()
{
shared.shutdown();
return;
}
@ -658,7 +689,10 @@ fn worker(shared: &Shared, ctx: &Ctx, tx: &mpsc::SyncSender<WalkEvent>) {
shared.shutdown();
return;
}
if tx.send(WalkEvent::File(prepare(path, Known::InDir(&rows), ctx))).is_err() {
if tx
.send(WalkEvent::File(prepare(path, Known::InDir(&rows), ctx)))
.is_err()
{
// Receiver gone: the run was stopped or failed. Not an error.
shared.shutdown();
return;
@ -940,7 +974,13 @@ pub fn walk_indexable_files(
})
};
ParallelWalk { rx: Some(rx), handles, prefetch: Some(prefetch), shared, ctx }
ParallelWalk {
rx: Some(rx),
handles,
prefetch: Some(prefetch),
shared,
ctx,
}
}
/// Pick a worker count for these roots.
@ -1072,7 +1112,13 @@ mod tests {
fn names(files: &[WalkedFile]) -> Vec<String> {
let mut n: Vec<String> = files
.iter()
.map(|f| Path::new(&f.path).file_name().unwrap().to_string_lossy().into_owned())
.map(|f| {
Path::new(&f.path)
.file_name()
.unwrap()
.to_string_lossy()
.into_owned()
})
.collect();
n.sort();
n
@ -1124,7 +1170,10 @@ mod tests {
let skipped = files.iter().find(|f| f.record.is_none()).unwrap();
assert!(matches!(skipped.action, FileIndexAction::Skip));
assert!(skipped.path.contains('\u{FFFD}'), "stored spelling is the lossy one");
assert!(
skipped.path.contains('\u{FFFD}'),
"stored spelling is the lossy one"
);
fs::remove_dir_all(&root).ok();
}
@ -1173,7 +1222,11 @@ mod tests {
.collect();
let second = walk(&root, &db_with("skip-second", &indexed));
assert_eq!(second.len(), 2, "unchanged files are still reported as seen");
assert_eq!(
second.len(),
2,
"unchanged files are still reported as seen"
);
for f in &second {
assert_eq!(f.action, FileIndexAction::Skip);
assert!(f.record.is_none(), "an unchanged file is never hashed");
@ -1237,7 +1290,9 @@ mod tests {
})
.collect();
let recorded = !w.unreadable().is_empty();
let covers = w.unreadable().covers(locked.join("inside.txt").to_str().unwrap());
let covers = w
.unreadable()
.covers(locked.join("inside.txt").to_str().unwrap());
fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).ok();
@ -1278,18 +1333,19 @@ mod tests {
let root = tmp_tree("symlink-file");
touch(&root.join("real/target.txt"));
fs::create_dir_all(root.join("links")).unwrap();
std::os::unix::fs::symlink(
root.join("real/target.txt"),
root.join("links/alias.txt"),
)
.unwrap();
std::os::unix::fs::symlink(root.join("real/target.txt"), root.join("links/alias.txt"))
.unwrap();
let files = walk_with(&root, &empty_db("symlink-file"), true, false);
let paths: HashSet<&String> = files.iter().map(|f| &f.path).collect();
assert_eq!(paths.len(), 1, "both routes report one canonical path");
let canonical = path_to_db_string(&root.join("real/target.txt").canonicalize().unwrap());
assert_eq!(*paths.into_iter().next().unwrap(), canonical, "the target, not the alias");
assert_eq!(
*paths.into_iter().next().unwrap(),
canonical,
"the target, not the alias"
);
// The alias itself is still reported, so its row is never mistaken for
// deleted — it is reported under the *target's* path.
@ -1330,6 +1386,50 @@ mod tests {
fs::remove_dir_all(&outside).ok();
}
/// Windows counterpart of the symlink tests: `canonicalize` spells a
/// junction's target `\\?\C:\…`, and the walker must strip that before
/// storing — otherwise everything beneath the junction is spelled
/// differently from the plainly-spelled roots, full-path ignore patterns
/// never match there, and the canonical-directory dedup fails.
#[test]
#[cfg(windows)]
fn a_followed_junction_stores_plain_paths() {
let root = tmp_tree("junction");
touch(&root.join("real").join("target.txt"));
// Junctions need no privileges, unlike symlinks; still, skip cleanly
// on filesystems where mklink refuses.
let made = std::process::Command::new("cmd")
.args([
"/C",
"mklink",
"/J",
root.join("jlink").to_str().unwrap(),
root.join("real").to_str().unwrap(),
])
.status()
.map(|s| s.success())
.unwrap_or(false);
if !made {
fs::remove_dir_all(&root).ok();
return;
}
let files = walk_with(&root, &empty_db("junction"), true, false);
for f in &files {
assert!(
!f.path.starts_with(r"\\?\"),
"stored path leaked a verbatim prefix: {}",
f.path
);
}
// The junction resolves to the same canonical directory the walk
// reaches directly, so the dedup visits it exactly once. A leaked
// prefix would spell it twice and report the file twice.
assert_eq!(names(&files), vec!["target.txt"], "visited once");
fs::remove_dir_all(&root).ok();
}
#[test]
fn hidden_and_ignored_entries_are_pruned() {
let root = tmp_tree("prune");
@ -1340,11 +1440,8 @@ mod tests {
touch(&root.join(".dotfile"));
touch(&root.join("node_modules/dep/index.js"));
let ignore = IgnoreSet::compile(&[
"*.tmp".to_string(),
"node_modules".to_string(),
])
.unwrap();
let ignore =
IgnoreSet::compile(&["*.tmp".to_string(), "node_modules".to_string()]).unwrap();
let files: Vec<WalkedFile> = files_only(walk_indexable_files(
&[root.to_string_lossy().into_owned()],
false,
@ -1362,7 +1459,14 @@ mod tests {
let files = walk_with(&root, &empty_db("prune-hidden"), false, true);
assert_eq!(
names(&files),
vec![".dotfile", "index.js", "inside.txt", "keep.txt", "keep2.txt", "skip.tmp"],
vec![
".dotfile",
"index.js",
"inside.txt",
"keep.txt",
"keep2.txt",
"skip.tmp"
],
"include_hidden with no ignore patterns keeps everything"
);
fs::remove_dir_all(&root).ok();
@ -1381,7 +1485,11 @@ mod tests {
let kept = path_to_db_string(&root.join("kept.txt"));
let db = db_with(
"reconcile",
&[(gone.clone(), 1), (gone_nested.clone(), 1), (kept.clone(), 1)],
&[
(gone.clone(), 1),
(gone_nested.clone(), 1),
(kept.clone(), 1),
],
);
let mut stale = stale_only(walk_indexable_files(
@ -1400,7 +1508,10 @@ mod tests {
let mut want = vec![gone, gone_nested];
want.sort();
assert_eq!(stale, want, "exactly the rows with no file, from both directories");
assert_eq!(
stale, want,
"exactly the rows with no file, from both directories"
);
fs::remove_dir_all(&root).ok();
}
@ -1435,7 +1546,10 @@ mod tests {
));
fs::set_permissions(&locked, fs::Permissions::from_mode(0o755)).ok();
assert!(stale.is_empty(), "an unreadable directory is not an empty one");
assert!(
stale.is_empty(),
"an unreadable directory is not an empty one"
);
fs::remove_dir_all(&root).ok();
}
@ -1472,7 +1586,10 @@ mod tests {
4,
));
assert!(files.len() < 500, "an already-stopped walk does not run to completion");
assert!(
files.len() < 500,
"an already-stopped walk does not run to completion"
);
fs::remove_dir_all(&root).ok();
}
@ -1591,7 +1708,10 @@ mod tests {
#[test]
fn local_temp_dir_is_not_detected_as_network() {
let root = tmp_tree("fstype");
assert_eq!(thread_count_for(&[root.to_string_lossy().into_owned()]), LOCAL_THREADS);
assert_eq!(
thread_count_for(&[root.to_string_lossy().into_owned()]),
LOCAL_THREADS
);
fs::remove_dir_all(&root).ok();
}
}

154
crates/quicksearch-core/src/watcher.rs
View file

@ -63,8 +63,10 @@ use std::sync::{mpsc, Arc, Mutex};
use std::thread::{self, JoinHandle};
use std::time::{Duration, Instant};
use notify::{Config as NotifyConfig, ErrorKind as NotifyErrorKind, Event as NotifyEvent, EventKind,
RecommendedWatcher, RecursiveMode, Watcher as NotifyWatcher};
use notify::{
Config as NotifyConfig, ErrorKind as NotifyErrorKind, Event as NotifyEvent, EventKind,
RecommendedWatcher, RecursiveMode, Watcher as NotifyWatcher,
};
use crate::config::IgnoreSet;
use crate::file_handling::{filtered_dirs, UnreadableDirs};
@ -89,7 +91,10 @@ pub enum FsEvent {
/// Rename where both endpoints arrived in the same notify event. For
/// split rename halves (From or To only) the watcher emits Remove/Create
/// instead.
Rename { from: PathBuf, to: PathBuf },
Rename {
from: PathBuf,
to: PathBuf,
},
}
/// Sink callback. Called on the watcher thread; implementors should keep
@ -110,11 +115,16 @@ pub struct WatchFilters {
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum WatchError {
/// The indexed roots hold more directories than the cap allows.
TooManyDirectories { dirs: usize, cap: usize },
TooManyDirectories {
dirs: usize,
cap: usize,
},
/// The kernel refused a watch before our own cap was reached —
/// `fs.inotify.max_user_watches` is lower than the cap, or other
/// processes have consumed the shared budget.
KernelLimit { registered: usize },
KernelLimit {
registered: usize,
},
Other(String),
}
@ -640,9 +650,7 @@ fn is_event_interesting(ctx: &LoopCtx, path: &Path) -> bool {
if ctx.filters.ignore.matches_path(path) {
return false;
}
if !ctx.filters.include_hidden
&& path_has_hidden_component_under(path, &ctx.roots)
{
if !ctx.filters.include_hidden && path_has_hidden_component_under(path, &ctx.roots) {
return false;
}
true
@ -661,8 +669,7 @@ fn handle_notify_event(
// file out of an ignored directory into a watched one is a real
// Create, and the reverse is a real Remove. `apply_fs_event`
// re-checks each end, so passing the pair through is safe.
if !is_event_interesting(ctx, &ev.paths[0])
&& !is_event_interesting(ctx, &ev.paths[1])
if !is_event_interesting(ctx, &ev.paths[0]) && !is_event_interesting(ctx, &ev.paths[1])
{
return;
}
@ -705,19 +712,16 @@ fn handle_notify_event(
}
}
fn enqueue(
throttle: &mut HashMap<PathBuf, DirThrottleEntry>,
path: PathBuf,
op: QueuedOp,
) {
let dir = path.parent().map(|p| p.to_path_buf()).unwrap_or_else(|| path.clone());
let entry = throttle
.entry(dir)
.or_insert_with(|| DirThrottleEntry {
record_time: Instant::now(),
queue: HashMap::new(),
immediate: true,
});
fn enqueue(throttle: &mut HashMap<PathBuf, DirThrottleEntry>, path: PathBuf, op: QueuedOp) {
let dir = path
.parent()
.map(|p| p.to_path_buf())
.unwrap_or_else(|| path.clone());
let entry = throttle.entry(dir).or_insert_with(|| DirThrottleEntry {
record_time: Instant::now(),
queue: HashMap::new(),
immediate: true,
});
// Coalesce: Remove after Create → drop both. Modify after Modify → one Modify.
match (op, entry.queue.get(&path).copied()) {
(QueuedOp::Remove, Some(QueuedOp::Create)) => {
@ -892,7 +896,11 @@ mod tests {
fn flush_ready_respects_max_dirs_per_tick() {
let mut map: HashMap<PathBuf, DirThrottleEntry> = HashMap::new();
for i in 0..10 {
enqueue(&mut map, PathBuf::from(format!("/dir{}/a", i)), QueuedOp::Create);
enqueue(
&mut map,
PathBuf::from(format!("/dir{}/a", i)),
QueuedOp::Create,
);
}
let (sink, got) = sink_to_vec();
let mut config = WatcherConfig::default();
@ -942,8 +950,13 @@ mod tests {
let dir = tmp_dir("e2e");
let (sink, got) = sink_to_vec();
let mut w =
Watcher::start(std::iter::once(&dir), default_filters(), fast_config(), sink).unwrap();
let mut w = Watcher::start(
std::iter::once(&dir),
default_filters(),
fast_config(),
sink,
)
.unwrap();
let f = dir.join("hello.txt");
std::fs::write(&f, "hi").unwrap();
@ -973,21 +986,28 @@ mod tests {
#[test]
fn ignored_and_hidden_dirs_are_not_registered() {
let dir = tmp_dir("filter");
for sub in ["keep", "keep/nested", ".git", ".git/objects", "node_modules",
"node_modules/pkg", ".hidden"] {
for sub in [
"keep",
"keep/nested",
".git",
".git/objects",
"node_modules",
"node_modules/pkg",
".hidden",
] {
std::fs::create_dir_all(dir.join(sub)).unwrap();
}
let w =
Watcher::start(std::iter::once(&dir), default_filters(), fast_config(), sink_to_vec().0)
.unwrap();
let w = Watcher::start(
std::iter::once(&dir),
default_filters(),
fast_config(),
sink_to_vec().0,
)
.unwrap();
// root + keep + keep/nested. The 4 ignored/hidden dirs cost nothing.
assert_eq!(
w.watched_dirs(),
3,
"expected root, keep, keep/nested only"
);
assert_eq!(w.watched_dirs(), 3, "expected root, keep, keep/nested only");
drop(w);
std::fs::remove_dir_all(&dir).ok();
}
@ -1001,8 +1021,13 @@ mod tests {
include_hidden: true,
..default_filters()
};
let w = Watcher::start(std::iter::once(&dir), filters, fast_config(), sink_to_vec().0)
.unwrap();
let w = Watcher::start(
std::iter::once(&dir),
filters,
fast_config(),
sink_to_vec().0,
)
.unwrap();
assert_eq!(w.watched_dirs(), 2, "root + .hidden");
drop(w);
@ -1022,8 +1047,13 @@ mod tests {
max_watched_dirs: 2,
..fast_config()
};
let err = Watcher::start(std::iter::once(&dir), default_filters(), config, sink_to_vec().0)
.unwrap_err();
let err = Watcher::start(
std::iter::once(&dir),
default_filters(),
config,
sink_to_vec().0,
)
.unwrap_err();
assert_eq!(
err,
@ -1044,8 +1074,12 @@ mod tests {
std::fs::create_dir_all(&locked).unwrap();
crate::platform::deny_read(&locked).unwrap();
let started =
Watcher::start(std::iter::once(&dir), default_filters(), fast_config(), sink_to_vec().0);
let started = Watcher::start(
std::iter::once(&dir),
default_filters(),
fast_config(),
sink_to_vec().0,
);
crate::platform::restore_read(&locked).ok();
let w = started.expect("an unreadable directory must not fail the watcher");
@ -1074,8 +1108,12 @@ mod tests {
max_watched_dirs: 2,
..fast_config()
};
let started =
Watcher::start(std::iter::once(&dir), default_filters(), config, sink_to_vec().0);
let started = Watcher::start(
std::iter::once(&dir),
default_filters(),
config,
sink_to_vec().0,
);
crate::platform::restore_read(&locked).ok();
// Whichever order the walk visits them in, the cap is what stops us.
@ -1096,8 +1134,13 @@ mod tests {
max_watched_dirs: 2,
..fast_config()
};
let w = Watcher::start(std::iter::once(&dir), default_filters(), config, sink_to_vec().0)
.unwrap();
let w = Watcher::start(
std::iter::once(&dir),
default_filters(),
config,
sink_to_vec().0,
)
.unwrap();
assert_eq!(w.watched_dirs(), 2);
assert!(!w.is_degraded());
drop(w);
@ -1111,8 +1154,13 @@ mod tests {
fn a_directory_created_after_start_is_watched() {
let dir = tmp_dir("newdir");
let (sink, got) = sink_to_vec();
let mut w =
Watcher::start(std::iter::once(&dir), default_filters(), fast_config(), sink).unwrap();
let mut w = Watcher::start(
std::iter::once(&dir),
default_filters(),
fast_config(),
sink,
)
.unwrap();
assert_eq!(w.watched_dirs(), 1, "only the root to begin with");
let sub = dir.join("later");
@ -1220,9 +1268,13 @@ mod tests {
max_watched_dirs: 2,
..fast_config()
};
let mut w =
Watcher::start(std::iter::once(&dir), default_filters(), config, sink_to_vec().0)
.unwrap();
let mut w = Watcher::start(
std::iter::once(&dir),
default_filters(),
config,
sink_to_vec().0,
)
.unwrap();
assert!(!w.is_degraded(), "one directory is under the cap of 2");
// Two more directories: the first fits, the second cannot.

93
crates/quicksearch-core/tests/cascade.rs
View file

@ -184,7 +184,10 @@ fn rank_classification_across_all_stages() {
}
// Full-text hits carry snippets with valid ranges.
for h in hits.iter().filter(|h| h.stage == 5 || h.stage == 6 || h.stage == 8) {
for h in hits
.iter()
.filter(|h| h.stage == 5 || h.stage == 6 || h.stage == 8)
{
let snip = h.snippet.as_ref().expect("full-text hit has a snippet");
for &(a, b) in &snip.ranges {
assert!(a < b && b <= snip.window.len());
@ -206,7 +209,9 @@ fn path_substring_tiers_split_by_case() {
let (hits, _) = run_collect(&conn, "Vacation", &SearchOptions::default());
assert_eq!(
hits.iter().map(|h| (h.file_id, h.stage)).collect::<Vec<_>>(),
hits.iter()
.map(|h| (h.file_id, h.stage))
.collect::<Vec<_>>(),
vec![(exact, 9), (anycase, 10)],
"exact-case path matches outrank any-case ones"
);
@ -274,7 +279,9 @@ fn path_tiers_respect_the_three_char_floor() {
// is off here so `ab.txt`, a 1-edit match for `abc`, stays out of it.)
let (long, _) = run_collect(&conn, "abc", &SearchOptions::default());
assert_eq!(
long.iter().map(|h| (h.file_id, h.stage)).collect::<Vec<_>>(),
long.iter()
.map(|h| (h.file_id, h.stage))
.collect::<Vec<_>>(),
vec![(dir_only, 9)],
"3-char term: the directory match surfaces"
);
@ -293,7 +300,9 @@ fn term_with_separator_matches_across_the_path() {
let (hits, _) = run_collect(&conn, "docs/report", &SearchOptions::default());
assert_eq!(
hits.iter().map(|h| (h.file_id, h.stage)).collect::<Vec<_>>(),
hits.iter()
.map(|h| (h.file_id, h.stage))
.collect::<Vec<_>>(),
vec![(nested, 9)],
"a term spanning a separator can only match the full path"
);
@ -383,12 +392,7 @@ fn occurrence_counts_order_within_rank() {
let mut s = Seeder::new(&p, true);
let one = s.add("one.txt", "/d", 1, Some("zebra"));
let three = s.add("three.txt", "/d", 2, Some("zebra zebra zebra"));
let thousand = s.add(
"thousand.txt",
"/d",
3,
Some(&"zebra ".repeat(1500)),
);
let thousand = s.add("thousand.txt", "/d", 3, Some(&"zebra ".repeat(1500)));
let conn = s.done();
let (hits, _) = run_collect(&conn, "zebra", &SearchOptions::default());
@ -551,7 +555,10 @@ fn session_ignores_hide_hits_before_the_cap() {
..SearchOptions::default()
};
let (hits, outcome) = run_collect(&conn, "match", &options);
assert_eq!(hits.iter().map(|h| h.file_id).collect::<Vec<_>>(), vec![keep]);
assert_eq!(
hits.iter().map(|h| h.file_id).collect::<Vec<_>>(),
vec![keep]
);
assert_eq!(outcome.total, 1, "ignored rows never count toward totals");
drop(conn);
@ -634,9 +641,17 @@ fn wildcard_name_ranks_through_the_same_tiers() {
let (hits, _) = run_collect(&conn, "report*", &SearchOptions::default());
assert_eq!(
hits.iter().map(|h| (h.file_id, h.stage)).collect::<Vec<_>>(),
hits.iter()
.map(|h| (h.file_id, h.stage))
.collect::<Vec<_>>(),
// Within rank 3 the tie breaks by name: "2024…" sorts first.
vec![(whole_cs, 1), (whole_ci, 2), (suffix, 3), (sub_cs, 3), (sub_ci, 4)],
vec![
(whole_cs, 1),
(whole_ci, 2),
(suffix, 3),
(sub_cs, 3),
(sub_ci, 4)
],
"wildcard terms rank exactly like literal ones"
);
@ -697,7 +712,11 @@ fn wildcard_leaves_like_metacharacters_literal() {
let (hits, _) = run_collect(&conn, "100*", &SearchOptions::default());
let mut ids: Vec<i64> = hits.iter().map(|h| h.file_id).collect();
ids.sort();
assert_eq!(ids, vec![percent, underscore], "star globs, % and _ stay literal");
assert_eq!(
ids,
vec![percent, underscore],
"star globs, % and _ stay literal"
);
drop(conn);
std::fs::remove_file(&p).ok();
@ -715,11 +734,16 @@ fn wildcard_fulltext_narrows_with_fts_and_verifies_order() {
let (hits, _) = run_collect(&conn, "wond*world", &SearchOptions::default());
assert_eq!(
hits.iter().map(|h| (h.file_id, h.stage)).collect::<Vec<_>>(),
hits.iter()
.map(|h| (h.file_id, h.stage))
.collect::<Vec<_>>(),
vec![(ordered, 5)],
"unordered FTS candidates must fail pattern verification"
);
let snip = hits[0].snippet.as_ref().expect("wildcard hit has a snippet");
let snip = hits[0]
.snippet
.as_ref()
.expect("wildcard hit has a snippet");
assert_eq!(snip.ranges.len(), 1);
let (a, b) = snip.ranges[0];
assert_eq!(&snip.window[a..b], "wondrous world");
@ -740,7 +764,9 @@ fn wildcard_with_short_segments_falls_back_to_a_full_scan() {
let (hits, _) = run_collect(&conn, "ab*cd", &SearchOptions::default());
assert_eq!(
hits.iter().map(|h| (h.file_id, h.stage)).collect::<Vec<_>>(),
hits.iter()
.map(|h| (h.file_id, h.stage))
.collect::<Vec<_>>(),
vec![(hit, 5)]
);
@ -764,7 +790,10 @@ fn wildcard_path_tier_and_filters() {
// Structured filters gate wildcard scans like any other.
let (kept, _) = run_collect(&conn, "Vac*tion path:/elsewhere", &SearchOptions::default());
assert_eq!(kept.iter().map(|h| h.file_id).collect::<Vec<_>>(), vec![_filtered]);
assert_eq!(
kept.iter().map(|h| h.file_id).collect::<Vec<_>>(),
vec![_filtered]
);
drop(conn);
std::fs::remove_file(&p).ok();
@ -803,7 +832,9 @@ fn contentless_wildcard_degrades_to_unranked_stage6() {
// text the row can't be pattern-verified and lands at count-unknown 6.
let (hits, _) = run_collect(&conn, "wal*rus", &SearchOptions::default());
assert_eq!(
hits.iter().map(|h| (h.file_id, h.stage)).collect::<Vec<_>>(),
hits.iter()
.map(|h| (h.file_id, h.stage))
.collect::<Vec<_>>(),
vec![(doc, 6)]
);
assert!(hits[0].snippet.is_none());
@ -829,7 +860,9 @@ fn regex_only_query_hits_name_content_and_path() {
let (hits, _) = run_collect(&conn, r"regex:qz\d+", &SearchOptions::default());
assert_eq!(
hits.iter().map(|h| (h.file_id, h.stage)).collect::<Vec<_>>(),
hits.iter()
.map(|h| (h.file_id, h.stage))
.collect::<Vec<_>>(),
vec![(by_name, 4), (by_content, 6), (by_path, 10)],
"regex-only reuses the name/content/path tiers in cascade order"
);
@ -859,7 +892,10 @@ fn regex_is_case_insensitive_by_default_and_respects_filters() {
let conn = s.done();
let (hits, _) = run_collect(&conn, r"regex:qz\d path:/keep", &SearchOptions::default());
assert_eq!(hits.iter().map(|h| h.file_id).collect::<Vec<_>>(), vec![keep]);
assert_eq!(
hits.iter().map(|h| h.file_id).collect::<Vec<_>>(),
vec![keep]
);
// Inline opt-out flips it back to case-sensitive.
let (cs, _) = run_collect(&conn, r"regex:(?-i:qz)\d", &SearchOptions::default());
@ -882,7 +918,9 @@ fn regex_alongside_a_term_is_an_accept_predicate() {
let (hits, _) = run_collect(&conn, r"budget regex:acme\d", &SearchOptions::default());
assert_eq!(
hits.iter().map(|h| (h.file_id, h.stage)).collect::<Vec<_>>(),
hits.iter()
.map(|h| (h.file_id, h.stage))
.collect::<Vec<_>>(),
vec![(kept, 3)],
"the term drives ranking; the regex gates acceptance"
);
@ -926,7 +964,10 @@ fn service_surfaces_invalid_regex_as_an_error() {
let mut message = None;
while std::time::Instant::now() < deadline {
match updates.recv_timeout(std::time::Duration::from_millis(200)) {
Ok(SearchUpdate::Error { generation: g, message: m }) if g == generation => {
Ok(SearchUpdate::Error {
generation: g,
message: m,
}) if g == generation => {
message = Some(m);
break;
}
@ -1073,7 +1114,11 @@ fn a_pass_hands_hits_over_before_the_scan_reaches_the_end() {
assert_eq!(outcome.total, 2, "both matches still reach the sink");
assert_eq!(
batches.first().map(|b| b.as_slice()).and_then(|b| b.first()).map(|h| h.file_id),
batches
.first()
.map(|b| b.as_slice())
.and_then(|b| b.first())
.map(|h| h.file_id),
Some(early_worse),
"the early hit should have gone out before the scan found the better one; \
batch sizes: {:?}",

13
crates/quicksearch-core/tests/encrypted.rs
View file

@ -103,7 +103,8 @@ fn encrypted_index_lifecycle() {
// Raw bytes must not leak the indexed content anywhere in the file.
let raw = std::fs::read(&db_path).unwrap();
assert!(
!raw.windows(b"zebrapayload".len()).any(|w| w == b"zebrapayload"),
!raw.windows(b"zebrapayload".len())
.any(|w| w == b"zebrapayload"),
"plaintext content leaked into the encrypted file"
);
@ -187,9 +188,8 @@ fn encrypted_index_lifecycle() {
drop(conn);
// The right password still unlocks...
db::verify_process_key(&db_path.to_string_lossy()).expect(
"a stale schema must not make the correct password look wrong",
);
db::verify_process_key(&db_path.to_string_lossy())
.expect("a stale schema must not make the correct password look wrong");
// ...and the wrong one is still refused, with the same tagged error —
// the relaxation must not have turned the check into a rubber stamp.
db::set_process_key(Some(wrong_key.clone()));
@ -200,7 +200,10 @@ fn encrypted_index_lifecycle() {
// indexer down its rebuild path.
db::set_process_key(Some(key.clone()));
let err = db::open_existing(&db_path.to_string_lossy(), false).unwrap_err();
assert!(err.contains("not a compatible QuickSearch index"), "got: {err}");
assert!(
err.contains("not a compatible QuickSearch index"),
"got: {err}"
);
// And the rebuild comes back encrypted and searchable under the same
// key, so the whole path a real user walks is covered.

159
crates/quicksearch-core/tests/full_index.rs
View file

@ -20,7 +20,10 @@ fn tmp_dir(tag: &str) -> PathBuf {
"quicksearch-e2e-{}-{}-{}",
tag,
std::process::id(),
SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos()
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&p).unwrap();
p
@ -116,7 +119,10 @@ fn reindexing_an_unchanged_tree_changes_nothing() {
// The whole point: a second run over an unchanged tree must not delete
// and re-insert anything. A wiped-and-rebuilt row would come back with
// content_state reset, throwing away extracted text for no reason.
assert_eq!(first, second, "an unchanged tree must re-index to an identical set");
assert_eq!(
first, second,
"an unchanged tree must re-index to an identical set"
);
std::fs::remove_dir_all(&root).ok();
std::fs::remove_dir_all(&db_dir).ok();
@ -140,9 +146,19 @@ fn deleted_files_are_removed_and_new_ones_added() {
let names: Vec<String> = rows(&db)
.into_iter()
.map(|(p, _, _)| Path::new(&p).file_name().unwrap().to_string_lossy().into_owned())
.map(|(p, _, _)| {
Path::new(&p)
.file_name()
.unwrap()
.to_string_lossy()
.into_owned()
})
.collect();
assert_eq!(names, vec!["added.txt", "keep.txt"], "stale cleanup still works");
assert_eq!(
names,
vec!["added.txt", "keep.txt"],
"stale cleanup still works"
);
std::fs::remove_dir_all(&root).ok();
std::fs::remove_dir_all(&db_dir).ok();
@ -439,7 +455,10 @@ fn index_roots_once(roots: &[&Path], db: &Path, config: &Config) {
let service = IndexingService::new();
service
.start_indexing(
roots.iter().map(|r| r.to_string_lossy().into_owned()).collect(),
roots
.iter()
.map(|r| r.to_string_lossy().into_owned())
.collect(),
db.to_string_lossy().into_owned(),
config.clone(),
)
@ -475,10 +494,16 @@ fn two_roots_walk_extract_and_clean_independently() {
// Imbalanced roots so the round-robin writer sees a firehose and a
// trickle in the same run.
for i in 0..60 {
touch(&root_a.join(format!("a{:03}.txt", i)), b"alpha corpus xylophone");
touch(
&root_a.join(format!("a{:03}.txt", i)),
b"alpha corpus xylophone",
);
}
for i in 0..5 {
touch(&root_b.join(format!("b{:03}.txt", i)), b"bravo corpus quagmire");
touch(
&root_b.join(format!("b{:03}.txt", i)),
b"bravo corpus quagmire",
);
}
index_roots_once(&[&root_a, &root_b], &db, &config);
@ -489,7 +514,11 @@ fn two_roots_walk_extract_and_clean_independently() {
.unwrap();
assert_eq!(total, 65, "both roots fully walked");
let pending: i64 = conn
.query_row("SELECT COUNT(*) FROM files WHERE content_state = 0", [], |r| r.get(0))
.query_row(
"SELECT COUNT(*) FROM files WHERE content_state = 0",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(pending, 0, "per-root extraction drained both roots");
// Content from EACH root is searchable.
@ -501,7 +530,11 @@ fn two_roots_walk_extract_and_clean_independently() {
|r| r.get(0),
)
.unwrap();
assert!(hits > 0, "content from both roots must be indexed ({})", term);
assert!(
hits > 0,
"content from both roots must be indexed ({})",
term
);
}
drop(conn);
@ -552,7 +585,13 @@ fn a_deleted_directory_takes_its_whole_subtree_out_of_the_index() {
let names: Vec<String> = rows(&db)
.into_iter()
.map(|(p, _, _)| Path::new(&p).file_name().unwrap().to_string_lossy().into_owned())
.map(|(p, _, _)| {
Path::new(&p)
.file_name()
.unwrap()
.to_string_lossy()
.into_owned()
})
.collect();
assert_eq!(names, vec!["keep.txt"], "the whole subtree is swept");
@ -598,7 +637,9 @@ fn a_symlink_target_in_an_unwalked_directory_survives_reindexing() {
let first = rows(&db);
assert_eq!(first.len(), 3, "both targets indexed under their own paths");
assert!(
first.iter().any(|(p, _, _)| p.ends_with(".pruned/inner.txt")),
first
.iter()
.any(|(p, _, _)| p.ends_with(".pruned/inner.txt")),
"the pruned-directory target is stored under its canonical path"
);
@ -650,7 +691,10 @@ fn a_modified_symlink_target_is_updated_not_silently_ignored() {
let after = rows(&db);
assert_eq!(after.len(), 1, "still exactly one row");
assert_eq!(after[0].0, before[0].0, "same path");
assert_ne!(after[0].1, before[0].1, "mtime was refreshed, so it was re-read");
assert_ne!(
after[0].1, before[0].1,
"mtime was refreshed, so it was re-read"
);
std::fs::remove_dir_all(&root).ok();
std::fs::remove_dir_all(&outside).ok();
@ -679,7 +723,11 @@ fn overlapping_roots_index_each_file_exactly_once() {
.iter()
.filter(|(p, _, _)| p.ends_with("shared.txt"))
.collect();
assert_eq!(shared.len(), 1, "the doubly-reachable file has exactly one row");
assert_eq!(
shared.len(),
1,
"the doubly-reachable file has exactly one row"
);
// And the overlap must not make anything look stale on a second pass.
index_roots_once(&[&outer, &inner], &db, &config);
@ -774,7 +822,10 @@ fn stored_text(db: &Path, suffix: &str) -> Option<String> {
/// A tree that exercises every branch of the inline decision at once.
fn seed_mixed_tree(root: &Path) {
let big = "lorem ipsum dolor sit amet ".repeat(600); // ~16 KiB, past any head
touch(&root.join("small.txt"), b"a small plaintext body with xylophone in it");
touch(
&root.join("small.txt"),
b"a small plaintext body with xylophone in it",
);
touch(&root.join("large.txt"), big.as_bytes());
touch(&root.join("empty.txt"), b"");
// Binary bytes with a .txt extension: claimed by the plaintext
@ -784,8 +835,14 @@ fn seed_mixed_tree(root: &Path) {
touch(&root.join("bad.txt"), &[0x68, 0x69, 0xff, 0xfe, 0x00, 0x41]);
// No extension table, magic, or text sniff has an answer for NUL soup:
// no MIME, no extractor.
touch(&root.join("blob.bin"), &[0x00, 0x01, 0x02, 0xfd, 0xfe, 0xff]);
touch(&root.join("nested/deep/note.md"), b"# heading\n\nquagmire body text\n");
touch(
&root.join("blob.bin"),
&[0x00, 0x01, 0x02, 0xfd, 0xfe, 0xff],
);
touch(
&root.join("nested/deep/note.md"),
b"# heading\n\nquagmire body text\n",
);
}
#[test]
@ -844,7 +901,11 @@ fn the_head_boundary_decides_inlining_without_changing_the_result() {
// Both are fully extracted; the boundary only decides *which pass* did it.
let conn = rusqlite::Connection::open(&db).unwrap();
let pending: i64 = conn
.query_row("SELECT COUNT(*) FROM files WHERE content_state != 1", [], |r| r.get(0))
.query_row(
"SELECT COUNT(*) FROM files WHERE content_state != 1",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(pending, 0, "both sides of the boundary end up extracted");
drop(conn);
@ -896,7 +957,10 @@ fn extensionless_text_files_are_indexed() {
let db_dir = tmp_dir("extless-db");
let db = db_dir.join("index.sqlite");
touch(&root.join("README"), b"QuickSearch indexes zanzibar contents.\n");
touch(
&root.join("README"),
b"QuickSearch indexes zanzibar contents.\n",
);
touch(&root.join("Makefile"), b"all:\n\tcargo build --release\n");
touch(&root.join("go.sum"), b"example.com/x v1.0.0 h1:abcdef=\n");
touch(&root.join("blob"), &[0x00, 0x01, 0xfe, 0xff]);
@ -937,17 +1001,25 @@ fn utf16_files_are_stored_as_utf8() {
let db_dir = tmp_dir("charset-db");
let db = db_dir.join("index.sqlite");
let reg_src = "Windows Registry Editor Version 5.00\r\n\r\n[HKEY_CURRENT_USER\\Software\\Xylograph]\r\n";
let reg_src =
"Windows Registry Editor Version 5.00\r\n\r\n[HKEY_CURRENT_USER\\Software\\Xylograph]\r\n";
let mut reg_body = vec![0xFF, 0xFE];
reg_body.extend(reg_src.encode_utf16().flat_map(|u| u.to_le_bytes()));
touch(&root.join("export.reg"), &reg_body);
// The same encoding behind no extension at all: BOM first, sniff after.
let mut extless = vec![0xFF, 0xFE];
extless.extend("utf16 notes about quokkas".encode_utf16().flat_map(|u| u.to_le_bytes()));
extless.extend(
"utf16 notes about quokkas"
.encode_utf16()
.flat_map(|u| u.to_le_bytes()),
);
touch(&root.join("NOTES16"), &extless);
touch(&root.join("legacy.txt"), b"un caf\xe9 tr\xe8s agr\xe9able pr\xe8s du mus\xe9e");
touch(
&root.join("legacy.txt"),
b"un caf\xe9 tr\xe8s agr\xe9able pr\xe8s du mus\xe9e",
);
index_once(&root, &db, &Config::default());
assert_eq!(stored_text(&db, "export.reg").as_deref(), Some(reg_src));
@ -987,7 +1059,12 @@ fn rtf_files_are_extracted() {
for name in ["small.rtf", "big.rtf"] {
let text = stored_text(&db, name).unwrap_or_else(|| panic!("{} has no stored text", name));
assert!(text.contains("pangolin budget"), "{}: {:?}", name, &text[..text.len().min(80)]);
assert!(
text.contains("pangolin budget"),
"{}: {:?}",
name,
&text[..text.len().min(80)]
);
assert!(!text.contains(r"\rtf"), "{} stored control words", name);
}
@ -1113,7 +1190,11 @@ fn the_content_extension_filter_still_excludes_small_text_files() {
}
}
drop(conn);
assert_eq!(stored_text(&db, "skipped.txt"), None, "no body stored for a filtered file");
assert_eq!(
stored_text(&db, "skipped.txt"),
None,
"no body stored for a filtered file"
);
std::fs::remove_dir_all(&root).ok();
std::fs::remove_dir_all(&db_dir).ok();
@ -1142,7 +1223,10 @@ fn contentless_mode_still_indexes_inlined_files_without_storing_bodies() {
|r| r.get(0),
)
.unwrap();
assert_eq!(hits, 1, "an inlined file is still searchable in contentless mode");
assert_eq!(
hits, 1,
"an inlined file is still searchable in contentless mode"
);
std::fs::remove_dir_all(&root).ok();
std::fs::remove_dir_all(&db_dir).ok();
@ -1165,7 +1249,9 @@ fn a_heavy_root_does_not_stall_a_light_one() {
// small `maximum_text_size` so the cost lands in extraction rather than in
// the writer's tokenising.
let heavy = tmp_dir("stall-heavy");
let body: Vec<u8> = "sphinx of black quartz judge my vow ".repeat(40_000).into_bytes();
let body: Vec<u8> = "sphinx of black quartz judge my vow "
.repeat(40_000)
.into_bytes();
for i in 0..200 {
touch(&heavy.join(format!("d{}/big{:04}.txt", i % 8, i)), &body);
}
@ -1240,7 +1326,10 @@ fn a_heavy_root_does_not_stall_a_light_one() {
// The fixed design's stall does not grow with the heavy root's cost — it is
// one round-robin pass plus one commit — so making that root heavier only
// widens the margin.
eprintln!("longest light-root stall while heavy extracted: {:?}", worst);
eprintln!(
"longest light-root stall while heavy extracted: {:?}",
worst
);
assert!(
worst < Duration::from_millis(100),
"the light root stalled for {:?} while the heavy root extracted",
@ -1269,7 +1358,9 @@ fn the_wal_stays_bounded_during_a_run() {
let root = tmp_dir("wal-bound");
// Wide and text-heavy: every file lands in the FTS index, which is what
// actually fills the log.
let body: Vec<u8> = "sphinx of black quartz judge my vow ".repeat(200).into_bytes();
let body: Vec<u8> = "sphinx of black quartz judge my vow "
.repeat(200)
.into_bytes();
for i in 0..4000 {
touch(&root.join(format!("d{}/f{:05}.txt", i % 40, i)), &body);
}
@ -1347,7 +1438,9 @@ fn the_wal_stays_bounded_during_a_run() {
#[test]
fn a_stopped_run_is_still_optimized() {
let root = tmp_dir("stop-optimize");
let body: Vec<u8> = "sphinx of black quartz judge my vow ".repeat(200).into_bytes();
let body: Vec<u8> = "sphinx of black quartz judge my vow "
.repeat(200)
.into_bytes();
for i in 0..4000 {
touch(&root.join(format!("d{}/f{:05}.txt", i % 40, i)), &body);
}
@ -1387,11 +1480,17 @@ fn a_stopped_run_is_still_optimized() {
IndexingStatus::Error(e) => panic!("indexing failed: {}", e),
_ => {}
}
assert!(Instant::now() < idle_by, "the stopped run never reached Idle");
assert!(
Instant::now() < idle_by,
"the stopped run never reached Idle"
);
std::thread::sleep(Duration::from_millis(1));
}
assert!(saw_optimizing, "a stopped run must still publish Optimizing");
assert!(
saw_optimizing,
"a stopped run must still publish Optimizing"
);
assert_eq!(
std::fs::metadata(&wal).map(|m| m.len()).unwrap_or(0),
0,

127
crates/quicksearch-core/tests/snippet_perf.rs
View file

@ -28,23 +28,85 @@ const PAGE_SIZE: usize = 50;
/// Word list we draw text from. Has enough variety that trigram posting
/// lists stay non-trivial (hundreds of terms, not "the" 10000 times).
const WORDS: &[&str] = &[
"alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta",
"iota", "kappa", "lambda", "mu", "nu", "xi", "omicron", "pi", "rho",
"sigma", "tau", "upsilon", "phi", "chi", "psi", "omega",
"quick", "brown", "fox", "jumps", "over", "lazy", "dog",
"rust", "cargo", "sqlite", "baloo", "indexer", "tokenizer", "trigram",
"contentless", "posting", "fts5", "snippet", "highlight",
"morning", "afternoon", "evening", "midnight", "yesterday", "today",
"ocean", "forest", "mountain", "river", "valley", "bridge", "tunnel",
"tokyo", "paris", "london", "berlin", "rome", "madrid", "vienna",
"alpha",
"beta",
"gamma",
"delta",
"epsilon",
"zeta",
"eta",
"theta",
"iota",
"kappa",
"lambda",
"mu",
"nu",
"xi",
"omicron",
"pi",
"rho",
"sigma",
"tau",
"upsilon",
"phi",
"chi",
"psi",
"omega",
"quick",
"brown",
"fox",
"jumps",
"over",
"lazy",
"dog",
"rust",
"cargo",
"sqlite",
"baloo",
"indexer",
"tokenizer",
"trigram",
"contentless",
"posting",
"fts5",
"snippet",
"highlight",
"morning",
"afternoon",
"evening",
"midnight",
"yesterday",
"today",
"ocean",
"forest",
"mountain",
"river",
"valley",
"bridge",
"tunnel",
"tokyo",
"paris",
"london",
"berlin",
"rome",
"madrid",
"vienna",
];
/// Query terms that appear in the seeded corpus, so every query returns
/// real hits (not zero rows, which would skew against both paths equally
/// but wouldn't exercise the snippet renderer at all).
const QUERIES: &[&str] = &[
"quick", "rust", "baloo", "morning", "paris",
"tokyo", "forest", "indexer", "contentless", "trigram",
"quick",
"rust",
"baloo",
"morning",
"paris",
"tokyo",
"forest",
"indexer",
"contentless",
"trigram",
];
fn seed_text(rng: &mut u64, target_words: usize) -> String {
@ -127,14 +189,10 @@ fn snippet_paths_perf_comparison() {
let tx = conn.unchecked_transaction().unwrap();
{
let mut ins_reg = tx
.prepare(
"INSERT INTO st_regular(rowid, name, text) VALUES (?1, ?2, ?3)",
)
.prepare("INSERT INTO st_regular(rowid, name, text) VALUES (?1, ?2, ?3)")
.unwrap();
let mut ins_con = tx
.prepare(
"INSERT INTO st_contentless(rowid, name, text) VALUES (?1, ?2, ?3)",
)
.prepare("INSERT INTO st_contentless(rowid, name, text) VALUES (?1, ?2, ?3)")
.unwrap();
let mut ins_blob = tx
.prepare(
@ -164,17 +222,19 @@ fn snippet_paths_perf_comparison() {
// Warm each table's page cache so the first run doesn't skew.
for q in QUERIES.iter().take(2) {
let mut s = conn
.prepare(
"SELECT rowid FROM st_regular WHERE st_regular MATCH ?1 LIMIT 50",
)
.prepare("SELECT rowid FROM st_regular WHERE st_regular MATCH ?1 LIMIT 50")
.unwrap();
let _ = s.query_map(params![q], |r| r.get::<_, i64>(0)).unwrap().count();
let _ = s
.query_map(params![q], |r| r.get::<_, i64>(0))
.unwrap()
.count();
let mut s = conn
.prepare(
"SELECT rowid FROM st_contentless WHERE st_contentless MATCH ?1 LIMIT 50",
)
.prepare("SELECT rowid FROM st_contentless WHERE st_contentless MATCH ?1 LIMIT 50")
.unwrap();
let _ = s.query_map(params![q], |r| r.get::<_, i64>(0)).unwrap().count();
let _ = s
.query_map(params![q], |r| r.get::<_, i64>(0))
.unwrap()
.count();
}
// Path A: SQLite's built-in snippet() on a regular FTS5 table.
@ -192,7 +252,11 @@ fn snippet_paths_perf_comparison() {
.unwrap();
let rows = stmt
.query_map(params![q, PAGE_SIZE as i64], |r| {
Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?, r.get::<_, String>(2)?))
Ok((
r.get::<_, i64>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
))
})
.unwrap();
for r in rows {
@ -225,10 +289,7 @@ fn snippet_paths_perf_comparison() {
.unwrap();
let rows = stmt
.query_map(params![q, PAGE_SIZE as i64], |r| {
Ok((
r.get::<_, i64>(0)?,
r.get::<_, Option<Vec<u8>>>(1)?,
))
Ok((r.get::<_, i64>(0)?, r.get::<_, Option<Vec<u8>>>(1)?))
})
.unwrap();
for row in rows {
@ -253,8 +314,10 @@ fn snippet_paths_perf_comparison() {
let a_per_row = dur_a.as_secs_f64() / rows_a_total as f64 * 1_000_000.0;
let b_per_row = dur_b.as_secs_f64() / rows_b_total as f64 * 1_000_000.0;
eprintln!();
eprintln!("snippet perf (NUM_DOCS={NUM_DOCS}, PAGE_SIZE={PAGE_SIZE}, QUERIES={}, reps={a_reps}):",
QUERIES.len());
eprintln!(
"snippet perf (NUM_DOCS={NUM_DOCS}, PAGE_SIZE={PAGE_SIZE}, QUERIES={}, reps={a_reps}):",
QUERIES.len()
);
eprintln!(
" A (SQLite snippet(), regular FTS5): {:.2?} total, {:.2} ms/query, {:.1} µs/row ({} rows)",
dur_a, a_per_query, a_per_row, rows_a_total

351
crates/quicksearch-gui/src/app.rs
View file

@ -33,6 +33,52 @@ enum Tab {
Help,
}
/// A navigation the unsaved-changes guard put on hold. The intent survives
/// while the guard walks the dirty editors (on Quit there can be two); once
/// nothing relevant is dirty, [`QuickSearchApp::complete_nav`] performs it.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum NavIntent {
/// Leave the Manage tab for this one.
SwitchTab(Tab),
/// Close the Options window.
CloseOptions,
/// Close the application window.
Quit,
}
/// Which editor the guard is currently asking about.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum UnsavedSource {
Manage,
Options,
}
/// A button (or Esc/backdrop click) in the unsaved-changes modal.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
enum UnsavedChoice {
Apply,
Discard,
Cancel,
}
/// Which editor the guard must ask about for `intent`, if any. Quit asks
/// about Options before Manage — sequential prompts, one decision each; a
/// combined prompt could not Apply both drafts safely, since each is a full
/// `Config` snapshot and the second apply would revert the first.
fn guard_source(
intent: NavIntent,
manage_dirty: bool,
options_dirty: bool,
) -> Option<UnsavedSource> {
match intent {
NavIntent::SwitchTab(_) => manage_dirty.then_some(UnsavedSource::Manage),
NavIntent::CloseOptions => options_dirty.then_some(UnsavedSource::Options),
NavIntent::Quit if options_dirty => Some(UnsavedSource::Options),
NavIntent::Quit if manage_dirty => Some(UnsavedSource::Manage),
NavIntent::Quit => None,
}
}
pub struct QuickSearchApp {
cfg: Config,
backend: Backend,
@ -63,6 +109,10 @@ pub struct QuickSearchApp {
/// In-flight security flow (enable/disable/change password), driven by
/// the Options window's Security block.
security_prompt: Option<SecurityPrompt>,
/// A navigation held by the unsaved-changes guard; see [`NavIntent`].
pending_nav: Option<NavIntent>,
/// The guard resolved a Quit: let the next close request through.
quit_confirmed: bool,
config_error: Option<String>,
}
@ -156,6 +206,8 @@ impl QuickSearchApp {
stale_index_prompt,
watch_cap_prompt: None,
security_prompt: None,
pending_nav: None,
quit_confirmed: false,
config_error,
})
}
@ -184,15 +236,17 @@ impl QuickSearchApp {
self.backend.start_duplicates(&cfg, ctx.clone());
}
/// Save + route an edited config to the running services.
fn apply_new_config(&mut self, ctx: &egui::Context, mut new: Config) {
/// Save + route an edited config to the running services. Reports
/// whether the config was accepted — a `false` means nothing was saved
/// and the caller must keep any staged edits alive.
fn apply_new_config(&mut self, ctx: &egui::Context, mut new: Config) -> bool {
pin_live_fields(&mut new, &self.cfg);
if let Some((child, parent)) = nested_roots(&new.paths.indexing_paths).first() {
self.config_error = Some(format!(
"Not applied: indexed folder {} is nested under {}",
child, parent
));
return;
return false;
}
// Warned-root memory only means anything for folders still indexed.
// Pruning here is what makes removing and re-adding a folder warn
@ -243,6 +297,7 @@ impl QuickSearchApp {
}
}
self.cfg = new;
true
}
/// Switch the indexing mode and write it to the config immediately.
@ -946,6 +1001,80 @@ impl QuickSearchApp {
self.clear_prompt = false;
}
}
/// Drive the unsaved-changes guard. Each frame the pending intent picks
/// the editor to ask about; Apply and Discard clean one editor and let
/// the next frame either move to the second (Quit with both dirty asks
/// about Options, then Manage) or fall through to the navigation
/// itself. Cancel — button, Esc, or a backdrop click — drops the intent
/// and stays put.
fn unsaved_prompt_ui(&mut self, ctx: &egui::Context) {
let Some(intent) = self.pending_nav else {
return;
};
let dirty = (self.manage.is_dirty(), self.options.is_dirty(&self.cfg));
let Some(source) = guard_source(intent, dirty.0, dirty.1) else {
return self.complete_nav(ctx, intent);
};
match unsaved_changes_modal(ctx, source) {
None => {}
Some(UnsavedChoice::Cancel) => self.pending_nav = None,
Some(UnsavedChoice::Discard) => match source {
UnsavedSource::Manage => self.manage.discard(),
UnsavedSource::Options => self.options.close_discard(),
},
Some(UnsavedChoice::Apply) => {
let ok = match source {
UnsavedSource::Manage => match self.manage.take_apply_config(&self.cfg) {
Some(cfg) => {
let ok = self.apply_new_config(ctx, cfg);
if ok {
self.manage.mark_applied();
}
ok
}
None => true,
},
UnsavedSource::Options => match self.options.draft_config() {
Some(cfg) => {
let ok = self.apply_new_config(ctx, cfg);
if ok {
self.options.close_discard();
}
ok
}
None => true,
},
};
if !ok {
// Rejected (nested roots): stay put, keep the staged
// edits; the error banner explains what to fix.
self.pending_nav = None;
}
}
}
}
/// Perform a navigation the guard has cleared.
fn complete_nav(&mut self, ctx: &egui::Context, intent: NavIntent) {
self.pending_nav = None;
match intent {
NavIntent::SwitchTab(tab) => {
let was = self.tab;
self.tab = tab;
// Same trigger the direct switch in `update` runs; a guarded
// switch lands here instead.
if tab == Tab::Duplicates && was != Tab::Duplicates {
self.start_duplicates_scan(ctx);
}
}
NavIntent::CloseOptions => self.options.close_discard(),
NavIntent::Quit => {
self.quit_confirmed = true;
ctx.send_viewport_cmd(egui::ViewportCommand::Close);
}
}
}
}
/// Overwrite the fields a config draft must never carry back.
@ -956,11 +1085,59 @@ impl QuickSearchApp {
/// change. A draft taken before one of those clicks still holds the old
/// value, so applying it would silently revert protection, the salt, or
/// the indexing mode.
fn pin_live_fields(new: &mut Config, live: &Config) {
pub(crate) fn pin_live_fields(new: &mut Config, live: &Config) {
new.security = live.security.clone();
new.indexing.auto_index = live.indexing.auto_index;
}
/// Body of the unsaved-changes guard; `Some(choice)` when the user decided
/// this frame. Esc and a click on the backdrop count as Cancel.
///
/// The one `egui::Modal` in the app, deliberately: unlike the centered
/// `egui::Window` the other prompts use, its backdrop blocks input to
/// everything behind it — this guard exists to force a decision, and a
/// click that lands on the tab strip or the Options ✕ behind the prompt
/// would re-trigger or bypass it.
///
/// A free function (not a method) so tests can render it, and click its
/// buttons, in a headless egui context.
fn unsaved_changes_modal(ctx: &egui::Context, source: UnsavedSource) -> Option<UnsavedChoice> {
let mut choice = None;
let modal = egui::Modal::new(egui::Id::new("unsaved-guard")).show(ctx, |ui| {
ui.set_max_width(420.0);
ui.heading("Unsaved changes");
ui.label(match source {
UnsavedSource::Manage => "The Manage Index tab has edits that have not been applied.",
UnsavedSource::Options => "The Options window has edits that have not been applied.",
});
ui.add_space(6.0);
ui.horizontal(|ui| {
if ui
.add(crate::ui_util::bordered_button(
"Apply & Save",
crate::ui_util::BLUE,
))
.clicked()
{
choice = Some(UnsavedChoice::Apply);
}
if ui
.button(egui::RichText::new("Discard changes").color(ui.visuals().error_fg_color))
.clicked()
{
choice = Some(UnsavedChoice::Discard);
}
if ui.button("Cancel").clicked() {
choice = Some(UnsavedChoice::Cancel);
}
});
});
if choice.is_none() && modal.should_close() {
choice = Some(UnsavedChoice::Cancel);
}
choice
}
/// The stale-index window's body. Returns whether the user asked for the
/// rebuild.
///
@ -1038,27 +1215,54 @@ impl eframe::App for QuickSearchApp {
fn update(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) {
self.drain_events();
self.tick_debounce(ctx);
// Quitting with unapplied edits gets the same guard as tab
// navigation. The close must be cancelled *this* frame — once the
// window is gone there is nothing left to ask — and re-sent from
// `complete_nav` if the user chooses to leave.
if ctx.input(|i| i.viewport().close_requested()) && !self.quit_confirmed {
if self.manage.is_dirty() || self.options.is_dirty(&self.cfg) {
ctx.send_viewport_cmd(egui::ViewportCommand::CancelClose);
// Quitting subsumes any narrower pending navigation.
self.pending_nav = Some(NavIntent::Quit);
}
}
self.status_bar(ctx);
let previous_tab = self.tab;
// Tab clicks land on a local first, so leaving a dirty Manage tab
// can be held for the unsaved-changes guard instead of committed.
let mut requested = self.tab;
egui::TopBottomPanel::top("tab-strip").show(ctx, |ui| {
ui.horizontal(|ui| {
ui.selectable_value(&mut self.tab, Tab::Search, "Search");
ui.selectable_value(&mut self.tab, Tab::Manage, "Manage Index");
ui.selectable_value(&mut self.tab, Tab::Duplicates, "Duplicates");
ui.selectable_value(&mut self.tab, Tab::Logs, "Logs");
ui.selectable_value(&mut self.tab, Tab::Help, "Help");
ui.selectable_value(&mut requested, Tab::Search, "Search");
ui.selectable_value(&mut requested, Tab::Manage, "Manage Index");
ui.selectable_value(&mut requested, Tab::Duplicates, "Duplicates");
ui.selectable_value(&mut requested, Tab::Logs, "Logs");
ui.selectable_value(&mut requested, Tab::Help, "Help");
ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| {
if ui.button("⚙").on_hover_text("Options").clicked() {
if self.options.open {
self.options.open = false;
} else {
if !self.options.open {
self.options.open_with(&self.cfg);
} else if self.options.is_dirty(&self.cfg) {
if self.pending_nav.is_none() {
self.pending_nav = Some(NavIntent::CloseOptions);
}
} else {
self.options.close_discard();
}
}
});
});
});
if requested != self.tab {
if self.tab == Tab::Manage && self.manage.is_dirty() && self.pending_nav.is_none() {
self.pending_nav = Some(NavIntent::SwitchTab(requested));
} else {
self.tab = requested;
}
}
// Entering the Duplicates tab kicks off a fresh scan.
if self.tab == Tab::Duplicates && previous_tab != Tab::Duplicates {
self.start_duplicates_scan(ctx);
@ -1121,7 +1325,9 @@ impl eframe::App for QuickSearchApp {
ui.ctx().request_repaint_after(Duration::from_millis(100));
}
if let Some(new_cfg) = actions.apply_config {
self.apply_new_config(ctx, new_cfg);
if self.apply_new_config(ctx, new_cfg) {
self.manage.mark_applied();
}
}
}
Tab::Duplicates => {
@ -1141,6 +1347,9 @@ impl eframe::App for QuickSearchApp {
if let Some(action) = options_out.security {
self.handle_security_action(action);
}
if options_out.close_requested && self.pending_nav.is_none() {
self.pending_nav = Some(NavIntent::CloseOptions);
}
self.rebuild_prompt_ui(ctx);
self.security_prompt_ui(ctx);
self.clear_prompt_ui(ctx);
@ -1150,6 +1359,8 @@ impl eframe::App for QuickSearchApp {
// user is actually looking at.
self.stale_index_prompt_ui(ctx);
self.watch_cap_prompt_ui(ctx);
// Last: the guard must sit above everything else on screen.
self.unsaved_prompt_ui(ctx);
}
fn on_exit(&mut self, _gl: Option<&eframe::glow::Context>) {
@ -1219,7 +1430,10 @@ mod tests {
}
}
}
assert!(fired.is_some(), "no clickable Rebuild button for {source:?}");
assert!(
fired.is_some(),
"no clickable Rebuild button for {source:?}"
);
}
}
@ -1251,4 +1465,113 @@ mod tests {
"the staged edit itself still applies"
);
}
/// The whole guard decision table. Quit walks Options before Manage —
/// two sequential prompts, because each draft is a full `Config`
/// snapshot and applying both in one step would let the second revert
/// the first.
#[test]
fn guard_source_orders_quit_prompts_options_first() {
use NavIntent::*;
let tab = SwitchTab(Tab::Search);
assert_eq!(guard_source(tab, true, true), Some(UnsavedSource::Manage));
assert_eq!(guard_source(tab, true, false), Some(UnsavedSource::Manage));
assert_eq!(
guard_source(tab, false, true),
None,
"options guard its own close"
);
assert_eq!(guard_source(tab, false, false), None);
assert_eq!(
guard_source(CloseOptions, true, true),
Some(UnsavedSource::Options)
);
assert_eq!(
guard_source(CloseOptions, false, true),
Some(UnsavedSource::Options)
);
assert_eq!(
guard_source(CloseOptions, true, false),
None,
"manage guards tab switches"
);
assert_eq!(guard_source(CloseOptions, false, false), None);
assert_eq!(guard_source(Quit, true, true), Some(UnsavedSource::Options));
assert_eq!(
guard_source(Quit, false, true),
Some(UnsavedSource::Options)
);
assert_eq!(guard_source(Quit, true, false), Some(UnsavedSource::Manage));
assert_eq!(guard_source(Quit, false, false), None);
}
fn modal_frame(
ctx: &egui::Context,
source: UnsavedSource,
events: Vec<egui::Event>,
) -> Option<UnsavedChoice> {
let input = egui::RawInput {
screen_rect: Some(egui::Rect::from_min_size(
egui::Pos2::ZERO,
egui::vec2(1000.0, 700.0),
)),
events,
..Default::default()
};
let mut choice = None;
let _ = ctx.run(input, |ctx| choice = unsaved_changes_modal(ctx, source));
choice
}
/// Every way out of the guard reports the right choice: all three
/// buttons fire, Esc cancels, and an untouched frame decides nothing.
/// A backdrop click also maps to Cancel — that is `should_close`'s
/// contract — so the sweep counts button hits by their distinct values.
#[test]
fn the_unsaved_modal_reports_each_choice() {
for source in [UnsavedSource::Manage, UnsavedSource::Options] {
let ctx = egui::Context::default();
assert_eq!(
modal_frame(&ctx, source, Vec::new()),
None,
"an untouched frame must not decide"
);
let mut seen = std::collections::HashSet::new();
for y in (250..450).step_by(3) {
for x in (250..760).step_by(6) {
let pos = egui::pos2(x as f32, y as f32);
if let Some(choice) = modal_frame(&ctx, source, click_at(pos)) {
seen.insert(choice);
}
}
}
for expected in [
UnsavedChoice::Apply,
UnsavedChoice::Discard,
UnsavedChoice::Cancel,
] {
assert!(
seen.contains(&expected),
"{expected:?} never fired ({source:?})"
);
}
let esc = modal_frame(
&ctx,
source,
vec![egui::Event::Key {
key: egui::Key::Escape,
physical_key: None,
pressed: true,
repeat: false,
modifiers: egui::Modifiers::NONE,
}],
);
assert_eq!(esc, Some(UnsavedChoice::Cancel), "Esc must cancel");
}
}
}

265
crates/quicksearch-gui/src/manage_tab.rs
View file

@ -85,9 +85,7 @@ impl ManageTab {
}
// The config changed elsewhere (Options apply, a filter persisted
// from the Search tab, the fuzzy toggle's direct save…).
let dirty = self.draft.as_ref() != Some(baseline)
|| self.ext_filter_text != baseline.indexing.content_extensions.join("\n");
if !dirty {
if !self.is_dirty() {
// Nothing staged, nothing to lose.
return self.resync(config);
}
@ -98,6 +96,10 @@ impl ManageTab {
let mut merged = config.clone();
merged.paths.indexing_paths = draft.paths.indexing_paths;
merged.indexing = draft.indexing;
// Live state, not a user edit: the mode buttons write `auto_index`
// straight to the config, and a stale copy frozen into the draft
// here would read as permanently dirty.
merged.indexing.auto_index = config.indexing.auto_index;
merged.processing = draft.processing;
for pat in &config.indexing.ignore_patterns {
if !baseline.indexing.ignore_patterns.contains(pat)
@ -116,6 +118,57 @@ impl ManageTab {
self.baseline = Some(config.clone());
}
/// Whether the editors hold changes not yet applied.
///
/// `security` and `indexing.auto_index` are neutralized before comparing
/// — they are live state the app pins on apply (`pin_live_fields`), not
/// user edits — and the extension text is compared parsed, so a trailing
/// newline never reads as dirty. False before the first sync.
pub fn is_dirty(&self) -> bool {
let (Some(draft), Some(baseline)) = (&self.draft, &self.baseline) else {
return false;
};
let mut d = draft.clone();
crate::app::pin_live_fields(&mut d, baseline);
d != *baseline
|| parse_lines(&self.ext_filter_text)
!= parse_lines(&baseline.indexing.content_extensions.join("\n"))
}
/// The Apply & Save action, callable from the app's unsaved-changes
/// modal as well as the button. Syncs against `live` first — the same
/// merge the per-frame sync does — so a config applied elsewhere moments
/// ago is not reverted. Does NOT clear `baseline`: the app calls
/// [`ManageTab::mark_applied`] only after the apply succeeds, so a
/// rejected apply (nested roots) keeps the staged edits.
pub fn take_apply_config(&mut self, live: &Config) -> Option<Config> {
self.sync_editors(live);
let draft = self.draft.as_ref()?;
let mut new_config = draft.clone();
new_config.indexing.content_extensions = parse_lines(&self.ext_filter_text);
let roots = new_config.paths.indexing_paths.clone();
new_config
.indexing
.root_workers
.retain(|root, _| roots.contains(root));
Some(new_config)
}
/// The last apply landed: resync from the applied config next frame.
pub fn mark_applied(&mut self) {
self.baseline = None;
}
/// Drop every staged edit and editor box; the next frame resyncs from
/// the live config.
pub fn discard(&mut self) {
self.draft = None;
self.baseline = None;
self.new_root.clear();
self.new_ignore.clear();
self.root_error = None;
}
pub fn ui(
&mut self,
ui: &mut egui::Ui,
@ -373,6 +426,7 @@ impl ManageTab {
self.new_ignore.clear();
}
});
crate::ui_util::pattern_hint_label(&mut cols[1], &self.new_ignore);
cols[1].label(
egui::RichText::new(
"Changes apply on Apply & Save (may trigger index rebuild).",
@ -396,23 +450,33 @@ impl ManageTab {
);
ui.add_space(8.0);
let apply = ui.add(crate::ui_util::bordered_button(
"Apply & Save",
crate::ui_util::BLUE,
));
#[cfg(test)]
tests::record_widget("apply", &apply);
if apply.clicked() {
let mut new_config = draft.clone();
new_config.indexing.content_extensions = parse_lines(&self.ext_filter_text);
let roots = new_config.paths.indexing_paths.clone();
new_config
.indexing
.root_workers
.retain(|root, _| roots.contains(root));
actions.apply_config = Some(new_config);
self.baseline = None;
}
let dirty = self.is_dirty();
ui.horizontal(|ui| {
let apply = ui.add(crate::ui_util::bordered_button(
"Apply & Save",
if dirty {
crate::ui_util::ORANGE
} else {
crate::ui_util::BLUE
},
));
#[cfg(test)]
tests::record_widget("apply", &apply);
// The label comes and goes with the dirty state; keep it
// off the ids of whatever sits after it.
crate::ui_util::stable_section(ui, |ui| {
if dirty {
ui.label(
egui::RichText::new("Unsaved changes")
.small()
.color(crate::ui_util::ORANGE),
);
}
});
if apply.clicked() {
actions.apply_config = self.take_apply_config(config);
}
});
});
crate::ui_util::more_below_hint(ui, &scroll);
@ -902,7 +966,13 @@ mod tests {
let mut tab = ManageTab::new();
let cfg = cfg_with_root();
frame(&ctx, &mut tab, &cfg, &running_state(&["/data"], None), vec![]);
frame(
&ctx,
&mut tab,
&cfg,
&running_state(&["/data"], None),
vec![],
);
let (baseline, _) = widget("workers");
for state in [
@ -935,7 +1005,13 @@ mod tests {
let mut tab = ManageTab::new();
let cfg = cfg_with_root();
frame(&ctx, &mut tab, &cfg, &running_state(&["/data"], None), vec![]);
frame(
&ctx,
&mut tab,
&cfg,
&running_state(&["/data"], None),
vec![],
);
let field = widget("workers").1.center();
frame(
&ctx,
@ -962,7 +1038,9 @@ mod tests {
// egui fires a click on release; give it the follow-up frame.
actions = frame(&ctx, &mut tab, &cfg, &busy, vec![]);
}
let applied = actions.apply_config.expect("Apply & Save produced a config");
let applied = actions
.apply_config
.expect("Apply & Save produced a config");
assert_eq!(applied.indexing.root_workers.get("/data"), Some(&8));
}
@ -1369,4 +1447,145 @@ mod tests {
.count();
assert_eq!(count, 1);
}
#[test]
fn a_fresh_tab_is_not_dirty_before_or_after_its_first_sync() {
let mut tab = ManageTab::new();
assert!(!tab.is_dirty(), "no draft yet");
tab.sync_editors(&cfg_with_root());
assert!(!tab.is_dirty(), "a fresh sync stages nothing");
}
#[test]
fn a_staged_edit_reads_dirty_and_discard_reverts_it() {
let cfg = cfg_with_root();
let mut tab = synced_tab(&cfg);
tab.draft
.as_mut()
.unwrap()
.indexing
.ignore_patterns
.push("*.jpg".into());
tab.new_ignore = "half-typ".into();
tab.root_error = Some("bad".into());
assert!(tab.is_dirty());
tab.discard();
assert!(!tab.is_dirty());
assert!(tab.new_ignore.is_empty(), "scratch boxes are cleared too");
assert!(tab.root_error.is_none());
tab.sync_editors(&cfg);
assert_eq!(tab.draft.as_ref(), Some(&cfg), "resynced from live");
assert!(!tab.is_dirty());
}
/// `parse_lines` drops blank lines and trims entries, so cosmetic
/// whitespace in the extension editor must not read as an edit.
#[test]
fn a_trailing_newline_in_the_extension_editor_is_not_dirty() {
let mut cfg = cfg_with_root();
cfg.indexing.content_extensions = vec!["txt".into(), "md".into()];
let mut tab = synced_tab(&cfg);
tab.ext_filter_text.push('\n');
assert!(!tab.is_dirty(), "a blank line is not an edit");
tab.ext_filter_text.push_str("pdf");
assert!(tab.is_dirty(), "a real entry is");
}
/// The mode buttons write `auto_index` straight to the live config. A
/// stale copy frozen into a staged draft used to read as permanently
/// dirty — and applying it would have reverted the stop.
#[test]
fn a_live_mode_flip_does_not_read_as_dirty() {
let mut cfg = cfg_with_root();
cfg.indexing.auto_index = true;
let mut tab = synced_tab(&cfg);
// Stop clicked, nothing staged: the tab resyncs and stays clean.
let mut stopped = cfg.clone();
stopped.indexing.auto_index = false;
tab.sync_editors(&stopped);
assert!(!tab.is_dirty());
// Staged edit, then Return to Automatic: dirty because of the edit
// only, and the draft adopts the live mode.
tab.draft
.as_mut()
.unwrap()
.indexing
.ignore_patterns
.push("*.jpg".into());
let mut auto_again = stopped.clone();
auto_again.indexing.auto_index = true;
tab.sync_editors(&auto_again);
assert!(tab.is_dirty(), "the staged pattern is still pending");
let applied = tab.take_apply_config(&auto_again).expect("a config");
assert!(
applied.indexing.auto_index,
"applying must not revert the live mode"
);
// Un-staging the edit reads clean again — not permanently dirty on
// a stale mode copy.
tab.draft.as_mut().unwrap().indexing.ignore_patterns.pop();
assert!(!tab.is_dirty());
}
/// `take_apply_config` must leave the editors intact: the app reports
/// back via `mark_applied` only when the apply landed, so a rejected
/// config (nested roots) keeps the user's staged edits on screen.
#[test]
fn a_rejected_apply_keeps_the_draft() {
let cfg = cfg_with_root();
let mut tab = synced_tab(&cfg);
tab.draft
.as_mut()
.unwrap()
.paths
.indexing_paths
.push("/data/nested".into());
let staged = tab.take_apply_config(&cfg).expect("a config to apply");
assert!(staged
.paths
.indexing_paths
.contains(&"/data/nested".to_string()));
assert!(tab.baseline.is_some(), "baseline survives the attempt");
assert!(tab.is_dirty(), "the staged root is still pending");
tab.mark_applied();
assert!(tab.baseline.is_none(), "a landed apply forces a resync");
assert!(!tab.is_dirty());
}
/// The dirty label sits after the Apply button inside a stable section:
/// its coming and going must never rename the button, which egui hangs
/// interaction state off.
#[test]
fn the_unsaved_label_appears_without_renaming_the_apply_button() {
let ctx = egui::Context::default();
let mut tab = ManageTab::new();
let cfg = cfg_with_root();
let clean = frame_text_with(&ctx, &mut tab, &cfg, &idle_state());
assert!(
!clean.iter().any(|t| t.contains("Unsaved changes")),
"a clean tab must not claim unsaved changes"
);
let (clean_id, _) = widget("apply");
tab.draft
.as_mut()
.unwrap()
.indexing
.ignore_patterns
.push("*.jpg".into());
let dirty = frame_text_with(&ctx, &mut tab, &cfg, &idle_state());
assert!(
dirty.iter().any(|t| t.contains("Unsaved changes")),
"painted: {:?}",
dirty
);
assert_eq!(widget("apply").0, clean_id, "the label renamed the button");
}
}

505
crates/quicksearch-gui/src/options.rs
View file

@ -31,6 +31,10 @@ pub struct OptionsOutput {
pub applied: Option<Config>,
/// A Security block action was clicked.
pub security: Option<SecurityAction>,
/// The title-bar close was clicked while the draft holds unapplied
/// edits. The window is held open; the app raises the unsaved-changes
/// guard.
pub close_requested: bool,
}
pub struct OptionsWindow {
@ -58,6 +62,47 @@ impl OptionsWindow {
self.keychain_probed_for = None;
}
/// Whether the draft differs from the live config. The fields the app
/// pins on apply (`security`, `indexing.auto_index`) are neutralized
/// first — the Security block acts on the live config directly and must
/// not make the window read as dirty.
pub fn is_dirty(&self, current: &Config) -> bool {
let Some(draft) = &self.draft else {
return false;
};
let mut d = draft.clone();
crate::app::pin_live_fields(&mut d, current);
d != *current
}
/// The draft as it stands, for the app's unsaved-changes guard.
pub fn draft_config(&self) -> Option<Config> {
self.draft.clone()
}
/// Close and drop the draft (Discard, or a clean close).
pub fn close_discard(&mut self) {
self.open = false;
self.draft = None;
}
/// Adopt the window's open flag for this frame. A dirty close is
/// intercepted: the window is held open and the caller is told to raise
/// the unsaved-changes guard instead.
fn intercept_close(&mut self, still_open: bool, current: &Config) -> bool {
self.open = still_open;
if self.open {
return false;
}
if self.is_dirty(current) {
self.open = true;
true
} else {
self.draft = None;
false
}
}
/// True when this index's key really is in the OS keychain: the
/// preference is on *and* the keychain answers with an entry (a dead
/// daemon, a locked keyring or a denied prompt all read as "no", which
@ -86,6 +131,7 @@ impl OptionsWindow {
let mut out = OptionsOutput::default();
let mut open = self.open;
let keychain_active = self.keychain_active(current);
let dirty = self.is_dirty(current);
let draft = self.draft.as_mut().unwrap();
egui::Window::new("Options")
@ -93,91 +139,109 @@ impl OptionsWindow {
.resizable(false)
.default_width(420.0)
.show(ctx, |ui| {
let scroll = egui::ScrollArea::vertical().max_height(480.0).show(ui, |ui| {
ui.heading(egui::RichText::new("Paths").strong());
egui::Grid::new("opt-paths").num_columns(2).show(ui, |ui| {
ui.label("Database file");
ui.add(
egui::TextEdit::singleline(&mut draft.paths.database_path)
.desired_width(260.0),
let scroll = egui::ScrollArea::vertical()
.max_height(480.0)
.show(ui, |ui| {
ui.heading(egui::RichText::new("Paths").strong());
egui::Grid::new("opt-paths").num_columns(2).show(ui, |ui| {
ui.label("Database file");
ui.add(
egui::TextEdit::singleline(&mut draft.paths.database_path)
.desired_width(260.0),
);
ui.end_row();
});
ui.label(
egui::RichText::new(
"Indexed folders are managed on the Manage Index tab.",
)
.small()
.weak(),
);
ui.end_row();
});
ui.label(
egui::RichText::new(
"Indexed folders are managed on the Manage Index tab.",
)
.small()
.weak(),
);
ui.separator();
ui.separator();
ui.heading(egui::RichText::new("Indexing").strong());
config_editor_ui(ui, draft, Section::Indexing);
ui.label(
egui::RichText::new(
"Automatic and manual indexing are switched on the \
ui.heading(egui::RichText::new("Indexing").strong());
config_editor_ui(ui, draft, Section::Indexing);
ui.label(
egui::RichText::new(
"Automatic and manual indexing are switched on the \
Manage Index tab.",
)
.small()
.weak(),
);
ui.separator();
)
.small()
.weak(),
);
ui.separator();
ui.heading(egui::RichText::new("Processing").strong());
config_editor_ui(ui, draft, Section::Processing);
ui.separator();
ui.heading(egui::RichText::new("Processing").strong());
config_editor_ui(ui, draft, Section::Processing);
ui.separator();
ui.heading(egui::RichText::new("Search").strong());
config_editor_ui(ui, draft, Section::Search);
ui.separator();
ui.heading(egui::RichText::new("Search").strong());
config_editor_ui(ui, draft, Section::Search);
ui.separator();
ui.heading(egui::RichText::new("Interface").strong());
egui::Grid::new("opt-ui").num_columns(2).show(ui, |ui| {
ui.label("UI scale");
ui.add(
egui::Slider::new(&mut draft.ui.scale, 0.5..=2.5)
.step_by(0.05)
.fixed_decimals(2),
)
.on_hover_text(
"Zooms the whole interface: fonts, spacing, and \
ui.heading(egui::RichText::new("Interface").strong());
egui::Grid::new("opt-ui").num_columns(2).show(ui, |ui| {
ui.label("UI scale");
ui.add(
egui::Slider::new(&mut draft.ui.scale, 0.5..=2.5)
.step_by(0.05)
.fixed_decimals(2),
)
.on_hover_text(
"Zooms the whole interface: fonts, spacing, and \
widgets. Ctrl +/- and Ctrl 0 adjust it temporarily \
at runtime.",
);
ui.end_row();
});
ui.separator();
);
ui.end_row();
});
ui.separator();
// Security acts on the live config, not the draft: each
// action opens its own confirmation flow immediately.
// The KDF salt is deliberately never shown here (or
// anywhere else in the GUI).
ui.heading(egui::RichText::new("Security").strong());
out.security = security_ui(ui, current, keychain_active);
});
// Security acts on the live config, not the draft: each
// action opens its own confirmation flow immediately.
// The KDF salt is deliberately never shown here (or
// anywhere else in the GUI).
ui.heading(egui::RichText::new("Security").strong());
out.security = security_ui(ui, current, keychain_active);
});
crate::ui_util::more_below_hint(ui, &scroll);
ui.separator();
ui.horizontal(|ui| {
if ui.button("Apply & Save").clicked() {
let apply = ui.add(crate::ui_util::bordered_button(
"Apply & Save",
if dirty {
crate::ui_util::ORANGE
} else {
crate::ui_util::BLUE
},
));
if apply.clicked() {
out.applied = Some(draft.clone());
}
ui.label(
egui::RichText::new(
"Changes to tokenizer, filters, hidden files, or hashing \
prompt an index rebuild.",
)
.small()
.weak(),
);
// Comes and goes with the dirty state; keep it off the
// ids of the hint that follows.
crate::ui_util::stable_section(ui, |ui| {
if dirty {
ui.label(
egui::RichText::new("Unsaved changes")
.small()
.color(crate::ui_util::ORANGE),
);
}
});
});
ui.label(
egui::RichText::new(
"Changes to tokenizer, filters, hidden files, or hashing \
prompt an index rebuild.",
)
.small()
.weak(),
);
});
self.open = open;
if !self.open {
self.draft = None;
}
out.close_requested = self.intercept_close(open, current);
out
}
}
@ -239,95 +303,104 @@ fn security_ui(
pub fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section) {
match section {
Section::Indexing => {
egui::Grid::new("cfg-indexing").num_columns(2).show(ui, |ui| {
// Automatic vs manual is deliberately absent: it is live
// state, switched (and saved) by the Stop / Return to
// Automatic buttons on the Manage Index tab. A staged copy
// of it here would fight those buttons.
ui.label("Full reindex every");
ui.horizontal(|ui| {
ui.add(
egui::DragValue::new(&mut config.indexing.reindex_interval_minutes)
.range(5..=60 * 24 * 30),
);
ui.label("minutes");
egui::Grid::new("cfg-indexing")
.num_columns(2)
.show(ui, |ui| {
// Automatic vs manual is deliberately absent: it is live
// state, switched (and saved) by the Stop / Return to
// Automatic buttons on the Manage Index tab. A staged copy
// of it here would fight those buttons.
ui.label("Full reindex every");
ui.horizontal(|ui| {
ui.add(
egui::DragValue::new(&mut config.indexing.reindex_interval_minutes)
.range(5..=60 * 24 * 30),
);
ui.label("minutes");
});
ui.end_row();
ui.label("Follow symlinks");
ui.checkbox(&mut config.indexing.follow_symlinks, "");
ui.end_row();
ui.label("Include hidden files");
ui.checkbox(&mut config.indexing.include_hidden, "");
ui.end_row();
});
ui.end_row();
ui.label("Follow symlinks");
ui.checkbox(&mut config.indexing.follow_symlinks, "");
ui.end_row();
ui.label("Include hidden files");
ui.checkbox(&mut config.indexing.include_hidden, "");
ui.end_row();
});
}
Section::Processing => {
egui::Grid::new("cfg-processing").num_columns(2).show(ui, |ui| {
ui.label("Tokenizer");
egui::ComboBox::from_id_salt("cfg-tokenize")
.selected_text(&config.processing.tokenize)
.show_ui(ui, |ui| {
for opt in ["trigram", "unicode61", "porter"] {
ui.selectable_value(
&mut config.processing.tokenize,
opt.to_string(),
opt,
);
}
});
ui.end_row();
egui::Grid::new("cfg-processing")
.num_columns(2)
.show(ui, |ui| {
ui.label("Tokenizer");
egui::ComboBox::from_id_salt("cfg-tokenize")
.selected_text(&config.processing.tokenize)
.show_ui(ui, |ui| {
for opt in ["trigram", "unicode61", "porter"] {
ui.selectable_value(
&mut config.processing.tokenize,
opt.to_string(),
opt,
);
}
});
ui.end_row();
ui.label("");
ui.hyperlink_to(
"Tokenizer documentation",
"https://www.sqlite.org/fts5.html#tokenizers",
);
ui.end_row();
ui.label("");
ui.hyperlink_to(
"Tokenizer documentation",
"https://www.sqlite.org/fts5.html#tokenizers",
);
ui.end_row();
ui.label("Hash sample size (bytes)");
ui.add(egui::DragValue::new(&mut config.processing.hash_length).range(512..=1_048_576));
ui.end_row();
ui.label("Hash sample size (bytes)");
ui.add(
egui::DragValue::new(&mut config.processing.hash_length)
.range(512..=1_048_576),
);
ui.end_row();
ui.label("Max stored text (bytes)");
ui.add(
egui::DragValue::new(&mut config.processing.maximum_text_size)
.range(1024..=16_777_216),
);
ui.end_row();
ui.label("Max stored text (bytes)");
ui.add(
egui::DragValue::new(&mut config.processing.maximum_text_size)
.range(1024..=16_777_216),
);
ui.end_row();
ui.label("Max text file size (bytes)");
ui.add(
egui::DragValue::new(&mut config.processing.maximum_text_file_size)
.range(1024..=1_073_741_824),
);
ui.end_row();
ui.label("Max text file size (bytes)");
ui.add(
egui::DragValue::new(&mut config.processing.maximum_text_file_size)
.range(1024..=1_073_741_824),
);
ui.end_row();
ui.label("Batch size");
ui.add(egui::DragValue::new(&mut config.processing.batch_size).range(10..=100_000));
ui.end_row();
ui.label("Batch size");
ui.add(
egui::DragValue::new(&mut config.processing.batch_size).range(10..=100_000),
);
ui.end_row();
ui.label("Max WAL size (bytes)");
ui.add(
egui::DragValue::new(&mut config.processing.maximum_wal_size)
.range(0u64..=8_589_934_592u64),
)
.on_hover_text(
"How large index.sqlite-wal may grow during a run before the \
ui.label("Max WAL size (bytes)");
ui.add(
egui::DragValue::new(&mut config.processing.maximum_wal_size)
.range(0u64..=8_589_934_592u64),
)
.on_hover_text(
"How large index.sqlite-wal may grow during a run before the \
indexer forces a checkpoint. 0 disables forced checkpoints; \
anything below 16 MiB is raised to it.",
);
ui.end_row();
ui.label("Store text for snippets");
ui.checkbox(&mut config.processing.store_text_for_snippets, "")
.on_hover_text(
"Off: smaller index, but no previews, occurrence ranking, \
case verification, or fuzzy full-text search",
);
ui.end_row();
});
ui.end_row();
ui.label("Store text for snippets");
ui.checkbox(&mut config.processing.store_text_for_snippets, "")
.on_hover_text(
"Off: smaller index, but no previews, occurrence ranking, \
case verification, or fuzzy full-text search",
);
ui.end_row();
});
}
Section::Search => {
egui::Grid::new("cfg-search").num_columns(2).show(ui, |ui| {
@ -388,3 +461,147 @@ pub fn config_editor_ui(ui: &mut egui::Ui, config: &mut Config, section: Section
}
}
}
#[cfg(test)]
mod tests {
use super::*;
// All headless-safe: `keychain_active` only probes the OS keychain when
// `use_keychain` is set, and no test here sets it.
#[test]
fn a_fresh_draft_is_not_dirty() {
let mut w = OptionsWindow::new();
let cfg = Config::default();
assert!(!w.is_dirty(&cfg), "no draft at all");
w.open_with(&cfg);
assert!(!w.is_dirty(&cfg));
}
#[test]
fn an_edited_draft_is_dirty_until_discarded() {
let mut w = OptionsWindow::new();
let cfg = Config::default();
w.open_with(&cfg);
w.draft.as_mut().unwrap().search.debounce_ms += 100;
assert!(w.is_dirty(&cfg));
w.close_discard();
assert!(!w.open);
assert!(!w.is_dirty(&cfg), "the draft is gone");
}
/// The Security block and the mode buttons act on the live config while
/// the window sits open; the stale copies in the draft are not edits.
#[test]
fn live_security_and_mode_changes_are_not_dirty() {
let mut w = OptionsWindow::new();
let mut cfg = Config::default();
w.open_with(&cfg);
cfg.security.use_keychain = !cfg.security.use_keychain;
cfg.indexing.auto_index = !cfg.indexing.auto_index;
assert!(!w.is_dirty(&cfg));
}
#[test]
fn a_dirty_close_is_held_and_a_clean_one_drops_the_draft() {
let mut w = OptionsWindow::new();
let cfg = Config::default();
w.open_with(&cfg);
w.draft.as_mut().unwrap().search.debounce_ms += 100;
assert!(
w.intercept_close(false, &cfg),
"dirty close raises the guard"
);
assert!(w.open, "the window is held open until the user decides");
assert!(w.draft.is_some(), "the draft survives");
assert!(!w.intercept_close(true, &cfg), "still open: nothing to do");
w.draft = Some(cfg.clone());
assert!(!w.intercept_close(false, &cfg), "a clean close just closes");
assert!(!w.open);
assert!(w.draft.is_none());
}
/// Where `needle` was painted this frame, as the center of its galley —
/// a click target that follows the layout instead of pinning it.
fn painted_text_center(out: &egui::FullOutput, needle: &str) -> Option<egui::Pos2> {
fn walk(shape: &egui::epaint::Shape, needle: &str, found: &mut Option<egui::Pos2>) {
match shape {
egui::epaint::Shape::Text(t) => {
if t.galley.text() == needle {
*found = Some(t.pos + t.galley.size() / 2.0);
}
}
egui::epaint::Shape::Vec(v) => {
for s in v {
walk(s, needle, found);
}
}
_ => {}
}
}
let mut found = None;
for clipped in &out.shapes {
walk(&clipped.shape, needle, &mut found);
}
found
}
/// One real frame of the window in a headless context: it renders, and
/// the Apply & Save click comes back out as `applied`.
#[test]
fn the_window_renders_and_apply_reports_the_draft() {
let ctx = egui::Context::default();
let cfg = Config::default();
let mut w = OptionsWindow::new();
w.open_with(&cfg);
w.draft.as_mut().unwrap().search.debounce_ms += 100;
let run = |w: &mut OptionsWindow, events: Vec<egui::Event>| {
let input = egui::RawInput {
screen_rect: Some(egui::Rect::from_min_size(
egui::Pos2::ZERO,
egui::vec2(1000.0, 900.0),
)),
events,
..Default::default()
};
let mut out = OptionsOutput::default();
let full = ctx.run(input, |ctx| out = w.ui(ctx, &cfg));
(out, full)
};
// A new egui window spends its first frames in sizing passes that
// suppress painting; run untouched frames until the settled button
// is actually on screen.
let mut target = None;
for _ in 0..5 {
let (untouched, full) = run(&mut w, vec![]);
assert!(untouched.applied.is_none());
assert!(!untouched.close_requested);
target = painted_text_center(&full, "Apply & Save");
if target.is_some() {
break;
}
}
let target = target.expect("the Apply & Save button was not painted");
let clicks = [true, false]
.into_iter()
.map(|pressed| egui::Event::PointerButton {
pos: target,
button: egui::PointerButton::Primary,
pressed,
modifiers: egui::Modifiers::default(),
})
.collect();
let (clicked, _) = run(&mut w, clicks);
let applied = clicked.applied.expect("the click did not report a config");
assert_eq!(
applied.search.debounce_ms,
Config::default().search.debounce_ms + 100,
"the click reported the edited draft"
);
}
}

56
crates/quicksearch-gui/src/search_tab.rs
View file

@ -29,6 +29,14 @@ pub struct IgnoreDialog {
pub persist: bool,
}
/// Glob ignoring everything under `dir`, spelled with the platform
/// separator. `Path::join` inserts a separator only where one is needed, so
/// a drive root yields `C:\*` rather than the never-matching `C:\/*` a
/// `format!("{}/*")` would produce.
fn dir_ignore_pattern(dir: &std::path::Path) -> String {
dir.join("*").to_string_lossy().into_owned()
}
/// What the tab asks the app to do after this frame.
#[derive(Default)]
pub struct SearchActions {
@ -51,12 +59,7 @@ pub struct SearchActions {
/// the table with whatever the scan happened to reach first and never show the
/// good ones. Dropping the worst-ranked instead means a rank-1 hit found late
/// in a scan still displaces a rank-10 one found early.
fn admit(
set: &mut Vec<SearchHit>,
incoming: Vec<SearchHit>,
limit: usize,
limited: &mut bool,
) {
fn admit(set: &mut Vec<SearchHit>, incoming: Vec<SearchHit>, limit: usize, limited: &mut bool) {
set.extend(incoming);
if set.len() > limit {
set.sort_by(|a, b| {
@ -644,8 +647,7 @@ impl SearchTab {
name_pattern: hit.name.clone(),
dir_pattern: std::path::Path::new(&hit.path)
.parent()
.and_then(|p| p.to_str())
.map(|p| format!("{}/*", p))
.map(dir_ignore_pattern)
.unwrap_or_default(),
persist: false,
});
@ -713,6 +715,9 @@ impl SearchTab {
}
});
});
// Inside a stable section, or the hint's appearance would
// rename the directory editor below and drop its focus.
crate::ui_util::pattern_hint_label(ui, &dialog.name_pattern);
ui.separator();
// --- Directory ---------------------------------------------
@ -1211,7 +1216,10 @@ mod tests {
tab.sort = (SortKey::Size, false);
tab.sort_dirty = true;
tab.resort();
assert_eq!(displayed(&tab), vec!["zucchini.txt", "mango.txt", "apple.txt"]);
assert_eq!(
displayed(&tab),
vec!["zucchini.txt", "mango.txt", "apple.txt"]
);
}
/// The user may re-key the sort at any time, including while results are
@ -1222,7 +1230,11 @@ mod tests {
let mut tab = streaming_tab();
batch(&mut tab, vec![hit(1, "delta.txt", 1.0, 30)]);
batch(&mut tab, vec![hit(2, "alpha.txt", 5.0, 10)]);
assert_eq!(displayed(&tab), vec!["delta.txt", "alpha.txt"], "rank order");
assert_eq!(
displayed(&tab),
vec!["delta.txt", "alpha.txt"],
"rank order"
);
// Header click, mid-search.
tab.sort = (SortKey::Name, true);
@ -1326,8 +1338,7 @@ mod tests {
batch(&mut tab, vec![hit(2, "better.txt", 1.0, 20)]);
let sel = tab.selected.expect("still selected");
assert_eq!(
tab.results[sel as usize].file_id,
1,
tab.results[sel as usize].file_id, 1,
"selection follows the file, not the slot"
);
}
@ -1351,4 +1362,25 @@ mod tests {
assert_eq!(tab.selected, Some(1));
assert!(egui::Popup::is_any_open(&ctx));
}
/// `Path::join` adds a separator only where one is needed, so the
/// pattern is spelled natively and a drive root does not become the
/// never-matching `C:\/*`.
#[test]
fn dir_ignore_patterns_use_the_platform_separator() {
use std::path::Path;
#[cfg(unix)]
{
assert_eq!(dir_ignore_pattern(Path::new("/home/x")), "/home/x/*");
assert_eq!(dir_ignore_pattern(Path::new("/")), "/*");
}
#[cfg(windows)]
{
assert_eq!(
dir_ignore_pattern(Path::new(r"C:\Users\x")),
r"C:\Users\x\*"
);
assert_eq!(dir_ignore_pattern(Path::new(r"C:\")), r"C:\*");
}
}
}

4
crates/quicksearch-gui/src/tracker.rs
View file

@ -21,7 +21,9 @@ pub struct SpeedTracker {
impl SpeedTracker {
pub fn new() -> SpeedTracker {
SpeedTracker { points: VecDeque::new() }
SpeedTracker {
points: VecDeque::new(),
}
}
/// Reset between phases (each phase restarts its counter).

86
crates/quicksearch-gui/src/ui_util.rs
View file

@ -47,6 +47,44 @@ pub fn ignore_pattern_valid(pattern: &str) -> bool {
!trimmed.is_empty() && IgnoreSet::compile(&[pattern.to_string()]).is_ok()
}
/// An informational note for a pattern that is valid but likely does not
/// mean what was typed, or `None`. Never an error: everything it fires on
/// compiles and matches exactly as described.
///
/// The dot-leading case fires for `.git` too — "matches only items named
/// exactly `.git`" is both true and the intended behavior there, so the
/// note stays factual rather than guessing intent.
pub fn pattern_hint(pattern: &str) -> Option<String> {
let p = pattern.trim();
// ".jpg" is an exact-name pattern, not an extension pattern — the trap
// behind "my ignore filters don't work" reports.
if p.len() >= 2 && p.starts_with('.') && !p.contains(['*', '?', '[', '/', '\\']) {
return Some(format!(
"Matches only files or folders named exactly \"{p}\". \
To ignore all {p} files, use \"*{p}\"."
));
}
// "D:" can only match a component literally named "D:", which nothing
// ever is; the working spelling keeps the separator.
let b = p.as_bytes();
if b.len() == 2 && b[0].is_ascii_alphabetic() && b[1] == b':' {
return Some(format!(
"\"{p}\" never matches anything — use \"{p}\\\" to ignore the whole drive."
));
}
None
}
/// Render [`pattern_hint`] as a small orange label inside a stable section,
/// so its appearance never shifts the ids of widgets below it.
pub fn pattern_hint_label(ui: &mut egui::Ui, pattern: &str) {
stable_section(ui, |ui| {
if let Some(hint) = pattern_hint(pattern) {
ui.label(egui::RichText::new(hint).small().color(ORANGE));
}
});
}
/// Border color for a pattern editor holding `text`, or `None` to keep the
/// theme's own border. A blank box is not wrong yet, just unfilled, so it
/// stays neutral; only text the user actually typed is judged.
@ -137,7 +175,53 @@ pub fn more_below_hint<R>(ui: &egui::Ui, out: &egui::scroll_area::ScrollAreaOutp
#[cfg(test)]
mod tests {
use super::{ignore_pattern_valid, pattern_border, INVALID_RED, VALID_GREEN};
use super::{ignore_pattern_valid, pattern_border, pattern_hint, INVALID_RED, VALID_GREEN};
/// The trap behind "my ignore filters don't work" reports: ".jpg" is an
/// exact-name pattern, and the hint must say so and offer "*.jpg".
#[test]
fn extension_like_patterns_get_a_hint() {
let hint = pattern_hint(".jpg").expect("hint for .jpg");
assert!(hint.contains("*.jpg"), "{}", hint);
assert!(hint.contains("exactly"), "{}", hint);
// Fires for genuine exact-name patterns too — the statement it
// makes is just as true for .git, so no allowlist.
assert!(pattern_hint(".git").is_some());
assert!(pattern_hint(" .venv ").is_some(), "trimmed first");
let targz = pattern_hint(".tar.gz").expect("hint for .tar.gz");
assert!(targz.contains("*.tar.gz"), "{}", targz);
}
#[test]
fn working_patterns_get_no_hint() {
for p in [
"*.jpg", // the fixed spelling itself
"node_modules", // plain name
".hidden*", // wildcard: the author knows about globs
".[jJ]pg", // character class counts as a wildcard
".git/", // separator: a path pattern
r".git\", // …either flavor
".", // too short to be an extension
"",
" ",
"D:/", // working drive-root spelling
r"D:\", // …either flavor
"cache-??",
] {
assert_eq!(pattern_hint(p), None, "hinted on {:?}", p);
}
}
#[test]
fn bare_drive_letters_get_a_hint() {
let hint = pattern_hint("D:").expect("hint for D:");
assert!(hint.contains("D:\\"), "{}", hint);
assert!(pattern_hint("d:").is_some(), "case does not matter");
assert_eq!(pattern_hint("DD:"), None, "not a drive letter");
assert_eq!(pattern_hint("4:"), None, "not a drive letter");
}
#[test]
fn blank_patterns_are_invalid() {

20
crates/quicksearch-gui/src/unlock.rs
View file

@ -57,7 +57,11 @@ impl Gate {
.map(|app| Gate::Running(Box::new(app)))
}
pub fn locked(cfg: Config, config_error: Option<String>, initial_query: Option<String>) -> Gate {
pub fn locked(
cfg: Config,
config_error: Option<String>,
initial_query: Option<String>,
) -> Gate {
Gate::Locked(UnlockScreen::new(cfg, config_error, initial_query))
}
}
@ -148,7 +152,11 @@ pub struct UnlockScreen {
}
impl UnlockScreen {
fn new(cfg: Config, config_error: Option<String>, initial_query: Option<String>) -> UnlockScreen {
fn new(
cfg: Config,
config_error: Option<String>,
initial_query: Option<String>,
) -> UnlockScreen {
let mode = match cfg.security.salt_bytes() {
Err(e) => Mode::BrokenSalt(e),
Ok(_) => {
@ -248,8 +256,8 @@ impl UnlockScreen {
_ => "Create index",
};
let clicked = ui.button(label).clicked();
let entered = field.lost_focus()
&& ui.input(|i| i.key_pressed(egui::Key::Enter));
let entered =
field.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter));
submitted = clicked || entered;
if self.focus_password && !busy {
field.request_focus();
@ -331,9 +339,7 @@ impl UnlockScreen {
drop(password);
db::set_process_key(Some(key.clone()));
let result = match &verify_against {
Some(db_path) => {
db::verify_process_key(&db_path.to_string_lossy()).map(|()| key)
}
Some(db_path) => db::verify_process_key(&db_path.to_string_lossy()).map(|()| key),
None => Ok(key),
};
let _ = tx.send(result);

12
rust-toolchain.toml Normal file
View file

@ -0,0 +1,12 @@
# Pinning the toolchain keeps CI from silently tracking whatever `stable` became
# overnight, and rustup honours this file automatically, so build.sh, build.bat
# and a plain `cargo build` all pick it up with no extra arguments.
#
# The floor is set by eframe/egui 0.32, which declare rust-version = "1.85".
#
# Listing the Windows target here is what lets the cross-compile CI job skip
# `rustup target add`: rustup installs everything named below on first use.
[toolchain]
channel = "1.94.1"
components = ["rustfmt"]
targets = ["x86_64-unknown-linux-gnu", "x86_64-pc-windows-gnu"]