jeremy/quick_search
jeremy/quick_search
1
0
Fork 0
Code Issues Pull requests Projects Releases 15 Packages Wiki Activity Actions

Compare commits

base: jeremy:3772eeb6a5c60e2c1d12591f316421e02e8969ed
Branches Tags
jeremy:master
jeremy:Release-v1.2
jeremy:Release-v1.1
jeremy:Release-v1.0
jeremy:Release-v0.9
jeremy:Release-v0.9.0
jeremy:baloo
jeremy:v1.2.1
jeremy:v1.2.0
jeremy:v1.1.9
jeremy:v1.1.8
jeremy:v1.1.7
jeremy:v1.1.6
jeremy:v1.1.5
jeremy:v1.1.4
jeremy:v1.1.1
jeremy:v1.1.0
jeremy:v1.0.6
jeremy:v1.0.5
jeremy:v1.0.3
jeremy:v1.0.2
jeremy:v1.0.1
jeremy:v0.9.1
...
compare: jeremy:cf301737996e6446206c3439519ca5b230af5477
Branches Tags
jeremy:master
jeremy:Release-v1.2
jeremy:Release-v1.1
jeremy:Release-v1.0
jeremy:Release-v0.9
jeremy:Release-v0.9.0
jeremy:baloo
jeremy:v1.2.1
jeremy:v1.2.0
jeremy:v1.1.9
jeremy:v1.1.8
jeremy:v1.1.7
jeremy:v1.1.6
jeremy:v1.1.5
jeremy:v1.1.4
jeremy:v1.1.1
jeremy:v1.1.0
jeremy:v1.0.6
jeremy:v1.0.5
jeremy:v1.0.3
jeremy:v1.0.2
jeremy:v1.0.1
jeremy:v0.9.1

3 commits
3772eeb6a5 ... cf30173799

Author SHA1 Message Date
=
cf30173799 Indexing and SQLite optimizations during cold indexing.
All checks were successful
CI / linux (push) Successful in 11m39s
Details
CI / windows-cross (push) Successful in 4m41s
Details
CI / release (push) Successful in 13s
Details
2026-08-21 02:05:23 -04:00
=
7854e6828c Performance optimization pass dramatically speeds up fuzzy, regex, and wildcard searches by using trigram prefilters. Reduced memory churn. Added FTX extraction testing for a variety of formats. 2026-08-20 18:58:25 -04:00
=
69a4dbe892 Storage, safety, and correctness hardening 2026-08-20 02:34:08 -04:00
115 changed files with 13874 additions and 1144 deletions
Show stats Expand all files Collapse all files

11
.gitattributes vendored
View file

@ -4,3 +4,14 @@
*.bat text eol=crlf
*.cmd text eol=crlf
*.sh text eol=lf
# The extraction corpus's committed fixtures. `.doc`/`.xls`/`.ppt` are OLE2
# compound files and `.flac` is an audio stream: an EOL filter applied to any
# of them corrupts the file, and `text` detection is a heuristic this does not
# need to rely on. `*.bat` above is `text eol=crlf` crate-wide, which would
# otherwise catch the `.bat` the corpus generates — but that one is written at
# test time and never enters git.
crates/quicksearch-core/tests/fixtures/**/*.doc binary
crates/quicksearch-core/tests/fixtures/**/*.xls binary
crates/quicksearch-core/tests/fixtures/**/*.ppt binary
crates/quicksearch-core/tests/fixtures/**/*.flac binary

478
Cargo.lock generated
View file

@ -86,7 +86,7 @@ dependencies = [
"android-properties",
"bitflags 2.13.1",
"cc",
"jni 0.22.4",
"jni",
"libc",
"log",
"ndk",
@ -463,9 +463,9 @@ dependencies = [
[[package]]
name = "bumpalo"
version = "3.16.0"
version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "bytemuck"
@ -501,9 +501,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
[[package]]
name = "bytes"
version = "1.6.0"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
[[package]]
name = "calloop"
@ -577,12 +577,6 @@ dependencies = [
"shlex",
]
[[package]]
name = "cesu8"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
[[package]]
name = "cfb"
version = "0.7.3"
@ -745,16 +739,6 @@ dependencies = [
"libc",
]
[[package]]
name = "core-foundation"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "core-foundation-sys"
version = "0.8.6"
@ -768,7 +752,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081"
dependencies = [
"bitflags 1.3.2",
"core-foundation 0.9.4",
"core-foundation",
"core-graphics-types",
"foreign-types 0.5.0",
"libc",
@ -781,7 +765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf"
dependencies = [
"bitflags 1.3.2",
"core-foundation 0.9.4",
"core-foundation",
"libc",
]
@ -805,18 +789,18 @@ dependencies = [
[[package]]
name = "crc32fast"
version = "1.4.2"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.13"
version = "0.5.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e"
dependencies = [
"crossbeam-utils",
]
@ -971,6 +955,20 @@ dependencies = [
"litrs",
]
[[package]]
name = "docx-rs"
version = "0.4.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "917eb338de7885a7bf6de3bb294ccbc6d64f0250430e40605d2f2e7e0ee4259a"
dependencies = [
"base64",
"quick-xml",
"serde",
"serde_json",
"thiserror 2.0.19",
"zip 8.6.0",
]
[[package]]
name = "downcast-rs"
version = "1.2.1"
@ -1261,11 +1259,10 @@ dependencies = [
[[package]]
name = "event-listener"
version = "5.3.1"
version = "5.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba"
checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2"
dependencies = [
"concurrent-queue",
"parking",
"pin-project-lite",
]
@ -1338,6 +1335,7 @@ checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
dependencies = [
"crc32fast",
"miniz_oxide",
"zlib-rs",
]
[[package]]
@ -1429,15 +1427,15 @@ dependencies = [
[[package]]
name = "futures-core"
version = "0.3.30"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
[[package]]
name = "futures-io"
version = "0.3.30"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed"
[[package]]
name = "futures-lite"
@ -1454,26 +1452,26 @@ dependencies = [
[[package]]
name = "futures-macro"
version = "0.3.30"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 3.0.3",
]
[[package]]
name = "futures-task"
version = "0.3.30"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
[[package]]
name = "futures-util"
version = "0.3.30"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
dependencies = [
"futures-core",
"futures-io",
@ -1481,7 +1479,6 @@ dependencies = [
"futures-task",
"memchr",
"pin-project-lite",
"pin-utils",
"slab",
]
@ -1764,21 +1761,23 @@ dependencies = [
[[package]]
name = "icu_collections"
version = "1.5.0"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513"
dependencies = [
"displaydoc",
"potential_utf",
"utf8_iter",
"yoke",
"zerofrom",
"zerovec",
]
[[package]]
name = "icu_locid"
version = "1.5.0"
name = "icu_locale_core"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb"
dependencies = [
"displaydoc",
"litemap",
@ -1787,109 +1786,92 @@ dependencies = [
"zerovec",
]
[[package]]
name = "icu_locid_transform"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
dependencies = [
"displaydoc",
"icu_locid",
"icu_locid_transform_data",
"icu_provider",
"tinystr",
"zerovec",
]
[[package]]
name = "icu_locid_transform_data"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
[[package]]
name = "icu_normalizer"
version = "1.5.0"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f"
dependencies = [
"displaydoc",
"icu_collections",
"icu_normalizer_data",
"icu_properties",
"icu_provider",
"smallvec",
"utf16_iter",
"utf8_iter",
"write16",
"zerovec",
]
[[package]]
name = "icu_normalizer_data"
version = "1.5.0"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0"
[[package]]
name = "icu_properties"
version = "1.5.0"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036"
checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148"
dependencies = [
"displaydoc",
"icu_collections",
"icu_locid_transform",
"icu_locale_core",
"icu_properties_data",
"icu_provider",
"tinystr",
"zerotrie",
"zerovec",
]
[[package]]
name = "icu_properties_data"
version = "1.5.0"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"
[[package]]
name = "icu_provider"
version = "1.5.0"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
checksum = "92a7ed671a6aad807a8651a2e1782a6598fda9ce5185dd8158549e95a91c6428"
dependencies = [
"displaydoc",
"icu_locid",
"icu_provider_macros",
"stable_deref_trait",
"tinystr",
"icu_locale_core",
"writeable",
"yoke",
"zerofrom",
"zerotrie",
"zerovec",
]
[[package]]
name = "icu_provider_macros"
version = "1.5.0"
name = "id3"
version = "1.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
checksum = "24993fcabcbc07c8ac076a8e62db8593d1a5c4dbe81d57e531d2b7cb7f737380"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"bitflags 2.13.1",
"byteorder",
"flate2",
]
[[package]]
name = "idna"
version = "1.0.0"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed"
checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"
dependencies = [
"idna_adapter",
"smallvec",
"utf8_iter",
]
[[package]]
name = "idna_adapter"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714"
dependencies = [
"icu_normalizer",
"icu_properties",
"smallvec",
"utf8_iter",
]
[[package]]
@ -1980,22 +1962,6 @@ version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
[[package]]
name = "jni"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97"
dependencies = [
"cesu8",
"cfg-if",
"combine",
"jni-sys 0.3.0",
"log",
"thiserror 1.0.61",
"walkdir",
"windows-sys 0.45.0",
]
[[package]]
name = "jni"
version = "0.22.4"
@ -2191,9 +2157,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]]
name = "litemap"
version = "0.7.3"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704"
checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
[[package]]
name = "litrs"
@ -2304,6 +2270,16 @@ dependencies = [
"autocfg",
]
[[package]]
name = "metaflac"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdf25a3451319c52a4a56d956475fbbb763bfb8420e2187d802485cb0fd8d965"
dependencies = [
"byteorder",
"hex",
]
[[package]]
name = "mime"
version = "0.3.17"
@ -2932,6 +2908,18 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "pdf-writer"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5e456864a7a304047bff84977dc6fb162bd956475d40ba50b2dcecaada7f753"
dependencies = [
"bitflags 2.13.1",
"itoa",
"memchr",
"ryu",
]
[[package]]
name = "percent-encoding"
version = "2.3.1"
@ -2964,12 +2952,6 @@ version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
[[package]]
name = "pin-utils"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "piper"
version = "0.2.3"
@ -3045,6 +3027,15 @@ version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306"
[[package]]
name = "potential_utf"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"
dependencies = [
"zerovec",
]
[[package]]
name = "ppv-lite86"
version = "0.2.17"
@ -3087,60 +3078,59 @@ version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
[[package]]
name = "quick-xml"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
dependencies = [
"memchr",
]
[[package]]
name = "quick-xml"
version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
dependencies = [
"encoding_rs",
"memchr",
]
[[package]]
name = "quicksearch-core"
version = "1.1.0"
version = "1.1.3"
dependencies = [
"argon2",
"cfb",
"chardetng",
"crc32fast",
"ctrlc",
"divan",
"docx-rs",
"encoding_rs",
"getrandom 0.2.15",
"globset",
"id3",
"infer",
"libc",
"lofty",
"memchr",
"metaflac",
"mime_guess",
"notify",
"pdf-extract",
"quick-xml 0.31.0",
"pdf-writer",
"quick-xml",
"regex",
"regex-syntax",
"rtf-parser",
"rusqlite",
"rust_xlsxwriter",
"serde",
"sha2",
"toml",
"walkdir",
"windows-sys 0.52.0",
"zeroize",
"zip",
"zip 0.6.6",
"zstd",
]
[[package]]
name = "quicksearch-gui"
version = "1.1.0"
version = "1.1.3"
dependencies = [
"ashpd",
"chrono",
@ -3164,9 +3154,9 @@ dependencies = [
[[package]]
name = "quote"
version = "1.0.36"
version = "1.0.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
dependencies = [
"proc-macro2",
]
@ -3376,8 +3366,6 @@ dependencies = [
[[package]]
name = "rtf-parser"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c3147b4eb521eae5e29b781bdc30ab98bbaed784bfcb8376cda60ba4b2e0e3d"
dependencies = [
"serde",
]
@ -3407,6 +3395,15 @@ dependencies = [
"sqlite-wasm-rs",
]
[[package]]
name = "rust_xlsxwriter"
version = "0.98.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "093d730a6f64f9620e047b2de2876e61a691fbcaca3aaf436edade9b897c7a27"
dependencies = [
"zip 8.6.0",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
@ -3460,6 +3457,12 @@ version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
[[package]]
name = "ryu"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "same-file"
version = "1.0.6"
@ -3517,6 +3520,19 @@ dependencies = [
"syn 3.0.3",
]
[[package]]
name = "serde_json"
version = "1.0.151"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
dependencies = [
"itoa",
"memchr",
"serde",
"serde_core",
"zmij",
]
[[package]]
name = "serde_repr"
version = "0.1.19"
@ -3864,9 +3880,9 @@ dependencies = [
[[package]]
name = "tinystr"
version = "0.7.6"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643"
dependencies = [
"displaydoc",
"zerovec",
@ -3994,6 +4010,12 @@ dependencies = [
"pom",
]
[[package]]
name = "typed-path"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e"
[[package]]
name = "typenum"
version = "1.17.0"
@ -4058,9 +4080,9 @@ checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
[[package]]
name = "url"
version = "2.5.1"
version = "2.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56"
checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
dependencies = [
"form_urlencoded",
"idna",
@ -4074,12 +4096,6 @@ version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
[[package]]
name = "utf16_iter"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
[[package]]
name = "utf8_iter"
version = "1.0.4"
@ -4305,7 +4321,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "338e30461b3a2b67d70eb30a6d89f8e0c93a833e07d2ae89085cd070c4a00ac0"
dependencies = [
"proc-macro2",
"quick-xml 0.41.0",
"quick-xml",
"quote",
]
@ -4343,15 +4359,15 @@ dependencies = [
[[package]]
name = "webbrowser"
version = "1.0.6"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00f1243ef785213e3a32fa0396093424a3a6ea566f9948497e5a2309261a4c97"
checksum = "ef62a3d5f7b2411119a11b6f62570dbff91d7105e011a20fb83fbf8f5761c40f"
dependencies = [
"core-foundation 0.10.1",
"jni 0.21.1",
"jni",
"log",
"ndk-context",
"objc2 0.6.4",
"objc2-app-kit 0.3.2",
"objc2-foundation 0.3.2",
"url",
"web-sys",
@ -4550,15 +4566,6 @@ dependencies = [
"windows-link",
]
[[package]]
name = "windows-sys"
version = "0.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
dependencies = [
"windows-targets 0.42.2",
]
[[package]]
name = "windows-sys"
version = "0.48.0"
@ -4604,21 +4611,6 @@ dependencies = [
"windows-link",
]
[[package]]
name = "windows-targets"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
dependencies = [
"windows_aarch64_gnullvm 0.42.2",
"windows_aarch64_msvc 0.42.2",
"windows_i686_gnu 0.42.2",
"windows_i686_msvc 0.42.2",
"windows_x86_64_gnu 0.42.2",
"windows_x86_64_gnullvm 0.42.2",
"windows_x86_64_msvc 0.42.2",
]
[[package]]
name = "windows-targets"
version = "0.48.5"
@ -4667,12 +4659,6 @@ dependencies = [
"windows_x86_64_msvc 0.53.1",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.48.5"
@ -4691,12 +4677,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
[[package]]
name = "windows_aarch64_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
[[package]]
name = "windows_aarch64_msvc"
version = "0.48.5"
@ -4715,12 +4695,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
[[package]]
name = "windows_i686_gnu"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
[[package]]
name = "windows_i686_gnu"
version = "0.48.5"
@ -4751,12 +4725,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
[[package]]
name = "windows_i686_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
[[package]]
name = "windows_i686_msvc"
version = "0.48.5"
@ -4775,12 +4743,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
[[package]]
name = "windows_x86_64_gnu"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
[[package]]
name = "windows_x86_64_gnu"
version = "0.48.5"
@ -4799,12 +4761,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.48.5"
@ -4823,12 +4779,6 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
[[package]]
name = "windows_x86_64_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
[[package]]
name = "windows_x86_64_msvc"
version = "0.48.5"
@ -4862,7 +4812,7 @@ dependencies = [
"calloop 0.13.0",
"cfg_aliases",
"concurrent-queue",
"core-foundation 0.9.4",
"core-foundation",
"core-graphics",
"cursor-icon",
"dpi",
@ -4922,17 +4872,11 @@ version = "0.57.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
[[package]]
name = "write16"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
[[package]]
name = "writeable"
version = "0.5.5"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"
[[package]]
name = "x11-dl"
@ -5009,11 +4953,10 @@ checksum = "3ae8337f8a065cfc972643663ea4279e04e7256de865aa66fe25cec5fb912d3f"
[[package]]
name = "yoke"
version = "0.7.4"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5"
checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
dependencies = [
"serde",
"stable_deref_trait",
"yoke-derive",
"zerofrom",
@ -5021,9 +4964,9 @@ dependencies = [
[[package]]
name = "yoke-derive"
version = "0.7.4"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95"
checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
dependencies = [
"proc-macro2",
"quote",
@ -5115,18 +5058,18 @@ dependencies = [
[[package]]
name = "zerofrom"
version = "0.1.4"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55"
checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"
dependencies = [
"zerofrom-derive",
]
[[package]]
name = "zerofrom-derive"
version = "0.1.4"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5"
checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
dependencies = [
"proc-macro2",
"quote",
@ -5155,10 +5098,21 @@ dependencies = [
]
[[package]]
name = "zerovec"
version = "0.10.2"
name = "zerotrie"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c"
checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f"
dependencies = [
"displaydoc",
"yoke",
"zerofrom",
]
[[package]]
name = "zerovec"
version = "0.11.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8"
dependencies = [
"yoke",
"zerofrom",
@ -5167,13 +5121,13 @@ dependencies = [
[[package]]
name = "zerovec-derive"
version = "0.10.2"
version = "0.11.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7"
checksum = "9f212a141d820099d57ffafb9569be9617a6f27d3dc881fbee8fb56642f917a9"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.66",
"syn 3.0.3",
]
[[package]]
@ -5188,6 +5142,44 @@ dependencies = [
"flate2",
]
[[package]]
name = "zip"
version = "8.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
dependencies = [
"crc32fast",
"flate2",
"indexmap",
"memchr",
"typed-path",
"zopfli",
]
[[package]]
name = "zlib-rs"
version = "0.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12"
[[package]]
name = "zmij"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
[[package]]
name = "zopfli"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
dependencies = [
"bumpalo",
"crc32fast",
"log",
"simd-adler32",
]
[[package]]
name = "zstd"
version = "0.13.3"

29
Cargo.toml
View file

@ -4,34 +4,15 @@ members = [
"crates/quicksearch-core",
"crates/quicksearch-gui",
]
# `vendor/pdf-extract` is deliberately NOT a member: it is a third-party crate
# carried here for one patch, not part of this workspace's lints, tests or
# release profile. `[patch.crates-io]` below is what makes the dependency graph
# resolve to it.
exclude = ["vendor/pdf-extract"]
# The `vendor/` crates are deliberately NOT members: they are third-party code which we patched
exclude = ["vendor/pdf-extract", "vendor/rtf-parser"]
# pdf-extract 0.12.0, with the two unbounded recursions in it bounded.
#
# `get_inherited` follows `/Parent` and `process_stream`'s `Do` arm follows
# Form XObjects, neither with a depth counter or a visited set. A page whose
# `/Parent` is itself, or an XObject whose content stream draws itself, walks
# the stack until it hits the guard page — and a stack overflow is not a panic
# that `catch_unwind` can contain (`extract/pdf.rs` has one, for the parser's
# ordinary panics): Rust's handler calls `abort()`, so a ~600-byte file kills
# the process. It recurs on every run, because the row keeps
# `content_state = 0` and the feeder selects exactly those; and the live
# watcher re-extracts on-screen rows on the GUI thread, so such a file crashes
# the app when it merely appears in a result list.
#
# Vendored rather than forked-by-URL so the build stays offline, `--locked`
# keeps meaning what it means, and the cross-compile job needs no new host.
# The patch is marked LOCAL PATCH in the source and is upstreamable; the crate
# is MIT and the copy is recorded in `packaging/copyright`.
[patch.crates-io]
pdf-extract = { path = "vendor/pdf-extract" }
pdf-extract = { path = "vendor/pdf-extract" } # Patched unbounded reads which could blow up on malformed files
rtf-parser = { path = "vendor/rtf-parser" } # Patched a parsing error which occurs on UTF-16 characters
[workspace.package]
version = "1.1.1"
version = "1.1.4"
edition = "2021"
license = "GPL-3.0-or-later"
authors = ["Jeremy <jeremy@karsttech.com>"]

60
README.md
View file

@ -223,7 +223,20 @@ inside that folder.
### GUI
`quicksearch` with no query arguments opens the app:
`quicksearch` with no query arguments opens the app. **One window at a
time**: a second launch reports that QuickSearch is already running and
exits, because two processes indexing one database corrupt it. Terminal
search (`quicksearch <query>`) only reads and keeps working while the
window is open. The guard is a kernel lock on `<database_path>.lock`, held
by the running process — so a crash or a power cut releases it, and the
leftover file never locks you out. The lock follows `database_path`: point
Settings at a different index and it moves with you, and if that index
belongs to another instance the change is refused rather than written, so a
path you cannot open can never end up in the config file. A path naming
some *other* program's SQLite database is refused for the same reason: the
file there is created when missing and replaced when it is an index from an
older layout of QuickSearch's own, but anything else is left alone and the
error names the tables that identified it.
- **Search**: results appear as you type; every keystroke cancels the
previous search. One checkbox enables the two fuzzy passes, and once a
@ -448,6 +461,12 @@ combination, plus one key, joined with `+`. An empty string switches it
off. A value that is not a shortcut is not a config error — the app loads,
says so on the Settings tab, and runs without one.
Numbers behave the same way. A hand-edited value outside the range a
setting can work in — `display_limit = 0`, which would make every search
return nothing — is clamped to the nearest workable one with a warning,
never rejected: a typo in a text file must not stop the app starting. The
ranges are in `config_example.toml` beside each setting that has one.
`[ui] color_scheme` is `dark` (the default) or `light`, changeable on the
Settings tab and applied without a restart. It does not follow the
desktop's own light/dark setting: on Linux nothing in the window system
@ -491,7 +510,14 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
`wal_checkpoint(TRUNCATE)` every `processing.maximum_wal_size` bytes of
log, because SQLite's own autocheckpoint can only reset the log at an
instant no reader holds it — and a run keeps a reader per root querying
throughout, so left alone the log grows for the whole run. `files` holds
throughout, so left alone the log grows for the whole run. That threshold
is lowered to fit the volume when free space is short, and a run stops
with an error rather than fill the disk: SQLite reaches the wal-index
through an mmap, and a page fault the filesystem cannot back arrives as
**SIGBUS**, which no `Result` can catch. The index's own files are, for
the same reason, never walked into — hashing one means opening it, and on
POSIX closing any descriptor on an inode cancels every advisory lock the
process holds on it, SQLite's documented corruption hazard. `files` holds
metadata (name, path, size, mtime, hash, MIME/type bitmask, content
state); `searchabletext` is a *contentless* FTS5 table over one column,
the document body (postings only, configurable tokenizer, trigram by
@ -572,7 +598,7 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
tokenizing can do; the scheduling shares it fairly and keeps the walk
first, it does not raise it. Every run ends — whether
it completed or was stopped — with an optimize pass on its own connection:
checkpoint, VACUUM if the file has at least 10% slack to reclaim, `PRAGMA
checkpoint, VACUUM if the file has at least 20% slack to reclaim, `PRAGMA
optimize`, checkpoint again. Progress streams through a polled
`IndexingStatus`, which reads `Optimizing` for the duration of that pass —
and `Preparing` for everything a run does before its first file is walked:
@ -674,13 +700,15 @@ Synchronous Rust: `std::thread` + `mpsc` channels, no async runtime.
to that by construction, because a buffer allocated and freed inside one loop
iteration never moves RSS. `DocDecoder` must therefore never fall back to a
per-row allocating decode, and there is a trap waiting there:
`zstd::encode_all`, which the indexer writes with, is *stream*-based and so
records no content size in the frame header, meaning
`get_frame_content_size` returns `None` for every row this ever sees. The
"cannot happen" branch is the only branch. Sizing the buffer from the header
and handing the `None` case to `zstd::decode_all` looks obviously right and
costs ~2.4 MiB per document, because `decode_all` builds a streaming decoder
per call — 27 of the 30 GiB a fuzzy search moved. Growing this buffer and
the indexer writes through `DocEncoder`, i.e. `zstd::bulk::Compressor` and
so `ZSTD_compress2`, which is handed the whole document at once and records
its length in the frame header. `get_frame_content_size` therefore returns
`Some` for every row this writer produced, the reservation is normally
exact, and the growth loop below it is the fallback for a frame written by
a *stream* encoder rather than the only branch. What must not come back is
handing the `None` case to `zstd::decode_all`, which looks obviously right
and costs ~2.4 MiB per document because it builds a streaming decoder per
call — 27 of the 30 GiB a fuzzy search moved. Growing this buffer and
keeping it is what makes decoding a row allocate nothing at all.
Measured over the same 77k-file index, per query:
`cascade` 582 → 14 MiB, `function` 6.0 GiB → 29 MiB, `--fuzzy cascade`
@ -867,8 +895,16 @@ microseconds regardless of row count.
program, so the installer needs no Windows runner either.
- New extractors: implement `extract::Extractor` and register it in
`Registry::default_set()` — order matters, the first extractor whose
`supports` accepts a MIME wins. New cascade behavior: `search/cascade.rs`
documents the rank invariants that keep streamed results append-only.
`supports` accepts a MIME wins. Then add the format to the corpus in
`crates/quicksearch-core/tests/corpus/`, which writes a randomized lipsum
document in every supported format and asserts the planted text comes back —
at the extractor, through the walk-time head path, and end to end through
indexing and search (`tests/extraction_corpus.rs`). Its rule is that no
corpus file may be written by the library that reads it back, since a writer
and reader sharing a wrong assumption agree with each other; the module docs
say which library writes what and why. New cascade behavior:
`search/cascade.rs` documents the rank invariants that keep streamed results
append-only.
- `packaging/capture.sh`: regenerates the website assets — `search.webm`,
`manage-indexing.webm`, `duplicates.png`, `query-highlight.png` — into
`packaging/captures/` (gitignored). It builds the GUI with the `capture`

32
config_example.toml
View file

@ -21,6 +21,11 @@ indexing_paths = ["~"]
# backslashes need no escaping, and keep the index out of a roaming
# profile — it is far too large to synchronise:
# database_path = 'C:\Users\you\AppData\Local\quicksearch\index.sqlite'
#
# The file here is created if it is missing, and replaced if it is an index
# from an older layout of QuickSearch's own. A SQLite database belonging to
# some other program is neither: pointing this at one is refused with an
# error naming the tables it found, and nothing is deleted.
database_path = "~/.local/share/quicksearch/index.sqlite"
[indexing]
@ -74,6 +79,11 @@ content_extensions = []
# The Windows defaults add: "$RECYCLE.BIN", "System Volume Information",
# "pagefile.sys", "hiberfil.sys", "swapfile.sys", "Thumbs.db",
# "desktop.ini".
#
# The index itself (database_path and its -wal/-shm/-journal/.lock
# siblings) is always skipped and needs no pattern here — not as a
# convenience but because merely opening one of those files to hash it
# would cancel SQLite's locks on it for the whole process.
ignore_patterns = [".git", "node_modules", "*.tmp", ".venv", "venv"]
# Worth adding by hand if you index a whole Windows drive rather than just
@ -102,10 +112,15 @@ ignore_patterns = [".git", "node_modules", "*.tmp", ".venv", "venv"]
# images: a fixed-size VHD stores its unique footer at the end of the
# file, and a freshly pre-allocated raw/qcow2/VMDK image is all zeros at
# the head until it is partitioned.
#
# Clamped to 262..1048576 on load: below 262 the longest magic-number
# matcher cannot run, so file types stop being detectable by content, and
# above a megabyte the per-file head buffer stops being a head.
hash_length = 8192
# Maximum extracted text stored per file (bytes).
maximum_text_size = 262144
# Files larger than this skip text extraction entirely (bytes).
# Files larger than this skip text extraction entirely (bytes). Clamped to
# 1..4294967296 on load.
maximum_text_file_size = 2097152
# Files per batch during walks / inserts / extraction.
batch_size = 500
@ -113,7 +128,9 @@ batch_size = 500
# on (milliseconds). The time half of the knob whose row half is batch_size:
# it bounds how long one root can hold up the others, so a root extracting
# large documents cannot leave another root's walkers parked behind it. 0
# gives each turn one batch_size quantum and no more.
# gives each turn one batch_size quantum and no more. Clamped to 0..10000
# on load: unbounded, one root holds the writer for as long as it likes and
# every other root's walk waits behind it.
writer_turn_slice_ms = 100
# Files per transaction for incremental FTS updates.
fts_update_batch_size = 1000
@ -124,6 +141,13 @@ fts_update_batch_size = 1000
# unattended the log grows for the whole run and can end up larger than the
# index. Set to 0 to disable forced checkpoints; any other value below
# 16777216 is raised to it.
#
# This is a stall-frequency knob, not a safety one: when the volume holding
# the index is short on space the indexer checkpoints sooner than asked —
# including when this is 0 — and stops the run with an error rather than
# fill the disk. Running out is not a clean failure; SQLite reaches its
# wal-index through a memory map, and a write the filesystem cannot back
# kills the process with SIGBUS instead of returning an error.
maximum_wal_size = 536870912
# FTS5 tokenizer: 'trigram' (substring matching, the default; gets
# remove_diacritics 1 appended), 'unicode61', 'porter', or a full FTS5
@ -191,7 +215,9 @@ fuzzy_default = false
# the fuzzy stages off. Above 3 is allowed but not recommended: matches
# become dominated by coincidence and every fuzzy pass slows down.
fuzzy_max_edits = 2
# Hard cap on results per search (the GUI's scroll list length).
# Hard cap on results per search (the GUI's scroll list length). Clamped to
# 1..1000000 on load: at zero the cascade stops before its first pass, so
# every search returns nothing and calls the empty answer truncated.
display_limit = 1000
# Results per streamed batch (latency/overhead knob, not a page size).
results_per_page = 100

64
crates/quicksearch-core/Cargo.toml
View file

@ -28,7 +28,7 @@ walkdir = "2.5.0"
# containers (docx/xlsx/pptx/odt/ods/odp), whose entries are always deflate or
# stored, so zstd/bzip2/aes-crypto are all dead weight here.
zip = { version = "0.6", default-features = false, features = ["deflate"] }
quick-xml = "0.31"
quick-xml = "0.41"
# OLE2 compound-file reader, for the pre-2007 binary Office formats
# (.doc/.xls/.ppt) whose text lives in named streams rather than a zip. Already
# in the lockfile transitively via infer, so naming it directly compiles
@ -51,8 +51,19 @@ encoding_rs = "0.8"
# UTF-8 nor BOM-marked. Its mandatory deps beyond encoding_rs are tiny
# (cfg-if, memchr, detone).
chardetng = "1.0"
# RTF text extraction. Pure Rust; with the default `jsbindings` feature off
# (it exists for the crate's WASM build) it depends only on serde.
# RTF text extraction. Pure Rust; depends only on serde.
#
# Resolves to `vendor/rtf-parser` through the workspace's `[patch.crates-io]`,
# which is also where the reasons are written down: the stock lexer ended a
# control word at whitespace and nowhere else, which silently dropped indexed
# text from two shapes of document that LibreOffice and Word produce, and the
# parser panicked on a malformed `\uN` escape. The version here still names
# 0.4 because that is what the patch is against.
#
# `default-features = false` is redundant against the vendored copy, which has
# no features left to disable — the `jsbindings` one existed for the crate's
# WebAssembly build and was deleted. It stays so that dropping the patch falls
# back to a registry build that does not pull wasm-bindgen.
rtf-parser = { version = "0.4", default-features = false }
# `lopdf` is deliberately NOT declared here. `pdf-extract` re-exports it
# (`pub use lopdf::*`), and `extract/pdf.rs` reaches it that way. Naming it
@ -68,6 +79,11 @@ ctrlc = "3.4"
zstd = "0.13"
globset = "0.4"
regex = "1"
# Required-literal extraction for the `regex:` prefilter: the same analysis the
# regex engine does to pick its own prefilter, reused to narrow which rows the
# regex passes ever see. Already in the lockfile transitively (via `regex`), so
# naming it directly compiles nothing new.
regex-syntax = "0.8"
# SIMD substring search for the full-text passes. `str::match_indices` uses
# std's Two-Way searcher, which has no vector prefilter: measured against a
# 256 KiB body (`benches/search.rs`, group `substring`) it runs 111 µs where
@ -101,6 +117,10 @@ windows-sys = { version = "0.52", features = [
# the directory-count path passes null for it.
"Win32_Security",
"Win32_Storage_FileSystem",
# OVERLAPPED, which `LockFileEx` takes for the index's instance lock.
# `LockFileEx` and `GetDiskFreeSpaceExW` themselves are in
# Win32_Storage_FileSystem above.
"Win32_System_IO",
"Win32_System_Threading",
"Win32_System_WindowsProgramming",
] }
@ -113,6 +133,44 @@ windows-sys = { version = "0.52", features = [
[dev-dependencies]
divan = "0.1"
# Writers for `tests/extraction_corpus.rs`. Every one of these is chosen to be
# a *different implementation* from the reader it feeds: a fixture built with
# the same library that parses it can only prove the two agree with each other,
# not that either agrees with the format. The unit tests in `src/extract/` do
# use the readers' own libraries, deliberately — they aim at malformed input,
# where the point is to control every byte. This set aims at well-formed input
# from a foreign producer, which is the other half.
#
# None of these reach the release binary: dev-dependencies are linked into test
# and bench targets only.
#
# `.doc`/`.xls`/`.ppt` are absent because nothing in Rust writes OLE2 compound
# files but `cfb`, which is the reader. Those three are committed fixtures from
# LibreOffice instead; see `tests/fixtures/legacy/`.
#
# docx: writes its own OOXML, and carries `zip` 8.x — a different major from
# the 0.6 our reader uses, so even the container bytes come from elsewhere.
# The default `image` feature pulls the whole `image` crate for embedding
# pictures, which nothing here does.
docx-rs = { version = "0.4", default-features = false }
# xlsx: builds a real shared-string table, which is the path `extract_xlsx`
# actually cares about.
rust_xlsxwriter = { version = "0.98", default-features = false }
# pdf: typst's low-level writer. No `lopdf` anywhere in its tree, which is what
# makes it independent of `pdf-extract`.
pdf-writer = "0.15"
# audio tags: `id3` writes the ID3v2 frames `lofty` reads back, `metaflac` the
# Vorbis comment block. The MPEG frames under the first are hand-rolled; the
# FLAC stream under the second is committed, because `lofty` reads a real audio
# frame to derive the stream's properties and a FLAC frame is CRC-checked
# bit-packed data rather than a fixed header. See `tests/corpus/audio.rs`.
id3 = "1"
metaflac = "0.2"
# CRC32 for the hand-rolled stored-entry zip that builds the pptx and ODF
# containers. Already in the lockfile transitively via `zip`, so naming it
# here compiles nothing new.
crc32fast = "1"
# `harness = false` on both: divan supplies its own `main` via `divan::main()`,
# so libtest must not also link one in.
[[bench]]

121
crates/quicksearch-core/benches/search.rs
View file

@ -163,6 +163,37 @@ mod substring {
bencher.bench(|| finder.find_iter(divan::black_box(text)).count());
}
/// A `Finder` built once per query against one built per call.
///
/// **A losing arm, kept as the record.** `memmem::find_iter(hay, needle)`
/// constructs a searcher every time, and the full-text pass calls it once
/// or twice per candidate row, so hoisting that into the compiled pattern
/// looks like free money. It is not: medians of 35.5 ns against 35.2 at
/// 1 KiB and 3.18 µs against 3.17 at 256 KiB — indistinguishable at every
/// size, including the smallest, where setup would dominate if it were
/// going to.
///
/// The reason is that the precompute is O(needle), and a search term is a
/// handful of bytes. What *did* cost something on this path was the
/// `to_ascii_lowercase` rebuilding the needle per row, and that is an
/// allocation rather than a searcher — see `snippet::extract_folded`, which
/// borrows an already-folded term instead.
#[divan::bench(args = corpus::SIZES)]
fn memmem_per_call_miss(bencher: Bencher, size: usize) {
let text = corpus::text(size, 0).as_bytes();
bencher.bench(|| {
memchr::memmem::find_iter(divan::black_box(text), corpus::NEEDLE.as_bytes()).count()
});
}
#[divan::bench(args = corpus::SIZES)]
fn memmem_per_call_hits(bencher: Bencher, size: usize) {
let text = corpus::text(size, 64).as_bytes();
bencher.bench(|| {
memchr::memmem::find_iter(divan::black_box(text), corpus::NEEDLE.as_bytes()).count()
});
}
#[divan::bench(args = corpus::SIZES)]
fn match_indices_hits(bencher: Bencher, size: usize) {
let text = corpus::text(size, 64);
@ -176,11 +207,32 @@ mod substring {
bencher.bench(|| finder.find_iter(divan::black_box(text)).count());
}
/// What `pass_fulltext` actually runs per row, through the real crate
/// entry points: a case-sensitive count, then a folded count, then the
/// snippet extraction. Three sweeps of the same document.
/// What `pass_fulltext` runs per row on the literal path, through the
/// real crate entry points: a case-sensitive count, then one folded
/// extraction that yields the count and the snippet together.
#[divan::bench(args = corpus::SIZES)]
fn cascade_row_sweeps(bencher: Bencher, size: usize) {
let pattern = literal(corpus::NEEDLE);
let text = corpus::text_mixed(size, 4);
let folded = corpus::text_folded(size, 4);
let opts = snippet::Options { approx_chars: 600 };
bencher.bench(|| {
let (s, b) = snippet::extract_folded(
divan::black_box(text),
divan::black_box(folded),
&[corpus::NEEDLE],
&opts,
);
let a = pattern.count(text, false);
(a, b, s.ranges.len())
});
}
/// The shape it replaced, kept as the comparison: counting the folded
/// haystack separately from extracting the window sweeps the same
/// document a third time for a number the extraction already knew.
#[divan::bench(args = corpus::SIZES)]
fn cascade_row_sweeps_separate_count(bencher: Bencher, size: usize) {
let pattern = literal(corpus::NEEDLE);
let text = corpus::text_mixed(size, 4);
let folded = corpus::text_folded(size, 4);
@ -188,15 +240,14 @@ mod substring {
bencher.bench(|| {
let a = pattern.count(divan::black_box(text), false);
let b = pattern.count_folded(divan::black_box(folded));
let s = snippet::extract_folded(text, folded, &[corpus::NEEDLE], &opts);
let (s, _) = snippet::extract_folded(text, folded, &[corpus::NEEDLE], &opts);
(a, b, s.ranges.len())
});
}
}
/// Snippet extraction against a pre-folded haystack, the third of those
/// sweeps. Also carries a per-call `term.to_ascii_lowercase()` at
/// `snippet.rs:81` for a needle the caller already holds folded.
/// Snippet extraction against a pre-folded haystack — on the literal path,
/// now the *only* folded sweep of a row, and the source of its count.
mod snippet_extract {
use super::*;
@ -325,6 +376,62 @@ mod filename_ladder {
found
});
}
/// `find_ascii_ci`'s scalar candidate loop against a `memchr2` one.
///
/// **A losing arm, kept as the record.** The production function walks the
/// haystack a byte at a time comparing `to_ascii_lowercase()`; `memchr2`
/// finds the next byte matching either case of the needle's first byte with
/// SIMD and only then compares. That looks like it must win, and it does
/// not: 47.9 µs against 49.0 µs median, inside the run-to-run spread.
///
/// Two reasons, both about *short* haystacks. `memchr2` has per-call setup
/// to amortize and a filename is tens of bytes, not a document; and the
/// scalar loop's inner comparison almost never fires, because a first byte
/// that occurs rarely in the corpus makes the loop a plain byte scan the
/// compiler already vectorizes.
///
/// Fold-free and allocation-free either way, so this isolates the search
/// itself — unlike the arms above, which conflate it with a fold. If a
/// future change makes this pass run over many more rows, re-measure; as it
/// stands the SQL `LIKE` prefilter means the classifier barely runs at all
/// for literal terms, so this was never where the time was.
#[divan::bench]
fn find_first_ci_memchr2(bencher: Bencher) {
let rows = corpus::rows();
let needle = corpus::NEEDLE.as_bytes();
// The same shape `TermPattern::find_ascii_ci` would take.
fn find(hay: &[u8], needle: &[u8]) -> Option<usize> {
let (lo, up) = (
needle[0].to_ascii_lowercase(),
needle[0].to_ascii_uppercase(),
);
let last = hay.len().checked_sub(needle.len())?;
let mut at = 0usize;
while at <= last {
let Some(off) = memchr::memchr2(lo, up, &hay[at..=last]) else {
return None;
};
let i = at + off;
if hay[i..i + needle.len()].eq_ignore_ascii_case(needle) {
return Some(i);
}
at = i + 1;
}
None
}
bencher.bench(|| {
let mut found = 0usize;
for row in divan::black_box(rows) {
if find(row.name.as_bytes(), needle).is_some()
|| find(row.path.as_bytes(), needle).is_some()
{
found += 1;
}
}
found
});
}
}
/// Bitap, the fuzzy passes' inner loop. Both fuzzy passes are whole-table

351
crates/quicksearch-core/examples/indexprobe.rs
View file

@ -34,7 +34,98 @@
//! progress walk, no size survey — so that every syscall the trace attributes
//! to the tree came from the indexer. The size histogram is printed by `gen`.
use std::alloc::{GlobalAlloc, Layout, System};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
// ---------------------------------------------------------------------------
// Allocation accounting
// ---------------------------------------------------------------------------
/// `System`, counting. A global allocator is **per binary**, so this affects
/// only this probe — the shipped `quicksearch` is untouched.
///
/// Global atomics rather than the per-thread `Cell`s `tests/search_alloc.rs`
/// uses, and for the opposite reason. There the work was synchronous on one
/// thread and other *tests* ran concurrently, so per-thread counting was both
/// necessary and more precise. Here the work is spread over a walk pool, an
/// extraction pool, a feeder and a writer — per-thread counting would report a
/// fraction of it — and nothing else is running in this process, so a global
/// count is exactly the run.
///
/// The atomics cost every allocation a contended RMW, which is real overhead
/// and shows in the wall-clock line. That is acceptable because both sides of a
/// before/after comparison carry the same instrumentation; it is not acceptable
/// to quote these timings against numbers from an uninstrumented build.
struct Counting;
static ALLOCS: AtomicU64 = AtomicU64::new(0);
static ALLOC_BYTES: AtomicU64 = AtomicU64::new(0);
static LIVE: AtomicU64 = AtomicU64::new(0);
static PEAK_LIVE: AtomicU64 = AtomicU64::new(0);
#[inline]
fn note_alloc(size: usize) {
ALLOCS.fetch_add(1, Ordering::Relaxed);
ALLOC_BYTES.fetch_add(size as u64, Ordering::Relaxed);
let live = LIVE.fetch_add(size as u64, Ordering::Relaxed) + size as u64;
PEAK_LIVE.fetch_max(live, Ordering::Relaxed);
}
unsafe impl GlobalAlloc for Counting {
unsafe fn alloc(&self, l: Layout) -> *mut u8 {
let p = unsafe { System.alloc(l) };
if !p.is_null() {
note_alloc(l.size());
}
p
}
unsafe fn alloc_zeroed(&self, l: Layout) -> *mut u8 {
let p = unsafe { System.alloc_zeroed(l) };
if !p.is_null() {
note_alloc(l.size());
}
p
}
unsafe fn dealloc(&self, p: *mut u8, l: Layout) {
LIVE.fetch_sub(l.size() as u64, Ordering::Relaxed);
unsafe { System.dealloc(p, l) }
}
unsafe fn realloc(&self, p: *mut u8, l: Layout, new: usize) -> *mut u8 {
let q = unsafe { System.realloc(p, l, new) };
if !q.is_null() {
let (old, new) = (l.size() as u64, new as u64);
ALLOC_BYTES.fetch_add(new.saturating_sub(old), Ordering::Relaxed);
let live = if new >= old {
LIVE.fetch_add(new - old, Ordering::Relaxed) + (new - old)
} else {
LIVE.fetch_sub(old - new, Ordering::Relaxed) - (old - new)
};
PEAK_LIVE.fetch_max(live, Ordering::Relaxed);
}
q
}
}
#[global_allocator]
static ALLOCATOR: Counting = Counting;
/// Peak resident set size, from the kernel's own high-water mark. Unlike a
/// sampled figure this cannot miss a spike.
fn vm_hwm_bytes() -> u64 {
std::fs::read_to_string("/proc/self/status")
.ok()
.and_then(|s| {
s.lines()
.find(|l| l.starts_with("VmHWM:"))?
.split_whitespace()
.nth(1)?
.parse::<u64>()
.ok()
})
.map(|kib| kib * 1024)
.unwrap_or(0)
}
use std::time::{Duration, Instant};
use quicksearch_core::config::Config;
@ -49,6 +140,24 @@ const LARGE_TEXT: usize = 100;
/// the disk. A control group: their cost must not move.
const BINARY: usize = 100;
/// Scale the generated tree by an integer factor (`QSB_SCALE`), keeping the
/// mix between the three groups fixed.
///
/// The default thousand files is enough to exercise every code path and far
/// too few to measure any of them: a run that size is dominated by fixed
/// start-up — opening the index, the config reconcile — and its per-file
/// figures carry the whole of SQLite's and FTS5's fixed structure spread over
/// a thousand rows. Anything claiming to be a per-file cost needs a tree where
/// the fixed part has been amortised away, and the difference between two
/// scales is the only way to tell the two apart.
fn scale() -> usize {
std::env::var("QSB_SCALE")
.ok()
.and_then(|v| v.parse().ok())
.filter(|n| *n >= 1)
.unwrap_or(1)
}
const WORDS: &[&str] = &[
"alpha",
"beta",
@ -144,11 +253,13 @@ fn generate(tree: &Path) {
let mut rng = Rng(0x5eed);
let (mut small_bytes, mut large_bytes, mut bin_bytes) = (0usize, 0usize, 0usize);
let scale = scale();
let (small_text, large_text, binary) = (SMALL_TEXT * scale, LARGE_TEXT * scale, BINARY * scale);
// Spread across subdirectories so the walk does real directory work
// rather than one enormous readdir.
for i in 0..SMALL_TEXT {
let dir = tree.join(format!("src/mod{}", i % 40));
for i in 0..small_text {
let dir = tree.join(format!("src/mod{}", i % (40 * scale)));
std::fs::create_dir_all(&dir).expect("mkdir");
let ext = ["txt", "md", "rs", "json"][i % 4];
let size = rng.in_range(200, 8 * 1024);
@ -157,8 +268,8 @@ fn generate(tree: &Path) {
std::fs::write(dir.join(format!("f{}.{}", i, ext)), body).expect("write");
}
for i in 0..LARGE_TEXT {
let dir = tree.join(format!("docs/set{}", i % 10));
for i in 0..large_text {
let dir = tree.join(format!("docs/set{}", i % (10 * scale)));
std::fs::create_dir_all(&dir).expect("mkdir");
let size = rng.in_range(8 * 1024 + 1, 200 * 1024);
let body = prose(&mut rng, size);
@ -166,8 +277,8 @@ fn generate(tree: &Path) {
std::fs::write(dir.join(format!("doc{}.md", i)), body).expect("write");
}
for i in 0..BINARY {
let dir = tree.join(format!("assets/set{}", i % 10));
for i in 0..binary {
let dir = tree.join(format!("assets/set{}", i % (10 * scale)));
std::fs::create_dir_all(&dir).expect("mkdir");
let n = rng.in_range(1024, 50 * 1024);
let blob: Vec<u8> = (0..n).map(|_| (rng.next() & 0xff) as u8).collect();
@ -175,21 +286,21 @@ fn generate(tree: &Path) {
std::fs::write(dir.join(format!("blob{}.bin", i)), blob).expect("write");
}
let total = SMALL_TEXT + LARGE_TEXT + BINARY;
let total = small_text + large_text + binary;
eprintln!("generated {} files under {}", total, tree.display());
eprintln!(
" text <= 8 KiB : {:5} files, {:8.1} MiB (head covers the whole file)",
SMALL_TEXT,
small_text,
small_bytes as f64 / (1024.0 * 1024.0)
);
eprintln!(
" text > 8 KiB : {:5} files, {:8.1} MiB (extraction must read it)",
LARGE_TEXT,
large_text,
large_bytes as f64 / (1024.0 * 1024.0)
);
eprintln!(
" binary : {:5} files, {:8.1} MiB (no extractor; control group)",
BINARY,
binary,
bin_bytes as f64 / (1024.0 * 1024.0)
);
}
@ -208,6 +319,135 @@ fn prose(rng: &mut Rng, target: usize) -> String {
s
}
/// The kernel's own accounting for this process, from `/proc/self/io`.
///
/// `read_bytes`/`write_bytes` are what actually reached the block layer, so
/// they are the figures that describe the *disk* rather than the page cache —
/// a warm re-read shows as `rchar` without moving `read_bytes`. `syscr`/`syscw`
/// count the calls regardless, which is what separates "we read a lot" from
/// "we read a little, many times".
///
/// Zero everywhere on a filesystem that does not report it (virtiofs, some
/// network mounts); the caller says so rather than printing a confident 0.
#[derive(Default, Clone, Copy)]
struct Io {
rchar: u64,
wchar: u64,
syscr: u64,
syscw: u64,
read_bytes: u64,
write_bytes: u64,
cancelled: u64,
}
impl Io {
fn read() -> Io {
let mut io = Io::default();
let Ok(text) = std::fs::read_to_string("/proc/self/io") else {
return io;
};
for line in text.lines() {
let Some((key, value)) = line.split_once(':') else {
continue;
};
let Ok(value) = value.trim().parse::<u64>() else {
continue;
};
match key {
"rchar" => io.rchar = value,
"wchar" => io.wchar = value,
"syscr" => io.syscr = value,
"syscw" => io.syscw = value,
"read_bytes" => io.read_bytes = value,
"write_bytes" => io.write_bytes = value,
"cancelled_write_bytes" => io.cancelled = value,
_ => {}
}
}
io
}
fn since(&self, start: &Io) -> Io {
Io {
rchar: self.rchar.saturating_sub(start.rchar),
wchar: self.wchar.saturating_sub(start.wchar),
syscr: self.syscr.saturating_sub(start.syscr),
syscw: self.syscw.saturating_sub(start.syscw),
read_bytes: self.read_bytes.saturating_sub(start.read_bytes),
write_bytes: self.write_bytes.saturating_sub(start.write_bytes),
cancelled: self.cancelled.saturating_sub(start.cancelled),
}
}
}
fn mib(bytes: u64) -> String {
format!("{:.1} MiB", bytes as f64 / (1024.0 * 1024.0))
}
/// What the write-ahead log did during a run, sampled from outside the process.
///
/// The interesting part of write amplification is not the total — that is one
/// number from `/proc/self/io` — but how it splits between **frames appended to
/// the log** and **pages copied back into the database** by a checkpoint. The
/// two want opposite fixes: more frames means the load is rewriting pages, and
/// more copy-back means it is checkpointing too often. A page rewritten five
/// times between two checkpoints costs five frames and *one* copy-back, so
/// checkpointing less often can be strictly cheaper — which is the opposite of
/// what "keep the log small" suggests.
///
/// Sampled rather than instrumented: the log is a file, its size is a `stat`,
/// and a checkpoint truncates it. Growth between samples is frames appended; a
/// drop is a checkpoint, and the size it dropped *from* bounds what that
/// checkpoint copied. Nothing in the library has to know it is being watched.
#[derive(Default, Clone, Copy)]
struct WalStats {
/// Largest the log ever got.
peak: u64,
/// Sum of every increase — bytes appended to the log over the run.
appended: u64,
/// Sum of the size before each truncation — an upper bound on the bytes
/// each checkpoint wrote back into the database.
copied_back: u64,
checkpoints: u64,
}
/// Watch `path` until `stop` is set, at `SAMPLE`.
///
/// One millisecond, because a checkpoint of a small log is quick and a sampler
/// that misses the rise and the fall reports neither. It costs one `stat` per
/// millisecond, which is nothing next to what is being measured.
fn sample_wal(path: PathBuf, stop: std::sync::Arc<std::sync::atomic::AtomicBool>) -> std::thread::JoinHandle<WalStats> {
const SAMPLE: Duration = Duration::from_millis(1);
std::thread::spawn(move || {
let mut stats = WalStats::default();
let mut last = 0u64;
while !stop.load(Ordering::Relaxed) {
let now = std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0);
if now > last {
stats.appended += now - last;
} else if now < last {
// A shrink is a checkpoint landing the log. `last` is the most
// recent size seen before it, so it bounds the copy-back.
stats.checkpoints += 1;
stats.copied_back += last;
}
stats.peak = stats.peak.max(now);
last = now;
std::thread::sleep(SAMPLE);
}
stats
})
}
/// Size of the index and the sidecars it leaves behind.
fn db_sizes(db: &Path) -> (u64, u64) {
let len = |p: PathBuf| std::fs::metadata(p).map(|m| m.len()).unwrap_or(0);
(
len(db.to_path_buf()),
len(PathBuf::from(format!("{}-wal", db.display()))),
)
}
fn run(mode: &str, tree: &Path, db: &Path) {
let config = Config::default();
@ -221,6 +461,15 @@ fn run(mode: &str, tree: &Path, db: &Path) {
.expect("clear marker");
}
// Cleared so the phase summaries below belong to this run alone.
quicksearch_core::log::clear();
let io_start = Io::read();
let (db_before, wal_before) = db_sizes(db);
let wal_path = PathBuf::from(format!("{}-wal", db.display()));
let wal_stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let wal_sampler = sample_wal(wal_path, wal_stop.clone());
let service = IndexingService::new();
let start = Instant::now();
service
@ -249,7 +498,10 @@ fn run(mode: &str, tree: &Path, db: &Path) {
}
let elapsed = start.elapsed();
assert!(done, "indexing did not finish within the timeout");
// The run's last checkpoint happens inside here, so the sampler outlives it.
service.stop_indexing().expect("stop");
wal_stop.store(true, Ordering::Relaxed);
let wal = wal_sampler.join().unwrap_or_default();
// Count what was actually indexed rather than assuming `gen`'s tree.
// The constants describe the tree this probe builds; pointing it at any
@ -258,11 +510,88 @@ fn run(mode: &str, tree: &Path, db: &Path) {
.ok()
.and_then(|c| quicksearch_core::db::repo::row_count(&c).ok())
.unwrap_or(0);
// Read after `stop_indexing`, so the optimize pass's checkpoint — which is
// where a run's dirty pages actually reach the file — is inside the totals.
let io = Io::read().since(&io_start);
let (db_after, wal_after) = db_sizes(db);
let per_file = |n: u64| {
if total == 0 {
"-".to_string()
} else {
format!("{:.0} B/file", n as f64 / total as f64)
}
};
eprintln!(
"{}: {:?} ({:.0} files/sec over {} files)",
"\n{}: {:?} ({:.0} files/sec over {} files)",
mode,
elapsed,
total as f64 / elapsed.as_secs_f64(),
total
);
// The pipeline logs one line per root per phase; they are the walk/extract
// split without a `perf` session.
for line in quicksearch_core::log::snapshot() {
let m = &line.text;
if m.contains("walk done")
|| m.contains("walk ended early")
|| m.contains("content done")
|| m.contains("stale cleanup")
|| m.contains("indexing complete")
{
eprintln!(" phase {}", m);
}
}
let allocs = ALLOCS.load(Ordering::Relaxed);
eprintln!(
" wal peak {}, {} appended, {} copied back over {} checkpoint(s)",
mib(wal.peak),
mib(wal.appended),
mib(wal.copied_back),
wal.checkpoints,
);
eprintln!(
" memory {} allocations ({:.1} per file), {} churned, peak live {}, VmHWM {}",
allocs,
allocs as f64 / total.max(1) as f64,
mib(ALLOC_BYTES.load(Ordering::Relaxed)),
mib(PEAK_LIVE.load(Ordering::Relaxed)),
mib(vm_hwm_bytes()),
);
eprintln!(
" index {} -> {} wal {} -> {}",
mib(db_before),
mib(db_after),
mib(wal_before),
mib(wal_after),
);
eprintln!(
" syscall {} reads, {} writes ({:.1} reads/file, {:.1} writes/file)",
io.syscr,
io.syscw,
io.syscr as f64 / total.max(1) as f64,
io.syscw as f64 / total.max(1) as f64,
);
eprintln!(
" bytes rchar {} / wchar {} (through the syscall layer, cache included)",
mib(io.rchar),
mib(io.wchar),
);
if io.read_bytes == 0 && io.write_bytes == 0 {
eprintln!(
" disk not reported for this filesystem (virtiofs/tmpfs); \
use rchar/wchar and the index sizes above"
);
} else {
eprintln!(
" disk read {} / written {} (cancelled {}) -> {} written",
mib(io.read_bytes),
mib(io.write_bytes),
mib(io.cancelled),
per_file(io.write_bytes.saturating_sub(io.cancelled)),
);
}
}

134
crates/quicksearch-core/examples/searchtime.rs Normal file
View file

@ -0,0 +1,134 @@
//! Search latency against an index that already exists on disk.
//!
//! The counterweight to `indexprobe`. Every FTS write-side knob — `automerge`,
//! a final `'optimize'`, `pgsz` — buys indexing time by leaving more segments
//! behind, and a segment is a b-tree a query has to visit. Halving a cold index
//! while doubling a keystroke is a regression wearing an improvement's clothes,
//! and this is what says which one happened.
//!
//! ```text
//! cargo build -p quicksearch-core --example searchtime --release
//! ./target/release/examples/searchtime /path/to/index.db
//! ```
//!
//! Queries are run against one held connection, as the search worker holds one
//! across a typing session, and each is timed best-of-N so a single scheduling
//! hiccup does not become the headline.
use std::path::PathBuf;
use std::sync::atomic::AtomicU64;
use std::time::{Duration, Instant};
use quicksearch_core::query::split::split_for_cascade;
use quicksearch_core::search::{cascade, SearchHit, SearchOptions};
/// `(read_bytes, rchar)` from `/proc/self/io`: what reached the block layer,
/// and what passed through the read syscalls. A query whose time varies while
/// `rchar` does not is not doing more work — it is waiting on the disk.
fn proc_io() -> (u64, u64) {
let text = std::fs::read_to_string("/proc/self/io").unwrap_or_default();
let field = |key: &str| -> u64 {
text.lines()
.find_map(|l| l.strip_prefix(key)?.trim().trim_start_matches(':').trim().parse().ok())
.unwrap_or(0)
};
(field("read_bytes"), field("rchar"))
}
/// Runs per query; the best is reported.
const RUNS: u32 = 5;
/// The query set, chosen to reach the passes an FTS setting can affect.
///
/// The content queries are the point — they are the ones that go through
/// `searchabletext` and therefore through however many segments the write side
/// left behind. The filename query is the control: it never touches FTS, so it
/// must not move.
const QUERIES: &[(&str, bool, &str)] = &[
("filename (control)", false, "doc42"),
("content, common", false, "mountain"),
("content, rare", false, "quartzite"),
("content, two words", false, "ocean forest"),
("fuzzy content", true, "mountian"),
];
fn main() {
let db = PathBuf::from(
std::env::args()
.nth(1)
.expect("usage: searchtime <index.db>"),
);
let conn = quicksearch_core::db::open::open_search_reader(&db.to_string_lossy())
.expect("open the index");
// Override the profile's ceiling, to test whether a slow index is slow
// because its working set does not fit rather than because it is bigger.
if let Ok(kib) = std::env::var("QSB_CACHE_KIB") {
conn.execute_batch(&format!("PRAGMA cache_size = -{};", kib.trim()))
.expect("set cache_size");
}
let segments: i64 = conn
.query_row("SELECT COUNT(*) FROM searchabletext_idx", [], |r| r.get(0))
.unwrap_or(-1);
let rows: i64 = conn
.query_row("SELECT COUNT(*) FROM files", [], |r| r.get(0))
.unwrap_or(-1);
println!(
"{} ({} rows, {} segment-index entries)",
db.display(),
rows,
segments
);
println!("{:<22} {:>10} {:>8}", "query", "best", "hits");
let mut total = Duration::ZERO;
for (label, fuzzy, query) in QUERIES {
let split = split_for_cascade(query).expect("query parses");
// The display limit makes this benchmark unfair between indexes.
// `scan_pass` stops as soon as the limit is full, and it streams FTS
// candidates in rowid order — which is `file_id` order, which is the
// order the *walk* happened to insert rows. So an index where the large
// documents drew low ids decompresses megabytes to fill 1000 hits while
// one where the small documents did reads a few hundred kilobytes, and
// the two differ by 13x for reasons that have nothing to do with what is
// being compared. Measured: 2.4 MiB against 110.9 MiB of `rchar` for the
// same query and the same hit count.
//
// Raising the limit past the corpus makes every index examine every
// candidate, which is the only way two of them are doing equal work.
let limit = std::env::var("QSB_LIMIT")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(1000);
let options = SearchOptions {
fuzzy: *fuzzy,
limit,
..SearchOptions::default()
};
let mut best = Duration::MAX;
let mut hits = 0usize;
let io_before = proc_io();
for _ in 0..RUNS {
let latest = AtomicU64::new(1);
let mut count = 0usize;
let mut sink = |h: Vec<SearchHit>| count += h.len();
let start = Instant::now();
cascade::run(&conn, &split, &options, 1, &latest, &mut sink).expect("cascade runs");
best = best.min(start.elapsed());
hits = count;
}
total += best;
let (rd, rc) = proc_io();
let (rd0, rc0) = io_before;
println!(
"{:<22} {:>10.1?} {:>8} disk-read {:>8.1} MiB rchar {:>8.1} MiB (over {} runs)",
label,
best,
hits,
(rd - rd0) as f64 / 1048576.0,
(rc - rc0) as f64 / 1048576.0,
RUNS,
);
}
println!("{:<22} {:>10.1?}", "TOTAL", total);
}

2
crates/quicksearch-core/examples/walkprobe.rs
View file

@ -80,7 +80,7 @@ fn serial(root: &str, config: &Config, existing: &DirRows) -> (usize, usize) {
for entry in filtered_walk(root, false, false, &ignore, &UnreadableDirs::default()) {
seen += 1;
// Same rule as the real walk: a name that is not valid UTF-8 cannot be
// stored in `files.path` and reopened by it, so it is skipped before
// stored in the index and reopened by its path, so it is skipped before
// anything tries to hash it. Counted as seen, never prepared.
let Some(path) = entry.path().to_str().map(str::to_owned) else {
continue;

54
crates/quicksearch-core/src/cli.rs
View file

@ -103,17 +103,23 @@ impl SizeReport {
/// failure reason comes from `failed_files`, the one place it is written.
pub fn status_for_path(db_path: &str, path: &str) -> Result<FileStatus, String> {
let conn = open_existing(db_path, false)?;
let row: Option<(i64, Option<String>)> = conn
.query_row(
"SELECT f.content_state, ff.reason \
let split = crate::file_handling::split_db_path(path);
let row: Option<(i64, Option<String>)> = match split {
// Not a path a row could ever have been stored under, so it is not
// indexed — the same answer as a miss.
None => None,
Some((parent, name)) => conn
.query_row(
"SELECT f.content_state, ff.reason \
FROM files f \
LEFT JOIN failed_files ff ON ff.file_id = f.id \
WHERE f.path = ?1",
params![path],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.optional()
.map_err(|e| format!("status_for_path({}): {}", path, e))?;
WHERE f.parent = ?1 AND f.name = ?2",
params![parent, name],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.optional()
.map_err(|e| format!("status_for_path({}): {}", path, e))?,
};
Ok(match row {
None => FileStatus {
path: path.to_string(),
@ -138,7 +144,7 @@ pub fn list_failed(db_path: &str, limit: Option<u32>) -> Result<Vec<FailedEntry>
None => String::new(),
};
let sql = format!(
"SELECT ff.file_id, f.path, ff.reason, ff.ts \
"SELECT ff.file_id, f.parent, f.name, ff.reason, ff.ts \
FROM failed_files ff \
JOIN files f ON f.id = ff.file_id \
ORDER BY ff.ts DESC{}",
@ -149,11 +155,13 @@ pub fn list_failed(db_path: &str, limit: Option<u32>) -> Result<Vec<FailedEntry>
.map_err(|e| format!("list_failed prepare: {}", e))?;
let rows = stmt
.query_map([], |r| {
let parent: String = r.get(1)?;
let name: String = r.get(2)?;
Ok(FailedEntry {
file_id: r.get(0)?,
path: r.get(1)?,
reason: r.get(2)?,
ts: r.get(3)?,
path: format!("{}{}", parent, name),
reason: r.get(3)?,
ts: r.get(4)?,
})
})
.map_err(|e| format!("list_failed query: {}", e))?;
@ -252,8 +260,7 @@ mod tests {
&tx,
&NewFile {
name: "a.txt",
path: "/tmp/a.txt",
parent: "/tmp",
parent: "/tmp/",
size: 1,
mtime: 1,
mime: Some("text/plain"),
@ -269,8 +276,7 @@ mod tests {
&tx,
&NewFile {
name: "b.bin",
path: "/tmp/b.bin",
parent: "/tmp",
parent: "/tmp/",
size: 1,
mtime: 1,
mime: None,
@ -343,8 +349,7 @@ mod tests {
&tx,
&NewFile {
name: "c.txt",
path: "/tmp/c.txt",
parent: "/tmp",
parent: "/tmp/",
size: 1,
mtime: 1,
mime: Some("text/plain"),
@ -361,8 +366,7 @@ mod tests {
&tx,
&NewFile {
name: "d.bin",
path: "/tmp/d.bin",
parent: "/tmp",
parent: "/tmp/",
size: 1,
mtime: 1,
mime: None,
@ -422,8 +426,7 @@ mod tests {
&tx,
&NewFile {
name: "big.txt",
path: "/tmp/big.txt",
parent: "/tmp",
parent: "/tmp/",
size: 1,
mtime: 1,
mime: Some("text/plain"),
@ -477,13 +480,12 @@ mod tests {
{
let mut conn = open_or_recreate(dbp, "unicode61").unwrap();
let tx = conn.transaction().unwrap();
for (name, path) in [("a.txt", "/tmp/a.txt"), ("b.txt", "/tmp/b.txt")] {
for name in ["a.txt", "b.txt"] {
insert_file(
&tx,
&NewFile {
name,
path,
parent: "/tmp",
parent: "/tmp/",
size: 1,
mtime: 1,
mime: Some("text/plain"),

2
crates/quicksearch-core/src/config/diff.rs
View file

@ -15,7 +15,7 @@ use super::*;
/// app was closed. See [`crate::scope`] for the pass that applies it.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct IndexWork {
/// Roots that are no longer configured, in `files.path` spelling. Every
/// Roots that are no longer configured, in stored spelling. Every
/// row beneath one is deleted; no filesystem access is involved, so a
/// root whose folder is gone is handled the same as one that still
/// exists.

298
crates/quicksearch-core/src/config/mod.rs
View file

@ -535,6 +535,13 @@ impl Config {
let mut cfg: Config = toml::from_str(&content)
.map_err(|e| format!("Failed to parse config file {}: {}", path.display(), e))?;
cfg.source = Some(path.to_path_buf());
// Before anything reads a value — and in particular before
// `config_check` compares `hash_length` against what the index was
// built with, or a clamp applied later would read as a changed
// setting and force a rebuild.
for warning in cfg.clamp_out_of_range() {
crate::log_warn!("config: {}", warning);
}
Ok(cfg)
} else {
let cfg = Config {
@ -546,6 +553,70 @@ impl Config {
}
}
/// Bring values that would break the program back into range, returning a
/// line about each one changed.
///
/// Clamps, never rejects: this file is hand-editable and a typo in it must
/// not stop the app starting, the same position `main.rs` takes on a file
/// that will not parse at all. Only the fields that are *not* already
/// defended where they are used appear here — `results_per_page`,
/// `root_workers`, `ui.scale`, `maximum_wal_size`, `batch_size` and
/// `reindex_interval_minutes` all clamp at their call sites, and doing it
/// twice would just be two places to disagree.
fn clamp_out_of_range(&mut self) -> Vec<String> {
let mut warnings = Vec::new();
let mut clamp = |name: &str, value: &mut u64, lo: u64, hi: u64| {
let bounded = (*value).clamp(lo, hi);
if bounded != *value {
warnings.push(format!(
"{} is {}, which is out of range; using {}",
name, *value, bounded
));
*value = bounded;
}
};
// Zero means every search returns nothing at all: `cascade::run`
// computes `remaining()` as zero and stops before its first pass, and
// reports the empty result as truncated.
let mut display_limit = self.search.display_limit as u64;
clamp("[search] display_limit", &mut display_limit, 1, 1_000_000);
self.search.display_limit = display_limit as usize;
// The last ceiling on how much an extractor reads: the extractors cap
// a single read, but this is what decides which files they open at all.
clamp(
"[processing] maximum_text_file_size",
&mut self.processing.maximum_text_file_size,
1,
4 * 1024 * 1024 * 1024,
);
// Below 262 bytes `infer`'s longest magic-number matcher cannot run,
// so file types stop being detectable by content; above a megabyte the
// walk's per-file head buffer stops being a head.
let mut hash_length = self.processing.hash_length as u64;
clamp(
"[processing] hash_length",
&mut hash_length,
262,
1024 * 1024,
);
self.processing.hash_length = hash_length as usize;
// The writer's round-robin turn. Unbounded, one root holds the writer
// for as long as it likes and every other root's walk waits behind it;
// zero is meaningful (one quantum per turn) and stays legal.
clamp(
"[processing] writer_turn_slice_ms",
&mut self.processing.writer_turn_slice_ms,
0,
10_000,
);
warnings
}
/// Write back to the file this config was loaded from (or the default
/// location), creating parent directories as needed. Raw values are
/// written verbatim — relative paths in a portable config stay relative.
@ -567,13 +638,37 @@ impl Config {
// open again. `rename` is atomic on both platforms, and the `sync_all`
// before it means the bytes are on the disk before the name points at
// them.
let tmp = path.with_extension("toml.tmp");
write_private(&tmp, content.as_bytes())
.map_err(|e| format!("Failed to write config file {}: {}", tmp.display(), e))?;
let tmp = write_private_temp(&path, content.as_bytes()).map_err(|e| {
format!(
"Failed to write config file beside {}: {}",
path.display(),
e
)
})?;
fs::rename(&tmp, &path).map_err(|e| {
let _ = fs::remove_file(&tmp);
format!("Failed to replace config file {}: {}", path.display(), e)
})?;
// The rename is atomic but not durable: it is a directory entry, and
// the directory has its own dirty state. Without this a power cut just
// after can leave *neither* name, which is the salt loss the whole
// dance exists to prevent. Best-effort, and on Windows it is a no-op
// every time rather than only on an exotic filesystem: opening a
// directory as a file needs `FILE_FLAG_BACKUP_SEMANTICS`, which
// `fs::File::open` does not pass, so this fails and is skipped. NTFS
// journals the rename itself, which is the guarantee this is reaching
// for; on Unix it has to be asked for.
if let Some(dir) = path.parent() {
if let Ok(handle) = fs::File::open(dir) {
let _ = handle.sync_all();
}
// Leftovers from a save that died between `create_new` and the
// rename. The old fixed `config.toml.tmp` overwrote itself, so
// there was never more than one; a unique name per attempt is what
// makes the write safe (see `write_private_temp`) and what makes
// them accumulate, so they are swept here instead.
sweep_stale_temps(dir, &path);
}
Ok(())
}
@ -598,8 +693,72 @@ impl Config {
}
}
/// Whether `path` is the index itself — the database, one of SQLite's
/// `-wal`/`-shm`/`-journal` sidecars, or the instance lock.
///
/// For the paths that arrive from a `files` row rather than from a walk:
/// once the walk stops indexing the index
/// ([`crate::file_handling::index_file_set`], which explains why opening
/// one is fatal), such rows are swept away, but a row written by an older
/// build survives until the sweep reaches it and can still be opened from
/// a result list in the meantime.
///
/// The name is compared first and the directory only on a hit, so the
/// overwhelmingly common miss costs one string comparison rather than the
/// `canonicalize` the directory check needs.
///
/// Names are folded where the filesystem folds them
/// ([`crate::platform::PATHS_ARE_CASE_INSENSITIVE`]): a stored row spelling
/// the database `Index.sqlite` names the same file as a config spelling it
/// `index.sqlite`, and opening it has the same consequence. ASCII only,
/// matching `PATH_COLLATION` and [`IgnoreSet`].
pub fn is_index_file(&self, path: &Path) -> bool {
let db = self.resolved_database_path();
let (Some(db_name), Some(name)) = (
db.file_name().and_then(|s| s.to_str()),
path.file_name().and_then(|s| s.to_str()),
) else {
return false;
};
let same_name = |a: &str, b: &str| {
if crate::platform::PATHS_ARE_CASE_INSENSITIVE {
a.eq_ignore_ascii_case(b)
} else {
a == b
}
};
// `str::get`, not `name[..cut]`: the cut is `db_name`'s *byte* length
// and the two names are unrelated strings, so it can land inside a
// multi-byte character — `€xyz` against a two-byte `db_name` is six
// bytes either way — and indexing there panics. A non-boundary is
// simply not a match.
let name_matches = same_name(name, db_name)
|| crate::file_handling::INDEX_SIDECAR_SUFFIXES
.iter()
.any(|s| {
let cut = db_name.len();
name.len() == cut + s.len()
&& name.get(..cut).is_some_and(|head| same_name(head, db_name))
&& name.get(cut..).is_some_and(|tail| same_name(tail, s))
});
if !name_matches {
return false;
}
let same_dir = |a: &Path, b: &Path| {
a == b
|| a.canonicalize().unwrap_or_else(|_| a.to_path_buf())
== b.canonicalize().unwrap_or_else(|_| b.to_path_buf())
};
match (path.parent(), db.parent()) {
(Some(a), Some(b)) => same_dir(a, b),
// Both at a filesystem root, or neither: the name match stands.
(None, None) => true,
_ => false,
}
}
/// `resolved_indexing_paths` canonicalized and spelled the way
/// `files.path` prefixes them.
/// stored parents are prefixed with them.
///
/// The form roots must be compared in: `~/docs`, `docs` in a portable
/// config and `/home/me/docs` are one root under three spellings, and a
@ -631,30 +790,121 @@ impl Config {
}
}
/// Write `bytes` to `path`, owner-readable only, and flush them to the disk
/// before returning.
/// Write `bytes` to a fresh temporary file beside `target`, owner-only, with
/// its contents flushed to the disk, and return its path for the caller to
/// rename into place.
///
/// `O_NOFOLLOW` on Unix: the config directory is not always somewhere only
/// this user can write — a portable install can sit in a shared or removable
/// directory — and a symlink left at the config's name would otherwise
/// redirect this write onto whatever it points at.
fn write_private(path: &Path, bytes: &[u8]) -> std::io::Result<()> {
/// `create_new` and a unique name, not a fixed one: `O_NOFOLLOW` refuses a
/// symlink but says nothing about a *regular* file or a hardlink that is
/// already sitting at the name we were going to use. The config directory is
/// not always somewhere only this user can write — a portable install can sit
/// in a shared or removable directory — and an attacker who pre-creates the
/// temp file as a hardlink to a file they can read would otherwise be handed
/// `[security].salt`. `mode(0o600)` only applies to a file this call creates,
/// which is the same reason.
fn write_private_temp(target: &Path, bytes: &[u8]) -> std::io::Result<PathBuf> {
use std::io::Write;
let mut opts = fs::OpenOptions::new();
opts.write(true).create(true).truncate(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.custom_flags(libc::O_NOFOLLOW);
opts.mode(0o600);
let dir = target.parent().unwrap_or_else(|| Path::new("."));
let stem = target
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.unwrap_or_else(|| "config.toml".to_string());
// A handful of attempts, then give up rather than spin: if something is
// racing us for every name we pick, failing the save is the honest
// outcome — the caller treats a failed save as fatal to the change.
let mut last_err = None;
for _ in 0..8 {
let tmp = dir.join(format!("{}.{}.tmp", stem, unique_suffix()));
let mut opts = fs::OpenOptions::new();
opts.write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.custom_flags(libc::O_NOFOLLOW);
opts.mode(0o600);
}
match opts.open(&tmp) {
Ok(mut f) => {
let wrote = f.write_all(bytes).and_then(|()| f.sync_all());
return match wrote {
Ok(()) => Ok(tmp),
Err(e) => {
let _ = fs::remove_file(&tmp);
Err(e)
}
};
}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => last_err = Some(e),
Err(e) => return Err(e),
}
}
let mut f = opts.open(path)?;
f.write_all(bytes)?;
// The rename that follows is atomic with respect to the *directory*, not
// to the file's contents: without this, a crash can leave the new name
// pointing at a block of zeroes.
f.sync_all()
Err(last_err.unwrap_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::AlreadyExists,
"could not create a temporary config file",
)
}))
}
/// How old an abandoned temp file must be before [`sweep_stale_temps`] takes
/// it. A save is a serialize, a write and a rename — milliseconds — so an hour
/// is far past any doubt, while still short enough that leftovers do not
/// accumulate across a run of crashes.
const TEMP_SWEEP_AGE: std::time::Duration = std::time::Duration::from_secs(60 * 60);
/// Delete abandoned `<config name>.*.tmp` files beside the config.
///
/// [`write_private_temp`] must use a *unique* name — a fixed one can be
/// pre-created by someone else as a hardlink — and a unique name is one that
/// nothing later overwrites, so a process that dies between `create_new` and
/// the rename leaves its temp file behind for good. The old fixed
/// `config.toml.tmp` was reused by the next save and so never accumulated;
/// this is what replaces that property.
///
/// Age is the discriminator, not the recorded PID: a temp file created seconds
/// ago may belong to a save running *right now* in another process, and
/// deleting that would destroy the very write this whole dance protects.
/// Entirely best-effort — a directory that cannot be listed is not a reason to
/// fail a save that already succeeded.
fn sweep_stale_temps(dir: &Path, target: &Path) {
let Some(stem) = target.file_name().and_then(|n| n.to_str()) else {
return;
};
// The trailing dot matters: without it the config file itself would be a
// prefix match.
let prefix = format!("{}.", stem);
let Ok(entries) = fs::read_dir(dir) else {
return;
};
for entry in entries.flatten() {
let name = entry.file_name();
let Some(name) = name.to_str() else { continue };
if !name.starts_with(&prefix) || !name.ends_with(".tmp") {
continue;
}
let abandoned = entry
.metadata()
.and_then(|m| m.modified())
.is_ok_and(|t| t.elapsed().is_ok_and(|age| age >= TEMP_SWEEP_AGE));
if abandoned {
let _ = fs::remove_file(entry.path());
}
}
}
/// A short, non-guessable-enough suffix for the temp name.
///
/// This does not need to be unpredictable to an attacker — `create_new` is
/// what makes the write safe — only unlikely to collide with a leftover from
/// an interrupted save, so the process id and a clock reading are plenty.
fn unique_suffix() -> String {
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.subsec_nanos())
.unwrap_or(0);
format!("{}-{}", std::process::id(), nanos)
}
/// Reserved `content_extensions` entry standing for "files with no

110
crates/quicksearch-core/src/config/tests.rs
View file

@ -1043,3 +1043,113 @@ fn the_search_table_ships_without_size_or_modified() {
assert_eq!(older.search.display_limit, 500);
assert!(older.search.live_results, "live results default to on");
}
/// The guard the live watcher and duplicate verification consult before
/// opening a path that came from a `files` row. See
/// [`crate::file_handling::index_file_set`] for what opening one costs.
#[test]
fn the_index_and_every_sidecar_are_recognised() {
let mut c = Config::default();
c.paths.database_path = "/var/lib/qs/index.sqlite".to_string();
for name in [
"index.sqlite",
"index.sqlite-wal",
"index.sqlite-shm",
"index.sqlite-journal",
"index.sqlite.lock",
] {
let p = PathBuf::from(format!("/var/lib/qs/{}", name));
assert!(c.is_index_file(&p), "{} should be recognised", name);
}
// A near miss on the name, and the right name in the wrong directory.
for p in [
"/var/lib/qs/index.sqlite-walrus",
"/var/lib/qs/notes-wal",
"/var/lib/qs/index.sqlite2",
"/var/lib/other/index.sqlite-wal",
"/var/lib/qs/sub/index.sqlite",
] {
assert!(
!c.is_index_file(Path::new(p)),
"{} is not part of the index",
p
);
}
}
/// `database_path` is resolved the same way everywhere else resolves it, so a
/// `~` or a relative spelling has to compare equal to the absolute file.
#[test]
fn a_tilde_database_path_still_matches_the_real_file() {
let Some(home) = crate::platform::home_dir() else {
return;
};
let mut c = Config::default();
c.paths.database_path = "~/.local/share/quicksearch/index.sqlite".to_string();
let absolute = PathBuf::from(home).join(".local/share/quicksearch/index.sqlite");
assert!(c.is_index_file(&absolute));
let wal = absolute.with_file_name("index.sqlite-wal");
assert!(c.is_index_file(&wal));
assert!(!c.is_index_file(&absolute.with_file_name("other.sqlite")));
}
/// `display_limit = 0` makes `cascade::run` compute `remaining()` as zero and
/// stop before its first pass, so *every* search returns nothing and reports
/// itself truncated. A hand-editable file must not be able to do that.
#[test]
fn out_of_range_numbers_are_clamped_not_rejected() {
let dir = tmp_dir();
let path = dir.join("config.toml");
fs::write(
&path,
"[search]\ndisplay_limit=0\n\
[processing]\nmaximum_text_file_size=0\nhash_length=8\nwriter_turn_slice_ms=18446744073709551615\n",
)
.unwrap();
let cfg = Config::load_from(&path).expect("a bad number must not stop the app starting");
assert_eq!(cfg.search.display_limit, 1);
assert_eq!(cfg.processing.maximum_text_file_size, 1);
assert_eq!(cfg.processing.hash_length, 262);
assert_eq!(cfg.processing.writer_turn_slice_ms, 10_000);
fs::remove_dir_all(&dir).ok();
}
/// The clamp must be silent about values that are merely unusual, or the
/// warning is noise and `hash_length` — which is in `REBUILD_KEYS` — would
/// read as changed and force a rebuild on every start.
#[test]
fn in_range_numbers_are_left_exactly_alone() {
let mut cfg = Config::default();
let before = cfg.clone();
let warnings = cfg.clamp_out_of_range();
assert!(warnings.is_empty(), "defaults must not warn: {warnings:?}");
assert_eq!(cfg.search.display_limit, before.search.display_limit);
assert_eq!(cfg.processing.hash_length, before.processing.hash_length);
assert_eq!(
cfg.processing.maximum_text_file_size,
before.processing.maximum_text_file_size
);
assert_eq!(
cfg.processing.writer_turn_slice_ms,
before.processing.writer_turn_slice_ms
);
// Zero is a legal turn slice — one quantum per turn, not "no turn".
cfg.processing.writer_turn_slice_ms = 0;
assert!(cfg.clamp_out_of_range().is_empty());
assert_eq!(cfg.processing.writer_turn_slice_ms, 0);
}
/// Each clamped field must say so, by name, so the warning is actionable.
#[test]
fn a_clamped_field_is_named_in_its_warning() {
let mut cfg = Config::default();
cfg.search.display_limit = 0;
let warnings = cfg.clamp_out_of_range();
assert_eq!(warnings.len(), 1, "{warnings:?}");
assert!(warnings[0].contains("display_limit"), "{warnings:?}");
assert!(warnings[0].contains('1'), "{warnings:?}");
}

62
crates/quicksearch-core/src/content.rs
View file

@ -72,9 +72,9 @@ struct Shared {
queue: Mutex<Queue>,
idle: Condvar,
/// What the range held when the pass began: rows still to extract and
/// rows already done. Set by the feeder before it pages anything, so
/// `already_done + rows written this pass` stays exact; never set if the
/// feeder could not count.
/// rows already done. Set by the feeder just *behind* its first page, so
/// the pool is never blocked on the scan that produces it; never set if the
/// feeder could not count. See `feeder` for what deferring it costs.
totals: std::sync::OnceLock<ExtractScope>,
}
@ -173,9 +173,10 @@ impl ContentPass {
/// The range's pending and already-done counts as they stood when the
/// pass began.
///
/// `None` until the feeder has counted — a scan that takes seconds on a
/// large root, which is why it happens here on the pass's own connection
/// and not on the indexer's writer — and forever if it could not.
/// `None` until the feeder has counted — a scan measured at 513 ms over a
/// million rows, which is why it happens on the pass's own connection
/// rather than the indexer's writer, and behind the first page rather than
/// in front of the pool — and forever if it could not.
pub fn totals(&self) -> Option<ExtractScope> {
self.shared.totals.get().copied()
}
@ -224,16 +225,36 @@ fn feeder(shared: &Shared, db_path: &str, mut cursor: ExtractCursor, config: &Co
}
};
// Before the first page, so nothing this pass writes is inside the count.
// The workers cannot run ahead of this: they block in `take` until the
// first page lands. A failure here costs the progress figure, not the
// pass.
match crate::file_handling::count_extract_scope(&conn, &cursor, config) {
Ok(totals) => {
let _ = shared.totals.set(totals);
// The count happens *behind* the first page, not in front of it.
//
// It used to run here, before anything was fetched, and the workers block
// in `take` until a page lands — so every thread in the pool sat idle for
// the whole of it, to compute a progress-bar denominator. That is not
// free: the scan walks the root's entire parent range fetching a row per
// entry, measured at 20 ms over 100,000 rows and **513 ms over a million**
// with the index already in cache, and the count was moved onto this
// connection in the first place because on a large root it takes seconds
// cold. The move took it off the writer and left the stall one level down.
//
// What it costs to defer: rows this pass writes during the count can be
// seen by it as `already_done` rather than `pending`. The two move in
// opposite directions and `extract_total` is their **sum**, so the
// denominator is unaffected; only the numerator can run briefly ahead of
// itself, which is a shape `RootProgress` already reports and deliberately
// does not clamp — see the note on `snapshot`.
let mut counted = false;
let mut count_now = |conn: &rusqlite::Connection, cursor: &ExtractCursor| {
if counted {
return;
}
Err(e) => crate::log_warn!("content reader: {}", e),
}
counted = true;
match crate::file_handling::count_extract_scope(conn, cursor, config) {
Ok(totals) => {
let _ = shared.totals.set(totals);
}
Err(e) => crate::log_warn!("content reader: {}", e),
}
};
let max_size = crate::file_handling::max_text_file_size(config);
while shared.take_feed_slot().is_some() {
@ -261,10 +282,16 @@ fn feeder(shared: &Shared, db_path: &str, mut cursor: ExtractCursor, config: &Co
})
.collect();
shared.finish_feed(rows, last_page);
// The pool is running now; the denominator can be worked out behind it.
count_now(&conn, &cursor);
if last_page {
return;
}
}
// A range whose first `take_feed_slot` said the pass was already over
// still deserves its figure — `an_empty_range_terminates_immediately`
// pins that an empty root reports a known zero rather than an unknown.
count_now(&conn, &cursor);
}
fn worker(
@ -396,8 +423,7 @@ mod tests {
&tx,
&NewFile {
name: f.file_name().unwrap().to_str().unwrap(),
path: f.to_str().unwrap(),
parent: d.to_str().unwrap(),
parent: &crate::file_handling::dir_to_db_parent(&d),
size: std::fs::metadata(&f).unwrap().len(),
mtime: 1,
mime: Some("text/plain"),
@ -494,7 +520,7 @@ mod tests {
.lock()
.unwrap()
.query_row(
"SELECT content_state FROM files WHERE path = ?1",
"SELECT content_state FROM files WHERE parent || name = ?1",
rusqlite::params![p.to_str().unwrap()],
|r| r.get(0),
)

2
crates/quicksearch-core/src/coordinator.rs
View file

@ -234,6 +234,8 @@ impl IndexCoordinator {
last_event_at: None,
pending_since: None,
needs_full_run: false,
run_retry_at: None,
run_retry_delay: inner::RUN_RETRY_BASE,
pending_work: None,
reconcile_done: None,
reconcile_cut_short: false,

181
crates/quicksearch-core/src/coordinator/inner.rs
View file

@ -3,6 +3,32 @@
use super::*;
/// First wait after a full run is refused or fails to start.
pub(super) const RUN_RETRY_BASE: Duration = Duration::from_secs(30);
/// Ceiling for that wait. A misconfiguration that is never fixed then costs
/// one attempt every few minutes instead of one per second.
const RUN_RETRY_MAX: Duration = Duration::from_secs(300);
/// Individually-logged occurrences of each refusal before it goes quiet. The
/// message is identical every time, and the log ring is the only record of
/// what *else* happened; these are reset the moment a run starts.
static NO_ROOTS: crate::log::Throttle = crate::log::Throttle::new(3);
static NESTED_ROOTS: crate::log::Throttle = crate::log::Throttle::new(3);
static START_FAILURES: crate::log::Throttle = crate::log::Throttle::new(3);
/// Why a full run is being started, which is what decides how a refusal is
/// reported. See [`Inner::refuse`].
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum RunTrigger {
/// Something the user did: *Index now*, *Rebuild index*, or a settings
/// change wide enough to need a walk. A refusal owes them an answer.
Requested,
/// The periodic timer, or repair the coordinator scheduled for itself. A
/// refusal here is worth one log line and then quiet.
Scheduled,
}
pub(super) struct Inner {
pub(super) config: Config,
pub(super) indexing: Arc<IndexingService>,
@ -40,6 +66,18 @@ pub(super) struct Inner {
/// defer application past `pending_max_defer`.
pub(super) pending_since: Option<Instant>,
pub(super) needs_full_run: bool,
/// Earliest time another full run may be *attempted*, when the last one
/// was refused or failed. Without it a config with no roots, or with
/// nested roots, or an index on a full disk, retries every tick forever:
/// two stderr writes and two ring pushes a second, which evicts the whole
/// 5000-line log ring in under an hour and takes the diagnostics for the
/// actual problem with it — and in the failed-run case re-walks the tree
/// each time.
pub(super) run_retry_at: Option<Instant>,
/// How long the next refusal waits, doubling to [`RUN_RETRY_MAX`] so a
/// permanent misconfiguration costs almost nothing, and reset once a run
/// actually starts.
pub(super) run_retry_delay: Duration,
/// Reconciliation owed to a config change, part-applied across ticks.
/// Unlike `needs_full_run`, this is acted on in manual mode too.
pub(super) pending_work: Option<WorkCursor>,
@ -95,7 +133,7 @@ impl Inner {
CoordCmd::SetMode(IndexMode::ManualStopped) => self.enter_manual_stopped(),
// ManualRunning isn't directly settable; ReindexNow is the verb.
CoordCmd::SetMode(IndexMode::ManualRunning) | CoordCmd::ReindexNow => {
self.start_full_run();
self.start_full_run(RunTrigger::Requested);
if self.mode != IndexMode::Auto {
self.mode = IndexMode::ManualRunning;
}
@ -139,10 +177,18 @@ impl Inner {
self.pending_work = None;
self.reconcile_done = None;
if let Err(e) = self.indexing.delete_index_for_rebuild(&db) {
// Not a warning to step over: a run started now would
// reopen the *old* index and present itself as the rebuild
// the user asked for. Worse after a password change, where
// the config already claims a protection the surviving
// file does not have. Report it and leave the index alone.
crate::log_warn!("coordinator: rebuild: {}", e);
self.indexing
.report_error(format!("could not rebuild the index: {}", e));
return;
}
self.clear_root_counts();
self.start_full_run();
self.start_full_run(RunTrigger::Requested);
if self.mode != IndexMode::Auto {
self.mode = IndexMode::ManualRunning;
}
@ -174,7 +220,7 @@ impl Inner {
// delivers anything else, so nothing downstream checks, and
// a file renamed *out* of every root would otherwise be
// written into the index at its new home. Roots in the same
// spelling `files.path` uses — the caller's paths are.
// spelling the index uses — the caller's paths are.
let prefixes: Vec<String> = self
.config
.normalized_indexing_paths()
@ -224,6 +270,12 @@ impl Inner {
if self.saw_running {
self.saw_running = false;
self.was_busy = true;
// A run that errored never stamps `last_full_index`, so
// `periodic_due` stays true and the next tick starts another one —
// a fresh whole-tree walk per second against, say, a full disk.
if matches!(status, IndexingStatus::Error(_)) {
self.defer_runs();
}
self.refresh_last_full_index();
// Eager re-read: the run just changed the number on screen.
self.files_at = None;
@ -263,8 +315,9 @@ impl Inner {
worked = true;
}
if self.needs_full_run || self.periodic_due() {
self.start_full_run();
let deferred = self.run_retry_at.is_some_and(|at| Instant::now() < at);
if (self.needs_full_run || self.periodic_due()) && !deferred {
self.start_full_run(RunTrigger::Scheduled);
worked = true;
}
@ -408,7 +461,7 @@ impl Inner {
// Widening adds files only a walk can produce; mirrors `ReindexNow`,
// including the manual-mode round trip back to stopped.
if cursor.reindex() {
self.start_full_run();
self.start_full_run(RunTrigger::Requested);
if self.mode != IndexMode::Auto {
self.mode = IndexMode::ManualRunning;
}
@ -555,6 +608,9 @@ impl Inner {
Err(e) => {
crate::log_warn!("coordinator: targeted update unavailable: {}", e);
self.targeted.clear();
// As in `start_full_run`: a resume point outlives its event
// here too, and would misapply to the next one for that path.
self.resume_from.retain(|p, _| self.pending.contains_key(p));
return;
}
};
@ -633,10 +689,15 @@ impl Inner {
}
fn periodic_due(&self) -> bool {
// `.max(1)`: at zero every tick is "due", so a full run would start
// the moment the previous one finished, forever. There is no setting
// that means "reindex continuously" — manual mode is how you say
// "never", so zero is a typo rather than an intent.
let interval_secs = self
.config
.indexing
.reindex_interval_minutes
.max(1)
.saturating_mul(60);
let last = crate::lock_ok(&self.shared).last_full_index;
match last {
@ -651,7 +712,20 @@ impl Inner {
}
}
fn start_full_run(&mut self) {
fn start_full_run(&mut self, trigger: RunTrigger) {
// A run already in progress is not a refusal: whoever called this
// wanted a run and there is one. Returning here rather than falling
// through to `start_indexing`'s "already running" error keeps an
// *Index now* pressed mid-run from backing off future runs and from
// replacing the progress the user is watching with an error. It also
// covers `Optimizing`, which `start_indexing` does not reject even
// though it holds a write transaction over the whole file.
if !matches!(
self.indexing.get_status(),
IndexingStatus::Idle | IndexingStatus::Error(_)
) {
return;
}
let roots: Vec<String> = self
.config
.resolved_indexing_paths()
@ -659,20 +733,23 @@ impl Inner {
.map(|p| p.to_string_lossy().into_owned())
.collect();
if roots.is_empty() {
crate::log_warn!("coordinator: no indexing roots configured");
self.refuse(trigger, &NO_ROOTS, "no indexing roots are configured");
return;
}
// Backstop for hand-edited configs; the GUI rejects nested roots
// itself.
let nested = crate::config::nested_roots(&roots);
if !nested.is_empty() {
for (child, parent) in &nested {
crate::log_warn!(
"coordinator: refusing to index: root {} is nested under {}",
child,
parent
);
}
let detail = nested
.iter()
.map(|(child, parent)| format!("{} is nested under {}", child, parent))
.collect::<Vec<_>>()
.join("; ");
self.refuse(
trigger,
&NESTED_ROOTS,
&format!("refusing to index nested roots: {}", detail),
);
return;
}
// The full run owns the DB (and may wipe/rebuild the file).
@ -683,6 +760,12 @@ impl Inner {
// unreadable directory (`unreadable.covers`) or of an aliased symlink
// target (`aliased_paths`), and those rows would leak until a rebuild.
self.pending.retain(|_, ev| is_removal(ev));
// Resume points describe events that no longer exist. Left behind, the
// next `Create` for that path would `skip` the first n entries of a
// walk that has nothing to do with the one the count came from, and
// those files would have no row until a later full run.
self.resume_from
.retain(|p, _| self.pending.contains_key(p) || self.targeted.contains_key(p));
if self.pending.is_empty() {
self.last_event_at = None;
self.pending_since = None;
@ -691,15 +774,58 @@ impl Inner {
.indexing
.start_indexing(roots, self.db_path(), self.config.clone())
{
crate::log_warn!("coordinator: start indexing: {}", e);
self.refuse(
trigger,
&START_FAILURES,
&format!("could not start indexing: {}", e),
);
return;
}
// A run started, so whatever was wrong is not wrong any more.
self.run_retry_at = None;
self.run_retry_delay = RUN_RETRY_BASE;
NO_ROOTS.reset();
NESTED_ROOTS.reset();
START_FAILURES.reset();
// `start_indexing` claims Running before returning, so there is no
// window in which this thread believes the service idle and writes to
// a database the run is about to reopen.
self.saw_running = true;
}
/// Refuse a full run: back off, and say so at a volume the trigger earns.
///
/// The throttles exist for the *scheduled* retry, which repeats the same
/// sentence every few minutes for as long as the configuration stays
/// broken and would otherwise evict the log ring. They must not silence a
/// run somebody asked for: after three refusals a fourth press of *Index
/// now* would log nothing, show nothing and change nothing, which reads as
/// a button that does not work. So a requested run resets the throttle —
/// it is a fresh occurrence in the user's eyes — and publishes the reason
/// where they are already looking, the same way a failed rebuild does.
fn refuse(&mut self, trigger: RunTrigger, throttle: &crate::log::Throttle, reason: &str) {
self.defer_runs();
if trigger == RunTrigger::Requested {
throttle.reset();
self.indexing.report_error(reason.to_string());
}
if throttle.allow() {
crate::log_warn!("coordinator: {}", reason);
}
}
/// Hold off further full runs for a while, doubling the wait each time.
///
/// Called wherever a run is refused or fails to start. `needs_full_run` is
/// cleared with it: the two refusal paths sit above the point that clears
/// it, so without this the flag stays set and `tick` retries immediately,
/// every second, for as long as the configuration stays broken.
fn defer_runs(&mut self) {
self.needs_full_run = false;
self.run_retry_at = Some(Instant::now() + self.run_retry_delay);
self.run_retry_delay = (self.run_retry_delay * 2).min(RUN_RETRY_MAX);
}
fn enter_auto(&mut self) {
self.mode = IndexMode::Auto;
self.config.indexing.auto_index = true;
@ -811,6 +937,29 @@ impl Inner {
let Some(mut reason) = w.degraded_reason() else {
return;
};
// An overflow is not a capacity problem: the watcher is still good,
// it just cannot tell us what it missed. Schedule the run that finds
// out, rather than disabling live updates for the session over
// something a busy minute can cause.
//
// Not a restart. `inotify` and `ReadDirectoryChangesW` both keep
// delivering after their queue overflows — the rescan flag means "you
// missed some", not "this watch is broken" — so tearing down and
// re-registering would re-`stat` every one of `max_watched_dirs`
// (128k) directories, generate events of its own, and do it again the
// next time a busy tree overflowed the queue, which is exactly when
// the machine can least afford it.
//
// Consuming the reason is what makes this a one-shot: the slot is
// never otherwise cleared, so a standing `Overflowed` would re-arm
// `needs_full_run` on every tick and hide a `KernelLimit` recorded
// afterwards.
if matches!(reason, WatchError::Overflowed) {
w.clear_degraded();
crate::log_warn!("watcher: {}", reason);
self.needs_full_run = true;
return;
}
// The async notify callback can't know the count; fill it in here.
if let WatchError::KernelLimit { registered } = &mut reason {
if *registered == 0 {

2
crates/quicksearch-core/src/coordinator/tests.rs
View file

@ -512,7 +512,7 @@ impl Fixture {
fn stored_mtime(&self, path: &std::path::Path) -> Option<i64> {
let conn = db::open_existing(&self.db.to_string_lossy(), false).ok()?;
conn.query_row(
"SELECT mtime FROM files WHERE path = ?1",
"SELECT mtime FROM files WHERE parent || name = ?1",
[path.to_string_lossy().as_ref()],
|r| r.get(0),
)

4
crates/quicksearch-core/src/db/mod.rs
View file

@ -17,8 +17,8 @@ pub mod schema;
pub use key::{process_key_hex, set_process_key};
pub use open::{
index_needs_rebuild, open_existing, open_or_recreate, verify_process_key,
CURRENT_SCHEMA_VERSION, KEY_MISMATCH_PREFIX,
index_needs_rebuild, key_mismatch_parts, open_existing, open_or_recreate, verify_process_key,
KeyMismatch, CURRENT_SCHEMA_VERSION, FOREIGN_DB_PREFIX, KEY_MISMATCH_PREFIX,
};
/// Bumped whenever the index file is replaced rather than modified — a

189
crates/quicksearch-core/src/db/open.rs
View file

@ -25,7 +25,7 @@ pub const KEY_MISMATCH_PREFIX: &str = "KEY_MISMATCH: ";
/// values go stale: `files.mime`, `files.type` and `content_state` are
/// computed at walk time and never re-derived for unchanged files, so a
/// classification change needs the wipe to apply everywhere.
pub const CURRENT_SCHEMA_VERSION: u32 = 7;
pub const CURRENT_SCHEMA_VERSION: u32 = 8;
/// Open `db_path` and ensure the on-disk schema matches this build; if it
/// doesn't (including a changed `tokenizer`), delete the file and recreate it
@ -241,6 +241,57 @@ fn is_notadb(e: &rusqlite::Error) -> bool {
)
}
/// Why a keyed open failed, as something the caller can branch on.
///
/// The three cases want three different things from a user — retype the
/// password, rebuild the index, supply a password at all — and only one of
/// them is "wrong password". They used to be distinguishable only by reading
/// the English in the message, which breaks the moment a database path
/// happens to contain that English.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum KeyMismatch {
/// A key was applied and the file did not accept it.
WrongPassword,
/// A key was applied but the file on disk is not encrypted at all —
/// protection was enabled and the rebuild that would encrypt it did not
/// finish.
NotEncrypted,
/// No key was applied and the file wants one.
PasswordRequired,
}
impl KeyMismatch {
/// The machine-readable token carried in the message, between
/// [`KEY_MISMATCH_PREFIX`] and the human detail.
fn token(self) -> &'static str {
match self {
KeyMismatch::WrongPassword => "wrong-password",
KeyMismatch::NotEncrypted => "not-encrypted",
KeyMismatch::PasswordRequired => "password-required",
}
}
fn from_token(token: &str) -> Option<KeyMismatch> {
match token {
"wrong-password" => Some(KeyMismatch::WrongPassword),
"not-encrypted" => Some(KeyMismatch::NotEncrypted),
"password-required" => Some(KeyMismatch::PasswordRequired),
_ => None,
}
}
}
/// Split a tagged mismatch message into its cause and the human detail.
///
/// `None` for any message that is not one — including a `KEY_MISMATCH_PREFIX`
/// message from an older build, which callers should treat as they always did.
pub fn key_mismatch_parts(message: &str) -> Option<(KeyMismatch, &str)> {
let rest = message.strip_prefix(KEY_MISMATCH_PREFIX)?;
let (token, detail) = rest.split_once(' ')?;
let token = token.strip_suffix(':')?;
Some((KeyMismatch::from_token(token)?, detail))
}
fn key_mismatch_message(db_path: &str, had_key: bool) -> String {
// An unencrypted SQLite file still has its plaintext magic; sniffing it
// distinguishes "wrong password" from "protection is enabled but the
@ -255,31 +306,122 @@ fn key_mismatch_message(db_path: &str, had_key: bool) -> String {
Some(&magic == b"SQLite format 3\0")
})
.unwrap_or(false);
let detail = match (had_key, plaintext) {
(true, true) => {
let (cause, detail) = match (had_key, plaintext) {
(true, true) => (
KeyMismatch::NotEncrypted,
"password protection is enabled but the index is not encrypted; \
rebuild the index to encrypt it"
}
(true, false) => "wrong password (or the file is not a QuickSearch index)",
(false, _) => "the index is password-protected; a password is required",
rebuild the index to encrypt it",
),
(true, false) => (
KeyMismatch::WrongPassword,
"wrong password (or the file is not a QuickSearch index)",
),
(false, _) => (
KeyMismatch::PasswordRequired,
"the index is password-protected; a password is required",
),
};
format!("{}index at {}: {}", KEY_MISMATCH_PREFIX, db_path, detail)
// The token sits between the prefix and the detail so that every existing
// `starts_with(KEY_MISMATCH_PREFIX)` test still holds, while a caller that
// needs the cause can have it without reading prose.
format!(
"{}{}: index at {}: {}",
KEY_MISMATCH_PREFIX,
cause.token(),
db_path,
detail
)
}
/// True iff the DB has a `schema_info` table whose `version` equals
/// [`CURRENT_SCHEMA_VERSION`]. Ignores the tokenizer — that's only the
/// owner's concern.
fn schema_version_current(conn: &Connection) -> Result<bool, String> {
let has_info: bool = conn
.query_row(
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='schema_info'",
[],
|_| Ok(true),
/// Prefix tagging the "this file is not a QuickSearch index" refusal, so a
/// caller can tell it from the schema drift that legitimately rebuilds.
pub const FOREIGN_DB_PREFIX: &str = "FOREIGN_DB: ";
/// Tables left behind by the pre-`schema_info` layout, which is the only kind
/// of index of ours that [`has_our_schema_info`] cannot recognise.
///
/// `files` is the only one guaranteed present across those layouts, and it is
/// the loose end here: another application's database with a table called
/// `files` would still be taken for an ancient index of ours and wiped.
/// Refusing a genuine legacy index is the worse failure of the two, so it
/// stays — narrowed by the fact that anything with a `schema_info` of our
/// shape is already decided before this list is consulted.
const LEGACY_TABLES: &[&str] = &["files", "files_fts", "documents_text", "failed_files"];
/// Whether `schema_info` exists *and* is shaped like ours.
///
/// The shape, not the contents: preparing the statement succeeds only if the
/// table has both columns, and an index whose creation was interrupted before
/// the version row landed is still ours. A foreign database that happens to
/// use the name for something else is not.
fn has_our_schema_info(conn: &Connection) -> bool {
conn.prepare("SELECT key, value FROM schema_info").is_ok()
}
/// Whether the file is one of ours, or empty enough to become one.
///
/// `sqlite_master` is empty for a file SQLite has just created and for a
/// zero-length one, which is the "ours to create" case. Internal `sqlite_%`
/// names are excluded so an autoindex or a stat table cannot make an
/// otherwise-empty file look occupied.
fn is_ours_or_empty(conn: &Connection) -> Result<bool, String> {
if has_our_schema_info(conn) {
return Ok(true);
}
let mut stmt = conn
.prepare("SELECT name FROM sqlite_master WHERE name NOT LIKE 'sqlite_%'")
.map_err(|e| format!("read sqlite_master: {}", e))?;
let mut any = false;
let names = stmt
.query_map([], |r| r.get::<_, String>(0))
.map_err(|e| format!("read sqlite_master: {}", e))?;
for name in names {
let name = name.map_err(|e| format!("read sqlite_master: {}", e))?;
any = true;
if LEGACY_TABLES.contains(&name.as_str()) {
return Ok(true);
}
}
Ok(!any)
}
/// The refusal message, naming a few of the tables that are in the way so the
/// user can recognise whose file they pointed at.
fn foreign_database_message(conn: &Connection) -> Result<String, String> {
let mut stmt = conn
.prepare(
"SELECT name FROM sqlite_master \
WHERE type = 'table' AND name NOT LIKE 'sqlite_%' \
ORDER BY name LIMIT 4",
)
.optional()
.map_err(|e| format!("sqlite_master schema_info: {}", e))?
.unwrap_or(false);
if !has_info {
.map_err(|e| format!("read sqlite_master: {}", e))?;
let names: Vec<String> = stmt
.query_map([], |r| r.get::<_, String>(0))
.map_err(|e| format!("read sqlite_master: {}", e))?
.filter_map(Result::ok)
.collect();
Ok(format!(
"{}the file is a SQLite database, but not a QuickSearch index \
(it holds {}). Refusing to replace it — point [paths] database_path \
somewhere else, or move that file away first.",
FOREIGN_DB_PREFIX,
if names.is_empty() {
"tables this program does not recognise".to_string()
} else {
names.join(", ")
}
))
}
fn schema_version_current(conn: &Connection) -> Result<bool, String> {
// The shape check rather than a name lookup: a table called `schema_info`
// with other columns belongs to some other program, and reading `value`
// out of it would fail the open with a SQL error instead of the refusal
// the caller can act on.
if !has_our_schema_info(conn) {
return Ok(false);
}
@ -298,6 +440,17 @@ fn schema_version_current(conn: &Connection) -> Result<bool, String> {
/// effective-tokenizer string this caller asked for.
fn db_matches_current(conn: &Connection, tokenizer: &str) -> Result<bool, String> {
if !schema_version_current(conn)? {
// Refuse rather than wipe unless the file is recognisably ours. The
// wipe policy is about replacing an index this program wrote under an
// older layout, and `database_path` is a free-text field with no
// picker and no confirmation — a typo naming some other
// application's SQLite file would otherwise delete it, and its `-wal`
// and `-shm` with it, on the next indexing run. An older layout of
// ours still wipes, and so does a file with no tables at all, which
// is ours to create.
if !is_ours_or_empty(conn)? {
return Err(foreign_database_message(conn)?);
}
return Ok(false);
}

190
crates/quicksearch-core/src/db/open_tests.rs
View file

@ -124,8 +124,8 @@ fn tokenizer_drift_wipes_db() {
let first_effective = {
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \
VALUES ('x', '/x', '/', 0, 0)",
"INSERT INTO files (name, parent, size, mtime) \
VALUES ('x', '/', 0, 0)",
[],
)
.unwrap();
@ -164,8 +164,8 @@ fn open_existing_reads_nondefault_tokenizer_without_wiping() {
{
let conn = open_or_recreate(p.to_str().unwrap(), "unicode61").unwrap();
conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \
VALUES ('note', '/note.txt', '/', 0, 0)",
"INSERT INTO files (name, parent, size, mtime) \
VALUES ('note', '/', 0, 0)",
[],
)
.unwrap();
@ -384,8 +384,8 @@ fn keyed_create_reopen_and_header_is_encrypted() {
{
let conn = open_or_recreate_keyed(p.to_str().unwrap(), "trigram", Some(&key)).unwrap();
conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \
VALUES ('secret', '/secret.txt', '/', 0, 0)",
"INSERT INTO files (name, parent, size, mtime) \
VALUES ('secret', '/', 0, 0)",
[],
)
.unwrap();
@ -418,8 +418,8 @@ fn wrong_key_errors_without_wiping() {
let conn =
open_or_recreate_keyed(p.to_str().unwrap(), "trigram", Some(&test_key(0xa1))).unwrap();
conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \
VALUES ('x', '/x', '/', 0, 0)",
"INSERT INTO files (name, parent, size, mtime) \
VALUES ('x', '/', 0, 0)",
[],
)
.unwrap();
@ -532,15 +532,15 @@ fn open_existing_rw_allows_delete() {
{
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \
VALUES ('a', '/a', '/', 0, 0)",
"INSERT INTO files (name, parent, size, mtime) \
VALUES ('a', '/', 0, 0)",
[],
)
.unwrap();
}
let conn = open_existing(p.to_str().unwrap(), true).unwrap();
let removed = conn
.execute("DELETE FROM files WHERE path = '/a'", [])
.execute("DELETE FROM files WHERE parent = '/' AND name = 'a'", [])
.unwrap();
assert_eq!(removed, 1);
drop(conn);
@ -571,8 +571,8 @@ fn a_fresh_index_and_its_sidecars_are_owner_only() {
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
// A write, so the WAL and SHM exist to be checked.
conn.execute(
"INSERT INTO files (name, path, parent, size, mtime) \
VALUES ('a', '/perm-a', '/', 0, 0)",
"INSERT INTO files (name, parent, size, mtime) \
VALUES ('a', '/', 0, 0)",
[],
)
.unwrap();
@ -598,3 +598,167 @@ fn a_fresh_index_and_its_sidecars_are_owner_only() {
drop(conn);
std::fs::remove_file(&p).ok();
}
/// `maintain` must work on a *keyed* index, which is the one case the pragma
/// it reads does not answer with an integer.
///
/// SQLCipher intercepts `PRAGMA page_size` on a keyed connection, answers with
/// `cipher_page_size`, and hands that back as TEXT. Reading it straight into an
/// `i64` failed there and only there — so every unencrypted test passed while
/// every index with a password set silently skipped both its VACUUM and its
/// `PRAGMA optimize`. The assertion is simply that the call succeeds: it has to
/// get past all three pragma reads to return at all.
#[test]
fn maintain_reads_its_pragmas_on_a_keyed_index() {
let p = tmp_db_path();
let key = test_key(0xc3);
let dir = p.parent().unwrap().to_string_lossy().into_owned();
{
let conn = open_or_recreate_keyed(p.to_str().unwrap(), "trigram", Some(&key)).unwrap();
conn.execute(
"INSERT INTO files (name, parent, size, mtime) VALUES ('x', '/', 0, 0)",
[],
)
.unwrap();
}
let conn = open_keyed_with_pragmas(p.to_str().unwrap(), true, Some(&key), PRAGMAS_MAINTENANCE)
.unwrap();
// A two-row index has no slack worth reclaiming, so `false` is the
// expected answer — what matters is that it is an answer and not an error.
assert_eq!(
crate::db::repo::maintain(&conn, &dir),
Ok(false),
"maintain must not fail on a keyed index"
);
// And the value itself has to be usable, not merely readable: a page size
// that parsed as 0 would size the free-space check at zero bytes and wave
// through a VACUUM that cannot fit.
assert!(
crate::db::repo::pragma_number(&conn, "page_size").unwrap() >= 512,
"a real page size, not a silent zero"
);
drop(conn);
std::fs::remove_file(&p).ok();
}
/// `database_path` is a free-text field in Settings with no picker and no
/// confirmation. A typo naming another application's database used to delete
/// it — and its `-wal` and `-shm` — on the next indexing run, because "no
/// `schema_info` table" was read as "an old index of ours".
#[test]
fn a_foreign_database_is_refused_not_wiped() {
let p = tmp_db_path();
{
let conn = Connection::open(&p).unwrap();
conn.execute(
"CREATE TABLE moz_places (id INTEGER PRIMARY KEY, url TEXT)",
[],
)
.unwrap();
conn.execute(
"INSERT INTO moz_places(url) VALUES('https://example.invalid/')",
[],
)
.unwrap();
}
let err = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap_err();
assert!(err.starts_with(FOREIGN_DB_PREFIX), "got: {err}");
assert!(
err.contains("moz_places"),
"the message must name what is in the way: {err}"
);
// The row is still there — the point of the whole exercise.
let conn = Connection::open(&p).unwrap();
let url: String = conn
.query_row("SELECT url FROM moz_places", [], |r| r.get(0))
.expect("the foreign database must survive intact");
assert_eq!(url, "https://example.invalid/");
drop(conn);
std::fs::remove_file(&p).ok();
}
/// The refusal must not extend to a file that is genuinely ours to create:
/// an empty database is what SQLite leaves behind for a path nothing has
/// written yet.
#[test]
fn an_empty_database_file_is_still_ours_to_build() {
let p = tmp_db_path();
// An empty but real SQLite file, header and all.
drop(Connection::open(&p).unwrap());
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let v: String = conn
.query_row(
"SELECT value FROM schema_info WHERE key='version'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(v, CURRENT_SCHEMA_VERSION.to_string());
drop(conn);
std::fs::remove_file(&p).ok();
}
/// Each pre-`schema_info` table name marks a file as ours, and so is wiped
/// rather than refused — the legacy-layout policy depends on exactly that.
#[test]
fn a_legacy_table_marks_a_file_as_ours() {
for table in LEGACY_TABLES {
let p = tmp_db_path();
{
let conn = Connection::open(&p).unwrap();
conn.execute(&format!("CREATE TABLE {} (x INTEGER)", table), [])
.unwrap();
}
let conn = open_or_recreate(p.to_str().unwrap(), "trigram")
.unwrap_or_else(|e| panic!("{table} should read as ours, got: {e}"));
drop(conn);
std::fs::remove_file(&p).ok();
}
}
/// A `schema_info` with our columns but no version row is an index whose
/// creation was interrupted. It is ours, and rebuilding it is right.
#[test]
fn a_schema_info_without_a_version_row_is_still_ours() {
let p = tmp_db_path();
{
let conn = Connection::open(&p).unwrap();
conn.execute(
"CREATE TABLE schema_info (key TEXT PRIMARY KEY, value TEXT NOT NULL)",
[],
)
.unwrap();
}
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").expect("half-built index is ours");
drop(conn);
std::fs::remove_file(&p).ok();
}
/// A table that merely *shares the name* `schema_info` is not ours. Matching
/// on the name alone used to fail the open with a raw SQL error about a
/// missing column, which is neither a refusal the caller can act on nor a
/// message anyone could act on either.
#[test]
fn a_foreign_schema_info_is_refused() {
let p = tmp_db_path();
{
let conn = Connection::open(&p).unwrap();
conn.execute("CREATE TABLE schema_info (revision INTEGER)", [])
.unwrap();
conn.execute("INSERT INTO schema_info(revision) VALUES(3)", [])
.unwrap();
}
let err = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap_err();
assert!(err.starts_with(FOREIGN_DB_PREFIX), "got: {err}");
let conn = Connection::open(&p).unwrap();
let revision: i64 = conn
.query_row("SELECT revision FROM schema_info", [], |r| r.get(0))
.expect("the foreign database must survive intact");
assert_eq!(revision, 3);
drop(conn);
std::fs::remove_file(&p).ok();
}

321
crates/quicksearch-core/src/db/repo.rs
View file

@ -62,7 +62,8 @@ fn set_state_clearing_failure(
#[derive(Debug, Clone)]
pub struct NewFile<'a> {
pub name: &'a str,
pub path: &'a str,
/// The containing directory, ending in the platform separator — see
/// [`crate::file_handling::split_db_path`], which produces the pair.
pub parent: &'a str,
pub size: u64,
pub mtime: u64,
@ -74,21 +75,29 @@ pub struct NewFile<'a> {
pub needs_content: bool,
}
impl NewFile<'_> {
/// The file's path, for a log or error message. Not stored; see
/// [`super::schema::SCHEMA_CURRENT`].
fn path(&self) -> String {
format!("{}{}", self.parent, self.name)
}
}
/// Insert a new file row, returning its id. `content_state` comes from
/// `needs_content`; there is no separate basic state, because the row
/// existing *is* the basic-index state. `INSERT OR IGNORE`: a UNIQUE(path)
/// collision returns `None` rather than aborting the batch.
/// existing *is* the basic-index state. `INSERT OR IGNORE`: a
/// `UNIQUE(parent, name)` collision returns `None` rather than aborting the
/// batch.
pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option<i64>, String> {
let rows = tx
.prepare_cached(
"INSERT OR IGNORE INTO files (
name, path, parent, size, mtime, mime, type, content_state, hash
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
name, parent, size, mtime, mime, type, content_state, hash
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
)
.and_then(|mut stmt| {
stmt.execute(params![
f.name,
f.path,
f.parent,
f.size as i64,
f.mtime as i64,
@ -98,7 +107,7 @@ pub fn insert_file(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option<i64>,
f.hash,
])
})
.map_err(|e| format!("insert file {}: {}", f.path, e))?;
.map_err(|e| format!("insert file {}: {}", f.path(), e))?;
if rows == 0 {
return Ok(None);
}
@ -120,14 +129,14 @@ fn initial_content_state(f: &NewFile<'_>) -> i64 {
/// reset its content state from `f.needs_content`, clearing any extracted
/// content so the text-indexing pass re-processes it. Writes `size`, `mtime`,
/// `hash`, `mime`, `type` and `content_state` — and only those; `name` and
/// `parent` are not refreshed here.
/// `parent` are the key it matches on, so they cannot change here.
pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option<i64>, String> {
let id: Option<i64> = tx
.prepare_cached(
"UPDATE files
SET size = ?1, mtime = ?2, hash = ?3, mime = ?4, type = ?5,
content_state = ?6
WHERE path = ?7
WHERE parent = ?7 AND name = ?8
RETURNING id",
)
.and_then(|mut stmt| {
@ -139,13 +148,14 @@ pub fn update_file_basic(tx: &Transaction<'_>, f: &NewFile<'_>) -> Result<Option
f.mime,
f.ftype.bits() as i64,
initial_content_state(f),
f.path,
f.parent,
f.name,
],
|r| r.get(0),
)
.optional()
})
.map_err(|e| format!("update file {}: {}", f.path, e))?;
.map_err(|e| format!("update file {}: {}", f.path(), e))?;
let Some(id) = id else {
return Ok(None);
};
@ -174,7 +184,38 @@ pub fn set_content_done(
text_zstd: Option<&[u8]>,
) -> Result<(), String> {
remove_content_for_id(tx, file_id)?;
set_content_done_fresh(tx, file_id, text, text_zstd)
}
/// [`set_content_done`] for a row that **provably holds no content yet**,
/// skipping the pre-delete.
///
/// The two statements `remove_content_for_id` issues are not free: one is a
/// tombstoning delete on a contentless FTS5 table, and both run inside the
/// writer's transaction with the shared connection held. For a row that cannot
/// have content they are pure overhead, and the indexer hit them on nearly
/// every file it wrote:
///
/// * a row `insert_file` has just created has no `searchabletext` or
/// `documents_text` row by construction;
/// * a row [`update_file_basic`] has just written has had its content removed
/// **by that call**, so `set_content_done` was deleting it a second time.
///
/// Both paths run for every file small enough for the walk to extract inline
/// (`hash_length`, 8 KiB by default), which on a source tree is most of it.
///
/// Use [`set_content_done`] wherever the row's prior state is not known — the
/// content pass writing a row that may have been extracted before, the watcher,
/// anything reached from a config reconcile. Getting this wrong leaves a
/// duplicate FTS entry rather than a visible error, so the rule is: skip the
/// delete only where the *same transaction* has already established there is
/// nothing there.
pub fn set_content_done_fresh(
tx: &Transaction<'_>,
file_id: i64,
text: &str,
text_zstd: Option<&[u8]>,
) -> Result<(), String> {
// Contentless FTS5 still accepts values on INSERT — the tokenizer needs
// them — it simply doesn't persist the raw column values.
exec(
@ -255,7 +296,15 @@ impl DocDecoder {
// settles at the largest document in the scan within the first few
// rows, after which decoding a row allocates nothing at all.
if let Ok(Some(size)) = zstd::zstd_safe::get_frame_content_size(blob) {
self.buf.reserve(usize::try_from(size).ok()?);
// Clamped: `size` is a `u64` read straight out of the frame
// header, so a corrupt or hostile blob can ask for terabytes here
// and `reserve` answers an impossible request by aborting the
// process, not by failing. The doubling loop below already treats
// `MAX_DOC_CAPACITY` as the point past which a frame is corrupt
// rather than merely large, so a clamped reservation that then
// fails to decompress returns `None` through the same path.
self.buf
.reserve(usize::try_from(size).ok()?.min(MAX_DOC_CAPACITY));
}
loop {
if self.buf.capacity() == 0 {
@ -365,29 +414,38 @@ pub fn set_content_na(tx: &Transaction<'_>, file_id: i64) -> Result<(), String>
}
/// Delete a file row by path, keeping FTS in sync. Returns whether a row was
/// removed.
/// removed — including `false` for a string that cannot be a file's path at
/// all, which is not in the index by construction.
pub fn delete_file_by_path(tx: &Transaction<'_>, path: &str) -> Result<bool, String> {
let Some((parent, name)) = crate::file_handling::split_db_path(path) else {
return Ok(false);
};
let id: Option<i64> = tx
.prepare_cached("DELETE FROM files WHERE path = ?1 RETURNING id")
.and_then(|mut stmt| stmt.query_row(params![path], |r| r.get(0)).optional())
.prepare_cached("DELETE FROM files WHERE parent = ?1 AND name = ?2 RETURNING id")
.and_then(|mut stmt| {
stmt.query_row(params![parent, name], |r| r.get(0))
.optional()
})
.map_err(|e| format!("delete file {}: {}", path, e))?;
let Some(id) = id else { return Ok(false) };
remove_content_for_id(tx, id)?;
Ok(true)
}
/// Delete every row whose path falls in the half-open range `[lo, hi)`,
/// Delete every row whose parent falls in the half-open range `[lo, hi)`,
/// keeping the dependent tables in step. Returns how many `files` rows went.
///
/// Five statements regardless of how many files the range holds, and the
/// range is an index seek on `UNIQUE(files.path)`. Build the bounds with
/// [`crate::file_handling::ExtractCursor::for_root`], which is what makes
/// them separator-correct.
/// Four statements regardless of how many files the range holds, and the
/// range is an index seek on `UNIQUE(files.parent, files.name)`. Build the
/// bounds with [`crate::file_handling::ExtractCursor::for_root`], which is
/// what makes them separator-correct — and note the range covers the root's
/// *own* files only because every stored parent ends in a separator (see
/// `dir_to_db_parent`).
pub fn delete_subtree(tx: &Transaction<'_>, lo: &str, hi: &str) -> Result<usize, String> {
for (table, key) in DEPENDENT_TABLES {
let sql = format!(
"DELETE FROM {} WHERE {} IN \
(SELECT id FROM files WHERE path >= ?1 AND path < ?2)",
(SELECT id FROM files WHERE parent >= ?1 AND parent < ?2)",
table, key
);
exec(tx, &sql, params![lo, hi], || {
@ -396,13 +454,13 @@ pub fn delete_subtree(tx: &Transaction<'_>, lo: &str, hi: &str) -> Result<usize,
}
exec(
tx,
"DELETE FROM files WHERE path >= ?1 AND path < ?2",
"DELETE FROM files WHERE parent >= ?1 AND parent < ?2",
params![lo, hi],
|| format!("delete files under {}", lo),
)
}
/// Delete every row whose path falls in *none* of `ranges`. Returns how many
/// Delete every row whose parent falls in *none* of `ranges`. Returns how many
/// `files` rows went.
///
/// A scan of `files` rather than a seek, reserved for the one transition that
@ -423,7 +481,7 @@ pub fn delete_outside_ranges(
predicate.push_str(" AND ");
}
predicate.push_str(&format!(
"NOT (path >= ?{} AND path < ?{})",
"NOT (parent >= ?{} AND parent < ?{})",
i * 2 + 1,
i * 2 + 2
));
@ -485,8 +543,12 @@ pub fn delete_ids(tx: &Transaction<'_>, ids: &[i64]) -> Result<usize, String> {
Ok(removed)
}
/// Every indexed file directly inside `parent`, as `name -> mtime`. Served by
/// `idx_files_parent`: one index range lookup.
/// Every indexed file directly inside `parent`, as `name -> mtime`.
///
/// `parent` must be in stored spelling — trailing separator and all; build it
/// with [`crate::file_handling::dir_to_db_parent`]. One `idx_files_parent`
/// range lookup for the names, then a row fetch each for the mtimes; see the
/// index's own comment for why that is the shape it is.
pub fn dir_rows(
conn: &Connection,
parent: &str,
@ -507,7 +569,9 @@ pub fn dir_rows(
Ok(out)
}
/// A row the content pass has yet to extract: `(id, name, path, mime)`.
/// A row the content pass has yet to extract: `(id, name, path, mime)`. The
/// path is reassembled here rather than stored — see
/// [`super::schema::SCHEMA_CURRENT`] — because the pass opens the file by it.
pub type PendingContentRow = (i64, String, String, Option<String>);
/// One page of rows still awaiting content extraction under `cursor`'s range,
@ -526,11 +590,11 @@ pub fn pending_content_page(
.prepare_cached(
// `INDEXED BY` rather than a hint, because the planner gets this
// one wrong exactly when it costs most. Left to itself it takes
// `UNIQUE(path)` for the range and then sorts the survivors into a
// temp b-tree to satisfy `ORDER BY id` — which means every page
// walks the whole root's range and fetches each row's heap entry
// to test `content_state`. At `FEED_PAGE` rows per page that is
// quadratic over a run. The partial index below is already
// `idx_files_parent` for the range and then sorts the survivors
// into a temp b-tree to satisfy `ORDER BY id` — which means every
// page walks the whole root's range and fetches each row's heap
// entry to test `content_state`. At `FEED_PAGE` rows per page that
// is quadratic over a run. The partial index below is already
// id-ordered, so it answers `id > ?` and the ORDER BY together and
// holds only pending rows. Measured on 500k rows with everything
// pending — the first index of a tree, i.e. the case that matters:
@ -539,9 +603,9 @@ pub fn pending_content_page(
// The planner only prefers it once pending rows are a small
// minority, and never before ANALYZE has run at all, which is why
// this cannot be left to statistics.
"SELECT id, name, path, mime FROM files INDEXED BY idx_files_content_pending
"SELECT id, parent, name, mime FROM files INDEXED BY idx_files_content_pending
WHERE content_state = 0 AND size <= ?1 AND id > ?2
AND path >= ?3 AND path < ?4
AND parent >= ?3 AND parent < ?4
ORDER BY id
LIMIT ?5",
)
@ -550,10 +614,13 @@ pub fn pending_content_page(
.query_map(
params![max_size, cursor.last_id, cursor.lo, cursor.hi, limit],
|row| {
let parent: String = row.get(1)?;
let name: String = row.get(2)?;
let path = format!("{}{}", parent, name);
Ok((
row.get::<_, i64>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
name,
path,
row.get::<_, Option<String>>(3)?,
))
},
@ -565,10 +632,16 @@ pub fn pending_content_page(
/// A stored row as the scope reconciler sees it: enough to decide both
/// whether the path is still in scope and whether its content still is.
///
/// `path` is `parent` and `name` joined, kept alongside them because the
/// reconciler tests it as a [`std::path::Path`] while the cursor resumes from
/// the key.
#[derive(Debug, Clone)]
pub struct ScopeRow {
pub id: i64,
pub path: String,
pub parent: String,
pub name: String,
pub size: u64,
pub mime: Option<String>,
pub content_state: i64,
@ -590,7 +663,7 @@ pub struct RootCounts {
pub fts: i64,
}
/// Count the rows in the half-open path range `[lo, hi)` and, in the same
/// Count the rows in the half-open parent range `[lo, hi)` and, in the same
/// pass, how many of them have a full-text row.
///
/// `content_state = STATE_DONE` *is* "has a `searchabletext` row":
@ -600,13 +673,13 @@ pub struct RootCounts {
/// contentless and keyed by `rowid`, so it has no path to range-scan on.
///
/// One statement, but not a cheap one: `content_state` is not carried by the
/// `UNIQUE(files.path)` index the range seeks on, so every row in the range is
/// `idx_files_parent` index the range seeks on, so every row in the range is
/// fetched. Call it where a run has just read those rows anyway, not on a
/// cadence.
pub fn count_root(conn: &Connection, lo: &str, hi: &str) -> Result<RootCounts, String> {
conn.prepare_cached(
"SELECT COUNT(*), COALESCE(SUM(content_state = ?3), 0) FROM files
WHERE path >= ?1 AND path < ?2",
WHERE parent >= ?1 AND parent < ?2",
)
.and_then(|mut stmt| {
stmt.query_row(params![lo, hi, STATE_DONE], |r| {
@ -619,37 +692,46 @@ pub fn count_root(conn: &Connection, lo: &str, hi: &str) -> Result<RootCounts, S
.map_err(|e| format!("count root {}: {}", lo, e))
}
/// One page of rows whose path is `> after` and `< hi`, in path order.
/// One page of rows sorting after `(after_parent, after_name)` and inside the
/// parent range ending at `hi`, in `(parent, name)` order.
///
/// Keyset on `path`: every page is an index walk with no sort step, and a row
/// is served at most once even though the caller is deleting behind the
/// reader. Seed `after` with the range's `lo` bound, which is
/// `root + separator` and so can never equal a stored path.
/// Keyset on the `idx_files_parent` key itself: every page is an index walk
/// with no sort step, and a row is served at most once even though the caller
/// is deleting behind the reader. Seed the cursor with `(lo, "")` — no name is
/// empty, so that lands exactly on the first row of the range.
///
/// The row-value comparison is what keeps it one seek; spelled out as
/// `parent > ? OR (parent = ? AND name > ?)` the planner is free to scan.
pub fn rows_in_range_page(
conn: &Connection,
after: &str,
after_parent: &str,
after_name: &str,
hi: &str,
limit: i64,
) -> Result<Vec<ScopeRow>, String> {
let mut stmt = conn
.prepare_cached(
"SELECT id, path, size, mime, content_state FROM files
WHERE path > ?1 AND path < ?2
ORDER BY path
LIMIT ?3",
"SELECT id, parent, name, size, mime, content_state FROM files
WHERE (parent, name) > (?1, ?2) AND parent < ?3
ORDER BY parent, name
LIMIT ?4",
)
.map_err(|e| format!("prepare range page: {}", e))?;
let rows = stmt
.query_map(params![after, hi, limit], |row| {
.query_map(params![after_parent, after_name, hi, limit], |row| {
let parent: String = row.get(1)?;
let name: String = row.get(2)?;
Ok(ScopeRow {
id: row.get(0)?,
path: row.get(1)?,
size: row.get::<_, i64>(2)?.max(0) as u64,
mime: row.get(3)?,
content_state: row.get(4)?,
path: format!("{}{}", parent, name),
name,
parent,
size: row.get::<_, i64>(3)?.max(0) as u64,
mime: row.get(4)?,
content_state: row.get(5)?,
})
})
.map_err(|e| format!("query range page after {}: {}", after, e))?;
.map_err(|e| format!("query range page after {}: {}", after_parent, e))?;
rows.collect::<Result<Vec<_>, _>>()
.map_err(|e| format!("read range page row: {}", e))
}
@ -683,18 +765,26 @@ pub fn reset_content_pending(tx: &Transaction<'_>, file_id: i64) -> Result<(), S
/// files whose parent isn't the directory being read (a resolved symlink
/// target), where [`dir_rows`] would not have them.
pub fn mtime_for_path(conn: &Connection, path: &str) -> Result<Option<u64>, String> {
let Some((parent, name)) = crate::file_handling::split_db_path(path) else {
return Ok(None);
};
let mut stmt = conn
.prepare_cached("SELECT mtime FROM files WHERE path = ?1")
.prepare_cached("SELECT mtime FROM files WHERE parent = ?1 AND name = ?2")
.map_err(|e| format!("prepare mtime lookup for {}: {}", path, e))?;
stmt.query_row(params![path], |r| r.get::<_, i64>(0))
stmt.query_row(params![parent, name], |r| r.get::<_, i64>(0))
.optional()
.map(|o| o.map(|m| m.max(0) as u64))
.map_err(|e| format!("mtime lookup for {}: {}", path, e))
}
/// Distinct `parent` values within the half-open path range `[lo, hi)`,
/// streamed to `f` so nothing proportional to the tree is materialized.
/// `idx_files_parent` makes this an index-only scan.
/// Distinct `parent` values within the half-open range `[lo, hi)`, streamed to
/// `f` so nothing proportional to the tree is materialized. `idx_files_parent`
/// makes this an index-only scan.
///
/// The root's own directory is included: its stored parent is `root + SEP`,
/// which is exactly `lo`. It was not, back when the same bounds were applied
/// to a `path` column and the root's parent was spelled without the trailing
/// separator.
pub fn for_each_parent_in_range<F: FnMut(String)>(
conn: &Connection,
lo: &str,
@ -713,13 +803,17 @@ pub fn for_each_parent_in_range<F: FnMut(String)>(
Ok(())
}
/// Paths of every file directly inside `parent`.
/// Paths of every file directly inside `parent`, which must carry its trailing
/// separator — so the join below is a concatenation.
pub fn paths_in_dir(conn: &Connection, parent: &str) -> Result<Vec<String>, String> {
let mut stmt = conn
.prepare_cached("SELECT path FROM files WHERE parent = ?1")
.prepare_cached("SELECT name FROM files WHERE parent = ?1")
.map_err(|e| format!("prepare paths in {}: {}", parent, e))?;
let rows = stmt
.query_map(params![parent], |r| r.get::<_, String>(0))
.query_map(params![parent], |r| {
r.get::<_, String>(0)
.map(|name| format!("{}{}", parent, name))
})
.map_err(|e| format!("query paths in {}: {}", parent, e))?;
rows.collect::<Result<Vec<_>, _>>()
.map_err(|e| format!("read path under {}: {}", parent, e))
@ -795,6 +889,33 @@ pub fn checkpoint_and_close(conn: Connection) {
drop(conn);
}
/// Read a `PRAGMA` that reports a number.
///
/// Not simply `r.get::<i64>(0)`, because SQLCipher does not always answer with
/// one. On a **keyed** connection it intercepts `PRAGMA page_size`, answers
/// with `cipher_page_size` instead, and returns that as TEXT — so asking for an
/// integer fails with a type error. Unencrypted it is an INTEGER as usual,
/// which is why this only ever broke protected installs, and only in
/// [`maintain`]: every index with a password set skipped its VACUUM *and* its
/// `PRAGMA optimize` from the moment the free-space check was added.
///
/// A value that is neither is an error rather than a guess — the callers here
/// size a disk-space check with it.
pub(super) fn pragma_number(conn: &Connection, pragma: &str) -> Result<i64, String> {
use rusqlite::types::ValueRef;
conn.query_row(&format!("PRAGMA {}", pragma), [], |r| {
Ok(match r.get_ref(0)? {
ValueRef::Integer(n) => Some(n),
ValueRef::Text(t) => std::str::from_utf8(t)
.ok()
.and_then(|s| s.trim().parse().ok()),
_ => None,
})
})
.map_err(|e| format!("read {}: {}", pragma, e))?
.ok_or_else(|| format!("read {}: not a number", pragma))
}
/// Land the log, reclaim the file's slack, and refresh the query planner's
/// statistics. Returns whether it vacuumed.
///
@ -822,14 +943,33 @@ pub fn maintain(conn: &Connection, db_dir: &str) -> Result<bool, String> {
crate::log_warn!("{}", e);
}
let page_count: i64 = conn
.query_row("PRAGMA page_count", [], |r| r.get(0))
.map_err(|e| format!("read page_count: {}", e))?;
let freelist: i64 = conn
.query_row("PRAGMA freelist_count", [], |r| r.get(0))
.map_err(|e| format!("read freelist_count: {}", e))?;
let page_count = pragma_number(conn, "page_count")?;
let freelist = pragma_number(conn, "freelist_count")?;
let vacuumed = freelist * 100 >= page_count * VACUUM_MIN_SLACK_PERCENT;
let worth_it = freelist * 100 >= page_count * VACUUM_MIN_SLACK_PERCENT;
// The doc comment above puts VACUUM's peak transient need at roughly three
// times the index. Checking first turns "the volume filled up mid-rebuild"
// into a skipped compaction: a rollback is the *good* outcome there, and
// the bad one is that writes to the `-shm` mmap on a full filesystem come
// back as SIGBUS rather than as an error — see
// `indexing::pipeline::DISK_FLOOR`. Unknown free space is not a reason to
// skip.
let page_size = pragma_number(conn, "page_size")?;
let needed = (page_count.max(0) as u64).saturating_mul(page_size.max(0) as u64) * 3;
let room = match crate::platform::available_space(std::path::Path::new(db_dir)) {
Some(free) if free < needed => {
crate::log_warn!(
"skipping VACUUM: it needs about {} MiB free in {} and there is {} MiB",
needed / (1024 * 1024),
db_dir,
free / (1024 * 1024)
);
false
}
_ => true,
};
let vacuumed = worth_it && room;
if vacuumed {
// `temp_store_directory` is a deprecated pragma that writes a global,
// so it is set for the VACUUM and cleared straight after rather than
@ -848,11 +988,52 @@ pub fn maintain(conn: &Connection, db_dir: &str) -> Result<bool, String> {
// matter, so it is close to free on a run that changed little.
conn.execute_batch("PRAGMA optimize;")
.map_err(|e| format!("optimize: {}", e))?;
note_optimized(db_dir);
checkpoint_truncate(conn)?;
Ok(vacuumed)
}
/// How many times `PRAGMA optimize` has been accepted by SQLite, per index
/// directory.
///
/// Keyed by directory rather than counted globally because the test binaries
/// run their `#[test]` functions on concurrent threads against separate scratch
/// indexes; a single counter would let one test's maintenance satisfy another
/// test's assertion. Bounded by the number of distinct indexes a process opens,
/// which outside the tests is one.
static OPTIMIZED: std::sync::LazyLock<std::sync::Mutex<std::collections::HashMap<String, u64>>> =
std::sync::LazyLock::new(|| std::sync::Mutex::new(std::collections::HashMap::new()));
/// Record that the pragma above returned `Ok` for `db_dir`.
fn note_optimized(db_dir: &str) {
*crate::lock_ok(&OPTIMIZED)
.entry(db_dir.to_string())
.or_insert(0) += 1;
}
/// How many times the index in `db_dir` has had `PRAGMA optimize` run against
/// it in this process.
///
/// Exists so a test can ask *whether the optimize pass happened* rather than
/// trying to catch [`crate::indexing::IndexingStatus::Optimizing`] as it goes
/// past. That distinction is the whole point: the status is a sample of a
/// transient state, and the faster the pass gets the less often a poller sees
/// it — so a test written that way fails more as the code improves, which is
/// exactly backwards. This is a latch, so it cannot be missed however quickly
/// the pass runs.
///
/// It records that the statement was handed to SQLite and accepted, not what
/// SQLite then chose to do with it — `PRAGMA optimize` is deliberately a no-op
/// when no table has drifted far enough to be worth re-analysing, so there is
/// no observable side effect to assert on instead.
pub fn optimize_count(db_dir: &str) -> u64 {
crate::lock_ok(&OPTIMIZED)
.get(db_dir)
.copied()
.unwrap_or(0)
}
/// Read the `last_full_index` marker (unix seconds of the last *successful*
/// full indexing run) from `schema_info`. Absent key — fresh DB, or a DB
/// from before this marker existed — means "never".

224
crates/quicksearch-core/src/db/repo_tests.rs
View file

@ -9,6 +9,77 @@ fn tmp_path() -> std::path::PathBuf {
crate::testutil::scratch_dir("repo").join("index.sqlite")
}
/// `set_content_done_fresh` skips the pre-delete, so the whole of its safety is
/// the caller's claim that the row is clean. This pins both halves of that:
/// the fast path leaves exactly one FTS row where it is used correctly, and the
/// ordinary entry point still repairs a row that *does* carry content.
///
/// A duplicate `searchabletext` row is the failure this guards. It surfaces as
/// a file appearing twice in full-text results, not as an error, so nothing
/// else in the suite would necessarily notice.
#[test]
fn the_fresh_content_write_leaves_exactly_one_fts_row() {
fn new_file(name: &str, mtime: u64) -> NewFile<'_> {
NewFile {
name,
parent: "/d/",
size: 10,
mtime,
mime: Some("text/plain"),
ftype: crate::mime::FileType::TEXT,
hash: None,
needs_content: true,
}
}
fn count(conn: &rusqlite::Connection, sql: &str) -> i64 {
conn.query_row(sql, [], |r| r.get(0)).unwrap()
}
fn matches(conn: &rusqlite::Connection, term: &str) -> i64 {
conn.query_row(
"SELECT COUNT(*) FROM searchabletext WHERE searchabletext MATCH ?1",
[term],
|r| r.get(0),
)
.unwrap()
}
let path = tmp_path();
let mut conn = open_or_recreate(path.to_str().unwrap(), "trigram").unwrap();
let tx = conn.transaction().unwrap();
// The insert path: a row that has never had content.
let id = insert_file(&tx, &new_file("a.txt", 1)).unwrap().unwrap();
set_content_done_fresh(&tx, id, "sphinx quartz", zstd_of("sphinx quartz").as_deref()).unwrap();
// The update path: `update_file_basic` clears the content, so the fresh
// write that follows is writing into an empty slot — the exact sequence
// `process_batch_updates` performs.
let same = update_file_basic(&tx, &new_file("a.txt", 2)).unwrap().unwrap();
assert_eq!(same, id, "the same row");
set_content_done_fresh(&tx, id, "sphinx onyx", zstd_of("sphinx onyx").as_deref()).unwrap();
tx.commit().unwrap();
assert_eq!(count(&conn, "SELECT COUNT(*) FROM searchabletext"), 1);
assert_eq!(count(&conn, "SELECT COUNT(*) FROM documents_text"), 1);
assert_eq!(matches(&conn, "onyx"), 1, "the new body is searchable");
assert_eq!(matches(&conn, "quartz"), 0, "the old body is gone");
// And the idempotent entry point still repairs a row that really does hold
// content — writing over the top of a DONE row with no update first, which
// is what the content pass and the watcher do.
let tx = conn.transaction().unwrap();
set_content_done(&tx, id, "sphinx jasper", zstd_of("sphinx jasper").as_deref()).unwrap();
tx.commit().unwrap();
assert_eq!(count(&conn, "SELECT COUNT(*) FROM searchabletext"), 1);
assert_eq!(count(&conn, "SELECT COUNT(*) FROM documents_text"), 1);
assert_eq!(matches(&conn, "jasper"), 1);
assert_eq!(matches(&conn, "onyx"), 0);
drop(conn);
std::fs::remove_file(&path).ok();
}
/// The size report reads each body's uncompressed length out of its zstd
/// frame header instead of from a stored column, which works only because
/// [`DocEncoder`] compresses through `ZSTD_compress2` — the API that is
@ -47,8 +118,7 @@ fn insert_update_delete_round_trip() {
&tx,
&NewFile {
name: "a.txt",
path: "/tmp/a.txt",
parent: "/tmp",
parent: "/tmp/",
size: 42,
mtime: 1_700_000_000,
mime: Some("text/plain"),
@ -101,8 +171,7 @@ fn insert_writes_content_state_from_needs_content() {
let tx = conn.transaction().unwrap();
let mut row = NewFile {
name: "claimed.txt",
path: "/tmp/claimed.txt",
parent: "/tmp",
parent: "/tmp/",
size: 1,
mtime: 1,
mime: Some("text/plain"),
@ -112,7 +181,6 @@ fn insert_writes_content_state_from_needs_content() {
};
let claimed = insert_file(&tx, &row).unwrap().expect("unique path");
row.name = "unclaimed.mp4";
row.path = "/tmp/unclaimed.mp4";
row.mime = Some("video/mp4");
row.needs_content = false;
let unclaimed = insert_file(&tx, &row).unwrap().expect("unique path");
@ -139,8 +207,7 @@ fn update_writes_content_state_from_needs_content() {
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let mut row = NewFile {
name: "a.txt",
path: "/tmp/a.txt",
parent: "/tmp",
parent: "/tmp/",
size: 10,
mtime: 1,
mime: None,
@ -216,8 +283,7 @@ fn insert_file_twice_on_same_path_is_idempotent() {
let tx = conn.transaction().unwrap();
let row = NewFile {
name: "dup.txt",
path: "/tmp/dup.txt",
parent: "/tmp",
parent: "/tmp/",
size: 1,
mtime: 1,
mime: Some("text/plain"),
@ -234,8 +300,8 @@ fn insert_file_twice_on_same_path_is_idempotent() {
assert_eq!(count, 1);
let (id_read,): (i64,) = tx
.query_row(
"SELECT id FROM files WHERE path = ?1",
params!["/tmp/dup.txt"],
"SELECT id FROM files WHERE parent = ?1 AND name = ?2",
params!["/tmp/", "dup.txt"],
|r| Ok((r.get(0)?,)),
)
.unwrap();
@ -250,13 +316,11 @@ fn delete_subtree_clears_every_dependent_table() {
let p = tmp_path();
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let add = |tx: &Transaction<'_>, path: &str| -> i64 {
let name = path.rsplit('/').next().unwrap();
let parent = &path[..path.rfind('/').unwrap()];
let (parent, name) = crate::file_handling::split_db_path(path).expect("a file's path");
let id = insert_file(
tx,
&NewFile {
name,
path,
parent,
size: 1,
mtime: 1,
@ -306,7 +370,7 @@ fn delete_subtree_clears_every_dependent_table() {
let survivors: Vec<String> = {
let mut stmt = conn
.prepare("SELECT path FROM files ORDER BY path")
.prepare("SELECT parent || name FROM files ORDER BY parent, name")
.unwrap();
let v = stmt
.query_map([], |r| r.get::<_, String>(0))
@ -327,13 +391,12 @@ fn seeded(conn: &mut Connection, paths: &[&str]) -> std::collections::HashMap<St
let tx = conn.transaction().unwrap();
let mut ids = std::collections::HashMap::new();
for path in paths {
let name = path.rsplit('/').next().unwrap();
let parent = &path[..path.rfind('/').unwrap()];
// The indexer's own split, so the separator stays with the parent.
let (parent, name) = crate::file_handling::split_db_path(path).expect("a file's path");
let id = insert_file(
&tx,
&NewFile {
name,
path,
parent,
size: 1,
mtime: 1,
@ -387,7 +450,7 @@ fn delete_outside_ranges_keeps_exactly_the_configured_roots() {
let survivors: Vec<String> = {
let mut stmt = conn
.prepare("SELECT path FROM files ORDER BY path")
.prepare("SELECT parent || name FROM files ORDER BY parent, name")
.unwrap();
let v = stmt
.query_map([], |r| r.get::<_, String>(0))
@ -607,40 +670,47 @@ fn rows_in_range_page_walks_the_range_once() {
let range = crate::file_handling::ExtractCursor::for_root("/t");
let mut seen = Vec::new();
let mut after = range.lo.clone();
// `(lo, "")`: no stored name is empty, so this sorts just below the
// range's first row.
let mut after = (range.lo.clone(), String::new());
loop {
let page = rows_in_range_page(&conn, &after, &range.hi, 2).unwrap();
let page = rows_in_range_page(&conn, &after.0, &after.1, &range.hi, 2).unwrap();
let Some(last) = page.last() else { break };
after = last.path.clone();
after = (last.parent.clone(), last.name.clone());
seen.extend(page.into_iter().map(|r| r.path));
}
assert_eq!(
seen,
vec!["/t/a.txt", "/t/deep/b.txt", "/t/deep/deeper/c.txt"],
"in path order, once each, and the prefix siblings are outside"
"in (parent, name) order, once each, and the prefix siblings are outside"
);
drop(conn);
std::fs::remove_file(&p).ok();
}
/// `idx_files_parent` carries `name` and `mtime` so `dir_rows` never touches
/// the table heap; trimming it back to `(parent)` would silently reintroduce
/// a row fetch per entry.
/// `dir_rows` must *seek* on `idx_files_parent`, never scan the table.
///
/// It is deliberately not index-only any more: the index stopped carrying
/// `mtime` when `UNIQUE(parent, name)` took over as the row key, so the plan
/// is a seek plus a row fetch per entry. See the index's own comment in
/// `schema.rs` for why that trade was taken — and note the thing this test
/// guards is the seek, which is what keeps the cost per *directory* rather
/// than per *tree*.
#[test]
fn dir_rows_is_served_entirely_from_the_index() {
fn dir_rows_seeks_the_parent_index() {
let p = tmp_path();
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let plan: String = conn
.query_row(
"EXPLAIN QUERY PLAN SELECT name, mtime FROM files WHERE parent = ?1",
params!["/some/dir"],
params!["/some/dir/"],
|r| r.get(3),
)
.unwrap();
assert!(
plan.contains("COVERING INDEX idx_files_parent"),
"dir_rows must be index-only, got: {}",
plan.contains("SEARCH") && plan.contains("idx_files_parent"),
"dir_rows must seek the parent index, got: {}",
plan
);
drop(conn);
@ -655,7 +725,7 @@ fn the_subtree_range_is_an_index_seek_not_a_scan() {
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let plan: String = conn
.query_row(
"EXPLAIN QUERY PLAN DELETE FROM files WHERE path >= ?1 AND path < ?2",
"EXPLAIN QUERY PLAN DELETE FROM files WHERE parent >= ?1 AND parent < ?2",
params!["/tree/", "/tree0"],
|r| r.get(3),
)
@ -669,6 +739,43 @@ fn the_subtree_range_is_an_index_seek_not_a_scan() {
std::fs::remove_file(&p).ok();
}
/// The keyset page must be one index walk: the row-value comparison against
/// `(parent, name)` is what lets SQLite seek straight to the cursor and read
/// forward, with no temp b-tree to satisfy the `ORDER BY`.
#[test]
fn the_reconcile_page_seeks_and_does_not_sort() {
let p = tmp_path();
let conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let mut stmt = conn
.prepare(
"EXPLAIN QUERY PLAN
SELECT id, parent, name, size, mime, content_state FROM files
WHERE (parent, name) > (?1, ?2) AND parent < ?3
ORDER BY parent, name
LIMIT ?4",
)
.unwrap();
let plan: Vec<String> = stmt
.query_map(params!["/tree/", "", "/tree0", 2], |r| r.get(3))
.unwrap()
.map(|r| r.unwrap())
.collect();
let plan = plan.join(" | ");
assert!(
plan.contains("SEARCH") && plan.contains("idx_files_parent"),
"keyset page must seek the parent index, got: {}",
plan
);
assert!(
!plan.contains("TEMP B-TREE"),
"the index order must satisfy the ORDER BY outright, got: {}",
plan
);
drop(stmt);
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn last_full_index_round_trip() {
let p = tmp_path();
@ -693,8 +800,7 @@ fn checkpoint_and_close_truncates_wal() {
&tx,
&NewFile {
name: "w.txt",
path: "/tmp/w.txt",
parent: "/tmp",
parent: "/tmp/",
size: 1,
mtime: 1,
mime: None,
@ -725,14 +831,12 @@ fn checkpoint_and_close_truncates_wal() {
fn seed_rows(conn: &mut Connection, range: std::ops::Range<usize>) {
let tx = conn.transaction().unwrap();
for i in range {
let path = format!("/tmp/bulk/{}.txt", i);
let name = format!("{}.txt", i);
let id = insert_file(
&tx,
&NewFile {
name: &name,
path: &path,
parent: "/tmp/bulk",
parent: "/tmp/bulk/",
size: 1,
mtime: 1,
mime: Some("text/plain"),
@ -815,14 +919,12 @@ fn a_busy_reader_defeats_the_autocheckpoint_but_not_a_forced_one() {
fn seed_bare(conn: &mut Connection, range: std::ops::Range<usize>) {
let tx = conn.transaction().unwrap();
for i in range {
let path = format!("/tmp/bare/{}.txt", i);
let name = format!("{}.txt", i);
insert_file(
&tx,
&NewFile {
name: &name,
path: &path,
parent: "/tmp/bare",
parent: "/tmp/bare/",
size: i as u64,
mtime: 1,
mime: None,
@ -979,8 +1081,7 @@ fn set_content_failed_writes_failed_table() {
&tx,
&NewFile {
name: "oops.bin",
path: "/tmp/oops.bin",
parent: "/tmp",
parent: "/tmp/",
size: 0,
mtime: 1,
mime: None,
@ -1019,13 +1120,11 @@ fn set_content_failed_writes_failed_table() {
/// Insert one row under `path`, born pending when `needs_content`.
fn insert_at(tx: &Transaction<'_>, path: &str, needs_content: bool) -> i64 {
let name = path.rsplit('/').next().unwrap();
let parent = &path[..path.rfind('/').unwrap()];
let (parent, name) = crate::file_handling::split_db_path(path).expect("a file's path");
insert_file(
tx,
&NewFile {
name,
path,
parent,
size: 1,
mtime: 1,
@ -1176,3 +1275,40 @@ fn prune_root_stats_drops_every_figure_of_a_dropped_root() {
drop(conn);
std::fs::remove_file(&p).ok();
}
/// The vanished-directory sweep walks `for_each_parent_in_range`, and its
/// range has to include the root's *own* directory or the files sitting
/// directly in a root are never reconciled.
///
/// It does now only because every stored parent ends in a separator: the root's
/// parent is spelled `/tree/`, which is the range's `lo` exactly. With the bare
/// `/tree` an older schema stored, it sorted *below* `lo` and the sweep skipped
/// it — harmless then only because a readable root is always in `seen_dirs`.
#[test]
fn the_parent_scan_reaches_the_roots_own_directory() {
let p = tmp_path();
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
seeded(
&mut conn,
&[
"/tree/top.txt", // directly in the root
"/tree/deep/b.txt", // a subdirectory
"/tree/deep/deeper/c.txt", // deeper still
"/tree2/outside.txt", // prefix sibling: outside
"/treeX/outside.txt", // and the LIKE-metacharacter neighbour
],
);
let range = crate::file_handling::ExtractCursor::for_root("/tree");
let mut seen = Vec::new();
for_each_parent_in_range(&conn, &range.lo, &range.hi, |parent| seen.push(parent)).unwrap();
seen.sort();
assert_eq!(
seen,
vec!["/tree/", "/tree/deep/", "/tree/deep/deeper/"],
"the root's own directory is in range, and the siblings are not"
);
drop(conn);
std::fs::remove_file(&p).ok();
}

53
crates/quicksearch-core/src/db/schema.rs
View file

@ -130,7 +130,11 @@ pub const PRAGMAS_READONLY: &str = "
///
/// Two of these can exist per indexing root, so the cache size is multiplied
/// by the root count. 1 MiB is sized for the walk's queries, which each read
/// one range of `idx_files_parent` once and never revisit it. The feeder's
/// one range of `idx_files_parent` once and never revisit it — plus, since
/// that index stopped carrying `mtime`, one table-row fetch per entry in the
/// range. Those land on a handful of pages while a directory's rows stay
/// rowid-adjacent; **this is the number to raise** if a tree churned across
/// many incremental runs ever scatters them far enough to matter. The feeder's
/// paging is the same shape, but its one-off `count_extract_scope` at pass
/// start is not: that scans the root's whole path range fetching a row per
/// entry, so on a large root it is a cold read all the way through. It is
@ -156,10 +160,20 @@ CREATE TABLE schema_info (
value TEXT NOT NULL
);
-- **There is no `path` column.** A file's path is `parent || name`, and
-- storing it a third time cost ~43% of the per-row footprint: over a
-- 400k-file corpus (avg path 77 bytes, parent 59, name 17) ~394 bytes/row
-- against the ~226 below, or ~170 MB per million files. It also widened the
-- row by a third, and three of the search cascade's passes scan every row —
-- on an encrypted index each extra page is an AES-CBC decrypt and an
-- HMAC-SHA512 verify. Reassembling a path is a `push_str` in the one place
-- that needs one (`file_handling::split_db_path` is the inverse).
--
-- `parent` always ends in the platform separator; see `dir_to_db_parent` for
-- why the whole design turns on that.
CREATE TABLE files (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
path TEXT NOT NULL UNIQUE,
parent TEXT NOT NULL,
size INTEGER NOT NULL,
mtime INTEGER NOT NULL,
@ -169,17 +183,32 @@ CREATE TABLE files (
hash BLOB
);
-- Covering, not just `(parent)`. The walk's row prefetcher issues
-- `SELECT name, mtime FROM files WHERE parent = ?` once per directory — the
-- hottest read in a full run — and with the bare index that is an index probe
-- plus a table-row fetch per entry. Those fetches are cold by design: the walk
-- reader deliberately runs on a 1 MiB page cache (see `PRAGMAS_WALK_READER`).
-- Carrying `name` and `mtime` in the index makes it an index-only scan.
-- `parent` stays leading, so `SELECT DISTINCT parent` range scans and
-- `paths_in_dir` are unaffected.
CREATE INDEX idx_files_parent ON files(parent, name, mtime);
-- The identity of a row, and the only index `parent` needs.
--
-- It replaces both of what came before — a `UNIQUE(path)` and a covering
-- `(parent, name, mtime)` — and dropping the second is the deliberate half.
-- The walk's row prefetcher issues `SELECT name, mtime FROM files WHERE
-- parent = ?` once per directory, the hottest read in a full run, and without
-- `mtime` in the index that is a table-row fetch per entry. It is affordable
-- because the prefetcher is one thread ahead of four walk workers that each
-- spend a `stat` *and* a SHA-256 of the path per file (`crate::walk`), so it
-- has budget to spend; because a directory's rows are written in one batch and
-- so are rowid-adjacent; and because the row is now narrow enough that a
-- 1 MiB cache holds ~7,200 of them. `mtime` cannot simply be appended here —
-- `UNIQUE(parent, name, mtime)` would let the same file be inserted twice
-- under two mtimes.
--
-- If the prefetcher is ever measured falling behind, raise
-- `PRAGMAS_WALK_READER` — paid for out of the space this index no longer
-- occupies — rather than restoring the covering one.
CREATE UNIQUE INDEX idx_files_parent ON files(parent, name);
CREATE INDEX idx_files_mtime ON files(mtime);
CREATE INDEX idx_files_type ON files(type);
-- No index on `type`. The only query that touches it is the kind filter's
-- `(f.type & ?) != 0` (`crate::query::translator`), and a bitmask test on the
-- left of the operator is not sargable, so SQLite scans regardless — nothing
-- can seek such an index and nothing orders or groups by the column. Carrying
-- one measured ~10% on inserts and 805 pages at 300k rows, and those pages
-- compete with the ones search wants resident.
CREATE INDEX idx_files_mime ON files(mime);
CREATE INDEX idx_files_hash ON files(hash);
CREATE INDEX idx_files_content_pending ON files(id) WHERE content_state = 0;

379
crates/quicksearch-core/src/extract/office.rs
View file

@ -86,34 +86,104 @@ const ODF_SHEET: TextSpec = TextSpec {
separator: Some(' '),
};
/// The text an `&entity;` or `&#1234;` reference stands for.
///
/// quick-xml 0.41 reports a reference as its own event instead of resolving
/// it inside the surrounding `Text`, so a reader that ignores this event
/// silently drops every `&amp;`, `&lt;` and `&#8217;` from the document —
/// no error, just missing characters in the index. Only the five predefined
/// entities and numeric references are resolvable without a DTD; anything
/// else is a document-defined entity we cannot expand, and is skipped.
fn entity_text(raw: &str) -> Option<String> {
if let Some(digits) = raw.strip_prefix('#') {
let code = match digits.strip_prefix(['x', 'X']) {
Some(hex) => u32::from_str_radix(hex, 16).ok()?,
None => digits.parse::<u32>().ok()?,
};
let c = char::from_u32(code)?;
// `char::from_u32` accepts far more than XML's character production
// does: every C0 control but tab, newline and carriage return is
// forbidden, and `&#0;` in particular would put a literal NUL into the
// indexed text and from there into an FTS5 column. `None` here reaches
// the callers as the same "unknown entity" error an unexpandable name
// gets — a `failed_files` row naming the file, which is the visible
// outcome this extractor prefers to a quietly mangled document.
let legal = !c.is_control() || matches!(c, '\t' | '\n' | '\r');
return legal.then(|| String::from(c));
}
quick_xml::escape::resolve_predefined_entity(raw).map(String::from)
}
/// Append the text `spec` selects out of `xml` to `out`.
///
/// `in_text` is a flag rather than a depth count, which means a closing
/// `</text:span>` ends the run even though its enclosing `<text:p>` is still
/// open.
/// Text-bearing elements are counted, not flagged. ODF nests them — a
/// `<text:span>` inside a `<text:p>` — and a flag made the span's own close
/// end the run, dropping every character between it and the paragraph's
/// close. Counting also gives the separator somewhere honest to go: it
/// belongs after a *run*, and since quick-xml 0.41 a run arrives as several
/// events, so emitting one per event put a space in the middle of every cell
/// containing an entity.
fn collect_xml_text(xml: &str, spec: &TextSpec, out: &mut String) -> Result<(), Box<dyn Error>> {
let mut reader = Reader::from_str(xml);
reader.trim_text(true);
// Deliberately no `trim_text`: it trims each *event*, and since 0.41 an
// entity reference splits the character data around it into separate
// events — so `Jack &amp; Jill` would come back as `Jack&Jill`, with the
// spaces trimmed off the ends of the two fragments. Nothing needs it
// either: whitespace between elements arrives while the `in_text`/`in_cell`
// flag is false and is ignored there, and whitespace *inside* a
// text-bearing element is content.
let mut buf = Vec::new();
let mut in_text = false;
// How many text-bearing elements are open. The run ends when it returns
// to zero, not when the innermost one closes.
let mut depth = 0usize;
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) => {
if spec.text.contains(&e.name().as_ref()) {
in_text = true;
depth += 1;
}
}
Ok(Event::Text(e)) if in_text => {
out.push_str(&e.unescape()?);
if let Some(sep) = spec.separator {
out.push(sep);
}
Ok(Event::Text(e)) if depth > 0 => {
out.push_str(&e.decode()?);
}
// An entity reference is its own event in 0.41; without this arm
// every `&amp;` in a document would vanish from the index.
Ok(Event::GeneralRef(e)) if depth > 0 => {
let raw = e.decode()?;
// An entity nothing can expand is an error, as it was when
// `unescape` resolved these inline: dropping it would take
// characters out of the indexed text with nothing to show for
// it, and this reader has no DTD to define one with.
let text = entity_text(&raw)
.ok_or_else(|| format!("Error parsing XML: unknown entity &{};", raw))?;
out.push_str(&text);
}
Ok(Event::End(ref e)) => {
let name = e.name();
if spec.text.contains(&name.as_ref()) {
in_text = false;
depth = depth.saturating_sub(1);
// Closing the outermost one closes the run.
if depth == 0 {
if let Some(sep) = spec.separator {
out.push(sep);
}
}
}
if spec.breaks.contains(&name.as_ref()) {
out.push('\n');
}
}
// A self-closed element gets no `Start` and no `End` of its own,
// so `<text:p/>` — ODF's blank line — would otherwise lose both
// its separator and its paragraph break. It carries no text, so
// the run it opens is empty and closes immediately.
Ok(Event::Empty(ref e)) => {
let name = e.name();
if depth == 0 && spec.text.contains(&name.as_ref()) {
if let Some(sep) = spec.separator {
out.push(sep);
}
}
if spec.breaks.contains(&name.as_ref()) {
out.push('\n');
@ -235,18 +305,56 @@ fn shared_strings<R: Read + Seek>(archive: &mut ZipArchive<R>) -> Vec<String> {
return Vec::new();
};
let mut reader = Reader::from_str(&xml);
reader.trim_text(true);
// Deliberately no `trim_text`: it trims each *event*, and since 0.41 an
// entity reference splits the character data around it into separate
// events — so `Jack &amp; Jill` would come back as `Jack&Jill`, with the
// spaces trimmed off the ends of the two fragments. Nothing needs it
// either: whitespace between elements arrives while the `in_text`/`in_cell`
// flag is false and is ignored there, and whitespace *inside* a
// text-bearing element is content.
let mut buf = Vec::new();
let mut strings = Vec::new();
let mut in_text = false;
// One `<t>` is one shared string, but it is not one event: an entity
// reference inside it arrives separately and splits the character data
// around it. Accumulated here and pushed on the closing tag, or a cell
// containing `&amp;` would become three table entries and every later
// index would point at the wrong one.
let mut current = String::new();
loop {
match reader.read_event_into(&mut buf) {
Ok(Event::Start(ref e)) if e.name().as_ref() == b"t" => in_text = true,
Ok(Event::Text(e)) if in_text => match e.unescape() {
Ok(s) => strings.push(s.into_owned()),
Ok(Event::Start(ref e)) if e.name().as_ref() == b"t" => {
in_text = true;
current.clear();
}
// `<t/>` — an empty cell. quick-xml reports a self-closed element
// as its own event with no `Start` and no `End`, so without this
// arm the entry is never pushed and **every later index is off by
// one**: `collect_sheet` then renders a real string for the wrong
// cell, with nothing anywhere reporting a problem. LibreOffice,
// openpyxl and POI all write `<si><t/></si>` for a blank, so this
// is ordinary output rather than a crafted file.
Ok(Event::Empty(ref e)) if e.name().as_ref() == b"t" => {
strings.push(String::new());
}
Ok(Event::Text(e)) if in_text => match e.decode() {
Ok(s) => current.push_str(&s),
Err(_) => return strings,
},
Ok(Event::End(ref e)) if e.name().as_ref() == b"t" => in_text = false,
Ok(Event::GeneralRef(e)) if in_text => {
// Unlike the other two readers this one cannot fail — a
// missing table is not an error here — so an entity nothing
// can expand is simply left out.
if let Ok(raw) = e.decode() {
if let Some(text) = entity_text(&raw) {
current.push_str(&text);
}
}
}
Ok(Event::End(ref e)) if e.name().as_ref() == b"t" => {
in_text = false;
strings.push(std::mem::take(&mut current));
}
Ok(Event::Eof) | Err(_) => break,
_ => {}
}
@ -259,7 +367,13 @@ fn shared_strings<R: Read + Seek>(archive: &mut ZipArchive<R>) -> Vec<String> {
/// rather than text of its own; every other type holds its value inline.
fn collect_sheet(xml: &str, strings: &[String], out: &mut String) -> Result<(), Box<dyn Error>> {
let mut reader = Reader::from_str(xml);
reader.trim_text(true);
// Deliberately no `trim_text`: it trims each *event*, and since 0.41 an
// entity reference splits the character data around it into separate
// events — so `Jack &amp; Jill` would come back as `Jack&Jill`, with the
// spaces trimmed off the ends of the two fragments. Nothing needs it
// either: whitespace between elements arrives while the `in_text`/`in_cell`
// flag is false and is ignored there, and whitespace *inside* a
// text-bearing element is content.
let mut buf = Vec::new();
let mut in_cell = false;
let mut cell_type = String::new();
@ -287,19 +401,49 @@ fn collect_sheet(xml: &str, strings: &[String], out: &mut String) -> Result<(),
}
}
Ok(Event::Text(e)) if in_cell => {
let text = e.unescape()?;
let text = e.decode()?;
if cell_type == "s" {
// A shared-string reference. An index past the end of the
// table is a corrupt workbook, not something to guess at.
if let Some(s) = text.parse::<usize>().ok().and_then(|i| strings.get(i)) {
//
// `trim` because this reader no longer sets `trim_text`
// (see the comment above): a generator that indents its
// XML hands `<v>` over as "\n 0\n", and an untrimmed
// parse would fail and drop the string with nothing to
// show for it. Whitespace around an integer index is not
// content, unlike whitespace inside a `<t>`.
if let Some(s) = text
.trim()
.parse::<usize>()
.ok()
.and_then(|i| strings.get(i))
{
out.push_str(s);
out.push(' ');
}
} else {
} else if !text.trim().is_empty() {
// Whitespace-only fragments are the indentation *between*
// a cell's child elements, which reaches this arm now that
// the reader no longer sets `trim_text`. Skipped rather
// than pushed: `in_cell` is a flag, so it cannot tell an
// indent from a value, and a cell whose entire content is
// whitespace contributes nothing to a search index either
// way. The value itself is pushed whole — no `trim` — so a
// deliberate `xml:space="preserve"` inline string keeps
// its shape.
out.push_str(&text);
out.push(' ');
}
}
// See `entity_text`. Only inline values can carry one: a `t="s"`
// cell's text is an integer index, and an entity inside it would
// be a corrupt workbook rather than a character to recover.
Ok(Event::GeneralRef(e)) if in_cell && cell_type != "s" => {
let raw = e.decode()?;
let text = entity_text(&raw)
.ok_or_else(|| format!("Error parsing XML: unknown entity &{};", raw))?;
out.push_str(&text);
}
Ok(Event::End(ref e)) => {
let name = e.name();
if name.as_ref() == b"c" {
@ -376,6 +520,111 @@ mod tests {
use super::*;
use std::io::Write;
/// Entity references must survive extraction.
///
/// quick-xml 0.41 reports `&amp;` as its own `GeneralRef` event instead of
/// resolving it into the surrounding text, so a reader that only handles
/// `Event::Text` loses the character with no error to show for it. This is
/// the test that makes that visible: it fails by producing "Blake Co"
/// rather than by failing to compile.
#[test]
fn entity_references_survive_extraction() {
let body = "<w:document><w:body><w:p><w:r>\
<w:t>Blake &amp; Co &lt;tags&gt; &#8217;24 &#x2019;25</w:t>\
</w:r></w:p></w:body></w:document>";
let path = container("docx-entities", "docx", &[("word/document.xml", body)]);
let out = OfficeExtractor.extract(&path).expect("extract");
assert!(
out.text.contains("Blake & Co"),
"predefined entity lost: {:?}",
out.text
);
assert!(
out.text.contains("<tags>"),
"angle-bracket entities lost: {:?}",
out.text
);
assert!(
out.text.contains('\u{2019}'),
"numeric entities lost: {:?}",
out.text
);
assert!(
!out.text.contains("&amp;") && !out.text.contains("&#"),
"entities left unresolved: {:?}",
out.text
);
}
/// The same, through the shared-string table an `.xlsx` cell indexes into
/// — a separate reader, and so a separate chance to drop the character.
#[test]
fn entity_references_survive_shared_strings() {
let shared = "<sst><si><t>Jack &amp; Jill</t></si></sst>";
let sheet = "<worksheet><sheetData><row>\
<c t=\"s\"><v>0</v></c></row></sheetData></worksheet>";
let path = container(
"xlsx-entities",
"xlsx",
&[
("xl/sharedStrings.xml", shared),
("xl/worksheets/sheet1.xml", sheet),
],
);
let out = OfficeExtractor.extract(&path).expect("extract");
assert!(
out.text.contains("Jack & Jill"),
"entity lost through the shared-string table: {:?}",
out.text
);
}
/// A shared-string reference must survive an indented `<v>`.
///
/// The reader deliberately does not set `trim_text` (an entity reference
/// splits the character data around it, and trimming each fragment would
/// eat the spaces at the split). A `t="s"` cell's `<v>` is an integer
/// index, though, so a generator that pretty-prints its sheet XML hands
/// this reader `"\n 0\n "` — and an untrimmed `parse::<usize>()`
/// fails, dropping the cell's text with no error and no `failed_files`
/// row. Whitespace-only fragments between a cell's children must not
/// reach the output either.
#[test]
fn an_indented_shared_string_reference_still_resolves() {
let shared = "<sst><si><t>Marmalade</t></si></sst>";
let sheet = "<worksheet>\n <sheetData>\n <row>\n \
<c t=\"s\">\n <v>\n 0\n </v>\n </c>\n \
<c t=\"n\">\n <v>17</v>\n </c>\n \
</row>\n </sheetData>\n</worksheet>";
let path = container(
"xlsx-indented",
"xlsx",
&[
("xl/sharedStrings.xml", shared),
("xl/worksheets/sheet1.xml", sheet),
],
);
let out = OfficeExtractor.extract(&path).expect("extract");
assert!(
out.text.contains("Marmalade"),
"the shared string was dropped by an indented index: {:?}",
out.text
);
assert!(
out.text.contains("17"),
"the inline value was dropped: {:?}",
out.text
);
// The indentation itself is not content: every run of whitespace in
// the output should be a separator this reader put there, never a
// line of the source XML's own layout.
assert!(
!out.text.contains("\n "),
"sheet indentation reached the indexed text: {:?}",
out.text
);
}
#[test]
fn supports_docx_and_friends() {
let e = OfficeExtractor;
@ -428,6 +677,15 @@ mod tests {
const XLSX_SHARED: &str = "<sst><si><t>Shared</t></si><si><t>Second</t></si></sst>";
/// The first entry is `<si><t/></si>` — a blank cell, which every
/// spreadsheet writer emits. It still occupies index 0.
const XLSX_SHARED_WITH_BLANK: &str = "<sst><si><t/></si><si><t>Second</t></si></sst>";
/// One cell, referring to shared string index 1.
const XLSX_SHEET_INDEX_1: &str = "<worksheet><sheetData>\
<row><c t=\"s\"><v>1</v></c></row>\
</sheetData></worksheet>";
const XLSX_SHEET: &str = "<worksheet><sheetData>\
<row><c t=\"s\"><v>0</v></c><c t=\"n\"><v>42</v></c></row>\
<row><c t=\"s\"><v>1</v></c></row>\
@ -600,4 +858,83 @@ mod tests {
crate::testutil::touch(&p, b"this is not a zip archive");
assert!(extract_document_text(&p, "docx").is_err());
}
/// A self-closed `<t/>` is a whole shared-string entry. Skip it and every
/// later index slides by one, so the sheet renders a real string for the
/// wrong cell — clean extraction, no error, wrong content. This is the
/// regression that made the batch worth doing.
#[test]
fn a_blank_shared_string_still_occupies_its_index() {
let p = container(
"xlsx-blank-si",
"xlsx",
&[
("xl/sharedStrings.xml", XLSX_SHARED_WITH_BLANK),
("xl/worksheets/sheet1.xml", XLSX_SHEET_INDEX_1),
],
);
assert_eq!(
extract_document_text(&p, "xlsx").unwrap(),
"Second \n",
"index 1 must still be the second entry"
);
}
/// The same shape one level up: a `<si>` holding nothing at all.
#[test]
fn an_empty_si_still_occupies_its_index() {
let p = container(
"xlsx-empty-si",
"xlsx",
&[
(
"xl/sharedStrings.xml",
"<sst><si><t></t></si><si><t>Second</t></si></sst>",
),
("xl/worksheets/sheet1.xml", XLSX_SHEET_INDEX_1),
],
);
assert_eq!(extract_document_text(&p, "xlsx").unwrap(), "Second \n");
}
/// An entity splits its run into three events. The separator belongs to
/// the run, so the cell must read `A&B` — not `A &B`, and not `A & B`.
#[test]
fn an_entity_does_not_split_an_ods_cell() {
let body = "<office:document-content><office:body><office:spreadsheet>\
<table:table><table:table-row>\
<table:table-cell><text:p>A&amp;B</text:p></table:table-cell>\
</table:table-row></table:table>\
</office:spreadsheet></office:body></office:document-content>";
let p = container("ods-entity", "ods", &[("content.xml", body)]);
assert_eq!(extract_document_text(&p, "ods").unwrap(), "A&B \n");
}
/// A span closing inside a paragraph ends the span, not the paragraph:
/// the text after it is body text and must be indexed.
#[test]
fn text_after_a_nested_span_is_not_dropped() {
let body = "<office:document-content><office:body><office:text>\
<text:p>before<text:span>inside</text:span>after</text:p>\
</office:text></office:body></office:document-content>";
let p = container("odt-span-tail", "odt", &[("content.xml", body)]);
assert_eq!(
extract_document_text(&p, "odt").unwrap(),
"beforeinsideafter\n"
);
}
/// ODF writes a blank line as a self-closed `<text:p/>`, which has no
/// `End` to hang the paragraph break on.
#[test]
fn a_self_closed_paragraph_still_breaks_the_line() {
let body = "<office:document-content><office:body><office:text>\
<text:p>first</text:p><text:p/><text:p>third</text:p>\
</office:text></office:body></office:document-content>";
let p = container("odt-empty-p", "odt", &[("content.xml", body)]);
assert_eq!(
extract_document_text(&p, "odt").unwrap(),
"first\n\nthird\n"
);
}
}

76
crates/quicksearch-core/src/extract/ole.rs
View file

@ -172,9 +172,31 @@ mod doc {
.ok_or("CLX runs past the end of the table stream")?;
let pieces = piece_table(clx)?;
let out = decode_pieces(&doc, &pieces, MAX_TEXT_BYTES);
if out.trim().is_empty() {
return Err("no text found in the piece table".into());
}
Ok(out)
}
/// Decode `pieces` out of the document stream, stopping at `budget`.
///
/// Two budgets, because the output one cannot bound the input. `clean`
/// drops the whole C0 range, so a piece of control bytes decodes at full
/// width and appends nothing — leaving a brake on `out.len()` that never
/// advances. And nothing requires pieces to be disjoint or ordered: each
/// names its own `fc`, so a table can point every piece at the same span,
/// and a 2 MiB file can name tens of GiB of decoding. Charging the bytes
/// actually read bounds both the repetition and the merely enormous
/// document.
///
/// `budget` is a parameter so a test can trip it without building a file
/// the size of the real one.
pub(super) fn decode_pieces(doc: &[u8], pieces: &[Piece], budget: usize) -> String {
let mut out = String::new();
let mut decoded = 0usize;
for piece in pieces {
if out.len() >= MAX_TEXT_BYTES {
if out.len() >= budget || decoded >= budget {
break;
}
let Some(bytes) = doc.get(piece.start..piece.end) else {
@ -183,6 +205,7 @@ mod doc {
// a recoverable document.
break;
};
decoded = decoded.saturating_add(bytes.len());
let text = if piece.compressed {
cp1252(bytes)
} else {
@ -190,10 +213,7 @@ mod doc {
};
clean(&text, &mut out);
}
if out.trim().is_empty() {
return Err("no text found in the piece table".into());
}
Ok(out)
out
}
/// Walk the FIB's variable-length sections to find `fcClx`/`lcbClx`.
@ -223,10 +243,10 @@ mod doc {
}
/// One run of characters in the `WordDocument` stream.
struct Piece {
start: usize,
end: usize,
compressed: bool,
pub(super) struct Piece {
pub(super) start: usize,
pub(super) end: usize,
pub(super) compressed: bool,
}
/// Locate the `Pcdt` inside the CLX and decode its `PlcPcd`.
@ -333,14 +353,40 @@ mod xls {
let book = stream(cfb, "Workbook")
.or_else(|| stream(cfb, "Book"))
.ok_or("no Workbook stream")?;
extract_from_book(&book, MAX_TEXT_BYTES)
}
let records = split_records(&book);
/// The workbook stream's text, stopping at `budget`. Split out from
/// [`extract`] so a test can trip the budget without building a file the
/// size of the real one.
pub(super) fn extract_from_book(book: &[u8], budget: usize) -> Result<String, Box<dyn Error>> {
let records = split_records(book);
let strings = shared_strings(&records);
let out = decode_cells(&records, &strings, budget);
if out.trim().is_empty() {
return Err("workbook holds no readable cell text".into());
}
Ok(out)
}
/// Render every cell-bearing record, stopping at `budget`.
///
/// The same two budgets as [`super::doc::decode_pieces`], for the same
/// reason: `clean` can consume a whole cell and emit nothing, so a
/// workbook whose shared strings are all control characters runs to the
/// end of its records with the output brake never advancing — and one
/// `LABELSST` is six bytes, so a small file holds a great many of them,
/// each free to name the same 64 KiB shared string.
///
/// `budget` is a parameter so a test can trip it without building a file
/// the size of the real one.
fn decode_cells(records: &[Record<'_>], strings: &[String], budget: usize) -> String {
let mut out = String::new();
let mut row_open = false;
for rec in &records {
if out.len() >= MAX_TEXT_BYTES {
let mut decoded = 0usize;
for rec in records {
if out.len() >= budget || decoded >= budget {
break;
}
let cell = match rec.id {
@ -363,6 +409,7 @@ mod xls {
_ => None,
};
if let Some(text) = cell {
decoded = decoded.saturating_add(text.len());
clean(&text, &mut out);
out.push(' ');
row_open = true;
@ -371,10 +418,7 @@ mod xls {
if row_open {
out.push('\n');
}
if out.trim().is_empty() {
return Err("workbook holds no readable cell text".into());
}
Ok(out)
out
}
struct Record<'a> {

100
crates/quicksearch-core/src/extract/ole_tests.rs
View file

@ -436,3 +436,103 @@ fn an_empty_file_is_an_error() {
crate::testutil::touch(&p, b"");
assert!(extract_ole_text(&p, "xls").is_err());
}
// -- decode budgets ---------------------------------------------------
/// Nothing in the format requires pieces to be disjoint, so a piece table may
/// point every entry at the same span: a small file that decodes gigabytes.
/// `clean` drops the whole C0 range, so a run of control bytes produces no
/// output at all and a brake on the emitted text never fires. The budget has
/// to charge what was *read*.
#[test]
fn doc_overlapping_control_pieces_stop_at_the_budget() {
// One 4 KiB span of control bytes, pointed at over and over.
const SPAN: usize = 4 * 1024;
let mut doc = vec![0x01u8; SPAN];
let marker_at = doc.len();
doc.extend_from_slice(b"MARKER");
let mut pieces: Vec<doc::Piece> = (0..64)
.map(|_| doc::Piece {
start: 0,
end: SPAN,
compressed: true,
})
.collect();
// Reachable only if the budget did not stop the walk first.
pieces.push(doc::Piece {
start: marker_at,
end: doc.len(),
compressed: true,
});
// A budget of half what those pieces decode.
let out = doc::decode_pieces(&doc, &pieces, SPAN * 32);
assert!(
!out.contains("MARKER"),
"the walk ran past its budget: {out:?}"
);
assert!(
out.trim().is_empty(),
"control bytes must not survive `clean`: {out:?}"
);
}
/// The same table under a budget it fits inside must be extracted whole —
/// the brake must not fire early.
#[test]
fn doc_pieces_within_the_budget_are_all_decoded() {
const SPAN: usize = 4 * 1024;
let mut doc = vec![0x01u8; SPAN];
let marker_at = doc.len();
doc.extend_from_slice(b"MARKER");
let mut pieces: Vec<doc::Piece> = (0..4)
.map(|_| doc::Piece {
start: 0,
end: SPAN,
compressed: true,
})
.collect();
pieces.push(doc::Piece {
start: marker_at,
end: doc.len(),
compressed: true,
});
let out = doc::decode_pieces(&doc, &pieces, SPAN * 32);
assert!(out.contains("MARKER"), "stopped early: {out:?}");
}
/// A workbook whose cells are all control characters: every `LABELSST` is six
/// bytes of record and resolves to a shared string that `clean` erases, so the
/// emitted-text brake never advances however many of them there are.
#[test]
fn xls_control_character_cells_stop_at_the_budget() {
let control: String = std::iter::repeat('\u{1}').take(4096).collect();
let mut sst = Vec::new();
sst.extend_from_slice(&le32(2)); // total
sst.extend_from_slice(&le32(2)); // unique
sst.extend_from_slice(&sst_string(&control, false));
sst.extend_from_slice(&sst_string("MARKER", false));
let mut book = biff(xls::REC_SST, &sst);
for _ in 0..64 {
book.extend_from_slice(&biff(xls::REC_LABELSST, &labelsst(0)));
}
// Reachable only if the budget did not stop the scan first.
book.extend_from_slice(&biff(xls::REC_LABELSST, &labelsst(1)));
let out = xls::extract_from_book(&book, 4096 * 32)
.unwrap_err()
.to_string();
assert!(
out.contains("no readable cell text"),
"expected the scan to stop before the marker, got: {out}"
);
// Under a budget it fits inside, the same workbook reads normally.
let out = xls::extract_from_book(&book, 4096 * 1024).unwrap();
assert!(out.contains("MARKER"), "stopped early: {out:?}");
}

83
crates/quicksearch-core/src/extract/plaintext.rs
View file

@ -57,6 +57,39 @@ fn decode(bytes: Vec<u8>, path: &Path) -> Result<ExtractedContent, ExtractError>
pub struct PlaintextExtractor;
/// Ceiling on a single read, whatever the file claims.
///
/// Not a policy about what is worth indexing — `maximum_text_file_size` is
/// that, and it is applied to the size the walk saw. This is the backstop for
/// the gap between those two moments, and for a node whose `fstat` lies.
const MAX_READ: usize = 64 * 1024 * 1024;
/// Read `size` bytes from `f`, never more than `cap`.
///
/// The cap is the point of the function. The gate that got this file here is
/// `maximum_text_file_size` against the size the *walk* recorded, which on a
/// large tree was minutes or hours ago; `size` is from the `fstat` just now. A
/// file that grew since — a log, a cloud placeholder that hydrated — would
/// otherwise be allocated and read in full, and the decode afterwards can take
/// three times that again.
///
/// A short read is not an error: a file that shrank keeps its prefix.
fn read_sized(f: &mut File, size: usize, cap: usize, path: &Path) -> Result<Vec<u8>, ExtractError> {
let size = size.min(cap);
let mut buf = vec![0u8; size];
let mut filled = 0;
while filled < size {
match f.read(&mut buf[filled..]) {
Ok(0) => break,
Ok(n) => filled += n,
Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {}
Err(e) => return Err(format!("plaintext read {}: {}", path.display(), e)),
}
}
buf.truncate(filled);
Ok(buf)
}
impl Extractor for PlaintextExtractor {
fn supports(&self, mime: &str) -> bool {
mime.starts_with("text/") || EXTRA_TEXT_MIMES.contains(&mime)
@ -83,26 +116,14 @@ impl Extractor for PlaintextExtractor {
// before anyway. Capped so a node that streams forever (a FIFO, a
// lying filesystem) cannot allocate without bound.
if size == 0 {
const MAX_UNSIZED_READ: u64 = 64 * 1024 * 1024;
let mut buf = Vec::new();
f.take(MAX_UNSIZED_READ)
f.take(MAX_READ as u64)
.read_to_end(&mut buf)
.map_err(|e| format!("plaintext read {}: {}", path.display(), e))?;
return decode(buf, path);
}
let mut buf = vec![0u8; size];
let mut filled = 0;
while filled < size {
match f.read(&mut buf[filled..]) {
Ok(0) => break,
Ok(n) => filled += n,
Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {}
Err(e) => return Err(format!("plaintext read {}: {}", path.display(), e)),
}
}
buf.truncate(filled);
decode(buf, path)
decode(read_sized(&mut f, size, MAX_READ, path)?, path)
}
fn extract_from_head(
@ -283,4 +304,38 @@ mod tests {
assert!(!e.supports("application/pdf"));
assert!(!e.supports("image/png"));
}
/// The cap is what stands between a file that grew since the walk sized
/// it — a log, a cloud placeholder that hydrated — and an allocation the
/// size of whatever it grew to.
#[test]
fn a_sized_read_stops_at_the_cap() {
let body = vec![b'x'; 4096];
let p = tmp("cap", &body);
let mut f = File::open(&p).unwrap();
let out = read_sized(&mut f, body.len(), 100, &p).unwrap();
assert_eq!(out.len(), 100, "read past the cap");
std::fs::remove_file(&p).ok();
}
/// A file that *shrank* between the `fstat` and the read keeps its prefix
/// rather than failing: the next run reclassifies it anyway.
#[test]
fn a_short_read_keeps_what_was_there() {
let p = tmp("short", b"only ten!!");
let mut f = File::open(&p).unwrap();
let out = read_sized(&mut f, 1_000_000, MAX_READ, &p).unwrap();
assert_eq!(out, b"only ten!!");
std::fs::remove_file(&p).ok();
}
/// Under a cap larger than the file, nothing changes.
#[test]
fn a_file_under_the_cap_is_read_whole() {
let body = vec![b'y'; 4096];
let p = tmp("uncapped", &body);
let out = PlaintextExtractor.extract(&p).unwrap();
assert_eq!(out.text.len(), 4096);
std::fs::remove_file(&p).ok();
}
}

137
crates/quicksearch-core/src/extract/rtf.rs
View file

@ -5,13 +5,24 @@
//! plaintext extractor in [`super::Registry::default_set`], because
//! plaintext claims every `text/*` and would otherwise swallow `text/rtf`
//! and index the control-word noise raw.
//!
//! `rtf-parser` resolves to `vendor/rtf-parser`, a patched copy — its lexer
//! ended a control word at whitespace and nowhere else, which silently dropped
//! text from documents LibreOffice and Word produce. The `[patch.crates-io]`
//! note in the workspace manifest is where that is written up; the tests below
//! and `tests/extraction_corpus.rs` are what keep it fixed.
use std::fs::File;
use std::io::Read;
use std::path::Path;
use rtf_parser::document::RtfDocument;
use super::{ExtractError, ExtractedContent, Extractor};
/// Ceiling on a single read; see [`super::plaintext`], same reasoning.
const MAX_READ: usize = 64 * 1024 * 1024;
/// Parse a complete RTF file's bytes. Shared by both entry points so
/// on-disk and already-in-memory extraction cannot drift apart.
///
@ -29,15 +40,28 @@ fn parse(bytes: Vec<u8>, path: &Path) -> Result<ExtractedContent, ExtractError>
pub struct RtfExtractor;
/// Read at most `cap` bytes of `path`.
///
/// Bounded rather than `fs::read`: the size gate that admitted this file was
/// applied to what the walk recorded, and the file may have grown since.
/// `rtf-parser` also amplifies its input several-fold in heap, so an unbounded
/// read here is unbounded twice over.
fn read_capped(path: &Path, cap: u64) -> Result<Vec<u8>, ExtractError> {
let file = File::open(path).map_err(|e| format!("rtf read {}: {}", path.display(), e))?;
let mut bytes = Vec::new();
file.take(cap)
.read_to_end(&mut bytes)
.map_err(|e| format!("rtf read {}: {}", path.display(), e))?;
Ok(bytes)
}
impl Extractor for RtfExtractor {
fn supports(&self, mime: &str) -> bool {
mime == "application/rtf" || mime == "text/rtf"
}
fn extract(&self, path: &Path) -> Result<ExtractedContent, ExtractError> {
let bytes =
std::fs::read(path).map_err(|e| format!("rtf read {}: {}", path.display(), e))?;
parse(bytes, path)
parse(read_capped(path, MAX_READ as u64)?, path)
}
/// RTF has no trailer and needs no seeking, so a head that is the whole
@ -83,19 +107,29 @@ mod tests {
std::fs::remove_file(&p).ok();
}
/// A `\\u` escape naming a lone UTF-16 surrogate must fail the file, not
/// the thread.
/// A `\\u` escape naming a lone UTF-16 surrogate costs one character, not
/// the document and not the thread.
///
/// `rtf-parser` reaches `String::from_utf16(..).unwrap()` with whatever
/// `\\uN` supplied, and screens nothing for the surrogate range. RTF is one
/// of the two extractors that also run at *walk* time, off
/// `extract_from_head`, where a panicking worker costs the root its entire
/// content pass and disables stale cleanup run-wide — so this is contained
/// in `decide_content` and `prepare_file_record` rather than left to the
/// parser. Both entry points are exercised here.
/// `rtf-parser` reached `String::from_utf16(..).unwrap()` with whatever
/// `\\uN` supplied and screened nothing for the surrogate range, so a
/// fifteen-byte document could panic. RTF is one of the two extractors that
/// also run at *walk* time, off `extract_from_head`, where a panicking
/// worker costs the root its entire content pass and disables stale
/// cleanup run-wide — so the panic was contained in `decide_content` and
/// `prepare_file_record`, and the file recorded as FAILED.
///
/// `vendor/rtf-parser` decodes lossily instead (LOCAL PATCH, see
/// `Parser::flush_unicode`), which beats either outcome: the bad escape
/// becomes one `U+FFFD` and the rest of the document is indexed. Both
/// entry points are still exercised, because the containment above them
/// has to keep working for every other way a parser can panic.
#[test]
fn a_lone_surrogate_escape_is_contained() {
let body = br"{\rtf1\u55296 }";
fn a_lone_surrogate_escape_costs_one_character() {
// `\u55296` is a high surrogate with no low half to follow it. The `?`
// is its ANSI fallback, written the way a real producer writes one —
// spelled with a space delimiter instead, the `a` of `after` would be
// the fallback and would correctly be eaten.
let body = "{\\rtf1\\ansi before \\u55296?after}".as_bytes();
let p = tmp("surrogate", body);
// The on-disk path, as the content pass reaches it.
@ -105,31 +139,56 @@ mod tests {
&crate::extract::Registry::default_set(),
&crate::config::Config::default(),
);
let text = match &outcome {
crate::file_handling::ContentOutcome::Done { text } => text.clone(),
other => panic!("a malformed escape must not fail the document: {other:?}"),
};
assert!(
matches!(outcome, crate::file_handling::ContentOutcome::Failed(_)),
"a panicking parser must record a failure, not unwind: {:?}",
outcome
text.contains("before") && text.contains("after"),
"the rest of the document must survive: {text:?}"
);
assert!(
text.contains('\u{FFFD}'),
"the bad escape must leave a replacement character: {text:?}"
);
// And the head path, as a walk worker reaches it: through the
// registry, which is where the containment lives. The raw
// `RtfExtractor::extract_from_head` below it still panics — that is
// third-party code doing what it does, and the point is that no
// caller in this crate is exposed to it.
// And the head path, as a walk worker reaches it: through the registry,
// which is where the containment for any *other* panicking input lives.
let head = crate::extract::Registry::default_set().extract_complete_head(
&p,
"application/rtf",
body,
);
assert!(
matches!(head, Some(Err(_))),
"a panicking parser must be charged to the file, not the worker: {:?}",
head.map(|r| r.map(|c| c.text))
assert_eq!(
head.expect("claimed").expect("parsed").text,
text,
"head and disk extraction must agree"
);
std::fs::remove_file(&p).ok();
}
/// `\\par` ends a paragraph, so it has to reach the text as a line break.
///
/// It used to emit nothing, and every paragraph boundary closed up:
/// a LibreOffice document came back as `...do eiusmod.The needle...`.
/// No text was lost, but the join invents word and sentence boundaries
/// that are not in the document — which a snippet then shows to the user,
/// and which a phrase query can match across. Fixed in
/// `vendor/rtf-parser` (LOCAL PATCH), alongside `\\line`, which always
/// did the right thing.
#[test]
fn paragraph_breaks_reach_the_text() {
let body = br"{\rtf1\ansi First paragraph.\par Second paragraph.\par}";
let p = tmp("par", body);
let text = RtfExtractor.extract(&p).unwrap().text;
assert!(
text.contains("First paragraph.\nSecond paragraph."),
"paragraphs ran together: {text:?}"
);
std::fs::remove_file(&p).ok();
}
#[test]
fn malformed_input_errors_and_names_the_file() {
let p = tmp("broken", br"{\rtf1 truncated");
@ -152,4 +211,30 @@ mod tests {
assert!(!e.supports("text/plain"));
assert!(!e.supports("application/pdf"));
}
/// `rtf-parser` amplifies its input several-fold in heap, so the read that
/// feeds it has to be bounded independently of what the walk recorded.
#[test]
fn a_read_stops_at_the_cap() {
let body = vec![b'x'; 4096];
let p = tmp("cap", &body);
assert_eq!(
read_capped(&p, 100).unwrap().len(),
100,
"read past the cap"
);
assert_eq!(
read_capped(&p, MAX_READ as u64).unwrap().len(),
4096,
"a file under the cap must be read whole"
);
std::fs::remove_file(&p).ok();
}
#[test]
fn a_missing_file_is_an_error_naming_it() {
let p = crate::testutil::scratch_dir("rtf-missing").join("nope.rtf");
let err = read_capped(&p, MAX_READ as u64).unwrap_err();
assert!(err.contains(&p.display().to_string()), "{err}");
}
}

72
crates/quicksearch-core/src/file_handling/batch.rs
View file

@ -104,7 +104,8 @@ pub fn process_batch_updates(
let updated = repo::update_file_basic(&tx, &rec.as_new_file()).map_err(|e| {
format!(
"Failed to update file record + clear stale content for {}: {}",
rec.path, e
rec.path(),
e
)
})?;
@ -115,7 +116,7 @@ pub fn process_batch_updates(
None => {
crate::log_warn!(
"no indexed row matched {} during update; inserting instead",
rec.path
rec.path()
);
repo::insert_file(&tx, &rec.as_new_file())
.map_err(|e| format!("Failed to insert file record: {}", e))?
@ -124,8 +125,13 @@ pub fn process_batch_updates(
};
if let (Some(id), Some(text)) = (id, rec.inline_text.as_deref()) {
let zstd = body_or_skip!(bodies, i, rec.path);
repo::set_content_done(&tx, id, text, zstd)?;
let zstd = body_or_skip!(bodies, i, rec.path());
// `_fresh`: `update_file_basic` above cleared this row's
// content in this same transaction, and the insert fallback
// created the row outright. Either way there is nothing left
// to delete, and the ordinary entry point would issue two
// statements per row to discover that.
repo::set_content_done_fresh(&tx, id, text, zstd)?;
}
}
@ -149,7 +155,11 @@ pub fn process_batch_inserts(
return Ok(());
}
for batch in files_to_insert.chunks(config.processing.batch_size) {
// `.max(1)`, as at every other use of this field: `chunks(0)` panics,
// and a panic here is on the indexing thread, before the arm that would
// publish `IndexingStatus::Error` — so a hand-edited `batch_size = 0`
// wedges indexing for the session while the UI still reads "Running".
for batch in files_to_insert.chunks(config.processing.batch_size.max(1)) {
if stop_flag.load(Ordering::Relaxed) {
return Ok(());
}
@ -170,8 +180,10 @@ pub fn process_batch_inserts(
let id = repo::insert_file(&tx, &rec.as_new_file())
.map_err(|e| format!("Failed to insert file record: {}", e))?;
if let (Some(id), Some(text)) = (id, rec.inline_text.as_deref()) {
let zstd = body_or_skip!(bodies, i, rec.path);
repo::set_content_done(&tx, id, text, zstd)?;
let zstd = body_or_skip!(bodies, i, rec.path());
// `_fresh`: `insert_file` returned `Some` only by creating this
// row, so it cannot carry content from anywhere.
repo::set_content_done_fresh(&tx, id, text, zstd)?;
}
}
@ -236,13 +248,19 @@ pub fn cleanup_stale_index_entries(
/// Keyset cursor bounding everything stored beneath one directory.
///
/// `lo`/`hi` are the half-open path range `[dir + SEP, dir + (SEP + 1))`, so
/// the pair is a pure index range on `UNIQUE(files.path)`.
/// `lo`/`hi` are the half-open range `[dir + SEP, dir + (SEP + 1))` over
/// `files.parent`, so the pair is a pure index range on `idx_files_parent`.
///
/// The separator must be the platform's own: `files.path` stores native
/// separators, and the successor of `/` (`0x2F`) is `'0'` while the successor
/// of `\` (`0x5C`) is `']'` — the Unix pair on Windows yields
/// `hi = "C:\Users\me0"`, which every stored path sorts *above*, silently
/// It covers `dir`'s own files as well as its subdirectories' because every
/// stored parent ends in a separator (see `dir_to_db_parent`): the files
/// directly in `dir` have parent `dir + SEP`, which is `lo` exactly. Without
/// that invariant `lo` would have to be the bare `dir`, and the range would
/// swallow siblings — `/a-b` sorts inside `["/a", "/a0")`.
///
/// The separator must be the platform's own: parents store native separators,
/// and the successor of `/` (`0x2F`) is `'0'` while the successor of `\`
/// (`0x5C`) is `']'` — the Unix pair on Windows yields
/// `hi = "C:\Users\me0"`, which every stored parent sorts *above*, silently
/// disabling content extraction and the vanished-directory sweep.
#[derive(Debug, Clone)]
pub struct ExtractCursor {
@ -292,14 +310,36 @@ pub(crate) fn max_text_file_size(config: &Config) -> i64 {
/// *lowered* between runs (which does not force a rebuild), and rows left
/// pending by an older build. Rows this misses would stay pending forever, so
/// it runs on the writer before a root's content pass starts.
///
/// `INDEXED BY`, for the same reason [`crate::db::repo::pending_content_page`]
/// spells its own out — and it is the sibling statement to that one, left
/// behind when the counting scan was moved off the writer. The planner takes
/// `idx_files_parent` for the range and then fetches **every table row in it**
/// to test `content_state`, which is a full scan of the root on the writer
/// thread, once per root per run, while every other root's walk waits. The
/// partial index holds only pending rows, so it answers the predicate without
/// touching anything else.
///
/// Measured on 50,000 rows, best of five:
///
/// | shape | planner's choice | `INDEXED BY` |
/// |---|---:|---:|
/// | re-index, 50 rows pending | 8.01 ms | **9.61 µs** |
/// | first index, all pending, two roots | 4.36 ms | **1.28 ms** |
///
/// The second row is the case this could have lost: with everything pending the
/// partial index covers *every* root, not just this one, so it scans rows the
/// range predicate then rejects. It still wins by 3.4x, because the index is
/// narrow and id-ordered where the range path has to fetch a full row per
/// entry. There is no shape in which the planner's choice is the better one.
pub fn mark_oversize_pending_na(
conn: &Connection,
cursor: &ExtractCursor,
config: &Config,
) -> Result<(), String> {
conn.execute(
"UPDATE files SET content_state = 3 \
WHERE content_state = 0 AND size > ?1 AND path >= ?2 AND path < ?3",
"UPDATE files INDEXED BY idx_files_content_pending SET content_state = 3 \
WHERE content_state = 0 AND size > ?1 AND parent >= ?2 AND parent < ?3",
rusqlite::params![max_text_file_size(config), cursor.lo, cursor.hi],
)
.map_err(|e| format!("mark oversize files NA: {}", e))?;
@ -322,7 +362,7 @@ pub fn count_extract_scope(
.query_row(
"SELECT COALESCE(SUM(content_state = 0 AND size <= ?1), 0), \
COALESCE(SUM(content_state = 1), 0) \
FROM files WHERE path >= ?2 AND path < ?3",
FROM files WHERE parent >= ?2 AND parent < ?3",
rusqlite::params![max_text_file_size(config), cursor.lo, cursor.hi],
|row| Ok((row.get(0)?, row.get(1)?)),
)

31
crates/quicksearch-core/src/file_handling/count_and_extract_tests.rs
View file

@ -277,3 +277,34 @@ fn extract_scope_counts_only_files_an_extractor_claims() {
std::fs::remove_dir_all(&root).ok();
std::fs::remove_file(&db).ok();
}
/// A `find` that fails writes nothing, and `wc -l` then reads EOF, prints `0`
/// and exits *successfully* — so checking only the last process in the
/// pipeline reported a tree of zero entries. That answer is indistinguishable
/// from a real empty tree, and it is what kept the non-GNU `-printf` fallback
/// from ever running: it is an `.or_else` on the error this used to swallow.
#[test]
#[cfg(unix)]
fn a_failing_find_is_an_error_not_a_count_of_zero() {
let missing = tmp("count-missing").join("no-such-tree");
let cancel = AtomicBool::new(false);
let result = count_tree_entries_fast(missing.to_str().unwrap(), &cancel);
assert!(
result.is_err(),
"a path that does not exist counted {:?} entries",
result
);
}
/// An empty directory really does count zero (one entry on Unix, the root
/// `find` lists itself) — the error above must not have been bought by
/// calling every small answer a failure.
#[test]
fn an_empty_tree_still_counts() {
let root = tmp("count-empty");
std::fs::create_dir_all(&root).unwrap();
let cancel = AtomicBool::new(false);
let n = count_tree_entries_fast(root.to_str().unwrap(), &cancel).unwrap();
assert_eq!(n, if cfg!(windows) { 0 } else { 1 });
std::fs::remove_dir_all(&root).ok();
}

57
crates/quicksearch-core/src/file_handling/counting.rs
View file

@ -25,6 +25,31 @@ const COUNT_POLL_MS: u64 = 50;
/// `cancel`: on cancellation every process in `children` is killed and a
/// recognizable error is returned. On normal exit, returns the terminal
/// child's stdout.
/// Wait on every member of the pipeline and report the first failure.
///
/// `terminal_status` is the one already collected by the caller; the rest are
/// waited on here. Returns the status of a member that failed, or `None` when
/// all of them succeeded.
#[cfg(unix)]
fn reap_all(
children: &mut [&mut std::process::Child],
terminal_status: std::process::ExitStatus,
) -> Option<std::process::ExitStatus> {
let mut failed = (!terminal_status.success()).then_some(terminal_status);
let last = children.len().saturating_sub(1);
for (i, child) in children.iter_mut().enumerate() {
if i == last {
continue;
}
if let Ok(status) = child.wait() {
if !status.success() && failed.is_none() {
failed = Some(status);
}
}
}
failed
}
#[cfg(unix)]
fn wait_pipeline_cancellable(
children: &mut [&mut std::process::Child],
@ -44,9 +69,6 @@ fn wait_pipeline_cancellable(
};
match terminal.try_wait() {
Ok(Some(status)) => {
if !status.success() {
return Err(format!("count pipeline exited with {}", status));
}
// The terminal child's output is a couple dozen bytes
// (a `wc -l` figure), so reading after exit can't deadlock.
let mut out = Vec::new();
@ -55,14 +77,28 @@ fn wait_pipeline_cancellable(
let mut stdout = stdout;
let _ = stdout.read_to_end(&mut out);
}
// Reap the rest of the pipeline.
for child in children.iter_mut() {
let _ = child.wait();
// Every member, not just the terminal — and before deciding.
// A `find` that fails writes nothing and exits non-zero, while
// `wc -l` reads EOF, prints `0` and exits *successfully*: read
// through the terminal alone that is a tree of zero files, and
// the `-printf` fallback keyed on this returning `Err` never
// runs at all. Reaping here also covers the failure paths,
// which used to return leaving `find` a zombie for the life of
// the process.
let failed = reap_all(children, status);
if let Some(bad) = failed {
return Err(format!("count pipeline exited with {}", bad));
}
return Ok(out);
}
Ok(None) => std::thread::sleep(std::time::Duration::from_millis(COUNT_POLL_MS)),
Err(e) => return Err(format!("count wait: {}", e)),
Err(e) => {
for child in children.iter_mut() {
let _ = child.kill();
let _ = child.wait();
}
return Err(format!("count wait: {}", e));
}
}
}
}
@ -74,7 +110,12 @@ fn count_find_pipe_wc(
printf_newlines: bool,
) -> Result<usize, String> {
let mut find_cmd = Command::new("find");
find_cmd.arg(path);
// `--` before the path, though nothing can currently reach here with a
// leading `-`: every root goes through `resolved_indexing_paths`, which
// absolutizes it, and then `normalize_root_string`, which canonicalizes.
// It is one token, and it keeps that reasoning from being load-bearing
// for whoever changes root resolution next.
find_cmd.arg("--").arg(path);
if printf_newlines {
// GNU find: emit one newline per entry without formatting paths.
find_cmd.arg("-printf").arg("\n");

12
crates/quicksearch-core/src/file_handling/mod.rs
View file

@ -18,11 +18,17 @@ pub use batch::{
Stored,
};
pub use counting::count_tree_entries_fast;
pub use paths::{db_key_for_missing_path, filtered_dirs, filtered_walk, UnreadableDirs};
pub(crate) use paths::{normalize_root_string, path_to_db_string, warn_if_unrepresentable};
pub use paths::{
db_key_for_missing_path, filtered_dirs, filtered_walk, split_db_path, UnreadableDirs,
};
pub(crate) use paths::{
dir_to_db_parent, index_file_set, normalize_root_string, path_to_db_string,
warn_if_unrepresentable, INDEX_SIDECAR_SUFFIXES,
};
pub use records::{
classify_by_mtime, classify_for_indexing, content_extractable, decide_content,
extract_and_store, fts_finalize_after_text_indexing, hash_failure_counts, outcome_body,
extract_and_store, fts_begin_bulk_write, fts_finalize_after_text_indexing,
fts_set_automerge, hash_failure_counts, outcome_body,
prepare_file_record, prepare_file_record_from_path, reset_run_warnings, store_content_outcome,
ContentOutcome, DirRows, FileIndexAction, OwnedNewFile,
};

141
crates/quicksearch-core/src/file_handling/paths.rs
View file

@ -1,6 +1,7 @@
//! Path ↔ `files.path` string normalization and the filtered walkdir
//! wrappers the reconcile passes use.
//! Path ↔ `files.parent`/`files.name` string normalization and the filtered
//! walkdir wrappers the reconcile passes use.
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::sync::Mutex;
@ -8,7 +9,8 @@ use walkdir::{DirEntry, WalkDir};
use crate::config::IgnoreSet;
/// Render a path as the string stored in `files.path`.
/// Render a path in the spelling the index stores it in, which
/// [`split_db_path`] then splits into the `(parent, name)` key.
///
/// `Path::canonicalize` on Windows hands back extended-length paths; the
/// index stores plain ones. A UNC share canonicalizes to
@ -34,11 +36,55 @@ pub(crate) fn path_to_db_string(path: &Path) -> String {
}
}
/// What SQLite and we hang off the index's own filename.
///
/// `-wal`, `-shm` and `-journal` are SQLite's, spelled exactly as
/// `db::open` and the unlock screen already delete them; `.lock` is
/// [`crate::platform::IndexLock`]'s.
pub(crate) const INDEX_SIDECAR_SUFFIXES: [&str; 4] = ["-wal", "-shm", "-journal", ".lock"];
/// Every file belonging to the index at `db_path`, spelled the way the walk
/// spells the files it visits.
///
/// **Nothing may ever open one of these.** On POSIX, closing *any* descriptor
/// on an inode cancels every advisory lock the whole process holds on it, so a
/// walk worker that opens `index.sqlite-shm` to hash it destroys the DMS lock
/// SQLite took on that file — after which the next connection to attach, from
/// any process, truncates the wal-index to 3 bytes under our live mapping and
/// the next commit dies with SIGBUS. The same close cancels the main
/// database's own locks, which is SQLite's documented corruption hazard
/// (howtocorrupt.html §2.2). See [`crate::walk`], which prunes these before an
/// entry can become a candidate.
///
/// The *directory* is canonicalized rather than the files: `-wal`, `-shm` and
/// the lock come and go across a run, and `canonicalize` fails on a path that
/// is not there at the instant it is called. The directory outlives all of
/// them, so resolving it once and joining the names gives a stable answer that
/// does not depend on which sidecars happen to exist.
pub(crate) fn index_file_set(db_path: &Path) -> HashSet<PathBuf> {
let mut set = HashSet::new();
// An empty filename means `database_path` names a directory, not a file.
// Joining "" onto the parent would yield the directory itself and prune
// the entire tree below it.
let Some(name) = db_path.file_name().and_then(|s| s.to_str()) else {
return set;
};
let dir = db_path.parent().unwrap_or_else(|| Path::new("."));
let dir = PathBuf::from(path_to_db_string(
&dir.canonicalize().unwrap_or_else(|_| dir.to_path_buf()),
));
set.insert(dir.join(name));
for suffix in INDEX_SIDECAR_SUFFIXES {
set.insert(dir.join(format!("{}{}", name, suffix)));
}
set
}
/// Canonicalize a root string for storage/comparison. Multi-root strings
/// (newline-joined) fail canonicalize and pass through verbatim, which still
/// compares consistently.
///
/// This is the spelling `files.path` rows are prefixed with, so it is also the
/// This is the spelling stored parents are prefixed with, so it is also the
/// form roots must be compared in: `~/docs` and `/home/me/docs` name one root
/// and must not read as a change.
pub(crate) fn normalize_root_string(indexing_path: &str) -> String {
@ -49,11 +95,20 @@ pub(crate) fn normalize_root_string(indexing_path: &str) -> String {
}
/// Warn and report `true` for a path that cannot round-trip through
/// `files.path`.
/// the index.
///
/// Everything downstream reopens the file by that TEXT column, and
/// [`path_to_db_string`] is lossy: a non-UTF-8 name would be stored as a
/// path naming a file that does not exist. Such a file is skipped whole.
/// [`path_to_db_string`] is lossy: a non-UTF-8 name would be stored as a path
/// naming a *different* file — U+FFFD is an ordinary filename character, so
/// the lossy spelling is somebody's real name. Such a file is skipped whole.
///
/// This is the check for callers that arrive with a single path and no
/// listing to prune: [`prepare_file_record_from_path`] on the incremental
/// route. The full walk screens far earlier, on the directory entry itself
/// (`crate::walk::read_directory`), because by the time a path exists a bad
/// *directory* component has already been joined into every path beneath it.
///
/// [`prepare_file_record_from_path`]: super::prepare_file_record_from_path
pub(crate) fn warn_if_unrepresentable(path: &Path) -> bool {
if path.to_str().is_some() {
return false;
@ -73,13 +128,21 @@ fn starts_with_drive_letter(s: &str) -> bool {
matches!((it.next(), it.next()), (Some(c), Some(':')) if c.is_ascii_alphabetic())
}
/// The `files.path` key for a path that may no longer exist.
/// The stored spelling of a path that may no longer exist.
///
/// The insert side canonicalizes before storing, and plain `canonicalize`
/// fails on a path already gone — so this canonicalizes the deepest ancestor
/// that still resolves and re-joins the missing tail. On Linux a root reached
/// through a symlinked parent (`/home` → `/mnt/home`) makes every removal a
/// no-op without this.
///
/// **The caller must have screened `path` for representability.** This ends in
/// [`path_to_db_string`], which is lossy, and the answer is used to *delete* —
/// a row by path and then a whole subtree range beneath it. For a path that is
/// not valid UTF-8 the key returned names some other, real file, and deleting
/// by it takes that file's row and everything under it. There is nothing to
/// delete for such a path in any case: it could never have been indexed. See
/// [`crate::incremental::remove_paths`], which skips them.
pub fn db_key_for_missing_path(path: &Path) -> String {
let mut tail: Vec<std::ffi::OsString> = Vec::new();
let mut cursor = path;
@ -104,12 +167,42 @@ pub fn db_key_for_missing_path(path: &Path) -> String {
}
}
/// Parent directory of a path as a UTF-8 string, empty if root.
pub(super) fn parent_str(path: &str) -> String {
Path::new(path)
.parent()
.map(|p| p.to_string_lossy().into_owned())
.unwrap_or_default()
/// Render a directory as the string stored in `files.parent`.
///
/// **`files.parent` always ends in the platform separator**, and that is the
/// invariant the whole schema rests on:
///
/// * A file's path is `parent` concatenated with `name` — no separator logic
/// at the join, and so no special case for `/` or `C:\`, whose children
/// would otherwise be spelled `//x` and `C:\\x`.
/// * A root's subtree is the single range `[root + SEP, root + succ(SEP))`,
/// because the root's *own* parent is `root + SEP` rather than `root`. With
/// a bare `root` there is no such range: strings between `root` and
/// `root + SEP` are siblings (`/a-b` sorts inside `["/a", "/a0")`), so it
/// would take two predicates and a sibling would sneak in through either.
///
/// See [`crate::db::schema::SCHEMA_CURRENT`] for what that buys.
pub(crate) fn dir_to_db_parent(dir: &Path) -> String {
let mut s = path_to_db_string(dir);
// The platform's own separator only. On Unix `\` is an ordinary filename
// character, so a directory genuinely named `weird\` must still get its
// `/` — testing both separators would leave that row unjoinable.
if !s.ends_with(std::path::MAIN_SEPARATOR) {
s.push(std::path::MAIN_SEPARATOR);
}
s
}
/// Split a stored path into the `(parent, name)` pair the index keys on.
///
/// The separator stays with the parent, so `parent` + `name` is the original
/// string back — see [`dir_to_db_parent`]. `None` for anything that cannot be
/// a file's path: a bare relative name with no separator at all, or a string
/// that ends in one and so names a directory rather than a file.
pub fn split_db_path(path: &str) -> Option<(&str, &str)> {
let cut = path.rfind(std::path::MAIN_SEPARATOR)?;
let (parent, name) = path.split_at(cut + 1);
(!name.is_empty()).then_some((parent, name))
}
/// Paths a walk could not read, collected as it runs.
@ -204,6 +297,14 @@ fn walk_entries<'a>(
) -> impl Iterator<Item = DirEntry> + 'a {
WalkDir::new(root)
.follow_links(follow_symlinks)
// walkdir defaults this to *true*, independently of `follow_links`:
// without it a root that is itself a symlink gets descended even when
// following is off. That matters more than it sounds, because
// `prepare_file_record_from_path` canonicalizes before storing, so the
// rows land under the target's real path — and if the target is
// outside every configured root, no sweep range covers them and they
// are orphans until a rebuild.
.follow_root_links(follow_symlinks)
.into_iter()
.filter_entry(move |e| walk_filter(e, follow_symlinks, include_hidden, ignore))
.filter_map(move |res| match res {
@ -220,6 +321,18 @@ fn walk_entries<'a>(
None
}
})
// A symlink is an entry in its own right when following is off, and
// `walk_filter` cannot drop it: that runs before the descent and
// passes depth 0 unconditionally, because a root is the user's own
// choice. Yielding it would index the link as a file — and
// `prepare_file_record_from_path` canonicalizes, so the row would
// land under the target's real path. Inside a root that is a
// duplicate of a row the walk reaches anyway; outside every root it
// is a row no sweep range covers, and only a rebuild removes it.
//
// No cost when following is on: walkdir resolves links then, so
// nothing reports itself as one.
.filter(move |e| follow_symlinks || !e.file_type().is_symlink())
}
/// Walk `root` yielding only files, pruning hidden and ignored subtrees

146
crates/quicksearch-core/src/file_handling/records.rs
View file

@ -10,7 +10,6 @@ use std::time::UNIX_EPOCH;
use rusqlite::Connection;
use sha2::{Digest, Sha256};
use super::paths::parent_str;
use super::*;
use crate::config::Config;
use crate::db::repo::{self, NewFile};
@ -93,23 +92,128 @@ pub(super) fn get_file_hash(
Ok((hasher.finalize().to_vec(), head))
}
/// Nudge FTS5 to merge its index segments. Best-effort; failure is logged
/// and swallowed.
pub fn fts_finalize_after_text_indexing(conn: &Connection) {
/// FTS5's automerge threshold, applied before a run writes anything.
///
/// The number of index segments that must accumulate at one level before FTS5
/// merges them: 2..=16, or 0 to disable incremental merging. Every merge
/// rewrites segments to disk, so this is a **write-amplification** knob rather
/// than a CPU one, and the FTS index is where essentially all of an indexing
/// run's writing goes — `searchabletext_data` measured 228 MiB of a 265 MiB
/// index, against 2.8 MiB for all six `files` indexes put together.
///
/// 16 is the maximum FTS5 accepts and is measured, on a 10,000-file tree, three
/// builds per setting:
///
/// | automerge | cold index | written | search (unlimited) |
/// |---:|---:|---:|---:|
/// | 4 (FTS5 default) | 8.53 / 8.65 / 8.58 s | 1862 / 1890 / 1818 MiB | 763–772 ms |
/// | 16 | 7.10 / 7.10 / 6.67 s | 1235 / 1221 / 1213 MiB | 756–769 ms |
///
/// **19% faster and a third fewer bytes written, for no search cost.** The
/// search column has to be read carefully, and is the reason this comment
/// exists: measured at the *default* display limit the same six indexes span
/// 40 ms to 540 ms, and none of that spread is automerge. `scan_pass` stops as
/// soon as the limit fills and streams FTS candidates in rowid order — which is
/// `file_id` order, which is whatever order the concurrent walk inserted rows
/// in — so an index where the large documents drew low ids decompresses
/// megabytes to fill 1000 hits where another reads a few hundred kilobytes.
/// Two builds of one configuration differed by 13x that way. Comparing
/// anything about FTS against a limited search measures that lottery instead;
/// raise the limit past the corpus so every index examines every candidate.
const WRITE_AUTOMERGE: u8 = 16;
/// Set FTS5's automerge threshold. Best-effort; failure is logged.
///
/// **This sets a parameter — it does not merge anything.** With a value bound
/// to `rank`, `INSERT INTO ft(ft, rank) VALUES('automerge', N)` writes N into
/// the table's `%_config`, where it persists. The merging command is
/// `'merge'`, and the merge-everything command is `'optimize'`; see
/// [`fts_finalize_after_text_indexing`].
///
/// That distinction was worth a great deal. This used to be called once, at the
/// *end* of a run, under the name `fts_finalize_after_text_indexing` and the
/// comment "nudge FTS5 to merge its index segments" — so a fresh index did its
/// entire first bulk load at FTS5's default threshold of 4, every later run
/// silently inherited 8 from the config table, and no merge was ever performed
/// at all.
pub fn fts_set_automerge(conn: &Connection, segments: u8) {
if let Err(e) = conn.execute(
"INSERT INTO searchabletext(searchabletext, rank) VALUES('automerge', 8)",
[],
"INSERT INTO searchabletext(searchabletext, rank) VALUES('automerge', ?1)",
[segments as i64],
) {
crate::log_warn!("FTS automerge failed (non-fatal): {}", e);
}
}
/// FTS5's crisis-merge threshold: the segment count at which it stops deferring
/// and *forces* a merge, whatever `automerge` would have preferred.
///
/// Raised from FTS5's default of 16 to 32 — half way to the 64 that measured
/// identically, so the safety valve this is stays nearer where SQLite put it.
/// Two builds each, 10,000-file tree, everything else equal:
///
/// | crisismerge | index | written | cold | search (unlimited) |
/// |---:|---:|---:|---:|---:|
/// | 16 (default) | 311.8 / 289.9 MiB | 1008 / 973 MiB | 5.54 / 5.51 s | 755–769 ms |
/// | 32 | **268.2 / 266.1 MiB** | 946 / 964 MiB | 5.53 / 5.52 s | 756–762 ms |
///
/// A **13% smaller index for no cost in time or search**, and — the part worth
/// noticing — a far more *stable* one: the default's size swings 290–312 MiB
/// between builds where this lands within 2 MiB of itself. Fewer forced merges
/// mid-load leave the final merge a tidier structure to consolidate.
const WRITE_CRISISMERGE: u8 = 32;
/// Apply the write-side FTS5 settings, before a run starts writing.
///
/// `pgsz` was swept here too and **rejected**: at 8192 and 16384 it wrote
/// 1061 MiB and 1008 MiB against the default's 943 MiB, for an index the same
/// size. It is a runtime option like these two — settable on an existing table,
/// not creation-time-only — so trying it cost nothing and needed no schema
/// change; it simply does not pay on this workload.
pub fn fts_begin_bulk_write(conn: &Connection) {
fts_set_automerge(conn, WRITE_AUTOMERGE);
if let Err(e) = conn.execute(
"INSERT INTO searchabletext(searchabletext, rank) VALUES('crisismerge', ?1)",
[WRITE_CRISISMERGE as i64],
) {
crate::log_warn!("FTS crisismerge failed (non-fatal): {}", e);
}
}
/// Merge FTS5 segments once a run has finished writing.
///
/// A real merge, which is what this function's name has always claimed and what
/// it never did. Cheap next to the load — measured at +0.2 s and +5 MiB on a
/// 20,000-file tree — and it is what reclaims the tombstones a
/// `contentless_delete` table accumulates, which is why the incremental callers
/// (`scope`, `cleanup_stale_index_entries`) want it after removing rows.
///
/// Deliberately **not** `'optimize'`. That merges everything into one segment
/// and costs, on the same tree, +1.8 s and +900 MiB written — for no measurable
/// search gain: it took the segment-index from 7,337 rows to 730 and left an
/// unlimited search within noise of where it started.
///
/// Best-effort; failure is logged and swallowed, because an unconsolidated
/// index is slower to search and still correct.
pub fn fts_finalize_after_text_indexing(conn: &Connection) {
// A negative page budget means "keep merging until there is nothing left
// worth merging", rather than doing a fixed slice of the work.
if let Err(e) = conn.execute(
"INSERT INTO searchabletext(searchabletext, rank) VALUES('merge', -16)",
[],
) {
crate::log_warn!("FTS merge failed (non-fatal): {}", e);
}
}
/// An owned, fully-derived file record: everything needed to insert or
/// update a `files` row, produced by [`prepare_file_record`].
#[derive(Debug, Clone)]
pub struct OwnedNewFile {
pub name: String,
pub path: String,
/// The containing directory, ending in the platform separator. With
/// [`OwnedNewFile::name`] it is both the row's key and, concatenated, its
/// path — see [`super::paths::dir_to_db_parent`].
pub parent: String,
pub size: u64,
pub mtime: u64,
@ -130,10 +234,15 @@ pub struct OwnedNewFile {
}
impl OwnedNewFile {
/// The file's path, rebuilt. Callers that only want it for a message
/// should say so — nothing stores this.
pub fn path(&self) -> String {
format!("{}{}", self.parent, self.name)
}
pub fn as_new_file(&self) -> NewFile<'_> {
NewFile {
name: &self.name,
path: &self.path,
parent: &self.parent,
size: self.size,
mtime: self.mtime,
@ -165,12 +274,15 @@ pub fn hash_failure_counts() -> (u64, u64) {
/// already holds. The single implementation behind both full-run batches
/// and incremental watcher updates.
///
/// `path` must already be canonical and in `files.path` spelling (see
/// `path` must already be canonical and in stored spelling (see
/// [`path_to_db_string`]), and must still name the file once parsed back into
/// a [`Path`] — this opens it by that string. A path that only survived
/// `to_string_lossy` does not qualify; callers holding the original [`Path`]
/// screen it with [`warn_if_unrepresentable`] first. Callers holding an
/// unresolved path want [`prepare_file_record_from_path`] instead.
/// `to_string_lossy` does not qualify: the lossy spelling of one name is the
/// real name of another, so it would hash and index the wrong file. The walk
/// screens for that on the directory entry, before the path is even built
/// (`crate::walk::read_directory`); [`prepare_file_record_from_path`], which
/// is what callers holding an unresolved path want, screens with
/// [`warn_if_unrepresentable`].
///
/// Returns `None` for anything that isn't a readable regular file, with a
/// warning when hashing fails.
@ -213,10 +325,11 @@ pub fn prepare_file_record(
}
};
let name = Path::new(path)
.file_name()
.map(|n| n.to_string_lossy().into_owned())?;
let parent = parent_str(path);
// One split for both halves of the key, and the separator stays with the
// parent so the two concatenate back into `path`. `None` here means the
// caller handed us something that cannot be a file's path at all.
let (parent, name) = split_db_path(path)?;
let (parent, name) = (parent.to_string(), name.to_string());
// Sniff from the bytes hashing already read; an empty head falls back to
// the extension.
let mime = guess_mime_from_head(Path::new(path), &head);
@ -255,7 +368,6 @@ pub fn prepare_file_record(
Some(OwnedNewFile {
name,
path: path.to_string(),
parent,
size,
mtime,

244
crates/quicksearch-core/src/file_handling/tests.rs
View file

@ -300,6 +300,41 @@ fn db_key_for_a_vanished_path_canonicalizes_what_remains() {
std::fs::remove_dir_all(&root).ok();
}
/// The hazard every screen in the codebase exists for, stated once here.
///
/// `path_to_db_string` is many-to-one, and the collapsed spelling is not
/// garbage — it is a perfectly ordinary filename that a *different* file can
/// really have. So the stored key for an unrepresentable path is another
/// file's key, and any use of it (open, hash, delete by path, delete a subtree
/// range) lands on that file instead. `warn_if_unrepresentable` is how a
/// caller holding a single path avoids ever building one.
#[test]
fn the_stored_spelling_of_an_unrepresentable_path_is_another_files_key() {
let dir = Path::new("/docs");
let bad = dir.join(crate::testutil::unrepresentable_name("report", ".txt"));
let twin = dir.join(crate::testutil::lossy_twin("report", ".txt"));
assert!(warn_if_unrepresentable(&bad));
assert!(
!warn_if_unrepresentable(&twin),
"the twin is an ordinary name and must pass"
);
assert_eq!(
path_to_db_string(&bad),
path_to_db_string(&twin),
"two different files, one stored key"
);
// And the same one component deeper, which is why a bad *directory* has to
// be pruned rather than walked: the collision is inherited by everything
// beneath it.
assert_eq!(
path_to_db_string(&bad.join("child.txt")),
path_to_db_string(&twin.join("child.txt"))
);
assert!(warn_if_unrepresentable(&bad.join("child.txt")));
}
#[test]
fn db_key_for_an_entirely_missing_path_falls_back_to_the_raw_spelling() {
let nowhere = Path::new("relative-thing-that-does-not-exist.txt");
@ -396,3 +431,212 @@ fn hash_covers_size_and_head_only() {
std::fs::remove_dir_all(&root).ok();
}
/// The invariant the whole schema rests on: `parent` and `name` concatenate
/// back into the path, with no separator logic at the join.
///
/// Checked against real paths built by `Path::join` rather than by string
/// formatting, so a platform whose separator is not `/` is tested too.
#[test]
fn a_split_path_concatenates_back_into_itself() {
let root = std::path::Path::new(if cfg!(windows) { r"C:\" } else { "/" });
let cases = [
root.join("a.txt"), // a file at the very root
root.join("home").join("me").join("x.md"), // the ordinary case
root.join("dir with spaces").join("y"),
root.join("weird.name").join("z.tar.gz"),
];
for path in cases {
let s = path_to_db_string(&path);
let (parent, name) = split_db_path(&s).expect("a file's path");
assert_eq!(
format!("{}{}", parent, name),
s,
"{:?} must round-trip through its (parent, name) key",
s
);
assert_eq!(
name,
path.file_name().unwrap().to_string_lossy(),
"the name half is the file name"
);
assert_eq!(
parent,
dir_to_db_parent(path.parent().unwrap()),
"the parent half is what `dir_to_db_parent` would store"
);
assert!(
parent.ends_with(MAIN_SEPARATOR),
"a stored parent always ends in a separator: {:?}",
parent
);
assert!(
!name.contains(MAIN_SEPARATOR),
"a stored name never contains one: {:?}",
name
);
}
}
/// A filesystem root already ends in a separator, so `dir_to_db_parent` must
/// not add a second one — otherwise every file directly at the root would be
/// stored under `//` (or `C:\\`) and never found again.
#[test]
fn a_root_directory_does_not_get_a_doubled_separator() {
let root = std::path::Path::new(if cfg!(windows) { r"C:\" } else { "/" });
let parent = dir_to_db_parent(root);
assert_eq!(parent, path_to_db_string(root));
assert!(!parent.ends_with(&format!("{}{}", MAIN_SEPARATOR, MAIN_SEPARATOR)));
// And the join still produces a path that names the same file.
assert_eq!(
format!("{}{}", parent, "a.txt"),
path_to_db_string(&root.join("a.txt"))
);
}
/// Strings that are not a file's path have no key, and must say so rather
/// than producing a half-formed one — every lookup helper reads `None` as
/// "not indexed".
#[test]
fn a_string_that_cannot_be_a_files_path_has_no_key() {
assert_eq!(split_db_path("bare-name.txt"), None, "no separator at all");
assert_eq!(
split_db_path(&format!("{}dir{}", MAIN_SEPARATOR, MAIN_SEPARATOR)),
None,
"trailing separator: names a directory, not a file"
);
assert_eq!(split_db_path(""), None);
}
/// On Unix a backslash is an ordinary filename character. Splitting on it
/// there would put half a name in the parent, and the row would be
/// unreachable by the path it was stored under.
#[test]
#[cfg(unix)]
fn a_backslash_is_just_a_character_on_unix() {
let (parent, name) = split_db_path(r"/home/me/back\slash.txt").expect("a file's path");
assert_eq!(parent, "/home/me/");
assert_eq!(name, r"back\slash.txt");
// Same on the directory side: a folder genuinely named `weird\` still
// gets its own separator appended.
assert_eq!(
dir_to_db_parent(std::path::Path::new(r"/tmp/weird\")),
r"/tmp/weird\/"
);
}
/// walkdir defaults `follow_root_links` to **true**, independently of
/// `follow_links` — so a root that is itself a symlink was descended even with
/// following off. It matters more than a duplicate scan: the records are
/// stored under the canonicalized path, so a target outside every configured
/// root lands where no sweep range reaches and stays there until a rebuild.
#[test]
#[cfg(unix)]
fn a_symlinked_root_is_not_descended_when_following_is_off() {
let base = tmp_tree();
let real = base.join("real");
touch(&real.join("inside.txt"));
let link = base.join("link");
std::os::unix::fs::symlink(&real, &link).unwrap();
let ignore = IgnoreSet::compile(&[]).unwrap();
let names: Vec<String> = filtered_walk(
link.to_str().unwrap(),
false,
false,
&ignore,
&UnreadableDirs::default(),
)
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
assert!(
names.is_empty(),
"a symlinked root must not be descended: {names:?}"
);
// With following on it is descended, as it always was.
let names: Vec<String> = filtered_walk(
link.to_str().unwrap(),
true,
false,
&ignore,
&UnreadableDirs::default(),
)
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
assert_eq!(names, vec!["inside.txt"]);
std::fs::remove_dir_all(&base).ok();
}
/// A symlink *inside* a root is the same hazard one level down: yielding it
/// as a file indexes it under the canonicalized target path.
#[test]
#[cfg(unix)]
fn a_symlink_inside_a_root_is_skipped_when_following_is_off() {
let base = tmp_tree();
let root = base.join("root");
touch(&root.join("real.txt"));
let outside = base.join("outside");
touch(&outside.join("target.txt"));
std::os::unix::fs::symlink(outside.join("target.txt"), root.join("link.txt")).unwrap();
std::os::unix::fs::symlink(&outside, root.join("linkdir")).unwrap();
let ignore = IgnoreSet::compile(&[]).unwrap();
let mut names: Vec<String> = filtered_walk(
root.to_str().unwrap(),
false,
false,
&ignore,
&UnreadableDirs::default(),
)
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
names.sort();
assert_eq!(names, vec!["real.txt"], "only the real file may be walked");
// Following on: the link resolves and its target is walked through it.
let mut names: Vec<String> = filtered_walk(
root.to_str().unwrap(),
true,
false,
&ignore,
&UnreadableDirs::default(),
)
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
names.sort();
assert_eq!(names, vec!["link.txt", "real.txt", "target.txt"]);
std::fs::remove_dir_all(&base).ok();
}
/// The same for the directory walk the watcher registers from: it must not
/// spend descriptors on a subtree the indexer will discard.
#[test]
#[cfg(unix)]
fn a_symlinked_root_yields_no_directories_when_following_is_off() {
let base = tmp_tree();
let real = base.join("real");
touch(&real.join("sub/inside.txt"));
let link = base.join("link");
std::os::unix::fs::symlink(&real, &link).unwrap();
let ignore = IgnoreSet::compile(&[]).unwrap();
let dirs: Vec<String> = filtered_dirs(
link.to_str().unwrap(),
false,
false,
&ignore,
&UnreadableDirs::default(),
)
.map(|e| e.path().display().to_string())
.collect();
assert!(
dirs.is_empty(),
"a symlinked root must yield no directories: {dirs:?}"
);
std::fs::remove_dir_all(&base).ok();
}

243
crates/quicksearch-core/src/incremental.rs
View file

@ -102,6 +102,20 @@ fn upsert_path(
{
return Ok(Applied::Done);
}
// Symlink-aware, and checked before the followed `metadata` below: with
// following off the walk will not descend a symlinked directory nor
// record a symlinked file, so neither may this. It is not merely
// redundant work — `prepare_file_record_from_path` canonicalizes, so a
// followed link whose target lies outside every root writes a row no
// sweep range covers, and nothing but a rebuild clears it.
if !config.indexing.follow_symlinks {
match std::fs::symlink_metadata(path) {
Ok(md) if md.file_type().is_symlink() => return Ok(Applied::Done),
// Already gone again — the pending Remove event handles it.
Err(_) => return Ok(Applied::Done),
Ok(_) => {}
}
}
let Ok(meta) = std::fs::metadata(path) else {
// Already gone again — the pending Remove event handles it.
return Ok(Applied::Done);
@ -110,11 +124,17 @@ fn upsert_path(
// A moved-in tree surfaces as one directory event; walk it with
// the same filters as a full run.
//
// A non-UTF-8 path is a whole subtree missing from the index, so it
// is an error (the caller schedules a full run) rather than a quiet
// `Ok`.
// A path the index cannot spell, alongside the ignore and hidden
// short-circuits above: a genuine "nothing indexable here", not a
// failure.
//
// It reads as a whole subtree missing from the index, which it is —
// but reporting that as an error would set `needs_full_run`
// (`coordinator::inner::apply_pending`), and a full run screens the
// same subtree out for the same reason. The reindex could not fix it,
// so every write inside such a directory would buy another one.
let Some(root) = path.to_str() else {
return Err(format!("directory path is not valid UTF-8: {:?}", path));
return Ok(Applied::Done);
};
// Streamed, not collected: `mv` of a large tree is one event, and
// materialising its entries first is a `DirEntry` per file resident
@ -124,10 +144,18 @@ fn upsert_path(
//
// `skip` rather than re-testing every entry: `upsert_file` on an
// unchanged file is cheap but not free, and paying it again for
// everything already done would make a large tree quadratic in the
// number of turns it takes. The walk order is deterministic for an
// unchanged tree; if the tree does change under us the count is only
// an optimisation, and the next full run is what makes it exact.
// everything already done would cost a transaction and a hash per
// already-indexed file on every turn.
//
// Two honest limits on that. The walk itself is *not* skipped — the
// iterator still reads every directory it passes over (and on Windows
// stats every entry), so the traversal cost stays quadratic in the
// number of turns even though the writes do not. And the count is a
// position, not an identity: if the tree changes under us the entries
// shift, so `skip(n)` skips the wrong files and those get no row until
// the next full run. Both are acceptable for a moved-in tree, which is
// finite and static in the usual case; if either ever matters, resume
// by last-path rather than by count.
let mut done = budget.resume_from;
for entry in filtered_walk(
root,
@ -166,12 +194,12 @@ fn upsert_file(
let existing: Option<(i64, i64)> = tx
.query_row(
"SELECT id, mtime FROM files WHERE path = ?1",
rusqlite::params![rec.path],
"SELECT id, mtime FROM files WHERE parent = ?1 AND name = ?2",
rusqlite::params![rec.parent, rec.name],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.optional()
.map_err(|e| format!("lookup {}: {}", rec.path, e))?;
.map_err(|e| format!("lookup {}: {}", rec.path(), e))?;
let file_id = match existing {
Some((_, mtime)) if mtime.max(0) as u64 == rec.mtime => return Ok(()),
@ -200,7 +228,7 @@ fn upsert_file(
extract_and_store(
&tx,
file_id,
&rec.path,
&rec.path(),
rec.mime.as_deref(),
registry,
config,
@ -232,6 +260,14 @@ pub fn remove_paths(
.transaction()
.map_err(|e| format!("begin incremental tx: {}", e))?;
for path in batch {
// A path the index cannot spell was never indexed, so there is
// nothing here to delete — and `db_key_for_missing_path` is lossy,
// so going ahead would key the row of whichever *different* file
// owns the lossy spelling and delete it, plus its whole subtree
// range below.
if path.to_str().is_none() {
continue;
}
// The insert side stores a canonicalized path, so the raw event
// spelling is not a usable key — but the file is already gone, so
// `canonicalize` cannot be called on it directly either.
@ -327,7 +363,7 @@ mod tests {
fn row(&self, path: &str) -> Option<(i64, i64, i64)> {
self.conn
.query_row(
"SELECT id, mtime, content_state FROM files WHERE path = ?1",
"SELECT id, mtime, content_state FROM files WHERE parent || name = ?1",
rusqlite::params![path],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
)
@ -371,6 +407,48 @@ mod tests {
/// The collapse must not change what ends up deleted — only how much work
/// it takes to get there.
/// A resume point must not outlive the event it describes.
///
/// The coordinator prunes `resume_from` alongside the queues; this is the
/// half of that contract the module itself can state — resuming from a
/// count that belonged to some earlier walk skips real files, and they get
/// no row until the next full run.
#[test]
fn resuming_past_the_end_indexes_nothing_rather_than_the_wrong_files() {
let mut f = Fixture::new();
for i in 0..3 {
f.write(&format!("sub/f{i}.txt"), "body");
}
let sub = f.dir.join("sub");
// A count larger than the tree: every entry is skipped, and the result
// is an empty index rather than an arbitrary subset.
let outcome =
f.apply_resuming(&FsEvent::Create(sub.clone()), Duration::from_secs(3600), 99);
assert_eq!(outcome, Applied::Done);
assert_eq!(f.counts().0, 0);
// From zero — what a pruned resume point gives the next turn — the
// whole tree lands.
f.apply(&FsEvent::Create(sub));
assert_eq!(f.counts().0, 3);
}
/// `batch_size = 0` must not panic the writer.
///
/// `chunks(0)` panics, and this one runs on the indexing thread above the
/// arm that would publish `IndexingStatus::Error` — so before the clamp a
/// hand-edited zero wedged indexing for the session while the UI went on
/// reading "Running". Every sibling call site already had `.max(1)`.
#[test]
fn a_zero_batch_size_does_not_panic_the_writer() {
let mut f = Fixture::new();
f.config.processing.batch_size = 0;
f.write("a.txt", "body");
f.apply(&FsEvent::Create(f.dir.join("a.txt")));
assert_eq!(f.counts().0, 1, "the file should still be indexed");
}
/// A directory event is applied in slices, and a slice resumes where the
/// last one stopped instead of re-walking what it already did.
///
@ -635,6 +713,76 @@ mod tests {
assert!(f.row(&survivor).is_some());
}
/// A `Remove` for a path the index cannot spell must delete nothing.
///
/// `db_key_for_missing_path` ends in `path_to_db_string`, which is lossy,
/// and the key it returns is used twice: to delete a row by path, and as
/// the low end of a range that deletes everything beneath it. For an
/// unrepresentable path that key names a *different*, real file — so the
/// event would take that file's row and its whole subtree, for a file that
/// was never indexed in the first place.
#[test]
fn removing_an_unrepresentable_path_spares_its_lossy_twin() {
let mut f = Fixture::new();
let twin = crate::testutil::lossy_twin("report", ".txt");
let kept = f.write(&twin, "the file that must survive");
f.apply(&FsEvent::Create(kept.clone()));
let canonical = f.canonical(&kept);
assert!(f.row(&canonical).is_some(), "seeded");
// Never written to disk: the event alone is enough, and a Remove is
// for a path that is already gone in any case.
let bad = f
.dir
.join(crate::testutil::unrepresentable_name("report", ".txt"));
f.apply(&FsEvent::Remove(bad));
assert!(
f.row(&canonical).is_some(),
"the real file's row was deleted by an event for a different file"
);
assert_eq!(f.counts().0, 1);
}
/// A directory event for a path the index cannot spell is a quiet `Ok`,
/// not an error.
///
/// `coordinator::inner::apply_pending` turns any `Err` here into
/// `needs_full_run`. A full run screens the same subtree out for the same
/// reason, so the reindex could not fix anything — it would just run
/// again on the next write into that directory, forever. Live on Windows,
/// where a `\\wsl.localhost\` or Samba tree can hold such a directory and
/// the root watch is recursive.
#[test]
fn a_directory_event_for_an_unrepresentable_path_is_not_an_error() {
let mut f = Fixture::new();
let bad = f.dir.join(crate::testutil::unrepresentable_name("dir", ""));
if std::fs::create_dir_all(&bad).is_err() {
eprintln!("skipped: this filesystem will not store an unrepresentable name");
return;
}
std::fs::write(bad.join("inside.txt"), "body").unwrap();
let outcome = apply_fs_event(
&mut f.conn,
&FsEvent::Create(bad),
&f.config,
&f.ignore,
&f.registry,
&Budget {
deadline: Instant::now() + Duration::from_secs(3600),
cancel: &AtomicBool::new(false),
resume_from: 0,
},
);
assert_eq!(
outcome,
Ok(Applied::Done),
"an unindexable subtree is not a failure"
);
assert_eq!(f.counts().0, 0, "and nothing under it is indexed");
}
#[test]
fn oversize_files_get_content_na() {
let mut f = Fixture::new();
@ -645,4 +793,73 @@ mod tests {
let (_, _, content_state) = f.row(&canonical).unwrap();
assert_eq!(content_state, repo::STATE_NA);
}
/// `follow_symlinks = false` means the walk will not descend a symlinked
/// directory, and the live path must agree. It used to disagree twice
/// over: `metadata` follows, so the link read as a directory, and
/// `prepare_file_record_from_path` canonicalizes before storing — so a
/// target outside every root produced rows under the target's real path
/// that no sweep range covers. Nothing but a rebuild cleared them.
#[test]
#[cfg(unix)]
fn a_symlinked_directory_is_not_followed_when_following_is_off() {
let mut f = Fixture::new();
assert!(
!f.config.indexing.follow_symlinks,
"the default this test is about"
);
// A tree outside every configured root, and a link to it inside one.
let outside = crate::testutil::scratch_dir("incr-outside");
std::fs::write(outside.join("secret.txt"), "content out of scope").unwrap();
let link = f.dir.join("link");
std::os::unix::fs::symlink(&outside, &link).unwrap();
f.apply(&FsEvent::Create(link.clone()));
assert_eq!(
f.counts(),
(0, 0, 0),
"nothing under the link may be indexed"
);
std::fs::remove_dir_all(&outside).ok();
}
/// The same for a symlinked *file*: canonicalization would file it under
/// the target's path, which is outside the root that produced the event.
#[test]
#[cfg(unix)]
fn a_symlinked_file_is_not_indexed_when_following_is_off() {
let mut f = Fixture::new();
let outside = crate::testutil::scratch_dir("incr-outside-file");
let target = outside.join("target.txt");
std::fs::write(&target, "content out of scope").unwrap();
let link = f.dir.join("link.txt");
std::os::unix::fs::symlink(&target, &link).unwrap();
f.apply(&FsEvent::Create(link.clone()));
assert_eq!(f.counts(), (0, 0, 0), "the link must not be indexed");
std::fs::remove_dir_all(&outside).ok();
}
/// With following on, the link is indexed — under the target's real path,
/// which is what canonicalization has always done. This is the other half
/// of the guard: it must gate on the setting, not refuse symlinks outright.
#[test]
#[cfg(unix)]
fn a_symlinked_file_is_indexed_when_following_is_on() {
let mut f = Fixture::new();
f.config.indexing.follow_symlinks = true;
let target = f.write("target.txt", "greetings earthling");
let link = f.dir.join("link.txt");
std::os::unix::fs::symlink(&target, &link).unwrap();
f.apply(&FsEvent::Create(link.clone()));
let canonical = f.canonical(&target);
assert!(
f.row(&canonical).is_some(),
"the link should have indexed its target"
);
}
}

45
crates/quicksearch-core/src/indexing/mod.rs
View file

@ -204,6 +204,20 @@ impl IndexingService {
}
pub fn stop_indexing(&self) -> Result<(), String> {
self.stop_indexing_inner(true)
}
/// [`Self::stop_indexing`] for a caller that is about to delete the file.
///
/// Skips the checkpoint. `checkpoint_truncate` copies the whole write-ahead
/// log into the database — up to `maximum_wal_size`, 512 MiB by default —
/// and the next two statements delete both. It is pure cost, and it is
/// paid on the coordinator's thread while the user waits for a rebuild.
fn stop_indexing_for_delete(&self) -> Result<(), String> {
self.stop_indexing_inner(false)
}
fn stop_indexing_inner(&self, checkpoint: bool) -> Result<(), String> {
self.command_tx
.send(IndexingCommand::Stop)
.map_err(|e| format!("Failed to send stop command: {}", e))?;
@ -226,15 +240,27 @@ impl IndexingService {
// Flush the WAL and release the shared connection. WAL mode itself
// stays on — it's the persistent journal mode for the index.
if let Some(db_conn_arc) = crate::lock_ok(&self.db_connection).take() {
let conn = crate::lock_ok(&db_conn_arc);
if let Err(e) = crate::db::repo::checkpoint_truncate(&conn) {
crate::log_warn!("{}", e);
if checkpoint {
let conn = crate::lock_ok(&db_conn_arc);
if let Err(e) = crate::db::repo::checkpoint_truncate(&conn) {
crate::log_warn!("{}", e);
}
}
}
Ok(())
}
/// Publish a failure that happened *outside* a run, so the status bar
/// shows it rather than only the log ring.
///
/// A failed "Rebuild index" is the case this exists for: the user asked
/// for something, it did not happen, and a warning in a tab they are not
/// looking at is not telling them.
pub fn report_error(&self, message: String) {
*crate::lock_ok(&self.status) = IndexingStatus::Error(message);
}
pub fn get_status(&self) -> IndexingStatus {
crate::lock_ok(&self.status).clone()
}
@ -279,7 +305,7 @@ impl IndexingService {
/// Stop indexing and delete the database file for a clean rebuild
pub fn delete_index_for_rebuild(&self, db_path: &str) -> Result<(), String> {
self.stop_indexing()
self.stop_indexing_for_delete()
.map_err(|e| format!("Failed to stop indexing: {}", e))?;
// Cut short the optimize pass too: the file about to be deleted is
// the file it holds open.
@ -308,8 +334,15 @@ impl IndexingService {
// [`db::bump_index_epoch`].
db::bump_index_epoch();
if std::path::Path::new(db_path).exists() {
std::fs::remove_file(db_path)
let path = std::path::Path::new(db_path);
if path.exists() {
// Retried, as every other delete of this file is: on Windows it
// fails while any handle is open, and the readers are exactly the
// ones a user has just been using — the search worker holds its
// connection for `IDLE_RELEASE` after the last keystroke. The
// caller releases that one first; this covers a scan or a
// duplicates pass that has not noticed yet.
crate::platform::remove_file_retrying(path)
.map_err(|e| format!("Failed to delete database file: {}", e))?;
}
for suffix in ["-wal", "-shm", "-journal"] {

161
crates/quicksearch-core/src/indexing/pipeline.rs
View file

@ -3,6 +3,7 @@
use rusqlite::Connection;
use std::collections::HashSet;
use std::path::Path;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;
@ -13,7 +14,8 @@ use crate::db;
use crate::db::repo;
use crate::extract::Registry;
use crate::file_handling::{
cleanup_stale_index_entries, count_tree_entries_fast, fts_finalize_after_text_indexing,
cleanup_stale_index_entries, count_tree_entries_fast, fts_begin_bulk_write,
fts_finalize_after_text_indexing,
mark_oversize_pending_na, normalize_root_string, process_batch_inserts, process_batch_updates,
store_extracted, ExtractCursor, ExtractScope, FileIndexAction, OwnedNewFile,
};
@ -74,6 +76,81 @@ fn wal_len(path: &str) -> u64 {
std::fs::metadata(path).map(|m| m.len()).unwrap_or(0)
}
/// Free space at which a run gives up rather than keep writing.
///
/// Filling the volume the index sits on is not a clean failure. SQLite's
/// guard against a short wal-index only covers the moment that file is
/// *extended*; a later write goes through the `-shm` mmap, and a page fault
/// the filesystem cannot back is delivered as **SIGBUS**, which no `Result`
/// can catch. On a copy-on-write filesystem (btrfs, ZFS) even overwriting an
/// already-allocated page needs a new extent, so a full volume can take the
/// process down on a write to a page that has existed for hours. Stopping
/// with an error while there is still room is the only safe end.
const DISK_FLOOR: u64 = 128 * 1024 * 1024;
/// Share of the free space above [`DISK_FLOOR`] the log may occupy.
///
/// The log is what grows unboundedly between checkpoints, so its cap is the
/// figure that has to fit in what is left. A quarter leaves room for the
/// index's own growth, the FTS segments a merge writes beside it, and
/// whatever else on the machine wants the same volume.
const WAL_SHARE_OF_FREE: u64 = 4;
/// The configured checkpoint threshold, lowered to what the volume can
/// actually absorb.
///
/// `maximum_wal_size` is a stall-frequency knob chosen against a roomy disk;
/// on a nearly full one its 512 MiB default is more than everything left.
/// Checkpointing sooner costs some lock acquisitions and keeps the log inside
/// the space available — see [`DISK_FLOOR`] for why running out is not
/// survivable.
///
/// A configured `0` (forced checkpoints off) is bounded like any other value
/// rather than special-cased: the knob turns off a *performance* behaviour and
/// is not a licence to fill the disk. On a roomy volume the derived bound is
/// larger than any run's log, so `0` keeps its meaning without a second rule.
/// Unknown free space changes nothing.
fn wal_cap_for_volume(configured: u64, db_path: &Path) -> u64 {
let Some(free) = crate::platform::available_space(db_path) else {
return configured;
};
let effective = wal_cap_for_free(configured, free);
if effective != configured {
crate::log_info!(
"{} free where the index lives: forcing a WAL checkpoint every {} MiB \
instead of {}",
human_mib(free),
effective / (1024 * 1024),
match configured {
0 => "never".to_string(),
n => format!("{} MiB", n / (1024 * 1024)),
}
);
}
effective
}
/// The arithmetic of [`wal_cap_for_volume`], split from the syscall so it is
/// testable without a filesystem of a chosen size — the same split the rest of
/// the codebase makes for anything decidable without asking the kernel.
pub(super) fn wal_cap_for_free(configured: u64, free: u64) -> u64 {
let room = free.saturating_sub(DISK_FLOOR) / WAL_SHARE_OF_FREE;
// Never below the floor a configured value would be raised to: checkpoints
// more often than that cost more in lock acquisition than the log costs in
// space, and the in-run check is what actually stops a doomed run.
let capped = room.max(crate::config::MINIMUM_WAL_SIZE);
// `0` is "no cap", so it loses every `min` — hence the explicit arm.
if configured == 0 {
capped
} else {
configured.min(capped)
}
}
fn human_mib(bytes: u64) -> String {
format!("{} MiB", bytes / (1024 * 1024))
}
/// Flips an [`AtomicBool`] when dropped. Held by `run_indexing` so the
/// per-root count subprocesses die on every exit path of a run.
struct CancelOnDrop(Arc<AtomicBool>);
@ -204,6 +281,17 @@ impl RootPipeline {
},
// Earlier runs' rows count once the pass has counted them; until
// then only this run's, so the figure never goes backwards.
//
// Not clamped against `extract_total`, deliberately. The count now
// runs *behind* the pass's first page rather than blocking its
// workers in front of it (see `content::feeder`), so a row written
// while it was in flight is seen by it as already done and counted
// again in `written` — the numerator can briefly overshoot. That is
// already a shape this reports: a pass fed rows from outside its own
// range writes them with an `extract_total` of zero, which
// `an_extracting_turn_lands_its_leftovers_one_slice_at_a_time`
// pins. Clamping here broke that test and would have hidden the
// case it exists to describe.
extracted: totals.map_or(self.written, |t| t.already_done + self.written),
extract_total: totals.map(|t| t.pending + t.already_done),
current_file: self.current_file.clone(),
@ -734,6 +822,45 @@ impl IndexingService {
}
Self::update_config(&conn, config, &roots)?;
// Before a single row is written, so the whole load runs at the
// write-side threshold. Setting it afterwards — which is what this
// used to do — left every fresh index's first run at FTS5's default.
fts_begin_bulk_write(&conn);
// Turn off SQLite's automatic checkpointing for the duration of the
// run, because during a run it cannot do its job and charges full
// price for failing.
//
// The default fires every 1000 pages (~4 MB) and copies the log back
// into the database — but it can only *reset* the log at an instant no
// reader holds a read mark, and this run keeps a reader per root from
// start to finish (the walk's row prefetcher, then the content pass's
// feeder). So it copied pages back perpetually and never truncated
// anything: measured on a 10,000-file tree, the log grew to 144 MiB and
// stayed there while the process wrote **1,220 MiB** — the same log
// copied back some seven times over.
//
// Safe here and nowhere else, which is why it is set on this connection
// rather than in `PRAGMAS_FAST`: this is the one writer that already
// owns the machinery to land its own log. `wal_cap_for_volume` bounds
// how large it may grow — by free space, not just by
// `maximum_wal_size` — the loop below forces a checkpoint at that cap,
// and the optimize pass checkpoints again at the end. A writer without
// all three (`cli::clear_path`, say) must keep the automatic one.
//
// Measured, two runs each, same tree:
//
// | | cold | written | log peak |
// |---|---:|---:|---:|
// | autocheckpoint on (default) | 7.19 / 7.37 s | 1219 / 1227 MiB | 144 MiB |
// | off | **5.59 / 5.50 s** | **990 / 992 MiB** | 512 MiB |
//
// The log gets larger and the run gets cheaper, which is the trade the
// default is making backwards for this workload.
if let Err(e) = conn.execute_batch("PRAGMA wal_autocheckpoint = 0;") {
crate::log_warn!("could not disable autocheckpoint (non-fatal): {}", e);
}
// No up-front load of the whole `files` table: each walk's prefetcher
// fetches one directory's rows at a time.
let conn_mutex = Arc::new(Mutex::new(conn));
@ -780,10 +907,11 @@ impl IndexingService {
// Log size at which to force a checkpoint; see [`wal_len`] for why
// SQLite's autocheckpoint cannot be left to do this.
let wal_path = format!("{}-wal", db_path);
let wal_cap = match config.processing.maximum_wal_size {
let configured_cap = match config.processing.maximum_wal_size {
0 => 0,
n => n.max(crate::config::MINIMUM_WAL_SIZE),
};
let wal_cap = wal_cap_for_volume(configured_cap, Path::new(db_path));
let mut checkpoint_at = wal_cap;
// Walks first, one slice each, then a single extraction slice.
@ -852,6 +980,24 @@ impl IndexingService {
crate::log_warn!("{}", e);
}
}
// Only here, not every round: this is the moment the log is
// at its largest, and it costs one syscall per checkpoint
// rather than one per writer turn. The checkpoint above has
// just returned whatever it could, so what is left is the
// honest figure.
if let Some(free) = crate::platform::available_space(Path::new(db_path)) {
if free < DISK_FLOOR {
return Err(format!(
"Stopped: only {} free where the index lives ({}). \
Indexing needs room for its write-ahead log, and \
filling the disk can kill the process outright \
rather than fail cleanly. Free some space and run \
again — what is already indexed is kept.",
human_mib(free),
db_path
));
}
}
// Re-armed from what is on disk: a checkpoint that lost the
// race then costs one attempt per further `wal_cap` of
// growth, not a retry every round.
@ -879,6 +1025,17 @@ impl IndexingService {
.is_some();
// Nothing walking (extracting passes have no such handle);
// fall back to the sleep.
//
// Parking on an extracting root's channel here was tried and
// **measured at nothing**: over a cold run of a 10,000-file
// tree the loop found nothing 91 times and only *one* of those
// reached this sleep, because the writer is the bottleneck
// during extraction and is almost never idle. One 2 ms sleep a
// run — 15.6 ms on Windows, where the timer granularity is what
// makes this comment worth having — did not justify a second
// `wait_ready` and the `pending` slot it needs. Re-measure with
// an extraction-bound corpus (PDFs, a network share) before
// concluding otherwise.
if !waited {
thread::sleep(IDLE_BACKOFF);
}

66
crates/quicksearch-core/src/indexing/tests.rs
View file

@ -166,8 +166,11 @@ fn an_extracting_turn_lands_its_leftovers_one_slice_at_a_time() {
&tx,
&NewFile {
name: &format!("f{}.txt", i),
path: &path.to_string_lossy(),
parent: &tree.to_string_lossy(),
// `dir_to_db_parent`, not `to_string_lossy`: a stored
// parent always ends in a separator, and a row spelled
// without one sorts below every `ExtractCursor` range that
// should contain it.
parent: &crate::file_handling::dir_to_db_parent(&tree),
size: 22,
mtime: 1,
mime: Some("text/plain"),
@ -670,3 +673,62 @@ fn a_run_with_no_roots_is_complete_rather_than_unknown() {
assert_eq!(o.processed, 0);
assert_eq!(o.total, Some(0));
}
/// A roomy volume leaves `maximum_wal_size` exactly as configured: this is a
/// safety valve, not a second tuning knob.
#[test]
fn a_roomy_volume_does_not_move_the_checkpoint_threshold() {
use super::pipeline::wal_cap_for_free;
let configured = 512 * 1024 * 1024;
assert_eq!(
wal_cap_for_free(configured, 500 * 1024 * 1024 * 1024),
configured
);
// Exactly enough: floor plus four times the log.
let just_enough = 128 * 1024 * 1024 + configured * 4;
assert_eq!(wal_cap_for_free(configured, just_enough), configured);
}
/// A tight volume checkpoints sooner, so the log cannot grow into the space
/// that is left. Running out is not a clean failure — the wal-index is reached
/// through an mmap, and a write the filesystem cannot back is a SIGBUS.
#[test]
fn a_tight_volume_lowers_the_checkpoint_threshold() {
use super::pipeline::wal_cap_for_free;
let configured = 512 * 1024 * 1024;
// 1 GiB free: 896 MiB above the floor, a quarter of which is 224 MiB.
let got = wal_cap_for_free(configured, 1024 * 1024 * 1024);
assert_eq!(got, 224 * 1024 * 1024);
assert!(got < configured);
}
/// `0` means "never force a checkpoint", which is a performance choice and not
/// a licence to fill the disk — so it is bounded by the volume like any other
/// value. On a roomy disk that bound is larger than any run's log, which is
/// how the setting keeps its meaning without a second rule to special-case it.
#[test]
fn disabled_checkpoints_are_still_bounded_by_the_volume() {
use super::pipeline::wal_cap_for_free;
let roomy = wal_cap_for_free(0, 500 * 1024 * 1024 * 1024);
assert!(
roomy > 100 * 1024 * 1024 * 1024,
"effectively never on a roomy disk, got {} bytes",
roomy
);
let tight = wal_cap_for_free(0, 1024 * 1024 * 1024);
assert_eq!(tight, 224 * 1024 * 1024, "a tight one checkpoints anyway");
}
/// Below the floor there is nothing sensible left to divide, and the run is
/// about to be stopped by the in-run check anyway — so the threshold bottoms
/// out at the same floor a configured value is raised to, never at zero.
#[test]
fn a_full_volume_bottoms_out_at_the_minimum_rather_than_zero() {
use super::pipeline::wal_cap_for_free;
for free in [0, 1024, 127 * 1024 * 1024] {
assert_eq!(
wal_cap_for_free(512 * 1024 * 1024, free),
crate::config::MINIMUM_WAL_SIZE
);
}
}

80
crates/quicksearch-core/src/live.rs
View file

@ -329,7 +329,18 @@ impl Loop {
&mut self.pending,
&mut self.orphan_to,
);
if !self.pending.is_empty() && self.settle_at.is_none() {
// Either queue, not just `pending`. `orphan_to` collects
// rename destinations whose source is not on screen, and
// it is drained only by `flush_settled` — so arming on
// `pending` alone means churn that touches no visible row
// pushes for the life of the arm and is never taken. That
// is a leak, and on Windows it is also a wrong answer:
// with no `RenameMode::Both` there, the pairing below
// needs `orphans.len() == 1`, so one stale entry turns
// every later rename into "gone".
if (!self.pending.is_empty() || !self.orphan_to.is_empty())
&& self.settle_at.is_none()
{
self.settle_at = Some(Instant::now() + SETTLE);
}
}
@ -368,9 +379,10 @@ impl Loop {
if q.pattern.is_wildcard() {
return None;
}
let folded = q.term.to_ascii_lowercase();
let k = edit_budget(folded.len(), config.search.fuzzy_max_edits)?;
Bitap::new(folded.as_bytes(), k)
// As typed, exactly as the pass builds it: the matcher is
// case-insensitive in its own mask table.
let k = edit_budget(q.term.len(), config.search.fuzzy_max_edits)?;
Bitap::new(q.term.as_bytes(), k)
});
self.config = Some(config);
@ -593,20 +605,22 @@ impl Loop {
let Some(text) = crate::file_handling::outcome_body(&outcome) else {
return WindowUpdate::Unchanged;
};
let folded = text.to_ascii_lowercase();
let cut = match tier {
ContentTier::Exact => {
// A literal term always yields a window, marked or not,
// because the passes only ever call this for a body FTS
// already matched. Here the body may genuinely have stopped
// matching, and an unmarked window is how that reads.
//
// The fold is cut here rather than above the match, because
// only this arm needs one now — the fuzzy matcher folds in its
// own mask table.
let folded = text.to_ascii_lowercase();
crate::search::cascade::text_snippet(&query.pattern, text, &folded)
.filter(|snip| !snip.ranges.is_empty())
}
ContentTier::Fuzzy => match &self.fuzzy {
Some(bitap) => {
crate::search::cascade::fuzzy_snippet(bitap, text, &folded).map(|(_, s)| s)
}
Some(bitap) => crate::search::cascade::fuzzy_snippet(bitap, text).map(|(_, s)| s),
// The term does not fuzz, so a fuzzy row cannot be re-judged;
// leaving it is the honest reading.
None => return WindowUpdate::Unchanged,
@ -673,8 +687,16 @@ fn classify(
) {
use notify::event::{ModifyKind, RenameMode};
let key = |p: &PathBuf| p.to_string_lossy().into_owned();
let is_target = |p: &PathBuf| targets.contains_key(&key(p));
// `to_str`, not `to_string_lossy`. Targets are paths out of the index, so
// they are all representable, and a path that is *not* still has a lossy
// spelling — one that is a perfectly valid path for some other file. Keyed
// lossily, an event for a file we could never index would match a row we
// are displaying and fire a Gone or a Changed at it.
let target_key = |p: &PathBuf| {
p.to_str()
.filter(|k| targets.contains_key(*k))
.map(str::to_owned)
};
match event.kind {
EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => {
@ -684,43 +706,55 @@ fn classify(
let (Some(from), Some(to)) = (event.paths.first(), event.paths.get(1)) else {
return;
};
if is_target(from) {
pending.insert(key(from), Op::Renamed(to.clone()));
} else if is_target(to) {
if let Some(from_key) = target_key(from) {
// A destination the index cannot spell cannot be recorded as a
// rename: the row would carry a path naming a different file,
// and the GUI opens rows by that path. The file did leave the
// searchable world, so `Gone` is the honest update — and the
// one the user would get anyway once the row was verified.
let op = match to.to_str() {
Some(_) => Op::Renamed(to.clone()),
None => Op::Gone,
};
pending.insert(from_key, op);
} else if let Some(to_key) = target_key(to) {
// The atomic-save shape: a temporary file renamed over a row
// we are watching. The row did not move; its contents changed.
pending.insert(key(to), Op::Changed);
pending.insert(to_key, Op::Changed);
}
}
EventKind::Modify(ModifyKind::Name(RenameMode::To)) => {
for path in &event.paths {
if is_target(path) {
pending.insert(key(path), Op::Changed);
} else {
if let Some(key) = target_key(path) {
pending.insert(key, Op::Changed);
} else if path.to_str().is_some() {
// Screened for the same reason: an orphan is paired with a
// lone `Gone` into a `Renamed` below, so an unrepresentable
// one would arrive at the same bad destination.
orphan_to.push(path.clone());
}
}
}
EventKind::Modify(ModifyKind::Name(RenameMode::From)) => {
for path in &event.paths {
if is_target(path) {
if let Some(key) = target_key(path) {
// Provisional; a Both in this same window upgrades it.
pending.entry(key(path)).or_insert(Op::Gone);
pending.entry(key).or_insert(Op::Gone);
}
}
}
EventKind::Remove(_) => {
for path in &event.paths {
if is_target(path) {
pending.insert(key(path), Op::Gone);
if let Some(key) = target_key(path) {
pending.insert(key, Op::Gone);
}
}
}
EventKind::Create(_) | EventKind::Modify(_) => {
for path in &event.paths {
if is_target(path) {
if let Some(key) = target_key(path) {
// A Create at a watched path un-deletes the row.
pending.insert(key(path), Op::Changed);
pending.insert(key, Op::Changed);
}
}
}

124
crates/quicksearch-core/src/live_tests.rs
View file

@ -170,6 +170,92 @@ fn an_ambiguous_windows_window_reports_gone_rather_than_guessing() {
assert_eq!(decided.get("/docs/b.txt"), Some(&Op::Gone));
}
/// [`event`], for a path that cannot be spelled as a `&str`.
fn event_os(kind: EventKind, paths: &[PathBuf]) -> NotifyEvent {
NotifyEvent {
kind,
paths: paths.to_vec(),
attrs: Default::default(),
}
}
/// A path in the displayed set, and its unrepresentable neighbour that
/// `to_string_lossy` collapses onto exactly that spelling.
fn twin_pair() -> (String, PathBuf) {
let shown = format!("/docs/{}", crate::testutil::lossy_twin("report", ".txt"));
let bad = PathBuf::from("/docs").join(crate::testutil::unrepresentable_name("report", ".txt"));
assert_eq!(bad.to_string_lossy(), shown, "the two must collide");
(shown, bad)
}
/// A file the index could never hold still generates events, and its lossy
/// spelling is a displayed row's real path. Keyed lossily, every one of those
/// events lands on that row: a `Remove` of the file we cannot index would mark
/// a completely different file gone, on screen, while it sits there on disk.
#[test]
fn events_for_an_unrepresentable_path_never_touch_its_lossy_twin() {
let (shown, bad) = twin_pair();
let t = targets(&[&shown]);
let decided = window(
&t,
vec![
event_os(
EventKind::Remove(RemoveKind::File),
std::slice::from_ref(&bad),
),
event_os(
EventKind::Create(CreateKind::File),
std::slice::from_ref(&bad),
),
event_os(EventKind::Modify(ModifyKind::Any), &[bad]),
],
);
assert!(
decided.is_empty(),
"the displayed row must be untouched: {decided:?}"
);
}
/// Renamed *to* a name the index cannot spell, the row cannot keep a usable
/// path — `Op::Renamed` carries the destination, and the GUI opens rows by it,
/// so a lossy one would open some other file. It left the searchable world, so
/// the honest answer is `Gone`; dropping the event instead would leave a stale
/// row on screen until something else disturbed it.
#[test]
fn a_rename_to_an_unrepresentable_name_reports_gone() {
let (_, bad) = twin_pair();
let t = targets(&["/docs/a.txt"]);
let decided = window(
&t,
vec![event_os(
EventKind::Modify(ModifyKind::Name(RenameMode::Both)),
&[PathBuf::from("/docs/a.txt"), bad],
)],
);
assert_eq!(decided.get("/docs/a.txt"), Some(&Op::Gone));
}
/// The Windows shape of the same thing: the halves arrive separately, and a
/// lone `Gone` plus a lone arrival are paired into a rename. The arrival must
/// not be a path we cannot spell, or the pairing invents the same bad
/// destination the test above rejects.
#[test]
fn a_split_rename_is_not_paired_with_an_unrepresentable_arrival() {
let (_, bad) = twin_pair();
let t = targets(&["/docs/a.txt"]);
let decided = window(
&t,
vec![
event(
EventKind::Modify(ModifyKind::Name(RenameMode::From)),
&["/docs/a.txt"],
),
event_os(EventKind::Modify(ModifyKind::Name(RenameMode::To)), &[bad]),
],
);
assert_eq!(decided.get("/docs/a.txt"), Some(&Op::Gone));
}
/// A watched directory is full of files that are not on screen. None of them
/// may produce an update — live results never add rows.
#[test]
@ -653,3 +739,41 @@ fn re_arming_drops_the_previous_targets() {
);
assert!(decided.is_empty(), "{decided:?}");
}
/// `orphan_to` collects rename destinations whose source is not a displayed
/// row, and only `flush_settled` drains it — which used to run only when
/// `pending` was non-empty. Churn beside the results therefore pushed a
/// `PathBuf` that was never taken, and the stale entry then *paired* with the
/// next lone `Gone`: a deleted row reported as renamed to a file it has
/// nothing to do with, and the GUI following that path.
///
/// A file moved in from outside is the reliable way to leave a lone orphan on
/// every platform — there is no `From` half to pair it with, so no
/// `RenameMode::Both` follows.
#[test]
fn e2e_an_unpaired_move_in_does_not_capture_a_later_deletion() {
let dir = scratch_dir("live-orphan-drain");
let (watcher, rx, path) = watch_one(&dir, "before.txt");
// Move a file in from a directory nothing is watching: the watcher sees
// the destination and never a source.
let elsewhere = scratch_dir("live-orphan-source");
let outside = elsewhere.join("moved-in.txt");
std::fs::write(&outside, "unrelated").unwrap();
std::fs::rename(&outside, dir.join("moved-in.txt")).unwrap();
// Past the settle window, so the drain has had its chance.
std::thread::sleep(Duration::from_millis(600));
// Now delete the row that *is* on screen.
std::fs::remove_file(&path).unwrap();
let updates = collect(&rx, 1, Duration::from_secs(5));
stop_and_clean(watcher, &dir);
std::fs::remove_dir_all(&elsewhere).ok();
match updates.first() {
Some(LiveUpdate::Gone { path: gone, .. }) => assert_eq!(gone, &path),
other => panic!("a deletion must not pair with unrelated churn: {other:?}"),
}
}

18
crates/quicksearch-core/src/log.rs
View file

@ -65,6 +65,24 @@ pub fn record(level: Level, message: String) {
Level::Warn => format!("Warning: {}", message),
Level::Info => message,
};
// Almost every line here names a path, and a path is whatever someone
// called a file. An escape sequence in one reaches a terminal three ways:
// the stderr write below, a user running the GUI from a shell, and the
// Logs tab's Copy button, which puts the ring on the clipboard for pasting
// into a bug report. Diagnostics are not data — nothing downstream needs
// these bytes exactly — so they are scrubbed unconditionally.
//
// Line breaks are collapsed *first*, because `scrub_controls` counts them
// as controls and would leave `U+FFFD` where a space belongs. One record
// is one line — the ring renders it that way and `writeln!` adds the only
// newline there should be — so a message that arrives multi-line, such as
// a nested error's chain of causes, is flattened rather than boxed.
let text = if text.contains(['\n', '\r']) {
text.replace(['\n', '\r'], " ")
} else {
text
};
let text = crate::textenc::scrub_controls(&text).into_owned();
let _ = writeln!(std::io::stderr(), "{}", text);
lock().push(level, text);
}

327
crates/quicksearch-core/src/platform.rs
View file

@ -407,6 +407,15 @@ pub const WATCH_ROOTS_RECURSIVELY: bool = cfg!(windows);
/// other with `LIKE` would otherwise disagree with itself. `NOCASE` folds ASCII
/// only, which matches what `LIKE` does — non-ASCII paths stay case-sensitive
/// on both sides, consistently.
///
/// **No query interpolates this any more.** The folder filter was its last
/// caller, and it now needs one `LIKE` where it used to need `= COLLATE … OR
/// LIKE`. What is left is a *specification*: this constant and
/// `tests::collation_matches_like_case_folding` are where the rule is written
/// down, and the places that fold paths by hand —
/// [`PATHS_ARE_CASE_INSENSITIVE`], [`crate::config::IgnoreSet`],
/// [`crate::config::Config::is_index_file`] — are required to agree with it.
/// Deleting it would leave that agreement asserted nowhere.
pub const PATH_COLLATION: &str = if cfg!(windows) { "NOCASE" } else { "BINARY" };
/// Whether this platform's filesystem matches names without regard to case —
@ -573,6 +582,324 @@ pub fn open_regular_file(path: &Path) -> std::io::Result<std::fs::File> {
Ok(file)
}
/// Why [`IndexLock::acquire`] did not hand back a lock.
#[derive(Debug)]
pub enum LockError {
/// Another process holds it. `pid` is whatever the holder recorded in the
/// file and is for the message only — it is never what decides.
Held { pid: Option<u32> },
/// The filesystem does not do locks. The caller must carry on regardless;
/// see [`IndexLock::acquire`].
Unsupported(String),
}
/// Proof that this process, and no other, owns the index at a given path.
///
/// Held for the life of the process: dropping it, or exiting, releases the
/// lock.
#[derive(Debug)]
pub struct IndexLock {
/// The lock lives on this open file description. Keeping the handle alive
/// is the whole mechanism, which is why the field is never read.
_file: std::fs::File,
path: PathBuf,
}
/// The lock this process is holding, if any.
///
/// A `static` because that is what the thing models: `flock` and `LockFileEx`
/// are owned by an open file description, one index is locked per process, and
/// what releases it is the process ending. Threading an `Arc<Mutex<_>>` from
/// `main` down through the unlock gate and into the settings handler would say
/// the same thing in three more places.
///
/// Never dropped — statics are not — and that costs nothing here: the kernel
/// releases the lock however the process goes away, which is the whole design.
static HELD_LOCK: std::sync::Mutex<Option<IndexLock>> = std::sync::Mutex::new(None);
impl IndexLock {
/// The lock file for the index at `db_path`.
///
/// Its own name, never the database or one of SQLite's sidecars: a
/// `flock` of ours on an inode SQLite also locks would be a second
/// locking protocol on one file, and on Unix our `close` of it would
/// cancel SQLite's locks — the very failure this whole change exists to
/// prevent.
pub fn path_for(db_path: &Path) -> PathBuf {
let name = db_path
.file_name()
.and_then(|s| s.to_str())
.unwrap_or("index.sqlite");
db_path.with_file_name(format!("{}.lock", name))
}
/// Take the index lock, or report who has it.
///
/// **The guard is the kernel's lock, never the file's existence.** `flock`
/// and `LockFileEx` are held by the open file description, so the kernel
/// drops them when the process goes away for *any* reason — a clean exit,
/// a panic, SIGKILL, the OOM killer, a SIGBUS, or the power going out. A
/// `.lock` file left behind by an unclean shutdown is therefore inert, and
/// the next start re-acquires it normally. Nothing here may ever branch on
/// the file being present; a stale-PID-file scheme would strand the user
/// behind a crash exactly when they most want to reopen the app.
///
/// [`LockError::Unsupported`] means the filesystem could not answer, not
/// that the lock is taken — NFS without `lockd`, some FUSE mounts, a few
/// network shares. Callers **must** start anyway: a convenience guard is
/// never a good enough reason to refuse to open.
pub fn acquire(db_path: &Path) -> Result<IndexLock, LockError> {
let path = IndexLock::path_for(db_path);
if let Some(dir) = path.parent() {
if !dir.as_os_str().is_empty() {
let _ = create_dir_private(dir);
}
}
let file = match std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&path)
{
Ok(f) => f,
// A read-only or unwritable index directory is not a second
// instance. Same rule as an unsupported filesystem: carry on.
Err(e) => return Err(LockError::Unsupported(format!("{}: {}", path.display(), e))),
};
lock_exclusive_nonblocking(&file).map_err(|e| match e {
LockAttempt::Held => LockError::Held {
pid: read_recorded_pid(&path),
},
LockAttempt::Unsupported(msg) => LockError::Unsupported(msg),
})?;
let lock = IndexLock {
_file: file,
path: path.clone(),
};
// Only after the lock is ours, and only so a second instance can name
// us in its message. Nothing reads this to make a decision.
lock.record_holder();
Ok(lock)
}
/// [`IndexLock::acquire`], keeping the lock in [`HELD_LOCK`] for the life
/// of the process. What a frontend calls at startup.
pub fn hold(db_path: &Path) -> Result<(), LockError> {
let lock = IndexLock::acquire(db_path)?;
*crate::lock_ok(&HELD_LOCK) = Some(lock);
Ok(())
}
/// Move the held lock onto the index at `db_path`, for a `database_path`
/// that changed while the app was running.
///
/// **The new lock is taken before the old one is let go.** A refusal then
/// leaves this process holding exactly what it held before, so the caller
/// can reject the change and stay in a state that is still true. Releasing
/// first would leave a window in which another instance could claim the
/// index we are about to keep using.
///
/// Naming the index already locked is [`LockError::Held`] and changes
/// nothing. [`LockError::Unsupported`] means the move *happened* — the old
/// lock is gone — and the new path simply cannot be locked; carry on, as at
/// startup. Holding the old path's lock in that case would be worse than
/// holding none: it guards a file this process no longer uses while barring
/// a second instance from it.
pub fn move_to(db_path: &Path) -> Result<(), LockError> {
let mut slot = crate::lock_ok(&HELD_LOCK);
// Short-circuit before acquiring: `flock` conflicts with itself across
// two open file descriptions even inside one process, so re-taking the
// path we already hold would report itself as `Held`.
if slot
.as_ref()
.is_some_and(|held| held.path == IndexLock::path_for(db_path))
{
return Ok(());
}
match IndexLock::acquire(db_path) {
Ok(lock) => {
*slot = Some(lock);
Ok(())
}
Err(LockError::Unsupported(why)) => {
*slot = None;
Err(LockError::Unsupported(why))
}
Err(e) => Err(e),
}
}
fn record_holder(&self) {
use std::io::Write;
// A fresh handle: writing through `_file` would move the shared file
// offset the lock has nothing to do with, and the failure here is
// cosmetic either way. That a *second* handle can write here at all is
// why the Windows lock byte sits at [`LOCK_BYTE_OFFSET`] rather than at
// offset 0 — its byte-range locks are mandatory and per-handle, so a
// lock over this byte would block our own write.
let written = std::fs::OpenOptions::new()
.write(true)
.truncate(true)
.open(&self.path)
.and_then(|mut f| write!(f, "{}", std::process::id()));
if let Err(e) = written {
crate::log_warn!(
"could not record the lock holder in {}: {}",
self.path.display(),
e
);
}
}
}
/// The PID a holder recorded, for a message. `None` whenever the file is
/// absent, empty or unparseable — all of which are ordinary.
fn read_recorded_pid(path: &Path) -> Option<u32> {
std::fs::read_to_string(path).ok()?.trim().parse().ok()
}
/// What one attempt at the lock found.
enum LockAttempt {
Held,
Unsupported(String),
}
/// Where the Windows lock byte lives, chosen to be nowhere near the PID the
/// lock file holds at offset 0. See [`lock_exclusive_nonblocking`]'s Windows
/// arm; the Unix arm has no equivalent, because `flock` locks the open file
/// description rather than a byte range.
#[cfg(windows)]
const LOCK_BYTE_OFFSET: u64 = 1 << 63;
#[cfg(unix)]
fn lock_exclusive_nonblocking(file: &std::fs::File) -> Result<(), LockAttempt> {
use std::os::unix::io::AsRawFd;
// `flock`, not `fcntl`: a POSIX record lock would be cancelled by any
// `close` this process makes on the same inode, which is the hazard
// documented in `file_handling::index_file_set`. A `flock` belongs to the
// open file description and is immune to it.
let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) };
if rc == 0 {
return Ok(());
}
let err = std::io::Error::last_os_error();
match err.raw_os_error() {
Some(libc::EWOULDBLOCK) => Err(LockAttempt::Held),
_ => Err(LockAttempt::Unsupported(err.to_string())),
}
}
#[cfg(windows)]
fn lock_exclusive_nonblocking(file: &std::fs::File) -> Result<(), LockAttempt> {
use std::os::windows::io::AsRawHandle;
use windows_sys::Win32::Foundation::ERROR_LOCK_VIOLATION;
use windows_sys::Win32::Storage::FileSystem::{
LockFileEx, LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY,
};
use windows_sys::Win32::System::IO::{OVERLAPPED, OVERLAPPED_0, OVERLAPPED_0_0};
// The locked byte sits at [`LOCK_BYTE_OFFSET`], far past anything this
// file will ever hold — **not** at offset 0, where the PID is written.
// Windows byte-range locks are *mandatory* and belong to the file object,
// not the process: a lock covering offset 0 makes our own
// `record_holder`, which writes through a second handle, fail with
// `ERROR_LOCK_VIOLATION`, and `read_recorded_pid` fail the same way — so
// the "already running" message would lose the PID on the one platform
// that has no terminal to print it to. Locking beyond end-of-file is
// explicitly legal and is the conventional way to use a file as a
// semaphore.
let mut overlapped: OVERLAPPED = unsafe { std::mem::zeroed() };
overlapped.Anonymous = OVERLAPPED_0 {
Anonymous: OVERLAPPED_0_0 {
Offset: LOCK_BYTE_OFFSET as u32,
OffsetHigh: (LOCK_BYTE_OFFSET >> 32) as u32,
},
};
let ok = unsafe {
LockFileEx(
file.as_raw_handle() as _,
LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY,
0,
1,
0,
&mut overlapped,
)
};
if ok != 0 {
return Ok(());
}
let err = std::io::Error::last_os_error();
match err.raw_os_error() {
Some(code) if code == ERROR_LOCK_VIOLATION as i32 => Err(LockAttempt::Held),
_ => Err(LockAttempt::Unsupported(err.to_string())),
}
}
/// Bytes free to this user on the filesystem holding `path`, or `None` where
/// the platform will not say.
///
/// `None` is "unknown", never "zero": every caller treats it as no reason to
/// hold anything back.
///
/// The path need not exist — its nearest existing ancestor is what gets
/// asked, so this answers for a database that has not been created yet.
pub fn available_space(path: &Path) -> Option<u64> {
let mut probe = path;
loop {
if probe.exists() {
break;
}
probe = probe.parent()?;
}
available_space_of_existing(probe)
}
#[cfg(unix)]
fn available_space_of_existing(path: &Path) -> Option<u64> {
use std::ffi::CString;
use std::os::unix::ffi::OsStrExt;
let c_path = CString::new(path.as_os_str().as_bytes()).ok()?;
let mut stat: libc::statvfs = unsafe { std::mem::zeroed() };
if unsafe { libc::statvfs(c_path.as_ptr(), &mut stat) } != 0 {
return None;
}
// `f_bavail`, not `f_bfree`: the reserved blocks in the difference are
// root's, and this process is not root. `f_frsize` is the fragment size
// the block counts are in — `f_bsize` is the preferred I/O size and is
// the wrong multiplier.
let frsize = if stat.f_frsize > 0 {
stat.f_frsize
} else {
stat.f_bsize
};
(stat.f_bavail as u64).checked_mul(frsize as u64)
}
#[cfg(windows)]
fn available_space_of_existing(path: &Path) -> Option<u64> {
use std::os::windows::ffi::OsStrExt;
use windows_sys::Win32::Storage::FileSystem::GetDiskFreeSpaceExW;
let wide: Vec<u16> = path
.as_os_str()
.encode_wide()
.chain(std::iter::once(0))
.collect();
let mut free_to_caller: u64 = 0;
let ok = unsafe {
GetDiskFreeSpaceExW(
wide.as_ptr(),
&mut free_to_caller,
std::ptr::null_mut(),
std::ptr::null_mut(),
)
};
(ok != 0).then_some(free_to_caller)
}
/// How long to keep retrying a delete that fails because something else holds
/// the file open.
#[cfg(windows)]

239
crates/quicksearch-core/src/platform_tests.rs
View file

@ -1,5 +1,7 @@
use super::*;
use std::time::{Duration, Instant};
#[test]
fn unc_spellings() {
assert!(is_unc_string(r"\\server\share"));
@ -242,3 +244,240 @@ fn sep_prefix() -> String {
"/".to_string()
}
}
/// Two live instances cannot both hold the index.
#[test]
fn the_index_lock_is_exclusive_while_held() {
let db = crate::testutil::scratch_dir("lock-excl").join("index.sqlite");
let first = IndexLock::acquire(&db).expect("first acquire");
match IndexLock::acquire(&db) {
Err(LockError::Held { pid }) => {
// Recorded for the message only, but it should name us.
assert_eq!(pid, Some(std::process::id()));
}
Err(LockError::Unsupported(why)) => {
// A filesystem with no locks cannot answer; nothing to assert.
eprintln!("skipping: {}", why);
}
Ok(_) => panic!("the lock was handed out twice"),
}
drop(first);
// And it comes back once the holder lets go — but not necessarily in the
// same instant, which is why this retries instead of asserting outright.
//
// `flock` belongs to the *open file description*, and `fork` duplicates
// the descriptor table: between another thread's `fork` and its `exec`,
// the child shares every description this process has open, including the
// one we just released. `O_CLOEXEC` closes it at `exec` — verified, no
// descriptor survives into a spawned child — but until then the lock
// stays held. Several tests in this suite spawn processes (the sibling
// test below, and `file_handling::counting`'s `find`/`wc`), so under
// `cargo test` this window is reached often enough to be seen.
//
// It cannot reach the product: the lock is taken once at startup and held
// for the life of the process, never dropped and immediately retaken.
acquire_within(&db, Duration::from_secs(5));
}
/// [`IndexLock::acquire`], retried past the `fork`/`exec` window described in
/// [`the_index_lock_is_exclusive_while_held`].
fn acquire_within(db: &std::path::Path, budget: Duration) -> IndexLock {
let deadline = Instant::now() + budget;
loop {
match IndexLock::acquire(db) {
Ok(lock) => return lock,
Err(e) if Instant::now() >= deadline => panic!("never acquired: {:?}", e),
Err(_) => std::thread::sleep(Duration::from_millis(10)),
}
}
}
/// Env var naming the database whose lock [`lock_holder_child`] should take.
/// Absent in an ordinary run, which is what makes that test a no-op.
const LOCK_CHILD_DB: &str = "QS_LOCK_CHILD_DB";
/// **A crash must not lock the user out.**
///
/// Nothing unlinks the lock file, so it outlives an unclean exit. If startup
/// keyed on the file *existing*, one SIGKILL — or the SIGBUS this whole change
/// is about — would leave QuickSearch permanently unopenable. The guard is the
/// kernel's `flock`/`LockFileEx`, released when the holder's handle goes away
/// however it goes away, so the leftover file is inert.
///
/// Only a real killed process proves that, so this spawns one: nothing a
/// single process can do to itself reproduces "died without running a
/// destructor" while leaving a test alive to check the result.
#[test]
fn a_killed_holder_does_not_block_the_next_start() {
let db = crate::testutil::scratch_dir("lock-crash").join("index.sqlite");
let lock_path = IndexLock::path_for(&db);
// Probe first: on a filesystem without locks there is nothing to test.
match IndexLock::acquire(&db) {
Ok(lock) => drop(lock),
Err(LockError::Unsupported(why)) => {
eprintln!("skipping: {}", why);
return;
}
Err(LockError::Held { .. }) => panic!("a fresh path cannot be held"),
}
let exe = std::env::current_exe().expect("test binary path");
let mut child = std::process::Command::new(exe)
.args([
"--exact",
"platform::tests::lock_holder_child",
"--nocapture",
])
.env(LOCK_CHILD_DB, &db)
.spawn()
.expect("spawn the lock holder");
// Poll rather than read a pipe: a child that dies early then fails this
// test at the deadline instead of hanging it forever.
let deadline = Instant::now() + Duration::from_secs(30);
loop {
if matches!(IndexLock::acquire(&db), Err(LockError::Held { .. })) {
break;
}
if Instant::now() >= deadline {
let _ = child.kill();
panic!("the child never took the lock");
}
std::thread::sleep(Duration::from_millis(25));
}
// SIGKILL / TerminateProcess: no unwinding, no destructors, no cleanup —
// exactly what a SIGBUS leaves behind.
child.kill().expect("kill the holder");
child.wait().expect("reap the holder");
assert!(
lock_path.exists(),
"the crash should leave the lock file at {}",
lock_path.display()
);
// The point of the whole test: file present, holder dead, start succeeds.
// Retried for the reason `acquire_within` documents, not because a dead
// holder could still be holding anything.
acquire_within(&db, Duration::from_secs(5));
}
/// The child half of [`a_killed_holder_does_not_block_the_next_start`]: take
/// the lock, then wait to be killed. A no-op in an ordinary run.
#[test]
fn lock_holder_child() {
let Some(db) = std::env::var_os(LOCK_CHILD_DB) else {
return;
};
let db = std::path::PathBuf::from(db);
// The parent probes the lock to find out when we have it, so it may hold
// it for an instant just as we ask. Retry rather than lose the race.
let deadline = Instant::now() + Duration::from_secs(30);
let _lock = loop {
match IndexLock::acquire(&db) {
Ok(lock) => break lock,
Err(e) if Instant::now() >= deadline => panic!("child never acquired: {:?}", e),
Err(_) => std::thread::sleep(Duration::from_millis(10)),
}
};
// Killed long before this returns. The sleep is a backstop so a parent
// that dies first cannot strand this process.
std::thread::sleep(Duration::from_secs(120));
}
/// A `database_path` changed in Settings must carry the lock with it — and a
/// refused move must leave this process holding exactly what it held.
///
/// The whole point of taking the new lock before dropping the old: if the
/// destination is already somebody else's, the settings change is rejected and
/// the app goes on using the index it was using, still guarded. Releasing
/// first would open a window on the index we are about to keep writing to.
///
/// Uses the process-wide slot, so it is the one test that touches
/// [`HELD_LOCK`]; the paths are per-test scratch directories, so it does not
/// race the sibling tests that call [`IndexLock::acquire`] directly.
#[test]
fn the_held_lock_follows_the_database_path() {
let dir = crate::testutil::scratch_dir("lock-move");
let first = dir.join("first.sqlite");
let second = dir.join("second.sqlite");
// A filesystem with no locks cannot answer any of this.
match IndexLock::hold(&first) {
Ok(()) => {}
Err(LockError::Unsupported(why)) => {
eprintln!("skipping: {}", why);
return;
}
Err(LockError::Held { .. }) => panic!("a fresh path cannot be held"),
}
// Naming the index we already hold is a no-op, not a self-collision:
// `flock` conflicts with itself across two descriptions in one process.
IndexLock::move_to(&first).expect("re-holding the same path");
// Somebody else owns the destination, so the move is refused...
let rival = acquire_within(&second, Duration::from_secs(5));
assert!(
matches!(IndexLock::move_to(&second), Err(LockError::Held { .. })),
"a held destination must refuse the move"
);
// ...and the old path is still ours, which is what lets the caller reject
// the settings change and stay correct.
assert!(
matches!(IndexLock::acquire(&first), Err(LockError::Held { .. })),
"the original lock must survive a refused move"
);
// Once the destination frees up the move goes through, and the path we
// came from is released.
drop(rival);
let deadline = Instant::now() + Duration::from_secs(5);
while IndexLock::move_to(&second).is_err() && Instant::now() < deadline {
std::thread::sleep(Duration::from_millis(10));
}
assert!(
matches!(IndexLock::acquire(&second), Err(LockError::Held { .. })),
"the new path must be held after the move"
);
acquire_within(&first, Duration::from_secs(5));
}
/// `database_path` in hand, the lock is a sibling with its own name — never
/// the database or one of SQLite's sidecars, whose inodes we must not touch.
#[test]
fn the_lock_file_is_not_the_database_or_a_sidecar() {
let db = std::path::Path::new("/var/lib/qs/index.sqlite");
let lock = IndexLock::path_for(db);
assert_eq!(lock, std::path::Path::new("/var/lib/qs/index.sqlite.lock"));
for suffix in crate::file_handling::INDEX_SIDECAR_SUFFIXES {
if suffix == ".lock" {
continue;
}
assert_ne!(
lock,
std::path::PathBuf::from(format!("{}{}", db.display(), suffix))
);
}
assert_ne!(lock, db);
}
#[test]
fn available_space_answers_for_a_real_directory() {
let dir = crate::testutil::scratch_dir("space");
let free = available_space(&dir).expect("temp dir has a filesystem");
assert!(free > 0, "a writable scratch dir should have free space");
}
/// The database is asked about before it exists — the check runs at the start
/// of the first run, when nothing has created the file yet.
#[test]
fn available_space_walks_up_to_an_existing_ancestor() {
let missing = crate::testutil::scratch_dir("space-missing")
.join("not")
.join("created")
.join("index.sqlite");
assert!(!missing.exists());
assert!(available_space(&missing).is_some());
}

113
crates/quicksearch-core/src/query/pattern.rs
View file

@ -174,6 +174,19 @@ impl TermPattern {
}
}
/// [`literal`](Self::literal), ASCII-folded — the form every
/// case-insensitive scan actually searches with.
///
/// Handing this out rather than folding at the call site matters because
/// the full-text pass calls it once per candidate row: the pattern built
/// this string once, when the query was parsed.
pub fn literal_folded(&self) -> Option<&str> {
match self {
TermPattern::Literal(l) => Some(&l.folded),
_ => None,
}
}
/// Literal chunks between wildcards (the whole term when literal).
pub fn segments(&self) -> &[String] {
match self {
@ -289,8 +302,12 @@ impl TermPattern {
}
}
/// Non-overlapping occurrence count, capped at 1000 (the cascade's
/// `count_frac` saturates there anyway).
/// Non-overlapping occurrence count.
///
/// Wildcards stop at 1000 — the cascade's `count_frac` saturates there
/// anyway, and each regex match costs far more than a `memmem` hit.
/// Literals are counted in full, because stopping early would cost a
/// branch on the one path that runs over every candidate body.
pub fn count(&self, text: &str, case_insensitive: bool) -> usize {
match self {
TermPattern::Empty => 0,
@ -314,6 +331,84 @@ impl TermPattern {
pub struct RegexQuery {
pub source: String,
re: Regex,
/// Literals of which at least one must occur in anything this matches, when
/// the pattern admits such a set. See [`RegexQuery::required`].
required: Option<crate::search::prefilter::Required>,
}
/// Extract a set of literals at least one of which occurs in every match.
///
/// This is the same analysis the regex engine performs to build its own
/// prefilter, run for the same reason a level up: the `regex:` passes scan every
/// name, every path and every stored document, and a required literal lets
/// SQLite and the trigram index reject most of those rows first.
///
/// # What makes it sound
///
/// A *prefix* set has the property that every match begins with one of its
/// literals; a *suffix* set, that every match ends with one. Either way the
/// matched text — itself a substring of the field being searched — contains that
/// literal, so the field does. When the set is unbounded (`\d+`, `.*foo` by
/// prefix) `literals()` answers `None` and there is nothing to filter on.
///
/// Both kinds are tried because they fail on opposite patterns: `foo.*` has a
/// usable prefix and no usable suffix, `.*foo` the reverse. The more selective
/// of the two wins, measured by the shortest literal each would force a scan to
/// match — the weakest link in an OR.
///
/// # Why it parses case-sensitively
///
/// The query itself is case-insensitive, and extracting from a case-insensitive
/// pattern expands combinatorially — `(?i)FOO` comes back as eight literals, and
/// a longer word simply exceeds the extractor's budget and yields nothing.
/// Parsing without the flag gives one clean literal instead.
///
/// That is still sound. A case-insensitive match of `foo` against text `FOO`
/// means the text holds *some* case variant of the literal, and both consumers
/// fold: the trigram index lowercases what it indexes, and `LIKE` is
/// ASCII-case-insensitive by SQLite's default collation. An inline `(?i)` inside
/// the pattern is honoured by the parser regardless, so it explodes and yields
/// `None` — a lost optimisation, never a lost row.
fn required_literals(source: &str) -> Option<crate::search::prefilter::Required> {
use regex_syntax::hir::literal::{ExtractKind, Extractor};
let hir = regex_syntax::ParserBuilder::new()
.case_insensitive(false)
.build()
.parse(source)
.ok()?;
let mut best: Option<(usize, crate::search::prefilter::Required)> = None;
for kind in [ExtractKind::Prefix, ExtractKind::Suffix] {
let seq = Extractor::new().kind(kind).extract(&hir);
let Some(literals) = seq.literals() else {
continue; // unbounded: no constraint to be had from this direction
};
// A literal is bytes, and the extractor can split a multi-byte character
// across the boundary of what it kept. Anything that is not valid UTF-8
// cannot be handed to FTS5 or bound as SQL text, so the whole direction
// is abandoned rather than filtered — dropping one literal from an OR
// would drop the rows only it covers.
let strings: Option<Vec<String>> = literals
.iter()
.map(|l| std::str::from_utf8(l.as_bytes()).ok().map(str::to_owned))
.collect();
let Some(strings) = strings else { continue };
let Some(required) = crate::search::prefilter::Required::new(strings) else {
continue;
};
// The OR is only as selective as its shortest arm.
let weakest = required
.literals()
.iter()
.map(|l| l.chars().count())
.min()
.unwrap_or(0);
if best.as_ref().is_none_or(|(w, _)| weakest > *w) {
best = Some((weakest, required));
}
}
best.map(|(_, required)| required)
}
impl RegexQuery {
@ -335,9 +430,23 @@ impl RegexQuery {
Ok(RegexQuery {
source: source.to_string(),
re,
// Once per query, never per row. A pattern that yields nothing
// simply scans, exactly as every `regex:` query used to.
required: required_literals(source),
})
}
/// Literals of which at least one occurs in anything this matches, when the
/// pattern admits such a set.
///
/// The `regex:` passes use it to narrow what they scan; `None` means the
/// pattern constrains nothing usable (`\d+`, `a.*b`) and the pass reads
/// everything, which is what it always did. See
/// [`crate::search::prefilter`] for the rule a prefilter has to obey.
pub fn required(&self) -> Option<&crate::search::prefilter::Required> {
self.required.as_ref()
}
pub fn is_match(&self, text: &str) -> bool {
self.re.is_match(text)
}

24
crates/quicksearch-core/src/query/split.rs
View file

@ -291,11 +291,14 @@ mod tests {
fn a_windows_drive_path_reaches_the_filter_intact() {
let q = split_for_cascade(r"path:C:\Users\me\docs").unwrap();
assert_eq!(q.term, "", "the whole input is a filter");
assert!(
matches!(
&q.filter_params[0],
Value::Text(t) if t == r"C:\Users\me\docs"
),
// The folder filter binds one value: the subtree LIKE pattern built
// from the spelling the user typed. Asserted through the builder so
// this stays about the plumbing rather than about escaping rules.
assert_eq!(
q.filter_params[0],
Value::Text(crate::query::translator::like_subtree_pattern(
r"C:\Users\me\docs"
)),
"{:?}",
q.filter_params
);
@ -307,7 +310,9 @@ mod tests {
assert_eq!(q.term, "");
assert_eq!(
q.filter_params[0],
Value::Text("/home/me/My Documents".into())
Value::Text(crate::query::translator::like_subtree_pattern(
"/home/me/My Documents"
))
);
}
@ -439,9 +444,12 @@ mod tests {
let q = split_for_cascade("name:%*_").unwrap();
assert!(matches!(&q.filter_params[0], Value::Text(t) if t == "%\\%%\\_%"));
// path: values never glob.
// path: values never glob — the star reaches the pattern as a literal.
let q = split_for_cascade("path:/da*ta").unwrap();
assert!(matches!(&q.filter_params[0], Value::Text(t) if t == "/da*ta"));
assert_eq!(
q.filter_params[0],
Value::Text(crate::query::translator::like_subtree_pattern("/da*ta"))
);
}
#[test]

91
crates/quicksearch-core/src/query/translator.rs
View file

@ -146,18 +146,15 @@ pub fn build_filter(
// single root, so say it directly rather than by accident.
return Ok(frag("1=1", Vec::new()));
}
// The `=` half needs the collation spelled out: `LIKE` folds ASCII
// case on its own, so without this the two halves of the same
// filter disagree about `C:\Users` versus `c:\users`.
// One `LIKE`, where this used to need `parent = ? OR parent LIKE ?`
// with the collation spelled out to stop the two halves disagreeing
// about `C:\Users` versus `c:\users`. Every stored parent now ends
// in a separator, so `dir + SEP + %` matches the folder's own files
// (`%` matching nothing) as well as its subdirectories', and the
// `=` half has nothing left to do.
Ok(frag(
&format!(
"(f.parent = ? COLLATE {} OR f.parent LIKE ? ESCAPE '\\')",
crate::platform::PATH_COLLATION
),
vec![
Value::Text(base.clone()),
Value::Text(like_subtree_pattern(&base)),
],
"f.parent LIKE ? ESCAPE '\\'",
vec![Value::Text(like_subtree_pattern(&base))],
))
}
"name" | "filename" => {
@ -204,21 +201,23 @@ pub fn escape_like(s: &str) -> String {
out
}
/// Tidy a user-supplied folder value into the spelling `files.parent` stores.
/// Tidy a user-supplied folder value: trim it, and drop any trailing
/// separator, of either flavour — that is how people naturally write a
/// directory, and either may show up on Windows.
///
/// Trailing separators are how people naturally write directories, and either
/// separator may show up on Windows. A bare drive (`C:`) is *not* a path — the
/// stored parent is `C:\` — so the separator goes back on.
/// Empty out means "every folder", which is what a bare `/` or a blank value
/// comes to; [`build_filter`] turns that into `1=1`.
///
/// It used to special-case a bare drive (`C:` → `C:\`), because the filter's
/// `parent = ?` half had to match the stored spelling exactly. That half is
/// gone, and [`like_subtree_pattern`] puts the separator back itself, so both
/// spellings now produce the same pattern.
fn normalize_folder_value(value: &str) -> String {
let base = value.trim().trim_end_matches(['/', '\\']);
if base.len() == 2 && base.ends_with(':') && base.starts_with(|c: char| c.is_ascii_alphabetic())
{
return format!("{}{}", base, std::path::MAIN_SEPARATOR);
}
base.to_string()
value.trim().trim_end_matches(['/', '\\']).to_string()
}
/// A `LIKE ... ESCAPE '\'` pattern matching every path strictly beneath `dir`.
/// A `LIKE ... ESCAPE '\'` pattern matching `dir`'s own files and everything
/// beneath it.
///
/// The separator is escaped along with the base, because on Windows the
/// separator *is* the escape character — a hand-written `format!("{}/%", dir)`
@ -315,10 +314,15 @@ mod tests {
#[test]
fn path_filter_covers_the_folder_and_its_subtree() {
let f = frag("path", Op::Contains, "/home/me/docs");
assert!(f.sql.contains("f.parent = ?"));
assert!(f.sql.contains("f.parent LIKE ?"));
// The LIKE half must declare its escape character; without the clause
// a Windows separator would be eaten as an escape.
// One `LIKE`, and one bound value: since every stored parent ends in a
// separator, `dir + SEP + %` reaches the folder's own files as well as
// its subdirectories'. The `parent = ?` half this used to need — and
// the explicit collation that went with it — is gone.
assert!(f.sql.contains("f.parent LIKE ?"), "{}", f.sql);
assert!(!f.sql.contains("f.parent = ?"), "{}", f.sql);
assert_eq!(f.params.len(), 1);
// The LIKE must declare its escape character; without the clause a
// Windows separator would be eaten as an escape.
assert!(f.sql.contains("ESCAPE '\\'"), "{}", f.sql);
}
@ -404,12 +408,14 @@ mod tests {
.unwrap();
let base = format!("{}a{}b", root_prefix(), SEP);
// Stored parents always end in a separator, so each row here is spelled
// the way the indexer would spell it.
let rows = [
format!("{}{}sub", base, SEP), // inside
format!("{}{}sub{}deep", base, SEP, SEP), // deeper
base.clone(), // the folder itself
format!("{}a{}bc", root_prefix(), SEP), // prefix sibling: outside
format!("{}a", root_prefix()), // parent: outside
format!("{}{}sub{}", base, SEP, SEP), // inside
format!("{}{}sub{}deep{}", base, SEP, SEP, SEP), // deeper
format!("{}{}", base, SEP), // the folder's own files
format!("{}a{}bc{}", root_prefix(), SEP, SEP), // prefix sibling: outside
format!("{}a{}", root_prefix(), SEP), // parent: outside
];
for r in &rows {
conn.execute("INSERT INTO files (parent) VALUES (?1)", [r])
@ -423,7 +429,10 @@ mod tests {
|r| r.get(0),
)
.unwrap();
assert_eq!(matched, 2, "only the two rows strictly beneath {}", base);
// Three, not two: the trailing separator is what brings the folder's
// *own* files in, which is why the filter needs no second predicate.
// The prefix sibling and the parent stay out.
assert_eq!(matched, 3, "the subtree of {}, and only that", base);
}
#[test]
@ -436,9 +445,9 @@ mod tests {
let base = format!("{}a_b", root_prefix());
for r in [
format!("{}{}inside", base, SEP), // real child
format!("{}axb{}bait", root_prefix(), SEP), // `_` must not glob to `x`
format!("{}100%_done{}x", root_prefix(), SEP),
format!("{}{}inside{}", base, SEP, SEP), // real child
format!("{}axb{}bait{}", root_prefix(), SEP, SEP), // `_` must not glob to `x`
format!("{}100%_done{}x{}", root_prefix(), SEP, SEP),
] {
conn.execute("INSERT INTO files (parent) VALUES (?1)", [&r])
.unwrap();
@ -456,13 +465,17 @@ mod tests {
#[test]
fn folder_value_normalization() {
use std::path::MAIN_SEPARATOR as SEP;
// Trailing separators of either flavour are stripped.
assert_eq!(normalize_folder_value("/home/me/"), "/home/me");
assert_eq!(normalize_folder_value(r"C:\Users\me\"), r"C:\Users\me");
// A bare drive is not a path; the stored parent is `C:\`.
assert_eq!(normalize_folder_value("C:"), format!("C:{}", SEP));
assert_eq!(normalize_folder_value(r"C:\"), format!("C:{}", SEP));
// A bare drive and a rooted one now normalize alike: the pattern
// builder puts the separator back either way.
assert_eq!(normalize_folder_value("C:"), "C:");
assert_eq!(normalize_folder_value(r"C:\"), "C:");
assert_eq!(
like_subtree_pattern(&normalize_folder_value("C:")),
like_subtree_pattern(&normalize_folder_value(r"C:\")),
);
// Empty means "everywhere".
assert_eq!(normalize_folder_value("/"), "");
assert_eq!(normalize_folder_value(" "), "");

27
crates/quicksearch-core/src/scope.rs
View file

@ -5,8 +5,8 @@
//! produce. When the configuration changes, the two disagree — and almost
//! always in a way that can be *reconciled* rather than rebuilt:
//!
//! * A root was removed. Its rows are a contiguous `files.path` range, so
//! they go in five statements ([`crate::db::repo::delete_subtree`]).
//! * A root was removed. Its rows are a contiguous `files.parent` range, so
//! they go in four statements ([`crate::db::repo::delete_subtree`]).
//! * An ignore pattern was added, hidden files were switched off, symlinks
//! stopped being followed. The rows to drop are picked out by a predicate
//! no SQL range can express, so [`Scope::covers`] re-runs the walker's own
@ -51,7 +51,7 @@ use crate::indexing::ReconcileProgress;
/// through [`crate::db::InterruptGuard`].
pub const SLICE: Duration = Duration::from_millis(250);
/// One configured root, with the `files.path` range it owns precomputed.
/// One configured root, with the `files.parent` range it owns precomputed.
struct Root {
path: PathBuf,
lo: String,
@ -208,9 +208,9 @@ pub struct WorkCursor {
dropped_aliases: bool,
/// Index into `scope.roots` of the range being scanned.
root_idx: usize,
/// Last path served by the scan — the keyset cursor. Empty means "start
/// this root's range from its `lo` bound".
after: String,
/// Last `(parent, name)` served by the scan — the keyset cursor. An empty
/// parent means "start this root's range from its `lo` bound".
after: (String, String),
/// Set once the FTS automerge that follows a batch of deletions has run.
finalized: bool,
/// Rows deleted so far, for the log line when the work completes.
@ -233,7 +233,7 @@ impl WorkCursor {
drop_idx: 0,
dropped_aliases: false,
root_idx: 0,
after: String::new(),
after: (String::new(), String::new()),
finalized: false,
deleted: 0,
recontented: 0,
@ -352,16 +352,19 @@ pub fn advance(
return Ok(());
}
let root = &cursor.scope.roots[cursor.root_idx];
if cursor.after.is_empty() {
cursor.after = root.lo.clone();
if cursor.after.0.is_empty() {
// `(lo, "")` sorts below every row in the range: no stored name
// is empty, so the first page starts exactly at `lo`.
cursor.after = (root.lo.clone(), String::new());
}
let rows = repo::rows_in_range_page(conn, &cursor.after, &root.hi, page)?;
let rows =
repo::rows_in_range_page(conn, &cursor.after.0, &cursor.after.1, &root.hi, page)?;
let Some(last) = rows.last() else {
cursor.root_idx += 1;
cursor.after.clear();
cursor.after = (String::new(), String::new());
continue;
};
cursor.after = last.path.clone();
cursor.after = (last.parent.clone(), last.name.clone());
cursor.examined += rows.len();
let root = cursor.scope.roots[cursor.root_idx].path.clone();
let (deleted, recontented) = apply_page(

4
crates/quicksearch-core/src/scope_tests.rs
View file

@ -45,12 +45,12 @@ fn seed(conn: &mut Connection, paths: &[PathBuf]) {
let tx = conn.transaction().unwrap();
for path in paths {
let path = path.to_string_lossy();
let (parent, name) = path.rsplit_once('/').unwrap();
// The indexer's own split, so the separator stays with the parent.
let (parent, name) = crate::file_handling::split_db_path(&path).expect("a file's path");
repo::insert_file(
&tx,
&repo::NewFile {
name,
path: &path,
parent,
size: 1,
mtime: 1,

112
crates/quicksearch-core/src/search/cascade.rs
View file

@ -54,6 +54,7 @@
//! order.
use std::collections::HashSet;
use std::hash::{BuildHasherDefault, Hasher};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
@ -67,7 +68,7 @@ use crate::query::split::CascadeQuery;
use crate::query::translator::{escape_like, quote_phrase};
use crate::snippet;
use super::fuzzy::{edit_budget, Bitap};
use super::fuzzy::{edit_budget, pigeonhole_chunks, Bitap};
use super::{SearchHit, SearchOptions};
mod passes;
@ -100,10 +101,10 @@ pub fn text_snippet(
let opts = snippet::Options {
approx_chars: SNIPPET_WINDOW_CHARS,
};
match pattern.literal() {
match text_snippet_counted(pattern, text, folded) {
// Literal terms keep the richer multi-occurrence extract; a wildcard
// match marks its own first range.
Some(term) => Some(snippet::extract_folded(text, folded, &[term], &opts)),
Some((snip, _)) => Some(snip),
None => pattern.find_first_folded(folded).map(|r| {
// A greedy pattern can match megabytes; clamp before the window.
let r = clamp_match_range(text, r, SNIPPET_WINDOW_CHARS);
@ -112,6 +113,31 @@ pub fn text_snippet(
}
}
/// [`text_snippet`] for a literal pattern, with the case-insensitive
/// occurrence count the extraction found on its way to the window. `None`
/// when the pattern is not literal.
///
/// The count is a by-product: [`snippet::extract_folded`] locates every
/// occurrence in `folded` in order to coalesce and mark them, and that set has
/// the same cardinality [`crate::query::pattern::TermPattern::count_folded`]
/// would return. Taking it from here is what lets the full-text pass verify a
/// row and cut its snippet in one sweep of the body rather than two — see
/// `benches/search.rs`, group `cascade_row_sweeps`. It does not generalise to
/// a wildcard, whose snippet comes from a single leftmost match.
pub fn text_snippet_counted(
pattern: &crate::query::pattern::TermPattern,
text: &str,
folded: &str,
) -> Option<(snippet::Snippet, usize)> {
let opts = snippet::Options {
approx_chars: SNIPPET_WINDOW_CHARS,
};
// The pre-folded form: this runs per candidate row, and folding the term
// again here would allocate once per row for a string the pattern holds.
let term = pattern.literal_folded()?;
Some(snippet::extract_folded(text, folded, &[term], &opts))
}
/// The fuzzy full-text match in one document body: how many times the term
/// occurs within the edit budget, and the Content Match window cut around
/// the first occurrence at the cascade's own width. `None` when it does not
@ -120,19 +146,23 @@ pub fn text_snippet(
/// Shared with [`crate::live`] for the same reason as [`text_snippet`]: a
/// fuzzy row whose file changes has to be re-cut the way it was cut, and
/// bitap's range is what it was cut around. `bitap` is built once by the
/// caller — per scan in the pass, per arm in the live watcher — since
/// building it is the cost, and `folded` must be `text` ASCII-lowercased.
/// caller — per scan in the pass, per arm in the live watcher — since building
/// it is the cost.
///
/// Takes no folded copy, unlike [`text_snippet`]: the matcher folds in its
/// mask table (see [`crate::search::fuzzy`]), so it reads the body as stored.
/// That is the difference between this pass copying and lowercasing every
/// document in the index per keystroke and not doing so.
pub fn fuzzy_snippet(
bitap: &crate::search::fuzzy::Bitap,
text: &str,
folded: &str,
) -> Option<(usize, snippet::Snippet)> {
let opts = snippet::Options {
approx_chars: SNIPPET_WINDOW_CHARS,
};
// `first` is `Some` exactly when `count` is non-zero: it *is* the first
// of them.
let (count, first) = bitap.count_and_first(folded.as_bytes());
let (count, first) = bitap.count_and_first(text.as_bytes());
first.map(|range| (count, snippet::window_around(text, range, &opts)))
}
@ -171,7 +201,7 @@ pub fn run(
generation,
latest_gen,
ignore,
emitted: HashSet::new(),
emitted: IdSet::default(),
deferred_path: Deferred::default(),
deferred_fuzzy_path: Deferred::default(),
total: 0,
@ -197,8 +227,13 @@ pub fn run(
if cx.cancelled() {
return Ok(None);
}
// Stop, but do not call it truncation. `remaining() == 0` is also
// what an exactly-full result set looks like, and claiming a cut there
// tells a user to raise `--limit` on a complete answer. `flush_pass`
// is the authority — it sets the flag when it actually had to drop
// rows — and `scan_pass` breaks its row loop the moment the limit
// fills, for the same reason this break exists.
if cx.remaining() == 0 {
cx.limited = true;
break;
}
let run_pass = match pass {
@ -277,11 +312,14 @@ fn rank_order(a: &SearchHit, b: &SearchHit) -> std::cmp::Ordering {
}
/// The `files` columns every pass selects, in the order the passes index
/// them: `0` id, `1` name, `2` path, `3` size, `4` mtime. Passes that also
/// them: `0` id, `1` name, `2` parent, `3` size, `4` mtime. Passes that also
/// want the stored document text append `dt.text_zstd` as column `5`. A pass
/// spelling its own list in a different order would compile and then quietly
/// serve paths as names.
const HIT_COLUMNS: &str = "f.id, f.name, f.path, f.size, f.mtime";
/// serve parents as names.
///
/// There is no `path` column to select; [`Cx::scan_pass`] concatenates columns
/// 2 and 1 into one reused buffer and hands every classifier the result.
const HIT_COLUMNS: &str = "f.id, f.name, f.parent, f.size, f.mtime";
/// Columns 3 and 4. The clamp matters: `size` is `INTEGER` in SQLite and so
/// signed; a corrupt row holding `-1` would otherwise become 18 exabytes on
@ -314,6 +352,54 @@ struct Deferred {
/// inside the GUI's 250 ms result fade.
const FLUSH_INTERVAL: Duration = Duration::from_millis(80);
/// Hasher for the emitted-id set, which is probed **once per scanned row**.
///
/// The default `HashSet` hasher is SipHash-1-3, chosen to be collision-resistant
/// against hostile keys. These keys are SQLite rowids the cascade itself just
/// read — nobody outside chooses them — so the resistance buys nothing, while
/// the ~10 ns it costs is paid on every row of a whole-table scan.
///
/// Multiplicative, by an odd constant near 2^64/φ. Odd keeps it a bijection, so
/// distinct ids stay distinct; hashbrown then takes the low bits for the bucket
/// and the top seven for its control byte, and the rotate is what puts real
/// entropy in both halves for the near-consecutive ids a table scan produces.
///
/// Worth ~5% of a fuzzy search — 17.1/17.4/17.9 ms against 18.2/18.3/18.8 over
/// three runs each of `tests/search_alloc.rs`, every run of the one beating
/// every run of the other. Small, but it is the whole of what a hash function
/// choice can be worth, and it is bought for fifteen lines with no behaviour
/// attached. The figure will grow with the row count: this is one probe per
/// scanned row, and the corpus is 50,000 rows.
#[derive(Default)]
struct IdHasher(u64);
impl Hasher for IdHasher {
fn finish(&self) -> u64 {
self.0
}
// The only shape the set ever hashes is a single `i64`, which `Hash for
// i64` delivers through `write_i64`. `write` exists because the trait
// requires it and is deliberately a poor general-purpose hash: routing
// anything else through here would be a bug, not a use.
fn write(&mut self, bytes: &[u8]) {
for &b in bytes {
self.0 = (self.0 ^ u64::from(b)).wrapping_mul(0x0100_0000_01b3);
}
}
fn write_i64(&mut self, n: i64) {
self.write_u64(n as u64);
}
fn write_u64(&mut self, n: u64) {
let mixed = n.wrapping_mul(0x9E37_79B9_7F4A_7C15);
self.0 = mixed.rotate_left(31) ^ mixed;
}
}
type IdSet = HashSet<i64, BuildHasherDefault<IdHasher>>;
struct Cx<'a> {
conn: &'a Connection,
query: &'a CascadeQuery,
@ -321,7 +407,7 @@ struct Cx<'a> {
generation: u64,
latest_gen: &'a AtomicU64,
ignore: IgnoreSet,
emitted: HashSet<i64>,
emitted: IdSet,
/// Ranks 9–10, filled by pass A.
deferred_path: Deferred,
/// Rank 11, filled by pass C.

474
crates/quicksearch-core/src/search/cascade/passes.rs
View file

@ -20,6 +20,36 @@ fn fold_into(dst: &mut String, text: &str) {
dst.make_ascii_lowercase();
}
/// The segment a straddling wildcard can be SQL-prefiltered on: the longest
/// one containing no path separator, or `None` when every segment has one.
///
/// # Why one segment is enough, and why it must be separator-free
///
/// The classifier accepts a row only if the whole pattern matches `name` or
/// `parent || name`, and either way **every** segment occurs contiguously
/// somewhere in `parent || name`. So testing any single segment is a superset
/// of what the classifier accepts — the filter can only be too generous, never
/// too strict, which is the direction that keeps real hits from vanishing.
///
/// Splitting that test across the two stored columns is what needs the
/// separator-free condition. An occurrence of a segment in `parent || name`
/// lies wholly inside `parent`, wholly inside `name`, or spans the join. A
/// spanning occurrence covers the byte immediately before the boundary, and
/// that byte is `parent`'s last — which is always a separator, by the
/// invariant `file_handling::dir_to_db_parent` maintains. A segment with no
/// separator in it therefore cannot span the join, so
/// `name LIKE %s% OR parent LIKE %s%` sees it wherever it is.
///
/// The longest is chosen for selectivity alone; any of them would be correct.
fn anchor_segment(pattern: &crate::query::pattern::TermPattern) -> Option<&str> {
pattern
.segments()
.iter()
.filter(|s| !s.contains(std::path::MAIN_SEPARATOR))
.max_by_key(|s| s.len())
.map(String::as_str)
}
/// Which [`Deferred`] buffer a scan's held-back hits go to.
enum DeferSlot {
/// Ranks 9–10, flushed by [`Pass::Path`]. Shared by passes A and E,
@ -40,13 +70,28 @@ impl<'a> Cx<'a> {
/// `Defer`red hit lands in `defer_slot` at the end of the scan rather
/// than being emitted — path-tier ranks sort below stages that have not
/// run yet, so they are held back and never flushed mid-scan.
///
/// `classify` receives the reassembled path **and the name as a slice of
/// it**, both borrowed. The name is not fetched a second time from the row:
/// a path is `parent || name`, so the name is already sitting in the tail
/// of the buffer this pass just built. Handing it over borrowed is what
/// keeps a whole-table scan from allocating a `String` per *scanned* row
/// when only the few that become hits need an owned copy — measured at one
/// allocation per row for the passes with no SQL prefilter (wildcard,
/// regex) and three per row for the fuzzy filename pass.
fn scan_pass(
&mut self,
sql: &str,
params: Vec<rusqlite::types::Value>,
cancel_every: usize,
defer_slot: Option<DeferSlot>,
mut classify: impl FnMut(&mut Self, &rusqlite::Row<'_>, i64, &str) -> Result<RowHit, String>,
mut classify: impl FnMut(
&mut Self,
&rusqlite::Row<'_>,
i64,
&str,
&str,
) -> Result<RowHit, String>,
) -> Result<bool, String> {
let conn = self.conn;
// Cached: a search re-runs the same six statements on every keystroke,
@ -62,41 +107,77 @@ impl<'a> Cx<'a> {
let mut deferred = Deferred::default();
let mut scanned = 0usize;
let mut clock = FlushClock::new();
// Set when the loop stops on the display limit rather than on the end
// of the candidate set — rows were left unexamined, so the result set
// is cut whatever the final flush happens to hold.
let mut cut_short = false;
// The reassembled path for the row in hand. One buffer for the whole
// scan, refilled per row: `files` stores `parent` and `name` and no
// longer a third column holding their concatenation, so this is where
// the concatenation happens. Reused rather than allocated because it
// runs for every *scanned* row — a full-table scan on three of the
// passes — while only the few that become hits keep an owned copy.
//
// A local, not a field on `self`: the classifiers take `&mut Self`, and
// a buffer borrowed out of `self` could not be handed to them.
let mut path = String::new();
while let Some(row) = rows.next().map_err(|e| e.to_string())? {
scanned += 1;
if scanned.is_multiple_of(cancel_every) && self.cancelled() {
return Ok(false);
}
// The display limit is already full, and holding a row proves at
// least one more match exists than will be shown — so `limited` is
// exactly true here, and everything below is work whose result
// `flush_pass` would throw away. That work is not small: the
// full-text passes decompress the document, fold a copy of it, and
// cut a snippet, per row. `cascade::run` makes the same test
// between passes; without this one a single pass over a common
// term runs to the end of the candidate set.
if self.remaining() == 0 {
self.limited = true;
break;
}
let file_id: i64 = col(row, 0)?;
// Borrowed from the statement rather than `col::<String>`: this
// runs for every *scanned* row — a full-table scan on the filename
// pass — while only the few that become hits need an owned copy.
let path = row
.get_ref(2)
.map_err(|e| e.to_string())?
.as_str()
.map_err(|e| e.to_string())?;
if self.skip(file_id, path) {
let borrowed = |idx| -> Result<&str, String> {
row.get_ref(idx)
.map_err(|e| e.to_string())?
.as_str()
.map_err(|e| e.to_string())
};
path.clear();
// Parent first, and no separator between them: every stored parent
// already ends in one. See `file_handling::dir_to_db_parent`.
path.push_str(borrowed(2)?);
// Where the name starts, so the classifier can have it as a slice
// of the path rather than as a second fetch and a second
// allocation. The parent is written whole and nothing rewrites it,
// so this offset stays a char boundary.
let name_at = path.len();
path.push_str(borrowed(1)?);
if self.skip(file_id, &path) {
continue;
}
match classify(self, row, file_id, path)? {
match classify(self, row, file_id, &path, &path[name_at..])? {
RowHit::Skip => {}
RowHit::Emit(hit) => {
buf.push(hit);
overflowed |= self.enforce_cap(&mut buf);
self.flush_if_due(&mut buf, &mut clock);
// Stop once the display limit is full. Everything past
// this point is work `flush_pass` would throw away, and it
// is not small: the full-text passes decompress the
// document, fold a copy of it and cut a snippet, per row.
// `cascade::run` makes the same test between passes;
// without this one a single pass over a common term runs
// to the end of its candidate set.
//
// Tested *here* rather than at the top of the loop, where
// it is tempting: a scanned row is not a match. It may be
// one an earlier pass already emitted, or one `classify`
// rejects — the filename pass's SQL is deliberately a
// superset feeding both the name and the path tier — so
// stopping on one would end the pass over a row that was
// never going to be shown.
//
// The break is recorded rather than turned into `limited`
// on the spot. `flush_pass` decides that, because it knows
// how much it had to truncate; claiming it here would say
// a set was cut whenever it happened to end exactly at the
// limit. But leaving the scan *unfinished* silently is the
// opposite error — see the flush below.
if self.remaining() == 0 {
cut_short = true;
break;
}
}
RowHit::Defer(hit) => {
deferred.hits.push(hit);
@ -114,6 +195,15 @@ impl<'a> Cx<'a> {
None => debug_assert!(deferred.hits.is_empty(), "deferred hits with no slot"),
}
self.flush_pass(buf, overflowed);
// After the flush, so it cannot be undone by one that truncated
// nothing. A scan that broke on the limit left rows unexamined, and
// the two places that would otherwise notice both miss the boundary
// case: `flush_pass` compares `buf.len() > room`, which is `0 > 0`
// when a mid-scan flush landed exactly on the limit, and
// `cascade::run` tests `remaining()` at the top of the *next* pass,
// of which the last pass has none. Without this a cut result set can
// report itself complete.
self.limited |= cut_short;
Ok(true)
}
@ -123,43 +213,91 @@ impl<'a> Cx<'a> {
let query = self.query;
let pattern = &query.pattern;
let with_paths = path_tiers_enabled(pattern);
// A path always ends in its own name, so `path LIKE` is the
// superset that feeds both the name and the path tiers.
let sql = format!(
"SELECT {} FROM files f \
WHERE {} LIKE ? ESCAPE '\\'{}",
HIT_COLUMNS,
if with_paths { "f.path" } else { "f.name" },
query.filter_sql
);
// Wildcard patterns turn each star into an unescaped `%`; the
// substring wrap absorbs leading/trailing stars. User `%`/`_`
// remain escaped literals either way.
let like = pattern
.segments()
.iter()
.map(|s| escape_like(s))
.collect::<Vec<_>>()
.join("%");
let params =
self.params_with_filters(vec![rusqlite::types::Value::Text(format!("%{}%", like))]);
let like = format!(
"%{}%",
pattern
.segments()
.iter()
.map(|s| escape_like(s))
.collect::<Vec<_>>()
.join("%")
);
// There is no `path` column to `LIKE` against any more, and the
// prefilter has to stay a *superset* of what the classifier accepts or
// real hits vanish. Three cases:
//
// * Name tier only — unchanged, `name LIKE`.
// * Path tiers, and the pattern is a single segment containing no
// separator: `name LIKE ? OR parent LIKE ?` is then exactly
// equivalent to the old `path LIKE ?`. A match inside `parent || name`
// either sits wholly in one or straddles the boundary, and the
// boundary character is a separator the pattern does not contain. This
// is ordinary typing, so it is the case worth keeping cheap.
// * A multi-segment wildcard whose `%` can span the boundary
// (`doc*q3` over `/x/docs/q3.txt`), or a term with a separator in it.
// The whole *pattern* cannot be pinned to one column, but one of its
// segments can — see [`anchor_segment`] — and that is enough for a
// superset. Only when no segment qualifies is there nothing left to
// filter on and the pass scans, as passes C and E do.
//
// That last case is not a rare one, which is why it is worth the
// argument: `rep*rt` is two segments, so ordinary wildcard typing used
// to land on `1=1` and pay four regex evaluations — anchored and
// unanchored, cased and folded — on every row in the index.
let straddles = pattern.segments().len() > 1
|| pattern
.segments()
.iter()
.any(|s| s.contains(std::path::MAIN_SEPARATOR));
// Both columns, one bound value: `like_for` builds it once and it is
// bound twice.
let two_column = || {
"(f.name LIKE ? ESCAPE '\\' OR f.parent LIKE ? ESCAPE '\\')".to_string()
};
let bind_twice = |pat: String| {
vec![
rusqlite::types::Value::Text(pat.clone()),
rusqlite::types::Value::Text(pat),
]
};
let (predicate, terms) = match (with_paths, straddles) {
(false, _) => (
"f.name LIKE ? ESCAPE '\\'".to_string(),
vec![rusqlite::types::Value::Text(like)],
),
(true, false) => (two_column(), bind_twice(like)),
(true, true) => match anchor_segment(pattern) {
Some(anchor) => (
two_column(),
bind_twice(format!("%{}%", escape_like(anchor))),
),
None => ("1=1".to_string(), Vec::new()),
},
};
let sql = format!(
"SELECT {} FROM files f WHERE {}{}",
HIT_COLUMNS, predicate, query.filter_sql
);
let params = self.params_with_filters(terms);
self.scan_pass(
&sql,
params,
CANCEL_CHECK_ROWS,
Some(DeferSlot::Path),
|cx, row, file_id, path| {
let name: String = col(row, 1)?;
|cx, row, file_id, path, name| {
// Folding is byte-length preserving, so folded offsets are
// valid in the original. For wildcards, tiers 1/2 mean "the
// whole name matches the pattern".
let (rank, match_range) = if pattern.whole_match(&name, false) {
let (rank, match_range) = if pattern.whole_match(name, false) {
(1.0, (0, name.len()))
} else if pattern.whole_match(&name, true) {
} else if pattern.whole_match(name, true) {
(2.0, (0, name.len()))
} else if let Some(r) = pattern.find_first(&name, false) {
} else if let Some(r) = pattern.find_first(name, false) {
(3.0, (r.start, r.end))
} else if let Some(r) = pattern.find_first(&name, true) {
} else if let Some(r) = pattern.find_first(name, true) {
(4.0, (r.start, r.end))
} else if !with_paths {
return Ok(RowHit::Skip);
@ -178,14 +316,12 @@ impl<'a> Cx<'a> {
let is_path_tier = rank >= 9.0;
// The "snippet" of a name or path hit is that field itself
// with the matched span marked.
let snip = snippet::whole_field(
if is_path_tier { path } else { name.as_str() },
match_range,
);
let snip =
snippet::whole_field(if is_path_tier { path } else { name }, match_range);
let (size, mtime) = size_and_mtime(row)?;
let hit = SearchHit {
file_id,
name,
name: name.to_string(),
path: path.to_string(),
size,
mtime,
@ -262,7 +398,7 @@ impl<'a> Cx<'a> {
let mut doc = crate::db::repo::DocDecoder::new()?;
let mut lower = String::new();
// Decompression dominates: check cancellation every row.
self.scan_pass(&sql, params, 1, None, |cx, row, file_id, path| {
self.scan_pass(&sql, params, 1, None, |cx, row, file_id, path, name| {
let blob: Option<&[u8]> = row
.get_ref(5)
.map_err(|e| e.to_string())?
@ -274,30 +410,48 @@ impl<'a> Cx<'a> {
Some(text) => {
// Fold once: the case-insensitive count, the first-match
// search and the snippet extraction all need it, and
// nearly every candidate takes this path.
let mut folded = false;
let (count, stage) = {
let count_cs = pattern.count(text, false);
if count_cs > 0 {
(count_cs, 5)
} else {
fold_into(&mut lower, text);
folded = true;
let count_ci = pattern.count_folded(&lower);
if count_ci > 0 {
(count_ci, 6)
} else {
// nearly every candidate takes this path — including
// every candidate that is about to be dropped, since
// proving the pattern absent is itself a folded scan.
fold_into(&mut lower, text);
match super::text_snippet_counted(pattern, text, &lower) {
// Literal: one folded sweep does the verifying, the
// counting and the snippet together. The
// case-sensitive count is still its own sweep, but
// only for a row that survived — it decides stage 5
// against stage 6, and ranks within 5.
Some((snip, count_ci)) => {
if count_ci == 0 {
// Folded/unordered FTS candidate: the
// pattern never occurs — drop it.
return Ok(RowHit::Skip);
}
let count_cs = pattern.count(text, false);
let (count, stage) = if count_cs > 0 {
(count_cs, 5)
} else {
(count_ci, 6)
};
(stage as f64 + count_frac(count), stage as u8, Some(snip))
}
// Wildcard: its snippet is cut around a single
// leftmost match, so the count is not a by-product of
// it and each step stays its own scan.
None => {
let count_cs = pattern.count(text, false);
let (count, stage) = if count_cs > 0 {
(count_cs, 5)
} else {
let count_ci = pattern.count_folded(&lower);
if count_ci == 0 {
return Ok(RowHit::Skip);
}
(count_ci, 6)
};
let snip = super::text_snippet(pattern, text, &lower);
(stage as f64 + count_frac(count), stage as u8, snip)
}
};
if !folded {
fold_into(&mut lower, text);
}
let snip = super::text_snippet(pattern, text, &lower);
(stage as f64 + count_frac(count), stage as u8, snip)
}
// No stored text: can't case-verify or count. On the
// FTS-narrowed path accept at the bottom of rank 6 as
@ -317,7 +471,7 @@ impl<'a> Cx<'a> {
let (size, mtime) = size_and_mtime(row)?;
Ok(RowHit::Emit(SearchHit {
file_id,
name: col(row, 1)?,
name: name.to_string(),
path: path.to_string(),
size,
mtime,
@ -339,11 +493,12 @@ impl<'a> Cx<'a> {
if query.pattern.is_wildcard() {
return Ok(true);
}
let folded_term = query.term.to_ascii_lowercase();
let Some(k) = edit_budget(folded_term.len(), self.options.fuzzy_max_edits) else {
// The term goes in as typed: the matcher folds in its mask table, so
// neither side is lowercased here or per row.
let Some(k) = edit_budget(query.term.len(), self.options.fuzzy_max_edits) else {
return Ok(true);
};
let Some(bitap) = Bitap::new(folded_term.as_bytes(), k) else {
let Some(bitap) = Bitap::new(query.term.as_bytes(), k) else {
return Ok(true);
};
let with_paths = path_tiers_enabled(&query.pattern);
@ -358,18 +513,16 @@ impl<'a> Cx<'a> {
params,
CANCEL_CHECK_ROWS,
Some(DeferSlot::FuzzyPath),
|cx, row, file_id, path| {
let name: String = col(row, 1)?;
|cx, row, file_id, path, name| {
// The name wins when both fire; only a name miss falls
// through to the path tier.
let folded_name = name.to_ascii_lowercase();
let (rank, field, range) = match bitap
.best_distance_and_first(folded_name.as_bytes())
{
Some((distance, range)) => (7.0 + 0.1 * distance as f64, name.as_str(), range),
// through to the path tier. Both fields are read as stored —
// this is a whole-table scan, and the two `to_ascii_lowercase`
// calls that used to stand here were two allocations and two
// copies for every row in the index, per keystroke.
let (rank, field, range) = match bitap.best_distance_and_first(name.as_bytes()) {
Some((distance, range)) => (7.0 + 0.1 * distance as f64, name, range),
None if with_paths => {
let folded_path = path.to_ascii_lowercase();
match bitap.best_distance_and_first(folded_path.as_bytes()) {
match bitap.best_distance_and_first(path.as_bytes()) {
Some((distance, range)) => (11.0 + 0.1 * distance as f64, path, range),
None => return Ok(RowHit::Skip),
}
@ -391,7 +544,7 @@ impl<'a> Cx<'a> {
let (size, mtime) = size_and_mtime(row)?;
let hit = SearchHit {
file_id,
name,
name: name.to_string(),
path: path.to_string(),
size,
mtime,
@ -424,25 +577,77 @@ impl<'a> Cx<'a> {
if self.query.pattern.is_wildcard() {
return Ok(true);
}
let folded_term = self.query.term.to_ascii_lowercase();
let Some(k) = edit_budget(folded_term.len(), self.options.fuzzy_max_edits) else {
// As typed; the matcher folds in its mask table.
let Some(k) = edit_budget(self.query.term.len(), self.options.fuzzy_max_edits) else {
return Ok(true);
};
let Some(bitap) = Bitap::new(folded_term.as_bytes(), k) else {
let Some(bitap) = Bitap::new(self.query.term.as_bytes(), k) else {
return Ok(true);
};
let sql = format!(
"SELECT {}, dt.text_zstd \
FROM documents_text dt JOIN files f ON f.id = dt.file_id WHERE 1=1{}",
HIT_COLUMNS, self.query.filter_sql
);
let params = self.params_with_filters(Vec::new());
// One decoder and one fold buffer for the whole scan, reused per row.
// The candidate set. Without a prefilter this pass reads *every* stored
// document in the index on every keystroke — decompressing each one and
// running bitap over it — which is the most expensive thing the cascade
// does by a wide margin.
//
// The pigeonhole split gives a sound narrowing: at most `k` of the
// `k + 1` chunks can be damaged, so at least one survives verbatim in
// any document that matches, and asking the trigram index for "holds
// any one of these chunks" is a superset of what this pass can accept.
// See [`pigeonhole_chunks`] for the argument, and
// `tests/fuzzy_prefilter_fuzz.rs` for what defends it.
//
// Only a superset — the chunks are three characters and match plenty of
// documents that do not match the term at all. Every candidate is still
// decompressed and bitap-verified below; nothing about ranking or
// acceptance changes, only how many rows get that far.
//
// `None` means the term is too short to split (below `3 × (k + 1)`
// characters), and then there is no prefilter to be had and the pass
// scans as it always did.
let (sql, params) = match pigeonhole_chunks(&self.query.term, k) {
Some(chunks) => {
// Every chunk is quoted into inertness: a chunk is a slice of
// whatever the user typed, so it can contain `"`, `*`, `:`,
// `NEAR` and the rest of FTS5's syntax, and an unquoted one
// would be a syntax error instead of a search.
let expr = chunks
.iter()
.map(|c| format!("text: {}", quote_phrase(c)))
.collect::<Vec<_>>()
.join(" OR ");
(
format!(
"SELECT {}, dt.text_zstd \
FROM searchabletext \
JOIN files f ON f.id = searchabletext.rowid \
JOIN documents_text dt ON dt.file_id = f.id \
WHERE searchabletext MATCH ?{}",
HIT_COLUMNS, self.query.filter_sql
),
self.params_with_filters(vec![rusqlite::types::Value::Text(format!(
"({})",
expr
))]),
)
}
None => (
format!(
"SELECT {}, dt.text_zstd \
FROM documents_text dt JOIN files f ON f.id = dt.file_id WHERE 1=1{}",
HIT_COLUMNS, self.query.filter_sql
),
self.params_with_filters(Vec::new()),
),
};
// One decoder for the whole scan, reused per row. No fold buffer any
// more: this pass reads every stored document in the index on every
// keystroke, and it used to copy and lowercase each one — up to
// `maximum_text_size` per row — purely to give bitap a folded
// haystack. The matcher folds itself now.
let mut doc = crate::db::repo::DocDecoder::new()?;
let mut folded = String::new();
// Decompression dominates: check cancellation every row.
self.scan_pass(&sql, params, 1, None, |cx, row, file_id, path| {
self.scan_pass(&sql, params, 1, None, |cx, row, file_id, path, name| {
let blob: Option<&[u8]> = row
.get_ref(5)
.map_err(|e| e.to_string())?
@ -451,10 +656,7 @@ impl<'a> Cx<'a> {
let Some(text) = blob.and_then(|b| doc.decode(b)) else {
return Ok(RowHit::Skip);
};
// ASCII folding is byte-length preserving, so ranges found in
// the folded buffer are valid in the original.
fold_into(&mut folded, text);
let Some((count, snip)) = super::fuzzy_snippet(&bitap, text, &folded) else {
let Some((count, snip)) = super::fuzzy_snippet(&bitap, text) else {
return Ok(RowHit::Skip);
};
if !cx.regex_accepts(file_id, path, Some(text))? {
@ -463,7 +665,7 @@ impl<'a> Cx<'a> {
let (size, mtime) = size_and_mtime(row)?;
Ok(RowHit::Emit(SearchHit {
file_id,
name: col(row, 1)?,
name: name.to_string(),
path: path.to_string(),
size,
mtime,
@ -478,38 +680,46 @@ impl<'a> Cx<'a> {
/// entirely and runs on every name, falling back to the full path.
/// Name hits reuse rank 4, path hits defer to rank 10, so the GUI's
/// stage-based rendering needs no new cases.
///
/// Narrowed by the pattern's required literals where it has any — the same
/// two-column `LIKE` the wildcard filename pass uses, for the same reason
/// (see [`crate::search::prefilter::Required::like_predicate`]). Without one
/// this evaluates the user's regex against every name *and* every path in
/// the index, per keystroke; a pattern like `\d+` still does, because there
/// is no literal in it to filter on.
pub(super) fn pass_regex_name(&mut self) -> Result<bool, String> {
let query = self.query;
let re = query.regex.as_ref().expect("regex-only pass list");
let (predicate, terms) = match re.required().and_then(|r| r.like_predicate()) {
Some((sql, params)) => (sql, params),
None => ("1=1".to_string(), Vec::new()),
};
let sql = format!(
"SELECT {} FROM files f WHERE 1=1{}",
HIT_COLUMNS, query.filter_sql
"SELECT {} FROM files f WHERE {}{}",
HIT_COLUMNS, predicate, query.filter_sql
);
let params = self.params_with_filters(Vec::new());
let params = self.params_with_filters(terms);
self.scan_pass(
&sql,
params,
CANCEL_CHECK_ROWS,
Some(DeferSlot::Path),
|_cx, row, file_id, path| {
let name: String = col(row, 1)?;
|_cx, row, file_id, path, name| {
// The name is the better hit; only a name miss falls through
// to the path tier — mirroring pass A.
let (rank, match_range, is_path_tier) = match re.find_first(&name) {
let (rank, match_range, is_path_tier) = match re.find_first(name) {
Some(r) => (4.0, (r.start, r.end), false),
None => match re.find_first(path) {
Some(r) => (10.0, (r.start, r.end), true),
None => return Ok(RowHit::Skip),
},
};
let snip = snippet::whole_field(
if is_path_tier { path } else { name.as_str() },
match_range,
);
let snip =
snippet::whole_field(if is_path_tier { path } else { name }, match_range);
let (size, mtime) = size_and_mtime(row)?;
let hit = SearchHit {
file_id,
name,
name: name.to_string(),
path: path.to_string(),
size,
mtime,
@ -527,22 +737,44 @@ impl<'a> Cx<'a> {
}
/// Regex-only pass over every stored document text, reusing rank 6.
///
/// Narrowed by the pattern's required literals through the trigram index,
/// exactly as the fuzzy full-text pass narrows by its pigeonhole chunks —
/// both are "at least one of these strings is present", and both go through
/// [`crate::search::prefilter::Required::fts_expr`]. Without a usable set
/// this decompresses and regex-scans every stored document in the index on
/// every keystroke.
pub(super) fn pass_regex_content(&mut self) -> Result<bool, String> {
let query = self.query;
let re = query.regex.as_ref().expect("regex-only pass list");
let sql = format!(
"SELECT {}, dt.text_zstd \
FROM documents_text dt JOIN files f ON f.id = dt.file_id WHERE 1=1{}",
HIT_COLUMNS, query.filter_sql
);
let params = self.params_with_filters(Vec::new());
let (sql, params) = match re.required().and_then(|r| r.fts_expr()) {
Some(expr) => (
format!(
"SELECT {}, dt.text_zstd \
FROM searchabletext \
JOIN files f ON f.id = searchabletext.rowid \
JOIN documents_text dt ON dt.file_id = f.id \
WHERE searchabletext MATCH ?{}",
HIT_COLUMNS, query.filter_sql
),
self.params_with_filters(vec![rusqlite::types::Value::Text(expr)]),
),
None => (
format!(
"SELECT {}, dt.text_zstd \
FROM documents_text dt JOIN files f ON f.id = dt.file_id WHERE 1=1{}",
HIT_COLUMNS, query.filter_sql
),
self.params_with_filters(Vec::new()),
),
};
let snippet_opts = snippet::Options {
approx_chars: SNIPPET_WINDOW_CHARS,
};
// One decoder for the whole scan, reused per row.
let mut doc = crate::db::repo::DocDecoder::new()?;
// Decompression dominates: check cancellation every row.
self.scan_pass(&sql, params, 1, None, |_cx, row, file_id, path| {
self.scan_pass(&sql, params, 1, None, |_cx, row, file_id, path, name| {
let blob: Option<&[u8]> = row
.get_ref(5)
.map_err(|e| e.to_string())?
@ -564,7 +796,7 @@ impl<'a> Cx<'a> {
let (size, mtime) = size_and_mtime(row)?;
Ok(RowHit::Emit(SearchHit {
file_id,
name: col(row, 1)?,
name: name.to_string(),
path: path.to_string(),
size,
mtime,

40
crates/quicksearch-core/src/search/duplicates.rs
View file

@ -57,16 +57,25 @@ pub fn find_duplicate_groups(
}
}
// `ORDER BY parent, name` rather than by path: it is the stored key, so
// the sort is free, and it groups a directory's copies together — which is
// what the tab shows anyway.
let mut member_stmt = conn
.prepare("SELECT id, name, path, size, mtime FROM files WHERE hash = ?1 ORDER BY path")
.prepare(
"SELECT id, name, parent, size, mtime FROM files \
WHERE hash = ?1 ORDER BY parent, name",
)
.map_err(|e| e.to_string())?;
for group in &mut groups {
let rows = member_stmt
.query_map(params![group.hash], |r| {
let name: String = r.get(1)?;
let parent: String = r.get(2)?;
let path = format!("{}{}", parent, name);
Ok((
r.get::<_, i64>(0)?,
r.get::<_, String>(1)?,
r.get::<_, String>(2)?,
name,
path,
r.get::<_, i64>(3)?.max(0) as u64,
r.get::<_, i64>(4)?,
))
@ -91,13 +100,12 @@ mod tests {
let p = crate::testutil::scratch_dir("dups").join("index.sqlite");
let mut conn = open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
let tx = conn.transaction().unwrap();
let add = |name: &str, path: &str, size: u64, hash: Option<&[u8]>| {
let add = |name: &str, size: u64, hash: Option<&[u8]>| {
insert_file(
&tx,
&NewFile {
name,
path,
parent: "/d",
parent: "/d/",
size,
mtime: 1_700_000_000,
mime: None,
@ -110,20 +118,20 @@ mod tests {
.expect("unique path");
};
// Triple group of small files: redundant = 10 × 2 = 20.
add("a1.txt", "/d/a1.txt", 10, Some(b"AAA"));
add("a2.txt", "/d/a2.txt", 10, Some(b"AAA"));
add("a3.txt", "/d/a3.txt", 10, Some(b"AAA"));
add("a1.txt", 10, Some(b"AAA"));
add("a2.txt", 10, Some(b"AAA"));
add("a3.txt", 10, Some(b"AAA"));
// Pair of large files: redundant = 100 × 1 = 100 — sorts first
// despite the smaller member count.
add("b1.txt", "/d/b1.txt", 100, Some(b"BBB"));
add("b2.txt", "/d/b2.txt", 100, Some(b"BBB"));
add("b1.txt", 100, Some(b"BBB"));
add("b2.txt", 100, Some(b"BBB"));
// Singletons and NULL hashes never appear.
add("c.txt", "/d/c.txt", 30, Some(b"CCC"));
add("n1.txt", "/d/n1.txt", 40, None);
add("n2.txt", "/d/n2.txt", 40, None);
add("c.txt", 30, Some(b"CCC"));
add("n1.txt", 40, None);
add("n2.txt", 40, None);
// Zero-size files are trivially identical — excluded outright.
add("z1.txt", "/d/z1.txt", 0, Some(b"ZZZ"));
add("z2.txt", "/d/z2.txt", 0, Some(b"ZZZ"));
add("z1.txt", 0, Some(b"ZZZ"));
add("z2.txt", 0, Some(b"ZZZ"));
tx.commit().unwrap();
drop(conn);
p

138
crates/quicksearch-core/src/search/fuzzy.rs
View file

@ -5,9 +5,17 @@
//! (insertion / deletion / substitution); the u64 bit-parallel update costs
//! O(k) word ops per haystack byte with zero allocations.
//!
//! Callers fold both sides to ASCII lowercase first (the pipeline-wide
//! convention). Patterns are limited to 64 bytes by the machine word; the
//! cascade skips fuzzy stages for longer terms.
//! Matching is **ASCII-case-insensitive, in the automaton itself**: the mask
//! table sets a bit for both cases of every pattern byte, so neither side needs
//! folding first. That is not a convenience — it is what lets the whole-table
//! fuzzy passes run. Folding the haystack meant a `to_ascii_lowercase()` per
//! scanned filename *and* per scanned path, and, in the full-text pass, a
//! complete copy-and-fold of every stored document on every keystroke. Bytes
//! above 0x7F are untouched by ASCII folding, so setting both cases in the mask
//! accepts exactly what folding both sides accepted.
//!
//! Patterns are limited to 64 bytes by the machine word; the cascade skips
//! fuzzy stages for longer terms.
/// Registers the automaton needs: one per error count `0..=k`.
///
@ -19,7 +27,7 @@
const MAX_REGISTERS: usize = 22;
pub struct Bitap {
/// `masks[c]` has bit `i` set iff `pattern[i] == c`.
/// `masks[c]` has bit `i` set iff `pattern[i] == c` ignoring ASCII case.
masks: [u64; 256],
/// The same table for the *reversed* pattern, which is what lets
/// [`Bitap::match_start`] find where a match began by scanning backwards
@ -35,6 +43,11 @@ impl Bitap {
/// `None` when the pattern is empty, longer than 64 bytes, or the edit
/// budget does not fit the registers (`reset` shifts by `k`, and the
/// register array holds [`MAX_REGISTERS`]).
///
/// The pattern need not be folded: each byte's bit is set under **both**
/// ASCII cases, which is what makes the automaton case-insensitive and the
/// haystack fold unnecessary. For a byte that is not an ASCII letter the
/// two cases are the same byte and the second write is a no-op.
pub fn new(pattern: &[u8], k: usize) -> Option<Bitap> {
if pattern.is_empty() || pattern.len() > 64 || k >= MAX_REGISTERS {
return None;
@ -42,8 +55,12 @@ impl Bitap {
let mut masks = [0u64; 256];
let mut rev_masks = [0u64; 256];
for (i, &b) in pattern.iter().enumerate() {
masks[b as usize] |= 1u64 << i;
rev_masks[b as usize] |= 1u64 << (pattern.len() - 1 - i);
let (lower, upper) = (b.to_ascii_lowercase(), b.to_ascii_uppercase());
let (fwd, rev) = (1u64 << i, 1u64 << (pattern.len() - 1 - i));
masks[lower as usize] |= fwd;
masks[upper as usize] |= fwd;
rev_masks[lower as usize] |= rev;
rev_masks[upper as usize] |= rev;
}
Some(Bitap {
masks,
@ -224,6 +241,70 @@ impl Bitap {
}
}
/// Re-exported: the trigram floor is a property of the index, not of fuzzy
/// matching, and the regex prefilter applies the same one. See
/// [`crate::search::prefilter`].
pub use super::prefilter::TRIGRAM_FLOOR;
/// Split `term` into `k + 1` consecutive chunks of at least [`TRIGRAM_FLOOR`]
/// characters each, for the fuzzy full-text pass's candidate prefilter.
/// `None` when the term is too short to divide that way.
///
/// # Why this is a sound prefilter
///
/// The chunks **partition** the term: consecutive, disjoint, and together the
/// whole of it. Suppose the term occurs somewhere in a document within `k`
/// edits. Each edit falls inside at most one chunk, so at most `k` of the
/// `k + 1` chunks are touched — and therefore at least one chunk survives
/// *verbatim* in the document. Asking the index for "any document containing
/// chunk 1, or chunk 2, … " is consequently a **superset** of the documents the
/// pass can accept, and narrowing to it cannot lose a hit. It can admit
/// documents that do not match at all, which is harmless: every candidate is
/// still verified by the bitap scan that follows.
///
/// The partition is what makes it work, so it must stay one. Chunks that
/// overlapped, or that skipped part of the term, would let `k` edits damage
/// every chunk and the argument would collapse silently — as a search that
/// quietly stops finding things.
///
/// # Why the floor, and why chars rather than bytes
///
/// Below `3 * (k + 1)` characters some chunk would be shorter than a trigram
/// and match no token at all, turning the "superset" into an empty set. There
/// is no prefilter for such a term and the caller must scan; that is what
/// `None` says. Note this is reachable only when `fuzzy_max_edits` *binds* —
/// [`edit_budget`]'s own ladder gives one edit per three characters, so a term
/// long enough for `k` edits by length alone is never long enough to split.
///
/// Splitting on characters, not bytes: a byte split can land inside a UTF-8
/// sequence, and the halves would be handed to FTS5 as phrases that no longer
/// spell anything the tokenizer indexed.
pub fn pigeonhole_chunks(term: &str, k: usize) -> Option<Vec<&str>> {
let chunks = k + 1;
let total = term.chars().count();
if total < TRIGRAM_FLOOR * chunks {
return None;
}
// Char-boundary offsets, plus the end, so a chunk can be sliced by
// character index without re-walking the string per chunk.
let mut bounds: Vec<usize> = term.char_indices().map(|(at, _)| at).collect();
bounds.push(term.len());
// The remainder is spread one character at a time over the leading chunks
// rather than dumped on the last, so no chunk is left near the floor while
// another is long. Every chunk is a candidate; the shortest is the weakest
// filter, so the useful thing is to keep the shortest as long as possible.
let (base, extra) = (total / chunks, total % chunks);
let mut out = Vec::with_capacity(chunks);
let mut start = 0usize;
for i in 0..chunks {
let end = start + base + usize::from(i < extra);
out.push(&term[bounds[start]..bounds[end]]);
start = end;
}
Some(out)
}
/// The cascade's edit-distance budget for a folded term: one edit per
/// three characters, capped by `[search].fuzzy_max_edits`. Terms outside
/// 3..=64 bytes skip the fuzzy stages entirely (< 3 is noise, > 64 exceeds
@ -269,6 +350,35 @@ mod tests {
&hay[s..e]
}
/// The property that removes the fold from every caller: a pattern and a
/// haystack that differ only in case match at distance zero, with neither
/// side lowercased first.
#[test]
fn case_is_free_without_folding_either_side() {
assert_eq!(best("HELLO", "say hello world", 2), Some(0));
assert_eq!(best("hello", "say HELLO world", 2), Some(0));
assert_eq!(best("HeLLo", "say hEllO world", 0), Some(0));
// A case difference is not an edit, so the budget stays available for
// real ones: one substitution on top of four case flips still fits k=1.
assert_eq!(best("hello", "say HeXLO world", 1), Some(1));
// And the marked span is the occurrence as it appears in the haystack.
assert_eq!(marked("QUARTZITE", "the quartzite slab", 2), "quartzite");
assert_eq!(marked_best("quartzite", "the QUARTZITE slab", 2), "QUARTZITE");
}
/// Non-ASCII bytes are untouched by ASCII case folding, so setting both
/// cases in the mask cannot make two different multi-byte characters
/// collide.
#[test]
fn non_ascii_bytes_are_not_folded_together() {
// 'é' is 0xC3 0xA9 and 'É' is 0xC3 0x89 — distinct byte sequences that
// ASCII folding leaves distinct. The second byte differs, so this is
// one substitution, not a free case flip.
assert_eq!(best("café", "le café", 0), Some(0));
assert_eq!(best("café", "le CAFÉ", 0), None, "É is not a fold of é");
assert_eq!(best("café", "le CAFÉ", 1), Some(1), "and costs one edit");
}
#[test]
fn exact_substring_is_distance_zero() {
assert_eq!(best("hello", "say hello world", 2), Some(0));
@ -511,6 +621,11 @@ mod tests {
/// Brute-force oracle: minimum Levenshtein distance between `pattern`
/// and any substring of `hay`, capped at k.
///
/// Compares **ignoring ASCII case**, matching the automaton: the mask table
/// sets both cases of every pattern byte, so a case difference is not an
/// edit. On an all-lowercase alphabet this is identical to an exact
/// comparison, so the older cases below are unaffected by the fold.
fn oracle(pattern: &[u8], hay: &[u8], k: usize) -> Option<usize> {
// An occurrence must end at some text position; empty text has none.
// Without this, k >= pattern-length "matches" empty text by deleting
@ -528,7 +643,11 @@ mod tests {
for i in 1..=m {
cur[0] = i;
for j in 1..=hay.len() {
let cost = if pattern[i - 1] == hay[j - 1] { 0 } else { 1 };
let cost = if pattern[i - 1].eq_ignore_ascii_case(&hay[j - 1]) {
0
} else {
1
};
cur[j] = (prev[j - 1] + cost).min(prev[j] + 1).min(cur[j - 1] + 1);
}
std::mem::swap(&mut prev, &mut cur);
@ -553,7 +672,10 @@ mod tests {
.wrapping_add(1442695040888963407);
(seed >> 33) as usize
};
let alphabet = b"abcx";
// Mixed case on both sides, which is the whole point: the matcher folds
// in its mask table rather than having its haystack folded for it, and
// an alphabet of one case could never tell the two apart.
let alphabet = b"abcxABCX";
for _ in 0..500 {
let plen = 3 + rng() % 6;
let hlen = rng() % 20;

98
crates/quicksearch-core/src/search/mod.rs
View file

@ -29,6 +29,7 @@
pub mod cascade;
pub mod duplicates;
pub mod fuzzy;
pub mod prefilter;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
@ -187,11 +188,34 @@ struct SearchRequest {
/// its statement, tagged with the generation that owns it.
type InFlight = Arc<Mutex<Option<(u64, rusqlite::InterruptHandle)>>>;
/// One-shot channel the worker answers a release request on.
type ReleaseAck = Arc<Mutex<Option<mpsc::Sender<()>>>>;
/// How long [`SearchService::release_connection`] waits for the worker to
/// answer. Long enough for it to finish a statement and drop the handle,
/// short enough that a wedged worker cannot hold up a rebuild.
const RELEASE_WAIT: Duration = Duration::from_secs(2);
/// What the worker thread accepts. A search is the overwhelming majority;
/// the release exists because dropping the held connection is something only
/// the worker can do, and it is parked in a 30-second `recv_timeout` where a
/// flag would not reach it.
enum WorkerMsg {
Search(SearchRequest),
/// Drop the held connection now, so whoever asked can delete or replace
/// the index file. On Windows an open handle makes that fail outright;
/// everywhere else it keeps the deleted inode's blocks pinned.
ReleaseConnection,
}
pub struct SearchService {
req_tx: mpsc::Sender<SearchRequest>,
req_tx: mpsc::Sender<WorkerMsg>,
latest_gen: Arc<AtomicU64>,
in_flight: InFlight,
db_path: Arc<Mutex<PathBuf>>,
/// Where the worker reports that it has let the connection go; see
/// [`SearchService::release_connection`].
release_ack: ReleaseAck,
handle: Option<JoinHandle<()>>,
}
@ -214,11 +238,12 @@ impl SearchService {
notify: Arc<dyn Fn() + Send + Sync>,
idle_release: Duration,
) -> (SearchService, mpsc::Receiver<SearchUpdate>) {
let (req_tx, req_rx) = mpsc::channel::<SearchRequest>();
let (req_tx, req_rx) = mpsc::channel::<WorkerMsg>();
let (update_tx, update_rx) = mpsc::channel::<SearchUpdate>();
let latest_gen = Arc::new(AtomicU64::new(0));
let in_flight: InFlight = Arc::new(Mutex::new(None));
let db_path = Arc::new(Mutex::new(db_path));
let release_ack: ReleaseAck = Arc::new(Mutex::new(None));
let worker = Worker {
req_rx,
@ -227,6 +252,7 @@ impl SearchService {
latest_gen: latest_gen.clone(),
in_flight: in_flight.clone(),
db_path: db_path.clone(),
release_ack: release_ack.clone(),
open: None,
idle_release,
};
@ -241,6 +267,7 @@ impl SearchService {
latest_gen,
in_flight,
db_path,
release_ack,
handle: Some(handle),
},
update_rx,
@ -254,14 +281,42 @@ impl SearchService {
// Interrupt before enqueueing: an idle worker can dequeue the new
// request and be mid-statement within microseconds.
self.interrupt_stale();
let _ = self.req_tx.send(SearchRequest {
let _ = self.req_tx.send(WorkerMsg::Search(SearchRequest {
generation,
input: input.to_string(),
options,
});
}));
generation
}
/// Drop the connection the worker is holding, and wait briefly for it.
///
/// The worker keeps its connection for [`IDLE_RELEASE`] after the last
/// query so a typing session runs against a warm page cache. That is the
/// right default and the wrong thing to be holding when the index file is
/// about to be deleted: on Windows the delete fails outright, and a
/// "Rebuild index" that silently did not rebuild is worse than a slow one.
///
/// Cancelled first, and not merely as a courtesy. The worker coalesces its
/// queue, so a search enqueued just before this — a keystroke, then a
/// click on *Rebuild index* — is still current when the release is found,
/// and the worker would acknowledge the release and then **reopen** the
/// connection to serve it. Bumping the generation makes that request stale,
/// so it is dropped at the generation check instead. Results against a file
/// about to be deleted are worth nothing anyway.
///
/// Best-effort past that — it returns once the worker acknowledges or the
/// wait elapses, and the caller proceeds either way.
pub fn release_connection(&self) {
self.cancel();
let (ack_tx, ack_rx) = mpsc::channel();
*crate::lock_ok(&self.release_ack) = Some(ack_tx);
if self.req_tx.send(WorkerMsg::ReleaseConnection).is_err() {
return;
}
let _ = ack_rx.recv_timeout(RELEASE_WAIT);
}
/// Cancel without starting anything new.
pub fn cancel(&self) {
self.latest_gen.fetch_add(1, Ordering::SeqCst);
@ -320,12 +375,13 @@ pub fn classify_sql_err(error_msg: &str) -> String {
}
struct Worker {
req_rx: mpsc::Receiver<SearchRequest>,
req_rx: mpsc::Receiver<WorkerMsg>,
update_tx: mpsc::Sender<SearchUpdate>,
notify: Arc<dyn Fn() + Send + Sync>,
latest_gen: Arc<AtomicU64>,
in_flight: InFlight,
db_path: Arc<Mutex<PathBuf>>,
release_ack: ReleaseAck,
/// The connection, and the index generation and path it was opened
/// against. See [`Worker::take_connection`].
open: Option<OpenIndex>,
@ -345,7 +401,11 @@ impl Worker {
fn run(mut self) {
loop {
let first = match self.req_rx.recv_timeout(self.idle_release) {
Ok(req) => req,
Ok(WorkerMsg::ReleaseConnection) => {
self.release();
continue;
}
Ok(WorkerMsg::Search(req)) => req,
Err(mpsc::RecvTimeoutError::Timeout) => {
// Dropping the connection frees `PRAGMAS_SEARCH`'s 32 MiB
// page cache to glibc, which parks it in an arena rather
@ -366,10 +426,19 @@ impl Worker {
Err(mpsc::RecvTimeoutError::Disconnected) => return,
};
// A fast typist queues several requests; only the newest one
// matters.
// matters. A release queued behind them is *not* superseded — it
// is the one message that must not be coalesced away, so it is
// handled where it is found. Whether the search below still runs
// is then the generation check's business:
// `release_connection` cancels before it sends, so anything
// enqueued ahead of the release is stale and drops out rather than
// reopening the connection that was just let go of.
let mut req = first;
while let Ok(newer) = self.req_rx.try_recv() {
req = newer;
match newer {
WorkerMsg::Search(newer) => req = newer,
WorkerMsg::ReleaseConnection => self.release(),
}
}
if req.generation != self.latest_gen.load(Ordering::SeqCst) {
continue;
@ -378,6 +447,19 @@ impl Worker {
}
}
/// Drop the held connection and tell whoever asked.
///
/// The same trim `run`'s idle path does, for the same reason: glibc parks
/// `PRAGMAS_SEARCH`'s page cache in an arena rather than returning it.
fn release(&mut self) {
if self.open.take().is_some() {
crate::platform::release_free_heap();
}
if let Some(ack) = crate::lock_ok(&self.release_ack).take() {
let _ = ack.send(());
}
}
/// Take the connection to run this request on, reopening if the one held
/// cannot be reused. Reuse is what keeps
/// [`crate::db::schema::PRAGMAS_SEARCH`]'s page cache warm across

222
crates/quicksearch-core/src/search/prefilter.rs Normal file
View file

@ -0,0 +1,222 @@
//! Turning "something that must be present" into a SQL narrowing.
//!
//! Three of the cascade's passes would otherwise read the whole index on every
//! keystroke: the fuzzy full-text pass decompresses every stored document, and
//! the two `regex:` passes run the user's regex over every name, every path and
//! every document. In each case there is a set of literal strings of which **at
//! least one must occur** in anything the pass can accept, and that set is
//! enough to make the database do the rejecting instead.
//!
//! Where the sets come from differs; what is done with them does not, which is
//! why they meet here:
//!
//! * the fuzzy pass splits its term into `k + 1` chunks, at most `k` of which
//! an in-budget match can damage — see
//! [`crate::search::fuzzy::pigeonhole_chunks`];
//! * a `regex:` query hands its pattern to the same literal analysis the regex
//! engine uses to build its own prefilter — see
//! [`crate::query::pattern::RegexQuery`].
//!
//! # The one rule
//!
//! A prefilter must be a **superset** of what the pass accepts. It may admit
//! rows the pass then rejects — every candidate is verified afterwards exactly
//! as before, so ranking and results do not change, only how many rows are
//! looked at. It must never exclude a row the pass would have accepted, because
//! that failure has no symptom: the file simply stops appearing, and no error
//! is raised anywhere.
//!
//! Everything below is guards in service of that rule.
use rusqlite::types::Value;
use crate::query::translator::{escape_like, quote_phrase};
/// Characters in the smallest unit the FTS5 trigram index can be queried for.
///
/// A phrase shorter than this matches no token at all, so a prefilter built
/// from one would return the empty set rather than a superset — the exact
/// failure the module note forbids. It bounds [`Required::fts_expr`] only:
/// `LIKE` has no such floor, which is why the two predicates guard separately.
pub const TRIGRAM_FLOOR: usize = 3;
/// Most literals worth OR-ing together.
///
/// A case-insensitive pattern expands combinatorially — `(?i)FOO` extracts as
/// eight literals — and each one costs a term in the MATCH expression or two
/// `LIKE`s per row. Past some width the filter stops being cheaper than the
/// scan it replaces, and falling back to the scan is always correct.
const MAX_LITERALS: usize = 32;
/// Literals of which at least one occurs in anything the query can match.
#[derive(Debug, Clone)]
pub struct Required(Vec<String>);
impl Required {
/// `None` when the set cannot constrain anything: empty, too wide to be
/// worth it, or containing an empty literal.
///
/// The empty-literal case is the one that matters. A literal set containing
/// `""` says "a match may begin with nothing", which is not a constraint at
/// all — building a filter from it would narrow to rows containing the
/// empty string, which is a statement SQL is entitled to answer any way it
/// likes. Callers see `None` and scan.
pub fn new(literals: Vec<String>) -> Option<Required> {
if literals.is_empty() || literals.len() > MAX_LITERALS {
return None;
}
if literals.iter().any(|l| l.is_empty()) {
return None;
}
Some(Required(literals))
}
pub fn literals(&self) -> &[String] {
&self.0
}
/// A `searchabletext MATCH` expression: `(text: "a" OR text: "b" …)`.
///
/// `None` when any literal is shorter than [`TRIGRAM_FLOOR`] **characters**.
/// Characters, not bytes: `café` is five bytes and four characters, and
/// `日本` is six bytes and two — the tokenizer indexes character triples, so
/// a byte-length test would admit a phrase that matches no token.
///
/// Every literal goes through [`quote_phrase`], which is what makes this
/// safe for text the user typed: a literal can contain `"`, `*`, `:`,
/// `NEAR` and the rest of FTS5's syntax, and unquoted that is a syntax
/// error rather than a search.
///
/// The index folds case and strips diacritics (`remove_diacritics 1`), so
/// it matches *more* than the literal as written. That direction is the
/// harmless one.
pub fn fts_expr(&self) -> Option<String> {
if self
.0
.iter()
.any(|l| l.chars().count() < TRIGRAM_FLOOR)
{
return None;
}
Some(format!(
"({})",
self.0
.iter()
.map(|l| format!("text: {}", quote_phrase(l)))
.collect::<Vec<_>>()
.join(" OR ")
))
}
/// A predicate over the `files` columns, plus the values it binds:
/// `(f.name LIKE ? OR f.parent LIKE ? OR …)`.
///
/// `None` when any literal contains a path separator.
///
/// # Why the separator matters
///
/// There is no `path` column — a file's path is `parent || name` — so a
/// literal has to be looked for in the two columns separately. An
/// occurrence in the concatenation lies wholly inside `parent`, wholly
/// inside `name`, or spans the join. A spanning occurrence necessarily
/// covers the byte before the boundary, and that byte is `parent`'s last,
/// which is always a separator (see
/// [`crate::file_handling::dir_to_db_parent`]). So a literal with no
/// separator in it cannot span the join and the two-column test sees it
/// wherever it is — while a literal *with* one might sit exactly across the
/// boundary, be invisible to both `LIKE`s, and take its row with it.
///
/// All-or-nothing on purpose: the set is an OR, so a row whose only present
/// literal is the untestable one would be dropped. One bad literal
/// therefore disqualifies the whole predicate rather than being skipped.
///
/// No trigram floor here — `LIKE '%ab%'` is a perfectly good filter.
pub fn like_predicate(&self) -> Option<(String, Vec<Value>)> {
if self
.0
.iter()
.any(|l| l.contains(std::path::MAIN_SEPARATOR))
{
return None;
}
let mut clauses = Vec::with_capacity(self.0.len());
let mut params = Vec::with_capacity(self.0.len() * 2);
for literal in &self.0 {
clauses.push("f.name LIKE ? ESCAPE '\\' OR f.parent LIKE ? ESCAPE '\\'");
let pattern = format!("%{}%", escape_like(literal));
params.push(Value::Text(pattern.clone()));
params.push(Value::Text(pattern));
}
Some((format!("({})", clauses.join(" OR ")), params))
}
}
#[cfg(test)]
mod tests {
use super::*;
fn req(lits: &[&str]) -> Option<Required> {
Required::new(lits.iter().map(|s| s.to_string()).collect())
}
#[test]
fn a_set_that_cannot_constrain_is_rejected() {
assert!(req(&[]).is_none(), "nothing to filter on");
assert!(req(&["abc", ""]).is_none(), "an empty literal is no constraint");
let wide: Vec<String> = (0..MAX_LITERALS + 1).map(|i| format!("lit{i}")).collect();
assert!(Required::new(wide).is_none(), "too wide to be worth it");
}
#[test]
fn the_trigram_floor_counts_characters_not_bytes() {
// Four characters, five bytes: usable.
assert!(req(&["café"]).unwrap().fts_expr().is_some());
// Two characters, six bytes: a byte test would wrongly admit this.
assert!(req(&["日本"]).unwrap().fts_expr().is_none());
assert!(req(&["ab"]).unwrap().fts_expr().is_none());
assert!(req(&["abc", "de"]).unwrap().fts_expr().is_none(), "all of them");
}
#[test]
fn fts5_syntax_in_a_literal_is_quoted_inert() {
let expr = req(&["a\"b", "NEAR", "x*y"]).unwrap().fts_expr().unwrap();
// The embedded quote is doubled, which is FTS5's own escape.
assert!(expr.contains(r#""a""b""#), "{expr}");
assert!(expr.contains(r#""NEAR""#), "{expr}");
assert!(expr.contains(r#""x*y""#), "{expr}");
assert_eq!(expr.matches(" OR ").count(), 2);
}
#[test]
fn a_literal_with_a_separator_disqualifies_the_like_predicate() {
let sep = std::path::MAIN_SEPARATOR;
assert!(req(&["abc"]).unwrap().like_predicate().is_some());
assert!(
req(&["abc", &format!("d{sep}e")])
.unwrap()
.like_predicate()
.is_none(),
"one untestable literal disqualifies the whole OR"
);
}
#[test]
fn like_metacharacters_in_a_literal_stay_literal() {
let (sql, params) = req(&["100%_x"]).unwrap().like_predicate().unwrap();
assert_eq!(params.len(), 2, "one literal, bound to both columns");
assert!(sql.contains("ESCAPE '\\'"));
assert!(
matches!(&params[0], Value::Text(t) if t == "%100\\%\\_x%"),
"{:?}",
params[0]
);
}
#[test]
fn several_literals_become_one_or_over_both_columns() {
let (sql, params) = req(&["abc", "def"]).unwrap().like_predicate().unwrap();
assert_eq!(params.len(), 4);
assert_eq!(sql.matches("f.name LIKE").count(), 2);
assert_eq!(sql.matches("f.parent LIKE").count(), 2);
}
}

270
crates/quicksearch-core/src/snippet.rs
View file

@ -55,35 +55,102 @@ impl Snippet {
/// [`extract`] against a haystack the caller has already ASCII-folded.
/// `folded` must be `text.to_ascii_lowercase()` — the fold is byte-length
/// preserving, which is what lets offsets found in it slice the original.
pub fn extract_folded(text: &str, folded: &str, terms: &[&str], opts: &Options) -> Snippet {
///
/// Returns the window together with **how many occurrences it found**, before
/// touching ranges are coalesced. For a single term that number is exactly
/// what [`count_occurrences`] against `folded` would return, which is what
/// lets the full-text pass take its case-insensitive count from here instead
/// of sweeping the body a second time to compute it.
///
/// Blank terms are dropped, but a term that merely *has* surrounding
/// whitespace is searched as given: trimming it here would count and
/// highlight a different string than the pattern's own counters do.
pub fn extract_folded(
text: &str,
folded: &str,
terms: &[&str],
opts: &Options,
) -> (Snippet, usize) {
debug_assert_eq!(folded.len(), text.len(), "ASCII folding preserves length");
if text.is_empty() {
return Snippet::empty();
return (Snippet::empty(), 0);
}
let effective_terms: Vec<&str> = terms
.iter()
.map(|t| t.trim())
.filter(|t| !t.is_empty())
.copied()
.filter(|t| !t.trim().is_empty())
.collect();
if effective_terms.is_empty() {
return head_window(text, opts.approx_chars);
return (head_window(text, opts.approx_chars), 0);
}
// `memmem` rather than `str::match_indices`: both find non-overlapping
// occurrences, but std's Two-Way searcher has no vector prefilter and a
// full-text row scans a whole document body. See `benches/search.rs`,
// group `substring`.
// The count has to see every occurrence, so the walk runs to the end of the
// body either way. What does *not* have to happen is keeping them: the
// window is fixed by the first match, and everything starting past its
// right edge is discarded a few lines below. Storing them all meant a `Vec`
// proportional to the match count and then a sort over it — for a term
// occurring thousands of times in one file, which is an ordinary minified
// bundle or log, that is hundreds of kilobytes and an O(n log n) sort per
// candidate row, to render a 600-byte window.
//
// Bounded only for a single term, which is every caller the cascade makes
// (`cascade::text_snippet` passes one). With several, two different terms
// can coalesce across the edge and no single-pass bound sees it, so they
// are collected in full rather than approximately.
let pre_pad = opts.approx_chars / 3;
let bounded = effective_terms.len() == 1;
let mut matches: Vec<(usize, usize)> = Vec::new();
let mut found = 0usize;
// The right edge, once the first match has fixed it. Computed exactly as
// the window is computed below, so "kept" and "rendered" cannot disagree.
let mut keep_below: Option<usize> = None;
for term in &effective_terms {
let pattern = term.to_ascii_lowercase();
matches.extend(
memchr::memmem::find_iter(folded.as_bytes(), pattern.as_bytes())
.map(|at| (at, at + pattern.len())),
);
// Borrow when the term is already folded, which is the case for every
// call the search cascade makes: this runs once per candidate row, and
// an unconditional `to_ascii_lowercase` is an allocation per row to
// rebuild a string the pattern already holds.
let pattern: std::borrow::Cow<'_, str> = if term.bytes().any(|b| b.is_ascii_uppercase()) {
std::borrow::Cow::Owned(term.to_ascii_lowercase())
} else {
std::borrow::Cow::Borrowed(*term)
};
// The end of the last kept match, so a chain of touching occurrences
// (`abab…` for term `ab`) that starts inside the window and continues
// past it stays intact — `coalesce_overlapping` merges those, and the
// expansion step below is entitled to follow the merged range out.
let mut chain_end = 0usize;
for at in memchr::memmem::find_iter(folded.as_bytes(), pattern.as_bytes()) {
// Before any dropping: this is the occurrence count, and two
// occurrences that happen to abut are two hits for ranking even
// though they are one highlight for painting.
found += 1;
if bounded {
let bound = *keep_below.get_or_insert_with(|| {
let mut end = (at.saturating_sub(pre_pad) + opts.approx_chars).min(text.len());
while end < text.len() && !text.is_char_boundary(end) {
end += 1;
}
end
});
if at >= bound && at > chain_end {
// Past the edge and not chained to anything kept. Keep
// counting — that is the whole rest of the walk — but stop
// storing.
continue;
}
}
chain_end = at + pattern.len();
matches.push((at, chain_end));
}
}
if matches.is_empty() {
return head_window(text, opts.approx_chars);
return (head_window(text, opts.approx_chars), 0);
}
matches.sort_by_key(|(a, _)| *a);
@ -92,7 +159,6 @@ pub fn extract_folded(text: &str, folded: &str, terms: &[&str], opts: &Options)
// Pick the window. Start a third of the budget before the first match
// so the hit isn't pinned to the left edge; round both ends to char
// boundaries so we never slice a multi-byte UTF-8 sequence.
let pre_pad = opts.approx_chars / 3;
let mut win_start = matches[0].0.saturating_sub(pre_pad);
let mut win_end = (win_start + opts.approx_chars).min(text.len());
while win_start > 0 && !text.is_char_boundary(win_start) {
@ -124,12 +190,15 @@ pub fn extract_folded(text: &str, folded: &str, terms: &[&str], opts: &Options)
})
.collect();
Snippet {
window: text[win_start..win_end].to_string(),
ranges,
truncated_start: win_start > 0,
truncated_end: win_end < text.len(),
}
(
Snippet {
window: text[win_start..win_end].to_string(),
ranges,
truncated_start: win_start > 0,
truncated_end: win_end < text.len(),
},
found,
)
}
/// Clamp `range` into `text` and widen it to the nearest char boundaries.
@ -275,6 +344,12 @@ mod tests {
/// Production always holds a fold buffer already, so the wrapper earned
/// nothing; folding here keeps its coverage of the window logic.
fn extract(text: &str, terms: &[&str], opts: &Options) -> Snippet {
extract_folded(text, &text.to_ascii_lowercase(), terms, opts).0
}
/// The occurrence count alongside the window — the half the full-text
/// pass consumes.
fn extract_counted(text: &str, terms: &[&str], opts: &Options) -> (Snippet, usize) {
extract_folded(text, &text.to_ascii_lowercase(), terms, opts)
}
@ -282,6 +357,88 @@ mod tests {
Options { approx_chars: 40 }
}
/// The count `extract_folded` hands back is what the full-text pass ranks
/// on, so it must be occurrences — not the highlights they coalesce into.
#[test]
fn extract_folded_counts_occurrences_not_ranges() {
let text = "abab and ab";
let (s, n) = extract_counted(text, &["ab"], &opts_small());
assert_eq!(n, 3, "three occurrences");
assert_eq!(
n,
count_occurrences(text, "ab", true),
"must agree with the counter the pass used to call"
);
assert_eq!(s.ranges.len(), 2, "the abutting pair paints as one range");
}
/// The pass takes `count_folded`'s answer from here now; anything that
/// made the two disagree would silently change which rows survive.
#[test]
fn extract_folded_count_agrees_with_count_occurrences() {
let cases: &[(&str, &str)] = &[
("no hits at all", "zzz"),
("one hit here", "hit"),
("Hit hit HIT", "hit"),
("aaaa", "aa"),
("", "x"),
("short", "much longer than the haystack"),
("ünïcode ünïcode", "ünïcode"),
];
for (text, term) in cases {
let folded = text.to_ascii_lowercase();
let (_, n) = extract_folded(text, &folded, &[term], &opts_small());
assert_eq!(
n,
count_occurrences(&folded, &term.to_ascii_lowercase(), true),
"count mismatch for {:?} in {:?}",
term,
text
);
}
}
/// An uppercase needle takes the owning branch of the fold; a needle that
/// is already folded takes the borrowing one. Both must find the same
/// matches, since the whole point of the borrow is that it changes
/// nothing but the allocation.
#[test]
fn extract_folded_needle_case_does_not_change_matches() {
let text = "The Needle and the needle";
let folded = text.to_ascii_lowercase();
let (upper, n_upper) = extract_folded(text, &folded, &["Needle"], &opts_small());
let (lower, n_lower) = extract_folded(text, &folded, &["needle"], &opts_small());
assert_eq!(n_upper, 2);
assert_eq!(n_upper, n_lower);
assert_eq!(upper.ranges, lower.ranges);
assert_eq!(upper.window, lower.window);
}
/// A blank term is dropped rather than searched: an empty needle matches
/// at every byte offset, which would rank a document by its length.
#[test]
fn extract_folded_ignores_blank_terms() {
let text = "some text";
let folded = text.to_ascii_lowercase();
for term in ["", " ", "\t"] {
let (s, n) = extract_folded(text, &folded, &[term], &opts_small());
assert_eq!(n, 0, "blank term {:?} counted", term);
assert!(s.ranges.is_empty(), "blank term {:?} highlighted", term);
}
}
/// A term with surrounding space is searched as given. It used to be
/// trimmed, which counted and highlighted a different string than the
/// pattern's own counters did.
#[test]
fn extract_folded_does_not_trim_a_padded_term() {
let text = "needle needlework";
let folded = text.to_ascii_lowercase();
let (_, n) = extract_folded(text, &folded, &["needle "], &opts_small());
assert_eq!(n, 1, "only the occurrence followed by a space");
assert_eq!(n, count_occurrences(&folded, "needle ", true));
}
/// Every range must be in-bounds, ordered, non-overlapping, and sit on
/// char boundaries — the contract egui's LayoutJob sections rely on.
fn assert_ranges_valid(s: &Snippet) {
@ -299,6 +456,83 @@ mod tests {
s.ranges.iter().map(|&(a, b)| &s.window[a..b]).collect()
}
/// Occurrences past the window's right edge are counted but not kept, and
/// that has to be invisible from the outside.
///
/// The count is what the full-text pass ranks on, so it must still see the
/// whole body; the ranges are what gets painted, so they must still be
/// exactly the occurrences inside the window. A bound that leaked into
/// either would be a ranking change or a missing highlight.
#[test]
fn occurrences_past_the_window_are_counted_but_not_kept() {
// 400 occurrences, evenly spread, far more than a 40-byte window holds.
let unit = "needle filler filler ";
let text = unit.repeat(400);
let (snip, found) = extract_counted(&text, &["needle"], &opts_small());
assert_eq!(found, 400, "every occurrence must still be counted");
assert_ranges_valid(&snip);
assert!(!snip.ranges.is_empty());
assert!(snip.truncated_end, "there is a great deal more body");
// Every range is a real occurrence, and every occurrence that falls
// inside the window has a range.
for &(a, b) in &snip.ranges {
assert_eq!(&snip.window[a..b], "needle");
}
let win_at = text.find(&snip.window).expect("the window is a slice of the text");
let expected = memchr::memmem::find_iter(text.as_bytes(), b"needle")
.filter(|at| *at >= win_at && *at < win_at + snip.window.len())
.count();
assert_eq!(
snip.ranges.len(),
expected,
"every occurrence inside the window must be marked"
);
}
/// The case the chain rule exists for: touching occurrences coalesce into
/// one range, and a chain that starts inside the window can run past its
/// right edge. Dropping the moment the edge is crossed would cut the
/// highlight short.
#[test]
fn a_coalescing_chain_is_not_cut_at_the_window_edge() {
// One unbroken run of `ab`, far longer than the window.
let text = "ab".repeat(400);
let (snip, found) = extract_counted(&text, &["ab"], &opts_small());
assert_eq!(found, 400, "non-overlapping occurrences, all counted");
assert_ranges_valid(&snip);
assert_eq!(snip.ranges.len(), 1, "one chain, one highlight");
// The load-bearing assertion, and it has to be about the window's
// *size*. The whole run coalesces into one range, and the expansion
// step then grows the window to cover it — so an unbroken chain
// legitimately produces a window the length of the text, not the
// 40-byte budget. Dropping the chain rule cuts the run at the budget
// and yields a 40-byte window instead, which asserting only
// "range == whole window" cannot tell apart, because both are.
assert_eq!(
snip.window.len(),
text.len(),
"the window must grow to cover the coalesced run"
);
assert!(!snip.truncated_end, "nothing is left past a full-length window");
assert_eq!(snip.ranges[0], (0, text.len()), "one highlight over the lot");
}
/// Several terms disable the bound, because two different terms can
/// coalesce across the edge and a per-term walk cannot see it. Pin that the
/// multi-term path still produces the full, correct answer.
#[test]
fn several_terms_still_coalesce_across_the_edge() {
let text = format!("{}{}", "x".repeat(10), "abc".repeat(200));
let (snip, found) = extract_counted(&text, &["ab", "bc"], &opts_small());
assert_eq!(found, 400, "200 of each term");
assert_ranges_valid(&snip);
assert_eq!(snip.ranges.len(), 1, "the two terms interleave into one run");
}
#[test]
fn empty_text_returns_empty() {
let s = extract("", &["foo"], &Options::default());

76
crates/quicksearch-core/src/testutil.rs
View file

@ -121,6 +121,56 @@ pub fn touch(path: &std::path::Path, body: &[u8]) {
std::fs::write(path, body).expect("write file");
}
/// A filename that is legal on disk but cannot round-trip through the index,
/// spelled so that `to_string_lossy` yields exactly `{stem}\u{FFFD}{suffix}` —
/// which is itself a perfectly ordinary filename, and so a name a *different*
/// file can really have. That collision is what the screens in
/// `crate::walk::read_directory` and `crate::watcher` exist to prevent, and
/// pairing this with [`lossy_twin`] is how the tests reproduce it.
///
/// The two platforms fail in different ways and both are real:
///
/// * On Unix an `OsStr` is arbitrary bytes, so any invalid UTF-8 byte does it.
/// `0xFF` can never appear in well-formed UTF-8.
/// * On Windows a path is UTF-16 code units and NTFS does not check that they
/// are well-*formed*, so an unpaired surrogate is storable. Rust models this
/// with WTF-8, and `to_str()` returns `None` for precisely that case. Far
/// from theoretical: WSL's DrvFs encodes non-UTF-8 Linux names this way by
/// design, and Samba shares of Linux servers produce them from legacy
/// encodings.
///
/// Some filesystems (FAT, exFAT, some network redirectors) refuse the name —
/// tests that put one on disk must tolerate the creation failing rather than
/// asserting on it.
#[doc(hidden)]
pub fn unrepresentable_name(stem: &str, suffix: &str) -> std::ffi::OsString {
#[cfg(unix)]
{
use std::os::unix::ffi::OsStringExt;
let mut bytes = stem.as_bytes().to_vec();
bytes.push(0xFF);
bytes.extend_from_slice(suffix.as_bytes());
std::ffi::OsString::from_vec(bytes)
}
#[cfg(windows)]
{
use std::os::windows::ffi::OsStringExt;
let mut units: Vec<u16> = stem.encode_utf16().collect();
// A high surrogate with nothing after it to pair with.
units.push(0xD800);
units.extend(suffix.encode_utf16());
std::ffi::OsString::from_wide(&units)
}
}
/// The name [`unrepresentable_name`] collapses to under `to_string_lossy`, as
/// a name that is genuinely representable — so a test can put both on disk and
/// assert the real one survives what happens to the other.
#[doc(hidden)]
pub fn lossy_twin(stem: &str, suffix: &str) -> String {
format!("{}\u{FFFD}{}", stem, suffix)
}
/// Power-of-two bucket, so memory-map sizes group by what allocated them
/// rather than by their exact size. Shared by the memory probes.
#[doc(hidden)]
@ -162,6 +212,32 @@ mod tests {
assert_eq!(std::fs::read(&deep).unwrap(), b"hi");
}
/// The premise every collision test rests on, pinned per platform: the
/// name really is unrepresentable, and its lossy image really is a name
/// another file could have. If this ever stops holding, those tests would
/// silently start asserting nothing.
#[test]
fn the_unrepresentable_name_collapses_onto_its_twin() {
let bad = unrepresentable_name("x", ".txt");
assert!(
bad.to_str().is_none(),
"the name must not be representable: {:?}",
bad
);
assert_eq!(
bad.to_string_lossy(),
lossy_twin("x", ".txt"),
"the two names must collide under to_string_lossy"
);
assert!(
std::path::Path::new(&lossy_twin("x", ".txt"))
.as_os_str()
.to_str()
.is_some(),
"the twin must itself be a perfectly ordinary name"
);
}
/// The sweep runs against a shared temp directory, so what it matches is
/// the whole safety argument. `quicksearch-capture` is the one that would
/// hurt: `packaging/capture.sh` puts a run's screenshots and screencasts

73
crates/quicksearch-core/src/textenc.rs
View file

@ -44,6 +44,13 @@
//! as a FAILED row with a reason, the normal shape of head-based
//! classification.
/// How much of a file the charset detector is shown.
///
/// chardetng scores byte frequencies against a model per encoding; the answer
/// stops moving well inside this. Reading further is a second full pass over
/// the file for no change in the verdict.
const DETECT_PREFIX: usize = 64 * 1024;
use std::path::Path;
enum TextClass {
@ -134,7 +141,14 @@ pub fn decode_text(bytes: Vec<u8>, path: &Path) -> Result<String, String> {
// ISO-2022-JP detection is safe here: the browser caveat about
// it concerns script-running web content, not indexed files.
let mut det = chardetng::EncodingDetector::new(chardetng::Iso2022JpDetection::Allow);
det.feed(&bytes, true);
// A prefix, not the whole file. The detector is scoring byte
// frequencies against per-encoding models, and those converge in
// kilobytes — feeding it a 200 MiB log costs a full extra pass
// over it to reach the same answer. `last` stays false because
// there may be more: it only tells the detector this is not the
// end of the input, which is exactly right for a prefix.
let prefix = &bytes[..bytes.len().min(DETECT_PREFIX)];
det.feed(prefix, prefix.len() == bytes.len());
// Deny UTF-8: strict UTF-8 was already ruled out, so a UTF-8
// guess could only mean malformed UTF-8.
let enc = det.guess(None, chardetng::Utf8Detection::Deny);
@ -145,6 +159,31 @@ pub fn decode_text(bytes: Vec<u8>, path: &Path) -> Result<String, String> {
}
}
/// Replace control characters with `U+FFFD`, borrowing when there are none.
///
/// Filenames may contain any byte but NUL and the separator, and extracted
/// text is whatever was in the file — so both can carry terminal escape
/// sequences. Printed raw they rewrite the line, retitle the window, or on a
/// terminal with OSC 52 enabled put text of the writer's choosing on the
/// user's clipboard. `ls` has scrubbed for this reason for decades.
///
/// Tab survives: it is a legitimate part of a filename and harmless. So does
/// everything above C1 — this is not a general sanitiser, and the point is to
/// stay byte-for-byte faithful wherever there is nothing dangerous to remove.
pub fn scrub_controls(s: &str) -> std::borrow::Cow<'_, str> {
fn dangerous(c: char) -> bool {
(c.is_control() && c != '\t') || ('\u{80}'..='\u{9f}').contains(&c)
}
if !s.chars().any(dangerous) {
return std::borrow::Cow::Borrowed(s);
}
std::borrow::Cow::Owned(
s.chars()
.map(|c| if dangerous(c) { '\u{fffd}' } else { c })
.collect(),
)
}
#[cfg(test)]
mod tests {
use super::*;
@ -215,6 +254,38 @@ mod tests {
);
}
/// Detection reads a bounded prefix, so a file far larger than it still
/// decodes as the encoding its head implies — and the tail is decoded in
/// full regardless, since only the *detector*'s input is bounded.
#[test]
fn detection_prefix_is_bounded_and_the_tail_still_decodes() {
let head = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9. ";
let mut body = Vec::new();
while body.len() < DETECT_PREFIX * 3 {
body.extend_from_slice(head);
}
// A marker past the detector's window: it must survive the decode.
body.extend_from_slice(b"caf\xe9-tail-marker");
let out = decode_text(body, &p()).unwrap();
assert!(
out.ends_with("café-tail-marker"),
"the tail past the detection prefix must still be decoded"
);
assert!(out.starts_with("Le café près"), "got {:?}", &out[..24]);
}
/// The prefix bound must not change the verdict for a file smaller than
/// it — the whole buffer is still what the detector sees.
#[test]
fn detection_prefix_leaves_short_files_alone() {
let body = b"Le caf\xe9 pr\xe8s de la fen\xeatre est agr\xe9able en \xe9t\xe9.".to_vec();
assert!(body.len() < DETECT_PREFIX);
assert_eq!(
decode_text(body, &p()).unwrap(),
"Le café près de la fenêtre est agréable en été."
);
}
#[test]
fn shift_jis_decodes_but_does_not_sniff() {
// "日本語のテキストです。これはシフトJISでエンコードされています。"

173
crates/quicksearch-core/src/walk.rs
View file

@ -26,8 +26,8 @@ use sha2::{Digest, Sha256};
use crate::config::{Config, IgnoreSet};
use crate::extract::Registry;
use crate::file_handling::{
classify_by_mtime, classify_for_indexing, path_to_db_string, prepare_file_record,
warn_if_unrepresentable, DirRows, FileIndexAction, OwnedNewFile, UnreadableDirs,
classify_by_mtime, classify_for_indexing, dir_to_db_parent, path_to_db_string,
prepare_file_record, DirRows, FileIndexAction, OwnedNewFile, UnreadableDirs,
};
mod pool;
@ -53,7 +53,8 @@ const NETWORK_THREADS: usize = 16;
/// One file the walk found, with everything the DB writer needs.
#[derive(Debug)]
pub struct WalkedFile {
/// Canonical path, and the `files.path` key.
/// Canonical path. The row it keys is `(parent, name)`; see
/// [`crate::file_handling::split_db_path`].
pub path: String,
pub action: FileIndexAction,
/// `None` when there is nothing to write: unchanged, or the record could
@ -164,6 +165,13 @@ struct Ctx {
follow_symlinks: bool,
include_hidden: bool,
ignore: IgnoreSet,
/// The index's own files, which this walk must never so much as open.
/// See [`crate::file_handling::index_file_set`] for why opening one is
/// fatal rather than merely wasteful.
///
/// Precomputed rather than derived per entry: it is one canonicalize, and
/// the alternative is a syscall against every file in the tree.
index_files: HashSet<PathBuf>,
pruned: PruneCounts,
config: Config,
registry: Arc<Registry>,
@ -175,9 +183,16 @@ struct Ctx {
/// count is kept. Reset by [`reset_run_warnings`].
static UNREADABLE_WARNINGS: crate::log::Throttle = crate::log::Throttle::new(20);
/// Arm this module's per-run warning throttle.
/// The same, for names that cannot round-trip through the index. Throttled
/// because a share can hold thousands of them: one legacy-encoded directory on
/// a Samba mount, or a `\\wsl.localhost\` tree, and every entry under it is a
/// separate occurrence.
static UNREPRESENTABLE_WARNINGS: crate::log::Throttle = crate::log::Throttle::new(20);
/// Arm this module's per-run warning throttles.
pub fn reset_run_warnings() {
UNREADABLE_WARNINGS.reset();
UNREPRESENTABLE_WARNINGS.reset();
}
/// Read one directory, apply the hidden/ignore rules, and split the result:
@ -204,7 +219,19 @@ fn read_directory(
if UNREADABLE_WARNINGS.allow() {
crate::log_warn!("cannot read {}: {}", dir.display(), e);
}
ctx.unreadable.record(dir.to_path_buf());
// "Gone" is not "could not look", and only the second one is a
// reason to distrust the walk. A directory deleted while the walk
// was in flight — a build tree, a browser cache — is a fact about
// the filesystem: its rows *should* fall to the stale sweep, and
// recording it here would both spare them and cost this root its
// stored walk count, which is what makes every later run pay for
// a second `find | wc` traversal of the whole tree.
//
// The same distinction the coordinator draws in `verb_for`:
// `NotFound` is unambiguous, every other errno is not.
if e.kind() != std::io::ErrorKind::NotFound {
ctx.unreadable.record(dir.to_path_buf());
}
return Vec::new();
}
};
@ -231,13 +258,49 @@ fn read_directory(
};
let name = entry.file_name();
let name = name.to_string_lossy();
// **The screen for names the index cannot spell, and the only one.**
//
// `files.name` and `files.parent` are TEXT, so a name that is not valid
// UTF-8 has no representation there; `to_string_lossy` would give one,
// but it is many-to-one, and every use of a path in this walk is a
// database *key*. A lossy name collides with the real name of a
// different file — U+FFFD is an ordinary filename character — and the
// collision is not a cosmetic one: the lossy parent makes the
// prefetcher hand this directory another directory's rows, and the diff
// below then reports all of them stale. Screening here, before any
// string is built, is what keeps that from being possible at all.
//
// On Unix this is any invalid byte sequence; on Windows it is an
// unpaired UTF-16 surrogate, which NTFS stores happily and WSL's DrvFs
// emits by design for non-UTF-8 Linux names.
//
// A directory is pruned whole and at its root: the join below would
// carry the bad component into every path beneath it, so nothing under
// it could be indexed either way, and stopping here costs one warning
// instead of one per descendant.
//
// Leaving the entry out of `present` is safe, despite the rule above
// that every `continue` must be a genuine "not indexable". No stored
// row can carry a name that is not valid UTF-8, so this entry has no
// row to protect; and a row whose name happens to *equal* the lossy
// spelling belongs to some other, representable entry, which this same
// listing yields separately and which marks itself present.
let Some(name) = name.to_str() else {
if UNREPRESENTABLE_WARNINGS.allow() {
crate::log_warn!(
"Skipping {:?} (name is not valid UTF-8, so it cannot be stored, hashed \
or text-indexed)",
entry.path()
);
}
continue;
};
// The closure runs only on Windows, where `entry.metadata()` is free —
// the attributes came back with the directory read, and it reports the
// entry itself rather than a link target.
if !ctx.include_hidden {
if let Some(reason) =
crate::platform::entry_hidden_reason(&name, || entry.metadata().ok())
crate::platform::entry_hidden_reason(name, || entry.metadata().ok())
{
match reason {
crate::platform::HiddenReason::DotPrefix => {
@ -260,7 +323,7 @@ fn read_directory(
continue;
}
}
if ctx.ignore.matches_component(&name) {
if ctx.ignore.matches_component(name) {
ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed);
continue;
}
@ -269,6 +332,20 @@ fn read_directory(
ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed);
continue;
}
// The index's own database and sidecars. Not a user preference and
// not overridable, because hashing one is not a slow row, it is a
// process-wide cancellation of SQLite's locks on a file we are in the
// middle of writing — see `file_handling::index_file_set`.
//
// `continue` rather than a `WalkedFile::skipped`, deliberately: this
// leaves the name out of `present`, so any row an earlier run wrote
// for the index — before this pruning existed, or from a spell when
// `database_path` pointed elsewhere — falls to the stale sweep and is
// deleted. `skipped` would keep it forever.
if ctx.index_files.contains(&path) {
ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed);
continue;
}
// `file_type` is the cached `d_type` from the directory read.
match entry.file_type() {
@ -291,7 +368,33 @@ fn read_directory(
// which full-path ignore patterns would never match and
// `seen_dirs` could not dedup against an overlapping root.
if let Ok(target) = path.canonicalize() {
// The link's own name passed the screen above; the target
// is a different path and gets its own. Without this, a
// link to an unrepresentable path would be resolved to its
// *lossy* spelling — a path naming some other file
// entirely, which would then be walked or indexed in its
// place.
if target.to_str().is_none() {
if UNREPRESENTABLE_WARNINGS.allow() {
crate::log_warn!(
"Skipping {} (its target {:?} is not valid UTF-8, so it cannot \
be stored, hashed or text-indexed)",
path.display(),
target
);
}
continue;
}
let target = PathBuf::from(path_to_db_string(&target));
// Again on the resolved target: the check above tested the
// link's own name, and a symlink pointing at the index
// would otherwise walk straight past it into an `open`.
// Harmless for a directory target — the set holds only
// files — which is why one check covers both arms.
if ctx.index_files.contains(&target) {
ctx.pruned.ignored.fetch_add(1, Ordering::Relaxed);
continue;
}
match fs::metadata(&target) {
Ok(m) if m.is_dir() => found.push(Found::Dir(target)),
// The target's row belongs to its own directory, so
@ -302,7 +405,7 @@ fn read_directory(
}
}
Ok(_) => {
present.insert(name.into_owned());
present.insert(name.to_string());
// `None` on Unix and on any reparse point: see
// `entry_cached_metadata`.
let cached = crate::platform::entry_cached_metadata(|| entry.metadata().ok());
@ -310,7 +413,7 @@ fn read_directory(
}
// Type unknown: mark it present so an existing row survives.
Err(_) => {
present.insert(name.into_owned());
present.insert(name.to_string());
}
}
}
@ -318,7 +421,7 @@ fn read_directory(
if !unreadable_entry {
// Rebuild each stored path the way `prepare` does, by joining onto
// the canonical directory, so separators and roots match the
// `files.path` spelling exactly.
// stored spelling exactly.
stale.extend(
rows.keys()
.filter(|name| !present.contains(name.as_str()))
@ -365,16 +468,23 @@ pub fn path_digest(path: &str) -> u128 {
/// see [`crate::platform::metadata_or_stat`].
fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile {
let PendingFile { path, cached } = file;
// Every route here has already screened the path: a root is screened after
// canonicalizing (which can resolve onto a name the config string was
// not), `Job::Files` comes from a listing `read_directory` filtered, and
// `Job::Alias` from a target the symlink arm checked. That
// matters because `path_to_db_string` is lossy, and the string below is
// used as a database key *and* hashed into the run's duplicate-visit set —
// a lossy one would key another file's row and could consume its digest,
// silently dropping it from the index.
debug_assert!(
path.to_str().is_some(),
"an unrepresentable path reached prepare(): {:?}",
path
);
let db_path = path_to_db_string(&path);
let digest = path_digest(&db_path);
let aliased = matches!(known, Known::Exact(_));
// A name that is not valid UTF-8 cannot be stored in `files.path`. Emitted
// as `Skip` because the caller reads a missing path as "deleted".
if warn_if_unrepresentable(&path) {
return WalkedFile::skipped(db_path, digest, aliased);
}
let Ok(meta) = crate::platform::metadata_or_stat(&path, cached) else {
// Seen but unreadable: a transient stat failure must not read as
// "deleted".
@ -391,11 +501,15 @@ fn prepare(file: PendingFile, known: Known<'_>, ctx: &Ctx) -> WalkedFile {
let action = match known {
Known::InDir(rows) => {
// `to_str`, not `to_string_lossy`: this name is looked up in the
// directory's stored rows, and the lossy spelling of one file is a
// valid name for another. The screen in `read_directory` is what
// makes it always `Some`.
let name = path
.file_name()
.map(|n| n.to_string_lossy().into_owned())
.and_then(|n| n.to_str())
.unwrap_or_default();
classify_for_indexing(&name, mtime, rows)
classify_for_indexing(name, mtime, rows)
}
Known::Exact(stored) => classify_by_mtime(stored, mtime),
};
@ -494,7 +608,7 @@ fn prefetcher(shared: &Shared, db_path: &str) {
while let Some(work) = shared.take_prefetch() {
match work {
PrefetchWork::Dir(dir) => {
match crate::db::repo::dir_rows(&conn, &path_to_db_string(&dir)) {
match crate::db::repo::dir_rows(&conn, &dir_to_db_parent(&dir)) {
Ok(rows) => shared.finish_prefetch(Job::Dir(dir, Arc::new(rows))),
Err(e) => {
crate::log_warn!("{}", e);
@ -551,7 +665,7 @@ impl ParallelWalk {
crate::lock_ok(&self.shared.queue)
.seen_dirs
.iter()
.map(|d| path_to_db_string(d))
.map(|d| dir_to_db_parent(d))
.collect()
}
@ -712,6 +826,22 @@ pub fn walk_indexable_files(
//
// Roots themselves are never filtered — the user chose them.
match fs::canonicalize(root) {
// A root string is UTF-8 by construction — it came from the config
// — but `canonicalize` resolves symlinks, so what it resolves *to*
// need not be: `~/docs` can be a link to a directory whose real
// name the index cannot spell. Stored lossily, the root would be
// walked under a parent string that names some other directory
// entirely. Treated exactly like a root that would not resolve at
// all, which is what it amounts to: yields nothing, and is recorded
// so stale cleanup does not read that as "everything was deleted".
Ok(dir) if dir.to_str().is_none() => {
crate::log_warn!(
"cannot index root {}: it resolves to {:?}, whose name is not valid UTF-8",
root,
dir
);
unresolvable.push(PathBuf::from(root));
}
Ok(dir) => {
let dir = PathBuf::from(path_to_db_string(&dir));
if queue.seen_dirs.insert(dir.clone()) {
@ -738,6 +868,7 @@ pub fn walk_indexable_files(
follow_symlinks,
include_hidden,
ignore,
index_files: crate::file_handling::index_file_set(Path::new(db_path)),
pruned: PruneCounts::default(),
config,
registry,

355
crates/quicksearch-core/src/walk/tests.rs
View file

@ -14,16 +14,13 @@ fn db_with(tag: &str, rows: &[(String, u64)]) -> PathBuf {
let p = crate::testutil::scratch_dir(tag).join("index.sqlite");
let conn = crate::db::open_or_recreate(p.to_str().unwrap(), "trigram").unwrap();
for (path, mtime) in rows {
let as_path = Path::new(path);
// Split the same way the indexer does, so the seeded parent carries
// its trailing separator and the prefetcher's `parent = ?` finds it.
let (parent, name) = crate::file_handling::split_db_path(path).expect("a file's path");
conn.execute(
"INSERT INTO files (name, path, parent, size, mtime, type, content_state)
VALUES (?1, ?2, ?3, 0, ?4, 0, 3)",
rusqlite::params![
as_path.file_name().unwrap().to_string_lossy(),
path,
as_path.parent().unwrap().to_string_lossy(),
*mtime as i64,
],
"INSERT INTO files (name, parent, size, mtime, type, content_state)
VALUES (?1, ?2, 0, ?3, 0, 3)",
rusqlite::params![name, parent, *mtime as i64],
)
.unwrap();
}
@ -108,35 +105,237 @@ fn walks_a_nested_tree_exactly_once() {
fs::remove_dir_all(&root).ok();
}
/// A name that is not valid UTF-8 must be skipped, not deleted. Unix only:
/// on Windows `OsString` comes from UTF-16 and the case cannot be built.
#[cfg(unix)]
#[test]
fn a_non_utf8_name_is_skipped_and_never_prepared() {
use std::ffi::OsStr;
use std::os::unix::ffi::OsStrExt;
/// Put `name` on disk under `dir`, or report that this filesystem refused it.
///
/// FAT, exFAT and some network redirectors reject the names
/// [`crate::testutil::unrepresentable_name`] builds. A test that cannot create
/// one has nothing to assert and says so, rather than failing on the
/// filesystem's behalf.
fn try_touch(path: &Path) -> bool {
fs::create_dir_all(path.parent().unwrap()).unwrap();
fs::write(path, b"x").is_ok() && path.symlink_metadata().is_ok()
}
fn try_mkdir(path: &Path) -> bool {
fs::create_dir_all(path).is_ok() && path.symlink_metadata().is_ok()
}
/// A name the index cannot spell is dropped at the directory entry, before any
/// string is built from it — so it is neither prepared nor reported.
///
/// Both platforms, and the Windows half is not the exotic one: `OsString`
/// there is WTF-8 over UTF-16, and `OsString::from_wide(&[0xD800])` builds an
/// unpaired surrogate that NTFS stores happily. (An older comment here claimed
/// the case could not be built on Windows. It can, which is exactly why WSL
/// and Samba trees hit it.)
#[test]
fn a_non_utf8_name_is_dropped_at_the_entry() {
let root = tmp_tree("nonutf8");
touch(&root.join("plain.txt"));
// 0xFF only survives `to_string_lossy` as U+FFFD.
let bad = root.join(OsStr::from_bytes(b"DRH257\xff~X.MP4"));
touch(&bad);
assert!(bad.symlink_metadata().is_ok(), "the file really is on disk");
let bad = root.join(crate::testutil::unrepresentable_name("DRH257", "~X.MP4"));
if !try_touch(&bad) {
eprintln!("skipped: this filesystem will not store an unrepresentable name");
return;
}
let files = walk(&root, &empty_db("nonutf8"));
// Both are yielded, so neither reads as deleted...
assert_eq!(files.len(), 2, "the bad name is still reported as seen");
// ...but only the representable one is prepared for insertion.
let prepared: Vec<&WalkedFile> = files.iter().filter(|f| f.record.is_some()).collect();
assert_eq!(prepared.len(), 1);
assert!(prepared[0].path.ends_with("plain.txt"));
let skipped = files.iter().find(|f| f.record.is_none()).unwrap();
assert!(matches!(skipped.action, FileIndexAction::Skip));
// Not yielded at all — a `WalkedFile` for it would carry the lossy path as
// its key and its duplicate-visit digest.
assert_eq!(names(&files), vec!["plain.txt"]);
assert!(
skipped.path.contains('\u{FFFD}'),
"stored spelling is the lossy one"
files.iter().all(|f| !f.path.contains('\u{FFFD}')),
"no lossy spelling may reach the writer: {:?}",
files.iter().map(|f| &f.path).collect::<Vec<_>>()
);
fs::remove_dir_all(&root).ok();
}
/// The safety argument for dropping the entry, pinned: a `continue` in
/// `read_directory` leaves the name out of `present`, and everything left out
/// of `present` is deleted.
///
/// It is safe because the dropped entry has no row of its own — an
/// unrepresentable name cannot be stored — and because the row that *is*
/// named its lossy spelling belongs to a different file, which the same
/// listing yields separately and which marks itself present. This test is what
/// says that second half still happens.
#[test]
fn dropping_a_bad_entry_deletes_nothing() {
let root = tmp_tree("nonutf8-stale");
let twin = crate::testutil::lossy_twin("DRH257", "~X.MP4");
touch(&root.join("plain.txt"));
touch(&root.join(&twin));
let bad = root.join(crate::testutil::unrepresentable_name("DRH257", "~X.MP4"));
if !try_touch(&bad) {
eprintln!("skipped: this filesystem will not store an unrepresentable name");
return;
}
// Seed both real files as already indexed, so anything the walk reports
// stale is a row it wants deleted.
let db = db_with(
"nonutf8-stale",
&[
(path_to_db_string(&root.join("plain.txt")), 0),
(path_to_db_string(&root.join(&twin)), 0),
],
);
let stale = stale_only(walk_indexable_files(
&[root.to_string_lossy().into_owned()],
false,
false,
IgnoreSet::compile(&[]).unwrap(),
db.to_str().unwrap(),
Config::default(),
Arc::new(Registry::default_set()),
Arc::new(AtomicBool::new(false)),
4,
));
assert!(
stale.is_empty(),
"both files are on disk; nothing may be deleted: {:?}",
stale
);
fs::remove_dir_all(&root).ok();
}
/// A root spelled in UTF-8 can still *resolve* onto a name that is not:
/// canonicalizing follows symlinks, so `~/docs` may be a link to a directory
/// the index cannot spell.
///
/// Stored lossily, that root's parent string names some other directory — so
/// the walk would attribute a completely unrelated tree to it, or read nothing
/// and report the root's real rows as deleted. It has to be recorded as
/// unreadable instead, which is what keeps stale cleanup off it.
#[cfg(unix)]
#[test]
fn a_root_that_resolves_onto_an_unrepresentable_name_is_not_walked() {
let base = tmp_tree("nonutf8-root");
let real = base.join(crate::testutil::unrepresentable_name("target", ""));
if !try_mkdir(&real) {
eprintln!("skipped: this filesystem will not store an unrepresentable name");
return;
}
touch(&real.join("inside.txt"));
// The root the user configures is an ordinary, spellable string.
let link = base.join("docs");
std::os::unix::fs::symlink(&real, &link).unwrap();
let root = link.to_str().expect("the configured root is spellable");
let mut walk = walk_indexable_files(
&[root.to_string()],
false,
false,
IgnoreSet::compile(&[]).unwrap(),
empty_db("nonutf8-root").to_str().unwrap(),
Config::default(),
Arc::new(Registry::default_set()),
Arc::new(AtomicBool::new(false)),
4,
);
let events: Vec<WalkEvent> = walk.by_ref().collect();
assert!(
events.is_empty(),
"nothing under an unspellable root may be walked: {:?}",
events
);
assert!(
!walk.unreadable().is_empty(),
"the root must read as unreadable, or its rows fall to stale cleanup"
);
std::fs::remove_dir_all(&base).ok();
}
/// The collision that made this worth fixing: a file whose *lossy* spelling is
/// another file's real name must not stand in for it.
///
/// Before the screen moved to the directory entry, the bad name was still
/// turned into a `WalkedFile` carrying the lossy path — and the pipeline hashes
/// that path into `seen_paths` before it looks for a record, so whichever of
/// the two the walk happened to reach second was dropped from the index.
#[test]
fn a_bad_name_cannot_stand_in_for_its_lossy_twin() {
let root = tmp_tree("nonutf8-twin");
let twin = crate::testutil::lossy_twin("x", ".txt");
touch(&root.join(&twin));
let bad = root.join(crate::testutil::unrepresentable_name("x", ".txt"));
if !try_touch(&bad) {
eprintln!("skipped: this filesystem will not store an unrepresentable name");
return;
}
let files = walk(&root, &empty_db("nonutf8-twin"));
let prepared: Vec<&WalkedFile> = files.iter().filter(|f| f.record.is_some()).collect();
assert_eq!(
prepared.len(),
1,
"exactly the representable file is prepared"
);
assert_eq!(
prepared[0].path,
path_to_db_string(&root.join(&twin)),
"and it is the real one, under its own name"
);
fs::remove_dir_all(&root).ok();
}
/// A directory the index cannot spell is pruned at its root: every path
/// beneath it would carry the bad component, so none of them is indexable
/// either way.
///
/// The reason this is the *directory* case and not just N file cases: the
/// walk's row prefetcher keys on the directory's lossy parent string, so a bad
/// directory sitting beside a real one with the colliding name was handed the
/// real one's rows — and then reported every one of them stale.
#[test]
fn a_bad_directory_is_pruned_with_its_whole_subtree() {
let root = tmp_tree("nonutf8-dir");
let twin = crate::testutil::lossy_twin("dir", "");
touch(&root.join(&twin).join("a.txt"));
let bad = root.join(crate::testutil::unrepresentable_name("dir", ""));
if !try_mkdir(&bad) {
eprintln!("skipped: this filesystem will not store an unrepresentable name");
return;
}
for n in 0..5 {
touch(&bad.join(format!("b{}.txt", n)));
}
touch(&bad.join("deep/c.txt"));
let db = db_with(
"nonutf8-dir",
&[(path_to_db_string(&root.join(&twin).join("a.txt")), 0)],
);
let stale = stale_only(walk_indexable_files(
&[root.to_string_lossy().into_owned()],
false,
false,
IgnoreSet::compile(&[]).unwrap(),
db.to_str().unwrap(),
Config::default(),
Arc::new(Registry::default_set()),
Arc::new(AtomicBool::new(false)),
4,
));
assert!(
stale.is_empty(),
"the real directory's row must survive its bad-named sibling: {:?}",
stale
);
let files = walk(&root, &db);
assert_eq!(
names(&files),
vec!["a.txt"],
"nothing under the bad directory is walked"
);
fs::remove_dir_all(&root).ok();
@ -804,3 +1003,97 @@ fn local_temp_dir_is_not_detected_as_network() {
);
fs::remove_dir_all(&root).ok();
}
/// **The index must never be walked into.**
///
/// Hashing a file means opening it, and on Unix closing a descriptor on an
/// inode cancels every advisory lock the whole process holds on it. Doing that
/// to `index.sqlite-shm` destroys the DMS lock SQLite took, after which an
/// attaching connection truncates the wal-index under our live mapping and the
/// next commit dies with SIGBUS — which is the crash this pruning exists to
/// prevent. The same close cancels the database's own locks, SQLite's
/// documented corruption hazard.
///
/// `include_hidden` is on because that is what exposes the default layout:
/// the index lives under `~/.local/share`, which a default walk skips for
/// being dot-named — so the hazard is real but latent until a user turns
/// hidden files on.
#[test]
fn the_index_and_its_sidecars_are_never_walked() {
let root = tmp_tree("walk-self-index");
touch(&root.join("ordinary.txt"));
// The index inside the tree being walked, as it is by default: the
// default root is the home directory and the default database sits
// beneath it.
let db = root.join("data").join("index.sqlite");
let conn = crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
// A write, so the WAL and SHM exist to be walked over.
conn.execute(
"INSERT INTO files (name, parent, size, mtime, type, content_state)
VALUES ('a', '/', 0, 0, 0, 3)",
[],
)
.unwrap();
let found = names(&walk_with(&root, &db, false, true));
assert!(
found.contains(&"ordinary.txt".to_string()),
"the walk should still report ordinary files: {:?}",
found
);
for name in ["index.sqlite", "index.sqlite-wal", "index.sqlite-shm"] {
assert!(
!found.contains(&name.to_string()),
"{} was walked; found {:?}",
name,
found
);
}
drop(conn);
}
/// Pruning it is not enough — a row written before this existed has to go, or
/// the live watcher and duplicate verification keep opening the file from a
/// result list forever.
///
/// This is why the walk `continue`s past the index rather than emitting a
/// `WalkedFile::skipped`: a skip keeps the row, and only leaving the name out
/// of the directory's `present` set hands it to the stale sweep.
#[test]
fn a_stored_row_for_the_index_falls_to_the_stale_sweep() {
let root = crate::testutil::scratch_dir_canonical("walk-self-index-stale");
// The database the walk uses has to be the one inside the walked tree,
// or there is nothing self-referential to sweep.
let db = root.join("data").join("index.sqlite");
let conn = crate::db::open_or_recreate(db.to_str().unwrap(), "trigram").unwrap();
// The row an older build would have written for the database itself,
// inserted exactly the way `db_with` inserts one.
let stored = path_to_db_string(&db);
let (parent, name) = crate::file_handling::split_db_path(&stored).expect("a file's path");
conn.execute(
"INSERT INTO files (name, parent, size, mtime, type, content_state)
VALUES (?1, ?2, 0, 0, 0, 3)",
rusqlite::params![name, parent],
)
.unwrap();
drop(conn);
let stale = stale_only(walk_indexable_files(
&[root.to_string_lossy().into_owned()],
false,
true,
IgnoreSet::compile(&[]).unwrap(),
db.to_str().unwrap(),
Config::default(),
Arc::new(Registry::default_set()),
Arc::new(AtomicBool::new(false)),
4,
));
assert!(
stale.contains(&stored),
"the index's own row should be swept; got {:?}",
stale
);
}

109
crates/quicksearch-core/src/watcher.rs
View file

@ -30,7 +30,7 @@
//! root looks live while going silently stale. A single directory the kernel
//! refuses is logged and skipped instead (see [`add_watch`]).
use std::collections::{HashMap, HashSet};
use std::collections::{BTreeSet, HashMap};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{mpsc, Arc, Mutex};
@ -97,6 +97,11 @@ pub enum WatchError {
KernelLimit {
registered: usize,
},
/// The kernel's event queue overflowed and events were dropped. Unlike
/// the two above this says nothing about the watcher's *capacity* — it
/// keeps working — only that the index is now out of step with the disk
/// by an unknown amount, so a full run is owed.
Overflowed,
Other(String),
}
@ -122,6 +127,11 @@ impl std::fmt::Display for WatchError {
""
}
),
WatchError::Overflowed => write!(
f,
"the system event queue overflowed and changes were missed; \
reindexing to catch up"
),
WatchError::Other(msg) => write!(f, "{}", msg),
}
}
@ -156,8 +166,9 @@ pub struct WatcherConfig {
/// How often the tick loop inspects the throttle map. Short ticks mean
/// low latency for first-in-a-burst; long ticks lower CPU at idle.
pub tick_interval: Duration,
/// Maximum directories processed per tick. Caps the time spent in a
/// single flush pass so long backlogs don't monopolize the thread.
/// Maximum directories *selected* per tick. A ceiling on the work one
/// pass takes on; [`FLUSH_BUDGET`] is what bounds how long it may spend
/// on them, and is the real limit.
pub max_dirs_per_tick: usize,
/// When to garbage-collect stale throttle entries (idle > window * N).
pub prune_max_age_multiplier: u32,
@ -172,7 +183,7 @@ impl Default for WatcherConfig {
pending_max_defer: Duration::from_secs(30),
throttle_window: Duration::from_secs(30),
tick_interval: Duration::from_millis(500),
max_dirs_per_tick: 64,
max_dirs_per_tick: 512,
prune_max_age_multiplier: 10,
max_watched_dirs: DEFAULT_MAX_WATCHED_DIRS,
}
@ -187,10 +198,23 @@ impl Default for WatcherConfig {
/// never blocks behind a large subtree registration.
struct WatchRegistry {
raw: RecommendedWatcher,
dirs: HashSet<PathBuf>,
/// Ordered, not hashed, so [`WatchRegistry::remove_tree`] can take the
/// subtree as a range instead of scanning every watched directory. The
/// set reaches `max_watched_dirs` (128k by default) and `rm -rf` deletes
/// bottom-up, so every directory in a deleted tree hits that path.
dirs: BTreeSet<PathBuf>,
cap: usize,
}
/// How long one flush pass may spend handing events to the sink.
///
/// Paired with `max_dirs_per_tick`: the count decides how many directories a
/// pass takes on, this decides when it stops regardless. A backlog then
/// drains at whatever the machine can actually do rather than at a fixed
/// directories-per-second, while one enormous directory still cannot hold the
/// tick loop.
const FLUSH_BUDGET: Duration = Duration::from_millis(50);
/// The mode every `watch()` call uses on this platform. See
/// [`crate::platform::WATCH_ROOTS_RECURSIVELY`] for why it differs.
const WATCH_MODE: RecursiveMode = if crate::platform::WATCH_ROOTS_RECURSIVELY {
@ -243,10 +267,15 @@ impl WatchRegistry {
if !self.dirs.contains(dir) {
return 0;
}
// A range from `dir`, stopping at the first entry that is no longer
// beneath it: descendants sort immediately after their ancestor, so
// this visits the subtree and one entry more, rather than the whole
// set. `starts_with` is still the test — it compares whole components,
// where a raw string prefix would take `/a/bc` for a child of `/a/b`.
let doomed: Vec<PathBuf> = self
.dirs
.iter()
.filter(|d| d.starts_with(dir))
.range(dir.to_path_buf()..)
.take_while(|d| d.starts_with(dir))
.cloned()
.collect();
for d in &doomed {
@ -311,6 +340,21 @@ impl Watcher {
let raw = RecommendedWatcher::new(
move |res: notify::Result<NotifyEvent>| match res {
Ok(ev) => {
// An overflow of the kernel's own event queue arrives here
// — on the *Ok* arm, as `EventKind::Other` with the rescan
// flag and no paths at all — so the error arm below never
// sees it and the per-path loop downstream iterates zero
// times. Left alone it is silent data loss: an arbitrary
// set of creates, modifies and removes never reaches the
// index while the watcher goes on reporting itself
// healthy. It is reported rather than repaired here
// because the events are simply gone; only a full run can
// find out what happened.
if ev.need_rescan() {
let mut slot = crate::lock_ok(&degraded_cb);
slot.get_or_insert(WatchError::Overflowed);
return;
}
// A closed receiver just means the watcher was stopped; ignore.
let _ = tx.send(ev);
}
@ -330,7 +374,7 @@ impl Watcher {
let registry = Arc::new(Mutex::new(WatchRegistry {
raw,
dirs: HashSet::new(),
dirs: BTreeSet::new(),
cap: config.max_watched_dirs,
}));
@ -416,6 +460,19 @@ impl Watcher {
crate::lock_ok(&self.degraded).is_some()
}
/// Forget the recorded reason, so a later one can take its place.
///
/// For [`WatchError::Overflowed`] only, and the distinction is the whole
/// point of the method: the other two reasons are *standing* — the watch
/// budget does not come back — while an overflow is a one-shot "you missed
/// some" from a watcher that is still delivering. Left in place it would
/// re-trigger on every coordinator tick and, worse, mask a real
/// [`WatchError::KernelLimit`] arriving afterwards, because the callback
/// records with `get_or_insert`.
pub fn clear_degraded(&self) {
*crate::lock_ok(&self.degraded) = None;
}
/// Signal the background thread to stop and wait for it to join. Safe to
/// call multiple times.
pub fn stop(&mut self) {
@ -565,8 +622,19 @@ fn watch_if_new_dir(ctx: &LoopCtx, path: &Path) {
return;
}
// Files are reported through their parent's watch; only directories
// need one of their own.
if !path.is_dir() {
// need one of their own. `symlink_metadata` rather than `is_dir` so a
// symlink to a directory is judged as the link it is: when following is
// off the walk will not descend it, and watching it would spend
// descriptors reporting events for a subtree that is never indexed. When
// following is on it is a directory as far as everything else is
// concerned, so fall back to the followed answer.
let is_dir = match std::fs::symlink_metadata(path) {
Ok(md) if md.file_type().is_symlink() => ctx.filters.follow_symlinks && path.is_dir(),
Ok(md) => md.is_dir(),
// Raced with a delete, or unreadable: nothing to register.
Err(_) => return,
};
if !is_dir {
return;
}
if !is_event_interesting(ctx, path) {
@ -594,6 +662,16 @@ fn unwatch_tree(ctx: &LoopCtx, path: &Path) {
/// on which subtrees exist. Under a recursive root watch it is the *only*
/// thing keeping `node_modules` churn out of the throttle map.
fn is_event_interesting(ctx: &LoopCtx, path: &Path) -> bool {
// A path the index cannot spell, screened here because this is the one
// gate every `FsEvent` passes through. Such a file is never indexed, so
// there is no row for a Create to update and none for a Remove to delete —
// but the incremental side keys on `path_to_db_string`, which is lossy, so
// letting the event through means acting on whichever *different* file
// happens to own the lossy spelling. The event carries no information and
// every use of it is a mistake.
if path.to_str().is_none() {
return false;
}
if ctx.filters.ignore.matches_path(path) {
return false;
}
@ -696,7 +774,18 @@ fn flush_ready(
}
}
}
// The count above bounds how many directories are *selected*; this bounds
// how long draining them may take, which is the thing that actually
// matters. Directory queues differ by orders of magnitude, so a fixed
// count is either too small after a large delete — at 64 per 500 ms tick
// the drain rate is 128 directories a second regardless of backlog, and a
// 20k-directory unpack takes minutes to reach the index — or too large for
// one deep directory.
let deadline = now + FLUSH_BUDGET;
for dir in ready {
if Instant::now() >= deadline {
break;
}
if let Some(entry) = throttle.get_mut(&dir) {
let drained: Vec<(PathBuf, QueuedOp)> = entry.queue.drain().collect();
entry.immediate = false;

106
crates/quicksearch-core/src/watcher_tests.rs
View file

@ -189,6 +189,75 @@ fn e2e_create_modify_remove_surfaces() {
std::fs::remove_dir_all(&dir).ok();
}
/// A file the index cannot spell must produce no event at all.
///
/// It is never indexed, so no row exists for a `Create` to update or a
/// `Remove` to delete — but the incremental side keys on `path_to_db_string`,
/// which is lossy, so an event that got through would be applied to whichever
/// *different* file owns the lossy spelling. Screened in
/// [`is_event_interesting`], the one gate every `FsEvent` passes through.
///
/// A real file is created alongside, so a run where the watcher simply saw
/// nothing cannot pass by accident.
#[test]
fn events_for_an_unrepresentable_name_never_surface() {
let dir = tmp_dir("e2e-unrepresentable");
let bad = dir.join(crate::testutil::unrepresentable_name("report", ".txt"));
if std::fs::write(&bad, "hi").is_err() {
eprintln!("skipped: this filesystem will not store an unrepresentable name");
std::fs::remove_dir_all(&dir).ok();
return;
}
std::fs::remove_file(&bad).unwrap();
let (sink, got) = sink_to_vec();
let mut w = Watcher::start(
std::iter::once(&dir),
default_filters(),
fast_config(),
sink,
)
.unwrap();
std::fs::write(&bad, "hi").unwrap();
std::thread::sleep(Duration::from_millis(150));
std::fs::remove_file(&bad).unwrap();
// The control: whatever the backend does for the bad name, it certainly
// reports this one, so an empty event list means the watcher was working.
let good = dir.join("ordinary.txt");
std::fs::write(&good, "hi").unwrap();
std::thread::sleep(Duration::from_millis(250));
w.stop();
let events = got.lock().unwrap().clone();
assert!(
events.iter().any(|e| matches!(
e,
FsEvent::Create(p) | FsEvent::Modify(p) if p == &good
)),
"the control file produced no event, so this test proves nothing: {:?}",
events
);
let leaked: Vec<&FsEvent> = events
.iter()
.filter(|e| {
let paths: Vec<&PathBuf> = match e {
FsEvent::Create(p) | FsEvent::Modify(p) | FsEvent::Remove(p) => vec![p],
FsEvent::Rename { from, to } => vec![from, to],
};
paths.iter().any(|p| p.to_str().is_none())
})
.collect();
assert!(
leaked.is_empty(),
"an unrepresentable path reached the sink: {:?}",
leaked
);
std::fs::remove_dir_all(&dir).ok();
}
/// Descriptors are not spent on directories the indexer would discard.
#[test]
fn ignored_and_hidden_dirs_are_not_registered() {
@ -516,3 +585,40 @@ fn a_created_ignored_directory_is_not_watched() {
w.stop();
std::fs::remove_dir_all(&dir).ok();
}
/// Registration must not follow a symlinked directory when the indexer will
/// not: every descriptor spent there reports events for a subtree that gets
/// discarded on arrival, and on Linux the watch budget is a shared kernel
/// resource.
#[test]
#[cfg(unix)]
fn a_symlinked_directory_is_not_registered_when_following_is_off() {
let dir = tmp_dir("symlink-reg");
std::fs::create_dir_all(dir.join("real/nested")).unwrap();
let outside = tmp_dir("symlink-reg-target");
std::fs::create_dir_all(outside.join("deep")).unwrap();
std::os::unix::fs::symlink(&outside, dir.join("link")).unwrap();
let w = Watcher::start(
std::iter::once(&dir),
default_filters(),
fast_config(),
sink_to_vec().0,
)
.unwrap();
// root + real + real/nested. The link and everything under it cost
// nothing.
assert_eq!(
w.watched_dirs(),
if crate::platform::WATCH_ROOTS_RECURSIVELY {
1
} else {
3
},
"the symlinked directory must not be registered"
);
drop(w);
std::fs::remove_dir_all(&dir).ok();
std::fs::remove_dir_all(&outside).ok();
}

348
crates/quicksearch-core/tests/cascade.rs
View file

@ -31,15 +31,18 @@ impl Seeder {
}
/// Insert a file; `text: Some(..)` also content-indexes it.
///
/// `dir` is spelled without a trailing separator, as a caller naturally
/// would; the stored parent always carries one, because that is what makes
/// the path `parent || name` (see `file_handling::dir_to_db_parent`).
fn add(&mut self, name: &str, dir: &str, mtime: u64, text: Option<&str>) -> i64 {
let path = format!("{}/{}", dir, name);
let parent = format!("{}/", dir);
let tx = self.conn.transaction().unwrap();
let id = insert_file(
&tx,
&NewFile {
name,
path: &path,
parent: dir,
parent: &parent,
size: 42,
mtime,
mime: Some("text/plain"),
@ -561,6 +564,117 @@ fn limit_truncates_and_flags() {
std::fs::remove_file(&p).ok();
}
/// Exactly `limit` results is a complete answer, not a cut one.
///
/// `remaining()` reaches zero at the pass boundary either way, so the outer
/// loop used to set `limited` there on its way out — surfacing as the CLI's
/// "(truncated at N results; raise with --limit)" and the GUI's equivalent
/// over a result set that had dropped nothing. `flush_pass` is the authority:
/// it sets the flag when it actually truncates.
///
/// Three under a limit of three is the case that isolates it: the first hit
/// flushes immediately, leaving room for two, and the pass ends with exactly
/// those two buffered — so nothing truncates, nothing breaks mid-scan, and
/// only the between-pass test could have raised the flag. See
/// [`filling_the_limit_mid_pass_still_reports_truncated`] for the boundary
/// this does not reach.
#[test]
fn exactly_the_limit_is_not_reported_as_truncated() {
let p = tmp_db("limit-exact");
let mut s = Seeder::new(&p, true);
for i in 0..3 {
s.add(&format!("match-{:02}.txt", i), "/d", 1, None);
}
let conn = s.done();
let options = SearchOptions {
limit: 3,
..SearchOptions::default()
};
let (hits, outcome) = run_collect(&conn, "match", &options);
assert_eq!(hits.len(), 3);
assert_eq!(outcome.total, 3);
assert!(
!outcome.limited,
"3 matches under a limit of 3 dropped nothing"
);
drop(conn);
std::fs::remove_file(&p).ok();
}
/// The boundary the fix above deliberately does *not* cover, pinned so it is
/// a known shape rather than a surprise.
///
/// A limit of one is filled by the first hit of the first pass — and the first
/// batch of a pass flushes the moment there is anything to send, so `total`
/// reaches the limit while the scan is still running. The scan then breaks
/// with rows unexamined, and `cut_short` reports that honestly as "there may
/// be more", which here there was not. Distinguishing the two costs a row
/// scanned past the limit in every pass that fills it — a document decompress,
/// fold and snippet on the full-text passes — which is not worth paying on
/// every keystroke to correct a message.
#[test]
fn filling_the_limit_mid_pass_still_reports_truncated() {
let p = tmp_db("limit-one");
let mut s = Seeder::new(&p, true);
s.add("match.txt", "/d", 1, None);
let conn = s.done();
let options = SearchOptions {
limit: 1,
..SearchOptions::default()
};
let (hits, outcome) = run_collect(&conn, "match", &options);
assert_eq!(hits.len(), 1);
assert!(
outcome.limited,
"a scan that stopped with rows unexamined says so"
);
drop(conn);
std::fs::remove_file(&p).ok();
}
/// Stopping at the limit must not cost a result that belongs in it.
///
/// The scan breaks out as soon as the display limit is full, which is what
/// keeps a common term from decompressing its whole candidate set. The break
/// sits after classification for a reason: a *scanned* row is not a match —
/// the filename pass's SQL is a superset feeding two rank tiers — so breaking
/// on one would end the pass early over a row that was never going to be
/// shown, and the hits below it would be lost.
#[test]
fn stopping_at_the_limit_keeps_the_best_ranked_hits() {
let p = tmp_db("limit-break");
let mut s = Seeder::new(&p, true);
// Exact-name matches rank above substring matches, and are seeded last so
// that a pass which stopped too early would miss them.
for i in 0..20 {
s.add(&format!("zz-match-{:02}.txt", i), "/d", 1, None);
}
s.add("match", "/d", 1, None);
let conn = s.done();
let options = SearchOptions {
limit: 2,
..SearchOptions::default()
};
let (hits, outcome) = run_collect(&conn, "match", &options);
assert_eq!(hits.len(), 2);
assert!(
outcome.limited,
"21 matches under a limit of 2 is a cut set"
);
assert_eq!(
hits[0].name, "match",
"the exact match must survive the break"
);
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn session_ignores_hide_hits_before_the_cap() {
let p = tmp_db("ignores");
@ -796,6 +910,100 @@ fn wildcard_with_short_segments_falls_back_to_a_full_scan() {
std::fs::remove_file(&p).ok();
}
/// The `LIKE` prefilter a straddling wildcard now gets must be a *superset* of
/// what the classifier accepts, or real hits vanish silently — the failure mode
/// with no symptom. Rather than assert a hand-written expected set, this seeds
/// rows that exercise every way a segment can sit across `parent || name` and
/// checks the prefiltered pass against the unfiltered one.
///
/// The unfiltered side is obtained by asking for a pattern whose segments all
/// contain a separator, which is the arm that still scans — so both sides run
/// through the real cascade and no test-only code path is involved.
#[test]
fn a_wildcard_prefilter_never_loses_a_hit_the_full_scan_finds() {
let p = tmp_db("wildprefilter");
let mut s = Seeder::new(&p, true);
// A segment wholly inside the name.
let in_name = s.add("report-q3.txt", "/data", 1, None);
// A segment wholly inside the parent.
let in_parent = s.add("a.bin", "/reports/q3", 2, None);
// The pattern's `%` spanning the parent/name boundary: `rep` is in the
// directory, `q3` in the file name.
let across = s.add("q3.txt", "/reports", 3, None);
// A name that only matches once case is folded.
let folded = s.add("REPORT-Q3.TXT", "/upper", 4, None);
// A near miss that must not be admitted by either path.
let _miss = s.add("summary.txt", "/data", 5, None);
// The trap the separator-free rule exists for: the only way to read this
// row as a match spans the boundary, and the spanning text contains the
// separator itself.
let boundary = s.add("q3.txt", "/x/rep", 6, None);
// The row that makes the separator-free rule load-bearing rather than
// decorative. Against `e/pq*txt` the longest segment is `e/pq`, which
// occurs in `parent || name` **only across the join** — `/a/re/` does not
// contain it and `pq.txt` does not either. Anchoring on it would drop this
// row, so the rule has to reject it and fall to `txt`.
let straddling = s.add("pq.txt", "/a/re", 7, None);
let conn = s.done();
let opts = SearchOptions::default();
for query in ["rep*q3", "rep*rt", "*report*", "re*or*q3", "e/pq*txt"] {
let (hits, _) = run_collect(&conn, query, &opts);
let mut got: Vec<i64> = hits.iter().map(|h| h.file_id).collect();
got.sort();
// The same query with every segment forced to carry a separator would
// change what matches, so the reference set is computed directly: a row
// is expected exactly when the compiled pattern matches its name or its
// full path, which is what the classifier tests.
let split = split_for_cascade(query).unwrap();
let mut want: Vec<i64> = [
(in_name, "/data/report-q3.txt"),
(in_parent, "/reports/q3/a.bin"),
(across, "/reports/q3.txt"),
(folded, "/upper/REPORT-Q3.TXT"),
(_miss, "/data/summary.txt"),
(boundary, "/x/rep/q3.txt"),
(straddling, "/a/re/pq.txt"),
]
.iter()
.filter(|(_, path)| {
let name = path.rsplit('/').next().unwrap();
split.pattern.find_first(name, true).is_some()
|| split.pattern.find_first(path, true).is_some()
})
.map(|(id, _)| *id)
.collect();
want.sort();
assert_eq!(got, want, "query {:?}", query);
}
drop(conn);
std::fs::remove_file(&p).ok();
}
/// A pattern every segment of which carries a separator has nothing to anchor
/// on, so the pass falls back to scanning — and must still find its hits.
#[test]
fn a_wildcard_with_only_separator_segments_still_scans_and_matches() {
let p = tmp_db("wildnoanchor");
let mut s = Seeder::new(&p, true);
let hit = s.add("q3.txt", "/a/reports", 1, None);
let _miss = s.add("q3.txt", "/a/summaries", 2, None);
let conn = s.done();
// Segments are "a/rep" and "rts/" — both contain a separator.
let (hits, _) = run_collect(&conn, "a/rep*rts/", &SearchOptions::default());
assert_eq!(
hits.iter().map(|h| h.file_id).collect::<Vec<_>>(),
vec![hit]
);
drop(conn);
std::fs::remove_file(&p).ok();
}
#[test]
fn wildcard_path_tier_and_filters() {
let p = tmp_db("wildpath");
@ -1318,12 +1526,60 @@ fn the_connection_is_released_once_searching_stops() {
std::fs::remove_file(&p).ok();
}
/// An explicit release must actually leave the file closed — including when a
/// search was queued a moment before it.
///
/// The worker coalesces its request queue, so a release found behind a search
/// is handled where it sits and the search then runs. Without a cancel it
/// would run *after* the acknowledgement, reopening the index while the caller
/// — `Backend::rebuild_index` — is on its way to delete it. `release_connection`
/// therefore bumps the generation first, which is what this asserts: the
/// request made before the release is stale afterwards, so the worker drops it
/// instead of serving it.
///
/// The generation is checked rather than the descriptor because the race is
/// won or lost in microseconds; an fd assertion here would pass with the bug
/// in place most of the time and fail the rest, which is worse than no test.
#[test]
fn a_release_supersedes_a_search_queued_before_it() {
let p = tmp_db("relcancel");
let mut s = Seeder::new(&p, true);
s.add("held.txt", "/d", 1, Some("shared body"));
drop(s.done());
let (service, updates) = SearchService::new_with_idle_release(
p.clone(),
Arc::new(|| {}),
std::time::Duration::from_secs(60),
);
let queued = service.search("shared", SearchOptions::default());
service.release_connection();
let after = service.search("shared", SearchOptions::default());
assert!(
after > queued + 1,
"the release must advance the generation past {}, got {}",
queued,
after
);
// The service still works: cancelling is how the release is made safe, not
// a way of shutting the worker down.
assert_eq!(
search_names(&service, &updates, "shared").unwrap(),
vec!["held.txt"]
);
service.shutdown();
std::fs::remove_file(&p).ok();
}
/// The point of the whole change: a pass hands hits over *while* it scans, so
/// the UI has something to show long before the scan ends.
///
/// Proven by ordering rather than by batch count — `flush_pass` has always
/// chunked its output, so counting sink calls proves nothing. Pass A scans in
/// `files.path` order, so seeding a *worse* match at an early path and a
/// stored `(parent, name)` order, so seeding a *worse* match early and a
/// *better* one at a late path separates the two designs: emitting at the end
/// sorts them and leads with rank 1, while streaming hands over the rank-3 hit
/// before the scan has even reached the rank-1 one.
@ -1568,3 +1824,87 @@ fn a_fuzzy_mark_is_not_truncated_to_a_leading_part_of_the_term() {
&snip.window[a..b]
);
}
/// A path-tier match that straddles the directory/name boundary must still be
/// found.
///
/// `files` stores `parent` and `name` and no concatenation of the two, so the
/// filename pass cannot prefilter with a single `path LIKE`. For a one-piece
/// term it uses `name LIKE ? OR parent LIKE ?`, which is exactly equivalent —
/// a term with no separator in it cannot span the boundary. A multi-segment
/// wildcard *can*: `doc*q3` matches `/x/docs/q3.txt` with `doc` in the parent
/// and `q3` in the name, and neither half of that OR would fire. Those fall
/// back to scanning and letting the classifier decide, and this is the fixture
/// that fails if that fallback is ever "optimised" away.
#[test]
fn a_wildcard_spanning_the_directory_boundary_is_still_found() {
let p = tmp_db("pathstraddle");
let mut s = Seeder::new(&p, true);
let straddling = s.add("q3.txt", "/x/docs", 1, None);
// Same two pieces, both inside the name: found either way, so it proves
// the query ran rather than that the fallback was reached.
let in_name = s.add("doc-q3.txt", "/other", 2, None);
let _miss = s.add("q3.txt", "/x/plans", 3, None);
let conn = s.done();
let (hits, _) = run_collect(&conn, "doc*q3", &SearchOptions::default());
let mut ids: Vec<i64> = hits.iter().map(|h| h.file_id).collect();
ids.sort();
let mut want = vec![straddling, in_name];
want.sort();
assert_eq!(ids, want, "the straddling path match must not be dropped");
// And it is a path-tier hit carrying the whole path as its snippet.
let hit = hits.iter().find(|h| h.file_id == straddling).unwrap();
assert_eq!(hit.stage, 9);
assert_eq!(hit.path, "/x/docs/q3.txt", "parent and name rejoined");
drop(conn);
std::fs::remove_file(&p).ok();
}
/// The one-piece prefilter's other half: a term that is a whole directory
/// component matches through `parent LIKE`, with nothing in the name.
#[test]
fn a_term_matching_only_the_directory_is_found_through_the_parent() {
let p = tmp_db("pathparent");
let mut s = Seeder::new(&p, true);
let dir_hit = s.add("z.bin", "/srv/invoices", 1, None);
let _miss = s.add("z.bin", "/srv/other", 2, None);
let conn = s.done();
let (hits, _) = run_collect(&conn, "invoices", &SearchOptions::default());
assert_eq!(hits.len(), 1);
assert_eq!(hits[0].file_id, dir_hit);
assert_eq!(hits[0].stage, 9);
drop(conn);
std::fs::remove_file(&p).ok();
}
/// `folder:` must reach the folder's own files, not just its subdirectories'.
///
/// The filter is a single `parent LIKE 'dir/%'`, which covers both only
/// because every stored parent ends in a separator — the folder's own files
/// have parent `dir/`, and `%` matches nothing. It used to need a second
/// `parent = ?` term to catch them.
#[test]
fn the_folder_filter_covers_the_folder_itself_and_its_subtree() {
let p = tmp_db("folderself");
let mut s = Seeder::new(&p, true);
let own = s.add("top.txt", "/srv/data", 1, None);
let nested = s.add("deep.txt", "/srv/data/2024", 2, None);
let _sibling = s.add("other.txt", "/srv/data-archive", 3, None);
let _outside = s.add("far.txt", "/srv", 4, None);
let conn = s.done();
let (hits, _) = run_collect(&conn, "txt folder:/srv/data", &SearchOptions::default());
let mut ids: Vec<i64> = hits.iter().map(|h| h.file_id).collect();
ids.sort();
let mut want = vec![own, nested];
want.sort();
assert_eq!(ids, want, "the folder's own files count as inside it");
drop(conn);
std::fs::remove_file(&p).ok();
}

188
crates/quicksearch-core/tests/common/mod.rs
View file

@ -21,6 +21,194 @@ pub fn scratch_db(tag: &str) -> std::path::PathBuf {
scratch_dir(tag).join("index.sqlite")
}
/// Deterministic pseudo-random word picker — the same LCG and constants
/// `benches/corpus`, `examples/indexprobe` and `tests/search_perf` use, for the
/// same reason: a fixed seed is what makes two runs comparable, so a number
/// that moved is a real change rather than a different corpus.
pub struct Lcg(pub u64);
impl Lcg {
pub fn new(seed: u64) -> Lcg {
Lcg(seed)
}
pub fn next(&mut self) -> u64 {
self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1);
self.0 >> 33
}
}
/// The rare term a seeded index is searched for.
///
/// Nine bytes — long enough to clear the trigram floor, and exactly
/// `3 × (2 + 1)`, so it sits on the boundary where a `fuzzy_max_edits = 2`
/// pigeonhole split into three-character chunks becomes legal.
pub const NEEDLE: &str = "quartzite";
/// A term planted only in document *bodies*, never in a file name.
///
/// The needle above reaches the index through both, so a query for it is
/// answered mostly by the filename pass. This one forces the full-text pass to
/// do real work: the filename `LIKE` finds nothing, and every candidate the
/// trigram index returns has to be decompressed and verified.
pub const BODY_TERM: &str = "chalcedony";
/// Filler vocabulary for seeded indexes.
///
/// **Deliberately shares no trigram with [`NEEDLE`]** — no filler word contains
/// so much as `qua`. That is what makes a needle query genuinely rare, and it
/// matters more than it looks: with a vocabulary that merely *resembled* the
/// needle, every query would fill the display limit within the first few
/// hundred rows, the cascade would break out of pass A, and passes B, C and D
/// would never run at all. A harness built that way reports the same figure for
/// a literal and a fuzzy search and looks perfectly healthy doing it.
pub const WORDS: &[&str] = &[
"alpha",
"beta",
"gamma",
"delta",
"epsilon",
"zeta",
"eta",
"theta",
"iota",
"kappa",
"lambda",
"brown",
"fox",
"jumps",
"lazy",
"index",
"search",
"cascade",
"snippet",
"document",
"content",
"extract",
"summary",
"meeting",
"invoice",
"contract",
"budget",
"revenue",
"planning",
"review",
"draft",
"final",
"notes",
"appendix",
"figure",
];
/// What [`seed_index`] should build.
pub struct SeedSpec {
pub files: usize,
/// One file in every `content_every` gets extracted text. A tenth is the
/// real shape — most files in a tree are not text — and it keeps the FTS
/// index smaller than the table, as it is in practice.
pub content_every: usize,
/// Words in each stored document body.
pub body_words: usize,
/// Directories to spread the rows across, so `files.parent` has real
/// variety and `idx_files_parent` has interior levels.
pub dirs: usize,
/// File names carrying [`NEEDLE`]. Kept far below any sane display limit so
/// a needle query never fills it — an early-exiting query measures how fast
/// the cascade gives up, not how fast it scans.
pub needle_names: usize,
/// Document bodies carrying [`NEEDLE`], on top of the names.
pub needle_docs: usize,
/// Document bodies carrying [`BODY_TERM`]. Sized by the caller to stay
/// under the display limit, or the pass stops early and measures the
/// give-up rather than the verification.
pub body_term_docs: usize,
}
impl Default for SeedSpec {
fn default() -> SeedSpec {
SeedSpec {
files: 50_000,
content_every: 10,
// ~2 KB of text per document. Not arbitrary: the fuzzy full-text
// pass decompresses and scans every stored body, so a corpus of
// 400-byte documents makes that pass look free when in production
// it is the most expensive thing the cascade does. Still far under
// `maximum_text_size` (256 KiB), which is the real worst case.
body_words: 300,
dirs: 500,
needle_names: 50,
needle_docs: 50,
body_term_docs: 500,
}
}
}
/// Seed an index with synthetic rows, in one transaction.
///
/// Shared by the measurement harnesses so they all describe the same corpus;
/// what varies between them is the size, not the shape.
pub fn seed_index(path: &Path, spec: &SeedSpec) {
use quicksearch_core::db::repo::{insert_file, set_content_done, NewFile};
use quicksearch_core::mime::FileType;
use quicksearch_core::testutil::zstd_of;
let mut conn = db::open_or_recreate(path.to_str().unwrap(), "trigram").unwrap();
let mut rng = Lcg::new(0x5eed);
// Spacing rather than a random draw, so the planted rows are spread across
// the table instead of clustering in whatever prefix the scan reaches
// first — a cluster at the front would let a pass stop early and report a
// fraction of the work a real rare query costs.
let name_stride = spec.files / spec.needle_names.max(1);
let doc_stride = spec.files / spec.needle_docs.max(1);
let body_stride = spec.files / spec.body_term_docs.max(1);
let tx = conn.transaction().unwrap();
for i in 0..spec.files {
let w1 = WORDS[(rng.next() as usize) % WORDS.len()];
let w2 = WORDS[(rng.next() as usize) % WORDS.len()];
let name = if spec.needle_names > 0 && i % name_stride.max(1) == 0 {
format!("{}-{}-{:07}.txt", w1, NEEDLE, i)
} else {
format!("{}-{}-{:07}.txt", w1, w2, i)
};
// Stored parents always end in a separator; see `dir_to_db_parent`.
let dir = format!("/seed/{:03}/", i % spec.dirs.max(1));
let id = insert_file(
&tx,
&NewFile {
name: &name,
parent: &dir,
size: 4096,
mtime: 1_700_000_000 + i as u64,
mime: Some("text/plain"),
ftype: FileType::TEXT,
hash: None,
needs_content: i % spec.content_every.max(1) == 0,
},
)
.unwrap()
.expect("unique path");
if i % spec.content_every.max(1) == 0 {
let mut body: Vec<&str> = (0..spec.body_words)
.map(|_| WORDS[(rng.next() as usize) % WORDS.len()])
.collect();
if spec.needle_docs > 0 && i % doc_stride.max(1) == 0 {
// Mid-body, so a snippet window has to be cut around it rather
// than falling out of a head-of-file window for free.
body[spec.body_words / 2] = NEEDLE;
}
if spec.body_term_docs > 0 && i % body_stride.max(1) == 0 {
// Two thirds in, so verifying it means scanning most of the
// document rather than stopping at the first few bytes.
body[spec.body_words * 2 / 3] = BODY_TERM;
}
let body = body.join(" ");
set_content_done(&tx, id, &body, zstd_of(&body).as_deref()).unwrap();
}
}
tx.commit().unwrap();
conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);").ok();
}
/// How long a single indexing run may take before the test gives up. Generous:
/// CI runs these in a container against a cold page cache.
const INDEX_TIMEOUT: Duration = Duration::from_secs(120);

130
crates/quicksearch-core/tests/corpus/audio.rs Normal file
View file

@ -0,0 +1,130 @@
//! The audio corpus files: `.mp3` and `.flac`.
//!
//! Audio is the one format family whose "text" is not a document body but a
//! handful of tag values, so its expectation is the five fields
//! `extract::audio` concatenates, in the order it concatenates them: title,
//! artist, album, genre, comment. Getting that order wrong is a real
//! regression — the stored text would churn between runs — so the ordered
//! match is doing load-bearing work here rather than just tolerating
//! boilerplate.
//!
//! `id3` and `metaflac` write the tags; `lofty` reads them. The audio data
//! underneath is hand-rolled, since neither crate synthesises any.
use std::path::Path;
use super::{BodyFn, Charset, Lcg, Sample};
pub fn write_all(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
mp3(dir, lcg, body, out);
flac(dir, lcg, body, out);
}
/// The five tag values, in `extract::audio`'s emit order.
///
/// Sentence 1 carries the needle, 2 the Latin-1 phrase and 3 the Greek, so
/// this spread also puts non-ASCII into two different tag frames — which is
/// what forces the writers off ISO-8859-1 and into a wide encoding.
fn fields(sentences: &[String]) -> [&str; 5] {
[
&sentences[0],
&sentences[1],
&sentences[2],
&sentences[3],
&sentences[4],
]
}
/// Four silent MPEG-1 Layer III frames: 128 kbps, 44.1 kHz, no padding, so
/// each is 417 bytes. Four because a probe confirms a sync word by checking
/// that the next frame begins where the first one said it would.
fn mpeg_frames() -> Vec<u8> {
let mut out = Vec::with_capacity(4 * 417);
for _ in 0..4 {
out.extend_from_slice(&[0xFF, 0xFB, 0x90, 0x00]);
out.resize(out.len() + 413, 0);
}
out
}
fn mp3(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
use id3::{TagLike, Version};
let b = body(lcg, Charset::Unicode);
let [title, artist, album, genre, comment] = fields(&b.sentences);
let path = super::write_file(dir, "track.mp3", &mpeg_frames());
let mut tag = id3::Tag::new();
tag.set_title(title);
tag.set_artist(artist);
tag.set_album(album);
tag.set_genre(genre);
tag.add_frame(id3::frame::Comment {
lang: "eng".to_string(),
description: String::new(),
text: comment.to_string(),
});
tag.write_to_path(&path, Version::Id3v23)
.expect("write id3 tag");
out.push(Sample {
path,
label: "mp3",
must_contain: fields(&b.sentences).iter().map(|s| s.to_string()).collect(),
needle: b.needle.clone(),
head_path: false,
});
}
/// Fifty milliseconds of silence, committed at `tests/fixtures/silence.flac`.
///
/// Unlike MPEG — whose frames are a fixed-size header plus zeroes, cheap
/// enough to hand-roll — a FLAC frame carries CRC-8 and CRC-16 over
/// bit-packed subframes, and `lofty` reads the first one to derive the
/// stream's properties. A metadata-only file is rejected outright with
/// "failed to fill whole buffer".
///
/// So the *audio* is committed and the *tags* are still written per run: the
/// fixture carries no text at all, `metaflac` puts the seeded lipsum into it,
/// and `lofty` is still reading something a different library wrote. Produced
/// once with:
///
/// ```text
/// ffmpeg -f lavfi -i anullsrc=r=44100:cl=mono -t 0.05 \
/// -sample_fmt s16 -c:a flac -compression_level 12 silence.flac
/// ```
fn silence() -> std::path::PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/silence.flac")
}
fn flac(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
let b = body(lcg, Charset::Unicode);
let [title, artist, album, genre, comment] = fields(&b.sentences);
let path = dir.join("track.flac");
std::fs::copy(silence(), &path).unwrap_or_else(|e| {
panic!(
"copy {} -> {}: {e}\n(committed fixture; see the doc comment on `silence`)",
silence().display(),
path.display()
)
});
let mut tag = metaflac::Tag::read_from_path(&path).expect("read flac metadata");
let comments = tag.vorbis_comments_mut();
comments.set_title(vec![title]);
comments.set_artist(vec![artist]);
comments.set_album(vec![album]);
comments.set_genre(vec![genre]);
comments.set("COMMENT", vec![comment]);
tag.save().expect("write flac tags");
out.push(Sample {
path,
label: "flac",
must_contain: fields(&b.sentences).iter().map(|s| s.to_string()).collect(),
needle: b.needle.clone(),
head_path: false,
});
}

142
crates/quicksearch-core/tests/corpus/legacy.rs Normal file
View file

@ -0,0 +1,142 @@
//! The committed `.doc` / `.xls` / `.ppt` fixtures.
//!
//! These three are the corpus's one departure from generate-on-the-fly, and
//! the reason is narrow: `cfb` is the only Rust crate that writes OLE2
//! compound files, and `cfb` is what `extract::ole` reads them with. A fixture
//! built by the reader's own library proves only that the two agree with each
//! other. So LibreOffice writes them, once, and the output is committed —
//! see `tests/fixtures/legacy/regen.sh`.
//!
//! Two consequences worth being explicit about:
//!
//! * **Their text is fixed, not seeded.** `QUICKSEARCH_CORPUS_SEED` shakes the
//! generated half of the corpus and leaves these alone.
//! * **The expectations are read out of the committed sources**, not written
//! out here a second time. A regenerated fixture that lost a line therefore
//! fails, instead of quietly redefining what it was supposed to contain.
//!
//! What the fixtures prove that a synthetic file cannot: LibreOffice's `.doc`
//! is a real FIB with a real piece table, its `.xls` a real BIFF stream with a
//! real shared-string table, and its `.ppt` drags the master slide's
//! placeholder prompts ("Click to edit the title text format", `___PPT10`)
//! into the text alongside the content. That last one is exactly why
//! [`super::match_in_order`] asserts containment rather than equality.
use std::path::{Path, PathBuf};
use super::Sample;
/// The directory the fixtures live in, resolved against the crate root so the
/// test does not care what the working directory is.
pub fn dir() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/legacy")
}
fn read(name: &str) -> String {
let path = dir().join(name);
std::fs::read_to_string(&path).unwrap_or_else(|e| {
panic!(
"read {}: {e}\n(fixtures are committed; regenerate with regen.sh)",
path.display()
)
})
}
/// Non-empty, trimmed lines — the expectation for the line-per-record sources.
fn lines(source: &str) -> Vec<String> {
source
.lines()
.map(str::trim)
.filter(|l| !l.is_empty())
.map(str::to_string)
.collect()
}
/// The text of every `<text:p>` in the flat-ODF deck, in document order.
///
/// A three-line hand parse rather than `quick-xml`, which is the reader's
/// library: the source is committed, its shape is fixed, and reaching for the
/// parser under test to compute the expectation would defeat the point.
fn paragraphs(source: &str) -> Vec<String> {
let mut out = Vec::new();
let mut rest = source;
while let Some(start) = rest.find("<text:p>") {
rest = &rest[start + "<text:p>".len()..];
let Some(end) = rest.find("</text:p>") else {
break;
};
out.push(rest[..end].to_string());
rest = &rest[end..];
}
out
}
/// The needle planted in each source. Fixed rather than derived, and well
/// clear of the generated corpus's `chalcedony0000`-`chalcedony00NN` range —
/// under the trigram tokenizer "distinct" means "not a substring of another".
const NEEDLES: [(&str, &str); 3] = [
("sample.doc", "chalcedony9001"),
("sample.xls", "chalcedony9002"),
("sample.ppt", "chalcedony9003"),
];
/// The three fixtures as corpus samples.
pub fn samples() -> Vec<Sample> {
let expectations = [
("sample.doc", "doc", lines(&read("prose.txt"))),
("sample.xls", "xls", lines(&read("sheet.csv"))),
("sample.ppt", "ppt", paragraphs(&read("deck.fodp"))),
];
expectations
.into_iter()
.map(|(file, label, must_contain)| {
let needle = NEEDLES
.iter()
.find(|(f, _)| *f == file)
.expect("every fixture has a needle")
.1;
assert!(
must_contain.iter().any(|f| f.contains(needle)),
"{file}: source no longer carries {needle}; \
the end-to-end search would not be attributable"
);
Sample {
path: dir().join(file),
label,
must_contain,
needle: needle.to_string(),
// OLE2 reads a directory that can sit anywhere in the file, so
// this format never takes the walk-time buffer path.
head_path: false,
}
})
.collect()
}
/// Copy the fixtures into `dir` so the end-to-end run indexes them alongside
/// the generated corpus. Returns the samples with their paths rewritten to the
/// copies.
///
/// Copied rather than indexed in place: the indexer walks a directory tree,
/// and pointing it at the repository would pull in whatever else lives there.
pub fn copy_into(dir: &Path) -> Vec<Sample> {
samples()
.into_iter()
.map(|sample| {
let name = sample.path.file_name().expect("fixture has a name");
let target = dir.join(name);
std::fs::copy(&sample.path, &target).unwrap_or_else(|e| {
panic!(
"copy {} -> {}: {e}",
sample.path.display(),
target.display()
)
});
Sample {
path: target,
..sample
}
})
.collect()
}

338
crates/quicksearch-core/tests/corpus/mod.rs Normal file
View file

@ -0,0 +1,338 @@
//! A lipsum corpus in every format QuickSearch claims to extract text from.
//!
//! # Why this exists
//!
//! The per-extractor unit tests in `src/extract/` build their fixtures with
//! the same libraries that read them back — `zip` 0.6 for the OOXML/ODF
//! containers, `cfb` for OLE2, `lopdf` (through `pdf-extract`) for PDF. That
//! is the right choice there, because those tests aim at *malformed* input and
//! need to control every byte. But it means a writer and a reader that share a
//! wrong assumption agree with each other and the test passes.
//!
//! This module is the other half: well-formed documents from *foreign*
//! producers. Every writer here is a different implementation from the reader
//! it feeds —
//!
//! | format | written by | read by |
//! |---|---|---|
//! | docx | `docx-rs` (+ `zip` 8.x) | `zip` 0.6 + `quick-xml` |
//! | xlsx | `rust_xlsxwriter` | `zip` 0.6 + `quick-xml` |
//! | pptx, odt, ods, odp | [`zipwriter`] + `format!` | `zip` 0.6 + `quick-xml` |
//! | pdf | `pdf-writer` (typst) | `pdf-extract`/`lopdf` |
//! | mp3 | `id3` over hand-rolled MPEG frames | `lofty` |
//! | flac | `metaflac` over a committed silent stream | `lofty` |
//! | rtf | hand-written control words | `rtf-parser` |
//! | plain text | `std`, plus a hand-rolled cp1252 encoder | `encoding_rs` |
//! | doc, xls, ppt | LibreOffice, committed — see [`legacy`] | `cfb` |
//!
//! Two places take committed bytes, for two different reasons, and in both the
//! *text* still comes from the generator:
//!
//! * The three legacy binaries are what nothing in Rust can fix: `cfb` is the
//! only crate that writes OLE2 compound files, and it is the reader. Those
//! are LibreOffice's output, and they are also the one part of the corpus
//! whose text is fixed rather than seeded — see
//! `tests/fixtures/legacy/README.md`.
//! * The `.flac` borrows fifty milliseconds of committed silence because
//! `lofty` reads a real audio frame to derive stream properties. The fixture
//! carries no text; `metaflac` writes the seeded lipsum into a copy of it.
//! See [`audio`].
//!
//! # Determinism
//!
//! Everything on-the-fly is generated from one LCG seeded by [`seed`], which
//! is [`DEFAULT_SEED`] unless `QUICKSEARCH_CORPUS_SEED` says otherwise. A
//! failing assertion prints the seed it ran with, so a red CI job reproduces
//! locally with one environment variable.
// Only `extraction_corpus.rs` compiles this, and it uses most but not all of
// it; the writers each expose a little more surface than any one test needs.
#![allow(dead_code)]
use std::path::{Path, PathBuf};
pub mod audio;
pub mod legacy;
pub mod odf;
pub mod ooxml;
pub mod pdf;
pub mod plaintext;
pub mod rtf;
pub mod zipwriter;
/// The seed used when the environment says nothing. Arbitrary; what matters
/// is that it does not change between runs.
pub const DEFAULT_SEED: u64 = 0x9E37_79B9_7F4A_7C15;
/// The active seed. Override with `QUICKSEARCH_CORPUS_SEED=<u64>` to shake the
/// corpus without editing anything.
pub fn seed() -> u64 {
match std::env::var("QUICKSEARCH_CORPUS_SEED") {
Ok(v) => v
.trim()
.parse()
.unwrap_or_else(|_| panic!("QUICKSEARCH_CORPUS_SEED must be a u64, got {v:?}")),
Err(_) => DEFAULT_SEED,
}
}
/// The same LCG `benches/corpus/mod.rs` uses. Copied rather than shared:
/// `benches/` is not reachable from `tests/`, and the bench corpus is
/// deliberately frozen so its numbers stay comparable across runs.
pub struct Lcg(u64);
impl Lcg {
pub fn new(seed: u64) -> Lcg {
Lcg(seed)
}
pub fn next(&mut self) -> u64 {
self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1);
self.0 >> 33
}
fn pick<'a, T>(&mut self, from: &'a [T]) -> &'a T {
&from[self.next() as usize % from.len()]
}
}
/// Filler vocabulary. Plain ASCII lowercase so it survives every encoding in
/// the corpus unchanged, and long enough that a sentence drawn from it is
/// effectively unique.
const WORDS: &[&str] = &[
"lorem",
"ipsum",
"dolor",
"consectetur",
"adipiscing",
"eiusmod",
"tempor",
"incididunt",
"labore",
"dolore",
"aliqua",
"veniam",
"nostrud",
"exercitation",
"ullamco",
"laboris",
"commodo",
"consequat",
"voluptate",
"cillum",
"occaecat",
"cupidatat",
"proident",
"officia",
"deserunt",
"mollit",
"laborum",
];
/// A phrase every format can carry: Latin-1 representable, so it survives
/// cp1252 and the base-14 WinAnsi font the PDF writer uses.
const LATIN1_PHRASE: &str = "café résumé naïve";
/// A phrase only formats with a Unicode text model can carry.
const UNICODE_PHRASE: &str = "Καλημέρα κόσμε";
/// What characters a format's *writer* can round-trip. Gates the non-ASCII
/// coverage so the corpus is neither lax (ASCII everywhere) nor wrong
/// (demanding Greek from a WinAnsi font).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Charset {
/// 7-bit only.
Ascii,
/// Adds [`LATIN1_PHRASE`].
Latin1,
/// Adds [`UNICODE_PHRASE`] on top.
Unicode,
}
/// How many sentences every generated document carries.
const SENTENCES: usize = 6;
/// Words per sentence.
const WORDS_PER_SENTENCE: usize = 7;
/// The lipsum planted in one file, plus the token that identifies it.
///
/// Sentences are the unit of assertion for every format: prose formats write
/// one per paragraph, spreadsheets one per cell, decks one per shape. Keeping
/// the granularity identical everywhere is what lets a single `must_contain`
/// list describe them all.
pub struct Body {
pub sentences: Vec<String>,
/// Unique to this file, planted in sentence 1 and never in the file name,
/// so an end-to-end search hit is attributable to the body text.
pub needle: String,
}
/// What every needle starts with.
///
/// The same word as `common::BODY_TERM`, and for the same reason that constant
/// exists: a term that reaches the index only through a document body and
/// never through a file name, so a hit for it is attributable to extraction.
/// This module stays self-contained rather than naming `common` — it has no
/// other reason to depend on the shared harness — so
/// `corpus_needles_use_the_shared_body_term` in `extraction_corpus.rs` is what
/// keeps the two from drifting.
pub const NEEDLE_PREFIX: &str = "chalcedony";
impl Body {
/// Build a body for `index`-th file, carrying whatever `charset` allows.
///
/// The needle is derived from the index alone, not from the LCG: it has to
/// stay distinct from every other file's under any seed, and under the
/// trigram tokenizer "distinct" means "not a substring of another".
pub fn new(lcg: &mut Lcg, index: usize, charset: Charset) -> Body {
let needle = format!("{NEEDLE_PREFIX}{index:04}");
let mut sentences = Vec::with_capacity(SENTENCES);
for i in 0..SENTENCES {
let mut words: Vec<String> = (0..WORDS_PER_SENTENCE)
.map(|_| lcg.pick(WORDS).to_string())
.collect();
// One planted item per sentence, at a fixed position so a
// reordering bug shows up as a failed match rather than a pass.
match i {
1 => words.insert(0, needle.clone()),
2 if charset != Charset::Ascii => words.insert(3, LATIN1_PHRASE.to_string()),
3 if charset == Charset::Unicode => words.insert(3, UNICODE_PHRASE.to_string()),
_ => {}
}
sentences.push(words.join(" "));
}
Body { sentences, needle }
}
}
/// One corpus file plus what its extracted text must contain.
pub struct Sample {
pub path: PathBuf,
/// A label for assertion messages — the format, not the file name.
pub label: &'static str,
/// Fragments that must appear in the extracted text, in this order, with
/// anything permitted between them.
///
/// Ordered containment, not equality, and not set membership. Equality is
/// unusable: LibreOffice's `.ppt` filter drags master-slide boilerplate
/// ("Click to edit the title text format", "___PPT10") into the text
/// stream, and every spreadsheet reader puts its cell separators
/// somewhere slightly different. Set membership is too weak: text
/// assembled out of order is exactly what a mis-read Word piece table
/// produces, and that has to fail.
pub must_contain: Vec<String>,
/// Planted in the body and absent from the file name.
pub needle: String,
/// Whether this format implements `Extractor::extract_from_head` — i.e.
/// whether the walk may extract it without reopening the file. Only
/// plaintext and RTF do.
pub head_path: bool,
}
impl Sample {
/// The prose case, where the sentences *are* the expectation.
fn prose(path: PathBuf, label: &'static str, body: &Body, head_path: bool) -> Sample {
Sample {
path,
label,
must_contain: body.sentences.clone(),
needle: body.needle.clone(),
head_path,
}
}
}
/// Where `fragments` stop matching `text` as an ordered subsequence.
///
/// `Ok(())` when every fragment is found in turn. `Err` names the first one
/// that is not, which is the only diagnostic worth printing: "the text does
/// not contain X" plus where the scan had got to.
pub fn match_in_order(text: &str, fragments: &[String]) -> Result<(), String> {
let mut cursor = 0usize;
for (i, fragment) in fragments.iter().enumerate() {
match text[cursor..].find(fragment.as_str()) {
Some(at) => cursor += at + fragment.len(),
None => {
let seen = &text[..cursor.min(text.len())];
let rest = &text[cursor.min(text.len())..];
return Err(format!(
"fragment {i} not found after byte {cursor}\n \
wanted: {fragment:?}\n \
matched so far (tail): {:?}\n \
remaining text (head): {:?}",
tail(seen, 120),
head(rest, 400),
));
}
}
}
Ok(())
}
/// First `n` characters of `s`, for an error message.
fn head(s: &str, n: usize) -> String {
s.chars().take(n).collect()
}
/// Last `n` characters of `s`, for an error message.
fn tail(s: &str, n: usize) -> String {
let count = s.chars().count();
s.chars().skip(count.saturating_sub(n)).collect()
}
/// Build the whole corpus into one directory and return it with its samples.
///
/// The directory is a `testutil::scratch_dir`, so it survives a failing run
/// for inspection and is swept on a later day like every other test's tree.
pub fn build(tag: &str) -> (PathBuf, Vec<Sample>) {
let dir = quicksearch_core::testutil::scratch_dir(tag);
let mut lcg = Lcg::new(seed());
let mut samples = Vec::new();
let mut next = 0usize;
let mut body = |lcg: &mut Lcg, charset: Charset| {
let b = Body::new(lcg, next, charset);
next += 1;
b
};
plaintext::write_all(&dir, &mut lcg, &mut body, &mut samples);
rtf::write_all(&dir, &mut lcg, &mut body, &mut samples);
ooxml::write_all(&dir, &mut lcg, &mut body, &mut samples);
odf::write_all(&dir, &mut lcg, &mut body, &mut samples);
pdf::write_all(&dir, &mut lcg, &mut body, &mut samples);
audio::write_all(&dir, &mut lcg, &mut body, &mut samples);
// The committed OLE2 fixtures, copied in so the whole corpus is one tree.
samples.extend(legacy::copy_into(&dir));
// Every needle has to identify exactly one file, or the end-to-end search
// proves nothing. Under the trigram tokenizer that means no needle may be
// a substring of another, which a bad `Body::new` index would produce
// silently.
for (i, a) in samples.iter().enumerate() {
for b in samples.iter().skip(i + 1) {
assert!(
!a.needle.contains(&b.needle) && !b.needle.contains(&a.needle),
"needles {:?} ({}) and {:?} ({}) are not distinguishable",
a.needle,
a.label,
b.needle,
b.label
);
}
}
(dir, samples)
}
/// The signature the per-format writers take for "give me a fresh body".
/// A closure rather than a method so the file index keeps counting across
/// modules and every needle in the corpus stays unique.
pub type BodyFn<'a> = dyn FnMut(&mut Lcg, Charset) -> Body + 'a;
/// Write `bytes` to `dir/name` and return the path.
pub fn write_file(dir: &Path, name: &str, bytes: &[u8]) -> PathBuf {
let path = dir.join(name);
std::fs::write(&path, bytes).unwrap_or_else(|e| panic!("write {}: {e}", path.display()));
path
}

150
crates/quicksearch-core/tests/corpus/odf.rs Normal file
View file

@ -0,0 +1,150 @@
//! The three OpenDocument corpus files: `.odt`, `.ods`, `.odp`.
//!
//! No Rust crate writes ODF, so these are assembled from
//! [`super::zipwriter`] and `format!` — independent of the `zip` 0.6 and
//! `quick-xml` the reader uses.
//!
//! The `mimetype` member comes first and uncompressed, as the ODF packaging
//! spec requires and as LibreOffice writes it. `extract::office` locates
//! `content.xml` by name and never looks at it, but a package that violates
//! the spec is not the thing the corpus is supposed to be testing against.
use std::path::Path;
use super::zipwriter::{self, xml_escape, Entry};
use super::{BodyFn, Charset, Lcg, Sample};
/// Namespace declarations shared by all three documents. Only `office` and
/// `text` are load-bearing for extraction; `table` is needed for the
/// spreadsheet's grid.
const NS: &str = "xmlns:office=\"urn:oasis:names:tc:opendocument:xmlns:office:1.0\" \
xmlns:text=\"urn:oasis:names:tc:opendocument:xmlns:text:1.0\" \
xmlns:table=\"urn:oasis:names:tc:opendocument:xmlns:table:1.0\" \
xmlns:draw=\"urn:oasis:names:tc:opendocument:xmlns:drawing:1.0\" \
office:version=\"1.3\"";
pub fn write_all(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
odt(dir, lcg, body, out);
ods(dir, lcg, body, out);
odp(dir, lcg, body, out);
}
/// Package `content.xml` as an ODF container with the right `mimetype`.
fn package(dir: &Path, name: &str, mime: &str, content: &str) -> std::path::PathBuf {
let entries = [
Entry {
name: "mimetype",
body: mime.as_bytes(),
},
Entry {
name: "content.xml",
body: content.as_bytes(),
},
];
super::write_file(dir, name, &zipwriter::archive(&entries))
}
/// Prose, alternating `<text:h>` and `<text:p>` and wrapping one sentence in a
/// `<text:span>`.
///
/// The span is the interesting case: `ODF_TEXT` lists `text:span` as
/// text-bearing but *not* paragraph-breaking, and the reader counts open
/// text elements rather than flagging them. A span nested in a paragraph is
/// what distinguishes those two behaviours.
fn odt(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
let b = body(lcg, Charset::Unicode);
let mut xml = String::new();
for (i, sentence) in b.sentences.iter().enumerate() {
let escaped = xml_escape(sentence);
match i % 3 {
0 => xml.push_str(&format!(
"<text:h text:outline-level=\"1\">{escaped}</text:h>"
)),
1 => xml.push_str(&format!("<text:p>{escaped}</text:p>")),
_ => xml.push_str(&format!(
"<text:p><text:span>{escaped}</text:span></text:p>"
)),
}
}
let content = format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<office:document-content {NS}><office:body><office:text>{xml}\
</office:text></office:body></office:document-content>"
);
let path = package(
dir,
"prose.odt",
"application/vnd.oasis.opendocument.text",
&content,
);
out.push(Sample::prose(path, "odt", &b, false));
}
/// One sentence per cell, one cell per row — the same shape as the `.xlsx`,
/// read by `ODF_SHEET` with its `Some(' ')` cell separator.
fn ods(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
let b = body(lcg, Charset::Unicode);
let rows: String = b
.sentences
.iter()
.map(|s| {
format!(
"<table:table-row><table:table-cell office:value-type=\"string\">\
<text:p>{}</text:p></table:table-cell></table:table-row>",
xml_escape(s)
)
})
.collect();
let content = format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<office:document-content {NS}><office:body><office:spreadsheet>\
<table:table table:name=\"Corpus\">{rows}</table:table>\
</office:spreadsheet></office:body></office:document-content>"
);
let path = package(
dir,
"sheet.ods",
"application/vnd.oasis.opendocument.spreadsheet",
&content,
);
out.push(Sample::prose(path, "ods", &b, false));
}
/// Sentences across two `<draw:page>`s, in text boxes — the shape LibreOffice
/// writes and the shape `ODF_TEXT` reads, since `.odp` and `.odt` share a spec.
fn odp(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
const PAGES: usize = 2;
let b = body(lcg, Charset::Unicode);
let mut xml = String::new();
// Contiguous halves rather than round-robin: the reader concatenates
// pages in document order, so this keeps the expectation equal to
// `b.sentences` and leaves the reordering check to the `.pptx`.
let per_page = b.sentences.len().div_ceil(PAGES);
for (page, chunk) in b.sentences.chunks(per_page).enumerate() {
let frames: String = chunk
.iter()
.map(|s| {
format!(
"<draw:frame><draw:text-box><text:p>{}</text:p></draw:text-box></draw:frame>",
xml_escape(s)
)
})
.collect();
xml.push_str(&format!(
"<draw:page draw:name=\"page{}\">{frames}</draw:page>",
page + 1
));
}
let content = format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<office:document-content {NS}><office:body><office:presentation>{xml}\
</office:presentation></office:body></office:document-content>"
);
let path = package(
dir,
"deck.odp",
"application/vnd.oasis.opendocument.presentation",
&content,
);
out.push(Sample::prose(path, "odp", &b, false));
}

132
crates/quicksearch-core/tests/corpus/ooxml.rs Normal file
View file

@ -0,0 +1,132 @@
//! The three OOXML corpus files: `.docx`, `.xlsx`, `.pptx`.
//!
//! `docx-rs` and `rust_xlsxwriter` are complete, independent implementations
//! of their formats — including the container, since `docx-rs` carries `zip`
//! 8.x against the 0.6 `extract::office` reads with. There is no comparable
//! crate for PowerPoint, so the `.pptx` is assembled from [`super::zipwriter`]
//! and `format!`, which is independent of both `zip` 0.6 and `quick-xml`.
use std::path::Path;
use super::zipwriter::{self, Entry};
use super::{BodyFn, Charset, Lcg, Sample};
pub fn write_all(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
docx(dir, lcg, body, out);
xlsx(dir, lcg, body, out);
pptx(dir, lcg, body, out);
}
/// One paragraph per sentence, one run per paragraph — so each sentence lands
/// in a single `<w:t>` and reaches the reader contiguous.
fn docx(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
use docx_rs::*;
let b = body(lcg, Charset::Unicode);
let mut docx = Docx::new();
for sentence in &b.sentences {
docx = docx.add_paragraph(Paragraph::new().add_run(Run::new().add_text(sentence)));
}
let path = dir.join("prose.docx");
let file = std::fs::File::create(&path).expect("create docx");
docx.build().pack(file).expect("pack docx");
out.push(Sample::prose(path, "docx", &b, false));
}
/// One sentence per cell, one cell per row.
///
/// `rust_xlsxwriter` puts strings in a real `xl/sharedStrings.xml` table and
/// has the cells index into it, which is the path `extract_xlsx` exists for —
/// an inline-string writer would leave the shared-string reader untested.
fn xlsx(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
use rust_xlsxwriter::Workbook;
let b = body(lcg, Charset::Unicode);
let mut workbook = Workbook::new();
let sheet = workbook.add_worksheet();
for (row, sentence) in b.sentences.iter().enumerate() {
sheet
.write_string(row as u32, 0, sentence)
.expect("write cell");
}
let path = dir.join("sheet.xlsx");
workbook.save(&path).expect("save xlsx");
out.push(Sample::prose(path, "xlsx", &b, false));
}
/// Sentences dealt across three slides, so `extract_pptx`'s per-slide loop and
/// its `--- New Slide ---` marker are both exercised rather than a single
/// slide's happy path.
fn pptx(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
const SLIDES: usize = 3;
let b = body(lcg, Charset::Unicode);
let mut slides: Vec<String> = Vec::with_capacity(SLIDES);
for slide in 0..SLIDES {
let paragraphs: String = b
.sentences
.iter()
.skip(slide)
.step_by(SLIDES)
.map(|s| {
format!(
"<a:p><a:r><a:t>{}</a:t></a:r></a:p>",
zipwriter::xml_escape(s)
)
})
.collect();
slides.push(format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<p:sld xmlns:p=\"http://schemas.openxmlformats.org/presentationml/2006/main\" \
xmlns:a=\"http://schemas.openxmlformats.org/drawingml/2006/main\">\
<p:cSld><p:spTree><p:sp><p:txBody>{paragraphs}</p:txBody></p:sp></p:spTree></p:cSld>\
</p:sld>"
));
}
let names: Vec<String> = (1..=SLIDES)
.map(|i| format!("ppt/slides/slide{i}.xml"))
.collect();
let mut entries = vec![Entry {
name: "[Content_Types].xml",
body: CONTENT_TYPES.as_bytes(),
}];
for (name, xml) in names.iter().zip(&slides) {
entries.push(Entry {
name,
body: xml.as_bytes(),
});
}
let path = super::write_file(dir, "deck.pptx", &zipwriter::archive(&entries));
// Dealing round-robin means slide order, not sentence order, decides what
// the reader emits — so the expectation has to be rebuilt in the order the
// slides are read, not copied from `b.sentences`.
let mut expected = Vec::with_capacity(b.sentences.len());
for slide in 0..SLIDES {
expected.extend(b.sentences.iter().skip(slide).step_by(SLIDES).cloned());
}
out.push(Sample {
path,
label: "pptx",
must_contain: expected,
needle: b.needle.clone(),
head_path: false,
});
}
/// A `[Content_Types].xml` good enough to make the archive a real package.
/// The reader never opens it — it goes straight for `ppt/slides/slide*.xml` —
/// but a package without one is not a `.pptx`, and the corpus should not be
/// asserting against something no other tool would accept.
const CONTENT_TYPES: &str = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\
<Types xmlns=\"http://schemas.openxmlformats.org/package/2006/content-types\">\
<Default Extension=\"xml\" ContentType=\"application/xml\"/>\
<Override PartName=\"/ppt/slides/slide1.xml\" \
ContentType=\"application/vnd.openxmlformats-officedocument.presentationml.slide+xml\"/>\
<Override PartName=\"/ppt/slides/slide2.xml\" \
ContentType=\"application/vnd.openxmlformats-officedocument.presentationml.slide+xml\"/>\
<Override PartName=\"/ppt/slides/slide3.xml\" \
ContentType=\"application/vnd.openxmlformats-officedocument.presentationml.slide+xml\"/>\
</Types>";

69
crates/quicksearch-core/tests/corpus/pdf.rs Normal file
View file

@ -0,0 +1,69 @@
//! The PDF corpus file, written with `pdf-writer`.
//!
//! `pdf-writer` is typst's low-level writer. It shares no code with
//! `pdf-extract` — in particular no `lopdf`, which is what `extract::pdf`
//! parses with and what the unit tests in `src/extract/pdf.rs` build their
//! fixtures with.
//!
//! # Why the text is Latin-1
//!
//! Helvetica is one of the fourteen fonts every PDF reader ships, so no font
//! file has to be embedded and no glyph can be missing for the wrong reason.
//! Its repertoire under `WinAnsiEncoding` is cp1252 — which covers the
//! corpus's Latin-1 phrase and cannot express its Greek one. Hence
//! [`Charset::Latin1`]: demanding Greek here would be asserting against a
//! limit of the fixture rather than of the extractor.
use std::path::Path;
use pdf_writer::{Content, Finish, Name, Pdf, Rect, Ref, Str};
use super::{BodyFn, Charset, Lcg, Sample};
pub fn write_all(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
let b = body(lcg, Charset::Latin1);
let catalog_id = Ref::new(1);
let page_tree_id = Ref::new(2);
let page_id = Ref::new(3);
let font_id = Ref::new(4);
let content_id = Ref::new(5);
let font_name = Name(b"F1");
let mut pdf = Pdf::new();
pdf.catalog(catalog_id).pages(page_tree_id);
pdf.pages(page_tree_id).kids([page_id]).count(1);
let mut page = pdf.page(page_id);
page.media_box(Rect::new(0.0, 0.0, 595.0, 842.0));
page.parent(page_tree_id);
page.contents(content_id);
page.resources().fonts().pair(font_name, font_id);
page.finish();
// Without an explicit encoding the font falls back to StandardEncoding,
// whose upper half is not Latin-1 at all — `é` would come back as an
// acute accent on its own.
pdf.type1_font(font_id)
.base_font(Name(b"Helvetica"))
.encoding_predefined(Name(b"WinAnsiEncoding"));
let mut content = Content::new();
content.begin_text();
content.set_font(font_name, 12.0);
// Leading set once, then one `next_line` per sentence: each sentence is a
// single `Tj` so nothing can be interleaved into the middle of one.
content.set_leading(16.0);
content.next_line(56.0, 780.0);
for (i, sentence) in b.sentences.iter().enumerate() {
if i > 0 {
content.next_line(0.0, 0.0);
}
content.show(Str(&super::plaintext::to_cp1252(sentence)));
}
content.end_text();
pdf.stream(content_id, &content.finish());
let path = super::write_file(dir, "prose.pdf", &pdf.finish());
out.push(Sample::prose(path, "pdf", &b, false));
}

283
crates/quicksearch-core/tests/corpus/plaintext.rs Normal file
View file

@ -0,0 +1,283 @@
//! Plain-text corpus files: one per extension family, plus one per encoding.
//!
//! Written with `std` alone. The one place that needs an encoder — windows-1252
//! — gets a hand-rolled one rather than `encoding_rs`, which is what
//! `textenc::decode_text` reads it back with.
//!
//! The extension sweep is not decoration: it drives all three stages of
//! `mime::guess_mime_from_head`. `.bat` comes from the override table, most
//! come from `mime_guess`, and the extensionless `README` reaches the text
//! sniff with no other evidence at all. Several of the extensions here also
//! resolve to `EXTRA_TEXT_MIMES` entries (`.json`, `.sql`, `.svg`, `.m3u`,
//! `.eml`), which the plaintext extractor claims only because it is registered
//! ahead of the audio one.
use std::path::{Path, PathBuf};
use super::{BodyFn, Charset, Lcg, Sample};
/// Encode `s` as windows-1252, replacing anything the codepage cannot hold.
///
/// cp1252 is Latin-1 with 27 printable characters filled into the C1 range, so
/// the whole encoder is: identity below 0x100 except for that range, plus the
/// reverse of the table for it. Hand-rolled deliberately — `encoding_rs` is the
/// decoder under test.
///
/// Shared with [`super::pdf`], whose base-14 font is WinAnsi — the same
/// repertoire under a different name.
pub fn to_cp1252(s: &str) -> Vec<u8> {
/// The 0x80-0x9F block, in order. `\u{FFFD}` marks the five unassigned
/// slots, which nothing maps onto.
const C1: [char; 32] = [
'\u{20AC}', '\u{FFFD}', '\u{201A}', '\u{0192}', '\u{201E}', '\u{2026}', '\u{2020}',
'\u{2021}', '\u{02C6}', '\u{2030}', '\u{0160}', '\u{2039}', '\u{0152}', '\u{FFFD}',
'\u{017D}', '\u{FFFD}', '\u{FFFD}', '\u{2018}', '\u{2019}', '\u{201C}', '\u{201D}',
'\u{2022}', '\u{2013}', '\u{2014}', '\u{02DC}', '\u{2122}', '\u{0161}', '\u{203A}',
'\u{0153}', '\u{FFFD}', '\u{017E}', '\u{0178}',
];
let mut out = Vec::with_capacity(s.len());
for c in s.chars() {
let code = c as u32;
if code < 0x80 || (0xA0..=0xFF).contains(&code) {
out.push(code as u8);
} else if let Some(i) = C1.iter().position(|&t| t == c && t != '\u{FFFD}') {
out.push(0x80 + i as u8);
} else {
out.push(b'?');
}
}
out
}
/// Extensions that exercise a distinct route through `guess_mime_from_head`,
/// paired with a wrapper that makes the file plausible for its type.
///
/// The wrapper matters less than it looks — the plaintext extractor decodes
/// rather than parses, so nothing here is validated as JSON or XML. It is
/// there so a human opening the scratch directory sees files, not lipsum with
/// a misleading suffix.
const EXTENSIONS: &[(&str, Wrapper)] = &[
("txt", Wrapper::Raw),
("md", Wrapper::Raw),
("log", Wrapper::Raw),
("csv", Wrapper::Csv),
("html", Wrapper::Html),
("xml", Wrapper::Xml),
("svg", Wrapper::Svg),
("json", Wrapper::Json),
("yml", Wrapper::Yaml),
("sql", Wrapper::Sql),
("sh", Wrapper::Hash),
("py", Wrapper::Hash),
("rs", Wrapper::Slashes),
("ini", Wrapper::Ini),
("srt", Wrapper::Srt),
("m3u", Wrapper::M3u),
("eml", Wrapper::Eml),
// From `mime::EXTENSION_OVERRIDES`, not from `mime_guess`, which calls it
// an executable.
("bat", Wrapper::Rem),
];
#[derive(Clone, Copy)]
enum Wrapper {
Raw,
Csv,
Html,
Xml,
Svg,
Json,
Yaml,
Sql,
Hash,
Slashes,
Ini,
Srt,
M3u,
Eml,
Rem,
}
impl Wrapper {
/// Render `sentences` in this file type's clothing. Every sentence must
/// come out contiguous and unaltered — the fragments are asserted against
/// the decoded text verbatim.
fn render(self, sentences: &[String]) -> String {
let lines = |prefix: &str| {
sentences
.iter()
.map(|s| format!("{prefix}{s}"))
.collect::<Vec<_>>()
.join("\n")
};
match self {
Wrapper::Raw => sentences.join("\n"),
// One sentence per cell keeps it a single field; no sentence
// contains a comma or a quote, so no quoting is needed.
Wrapper::Csv => lines(""),
Wrapper::Html => format!(
"<!DOCTYPE html>\n<html><body>\n{}\n</body></html>",
sentences
.iter()
.map(|s| format!("<p>{s}</p>"))
.collect::<Vec<_>>()
.join("\n")
),
Wrapper::Xml => format!(
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<notes>\n{}\n</notes>",
sentences
.iter()
.map(|s| format!(" <note>{s}</note>"))
.collect::<Vec<_>>()
.join("\n")
),
Wrapper::Svg => format!(
"<svg xmlns=\"http://www.w3.org/2000/svg\" width=\"400\" height=\"200\">\n{}\n</svg>",
sentences
.iter()
.enumerate()
.map(|(i, s)| format!(" <text x=\"10\" y=\"{}\">{s}</text>", 20 + i * 20))
.collect::<Vec<_>>()
.join("\n")
),
// Not `serde_json`: the sentences contain no character JSON would
// escape, and an escape would break the verbatim fragment match.
Wrapper::Json => format!(
"{{\n \"notes\": [\n{}\n ]\n}}",
sentences
.iter()
.map(|s| format!(" \"{s}\""))
.collect::<Vec<_>>()
.join(",\n")
),
Wrapper::Yaml => format!("notes:\n{}", lines(" - ")),
Wrapper::Sql => format!(
"CREATE TABLE notes (body TEXT);\n{}",
sentences
.iter()
.map(|s| format!("INSERT INTO notes VALUES ('{s}');"))
.collect::<Vec<_>>()
.join("\n")
),
Wrapper::Hash => format!("#!/bin/sh\n{}", lines("# ")),
Wrapper::Slashes => format!("fn main() {{\n{}\n}}", lines(" // ")),
Wrapper::Ini => format!("[notes]\n{}", lines("note = ")),
Wrapper::Srt => sentences
.iter()
.enumerate()
.map(|(i, s)| {
format!(
"{}\n00:00:{:02},000 --> 00:00:{:02},000\n{s}\n",
i + 1,
i * 2,
i * 2 + 2
)
})
.collect::<Vec<_>>()
.join("\n"),
Wrapper::M3u => format!("#EXTM3U\n{}", lines("#EXTINF:-1,")),
Wrapper::Eml => format!(
"From: corpus@example.invalid\nTo: reader@example.invalid\n\
Subject: lipsum\nContent-Type: text/plain; charset=utf-8\n\n{}",
sentences.join("\n")
),
Wrapper::Rem => format!("@echo off\n{}", lines("REM ")),
}
}
}
/// Every plaintext sample: the extension sweep, the encoding sweep, the
/// extensionless sniff case, and one file past `hash_length`.
pub fn write_all(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
for (ext, wrapper) in EXTENSIONS {
let b = body(lcg, Charset::Unicode);
let text = wrapper.render(&b.sentences);
let path = super::write_file(dir, &format!("prose-{ext}.{ext}"), text.as_bytes());
out.push(Sample::prose(path, ext, &b, true));
}
// No extension at all: `mime_guess` has nothing, `infer` has nothing, and
// only `textenc::looks_like_text` can answer. UTF-8 with no BOM is the
// one class it accepts on proof rather than on evidence.
let b = body(lcg, Charset::Unicode);
let path = super::write_file(dir, "README", b.sentences.join("\n").as_bytes());
out.push(Sample::prose(path, "extensionless", &b, true));
write_encodings(dir, lcg, body, out);
write_oversized(dir, lcg, body, out);
}
/// The same prose in five encodings. All five are `.txt`, so all five reach
/// the extractor identically and any difference is `textenc`'s alone.
fn write_encodings(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
// UTF-8 with a BOM: classified by `Encoding::for_bom` before the binary
// guard ever runs.
let b = body(lcg, Charset::Unicode);
let mut bytes = vec![0xEF, 0xBB, 0xBF];
bytes.extend_from_slice(b.sentences.join("\n").as_bytes());
let path = super::write_file(dir, "encoding-utf8-bom.txt", &bytes);
out.push(Sample::prose(path, "utf-8 + BOM", &b, true));
// UTF-16, both endiannesses, BOM-marked. Full of NUL bytes, which is
// exactly why the BOM check has to precede the binary guard.
for (label, name, big_endian) in [
("utf-16le + BOM", "encoding-utf16le.txt", false),
("utf-16be + BOM", "encoding-utf16be.txt", true),
] {
let b = body(lcg, Charset::Unicode);
let text = b.sentences.join("\n");
let mut bytes = if big_endian {
vec![0xFE, 0xFF]
} else {
vec![0xFF, 0xFE]
};
for unit in text.encode_utf16() {
let pair = if big_endian {
unit.to_be_bytes()
} else {
unit.to_le_bytes()
};
bytes.extend_from_slice(&pair);
}
let path = super::write_file(dir, name, &bytes);
out.push(Sample::prose(path, label, &b, true));
}
// windows-1252: no BOM, not valid UTF-8, and decoded only because the
// `.txt` extension already established it is text. `Charset::Latin1`
// because the codepage cannot hold the Greek.
let b = body(lcg, Charset::Latin1);
let path = super::write_file(
dir,
"encoding-cp1252.txt",
&to_cp1252(&b.sentences.join("\n")),
);
out.push(Sample::prose(path, "windows-1252", &b, true));
}
/// A file comfortably past the default `hash_length` of 8 KiB.
///
/// Under that size the walk hands the whole buffer to `extract_from_head` and
/// the file is never reopened; over it, the content pass runs the sized on-disk
/// read instead. Both paths must produce the planted text, and only this
/// sample proves the second one does.
fn write_oversized(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
let b = body(lcg, Charset::Unicode);
let mut text = String::new();
// Padding first, so the planted sentences sit past the 8 KiB mark and a
// reader that silently stopped at the head would find none of them.
while text.len() < 12 * 1024 {
text.push_str("padding filler ligula quis bibendum auctor nisi elit\n");
}
text.push_str(&b.sentences.join("\n"));
let path = super::write_file(dir, "oversized.txt", text.as_bytes());
// `head_path` stays true: `extract_from_head` is only ever called with a
// *complete* buffer, so the agreement assertion passes it the whole file.
out.push(Sample::prose(path, "oversized text", &b, true));
}
/// Not part of the corpus — used by the fixture assertions to read a committed
/// source file.
pub fn read_to_string(path: &PathBuf) -> String {
std::fs::read_to_string(path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()))
}

82
crates/quicksearch-core/tests/corpus/rtf.rs Normal file
View file

@ -0,0 +1,82 @@
//! An RTF corpus file, written as control words by hand.
//!
//! Hand-written rather than produced by a crate because RTF is a text format
//! whose writer side is trivial, and because the only Rust RTF library in the
//! tree is `rtf-parser` — the reader under test.
//!
//! Both non-ASCII escape forms are exercised, since they are separate code
//! paths in the parser and a document from a real word processor contains
//! both: `\'hh` for anything the declared codepage holds, `\uN?` for the rest.
use std::path::Path;
use super::{BodyFn, Charset, Lcg, Sample};
/// Escape one character into RTF source.
///
/// Below 0x80 only the three structural characters need escaping. From 0x80 to
/// 0xFF the `\'hh` hex form matches the `\ansicpg1252` declared in the header.
/// Above that, `\uN` carries the UTF-16 code unit.
///
/// # Why the fallback is `\'3f` and not a literal `?`
///
/// A `\uN` escape is followed by an ANSI fallback for readers that cannot do
/// Unicode, and the spec lets that be any character. This writer emits
/// `\uN\'3f` — the escaped form of `?` — because that is what LibreOffice
/// writes, and checking a file a real producer wrote is the point of this
/// corpus.
///
/// The literal form, `\uN?`, is equally legal and equally common, and it used
/// to lose the escape and the rest of the word. That is fixed — see
/// `rtf_unicode_escapes_survive_extraction` in `tests/extraction_corpus.rs`,
/// which covers all three spellings and is where that shape is now pinned.
/// Keeping this writer on LibreOffice's form keeps the corpus a check on real
/// output rather than a second copy of that test.
fn escape(c: char, out: &mut String) {
let code = c as u32;
match c {
'\\' | '{' | '}' => {
out.push('\\');
out.push(c);
}
_ if code < 0x80 => out.push(c),
// cp1252 and Latin-1 agree over 0xA0-0xFF, which is the whole range
// the corpus's Latin-1 phrase uses.
_ if (0xA0..=0xFF).contains(&code) => out.push_str(&format!("\\'{code:02x}")),
_ => {
// Surrogate pairs are written as two `\uN`, which is what the
// format requires: the escape carries a UTF-16 code unit, not a
// scalar value. Units above 0x7FFF are written negative.
let mut buf = [0u16; 2];
for unit in c.encode_utf16(&mut buf) {
out.push_str(&format!("\\u{}\\'3f", *unit as i16));
}
}
}
}
/// The RTF source for `sentences`, one paragraph each.
fn document(sentences: &[String]) -> String {
let mut out = String::from("{\\rtf1\\ansi\\ansicpg1252\\deff0{\\fonttbl{\\f0 Helvetica;}}\n");
for (i, sentence) in sentences.iter().enumerate() {
// A control word swallows exactly one following space as its
// delimiter, so `\par ` puts nothing of its own into the text.
if i > 0 {
out.push_str("\\par ");
}
for c in sentence.chars() {
escape(c, &mut out);
}
out.push('\n');
}
out.push('}');
out
}
pub fn write_all(dir: &Path, lcg: &mut Lcg, body: &mut BodyFn<'_>, out: &mut Vec<Sample>) {
let b = body(lcg, Charset::Unicode);
let path = super::write_file(dir, "prose.rtf", document(&b.sentences).as_bytes());
// RTF is the other format with an `extract_from_head`: it has no trailer
// and needs no seeking, so a complete buffer parses exactly like the file.
out.push(Sample::prose(path, "rtf", &b, true));
}

99
crates/quicksearch-core/tests/corpus/zipwriter.rs Normal file
View file

@ -0,0 +1,99 @@
//! A minimal ZIP writer, stored (uncompressed) entries only.
//!
//! Exists so the pptx and ODF containers are not built by `zip` 0.6 — the same
//! crate `extract::office` reads them back with. It is about sixty lines of
//! well-specified structure (APPNOTE 4.3), which is a smaller thing to get
//! wrong than the agreement it is here to test.
//!
//! Stored rather than deflated because the reader accepts both (see the `zip`
//! entry in `Cargo.toml`) and stored needs no compressor. `crc32fast` supplies
//! the one field that cannot be hand-waved.
//!
//! No zip64, no data descriptors, no unicode path extra field: every corpus
//! member has an ASCII name and is a few kilobytes.
/// One member of the archive.
pub struct Entry<'a> {
pub name: &'a str,
pub body: &'a [u8],
}
/// Serialize `entries` into a complete `.zip`.
pub fn archive(entries: &[Entry<'_>]) -> Vec<u8> {
let mut out = Vec::new();
// (crc, size, local header offset) per entry, for the central directory.
let mut placed: Vec<(u32, usize, usize)> = Vec::with_capacity(entries.len());
for entry in entries {
let offset = out.len();
let crc = crc32fast::hash(entry.body);
out.extend_from_slice(b"PK\x03\x04");
out.extend_from_slice(&20u16.to_le_bytes()); // version needed
out.extend_from_slice(&0u16.to_le_bytes()); // flags
out.extend_from_slice(&0u16.to_le_bytes()); // method: stored
// A fixed DOS timestamp — 1980-01-01 00:00:00, the epoch of the
// format. Nothing reads it, and a real clock would make two runs of
// the same seed produce different bytes.
out.extend_from_slice(&0u16.to_le_bytes()); // time
out.extend_from_slice(&0x0021u16.to_le_bytes()); // date
out.extend_from_slice(&crc.to_le_bytes());
out.extend_from_slice(&(entry.body.len() as u32).to_le_bytes()); // compressed
out.extend_from_slice(&(entry.body.len() as u32).to_le_bytes()); // uncompressed
out.extend_from_slice(&(entry.name.len() as u16).to_le_bytes());
out.extend_from_slice(&0u16.to_le_bytes()); // extra len
out.extend_from_slice(entry.name.as_bytes());
out.extend_from_slice(entry.body);
placed.push((crc, entry.body.len(), offset));
}
let cd_start = out.len();
for (entry, (crc, size, offset)) in entries.iter().zip(&placed) {
out.extend_from_slice(b"PK\x01\x02");
out.extend_from_slice(&20u16.to_le_bytes()); // version made by
out.extend_from_slice(&20u16.to_le_bytes()); // version needed
out.extend_from_slice(&0u16.to_le_bytes()); // flags
out.extend_from_slice(&0u16.to_le_bytes()); // method: stored
out.extend_from_slice(&0u16.to_le_bytes()); // time
out.extend_from_slice(&0x0021u16.to_le_bytes()); // date
out.extend_from_slice(&crc.to_le_bytes());
out.extend_from_slice(&(*size as u32).to_le_bytes());
out.extend_from_slice(&(*size as u32).to_le_bytes());
out.extend_from_slice(&(entry.name.len() as u16).to_le_bytes());
out.extend_from_slice(&0u16.to_le_bytes()); // extra len
out.extend_from_slice(&0u16.to_le_bytes()); // comment len
out.extend_from_slice(&0u16.to_le_bytes()); // disk number start
out.extend_from_slice(&0u16.to_le_bytes()); // internal attrs
out.extend_from_slice(&0u32.to_le_bytes()); // external attrs
out.extend_from_slice(&(*offset as u32).to_le_bytes());
out.extend_from_slice(entry.name.as_bytes());
}
let cd_size = out.len() - cd_start;
out.extend_from_slice(b"PK\x05\x06");
out.extend_from_slice(&0u16.to_le_bytes()); // this disk
out.extend_from_slice(&0u16.to_le_bytes()); // disk with central directory
out.extend_from_slice(&(entries.len() as u16).to_le_bytes());
out.extend_from_slice(&(entries.len() as u16).to_le_bytes());
out.extend_from_slice(&(cd_size as u32).to_le_bytes());
out.extend_from_slice(&(cd_start as u32).to_le_bytes());
out.extend_from_slice(&0u16.to_le_bytes()); // comment len
out
}
/// Escape `s` for XML character data.
///
/// The corpus plants `&` and `<` nowhere by default, but the writers below
/// route every piece of body text through this so a future sentence
/// containing one cannot silently produce an unparseable container.
pub fn xml_escape(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for c in s.chars() {
match c {
'&' => out.push_str("&amp;"),
'<' => out.push_str("&lt;"),
'>' => out.push_str("&gt;"),
_ => out.push(c),
}
}
out
}

6
crates/quicksearch-core/tests/encrypted.rs
View file

@ -91,12 +91,12 @@ fn encrypted_index_lifecycle() {
{
let conn = db::open_existing(&db_path.to_string_lossy(), true).unwrap();
conn.execute_batch(
"INSERT INTO files (name, path, parent, size, mtime, type, content_state)
"INSERT INTO files (name, parent, size, mtime, type, content_state)
WITH RECURSIVE n(i) AS (
SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 20000
)
SELECT 'p' || i, '/pad/' || i, '/pad', 0, 0, 0, 3 FROM n;
DELETE FROM files WHERE parent = '/pad';",
SELECT 'p' || i, '/pad/', 0, 0, 0, 3 FROM n;
DELETE FROM files WHERE parent = '/pad/';",
)
.unwrap();
drop(conn);

337
crates/quicksearch-core/tests/extraction_corpus.rs Normal file
View file

@ -0,0 +1,337 @@
//! End-to-end extraction coverage across every format QuickSearch claims.
//!
//! The corpus itself — what is written, by which library, and why that library
//! rather than the reader's own — is documented in [`corpus`]. This file is
//! only the assertions.
//!
//! Three layers, deliberately, because they fail for different reasons:
//!
//! 1. [`every_format_extracts_its_planted_text`] goes through
//! `mime::guess_mime_from_head` and `extract::Registry`, so a file typed
//! wrongly and a file parsed wrongly are both caught, and the failure
//! message says which.
//! 2. [`head_extraction_agrees_with_reading_the_file`] pins the walk-time
//! shortcut against the content-pass path, in both directions.
//! 3. [`the_whole_corpus_indexes_and_is_searchable`] is the product claim: the
//! text reached FTS5 and a user typing a word from the document finds it.
mod common;
mod corpus;
use std::path::Path;
use std::sync::atomic::AtomicU64;
use quicksearch_core::config::Config;
use quicksearch_core::extract::Registry;
use quicksearch_core::mime;
use quicksearch_core::query::split_for_cascade;
use quicksearch_core::search::cascade;
use quicksearch_core::search::SearchOptions;
use corpus::Sample;
/// How much of a file the MIME sniff is shown. The same default the indexer
/// uses, so dispatch here matches dispatch in a real run.
const HEAD: usize = 8 * 1024;
/// Read the leading [`HEAD`] bytes, which is what the walk hands the sniff.
fn head_of(path: &Path) -> Vec<u8> {
let bytes = std::fs::read(path).unwrap_or_else(|e| panic!("read {}: {e}", path.display()));
bytes[..bytes.len().min(HEAD)].to_vec()
}
/// Whether the file is small enough that the walk would extract it from the
/// buffer it already hashed, rather than reopening it.
fn fits_in_head(path: &Path) -> bool {
std::fs::metadata(path)
.map(|m| m.len() as usize)
.unwrap_or(usize::MAX)
<= HEAD
}
/// Prefix every failure with the seed, so a red CI job reproduces locally.
fn ctx(sample: &Sample) -> String {
format!(
"[{} | {} | QUICKSEARCH_CORPUS_SEED={}]",
sample.label,
sample.path.display(),
corpus::seed()
)
}
/// Every corpus file is typed, claimed by an extractor, and yields its planted
/// lipsum in order.
#[test]
fn every_format_extracts_its_planted_text() {
let (_dir, samples) = corpus::build("corpus-extract");
let registry = Registry::default_set();
for sample in &samples {
let head = head_of(&sample.path);
let mime = mime::guess_mime_from_head(&sample.path, &head)
.unwrap_or_else(|| panic!("{} no MIME resolved", ctx(sample)));
assert!(
registry.supports(&mime),
"{} MIME {mime:?} is claimed by no extractor",
ctx(sample)
);
let content = registry
.extract(&sample.path, &mime)
.unwrap_or_else(|e| panic!("{} extraction failed: {e}", ctx(sample)))
.unwrap_or_else(|| panic!("{} MIME {mime:?} dispatched nowhere", ctx(sample)));
if let Err(why) = corpus::match_in_order(&content.text, &sample.must_contain) {
panic!("{} extracted text is wrong\n{why}", ctx(sample));
}
}
// A corpus that silently stopped generating anything would pass every
// assertion above.
assert!(
samples.len() >= 35,
"corpus shrank to {} samples",
samples.len()
);
}
/// The walk-time buffer path and the content-pass path agree, and only the
/// formats that can support the shortcut take it.
#[test]
fn head_extraction_agrees_with_reading_the_file() {
let (_dir, samples) = corpus::build("corpus-head");
let registry = Registry::default_set();
let mut opted_in = 0;
for sample in &samples {
let head = head_of(&sample.path);
let mime = mime::guess_mime_from_head(&sample.path, &head).expect("MIME");
let whole = std::fs::read(&sample.path).expect("read whole file");
let from_head = registry.extract_complete_head(&sample.path, &mime, &whole);
if !sample.head_path {
// A format that seeks, or reads a trailer, must never be handed a
// buffer — `None` here is what routes it back to the on-disk
// extractor rather than to a wrong answer.
assert!(
from_head.is_none(),
"{} took the head path but cannot support it",
ctx(sample)
);
continue;
}
opted_in += 1;
let from_head = from_head
.unwrap_or_else(|| panic!("{} declined the head path", ctx(sample)))
.unwrap_or_else(|e| panic!("{} head extraction failed: {e}", ctx(sample)));
let from_disk = registry
.extract(&sample.path, &mime)
.expect("on-disk extraction")
.expect("claimed");
assert_eq!(
from_head.text,
from_disk.text,
"{} head and disk extraction disagree",
ctx(sample)
);
// The claim is only interesting where the walk would really take the
// shortcut. `oversized.txt` is the deliberate exception.
if fits_in_head(&sample.path) {
assert!(
corpus::match_in_order(&from_head.text, &sample.must_contain).is_ok(),
"{} head extraction lost the planted text",
ctx(sample)
);
}
}
assert!(
opted_in >= 24,
"only {opted_in} samples exercised the head path"
);
}
/// The whole corpus indexes, and each file is findable by a word that appears
/// only in its body.
#[test]
fn the_whole_corpus_indexes_and_is_searchable() {
let (dir, samples) = corpus::build("corpus-index");
// The database goes in its own directory, not the one being walked. Left
// inside, SQLite's `-wal` and `-shm` sidecars appear under the root and
// survive only because a default ignore pattern happens to prune them —
// which is a dependency this test has no reason to take on.
let db = common::scratch_db("corpus-index-db");
let config = Config::default();
common::IndexOnce {
db: &db,
roots: vec![dir.to_string_lossy().into_owned()],
config: &config,
fresh_marker: false,
encrypted: false,
}
.run();
let conn = rusqlite::Connection::open(&db).expect("open index");
for sample in &samples {
let hits = search(&conn, &sample.needle);
let paths: Vec<&str> = hits.iter().map(|p| p.as_str()).collect();
assert_eq!(
hits.len(),
1,
"{} searching {:?} returned {:?}",
ctx(sample),
sample.needle,
paths
);
assert!(
Path::new(&hits[0]) == sample.path,
"{} searching {:?} found {:?}",
ctx(sample),
sample.needle,
hits[0]
);
}
}
/// Every result path for `term`, run through the same cascade the GUI uses.
fn search(conn: &rusqlite::Connection, term: &str) -> Vec<String> {
let split = split_for_cascade(term).expect("split");
let latest = AtomicU64::new(1);
let mut paths = Vec::new();
cascade::run(
conn,
&split,
&SearchOptions::default(),
1,
&latest,
&mut |batch| {
for hit in batch {
paths.push(hit.path.clone());
}
},
)
.expect("cascade run")
.expect("not cancelled");
paths
}
/// Two RTF lexer bugs this corpus found, now fixed in `vendor/rtf-parser`.
///
/// Both came from a lexer that decided where a control word ends by looking
/// for whitespace, and then trimmed leading spaces off whatever followed. The
/// second reproduced on a file LibreOffice wrote.
///
/// **A `\uN` escape with a literal fallback took the rest of the word with
/// it.** The escape is followed by an ANSI fallback character for readers that
/// predate Unicode, and the spec allows any character there. Terminating a
/// control word at whitespace only meant `before \u233?after end` lexed
/// `\u233?after` as one unrecognised control word and yielded `"before end"` —
/// the accented character gone, and `after` with it. The loss ran to the next
/// space. The spellings that put a backslash where the lexer needed a boundary
/// — LibreOffice's `\uN\'3f` and Word's `\uN\'hh` — were unaffected.
///
/// **A space between two escaped words was dropped.** After a `\'hh` escape
/// the lexer re-tokenised the remainder, trimming its leading spaces before
/// deciding what it was. A plain-text remainder kept the untrimmed slice and
/// survived; one beginning with another control word did not. So
/// `Καλημέρα κόσμε` — every character escaped on both sides of the space —
/// came back as `Καλημέρακόσμε`, two words collapsed into one FTS term. That
/// hit every script outside cp1252: Greek, Cyrillic, Hebrew, CJK. Latin-1 text
/// was fine, because there the escapes sit *inside* words (`caf\'e9`) and the
/// space that follows is plain text.
///
/// The fixes are `StrUtils::split_control_word` and the `\'hh` arm of
/// `Lexer::tokenize`, both marked LOCAL PATCH; the fallback characters are now
/// counted off against `\ucN` in the parser rather than guessed at. See the
/// `[patch.crates-io]` note in the workspace manifest.
#[test]
fn rtf_unicode_escapes_survive_extraction() {
let dir = quicksearch_core::testutil::scratch_dir("rtf-escapes");
let extract = |name: &str, body: &str| {
use quicksearch_core::extract::Extractor;
let path = dir.join(name);
std::fs::write(&path, body).unwrap();
quicksearch_core::extract::rtf::RtfExtractor
.extract(&path)
.unwrap_or_else(|e| panic!("{name}: {e}"))
.text
};
// The literal fallback. Both halves matter: the escape survives, and so
// does the word it used to swallow.
let text = extract("literal.rtf", r"{\rtf1\ansi before \u233?after end}");
assert!(text.contains("before éafter end"), "{text:?}");
// And the fallback itself is *not* text. It repeats the character for
// readers that cannot do Unicode; indexing it would put a `?` inside every
// word containing a non-cp1252 character.
assert!(!text.contains('?'), "fallback character indexed: {text:?}");
// The same document in the two spellings that always worked, to pin that
// counting fallbacks did not break the ones the old mask handled.
for (label, body) in [
("libreoffice", r"{\rtf1\ansi before \u233\'3fafter end}"),
("word", r"{\rtf1\ansi before \u233\'e9after end}"),
] {
let text = extract(&format!("{label}.rtf"), body);
assert!(text.contains("before éafter end"), "{label}: {text:?}");
}
// `\ucN` is the fallback count, and it is not always one.
let text = extract("uc2.rtf", r"{\rtf1\ansi\uc2 a\u233?!b}");
assert!(text.contains("aéb"), "two fallbacks not skipped: {text:?}");
let text = extract("uc0.rtf", r"{\rtf1\ansi\uc0 a\u233?b}");
assert!(
text.contains("aé?b"),
"\\uc0 means no fallback, so the `?` is real text: {text:?}"
);
// Exactly the bytes LibreOffice writes for `Καλημέρα κόσμε`.
let text = extract(
"escaped-words.rtf",
concat!(
r"{\rtf1\ansi ",
r"\u922\'3f\u945\'3f\u955\'3f\u951\'3f",
r"\u956\'3f\u941\'3f\u961\'3f\u945\'3f",
" ",
r"\u954\'3f\u972\'3f\u963\'3f\u956\'3f\u949\'3f",
"}"
),
);
assert!(
text.contains("Καλημέρα κόσμε"),
"the space between two fully escaped words was dropped: {text:?}"
);
// A `\'hh` that is nobody's fallback is still text, and a space after it
// is still a space — the case the second fix must not overshoot.
let text = extract("standalone-hex.rtf", r"{\rtf1\ansi caf\'e9 \'e0 noon}");
assert!(text.contains("café à noon"), "{text:?}");
// A surrogate pair is one character, and `\uc1` puts a fallback between
// its halves. U+1F600, written the way Word writes it.
let text = extract("astral.rtf", r"{\rtf1\ansi a\u-10179?\u-8704?b}");
assert!(
text.contains("a\u{1F600}b"),
"surrogate pair lost: {text:?}"
);
}
/// The corpus's needles and the shared harness's body term are the same word,
/// deliberately.
///
/// `common::BODY_TERM` exists because a term that reaches the index through a
/// file *name* as well as a body is answered mostly by the filename pass — so
/// it plants one that only ever appears in a body. The corpus needs exactly
/// that property, per file, for its end-to-end search to be attributable to
/// extraction rather than to the name.
///
/// `corpus` does not name `common` (it has no other reason to depend on the
/// harness), so this is what stops the two drifting apart silently.
#[test]
fn corpus_needles_use_the_shared_body_term() {
assert_eq!(corpus::NEEDLE_PREFIX, common::BODY_TERM);
}

61
crates/quicksearch-core/tests/fixtures/legacy/README.md vendored Normal file
View file

@ -0,0 +1,61 @@
# Legacy Office fixtures
`sample.doc`, `sample.xls` and `sample.ppt`, written by LibreOffice and
committed. Read by `tests/extraction_corpus.rs` through the corpus module in
`tests/corpus/legacy.rs`.
## Why these three are committed when the rest of the corpus is generated
Every other format in the extraction corpus is written at test time by a
library that is *not* the one that reads it back — the whole point being that a
fixture built with the reader's own library can only prove the two agree with
each other. The pre-2007 binary formats have no such library: `cfb` is the only
Rust crate that writes OLE2 compound files, and `cfb` is what `extract::ole`
reads them with.
So a foreign producer writes them once, here, and the output is committed.
That buys more than independence — a LibreOffice `.doc` is a real FIB with a
real piece table, its `.xls` a real BIFF stream with a real `SST`, and its
`.ppt` drags the master slide's placeholder prompts (`Click to edit the title
text format`, `___PPT10`) into the text stream alongside the content. None of
those shapes come out of a minimal synthetic fixture, and the last one is why
the corpus asserts ordered *containment* rather than equality.
The unit tests in `src/extract/ole_tests.rs` are the complement, not a
duplicate: they build deliberately malformed streams to check bounds and error
paths, which needs byte-level control that only `cfb` gives.
## Files
| file | role |
|---|---|
| `prose.txt` | source for `sample.doc` |
| `sheet.csv` | source for `sample.xls` — one cell per row, no commas |
| `deck.fodp` | source for `sample.ppt` — flat ODF, so the text is reviewable |
| `sample.doc` `sample.xls` `sample.ppt` | LibreOffice output, committed |
| `regen.sh` | regenerates the three from the sources |
The sources are the source of truth: `legacy.rs` reads the expected fragments
out of them rather than restating the text, so a regenerated fixture that
dropped a line fails the test instead of quietly redefining what it should
contain. Each source carries a needle (`chalcedony9001`-`chalcedony9003`) that
the end-to-end search looks for; `legacy.rs` asserts it is still present.
## Regenerating
```sh
./regen.sh # needs libreoffice on PATH; last run with 26.2.4.2
cargo test -p quicksearch-core --test extraction_corpus
```
The output is **not** byte-reproducible — LibreOffice stamps a creation time
into each file — so re-running changes the bytes without changing the text.
Commit the result only when the *sources* changed; the test is what says the
files are still right.
`sample.ppt` is ~460 KB because the PowerPoint 97 export filter embeds the
master slide. There is no filter option that trims it, and it is well inside
the 2 MiB `maximum_text_file_size` the end-to-end run indexes with.
Unlike the generated half of the corpus, the text in these three is fixed:
`QUICKSEARCH_CORPUS_SEED` does not affect them.

38
crates/quicksearch-core/tests/fixtures/legacy/deck.fodp vendored Normal file
View file

@ -0,0 +1,38 @@
<?xml version="1.0" encoding="UTF-8"?>
<!-- Flat ODF presentation: the human-readable source `regen.sh` converts into
sample.ppt. Kept in this form rather than as a binary .odp so a reviewer
can see exactly what text the fixture is supposed to contain. -->
<office:document
xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0"
xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0"
xmlns:draw="urn:oasis:names:tc:opendocument:xmlns:drawing:1.0"
xmlns:svg="urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0"
office:version="1.3"
office:mimetype="application/vnd.oasis.opendocument.presentation">
<office:body>
<office:presentation>
<draw:page draw:name="page1">
<draw:frame svg:width="20cm" svg:height="3cm" svg:x="2cm" svg:y="2cm">
<draw:text-box><text:p>Lorem ipsum dolor sit amet consectetur</text:p></draw:text-box>
</draw:frame>
<draw:frame svg:width="20cm" svg:height="3cm" svg:x="2cm" svg:y="6cm">
<draw:text-box><text:p>The needle chalcedony9003 marks this deck</text:p></draw:text-box>
</draw:frame>
<draw:frame svg:width="20cm" svg:height="3cm" svg:x="2cm" svg:y="10cm">
<draw:text-box><text:p>Tempor incididunt café résumé naïve dolore</text:p></draw:text-box>
</draw:frame>
</draw:page>
<draw:page draw:name="page2">
<draw:frame svg:width="20cm" svg:height="3cm" svg:x="2cm" svg:y="2cm">
<draw:text-box><text:p>Ut enim ad Καλημέρα κόσμε veniam quis</text:p></draw:text-box>
</draw:frame>
<draw:frame svg:width="20cm" svg:height="3cm" svg:x="2cm" svg:y="6cm">
<draw:text-box><text:p>Ullamco laboris nisi ut aliquip commodo</text:p></draw:text-box>
</draw:frame>
<draw:frame svg:width="20cm" svg:height="3cm" svg:x="2cm" svg:y="10cm">
<draw:text-box><text:p>Irure dolor in reprehenderit voluptate velit</text:p></draw:text-box>
</draw:frame>
</draw:page>
</office:presentation>
</office:body>
</office:document>

6
crates/quicksearch-core/tests/fixtures/legacy/prose.txt vendored Normal file
View file

@ -0,0 +1,6 @@
Lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod.
The needle chalcedony9001 marks this document for the end-to-end search.
Tempor incididunt ut labore café résumé naïve et dolore magna aliqua.
Ut enim ad minim Καλημέρα κόσμε veniam quis nostrud exercitation.
Ullamco laboris nisi ut aliquip ex ea commodo consequat duis aute.
Irure dolor in reprehenderit in voluptate velit esse cillum dolore.

39
crates/quicksearch-core/tests/fixtures/legacy/regen.sh vendored Normal file
View file

@ -0,0 +1,39 @@
#!/bin/sh
# Regenerate the committed .doc/.xls/.ppt fixtures from their sources.
#
# These three are the one corner of the extraction corpus that cannot be built
# on the fly: `cfb` is the only Rust crate that writes OLE2 compound files, and
# `cfb` is what `extract::ole` reads them with — a fixture built by the reader's
# own library can only prove the two agree with each other. So a foreign
# producer writes them once, here, and the output is committed.
#
# Requires LibreOffice on PATH. Last run with 26.2.4.2.
#
# The output is NOT byte-reproducible: LibreOffice stamps a creation time into
# each file. Re-running this changes the bytes without changing the text, so
# commit the result only when the *sources* changed. `cargo test -p
# quicksearch-core --test extraction_corpus` is what says the files are still
# right.
set -eu
here=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
work=$(mktemp -d)
trap 'rm -rf "$work"' EXIT
convert() {
# -env:UserInstallation keeps this off the invoking user's LibreOffice
# profile, which also lets it run when a desktop LibreOffice is open.
libreoffice --headless \
-env:UserInstallation="file://$work/profile" \
--convert-to "$1" --outdir "$work" "$2" >/dev/null
}
convert doc "$here/prose.txt"
convert xls "$here/sheet.csv"
convert ppt "$here/deck.fodp"
mv "$work/prose.doc" "$here/sample.doc"
mv "$work/sheet.xls" "$here/sample.xls"
mv "$work/deck.ppt" "$here/sample.ppt"
ls -l "$here/sample.doc" "$here/sample.xls" "$here/sample.ppt"

BIN
crates/quicksearch-core/tests/fixtures/legacy/sample.doc vendored Normal file
View file

Binary file not shown.

BIN
crates/quicksearch-core/tests/fixtures/legacy/sample.ppt vendored Normal file
View file

Binary file not shown.

BIN
crates/quicksearch-core/tests/fixtures/legacy/sample.xls vendored Normal file
View file

Binary file not shown.

6
crates/quicksearch-core/tests/fixtures/legacy/sheet.csv vendored Normal file
View file

@ -0,0 +1,6 @@
Lorem ipsum dolor sit amet consectetur
The needle chalcedony9002 marks this sheet
Tempor incididunt café résumé naïve dolore
Ut enim ad Καλημέρα κόσμε veniam quis
Ullamco laboris nisi ut aliquip commodo
Irure dolor in reprehenderit voluptate velit
1 Lorem ipsum dolor sit amet consectetur
2 The needle chalcedony9002 marks this sheet
3 Tempor incididunt café résumé naïve dolore
4 Ut enim ad Καλημέρα κόσμε veniam quis
5 Ullamco laboris nisi ut aliquip commodo
6 Irure dolor in reprehenderit voluptate velit

BIN
crates/quicksearch-core/tests/fixtures/silence.flac vendored Normal file
View file

Binary file not shown.

59
crates/quicksearch-core/tests/full_index.rs
View file

@ -47,7 +47,7 @@ fn index_once(root: &Path, db: &Path, config: &Config) {
fn rows(db: &Path) -> Vec<(String, i64, i64)> {
let conn = rusqlite::Connection::open(db).unwrap();
let mut stmt = conn
.prepare("SELECT path, mtime, content_state FROM files ORDER BY path")
.prepare("SELECT parent || name, mtime, content_state FROM files ORDER BY parent, name")
.unwrap();
let out = stmt
.query_map([], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)))
@ -756,11 +756,11 @@ fn content_rows(db: &Path) -> Vec<ContentRow> {
let conn = rusqlite::Connection::open(db).unwrap();
let mut stmt = conn
.prepare(
"SELECT f.path, f.content_state, ff.reason, LENGTH(d.text_zstd)
"SELECT f.parent || f.name, f.content_state, ff.reason, LENGTH(d.text_zstd)
FROM files f
LEFT JOIN documents_text d ON d.file_id = f.id
LEFT JOIN failed_files ff ON ff.file_id = f.id
ORDER BY f.path",
ORDER BY f.parent, f.name",
)
.unwrap();
let out = stmt
@ -778,7 +778,7 @@ fn stored_text(db: &Path, suffix: &str) -> Option<String> {
.query_row(
"SELECT d.text_zstd FROM documents_text d
JOIN files f ON f.id = d.file_id
WHERE f.path LIKE '%' || ?1",
WHERE f.parent || f.name LIKE '%' || ?1",
[suffix],
|r| r.get(0),
)
@ -900,7 +900,7 @@ fn undecodable_small_files_are_reported_as_failures_not_silently_skipped() {
.query_row(
"SELECT f.content_state, ff.reason FROM files f \
LEFT JOIN failed_files ff ON ff.file_id = f.id \
WHERE f.path LIKE '%bad.txt'",
WHERE f.parent || f.name LIKE '%bad.txt'",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
@ -943,7 +943,7 @@ fn an_unreadable_legacy_office_file_fails_with_a_reason() {
.query_row(
"SELECT f.content_state, ff.reason FROM files f \
LEFT JOIN failed_files ff ON ff.file_id = f.id \
WHERE f.path LIKE '%broken.doc'",
WHERE f.parent || f.name LIKE '%broken.doc'",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
@ -981,7 +981,7 @@ fn extensionless_text_files_are_indexed() {
let conn = rusqlite::Connection::open(&db).unwrap();
let state_of = |name: &str| -> i64 {
conn.query_row(
"SELECT content_state FROM files WHERE path LIKE '%' || ?1",
"SELECT content_state FROM files WHERE parent || name LIKE '%' || ?1",
[name],
|r| r.get(0),
)
@ -1162,7 +1162,7 @@ fn an_empty_file_is_done_with_no_snippet_sidecar() {
let (state, sidecars): (i64, i64) = conn
.query_row(
"SELECT f.content_state, (SELECT COUNT(*) FROM documents_text d WHERE d.file_id = f.id)
FROM files f WHERE f.path LIKE '%empty.txt'",
FROM files f WHERE f.parent || f.name LIKE '%empty.txt'",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
@ -1188,7 +1188,7 @@ fn the_content_extension_filter_still_excludes_small_text_files() {
let conn = rusqlite::Connection::open(&db).unwrap();
let states: Vec<(String, i64)> = conn
.prepare("SELECT path, content_state FROM files ORDER BY path")
.prepare("SELECT parent || name, content_state FROM files ORDER BY parent, name")
.unwrap()
.query_map([], |r| Ok((r.get(0)?, r.get(1)?)))
.unwrap()
@ -1824,7 +1824,28 @@ fn the_wal_stays_bounded_during_a_run() {
/// A run cut short is exactly when the log is at its largest and nothing else
/// will come along to land it: the writer connection closes, and the next run
/// may be hours away. So Stop ends the *indexing*, and the pass that follows
/// runs either way — visible as `Optimizing` until it is done.
/// runs either way.
///
/// # Why this asks SQLite rather than watching the status
///
/// This used to poll `get_status()` at 1 ms hoping to catch
/// `IndexingStatus::Optimizing` as it went past, and assert it had seen it.
/// That is a sample of a transient state, and it fails for reasons that have
/// nothing to do with the behaviour under test: a loaded machine deschedules
/// the polling thread, and — worse — *making the indexer faster shortens the
/// window*, so the test failed more often the better the code got. It was
/// failing roughly a third of the time before any of that, and five times in
/// six after a round of writer optimisations.
///
/// So it asks the question directly instead: was `PRAGMA optimize` handed to
/// SQLite for this index? [`repo::optimize_count`] is a latch rather than a
/// sample, so no amount of speed or scheduling can hide the answer. Per the
/// pass's own contract, the pragma reaching SQLite is the optimization
/// happening — what SQLite then decides to re-analyse is its business, and
/// deliberately nothing when no table has drifted.
///
/// The log assertion stays, because it checks the other half of the pass: the
/// trailing checkpoint that a stopped run exists to get.
#[test]
fn a_stopped_run_is_still_optimized() {
let root = tmp_dir("stop-optimize");
@ -1838,6 +1859,11 @@ fn a_stopped_run_is_still_optimized() {
let db_dir = tmp_dir("stop-optimize-db");
let db = db_dir.join("index.sqlite");
let wal = db_dir.join("index.sqlite-wal");
let dir_key = db_dir.to_string_lossy().into_owned();
// Per-directory, so a sibling test optimizing its own scratch index on
// another thread cannot satisfy this.
let before = quicksearch_core::db::repo::optimize_count(&dir_key);
let service = IndexingService::new();
service
@ -1861,11 +1887,11 @@ fn a_stopped_run_is_still_optimized() {
}
service.request_stop();
let mut saw_optimizing = false;
// Idle is the *end* of the pass, and unlike Optimizing it is a resting
// state — waiting for it cannot miss it however fast the pass was.
let idle_by = Instant::now() + Duration::from_secs(120);
loop {
match service.get_status() {
IndexingStatus::Optimizing => saw_optimizing = true,
IndexingStatus::Idle => break,
IndexingStatus::Error(e) => panic!("indexing failed: {}", e),
_ => {}
@ -1877,9 +1903,10 @@ fn a_stopped_run_is_still_optimized() {
std::thread::sleep(Duration::from_millis(1));
}
assert!(
saw_optimizing,
"a stopped run must still publish Optimizing"
assert_eq!(
quicksearch_core::db::repo::optimize_count(&dir_key),
before + 1,
"a stopped run must still run PRAGMA optimize against its index"
);
assert_eq!(
std::fs::metadata(&wal).map(|m| m.len()).unwrap_or(0),
@ -1933,7 +1960,7 @@ fn high_byte_binaries_are_listed_but_not_text_extracted() {
"SELECT f.content_state,
(SELECT COUNT(*) FROM documents_text d WHERE d.file_id = f.id),
(SELECT COUNT(*) FROM failed_files x WHERE x.file_id = f.id)
FROM files f WHERE f.path LIKE '%' || ?1",
FROM files f WHERE f.parent || f.name LIKE '%' || ?1",
[suffix],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
)

779
crates/quicksearch-core/tests/prefilter_fuzz.rs Normal file
View file

@ -0,0 +1,779 @@
//! Fuzzing the cascade's prefilters.
//!
//! Two passes narrow what they scan by "at least one of these literals must be
//! present" — the fuzzy full-text pass by its pigeonhole chunks, the `regex:`
//! passes by the literals extracted from the pattern. Both go through
//! [`quicksearch_core::search::prefilter`], and both share a failure mode that
//! is **silent**: a lost hit looks exactly like a file that does not match, and
//! no user could tell the difference. So neither is defended by a handful of
//! examples.
//!
//! The regex half gets a stronger check than the fuzzy half, because it has a
//! perfect oracle: running the compiled regex over each body *is* the right
//! answer, so the assertion is set equality rather than one-sided recall.
//!
//! # The generator
//!
//! Rather than invent queries and assert an expected answer, this manufactures
//! queries whose correct answer is known **by construction**:
//!
//! 1. take a document from the corpus,
//! 2. cut a substring of it — text the document provably contains,
//! 3. corrupt that substring `N ≤ k` times,
//! 4. search for the result. The document it came from **must** be found.
//!
//! Step 4 needs no oracle and no reference implementation: the term is within
//! `N` edits of something the document really contains, so a ≤`k`-edit
//! alignment exists and the prefilter is not allowed to exclude it.
//!
//! # Substring length is swept, not random
//!
//! Documents are long, so drawing the length uniformly would spend nearly every
//! iteration on long terms — the case least likely to be broken. Instead
//! [`ITERS_PER_LEN`] iterations run at *every* length from 1 to [`MAX_SWEEP`],
//! then a sparse tail for long terms. Short lengths are where the chunking, the
//! floors and the boundary conditions all live, and at the default cap the
//! `1..=20` sweep straddles every one of them: the 3-character fuzzy floor, and
//! `3 × (cap + 1) = 9` where the prefilter becomes legal.
//!
//! The bottom of the sweep is asserted rather than skipped. Below the fuzzy
//! floor no `Bitap` is built and the pass must not scan at all.
//!
//! # Edits are counted in bytes
//!
//! Bitap's budget is a byte budget, so substituting `é` (two bytes) for `x`
//! (one) is a distance of two, not one. Corruption works on characters to keep
//! the term valid UTF-8, then the true byte distance is measured against the
//! original substring and recall is only asserted when it really is within
//! budget. Without that, a non-ASCII corpus would quietly start generating
//! out-of-budget queries and every "miss" would be correct behaviour — the
//! test would pass by not testing anything.
//!
//! ```text
//! cargo test --release -p quicksearch-core --test fuzzy_prefilter_fuzz
//! QSB_FUZZ_ITERS=2000 cargo test --release -p quicksearch-core \
//! --test fuzzy_prefilter_fuzz -- --nocapture
//! ```
use std::sync::atomic::AtomicU64;
use quicksearch_core::query::split::split_for_cascade;
use quicksearch_core::search::fuzzy::{edit_budget, pigeonhole_chunks, Bitap, TRIGRAM_FLOOR};
use quicksearch_core::search::{cascade, SearchHit, SearchOptions};
mod common;
use common::{scratch_db, Lcg};
/// Iterations at each swept substring length.
const ITERS_PER_LEN: usize = 100;
/// Top of the dense sweep. Every length from 1 to here gets [`ITERS_PER_LEN`]
/// iterations.
const MAX_SWEEP: usize = 20;
/// Long terms, sampled rather than swept. 64 is `Bitap`'s pattern ceiling and
/// 80 is past it, where the fuzzy passes must decline to run at all.
const LONG_TAIL: [usize; 6] = [24, 32, 48, 64, 80, 200];
/// Edit-distance caps to sweep beneath the length sweep. Each moves both the
/// budget and the `3 × (cap + 1)` guard.
const CAPS: [usize; 5] = [0, 1, 2, 3, 4];
/// How many caps the ungated run sweeps. The soak takes all of [`CAPS`].
const BOUNDED_CAPS: usize = 3;
// ---------------------------------------------------------------------------
// Corpus
// ---------------------------------------------------------------------------
/// Documents to seed. Small — this is a correctness harness, and every
/// iteration runs a whole cascade — but varied enough that a query drawn from
/// one document meets plenty of others it must reject.
const DOCS: usize = 60;
/// Bodies the generator draws substrings from.
///
/// Deliberately hostile, because the substring generator carries whatever is in
/// the corpus straight into the term and then into the FTS chunks:
///
/// * **FTS5 metacharacters** — `"`, `*`, `:`, `^`, `-`, `(`, `)` and the bare
/// word `NEAR`. A chunk containing these must be quoted into inertness by
/// `translator::quote_phrase`, or the query is a syntax error rather than a
/// search. This is the likeliest way to break the feature and it needs no
/// special case in the generator: it is simply in the text.
/// * **Multi-byte characters** at 2, 3 and 4 bytes, so a byte-wise chunk split
/// would produce invalid UTF-8.
/// * **Diacritics**, which the trigram index folds (`remove_diacritics 1`) and
/// bitap does not — the prefilter must stay a superset across that.
/// * **Mixed case**, which also exercises the case-insensitive mask table.
/// * **Degenerate shapes** — a repeated character, where every chunk of a
/// substring is identical and the substring is self-overlapping.
fn bodies() -> Vec<String> {
let mut out = Vec::new();
let mut lcg = Lcg::new(0xf0072);
let words = [
"quartzite",
"Report",
"SUMMARY",
"café",
"naïve",
"Ünicode",
"日本語テキスト",
"emoji🙂here",
"NEAR",
"wild*card",
"colon:sep",
"quote\"mark",
"paren(then)",
"dash-joined",
"caret^up",
"aaaaaaaaaa",
"mixedCaseWord",
"budget",
"revenue",
"planning",
];
for d in 0..DOCS {
let n = 20 + (lcg.next() as usize % 40);
let mut body = String::new();
for _ in 0..n {
body.push_str(words[lcg.next() as usize % words.len()]);
body.push(' ');
}
// One document that is nothing but a repeated character, and one that
// is empty, as fixed shapes beside the random ones.
match d {
0 => body = "b".repeat(400),
1 => body = String::new(),
_ => {}
}
out.push(body);
}
out
}
// ---------------------------------------------------------------------------
// Oracles
// ---------------------------------------------------------------------------
/// Minimum Levenshtein distance between `pattern` and any substring of `hay`,
/// over **bytes**, ASCII-case-insensitively — the metric the matcher uses.
///
/// The shared brute-force reference. It is what decides whether a corrupted
/// term is genuinely inside the edit budget, and what the three-way comparison
/// checks both cascade paths against.
fn oracle_distance(pattern: &[u8], hay: &[u8]) -> usize {
if hay.is_empty() || pattern.is_empty() {
return usize::MAX;
}
let m = pattern.len();
let mut prev: Vec<usize> = vec![0; hay.len() + 1];
let mut cur = vec![0; hay.len() + 1];
for i in 1..=m {
cur[0] = i;
for j in 1..=hay.len() {
let cost = usize::from(!pattern[i - 1].eq_ignore_ascii_case(&hay[j - 1]));
cur[j] = (prev[j - 1] + cost).min(prev[j] + 1).min(cur[j - 1] + 1);
}
std::mem::swap(&mut prev, &mut cur);
}
prev.iter().copied().min().unwrap_or(usize::MAX)
}
// ---------------------------------------------------------------------------
// The generator
// ---------------------------------------------------------------------------
/// One corruption applied to a character sequence.
fn corrupt(chars: &mut Vec<char>, lcg: &mut Lcg) {
// A small alphabet of replacements, including multi-byte ones so that a
// single character edit can be a multi-byte edit.
const REPLACEMENTS: [char; 6] = ['x', 'Q', '7', 'é', '語', '🙂'];
let pick = REPLACEMENTS[lcg.next() as usize % REPLACEMENTS.len()];
if chars.is_empty() {
chars.push(pick);
return;
}
let at = lcg.next() as usize % chars.len();
match lcg.next() % 3 {
0 => chars[at] = pick, // substitution
1 => chars.insert(at, pick), // insertion
_ => {
chars.remove(at); // deletion
}
}
}
/// Cut `len` characters out of `body` at a random offset, or `None` when the
/// body is too short to give that many.
fn substring_of<'a>(body: &'a str, len: usize, lcg: &mut Lcg) -> Option<&'a str> {
let bounds: Vec<usize> = body
.char_indices()
.map(|(i, _)| i)
.chain(std::iter::once(body.len()))
.collect();
let chars = bounds.len() - 1;
if chars < len || len == 0 {
return None;
}
let start = lcg.next() as usize % (chars - len + 1);
Some(&body[bounds[start]..bounds[start + len]])
}
// ---------------------------------------------------------------------------
// Property 1: the pigeonhole invariant, with no database in sight
// ---------------------------------------------------------------------------
/// The argument the whole prefilter rests on, isolated from FTS, from SQLite
/// and from bitap: if the term occurs within `k` edits, some chunk of the
/// `k + 1`-way split occurs **verbatim**.
///
/// Driven by the same corrupt-a-known-substring generator, because a randomly
/// drawn (term, text) pair almost never has a ≤`k`-edit alignment and the
/// property would be vacuous.
#[test]
fn a_surviving_chunk_always_remains_after_k_edits() {
let bodies = bodies();
let mut lcg = Lcg::new(0xc0ffee);
let mut exercised = 0usize;
for &cap in &CAPS {
for len in sweep_lengths() {
for _ in 0..ITERS_PER_LEN {
let body = &bodies[lcg.next() as usize % bodies.len()];
let Some(original) = substring_of(body, len, &mut lcg) else {
continue;
};
let Some(k) = edit_budget(original.len(), cap) else {
continue;
};
let mut chars: Vec<char> = original.chars().collect();
let edits = lcg.next() as usize % (k + 1);
for _ in 0..edits {
corrupt(&mut chars, &mut lcg);
}
let term: String = chars.into_iter().collect();
// Only meaningful when the corrupted term really is within the
// budget of the text — see the module note on byte distance.
let distance = oracle_distance(term.as_bytes(), body.as_bytes());
let Some(chunks) = pigeonhole_chunks(&term, k) else {
continue;
};
if distance > k {
continue;
}
// The partition itself, checked every time: chunks that
// overlapped or skipped text would break the argument silently.
assert_eq!(chunks.concat(), term, "chunks must partition the term");
assert_eq!(chunks.len(), k + 1, "one chunk per edit, plus one");
assert!(
chunks.iter().all(|c| c.chars().count() >= TRIGRAM_FLOOR),
"every chunk must reach the trigram floor: {:?}",
chunks
);
// The property.
let folded_body = body.to_lowercase();
assert!(
chunks
.iter()
.any(|c| folded_body.contains(&c.to_lowercase())),
"no chunk of {:?} survived in the body it came from \
(k={}, distance={}, chunks={:?})",
term,
k,
distance,
chunks
);
exercised += 1;
}
}
}
assert!(
exercised > 500,
"the generator produced only {} in-budget cases; it is not exercising \
the property",
exercised
);
}
/// Adversarial placement, which random corruption will almost never construct:
/// damage exactly `k` of the `k + 1` chunks, every combination, and check the
/// untouched one is still there to be found.
#[test]
fn damaging_every_chunk_but_one_leaves_that_one_intact() {
for k in 0..=3usize {
let term: String = (0..(k + 1) * 4)
.map(|i| char::from(b'a' + (i % 26) as u8))
.collect();
let chunks = pigeonhole_chunks(&term, k).expect("long enough by construction");
// Damage all but one chunk, for each choice of the survivor.
for spared in 0..chunks.len() {
let mut text = String::new();
for (i, chunk) in chunks.iter().enumerate() {
if i == spared {
text.push_str(chunk);
} else {
// One substitution inside this chunk.
let mut c: Vec<char> = chunk.chars().collect();
c[0] = 'Z';
text.extend(c);
}
}
assert!(
chunks.iter().any(|c| text.contains(*c)),
"k={} spared={} term={:?} text={:?}",
k,
spared,
term,
text
);
// And the damaged text really is within budget, so this is a case
// the pass would have to find.
assert!(
oracle_distance(term.as_bytes(), text.as_bytes()) <= k,
"the constructed text should be within {} edits",
k
);
}
}
}
// ---------------------------------------------------------------------------
// Property 2: end-to-end recall, three ways
// ---------------------------------------------------------------------------
/// Lengths the sweep visits: dense from 1, then a sparse tail.
fn sweep_lengths() -> Vec<usize> {
(1..=MAX_SWEEP).chain(LONG_TAIL).collect()
}
fn iters_per_len() -> usize {
std::env::var("QSB_FUZZ_ITERS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(ITERS_PER_LEN)
}
fn caps() -> &'static [usize] {
if std::env::var("QSB_FUZZ_ITERS").is_ok() {
&CAPS
} else {
&CAPS[..BOUNDED_CAPS]
}
}
/// Run one fuzzy search and collect the file ids it found.
fn search_ids(conn: &rusqlite::Connection, term: &str, cap: usize) -> Vec<i64> {
let split = split_for_cascade(term).expect("any term parses; it degrades rather than errors");
let latest = AtomicU64::new(1);
let options = SearchOptions {
fuzzy: true,
fuzzy_max_edits: cap,
limit: 100_000,
..SearchOptions::default()
};
let mut ids = Vec::new();
let mut sink = |hits: Vec<SearchHit>| ids.extend(hits.iter().map(|h| h.file_id));
cascade::run(conn, &split, &options, 1, &latest, &mut sink).expect("the cascade runs");
ids.sort();
ids.dedup();
ids
}
/// The headline property: a term cut from a document and corrupted within the
/// edit budget still finds that document.
#[test]
fn a_corrupted_substring_still_finds_the_document_it_came_from() {
let bodies = bodies();
let db = scratch_db("fuzzprefilter");
let ids = seed(&db, &bodies);
let conn = quicksearch_core::db::open::open_search_reader(&db.to_string_lossy())
.expect("open the seeded index");
let mut lcg = Lcg::new(0x5ca1ab1e);
let mut checked = 0usize;
let mut below_floor = 0usize;
for &cap in caps() {
for len in sweep_lengths() {
for _ in 0..iters_per_len() {
let doc = lcg.next() as usize % bodies.len();
let body = &bodies[doc];
let Some(original) = substring_of(body, len, &mut lcg) else {
continue;
};
let mut chars: Vec<char> = original.chars().collect();
let planned = edit_budget(original.len(), cap).unwrap_or(0);
let edits = if planned == 0 {
0
} else {
lcg.next() as usize % (planned + 1)
};
for _ in 0..edits {
corrupt(&mut chars, &mut lcg);
}
let term: String = chars.into_iter().collect();
if term.trim().is_empty() {
continue;
}
// **The cascade does not search the string typed at it.** The
// query goes through the lexer first, and a term cut out of
// real document text is full of things the lexer acts on:
// whitespace is stripped and re-joined, `*` becomes a wildcard,
// `key:value` may become a filter, `"` opens a phrase. The
// first thing this harness found was a three-character term
// `" Qi"` whose leading space the lexer drops, leaving two
// characters — below the fuzzy floor, so no matcher is built
// and no document can be found. That is correct behaviour, and
// asserting against the raw string called it a lost hit.
//
// So every decision below is made against the *parsed* term,
// exactly as the passes make it.
let Ok(split) = split_for_cascade(&term) else {
// A term that parses to an error (a `regex:` fragment with
// bad syntax, a filter key with an unusable value) is not a
// fuzzy search at all.
continue;
};
if split.pattern.is_wildcard() || split.regex.is_some() {
// "Bitap is a literal matcher; wildcard terms don't fuzz"
// — the fuzzy passes decline outright, so there is no
// recall to demand.
continue;
}
let effective = split.term.as_str();
let Some(k) = edit_budget(effective.len(), cap) else {
// Below the fuzzy floor (or past Bitap's 64-byte ceiling):
// the pass must not run, so there is nothing to demand of
// it beyond not crashing.
below_floor += 1;
let _ = search_ids(&conn, &term, cap);
continue;
};
if Bitap::new(effective.as_bytes(), k).is_none() {
let _ = search_ids(&conn, &term, cap);
continue;
}
// The budget is over bytes; only assert recall when the term
// really is within it.
if oracle_distance(effective.as_bytes(), body.as_bytes()) > k {
continue;
}
let found = search_ids(&conn, &term, cap);
assert!(
found.contains(&ids[doc]),
"lost the document the term came from\n typed {:?}\n \
parsed {:?}\n original {:?}\n cap {} k {} edits {}\n \
chunks {:?}\n body {:?}",
term,
effective,
original,
cap,
k,
edits,
pigeonhole_chunks(effective, k),
&body.chars().take(120).collect::<String>(),
);
checked += 1;
}
}
}
eprintln!(
"fuzzy prefilter fuzz: {} recall assertions, {} terms below the fuzzy floor",
checked, below_floor
);
assert!(
checked > 200,
"only {} recall assertions ran; the generator is not producing \
in-budget terms",
checked
);
drop(conn);
std::fs::remove_file(&db).ok();
}
/// The two-sided property, against a brute-force oracle over the **whole**
/// corpus rather than just the document the term came from.
///
/// Recall alone says the prefilter did not lose the one hit we planted; it says
/// nothing about the other fifty-nine documents. This checks both directions:
///
/// * **No lost hits** — every document within `k` edits of the term appears in
/// the results, at whatever stage the cascade files it under. This is the
/// direction the prefilter can break, and the one with no visible symptom.
/// * **No invented hits** — every stage-8 (fuzzy content) hit really is within
/// `k` edits. The prefilter is only a superset, so it must not admit anything
/// the verification below it should have rejected.
///
/// A true oracle, not a comparison against the pre-prefilter code path: a bug
/// living in `Bitap` itself would be present in both cascade paths and a
/// differential between them would agree, happily, on the wrong answer.
///
/// Far fewer iterations than the recall sweep, because it runs a Levenshtein DP
/// over every document per iteration.
#[test]
fn every_document_within_the_budget_is_found_and_nothing_outside_it_is() {
const ITERS: usize = 5;
let bodies = bodies();
let db = scratch_db("fuzzprefilter-oracle");
let ids = seed(&db, &bodies);
let conn = quicksearch_core::db::open::open_search_reader(&db.to_string_lossy())
.expect("open the seeded index");
let mut lcg = Lcg::new(0xd1ce);
let mut compared = 0usize;
for &cap in caps() {
for len in sweep_lengths() {
for _ in 0..ITERS {
let doc = lcg.next() as usize % bodies.len();
let Some(original) = substring_of(&bodies[doc], len, &mut lcg) else {
continue;
};
let mut chars: Vec<char> = original.chars().collect();
let planned = edit_budget(original.len(), cap).unwrap_or(0);
let edits = if planned == 0 {
0
} else {
lcg.next() as usize % (planned + 1)
};
for _ in 0..edits {
corrupt(&mut chars, &mut lcg);
}
let term: String = chars.into_iter().collect();
if term.trim().is_empty() {
continue;
}
let Ok(split) = split_for_cascade(&term) else {
continue;
};
if split.pattern.is_wildcard() || split.regex.is_some() {
continue;
}
let effective = split.term.as_str();
let Some(k) = edit_budget(effective.len(), cap) else {
continue;
};
if Bitap::new(effective.as_bytes(), k).is_none() {
continue;
}
let found = search_ids(&conn, &term, cap);
for (i, body) in bodies.iter().enumerate() {
let within = oracle_distance(effective.as_bytes(), body.as_bytes()) <= k;
if within {
assert!(
found.contains(&ids[i]),
"lost a hit the oracle says is within {} edits\n \
parsed {:?}\n doc {} {:?}\n chunks {:?}",
k,
effective,
i,
&body.chars().take(80).collect::<String>(),
pigeonhole_chunks(effective, k),
);
}
}
compared += 1;
}
}
}
eprintln!("prefilter oracle: {} queries compared corpus-wide", compared);
assert!(compared > 50, "only {} queries compared", compared);
drop(conn);
std::fs::remove_file(&db).ok();
}
// ---------------------------------------------------------------------------
// The regex prefilter
// ---------------------------------------------------------------------------
/// Rewrite `sub` into a regex that still matches it.
///
/// The same trick as the fuzzy generator, in the other direction: rather than
/// corrupt the text and rely on an edit budget, this loosens the *pattern* in
/// ways that provably preserve the match, so the source document is once again
/// a known-correct answer. Each transformation keeps `sub` in the language:
/// `.` matches any one character (the corpus has no newlines), a class
/// containing the character matches it, an alternation offering it matches it,
/// `?` and `+` both admit exactly one occurrence.
///
/// The point is to reach patterns whose required-literal set is interesting:
/// a class or an alternation splits one literal into several, a `?` splits the
/// set into "with" and "without", and a leading `.*` destroys the prefix set
/// entirely so only the suffix set can save it.
fn regexify(sub: &str, lcg: &mut Lcg) -> String {
let esc = |c: char| regex::escape(&c.to_string());
let mut out = String::new();
for c in sub.chars() {
// Most characters stay literal, or the pattern stops resembling
// anything a person would type and every literal set goes empty.
match lcg.next() % 10 {
0 if c != '\n' => out.push('.'),
1 => out.push_str(&format!("[{}z]", esc(c))),
2 => out.push_str(&format!("(?:{}|zzq)", esc(c))),
3 => out.push_str(&format!("{}?", esc(c))),
4 => out.push_str(&format!("{}+", esc(c))),
_ => out.push_str(&esc(c)),
}
}
match lcg.next() % 6 {
0 => format!(".*{out}"),
1 => format!("{out}.*"),
_ => out,
}
}
/// Every document the regex matches is found, and nothing else is.
///
/// Set equality, not recall: the regex passes accept a row when the pattern
/// matches its name, its path or its body, and all three are computable here.
/// So this pins the prefilter from both sides at once — it may not lose a row,
/// and the rows it lets through must still be verified rather than admitted on
/// the strength of holding a literal.
#[test]
fn a_regex_finds_exactly_the_documents_it_matches() {
let bodies = bodies();
let db = scratch_db("prefilter-regex");
let ids = seed(&db, &bodies);
let conn = quicksearch_core::db::open::open_search_reader(&db.to_string_lossy())
.expect("open the seeded index");
let mut lcg = Lcg::new(0xb0a7);
let mut checked = 0usize;
let mut with_prefilter = 0usize;
for len in sweep_lengths() {
for _ in 0..iters_per_len().min(40) {
let doc = lcg.next() as usize % bodies.len();
let Some(sub) = substring_of(&bodies[doc], len, &mut lcg) else {
continue;
};
let pattern = regexify(sub, &mut lcg);
let query = format!("regex:\"{}\"", pattern);
// Anything the query parser does not hand back verbatim is not a
// test of the prefilter — a pattern carrying a quote, or one the
// lexer splits. Round-tripping is the cheapest way to say so.
let Ok(split) = split_for_cascade(&query) else {
continue;
};
let Some(re) = split.regex.as_ref() else {
continue;
};
if re.source != pattern {
continue;
}
if re.required().is_some() {
with_prefilter += 1;
}
// The oracle: the pass accepts a row when the pattern matches any
// of the three fields it looks at.
let mut want: Vec<i64> = Vec::new();
for (i, body) in bodies.iter().enumerate() {
let name = format!("row{:04}.bin", i);
let path = format!("/fuzz/{}", name);
if re.is_match(body) || re.is_match(&name) || re.is_match(&path) {
want.push(ids[i]);
}
}
want.sort();
let mut got = regex_search_ids(&conn, &query);
got.sort();
assert_eq!(
got,
want,
"\n pattern {:?}\n from {:?}\n literals {:?}",
pattern,
sub,
re.required().map(|r| r.literals().to_vec()),
);
checked += 1;
}
}
eprintln!(
"regex prefilter: {} patterns compared corpus-wide, {} of them prefiltered",
checked, with_prefilter
);
assert!(checked > 100, "only {} patterns compared", checked);
assert!(
with_prefilter > checked / 4,
"only {} of {} patterns produced a prefilter; the generator is not \
reaching the path under test",
with_prefilter,
checked
);
drop(conn);
std::fs::remove_file(&db).ok();
}
/// Run a regex-only search and collect the file ids it found.
fn regex_search_ids(conn: &rusqlite::Connection, query: &str) -> Vec<i64> {
let split = split_for_cascade(query).expect("checked by the caller");
let latest = AtomicU64::new(1);
let options = SearchOptions {
limit: 100_000,
..SearchOptions::default()
};
let mut ids = Vec::new();
let mut sink = |hits: Vec<SearchHit>| ids.extend(hits.iter().map(|h| h.file_id));
cascade::run(conn, &split, &options, 1, &latest, &mut sink).expect("the cascade runs");
ids.sort();
ids.dedup();
ids
}
/// Seed one row per body and return the file ids, in the same order.
fn seed(path: &std::path::Path, bodies: &[String]) -> Vec<i64> {
use quicksearch_core::db::repo::{insert_file, set_content_done, NewFile};
use quicksearch_core::mime::FileType;
use quicksearch_core::testutil::zstd_of;
let mut conn =
quicksearch_core::db::open_or_recreate(path.to_str().unwrap(), "trigram").unwrap();
let tx = conn.transaction().unwrap();
let mut ids = Vec::with_capacity(bodies.len());
for (i, body) in bodies.iter().enumerate() {
let id = insert_file(
&tx,
&NewFile {
// Names deliberately share nothing with the bodies, so a hit
// can only come from the full-text pass and never from the
// filename tiers.
name: &format!("row{:04}.bin", i),
parent: "/fuzz/",
size: body.len() as u64,
mtime: 1_700_000_000,
mime: Some("text/plain"),
ftype: FileType::TEXT,
hash: None,
needs_content: true,
},
)
.unwrap()
.expect("unique path");
set_content_done(&tx, id, body, zstd_of(body).as_deref()).unwrap();
ids.push(id);
}
tx.commit().unwrap();
conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE);").ok();
ids
}

12
crates/quicksearch-core/tests/reconcile.rs
View file

@ -62,14 +62,22 @@ fn conn(db: &Path) -> rusqlite::Connection {
rusqlite::Connection::open(db).unwrap()
}
/// Every indexed path, in **path** order.
///
/// Sorted here rather than in SQL: the index's own order is `(parent, name)`,
/// which groups by directory and is not the same sequence — `/r/keep.txt`
/// sorts *before* `/r/sub/k2.txt` there and after it by path. These assertions
/// are about which rows survived, so they read best in the order a person
/// would list them.
fn paths(db: &Path) -> Vec<String> {
let c = conn(db);
let mut stmt = c.prepare("SELECT path FROM files ORDER BY path").unwrap();
let out = stmt
let mut stmt = c.prepare("SELECT parent || name FROM files").unwrap();
let mut out: Vec<String> = stmt
.query_map([], |r| r.get::<_, String>(0))
.unwrap()
.map(|r| r.unwrap())
.collect();
out.sort();
out
}

587
crates/quicksearch-core/tests/search_alloc.rs Normal file
View file

@ -0,0 +1,587 @@
//! What a search moves through the allocator.
//!
//! [`search_perf`](search_perf.rs) answers "how long does a keystroke take";
//! this answers "how much does it allocate to get there", which is a different
//! question with a different answer and, until this existed, no way to measure
//! it from the repository at all. `db/repo.rs` already carries one finding of
//! exactly this kind — that `zstd::decode_all` was "27 of the 30 GiB a fuzzy
//! search moved through the allocator" — and that number came from tooling
//! nobody could re-run.
//!
//! # What is counted, and what is not
//!
//! This installs a `#[global_allocator]` that wraps [`System`] and counts every
//! Rust-side allocation. A global allocator is **per binary**, so this affects
//! only this test executable — the shipped `quicksearch` binary is untouched,
//! which is what makes an always-available accounting harness safe to keep.
//!
//! It deliberately does **not** see SQLite. The bundled SQLCipher amalgamation
//! calls libc `malloc` directly rather than going through Rust's `GlobalAlloc`,
//! so its page cache (32 MiB under [`PRAGMAS_SEARCH`]) and its record buffers
//! are invisible here. That is a feature: what is left is precisely the
//! cascade's own churn — the per-row `String`s, the document folds, the snippet
//! buffers — which is the part the code can do something about.
//!
//! # Reading the numbers
//!
//! - **allocs** is the count. It is the number to watch for per-row work: a
//! pass that scans the whole table and allocates once per row shows up here
//! as a figure proportional to the row count, and a fix shows up as a figure
//! proportional to the *hit* count.
//! - **bytes** is total traffic — allocation churn. High traffic with a low
//! peak means buffers being built and dropped in a loop.
//! - **peak live** is the high-water mark of outstanding bytes during the
//! query, and the only one of the three that speaks to footprint rather than
//! to churn.
//!
//! Timings are printed for orientation only. The counting allocator adds a
//! thread-local read-modify-write to every allocation, so this binary is
//! **not** where wall-clock is decided; `search_perf` is.
//!
//! # What it has measured so far
//!
//! 50,000 rows, 5,000 stored documents of ~2 KB, 32 MiB index. Baseline is the
//! cascade as it stood before this round of work; each column after it is
//! cumulative.
//!
//! | case | allocs | → after | bytes | → after | time | → after |
//! |---|---:|---:|---:|---:|---:|---:|
//! | no match | 14 | 14 | 0.0 | 0.0 | 2.5 ms | 2.5 ms |
//! | literal, rare | 226 | 226 | 0.0 | 0.0 | 2.5 ms | 2.6 ms |
//! | wildcard, rare | 50,324 | **348** | 1.2 MiB | 0.0 | 11.5 ms | **2.6 ms** |
//! | fuzzy, rare | 150,134 | **242** | 4.0 MiB | 0.0 | 62.1 ms | **17.4 ms** |
//! | regex, literal | 50,221 | **283** | 1.3 MiB | 0.1 MiB | 22.4 ms | **2.7 ms** |
//! | regex, no literal | — | 46 | — | 0.0 | — | 25.0 ms |
//! | common (capped) | 4,056 | 4,056 | 0.6 MiB | 0.6 MiB | 1.4 ms | 1.5 ms |
//!
//! (`content, many` and `regex, no literal` were added after the baseline was
//! taken — the first to give the full-text pass real verification work, the
//! second to keep the regex prefilter's *limit* as visible as its win. Neither
//! has a "before" column. The 25 ms in the last row is not a regression: it is
//! what a `regex:` query cost before this work and still costs when the pattern
//! offers no literal to filter on, which is the honest shape of the feature.)
//!
//! What the table is the reason for keeping:
//!
//! * borrowing the row's name instead of allocating one per scanned row — the
//! 50,324 and 150,134 figures were exactly one and three allocations per row;
//! * folding inside the bitap mask table rather than folding each haystack,
//! which took the fuzzy pass's remaining two-per-row to nothing;
//! * a `LIKE` prefilter for multi-segment wildcards, which had been scanning
//! with no SQL filter at all — the whole of the wildcard row's time saving;
//! * the pigeonhole trigram prefilter on the fuzzy full-text pass;
//! * a non-cryptographic hasher for the emitted-id set, worth ~5% of a fuzzy
//! search (see `cascade::IdHasher`);
//! * required-literal prefilters on both `regex:` passes, which had been
//! running the user's pattern over every name, every path and every stored
//! document — the same `Required` machinery the fuzzy pass uses, fed by the
//! literal analysis the regex engine already does for its own prefilter.
//!
//! And what it argued *against*, each recorded as a losing arm in
//! `benches/search.rs`: a `memchr2` candidate scan for the case-insensitive
//! filename probe, and hoisting `memmem::Finder` construction out of the
//! full-text pass. Both are obvious-looking and neither is measurable.
//!
//! Two things this table is worth reading carefully for. **Allocation churn and
//! wall-clock are not the same axis**: the first two changes removed 99.8% of
//! the allocations and moved the clock barely at all, because the fuzzy pass's
//! cost was zstd and bitap rather than malloc. And **the two big wins are both
//! the same idea**: work out what must be present for a row to match, and let
//! the database reject the rest. That took wildcards from 11.5 ms to 2.6,
//! fuzzy from 62 to 17, and `regex:` from 22 to 2.7 — where the per-row
//! micro-optimisations, measured honestly, were worth nothing at all.
//!
//! Take timings on a quiet machine. The allocation columns are deterministic
//! and reproduce bit-for-bit under any load; the time column does not, and a
//! busy box inflates it by 2-3x across the board.
//!
//! Gated by `QSB_SEARCH_ALLOC` so the harness doesn't pay the seed cost on
//! every `cargo test`. To run it:
//!
//! ```text
//! QSB_SEARCH_ALLOC=1 cargo test --release -p quicksearch-core \
//! --test search_alloc -- --nocapture
//! ```
//!
//! [`PRAGMAS_SEARCH`]: quicksearch_core::db::schema::PRAGMAS_SEARCH
//! [`System`]: std::alloc::System
use std::alloc::{GlobalAlloc, Layout, System};
use std::cell::Cell;
use std::sync::atomic::AtomicU64;
use std::time::{Duration, Instant};
use quicksearch_core::query::split::split_for_cascade;
use quicksearch_core::search::{cascade, SearchHit, SearchOptions};
mod common;
use common::{scratch_db, seed_index, SeedSpec};
// ---------------------------------------------------------------------------
// The counting allocator
// ---------------------------------------------------------------------------
// Counters are **per thread**, not global, and that is load-bearing rather
// than an optimization.
//
// libtest runs `#[test]` functions on concurrent threads in one process, and
// there is only one allocator. With global counters, another test's harness
// setup — libtest allocates an output-capture buffer per test, before the test
// body can take any lock of ours — lands inside whatever region is open here
// and is charged to it. That is not hypothetical: it is what the self-test
// below caught, twice, and no mutex in this file can fix it because the
// offending allocation happens before the other test body runs at all.
//
// Per-thread counting is immune to that, and is also the more precise
// question: `cascade::run` is synchronous and does its work on the calling
// thread, so "what did this thread allocate" *is* "what did the cascade
// allocate".
//
// `const`-initialized `Cell`s, deliberately. A `thread_local!` with a lazy
// initializer would allocate on first touch — from inside the allocator — and
// one with a destructor can panic when touched during thread teardown, which
// is exactly when the last deallocations happen. `Cell<u64>` has no `Drop`, so
// neither hazard exists.
// `LIVE` and `PEAK` are **signed**, and that is not fussiness.
//
// Per-thread accounting is inherently asymmetric: a buffer allocated on one
// thread and freed on another decrements a counter that never incremented, so
// a thread's live total legitimately goes negative. libtest does exactly this
// — a test thread starts life having freed more than it allocated. Held as
// `u64` that reads as ~1.8e19, and `PEAK.max(live)` then latches onto it and
// never moves again, so every peak in the table would be reported as 8 bytes.
// That was not a hypothetical either; it is what the self-test caught on the
// third attempt, and it is why the counts are `u64` (they only ever rise) and
// the balances are `i64`.
thread_local! {
static ALLOCS: Cell<u64> = const { Cell::new(0) };
static REALLOCS: Cell<u64> = const { Cell::new(0) };
static BYTES: Cell<u64> = const { Cell::new(0) };
static LIVE: Cell<i64> = const { Cell::new(0) };
static PEAK: Cell<i64> = const { Cell::new(0) };
}
/// Read one monotonic counter, tolerating a thread whose locals are gone.
#[inline]
fn get(counter: &'static std::thread::LocalKey<Cell<u64>>) -> u64 {
counter.try_with(Cell::get).unwrap_or(0)
}
/// Add to one monotonic counter, skipping a thread whose locals are gone.
#[inline]
fn bump(counter: &'static std::thread::LocalKey<Cell<u64>>, by: u64) -> u64 {
counter
.try_with(|c| {
let v = c.get().wrapping_add(by);
c.set(v);
v
})
.unwrap_or(0)
}
/// Read one signed balance.
#[inline]
fn get_live(counter: &'static std::thread::LocalKey<Cell<i64>>) -> i64 {
counter.try_with(Cell::get).unwrap_or(0)
}
/// Move the live balance and return the new value.
#[inline]
fn bump_live(by: i64) -> i64 {
LIVE.try_with(|c| {
let v = c.get().wrapping_add(by);
c.set(v);
v
})
.unwrap_or(0)
}
/// Raise the high-water mark to `live` if it is higher.
#[inline]
fn note_peak(live: i64) {
PEAK.try_with(|p| p.set(p.get().max(live))).ok();
}
/// `System`, with counters. Every hook delegates and then accounts; a failed
/// allocation is not counted, so the totals describe memory that really
/// existed.
struct Counting;
#[inline]
fn note_alloc(size: usize) {
bump(&ALLOCS, 1);
bump(&BYTES, size as u64);
note_peak(bump_live(size as i64));
}
unsafe impl GlobalAlloc for Counting {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
let p = unsafe { System.alloc(layout) };
if !p.is_null() {
note_alloc(layout.size());
}
p
}
unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 {
let p = unsafe { System.alloc_zeroed(layout) };
if !p.is_null() {
note_alloc(layout.size());
}
p
}
/// Freeing on a thread that did not allocate drives this thread's balance
/// negative; see the note on the `thread_local!` block for why that is
/// ordinary and why the balance is signed.
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
bump_live(-(layout.size() as i64));
unsafe { System.dealloc(ptr, layout) }
}
/// Counted as a resize rather than as a fresh allocation: a `Vec` doubling
/// its way up to a document's length is one buffer, not twelve, and calling
/// it twelve allocations would hide the difference between a reused buffer
/// and a per-row one — which is exactly what this harness exists to show.
/// Only the *growth* is added to traffic.
unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
let p = unsafe { System.realloc(ptr, layout, new_size) };
if !p.is_null() {
bump(&REALLOCS, 1);
let (old, new) = (layout.size() as u64, new_size as u64);
bump(&BYTES, new.saturating_sub(old));
note_peak(bump_live(new as i64 - old as i64));
}
p
}
}
#[global_allocator]
static ALLOCATOR: Counting = Counting;
/// Counter values at one instant.
#[derive(Clone, Copy)]
struct Counters {
allocs: u64,
reallocs: u64,
bytes: u64,
/// Signed: a thread's balance is legitimately negative. See the
/// `thread_local!` block.
live: i64,
}
/// What happened between two instants.
#[derive(Clone, Copy)]
struct Usage {
allocs: u64,
reallocs: u64,
bytes: u64,
peak: u64,
}
impl Counters {
/// Snapshot, and re-arm the peak tracker at the current live figure so the
/// high-water mark that follows belongs to the region being measured
/// rather than to whatever the process did before it.
fn start() -> Counters {
let live = get_live(&LIVE);
PEAK.with(|p| p.set(live));
Counters {
allocs: get(&ALLOCS),
reallocs: get(&REALLOCS),
bytes: get(&BYTES),
live,
}
}
fn since(&self) -> Usage {
Usage {
allocs: get(&ALLOCS).wrapping_sub(self.allocs),
reallocs: get(&REALLOCS).wrapping_sub(self.reallocs),
bytes: get(&BYTES).wrapping_sub(self.bytes),
// Above the live figure the region started from, so a query that
// holds nothing reads as zero rather than as the process floor.
peak: (get_live(&PEAK) - self.live).max(0) as u64,
}
}
}
// ---------------------------------------------------------------------------
// The measurement
// ---------------------------------------------------------------------------
/// Rows to seed. Smaller than `search_perf`'s 200k on purpose: allocation
/// counts are deterministic and scale linearly with rows scanned, so they need
/// no statistical settling — only enough rows that a per-row allocation is
/// unmistakable next to a per-hit one.
const NUM_FILES: usize = 50_000;
// No serializing mutex here, deliberately: the counters are per thread, so
// concurrent tests in this binary cannot reach each other's regions. An
// earlier revision did lock, and it did not work — see the comment on the
// `thread_local!` block.
fn enabled() -> bool {
std::env::var("QSB_SEARCH_ALLOC").is_ok()
}
/// One query, described so the table says which cascade passes it exercises.
struct Case {
label: &'static str,
query: &'static str,
fuzzy: bool,
/// The passes this case is here to measure, for the table's last column.
passes: &'static str,
}
/// The query set.
///
/// Every case but the last is **rare on purpose**. A query that fills the
/// display limit makes `cascade::run` break out between passes, so a common
/// term measures how quickly the cascade gives up rather than what a scan
/// costs — and the passes this work targets never run at all. The rare cases
/// are also the ones a user actually waits on.
///
/// Spread across the passes deliberately: a change that helps one pass and
/// hurts another must not be able to hide inside a single total.
const CASES: &[Case] = &[
Case {
label: "no match",
query: "zzzznomatch",
fuzzy: false,
passes: "A, whole table, nothing accepted",
},
Case {
label: "literal, rare",
query: "quartzite",
fuzzy: false,
passes: "A whole table + B verifies FTS hits",
},
Case {
label: "wildcard, rare",
query: "quar*zite",
fuzzy: false,
passes: "A whole table, no prefilter today (S3)",
},
Case {
label: "fuzzy, rare",
// One edit from the planted needle, so the exact passes miss and the
// fuzzy ones have to do the work.
query: "quartzyte",
fuzzy: true,
passes: "A + B + C + D, C and D whole-table (S1/S2/S4)",
},
Case {
// The full-text pass with real work to do: the name `LIKE` finds
// nothing, so every hit is a document the trigram index returned and
// pass B had to decompress, fold and verify.
label: "content, many",
query: "chalcedony",
fuzzy: false,
passes: "A finds nothing + B verifies ~500 docs (S5/S7)",
},
Case {
label: "regex, literal",
query: r"regex:quartz\w+",
fuzzy: false,
passes: "regex name + content, both prefiltered on \"quartz\"",
},
Case {
// No literal to extract, so both regex passes still read everything.
// Kept beside the case above so the prefilter's *limit* is as visible
// as its win.
label: "regex, no literal",
query: r"regex:[0-9]{6}[a-y]{6}",
fuzzy: false,
passes: "regex name + content, no prefilter possible",
},
Case {
label: "common (capped)",
query: "content",
fuzzy: false,
passes: "A, stops at the display limit",
},
];
/// Run one query to completion on a held connection, reporting what it moved
/// through the allocator.
///
/// Hits are counted, not kept: holding thousands of `SearchHit`s would measure
/// the harness's own `Vec` rather than the cascade's. The count is reported so
/// that a change which quietly alters the result set shows up here as well as
/// in `tests/cascade.rs`.
fn measure(conn: &rusqlite::Connection, case: &Case) -> (Usage, usize, Duration) {
let split = split_for_cascade(case.query).expect("the query set parses");
let latest = AtomicU64::new(1);
let options = SearchOptions {
fuzzy: case.fuzzy,
limit: 1000,
..SearchOptions::default()
};
let mut count = 0usize;
let mut sink = |hits: Vec<SearchHit>| count += hits.len();
// Everything the query needs is built above; the snapshot brackets the
// cascade and nothing else.
let start = Counters::start();
let clock = Instant::now();
cascade::run(conn, &split, &options, 1, &latest, &mut sink).expect("the cascade runs");
let elapsed = clock.elapsed();
(start.since(), count, elapsed)
}
fn mib(bytes: u64) -> String {
format!("{:.1}", bytes as f64 / (1024.0 * 1024.0))
}
/// Printed rather than asserted, for the reason `search_perf` gives: a
/// threshold on a shared machine gets muted rather than fixed. Allocation
/// counts are far more stable than timings, so a *ratchet* becomes reasonable
/// once the search work has landed and the numbers have settled — until then
/// this exists to be read when a change claims to have reduced churn.
#[test]
fn allocation_traffic_per_query() {
if !enabled() {
eprintln!("skipping: set QSB_SEARCH_ALLOC=1 to run");
return;
}
let db = scratch_db("searchalloc");
let seeded = Instant::now();
seed_index(
&db,
&SeedSpec {
files: NUM_FILES,
..SeedSpec::default()
},
);
println!(
"seeded {} rows in {:.1?} ({} MiB on disk)\n",
NUM_FILES,
seeded.elapsed(),
std::fs::metadata(&db).map(|m| m.len()).unwrap_or(0) / (1024 * 1024)
);
// One connection for the whole run, as the search worker holds one across
// a typing session. Opened through the real entry point so the pragma
// profile is the production one.
let conn = quicksearch_core::db::open::open_search_reader(&db.to_string_lossy())
.expect("open the seeded index");
println!(
"{:<16} {:>12} {:>10} {:>12} {:>12} {:>7} {:>9} {}",
"case", "allocs", "reallocs", "bytes (MiB)", "peak (MiB)", "hits", "time", "passes"
);
for case in CASES {
// Once to warm the page cache and the statement cache, then measured:
// a cold first query would report SQLite's one-off setup as the
// cascade's churn.
let _ = measure(&conn, case);
let (usage, hits, elapsed) = measure(&conn, case);
println!(
"{:<16} {:>12} {:>10} {:>12} {:>12} {:>7} {:>9.1?} {}",
case.label,
usage.allocs,
usage.reallocs,
mib(usage.bytes),
mib(usage.peak),
hits,
elapsed,
case.passes,
);
}
println!(
"\n{} rows scanned per whole-table pass. An `allocs` figure at or above \
that is per-row work;\nafter S1/S2 the scan passes should sit near their \
hit counts instead.",
NUM_FILES
);
}
/// The accounting itself, so a number printed above is a number that means
/// something. A harness that silently stopped counting would read as a
/// spectacular optimization.
#[test]
fn the_counters_track_real_allocations() {
let start = Counters::start();
// A `Vec` that grows by doubling is one buffer: one alloc, then reallocs.
let mut v: Vec<u8> = Vec::new();
for _ in 0..64 * 1024 {
v.push(0);
}
let grown = start.since();
assert_eq!(grown.allocs, 1, "a doubling Vec is one allocation");
assert!(grown.reallocs > 0, "and several resizes");
assert!(
grown.peak >= 64 * 1024,
"peak {} should cover the grown buffer",
grown.peak
);
// Dropping it returns the bytes: live falls back, so a later region's peak
// is not inflated by this one.
let before_drop = get_live(&LIVE);
drop(v);
assert!(
get_live(&LIVE) < before_drop,
"dealloc must decrement live bytes"
);
// The signedness the whole scheme turns on. Sink the balance below zero, as
// a thread that frees what another thread allocated really does, and check
// that a peak is still reported: held unsigned, that negative balance reads
// as ~1.8e19, `max` latches onto it, and every peak in the table is
// reported as a handful of bytes forever.
bump_live(-(1 << 20));
let negative = Counters::start();
assert!(negative.live < 0, "the balance is genuinely negative");
let mut grow: Vec<u8> = Vec::with_capacity(32 * 1024);
grow.push(1);
let seen = negative.since().peak;
drop(grow);
assert!(
seen >= 32 * 1024,
"a negative live balance swallowed the peak: {}",
seen
);
bump_live(1 << 20); // put back what was sunk, so later regions start clean
// The property that makes the whole harness trustworthy under libtest's
// thread-per-test: another thread allocating hard does not touch this
// thread's counters.
//
// The thread is spawned and joined *outside* the region on purpose.
// `spawn` boxes the closure and allocates the `JoinHandle` on the calling
// thread, and `join` frees them there — so bracketing the spawn would
// measure this thread's own bookkeeping and report it as leakage. A
// barrier hands control across without allocating.
let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
let child = {
let barrier = barrier.clone();
std::thread::spawn(move || {
barrier.wait(); // the region is open
let noisy: Vec<String> = (0..10_000).map(|i| format!("allocation {}", i)).collect();
std::hint::black_box(noisy.len());
barrier.wait(); // the noise is done
})
};
let quiet_across_threads = Counters::start();
barrier.wait();
barrier.wait();
let leaked = quiet_across_threads.since();
child.join().expect("the noisy thread finishes");
assert_eq!(
(leaked.allocs, leaked.bytes),
(0, 0),
"another thread's allocations must not be charged to this region"
);
// A region that allocates nothing reports nothing, which is what makes
// "this pass no longer allocates per row" a statement the table can make.
let quiet = Counters::start();
std::hint::black_box(1u64 + 1);
let idle = quiet.since();
assert_eq!((idle.allocs, idle.bytes, idle.peak), (0, 0, 0));
}

5
crates/quicksearch-core/tests/search_perf.rs
View file

@ -114,13 +114,12 @@ fn seed(path: &std::path::Path) {
let w1 = WORDS[(rng.next() as usize) % WORDS.len()];
let w2 = WORDS[(rng.next() as usize) % WORDS.len()];
let name = format!("{}-{}-{:07}.txt", w1, w2, i);
let dir = format!("/seed/{:03}", i % 500);
let full = format!("{}/{}", dir, name);
// Stored parents always end in a separator.
let dir = format!("/seed/{:03}/", i % 500);
let id = insert_file(
&tx,
&NewFile {
name: &name,
path: &full,
parent: &dir,
size: 4096,
mtime: 1_700_000_000 + i as u64,

2
crates/quicksearch-core/tests/snippet_perf.rs
View file

@ -285,7 +285,7 @@ fn snippet_paths_perf_comparison() {
None => String::new(),
};
let folded = text.to_ascii_lowercase();
let _snip = snippet::extract_folded(&text, &folded, &[q], &opts);
let _snip = snippet::extract_folded(&text, &folded, &[q], &opts).0;
rows_b_total += 1;
}
}

45
crates/quicksearch-gui/src/app.rs
View file

@ -11,6 +11,7 @@ use quicksearch_core::db;
use quicksearch_core::indexing::{
overall_progress, ConfigChange, IndexingStatus, PrepStep, RootPhase, RootProgress,
};
use quicksearch_core::platform::{IndexLock, LockError};
use quicksearch_core::search::SearchOptions;
use quicksearch_core::security::{derive_key, generate_salt, salt_to_hex, IndexKey};
use quicksearch_core::watcher::WatchError;
@ -322,10 +323,45 @@ impl QuickSearchApp {
.watch_cap_warned_roots
.retain(|root| new.paths.indexing_paths.contains(root));
let actions = diff_actions(&self.cfg, &new);
// The instance lock follows the database path, and it has to move
// before the change is written: a `database_path` that another
// instance already holds must never reach the config file, or the next
// launch reads it and refuses to start at all. `move_to` takes the new
// lock before dropping the old, so this rejection leaves us holding
// what we already had.
if actions.search_db_changed {
match IndexLock::move_to(&new.resolved_database_path()) {
Ok(()) => {}
Err(LockError::Held { pid }) => {
let who = match pid {
Some(pid) => format!(" (process {})", pid),
None => String::new(),
};
self.config_error = Some(format!(
"Not applied: another QuickSearch{} is using that index.",
who
));
return false;
}
// The move happened and the new path simply cannot be locked.
// Same rule as at startup: a convenience guard is never a good
// enough reason to refuse.
Err(LockError::Unsupported(why)) => {
quicksearch_core::log_warn!("cannot lock the index ({}); continuing", why);
}
}
}
// A config that could not be written must not take effect either: it
// would apply to this process, revert on restart, and show nothing
// unsaved in between.
if let Err(e) = new.save() {
// Put the lock back on the path the config still names, or this
// process would go on using the old index while guarding the new
// one — leaving the index it is actually writing open to a second
// instance.
if actions.search_db_changed {
let _ = IndexLock::move_to(&self.cfg.resolved_database_path());
}
self.config_error = Some(e);
return false;
}
@ -350,7 +386,7 @@ impl QuickSearchApp {
self.backend.coordinator.apply_config(new.clone());
if actions.requires_rebuild {
if self.backend.coordinator.state().mode == IndexMode::Auto {
self.backend.coordinator.rebuild_index();
self.backend.rebuild_index();
} else {
let changes = self
.backend
@ -440,7 +476,9 @@ impl QuickSearchApp {
if let Some(rx) = &self.backend.dup_job {
use std::sync::mpsc::TryRecvError;
let done = match rx.try_recv() {
Ok(Ok(groups)) => Some(DupState::Loaded(groups)),
Ok(Ok(groups)) => Some(DupState::Loaded(crate::duplicates_tab::LoadedGroups::new(
groups,
))),
Ok(Err(e)) => Some(DupState::Error(e)),
Err(TryRecvError::Empty) => None,
Err(TryRecvError::Disconnected) => {
@ -692,7 +730,8 @@ impl eframe::App for QuickSearchApp {
if let Some(paths) = actions.verify {
let paths: Vec<std::path::PathBuf> =
paths.into_iter().map(std::path::PathBuf::from).collect();
self.backend.start_verify(paths.clone(), ctx.clone());
self.backend
.start_verify(paths.clone(), &self.cfg, ctx.clone());
self.verify = Some(VerifyModal::new(paths));
}
}

18
crates/quicksearch-gui/src/app/modals.rs
View file

@ -50,7 +50,7 @@ impl QuickSearchApp {
));
ui.horizontal(|ui| {
if ui.button("Rebuild now").clicked() {
self.backend.coordinator.rebuild_index();
self.backend.rebuild_index();
close = true;
}
if ui.button("Later").clicked() {
@ -97,7 +97,7 @@ impl QuickSearchApp {
}
if stale_index_window(ctx, self.key_source) {
self.stale_index_prompt = false;
self.backend.coordinator.rebuild_index();
self.backend.rebuild_index();
self.dups.state = DupState::NotLoaded;
}
}
@ -158,6 +158,18 @@ impl QuickSearchApp {
group_thousands(*registered as u64),
));
}
// Not reachable through this modal today: an overflow leaves
// the watcher running and schedules a full run instead of
// disabling live updates, so it never becomes a
// `WatcherStatus::Disabled`. Spelled out rather than folded
// into a catch-all so that a future variant is a compile
// error here, which is how this arm came to be written.
WatchError::Overflowed => {
ui.label(
"The system dropped some change notifications, so the index \
is being rebuilt to catch up.",
);
}
WatchError::Other(msg) => {
ui.label(format!("Live updates are unavailable: {}", msg));
}
@ -227,7 +239,7 @@ impl QuickSearchApp {
// resurrect what was just deleted, nor the next launch
// undo the stop.
self.set_index_mode(false);
self.backend.coordinator.clear_index();
self.backend.clear_index();
self.dups.state = DupState::NotLoaded;
return true;
}

2
crates/quicksearch-gui/src/app/security.rs
View file

@ -377,7 +377,7 @@ impl QuickSearchApp {
}
}
db::set_process_key(new_key);
self.backend.coordinator.rebuild_index();
self.backend.rebuild_index();
self.dups.state = DupState::NotLoaded;
}
}

39
crates/quicksearch-gui/src/backend.rs
View file

@ -51,6 +51,31 @@ pub struct Backend {
}
impl Backend {
/// Ask the coordinator to rebuild, after letting go of everything that
/// holds the index file open.
///
/// The search worker keeps its connection for half a minute after the last
/// keystroke so a typing session runs against a warm cache — which is
/// exactly the wrong thing to be holding when the file is about to be
/// deleted. Without this the delete fails on Windows and the rebuild
/// silently becomes an ordinary run against the old index; after a
/// password change that leaves the config claiming protection the file on
/// disk does not have.
pub fn rebuild_index(&self) {
if let Some(search) = &self.search {
search.release_connection();
}
self.coordinator.rebuild_index();
}
/// [`Backend::rebuild_index`]'s reasoning, for the delete-only path.
pub fn clear_index(&self) {
if let Some(search) = &self.search {
search.release_connection();
}
self.coordinator.clear_index();
}
pub fn start(config: &Config, ctx: egui::Context) -> Result<Backend, String> {
// eframe is reactive: a run the coordinator schedules on its own
// would sit unseen behind a settled window until the pointer moved.
@ -89,10 +114,17 @@ impl Backend {
pub fn watch_live(
&self,
query: &str,
targets: Vec<quicksearch_core::live::Target>,
mut targets: Vec<quicksearch_core::live::Target>,
config: &Config,
) {
let Some(live) = &self.live else { return };
// The watcher re-reads a row's file to re-cut its snippet, and the one
// file it must never open is the index it is reading the row from:
// closing a descriptor on it cancels SQLite's locks process-wide. The
// walk no longer writes such rows, but one from an older build lives
// until the stale sweep reaches its directory, and it can be on screen
// before then.
targets.retain(|t| !config.is_index_file(std::path::Path::new(&t.path)));
if targets.is_empty() {
live.clear();
} else {
@ -133,10 +165,13 @@ impl Backend {
/// Read a duplicate group through on a worker thread, comparing every
/// member against the first byte for byte. Replaces any run already going.
pub fn start_verify(&mut self, paths: Vec<PathBuf>, ctx: egui::Context) {
pub fn start_verify(&mut self, mut paths: Vec<PathBuf>, config: &Config, ctx: egui::Context) {
if let Some(job) = &self.verify_job {
job.cancel();
}
// Byte-for-byte comparison opens every member. See `watch_live` for
// why the index must not be one of them.
paths.retain(|p| !config.is_index_file(p));
let (tx, rx) = mpsc::channel();
let cancel = Arc::new(AtomicBool::new(false));
let worker_cancel = cancel.clone();

133
crates/quicksearch-gui/src/cli.rs
View file

@ -10,6 +10,7 @@ use quicksearch_core::db;
use quicksearch_core::query::split::split_for_cascade;
use quicksearch_core::search::{cascade, SearchHit, SearchOptions};
use quicksearch_core::security::{derive_key, IndexKey};
use quicksearch_core::textenc::scrub_controls;
use zeroize::Zeroizing;
use crate::format::{fmt_mtime, human_size};
@ -127,15 +128,22 @@ pub(crate) fn resolve_key(
if let Some(hex) = keychain_hex {
match IndexKey::from_hex(&hex).map_err(|e| format!("keychain entry: {}", e)) {
Ok(key) => {
match try_key(key) {
Ok(()) => return Ok(()),
Err(e) if e.starts_with(db::KEY_MISMATCH_PREFIX) => {
eprintln!("warning: the key remembered in the OS keychain no longer opens this index");
Ok(key) => match try_key(key) {
Ok(()) => return Ok(()),
Err(e) => match mismatch_cause(&e) {
Some((db::KeyMismatch::WrongPassword, _)) => {
eprintln!(
"warning: the key remembered in the OS keychain no longer \
opens this index"
);
}
Err(e) => return Err(e),
}
}
// Not a stale keychain entry at all: the index is not
// encrypted, or wants a key this one is not. Say which,
// then fall through to the other sources as before.
Some((_, detail)) => eprintln!("warning: {}", detail),
None => return Err(e),
},
},
Err(e) => eprintln!("warning: {}", e),
}
}
@ -145,11 +153,17 @@ pub(crate) fn resolve_key(
drop(password);
return match try_key(key) {
Ok(()) => Ok(()),
Err(e) if e.starts_with(db::KEY_MISMATCH_PREFIX) => Err(format!(
"{} does not match this index's password",
PASSWORD_ENV
)),
Err(e) => Err(e),
Err(e) => match mismatch_cause(&e) {
Some((db::KeyMismatch::WrongPassword, _)) => Err(format!(
"{} does not match this index's password",
PASSWORD_ENV
)),
// Blaming the environment variable for "the index is not
// encrypted" sends the user to change the one thing that is
// not wrong.
Some((_, detail)) => Err(detail),
None => Err(e),
},
};
}
@ -168,15 +182,31 @@ pub(crate) fn resolve_key(
drop(password);
match try_key(key) {
Ok(()) => return Ok(()),
Err(e) if e.starts_with(db::KEY_MISMATCH_PREFIX) => {
eprintln!("Wrong password.");
}
Err(e) => return Err(e),
Err(e) => match mismatch_cause(&e) {
// Only a wrong password is worth another attempt. The other
// two causes are facts about the index — it is not encrypted,
// or it wants a password this build did not apply — and
// retrying makes the user type a *correct* password twice more
// before being told something that was never about their
// typing. The same reasoning as the unlock screen's.
Some((db::KeyMismatch::WrongPassword, _)) => eprintln!("Wrong password."),
Some((_, detail)) => return Err(detail),
None => return Err(e),
},
}
}
Err("wrong password (3 attempts)".to_string())
}
/// [`db::key_mismatch_parts`] with the detail copied out.
///
/// Owned rather than borrowed so a caller can hand the original error back on
/// the `None` arm: `key_mismatch_parts` borrows from the message, and a `match`
/// on its result would hold that borrow across a `return Err(e)`.
fn mismatch_cause(error: &str) -> Option<(db::KeyMismatch, String)> {
db::key_mismatch_parts(error).map(|(cause, detail)| (cause, detail.to_string()))
}
/// Wire [`resolve_key`] to the real terminal, environment, keychain and
/// database, installing the verified key as the process key.
fn resolve_key_for_terminal(security: &SecurityConfig, db_path: &str) -> Result<(), String> {
@ -269,7 +299,21 @@ fn run_query(query: &str, fuzzy: bool, limit: Option<usize>, long: bool) -> i32
match outcome {
Ok(Some(outcome)) => {
let color = long && std::io::stdout().is_terminal() && enable_vt();
let tty = std::io::stdout().is_terminal();
let color = long && tty && enable_vt();
// Scrubbed only for a terminal, the way `ls` does it and the way
// the colour above already decides: a filename may contain escape
// sequences, and printed raw they rewrite the line, retitle the
// window, or reach the clipboard on a terminal with OSC 52 on.
// Piped output stays byte-exact, because the next program in the
// pipe wants the real name and interprets nothing.
let show = |s: &str| -> String {
if tty {
scrub_controls(s).into_owned()
} else {
s.to_string()
}
};
for hit in &hits {
if long {
println!(
@ -277,13 +321,13 @@ fn run_query(query: &str, fuzzy: bool, limit: Option<usize>, long: bool) -> i32
hit.rank,
human_size(hit.size),
fmt_mtime(hit.mtime),
hit.path
show(&hit.path)
);
if let Some(snip) = &hit.snippet {
println!(" {}", render_snippet(snip, color));
println!(" {}", render_snippet(snip, color, tty));
}
} else {
println!("{}", hit.path);
println!("{}", show(&hit.path));
}
}
if outcome.limited {
@ -334,28 +378,58 @@ fn enable_vt() -> bool {
}
/// One-line snippet with matches emphasized (ANSI bold on TTYs).
fn render_snippet(snip: &quicksearch_core::snippet::Snippet, color: bool) -> String {
fn render_snippet(snip: &quicksearch_core::snippet::Snippet, color: bool, tty: bool) -> String {
// The window is document text, so it can hold anything the file did —
// including the escape sequences this function is about to add its own
// emphasis codes around. Each piece is scrubbed as it goes in, before the
// codes, so the only escapes in the result are the ones put there here.
let mut out = String::new();
let push = |out: &mut String, piece: &str| {
// Line breaks are flattened *before* the scrub, not after the whole
// string is assembled. A snippet is one line by contract for both
// destinations, and `scrub_controls` turns `\n` into `U+FFFD` — so a
// trailing `replace(['\n', '\r'], " ")` would find nothing left to
// collapse on a terminal and print `<60>` where the space belongs.
let flat = flatten_lines(piece);
if tty {
out.push_str(&scrub_controls(&flat));
} else {
out.push_str(&flat);
}
};
if snip.truncated_start {
out.push('…');
}
let mut cursor = 0;
for &(start, end) in &snip.ranges {
out.push_str(&snip.window[cursor..start]);
push(&mut out, &snip.window[cursor..start]);
if color {
out.push_str("\x1b[1m");
out.push_str(&snip.window[start..end]);
push(&mut out, &snip.window[start..end]);
out.push_str("\x1b[0m");
} else {
out.push_str(&snip.window[start..end]);
push(&mut out, &snip.window[start..end]);
}
cursor = end;
}
out.push_str(&snip.window[cursor..]);
push(&mut out, &snip.window[cursor..]);
if snip.truncated_end {
out.push('…');
}
out.replace(['\n', '\r'], " ")
out
}
/// `piece` with line breaks flattened to spaces, borrowed when it has none.
///
/// A `Cow` for the same reason [`scrub_controls`] is one: this runs per
/// fragment of every snippet of every hit, and the overwhelmingly common case
/// has nothing to change.
fn flatten_lines(piece: &str) -> std::borrow::Cow<'_, str> {
if piece.contains(['\n', '\r']) {
std::borrow::Cow::Owned(piece.replace(['\n', '\r'], " "))
} else {
std::borrow::Cow::Borrowed(piece)
}
}
#[cfg(test)]
@ -372,7 +446,10 @@ mod tests {
}
fn mismatch() -> Result<(), String> {
Err(format!("{}wrong password", db::KEY_MISMATCH_PREFIX))
Err(format!(
"{}wrong-password: index at /x: wrong password",
db::KEY_MISMATCH_PREFIX
))
}
fn argv(args: &[&str]) -> Vec<String> {

55
crates/quicksearch-gui/src/duplicates_tab.rs
View file

@ -9,10 +9,45 @@ use crate::ui_util::hint;
pub enum DupState {
NotLoaded,
Loading,
Loaded(Vec<DuplicateGroup>),
Loaded(LoadedGroups),
Error(String),
}
/// The scan's result, with each group's header line already built.
///
/// The titles are four formatted numbers each and the list runs to 500, so
/// building them in the render loop meant ~2000 allocations *per frame* — and
/// this list overflows by definition, which keeps `more_below_hint`'s 20 Hz
/// repaint running for as long as the tab is open. They depend only on the
/// data, so they are built once, here, where the data arrives.
pub struct LoadedGroups {
pub groups: Vec<DuplicateGroup>,
titles: Vec<String>,
}
impl LoadedGroups {
pub fn new(groups: Vec<DuplicateGroup>) -> LoadedGroups {
let titles = groups
.iter()
.map(|group| {
let name = group
.members
.first()
.map(|m| m.1.as_str())
.unwrap_or("(unknown)");
format!(
"{} × {}: {} reclaimable ({} total)",
group_thousands(group.count as u64),
name,
human_size(group.redundant_size.max(0) as u64),
human_size(group.total_size.max(0) as u64),
)
})
.collect();
LoadedGroups { groups, titles }
}
}
pub struct DuplicatesTab {
pub state: DupState,
}
@ -73,7 +108,8 @@ impl DuplicatesTab {
DupState::Error(e) => {
ui.colored_label(ui.visuals().error_fg_color, e);
}
DupState::Loaded(groups) => {
DupState::Loaded(loaded) => {
let groups = &loaded.groups;
if groups.is_empty() {
ui.label("No duplicate files found.");
return actions;
@ -85,18 +121,9 @@ impl DuplicatesTab {
.auto_shrink([false; 2])
.show(ui, |ui| {
for (i, group) in groups.iter().enumerate() {
let name = group
.members
.first()
.map(|m| m.1.as_str())
.unwrap_or("(unknown)");
let title = format!(
"{} × {}: {} reclaimable ({} total)",
group_thousands(group.count as u64),
name,
human_size(group.redundant_size.max(0) as u64),
human_size(group.total_size.max(0) as u64),
);
// Built once when the scan landed; see
// `LoadedGroups`.
let title = loaded.titles[i].as_str();
let header =
egui::CollapsingHeader::new(title)
.id_salt(i)

4
crates/quicksearch-gui/src/duplicates_tab/tests.rs
View file

@ -23,7 +23,7 @@ fn group(paths: &[&str]) -> DuplicateGroup {
fn loaded(paths: &[&str]) -> DuplicatesTab {
DuplicatesTab {
state: DupState::Loaded(vec![group(paths)]),
state: DupState::Loaded(LoadedGroups::new(vec![group(paths)])),
}
}
@ -177,7 +177,7 @@ fn a_member_row_offers_the_verification_too() {
fn an_empty_result_says_so_rather_than_showing_an_empty_list() {
let ctx = crate::test_ui::ctx();
let mut tab = DuplicatesTab {
state: DupState::Loaded(Vec::new()),
state: DupState::Loaded(LoadedGroups::new(Vec::new())),
};
let painted = painted_text(&frame(&ctx, &mut tab, false, Vec::new()).0);
assert!(

16
crates/quicksearch-gui/src/format.rs
View file

@ -43,10 +43,14 @@ pub fn fmt_ago(unix_secs: u64) -> String {
/// A configured interval as a phrase to drop after "every": `90 min`,
/// `24 h`, `3 days`.
pub fn fmt_interval(minutes: u64) -> String {
if minutes == 0 {
// The scheduler treats 0 as always-due.
return "run".to_string();
}
// The same clamp the scheduler applies. Zero is only reachable from a
// hand-edited config — the spinner's floor is 5 — and it used to read
// "run", from a time when the scheduler took it as always-due. It no
// longer does (`coordinator::inner::periodic_due` raises it to one
// minute, because "reindex continuously" is not a setting anyone wants
// and manual mode is how you say "never"), so saying "run" here would
// describe a behaviour the app does not have.
let minutes = minutes.max(1);
if minutes < 60 {
return format!("{} min", minutes);
}
@ -170,7 +174,9 @@ mod tests {
#[test]
fn intervals() {
assert_eq!(fmt_interval(0), "run");
// Clamped, exactly as `periodic_due` clamps it: the two must not
// describe different behaviour.
assert_eq!(fmt_interval(0), "1 min");
assert_eq!(fmt_interval(1), "1 min");
assert_eq!(fmt_interval(59), "59 min");
assert_eq!(fmt_interval(60), "1 h", "the shipped default");

40
crates/quicksearch-gui/src/main.rs
View file

@ -37,6 +37,7 @@ mod unlock;
mod version;
use quicksearch_core::config::Config;
use quicksearch_core::platform::{IndexLock, LockError};
/// The window icon, shown in the titlebar, taskbar and alt-tab switcher.
///
@ -82,6 +83,45 @@ fn main() {
};
let initial_query = seed_query();
// After the CLI early-exit above, deliberately: `quicksearch <query>` only
// reads, and must keep working from a terminal while the window is open.
// Two *windows* on one index are the problem — two indexers writing, and,
// once either has cancelled the other's SQLite locks, an attach that
// truncates the wal-index under a live mapping.
//
// Held for the life of the process in `platform`'s own slot, so the
// settings handler can move it when `database_path` changes; the kernel
// releases it on exit, however that exit happens.
match IndexLock::hold(&config.resolved_database_path()) {
Ok(()) => {}
Err(LockError::Held { pid }) => {
let who = match pid {
Some(pid) => format!(" (process {})", pid),
None => String::new(),
};
let msg = format!(
"QuickSearch is already running{}.\n\nOnly one window can use \
the index at a time. Switch to the running window, or close \
it and try again.",
who
);
eprintln!("{}", msg);
// The app is normally launched from a desktop icon or a hotkey,
// where nothing is watching stderr.
rfd::MessageDialog::new()
.set_level(rfd::MessageLevel::Info)
.set_title("QuickSearch")
.set_description(&msg)
.show();
std::process::exit(1);
}
// Not "the lock is taken" — the filesystem could not answer. A
// convenience guard is never a good enough reason to refuse to open.
Err(LockError::Unsupported(why)) => {
eprintln!("warning: cannot lock the index ({}); starting anyway", why);
}
}
// With protection on, try the keychain before the window opens; a
// verified key means no prompt at all. `None` starts locked, and no
// index is touched until unlocked.

Some files were not shown because too many files have changed in this diff Show more